From 3961b2636329819f16b1f32739274f3a1d3273de Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 10 Aug 2022 11:26:08 +0000 Subject: [PATCH] wasm refactoring part 2 of (apparently) 2: moved ext/fiddle/... into ext/wasm and restructured the core API-related parts of the JS/WASM considerably. FossilOrigin-Name: 27f9da4eaaff39d1d58e9ffef7ddccf1e41b3726914f754b920e3e1fb572cba6 --- Makefile.in | 4 +- ext/fiddle/SqliteTestUtil.js | 144 -- ext/fiddle/sqlite3-api.js | 2068 ----------------- ext/fiddle/sqlite3-worker.js | 44 - ext/fiddle/testing1.js | 239 -- ext/fiddle/testing2.js | 235 -- ext/fiddle/wasm_util.c | 127 - ext/wasm/GNUmakefile | 270 ++- .../api}/EXPORTED_FUNCTIONS.sqlite3-api | 21 +- .../api/EXPORTED_RUNTIME_METHODS.sqlite3-api | 3 + ext/wasm/api/post-js-footer.js | 3 + ext/wasm/api/post-js-header.js | 26 + ext/wasm/api/sqlite3-api-cleanup.js | 44 + ext/wasm/api/sqlite3-api-glue.js | 211 ++ ext/wasm/api/sqlite3-api-oo1.js | 1438 ++++++++++++ ext/wasm/api/sqlite3-api-opfs.js | 394 ++++ ext/wasm/api/sqlite3-api-prologue.js | 593 +++++ ext/wasm/api/sqlite3-api-worker.js | 421 ++++ ext/wasm/api/sqlite3-wasi.h | 69 + ext/wasm/api/sqlite3-wasm.c | 413 ++++ ext/wasm/api/sqlite3-worker.js | 31 + ext/wasm/common/SqliteTestUtil.js | 173 ++ ext/wasm/common/emscripten.css | 24 + ext/{fiddle => wasm/common}/testing.css | 1 + ext/wasm/common/whwasmutil.js | 1548 ++++++++++++ ext/wasm/jaccwabyt/jaccwabyt.js | 737 ++++++ ext/wasm/jaccwabyt/jaccwabyt.md | 1078 +++++++++ ext/wasm/jaccwabyt/jaccwabyt_test.c | 178 ++ ext/wasm/jaccwabyt/jaccwabyt_test.exports | 10 + ext/{fiddle => wasm}/testing1.html | 13 +- ext/wasm/testing1.js | 1088 +++++++++ ext/{fiddle => wasm}/testing2.html | 11 +- ext/wasm/testing2.js | 340 +++ manifest | 63 +- manifest.uuid | 2 +- 35 files changed, 9162 insertions(+), 2902 deletions(-) delete mode 100644 ext/fiddle/SqliteTestUtil.js delete mode 100644 ext/fiddle/sqlite3-api.js delete mode 100644 ext/fiddle/sqlite3-worker.js delete mode 100644 ext/fiddle/testing1.js delete mode 100644 ext/fiddle/testing2.js delete mode 100644 ext/fiddle/wasm_util.c rename ext/{fiddle => wasm/api}/EXPORTED_FUNCTIONS.sqlite3-api (71%) create mode 100644 ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api create mode 100644 ext/wasm/api/post-js-footer.js create mode 100644 ext/wasm/api/post-js-header.js create mode 100644 ext/wasm/api/sqlite3-api-cleanup.js create mode 100644 ext/wasm/api/sqlite3-api-glue.js create mode 100644 ext/wasm/api/sqlite3-api-oo1.js create mode 100644 ext/wasm/api/sqlite3-api-opfs.js create mode 100644 ext/wasm/api/sqlite3-api-prologue.js create mode 100644 ext/wasm/api/sqlite3-api-worker.js create mode 100644 ext/wasm/api/sqlite3-wasi.h create mode 100644 ext/wasm/api/sqlite3-wasm.c create mode 100644 ext/wasm/api/sqlite3-worker.js create mode 100644 ext/wasm/common/SqliteTestUtil.js create mode 100644 ext/wasm/common/emscripten.css rename ext/{fiddle => wasm/common}/testing.css (93%) create mode 100644 ext/wasm/common/whwasmutil.js create mode 100644 ext/wasm/jaccwabyt/jaccwabyt.js create mode 100644 ext/wasm/jaccwabyt/jaccwabyt.md create mode 100644 ext/wasm/jaccwabyt/jaccwabyt_test.c create mode 100644 ext/wasm/jaccwabyt/jaccwabyt_test.exports rename ext/{fiddle => wasm}/testing1.html (76%) create mode 100644 ext/wasm/testing1.js rename ext/{fiddle => wasm}/testing2.html (79%) create mode 100644 ext/wasm/testing2.js diff --git a/Makefile.in b/Makefile.in index 3b9a7b56bc..193095c681 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1564,9 +1564,9 @@ fiddle_generated = $(fiddle_module_js) $(fiddle_module_js).gz \ $(fiddle_dir)/fiddle-module.wasm.gz \ $(fiddle_dir)/fiddle.js.gz -clean-wasm: +clean-fiddle: rm -f $(fiddle_generated) -clean: clean-wasm +clean: clean-fiddle fiddle: $(fiddle_module_js) $(fiddle_dir)/fiddle.js.gz wasm: fiddle ######################################################################## diff --git a/ext/fiddle/SqliteTestUtil.js b/ext/fiddle/SqliteTestUtil.js deleted file mode 100644 index bcbdc59c51..0000000000 --- a/ext/fiddle/SqliteTestUtil.js +++ /dev/null @@ -1,144 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file contains bootstrapping code used by various test scripts - which live in this file's directory. -*/ -(function(){ - /* querySelectorAll() proxy */ - const EAll = function(/*[element=document,] cssSelector*/){ - return (arguments.length>1 ? arguments[0] : document) - .querySelectorAll(arguments[arguments.length-1]); - }; - /* querySelector() proxy */ - const E = function(/*[element=document,] cssSelector*/){ - return (arguments.length>1 ? arguments[0] : document) - .querySelector(arguments[arguments.length-1]); - }; - - /** - Helpers for writing sqlite3-specific tests. - */ - self/*window or worker*/.SqliteTestUtil = { - /** Running total of the number of tests run via - this API. */ - counter: 0, - /** - If expr is a function, it is called and its result - is returned, coerced to a bool, else expr, coerced to - a bool, is returned. - */ - toBool: function(expr){ - return (expr instanceof Function) ? !!expr() : !!expr; - }, - /** abort() if expr is false. If expr is a function, it - is called and its result is evaluated. - */ - assert: function f(expr, msg){ - if(!f._){ - f._ = ('undefined'===typeof abort - ? (msg)=>{throw new Error(msg)} - : abort); - } - ++this.counter; - if(!this.toBool(expr)){ - f._(msg || "Assertion failed."); - } - return this; - }, - /** Identical to assert() but throws instead of calling - abort(). */ - affirm: function(expr, msg){ - ++this.counter; - if(!this.toBool(expr)) throw new Error(msg || "Affirmation failed."); - return this; - }, - /** Calls f() and squelches any exception it throws. If it - does not throw, this function throws. */ - mustThrow: function(f, msg){ - ++this.counter; - let err; - try{ f(); } catch(e){err=e;} - if(!err) throw new Error(msg || "Expected exception."); - return this; - }, - /** Throws if expr is truthy or expr is a function and expr() - returns truthy. */ - throwIf: function(expr, msg){ - ++this.counter; - if(this.toBool(expr)) throw new Error(msg || "throwIf() failed"); - return this; - }, - /** Throws if expr is falsy or expr is a function and expr() - returns falsy. */ - throwUnless: function(expr, msg){ - ++this.counter; - if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed"); - return this; - } - }; - - - /** - This is a module object for use with the emscripten-installed - sqlite3InitModule() factory function. - */ - self.sqlite3TestModule = { - postRun: [ - /* function(theModule){...} */ - ], - //onRuntimeInitialized: function(){}, - /* Proxy for C-side stdout output. */ - print: function(){ - console.log.apply(console, Array.prototype.slice.call(arguments)); - }, - /* Proxy for C-side stderr output. */ - printErr: function(){ - console.error.apply(console, Array.prototype.slice.call(arguments)); - }, - /** - Called by the module init bits to report loading - progress. It gets passed an empty argument when loading is - done (after onRuntimeInitialized() and any this.postRun - callbacks have been run). - */ - setStatus: function f(text){ - if(!f.last){ - f.last = { text: '', step: 0 }; - f.ui = { - status: E('#module-status'), - progress: E('#module-progress'), - spinner: E('#module-spinner') - }; - } - if(text === f.last.text) return; - f.last.text = text; - if(f.ui.progress){ - f.ui.progress.value = f.last.step; - f.ui.progress.max = f.last.step + 1; - } - ++f.last.step; - if(text) { - f.ui.status.classList.remove('hidden'); - f.ui.status.innerText = text; - }else{ - if(f.ui.progress){ - f.ui.progress.remove(); - f.ui.spinner.remove(); - delete f.ui.progress; - delete f.ui.spinner; - } - f.ui.status.classList.add('hidden'); - } - } - }; -})(self/*window or worker*/); diff --git a/ext/fiddle/sqlite3-api.js b/ext/fiddle/sqlite3-api.js deleted file mode 100644 index 01eb4c57fd..0000000000 --- a/ext/fiddle/sqlite3-api.js +++ /dev/null @@ -1,2068 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This file is intended to be appended to the emcc-generated - sqlite3.js via emcc: - - emcc ... -sMODULARIZE -sEXPORT_NAME=sqlite3InitModule --post-js=THIS_FILE - - It is loaded by importing the emcc-generated sqlite3.js, then: - - sqlite3InitModule({module object}).then( - function(theModule){ - theModule.sqlite3 == an object containing this file's - deliverables: - { - api: bindings for much of the core sqlite3 APIs, - SQLite3: high-level OO API wrapper - } - }); - - It is up to the caller to provide a module object compatible with - emcc, but it can be a plain empty object. The object passed to - sqlite3InitModule() will get populated by the emscripten-generated - bits and, in part, by the code from this file. Specifically, this file - installs the `theModule.sqlite3` part shown above. - - The resulting sqlite3.api object wraps the standard sqlite3 C API in - a way as close to its native form as JS allows for. The - sqlite3.SQLite3 object provides a higher-level wrapper more - appropriate for general client-side use in JS. - - Because using certain parts of the low-level API properly requires - some degree of WASM-related magic, it is not recommended that that - API be used as-is in client-level code. Rather, client code is - encouraged use the higher-level OO API or write a custom wrapper on - top of the lower-level API. In short, most of the C-style API is - used in an intuitive manner from JS but any C-style APIs which take - pointers-to-pointer arguments require WASM-specific interfaces - installed by Emscripten-generated code. Those which take or return - only integers, doubles, strings, or "plain" pointers to db or - statement objects can be used in "as normal," noting that "pointers" - in WASM are simply 32-bit integers. - - - Specific goals of this project: - - - Except where noted in the non-goals, provide a more-or-less - complete wrapper to the sqlite3 C API, insofar as WASM feature - parity with C allows for. In fact, provide at least 3... - - 1) Bind a low-level sqlite3 API which is as close to the native - one as feasible in terms of usage. - - 2) A higher-level API, more akin to sql.js and node.js-style - implementations. This one speaks directly to the low-level - API. This API must be used from the same thread as the - low-level API. - - 3) A second higher-level API which speaks to the previous APIs via - worker messages. This one is intended for use in the main - thread, with the lower-level APIs installed in a Worker thread, - and talking to them via Worker messages. Because Workers are - asynchronouns and have only a single message channel, some - acrobatics are needed here to feed async work results back to - the client (as we cannot simply pass around callbacks between - the main and Worker threads). - - - Insofar as possible, support client-side storage using JS - filesystem APIs. As of this writing, such things are still very - much TODO. - - - Specific non-goals of this project: - - - As WASM is a web-centric technology and UTF-8 is the King of - Encodings in that realm, there are no currently plans to support - the UTF16-related sqlite3 APIs. They would add a complication to - the bindings for no appreciable benefit. - - - Supporting old or niche-market platforms. WASM is built for a - modern web and requires modern platforms. - - - Attribution: - - Though this code is not a direct copy/paste, much of the - functionality in this file is strongly influenced by the - corresponding features in sql.js: - - https://github.com/sql-js/sql.js - - sql.js was an essential stepping stone in this code's development as - it demonstrated how to handle some of the WASM-related voodoo (like - handling pointers-to-pointers and adding JS implementations of - C-bound callback functions). These APIs have a considerably - different shape than sql.js's, however. -*/ -if(!Module.postRun) Module.postRun = []; -/* ^^^^ the name Module is, in this setup, scope-local in the generated - file sqlite3.js, with which this file gets combined at build-time. */ -Module.postRun.push(function(namespace/*the module object, the target for - installed features*/){ - 'use strict'; - /** - */ - const SQM/*interal-use convenience alias*/ = namespace/*the sqlite module object */; - - /** Throws a new Error, the message of which is the concatenation - all args with a space between each. */ - const toss = function(){ - throw new Error(Array.prototype.join.call(arguments, ' ')); - }; - - /** Returns true if n is a 32-bit (signed) integer, else false. */ - const isInt32 = function(n){ - return !!(n===(n|0) && n<=2147483647 && n>=-2147483648); - }; - - /** Returns v if v appears to be a TypedArray, else false. */ - const isTypedArray = (v)=>{ - return (v && v.constructor && isInt32(v.constructor.BYTES_PER_ELEMENT)) ? v : false; - }; - - /** - Returns true if v appears to be one of our bind()-able - TypedArray types: Uint8Array or Int8Array. Support for - TypedArrays with element sizes >1 is a potential TODO. - */ - const isBindableTypedArray = (v)=>{ - return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); - }; - - /** - Returns true if v appears to be one of the TypedArray types - which is legal for holding SQL code (as opposed to binary blobs). - - Currently this is the same as isBindableTypedArray() but it - seems likely that we'll eventually want to add Uint32Array - and friends to the isBindableTypedArray() list but not to the - isSQLableTypedArray() list. - */ - const isSQLableTypedArray = isBindableTypedArray; - - /** Returns true if isBindableTypedArray(v) does, else throws with a message - that v is not a supported TypedArray value. */ - const affirmBindableTypedArray = (v)=>{ - return isBindableTypedArray(v) - || toss("Value is not of a supported TypedArray type."); - }; - - /** - The main sqlite3 binding API gets installed into this object, - mimicking the C API as closely as we can. The numerous members - names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as - possible, identically to the C-native counterparts, as documented at: - - https://www.sqlite.org/c3ref/intro.html - - A very few exceptions require an additional level of proxy - function or may otherwise require special attention in the WASM - environment, and all such cases are document here. Those not - documented here are installed as 1-to-1 proxies for their C-side - counterparts. - */ - const api = { - /** - When using sqlite3_open_v2() it is important to keep the following - in mind: - - https://www.sqlite.org/c3ref/open.html - - - The flags for use with its 3rd argument are installed in this - object using the C-cide names, e.g. SQLITE_OPEN_CREATE. - - - If the combination of flags passed to it are invalid, - behavior is undefined. Thus is is never okay to call this - with fewer than 3 arguments, as JS will default the - missing arguments to `undefined`, which will result in a - flag value of 0. Most of the available SQLITE_OPEN_xxx - flags are meaningless in the WASM build, e.g. the mutext- - and cache-related flags, but they are retained in this - API for consistency's sake. - - - The final argument to this function specifies the VFS to - use, which is largely (but not entirely!) meaningless in - the WASM environment. It should always be null or - undefined, and it is safe to elide that argument when - calling this function. - */ - sqlite3_open_v2: function(filename,dbPtrPtr,flags,vfsStr){}/*installed later*/, - - /** - The sqlite3_prepare_v2() binding handles two different uses - with differing JS/WASM semantics: - - 1) sqlite3_prepare_v2(pDb, sqlString, -1, ppStmt [, null]) - - 2) sqlite3_prepare_v2(pDb, sqlPointer, -1, ppStmt, sqlPointerToPointer) - - Note that the SQL length argument (the 3rd argument) must - always be negative because it must be a byte length and - that value is expensive to calculate from JS (where only - the character length of strings is readily available). It - is retained in this API's interface for code/documentation - compatibility reasons but is currently _always_ - ignored. When using the 2nd form of this call, it is - critical that the custom-allocated string be terminated - with a 0 byte. (Potential TODO: if the 3rd argument is >0, - assume the caller knows precisely what they're doing, vis a - vis WASM memory management, and pass it on as-is. That - approach currently seems fraught with peril.) - - In usage (1), the 2nd argument must be of type string, - Uint8Array, or Int8Array (either of which is assumed to - hold SQL). If it is, this function assumes case (1) and - calls the underyling C function with: - - (pDb, sqlAsString, -1, ppStmt, null) - - The pzTail argument is ignored in this case because its result - is meaningless when a string-type value is passed through - (because the string goes through another level of internal - conversion for WASM's sake and the result pointer would refer - to that conversion's memory, not the passed-in string). - - If sql is not a string or supported TypedArray, it must be - a _pointer_ to a string which was allocated via - api.wasm.allocateUTF8OnStack(), api.wasm._malloc(), or - equivalent. In that case, the final argument may be - 0/null/undefined or must be a pointer to which the "tail" - of the compiled SQL is written, as documented for the - C-side sqlite3_prepare_v2(). In case (2), the underlying C - function is called with: - - (pDb, sqlAsPointer, -1, ppStmt, pzTail) - - It returns its result and compiled statement as documented - in the C API. Fetching the output pointers (4th and 5th - parameters) requires using api.wasm.getValue() and the - pzTail will point to an address relative to the - sqlAsPointer value. - - If passed an invalid 2nd argument type, this function will - throw. That behavior is in strong constrast to all of the - other C-bound functions (which return a non-0 result code - on error) but is necessary because we have to way to set - the db's error state such that this function could return a - non-0 integer and the client could call sqlite3_errcode() - or sqlite3_errmsg() to fetch it. - */ - sqlite3_prepare_v2: function(dbPtr, sql, sqlByteLen, stmtPtrPtr, strPtrPtr){}/*installed later*/, - - /** - Holds state which are specific to the WASM-related - infrastructure and glue code. It is not expected that client - code will normally need these, but they're exposed here in case it - does. - - Note that a number of members of this object are injected - dynamically after the api object is fully constructed, so - not all are documented inline here. - */ - wasm: { - /** - api.wasm._malloc()'s srcTypedArray.byteLength bytes, - populates them with the values from the source - TypedArray, and returns the pointer to that memory. The - returned pointer must eventually be passed to - api.wasm._free() to clean it up. - - As a special case, to avoid further special cases where - this is used, if srcTypedArray.byteLength is 0, it - allocates a single byte and sets it to the value - 0. Even in such cases, calls must behave as if the - allocated memory has exactly srcTypedArray.byteLength - bytes. - - ACHTUNG: this currently only works for Uint8Array and - Int8Array types and will throw if srcTypedArray is of - any other type. - */ - mallocFromTypedArray: function(srcTypedArray){ - affirmBindableTypedArray(srcTypedArray); - const pRet = api.wasm._malloc(srcTypedArray.byteLength || 1); - this.heapForSize(srcTypedArray).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); - return pRet; - }, - /** Convenience form of this.heapForSize(8,false). */ - HEAP8: ()=>SQM['HEAP8'], - /** Convenience form of this.heapForSize(8,true). */ - HEAPU8: ()=>SQM['HEAPU8'], - /** - Requires n to be one of (8, 16, 32) or a TypedArray - instance of Int8Array, Int16Array, Int32Array, or their - Uint counterparts. - - Returns the current integer-based TypedArray view of - the WASM heap memory buffer associated with the given - block size. If unsigned is truthy then the "U" - (unsigned) variant of that view is returned, else the - signed variant is returned. If passed a TypedArray - value and no 2nd argument then the 2nd argument - defaults to the signedness of that array. Note that - Float32Array and Float64Array views are not supported - by this function. - - Note that growth of the heap will invalidate any - references to this heap, so do not hold a reference - longer than needed and do not use a reference - after any operation which may allocate. - - Throws if passed an invalid n - */ - heapForSize: function(n,unsigned = true){ - if(isTypedArray(n)){ - if(1===arguments.length){ - unsigned = n instanceof Uint8Array || n instanceof Uint16Array - || n instanceof Uint32Array; - } - n = n.constructor.BYTES_PER_ELEMENT * 8; - } - switch(n){ - case 8: return SQM[unsigned ? 'HEAPU8' : 'HEAP8']; - case 16: return SQM[unsigned ? 'HEAPU16' : 'HEAP16']; - case 32: return SQM[unsigned ? 'HEAPU32' : 'HEAP32']; - } - toss("Invalid heapForSize() size: expecting 8, 16, or 32."); - } - } - }; - [/* C-side functions to bind. Each entry is an array with 3 elements: - - ["c-side name", - "result type" (cwrap() syntax), - [arg types in cwrap() syntax] - ] - */ - ["sqlite3_bind_blob","number",["number", "number", "number", "number", "number"]], - ["sqlite3_bind_double","number",["number", "number", "number"]], - ["sqlite3_bind_int","number",["number", "number", "number"]], - /*Noting that JS/wasm combo does not currently support 64-bit integers: - ["sqlite3_bind_int64","number",["number", "number", "number"]],*/ - ["sqlite3_bind_null","void",["number"]], - ["sqlite3_bind_parameter_count", "number", ["number"]], - ["sqlite3_bind_parameter_index","number",["number", "string"]], - ["sqlite3_bind_text","number",["number", "number", "number", "number", "number"]], - ["sqlite3_changes", "number", ["number"]], - ["sqlite3_clear_bindings","number",["number"]], - ["sqlite3_close_v2", "number", ["number"]], - ["sqlite3_column_blob","number", ["number", "number"]], - ["sqlite3_column_bytes","number",["number", "number"]], - ["sqlite3_column_count", "number", ["number"]], - ["sqlite3_column_count","number",["number"]], - ["sqlite3_column_double","number",["number", "number"]], - ["sqlite3_column_int","number",["number", "number"]], - /*Noting that JS/wasm combo does not currently support 64-bit integers: - ["sqlite3_column_int64","number",["number", "number"]],*/ - ["sqlite3_column_name","string",["number", "number"]], - ["sqlite3_column_text","string",["number", "number"]], - ["sqlite3_column_type","number",["number", "number"]], - ["sqlite3_compileoption_get", "string", ["number"]], - ["sqlite3_compileoption_used", "number", ["string"]], - ["sqlite3_create_function_v2", "number", - ["number", "string", "number", "number","number", - "number", "number", "number", "number"]], - ["sqlite3_data_count", "number", ["number"]], - ["sqlite3_db_filename", "string", ["number", "string"]], - ["sqlite3_errmsg", "string", ["number"]], - ["sqlite3_exec", "number", ["number", "string", "number", "number", "number"]], - ["sqlite3_extended_result_codes", "number", ["number", "number"]], - ["sqlite3_finalize", "number", ["number"]], - ["sqlite3_interrupt", "void", ["number"]], - ["sqlite3_libversion", "string", []], - ["sqlite3_open", "number", ["string", "number"]], - ["sqlite3_open_v2", "number", ["string", "number", "number", "string"]], - /* sqlite3_prepare_v2() is handled separately due to us requiring two - different sets of semantics for that function. */ - ["sqlite3_reset", "number", ["number"]], - ["sqlite3_result_blob",null,["number", "number", "number", "number"]], - ["sqlite3_result_double",null,["number", "number"]], - ["sqlite3_result_error",null,["number", "string", "number"]], - ["sqlite3_result_int",null,["number", "number"]], - ["sqlite3_result_null",null,["number"]], - ["sqlite3_result_text",null,["number", "string", "number", "number"]], - ["sqlite3_sourceid", "string", []], - ["sqlite3_sql", "string", ["number"]], - ["sqlite3_step", "number", ["number"]], - ["sqlite3_value_blob", "number", ["number"]], - ["sqlite3_value_bytes","number",["number"]], - ["sqlite3_value_double","number",["number"]], - ["sqlite3_value_text", "string", ["number"]], - ["sqlite3_value_type", "number", ["number"]] - //["sqlite3_normalized_sql", "string", ["number"]] - ].forEach((a)=>api[a[0]] = SQM.cwrap.apply(this, a)); - - /** - Proxies for variants of sqlite3_prepare_v2() which have - differing JS/WASM binding semantics. - */ - const prepareMethods = { - /** - This binding expects a JS string as its 2nd argument and - null as its final argument. In order to compile multiple - statements from a single string, the "full" impl (see - below) must be used. - */ - basic: SQM.cwrap('sqlite3_prepare_v2', - "number", ["number", "string", "number"/*MUST always be negative*/, - "number", "number"/*MUST be 0 or null or undefined!*/]), - /* Impl which requires that the 2nd argument be a pointer to - the SQL string, instead of being converted to a - string. This variant is necessary for cases where we - require a non-NULL value for the final argument - (exec()'ing multiple statements from one input - string). For simpler cases, where only the first statement - in the SQL string is required, the wrapper named - sqlite3_prepare_v2() is sufficient and easier to use - because it doesn't require dealing with pointers. - - TODO: hide both of these methods behind a single hand-written - sqlite3_prepare_v2() wrapper which dispatches to the appropriate impl. - */ - full: SQM.cwrap('sqlite3_prepare_v2', - "number", ["number", "number", "number"/*MUST always be negative*/, - "number", "number"]), - }; - - /* Import C-level constants... */ - //console.log("wasmEnum=",SQM.ccall('sqlite3_wasm_enum_json', 'string', [])); - const wasmEnum = JSON.parse(SQM.ccall('sqlite3_wasm_enum_json', 'string', [])); - ['blobFinalizers', 'dataTypes','encodings', - 'openFlags', 'resultCodes','udfFlags' - ].forEach(function(t){ - Object.keys(wasmEnum[t]).forEach(function(k){ - api[k] = wasmEnum[t][k]; - }); - }); - - const utf8Decoder = new TextDecoder('utf-8'); - const typedArrayToString = (str)=>utf8Decoder.decode(str); - //const stringToUint8 = (sql)=>new TextEncoder('utf-8').encode(sql); - - /* Documented inline in the api object. */ - api.sqlite3_prepare_v2 = function(pDb, sql, sqlLen, ppStmt, pzTail){ - if(isSQLableTypedArray(sql)) sql = typedArrayToString(sql); - switch(typeof sql){ - case 'string': return prepareMethods.basic(pDb, sql, -1, ppStmt, null); - case 'number': return prepareMethods.full(pDb, sql, -1, ppStmt, pzTail); - default: toss("Invalid SQL argument type for sqlite3_prepare_v2()."); - } - }; - - /** - Populate api.wasm with several members of the module object. Some of these - will be required by higher-level code. At a minimum: - - getValue(), setValue(), stackSave(), stackRestore(), stackAlloc(), _malloc(), - _free(), addFunction(), removeFunction() - - The rest are exposed primarily for internal use in this API but may well - be useful from higher-level client code. - - All of the functions injected here are part of the - Emscripten-exposed APIs and are documented "elsewhere". Some - are documented in the Emscripten-generated `sqlite3.js` and - some are documented (if at all) in places unknown, possibly - even inaccessible, to us. - */ - [ - // Memory management: - 'getValue','setValue', 'stackSave', 'stackRestore', 'stackAlloc', - 'allocateUTF8OnStack', '_malloc', '_free', - // String utilities: - 'intArrayFromString', 'lengthBytesUTF8', 'stringToUTF8Array', - // The obligatory "misc" category: - 'addFunction', 'removeFunction' - ].forEach(function(m){ - if(undefined === (api.wasm[m] = SQM[m])){ - toss("Internal init error: Module."+m+" not found."); - } - }); - - /* What follows is colloquially known as "OO API #1". It is a - binding of the sqlite3 API which is designed to be run within - the same thread (main or worker) as the one in which the - sqlite3 WASM binding was initialized. This wrapper cannot use - the sqlite3 binding if, e.g., the wrapper is in the main thread - and the sqlite3 API is in a worker. */ - - /** - The DB class wraps a sqlite3 db handle. - - It accepts the following argument signatures: - - - () - - (undefined) (same effect as ()) - - (filename[,buffer]) - - (buffer) - - Where a buffer indicates a Uint8Array holding an sqlite3 db - image. - - If the filename is provided, only the last component of the - path is used - any path prefix is stripped and certain - "special" characters are replaced with `_`. If no name is - provided, a random name is generated. The resulting filename is - the one used for accessing the db file within root directory of - the emscripten-supplied virtual filesystem, and is set (with no - path part) as the DB object's `filename` property. - - Note that the special sqlite3 db names ":memory:" and "" - (temporary db) have no special meanings here. We can apparently - only export images of DBs which are stored in the - pseudo-filesystem provided by the JS APIs. Since exporting and - importing images is an important usability feature for this - class, ":memory:" DBs are not supported (until/unless we can - find a way to export those as well). The naming semantics will - certainly evolve as this API does. - - The db is opened with a fixed set of flags: - (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXRESCODE). This API may change in the future - permit the caller to provide those flags via an additional - argument. - */ - const DB = function(arg){ - let buffer, fn; - if(arg instanceof Uint8Array){ - buffer = arg; - arg = undefined; - }else if(arguments.length){ /*(filename[,buffer])*/ - if('string'===typeof arg){ - const p = arg.split('/').pop().replace(':','_'); - if(p) fn = p; - if(arguments.length>1){ - buffer = arguments[1]; - } - }else if(undefined!==arg){ - toss("Invalid arguments to DB constructor.", - "Expecting (), (undefined), (name,buffer),", - "or (buffer), where buffer an sqlite3 db ", - "as a Uint8Array."); - } - } - if(!fn){ - fn = "db-"+((Math.random() * 10000000) | 0)+ - "-"+((Math.random() * 10000000) | 0)+".sqlite3"; - } - if(buffer){ - if(!(buffer instanceof Uint8Array)){ - toss("Expecting Uint8Array image of db contents."); - } - FS.createDataFile("/", fn, buffer, true, true); - } - const stack = api.wasm.stackSave(); - const ppDb = api.wasm.stackAlloc(4) /* output (sqlite3**) arg */; - api.wasm.setValue(ppDb, 0, "i32"); - try { - this.checkRc(api.sqlite3_open_v2(fn, ppDb, api.SQLITE_OPEN_READWRITE - | api.SQLITE_OPEN_CREATE - | api.SQLITE_OPEN_EXRESCODE, null)); - } - finally{api.wasm.stackRestore(stack);} - this._pDb = api.wasm.getValue(ppDb, "i32"); - this.filename = fn; - this._statements = {/*map of open Stmt _pointers_ to Stmt*/}; - this._udfs = {/*map of UDF names to wasm function _pointers_*/}; - }; - - /** - Internal-use enum for mapping JS types to DB-bindable types. - These do not (and need not) line up with the SQLITE_type - values. All values in this enum must be truthy and distinct - but they need not be numbers. - */ - const BindTypes = { - null: 1, - number: 2, - string: 3, - boolean: 4, - blob: 5 - }; - BindTypes['undefined'] == BindTypes.null; - - /** - This class wraps sqlite3_stmt. Calling this constructor - directly will trigger an exception. Use DB.prepare() to create - new instances. - */ - const Stmt = function(){ - if(BindTypes!==arguments[2]){ - toss("Do not call the Stmt constructor directly. Use DB.prepare()."); - } - this.db = arguments[0]; - this._pStmt = arguments[1]; - this.columnCount = api.sqlite3_column_count(this._pStmt); - this.parameterCount = api.sqlite3_bind_parameter_count(this._pStmt); - }; - - /** Throws if the given DB has been closed, else it is returned. */ - const affirmDbOpen = function(db){ - if(!db._pDb) toss("DB has been closed."); - return db; - }; - - /** - Expects to be passed (arguments) from DB.exec() and - DB.execMulti(). Does the argument processing/validation, throws - on error, and returns a new object on success: - - { sql: the SQL, opt: optionsObj, cbArg: function} - - cbArg is only set if the opt.callback is set, in which case - it's a function which expects to be passed the current Stmt - and returns the callback argument of the type indicated by - the input arguments. - */ - const parseExecArgs = function(args){ - const out = {opt:{}}; - switch(args.length){ - case 1: - if('string'===typeof args[0] || isSQLableTypedArray(args[0])){ - out.sql = args[0]; - }else if(args[0] && 'object'===typeof args[0]){ - out.opt = args[0]; - out.sql = out.opt.sql; - } - break; - case 2: - out.sql = args[0]; - out.opt = args[1]; - break; - default: toss("Invalid argument count for exec()."); - }; - if(isSQLableTypedArray(out.sql)){ - out.sql = typedArrayToString(out.sql); - }else if(Array.isArray(out.sql)){ - out.sql = out.sql.join(''); - }else if('string'!==typeof out.sql){ - toss("Missing SQL argument."); - } - if(out.opt.callback || out.opt.resultRows){ - switch((undefined===out.opt.rowMode) - ? 'stmt' : out.opt.rowMode) { - case 'object': out.cbArg = (stmt)=>stmt.get({}); break; - case 'array': out.cbArg = (stmt)=>stmt.get([]); break; - case 'stmt': out.cbArg = (stmt)=>stmt; break; - default: toss("Invalid rowMode:",out.opt.rowMode); - } - } - return out; - }; - - /** If object opts has _its own_ property named p then that - property's value is returned, else dflt is returned. */ - const getOwnOption = (opts, p, dflt)=> - opts.hasOwnProperty(p) ? opts[p] : dflt; - - DB.prototype = { - /** - Expects to be given an sqlite3 API result code. If it is - falsy, this function returns this object, else it throws an - exception with an error message from sqlite3_errmsg(), - using this object's db handle. Note that if it's passed a - non-error code like SQLITE_ROW or SQLITE_DONE, it will - still throw but the error string might be "Not an error." - The various non-0 non-error codes need to be checked for in - client code where they are expected. - */ - checkRc: function(sqliteResultCode){ - if(!sqliteResultCode) return this; - toss("sqlite result code",sqliteResultCode+":", - api.sqlite3_errmsg(this._pDb) || "Unknown db error."); - }, - /** - Finalizes all open statements and closes this database - connection. This is a no-op if the db has already been - closed. If the db is open and alsoUnlink is truthy then the - this.filename entry in the pseudo-filesystem will also be - removed (and any error in that attempt is silently - ignored). - */ - close: function(alsoUnlink){ - if(this._pDb){ - let s; - const that = this; - Object.keys(this._statements).forEach(function(k,s){ - delete that._statements[k]; - if(s && s._pStmt) s.finalize(); - }); - Object.values(this._udfs).forEach(api.wasm.removeFunction); - delete this._udfs; - delete this._statements; - api.sqlite3_close_v2(this._pDb); - delete this._pDb; - if(this.filename){ - if(alsoUnlink){ - try{SQM.FS.unlink('/'+this.filename);} - catch(e){/*ignored*/} - } - delete this.filename; - } - } - }, - /** - Similar to this.filename but will return NULL for - special names like ":memory:". Not of much use until - we have filesystem support. Throws if the DB has - been closed. If passed an argument it then it will return - the filename of the ATTACHEd db with that name, else it assumes - a name of `main`. - */ - fileName: function(dbName){ - return api.sqlite3_db_filename(affirmDbOpen(this)._pDb, dbName||"main"); - }, - /** - Compiles the given SQL and returns a prepared Stmt. This is - the only way to create new Stmt objects. Throws on error. - - The given SQL must be a string, a Uint8Array holding SQL, - or a WASM pointer to memory allocated using - api.wasm.allocateUTF8OnStack() (or equivalent (a term which - is yet to be defined precisely)). - */ - prepare: function(sql){ - affirmDbOpen(this); - const stack = api.wasm.stackSave(); - const ppStmt = api.wasm.stackAlloc(4)/* output (sqlite3_stmt**) arg */; - api.wasm.setValue(ppStmt, 0, "i32"); - try {this.checkRc(api.sqlite3_prepare_v2(this._pDb, sql, -1, ppStmt, null));} - finally {api.wasm.stackRestore(stack);} - const pStmt = api.wasm.getValue(ppStmt, "i32"); - if(!pStmt) toss("Cannot prepare empty SQL."); - const stmt = new Stmt(this, pStmt, BindTypes); - this._statements[pStmt] = stmt; - return stmt; - }, - /** - This function works like execMulti(), and takes most of the - same arguments, but is more efficient (performs much less - work) when the input SQL is only a single statement. If - passed a multi-statement SQL, it only processes the first - one. - - This function supports the following additional options not - supported by execMulti(): - - - .multi: if true, this function acts as a proxy for - execMulti() and behaves identically to that function. - - - .resultRows: if this is an array, each row of the result - set (if any) is appended to it in the format specified - for the `rowMode` property, with the exception that the - only legal values for `rowMode` in this case are 'array' - or 'object', neither of which is the default. It is legal - to use both `resultRows` and `callback`, but `resultRows` - is likely much simpler to use for small data sets and can - be used over a WebWorker-style message interface. - - - .columnNames: if this is an array and the query has - result columns, the array is passed to - Stmt.getColumnNames() to append the column names to it - (regardless of whether the query produces any result - rows). If the query has no result columns, this value is - unchanged. - - The following options to execMulti() are _not_ supported by - this method (they are simply ignored): - - - .saveSql - */ - exec: function(/*(sql [,optionsObj]) or (optionsObj)*/){ - affirmDbOpen(this); - const arg = parseExecArgs(arguments); - if(!arg.sql) return this; - else if(arg.opt.multi){ - return this.execMulti(arg, undefined, BindTypes); - } - const opt = arg.opt; - let stmt, rowTarget; - try { - if(Array.isArray(opt.resultRows)){ - if(opt.rowMode!=='array' && opt.rowMode!=='object'){ - toss("Invalid rowMode for resultRows array: must", - "be one of 'array' or 'object'."); - } - rowTarget = opt.resultRows; - } - stmt = this.prepare(arg.sql); - if(stmt.columnCount && Array.isArray(opt.columnNames)){ - stmt.getColumnNames(opt.columnNames); - } - if(opt.bind) stmt.bind(opt.bind); - if(opt.callback || rowTarget){ - while(stmt.step()){ - const row = arg.cbArg(stmt); - if(rowTarget) rowTarget.push(row); - if(opt.callback){ - stmt._isLocked = true; - opt.callback(row, stmt); - stmt._isLocked = false; - } - } - }else{ - stmt.step(); - } - }finally{ - if(stmt){ - delete stmt._isLocked; - stmt.finalize(); - } - } - return this; - - }/*exec()*/, - /** - Executes one or more SQL statements in the form of a single - string. Its arguments must be either (sql,optionsObject) or - (optionsObject). In the latter case, optionsObject.sql - must contain the SQL to execute. Returns this - object. Throws on error. - - If no SQL is provided, or a non-string is provided, an - exception is triggered. Empty SQL, on the other hand, is - simply a no-op. - - The optional options object may contain any of the following - properties: - - - .sql = the SQL to run (unless it's provided as the first - argument). This must be of type string, Uint8Array, or an - array of strings (in which case they're concatenated - together as-is, with no separator between elements, - before evaluation). - - - .bind = a single value valid as an argument for - Stmt.bind(). This is ONLY applied to the FIRST non-empty - statement in the SQL which has any bindable - parameters. (Empty statements are skipped entirely.) - - - .callback = a function which gets called for each row of - the FIRST statement in the SQL which has result - _columns_, but only if that statement has any result - _rows_. The second argument passed to the callback is - always the current Stmt object (so that the caller may - collect column names, or similar). The first argument - passed to the callback defaults to the current Stmt - object but may be changed with ... - - - .rowMode = a string describing what type of argument - should be passed as the first argument to the callback. A - value of 'object' causes the results of `stmt.get({})` to - be passed to the object. A value of 'array' causes the - results of `stmt.get([])` to be passed to the callback. - A value of 'stmt' is equivalent to the default, passing - the current Stmt to the callback (noting that it's always - passed as the 2nd argument). Any other value triggers an - exception. - - - saveSql = an optional array. If set, the SQL of each - executed statement is appended to this array before the - statement is executed (but after it is prepared - we - don't have the string until after that). Empty SQL - statements are elided. - - See also the exec() method, which is a close cousin of this - one. - - ACHTUNG #1: The callback MUST NOT modify the Stmt - object. Calling any of the Stmt.get() variants, - Stmt.getColumnName(), or similar, is legal, but calling - step() or finalize() is not. Routines which are illegal - in this context will trigger an exception. - - ACHTUNG #2: The semantics of the `bind` and `callback` - options may well change or those options may be removed - altogether for this function (but retained for exec()). - Generally speaking, neither bind parameters nor a callback - are generically useful when executing multi-statement SQL. - */ - execMulti: function(/*(sql [,obj]) || (obj)*/){ - affirmDbOpen(this); - const arg = (BindTypes===arguments[2] - /* ^^^ Being passed on from exec() */ - ? arguments[0] : parseExecArgs(arguments)); - if(!arg.sql) return this; - const opt = arg.opt; - const stack = api.wasm.stackSave(); - let stmt; - let bind = opt.bind; - let rowMode = ( - (opt.callback && opt.rowMode) - ? opt.rowMode : false); - try{ - const sql = isSQLableTypedArray(arg.sql) - ? typedArrayToString(arg.sql) - : arg.sql; - let pSql = api.wasm.allocateUTF8OnStack(sql) - const ppStmt = api.wasm.stackAlloc(8) /* output (sqlite3_stmt**) arg */; - const pzTail = ppStmt + 4 /* final arg to sqlite3_prepare_v2_sqlptr() */; - while(api.wasm.getValue(pSql, "i8")){ - api.wasm.setValue(ppStmt, 0, "i32"); - api.wasm.setValue(pzTail, 0, "i32"); - this.checkRc(api.sqlite3_prepare_v2( - this._pDb, pSql, -1, ppStmt, pzTail - )); - const pStmt = api.wasm.getValue(ppStmt, "i32"); - pSql = api.wasm.getValue(pzTail, "i32"); - if(!pStmt) continue; - if(opt.saveSql){ - opt.saveSql.push(api.sqlite3_sql(pStmt).trim()); - } - stmt = new Stmt(this, pStmt, BindTypes); - if(bind && stmt.parameterCount){ - stmt.bind(bind); - bind = null; - } - if(opt.callback && null!==rowMode && stmt.columnCount){ - while(stmt.step()){ - stmt._isLocked = true; - callback(arg.cbArg(stmt), stmt); - stmt._isLocked = false; - } - rowMode = null; - }else{ - // Do we need to while(stmt.step()){} here? - stmt.step(); - } - stmt.finalize(); - stmt = null; - } - }finally{ - if(stmt){ - delete stmt._isLocked; - stmt.finalize(); - } - api.wasm.stackRestore(stack); - } - return this; - }/*execMulti()*/, - /** - Creates a new scalar UDF (User-Defined Function) which is - accessible via SQL code. This function may be called in any - of the following forms: - - - (name, function) - - (name, function, optionsObject) - - (name, optionsObject) - - (optionsObject) - - In the final two cases, the function must be defined as the - 'callback' property of the options object. In the final - case, the function's name must be the 'name' property. - - This can only be used to create scalar functions, not - aggregate or window functions. UDFs cannot be removed from - a DB handle after they're added. - - On success, returns this object. Throws on error. - - When called from SQL, arguments to the UDF, and its result, - will be converted between JS and SQL with as much fidelity - as is feasible, triggering an exception if a type - conversion cannot be determined. Some freedom is afforded - to numeric conversions due to friction between the JS and C - worlds: integers which are larger than 32 bits will be - treated as doubles, as JS does not support 64-bit integers - and it is (as of this writing) illegal to use WASM - functions which take or return 64-bit integers from JS. - - The optional options object may contain flags to modify how - the function is defined: - - - .arity: the number of arguments which SQL calls to this - function expect or require. The default value is the - callback's length property (i.e. the number of declared - parameters it has). A value of -1 means that the function - is variadic and may accept any number of arguments, up to - sqlite3's compile-time limits. sqlite3 will enforce the - argument count if is zero or greater. - - The following properties correspond to flags documented at: - - https://sqlite.org/c3ref/create_function.html - - - .deterministic = SQLITE_DETERMINISTIC - - .directOnly = SQLITE_DIRECTONLY - - .innocuous = SQLITE_INNOCUOUS - - Maintenance reminder: the ability to add new - WASM-accessible functions to the runtime requires that the - WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` - flag. - */ - createFunction: function f(name, callback,opt){ - switch(arguments.length){ - case 1: /* (optionsObject) */ - opt = name; - name = opt.name; - callback = opt.callback; - break; - case 2: /* (name, callback|optionsObject) */ - if(!(callback instanceof Function)){ - opt = callback; - callback = opt.callback; - } - break; - default: break; - } - if(!opt) opt = {}; - if(!(callback instanceof Function)){ - toss("Invalid arguments: expecting a callback function."); - }else if('string' !== typeof name){ - toss("Invalid arguments: missing function name."); - } - if(!f._extractArgs){ - /* Static init */ - f._extractArgs = function(argc, pArgv){ - let i, pVal, valType, arg; - const tgt = []; - for(i = 0; i < argc; ++i){ - pVal = api.wasm.getValue(pArgv + (4 * i), "i32"); - valType = api.sqlite3_value_type(pVal); - switch(valType){ - case api.SQLITE_INTEGER: - case api.SQLITE_FLOAT: - arg = api.sqlite3_value_double(pVal); - break; - case api.SQLITE_TEXT: - arg = api.sqlite3_value_text(pVal); - break; - case api.SQLITE_BLOB:{ - const n = api.sqlite3_value_bytes(pVal); - const pBlob = api.sqlite3_value_blob(pVal); - arg = new Uint8Array(n); - let i; - const heap = n ? api.wasm.HEAP8() : false; - for(i = 0; i < n; ++i) arg[i] = heap[pBlob+i]; - break; - } - default: - arg = null; break; - } - tgt.push(arg); - } - return tgt; - }/*_extractArgs()*/; - f._setResult = function(pCx, val){ - switch(typeof val) { - case 'boolean': - api.sqlite3_result_int(pCx, val ? 1 : 0); - break; - case 'number': { - (isInt32(val) - ? api.sqlite3_result_int - : api.sqlite3_result_double)(pCx, val); - break; - } - case 'string': - api.sqlite3_result_text(pCx, val, -1, api.SQLITE_TRANSIENT); - break; - case 'object': - if(null===val) { - api.sqlite3_result_null(pCx); - break; - }else if(isBindableTypedArray(val)){ - const pBlob = api.wasm.mallocFromTypedArray(val); - api.sqlite3_result_blob(pCx, pBlob, val.byteLength, - api.SQLITE_TRANSIENT); - api.wasm._free(pBlob); - break; - } - // else fall through - default: - toss("Don't not how to handle this UDF result value:",val); - }; - }/*_setResult()*/; - }/*static init*/ - const wrapper = function(pCx, argc, pArgv){ - try{ - f._setResult(pCx, callback.apply(null, f._extractArgs(argc, pArgv))); - }catch(e){ - api.sqlite3_result_error(pCx, e.message, -1); - } - }; - const pUdf = api.wasm.addFunction(wrapper, "viii"); - let fFlags = 0; - if(getOwnOption(opt, 'deterministic')) fFlags |= api.SQLITE_DETERMINISTIC; - if(getOwnOption(opt, 'directOnly')) fFlags |= api.SQLITE_DIRECTONLY; - if(getOwnOption(opt, 'innocuous')) fFlags |= api.SQLITE_INNOCUOUS; - name = name.toLowerCase(); - try { - this.checkRc(api.sqlite3_create_function_v2( - this._pDb, name, - (opt.hasOwnProperty('arity') ? +opt.arity : callback.length), - api.SQLITE_UTF8 | fFlags, null/*pApp*/, pUdf, - null/*xStep*/, null/*xFinal*/, null/*xDestroy*/)); - }catch(e){ - api.wasm.removeFunction(pUdf); - throw e; - } - if(this._udfs.hasOwnProperty(name)){ - api.wasm.removeFunction(this._udfs[name]); - } - this._udfs[name] = pUdf; - return this; - }/*createFunction()*/, - /** - Prepares the given SQL, step()s it one time, and returns - the value of the first result column. If it has no results, - undefined is returned. If passed a second argument, it is - treated like an argument to Stmt.bind(), so may be any type - supported by that function. Throws on error (e.g. malformed - SQL). - */ - selectValue: function(sql,bind){ - let stmt, rc; - try { - stmt = this.prepare(sql).bind(bind); - if(stmt.step()) rc = stmt.get(0); - }finally{ - if(stmt) stmt.finalize(); - } - return rc; - }, - - /** - Exports a copy of this db's file as a Uint8Array and - returns it. It is technically not legal to call this while - any prepared statement are currently active because, - depending on the platform, it might not be legal to read - the db while a statement is locking it. Throws if this db - is not open or has any opened statements. - - The resulting buffer can be passed to this class's - constructor to restore the DB. - - Maintenance reminder: the corresponding sql.js impl of this - feature closes the current db, finalizing any active - statements and (seemingly unnecessarily) destroys any UDFs, - copies the file, and then re-opens it (without restoring - the UDFs). Those gymnastics are not necessary on the tested - platform but might be necessary on others. Because of that - eventuality, this interface currently enforces that no - statements are active when this is run. It will throw if - any are. - */ - exportBinaryImage: function(){ - affirmDbOpen(this); - if(Object.keys(this._statements).length){ - toss("Cannot export with prepared statements active!", - "finalize() all statements and try again."); - } - return FS.readFile(this.filename, {encoding:"binary"}); - } - }/*DB.prototype*/; - - - /** Throws if the given Stmt has been finalized, else stmt is - returned. */ - const affirmStmtOpen = function(stmt){ - if(!stmt._pStmt) toss("Stmt has been closed."); - return stmt; - }; - - /** Returns an opaque truthy value from the BindTypes - enum if v's type is a valid bindable type, else - returns a falsy value. As a special case, a value of - undefined is treated as a bind type of null. */ - const isSupportedBindType = function(v){ - let t = BindTypes[(null===v||undefined===v) ? 'null' : typeof v]; - switch(t){ - case BindTypes.boolean: - case BindTypes.null: - case BindTypes.number: - case BindTypes.string: - return t; - default: - //console.log("isSupportedBindType",t,v); - return isBindableTypedArray(v) ? BindTypes.blob : undefined; - } - }; - - /** - If isSupportedBindType(v) returns a truthy value, this - function returns that value, else it throws. - */ - const affirmSupportedBindType = function(v){ - //console.log('affirmSupportedBindType',v); - return isSupportedBindType(v) || toss("Unsupported bind() argument type:",typeof v); - }; - - /** - If key is a number and within range of stmt's bound parameter - count, key is returned. - - If key is not a number then it is checked against named - parameters. If a match is found, its index is returned. - - Else it throws. - */ - const affirmParamIndex = function(stmt,key){ - const n = ('number'===typeof key) - ? key : api.sqlite3_bind_parameter_index(stmt._pStmt, key); - if(0===n || !isInt32(n)){ - toss("Invalid bind() parameter name: "+key); - } - else if(n<1 || n>stmt.parameterCount) toss("Bind index",key,"is out of range."); - return n; - }; - - /** Throws if ndx is not an integer or if it is out of range - for stmt.columnCount, else returns stmt. - - Reminder: this will also fail after the statement is finalized - but the resulting error will be about an out-of-bounds column - index. - */ - const affirmColIndex = function(stmt,ndx){ - if((ndx !== (ndx|0)) || ndx<0 || ndx>=stmt.columnCount){ - toss("Column index",ndx,"is out of range."); - } - return stmt; - }; - - /** - If stmt._isLocked is truthy, this throws an exception - complaining that the 2nd argument (an operation name, - e.g. "bind()") is not legal while the statement is "locked". - Locking happens before an exec()-like callback is passed a - statement, to ensure that the callback does not mutate or - finalize the statement. If it does not throw, it returns stmt. - */ - const affirmUnlocked = function(stmt,currentOpName){ - if(stmt._isLocked){ - toss("Operation is illegal when statement is locked:",currentOpName); - } - return stmt; - }; - - /** - Binds a single bound parameter value on the given stmt at the - given index (numeric or named) using the given bindType (see - the BindTypes enum) and value. Throws on error. Returns stmt on - success. - */ - const bindOne = function f(stmt,ndx,bindType,val){ - affirmUnlocked(stmt, 'bind()'); - if(!f._){ - f._ = { - string: function(stmt, ndx, val, asBlob){ - if(1){ - /* _Hypothetically_ more efficient than the impl in the 'else' block. */ - const stack = api.wasm.stackSave(); - try{ - const n = api.wasm.lengthBytesUTF8(val)+1/*required for NUL terminator*/; - const pStr = api.wasm.stackAlloc(n); - api.wasm.stringToUTF8Array(val, api.wasm.HEAPU8(), pStr, n); - const f = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; - return f(stmt._pStmt, ndx, pStr, n-1, api.SQLITE_TRANSIENT); - }finally{ - api.wasm.stackRestore(stack); - } - }else{ - const bytes = api.wasm.intArrayFromString(val,true); - const pStr = api.wasm._malloc(bytes.length || 1); - api.wasm.HEAPU8().set(bytes.length ? bytes : [0], pStr); - try{ - const f = asBlob ? api.sqlite3_bind_blob : api.sqlite3_bind_text; - return f(stmt._pStmt, ndx, pStr, bytes.length, api.SQLITE_TRANSIENT); - }finally{ - api.wasm._free(pStr); - } - } - } - }; - } - affirmSupportedBindType(val); - ndx = affirmParamIndex(stmt,ndx); - let rc = 0; - switch((null===val || undefined===val) ? BindTypes.null : bindType){ - case BindTypes.null: - rc = api.sqlite3_bind_null(stmt._pStmt, ndx); - break; - case BindTypes.string:{ - rc = f._.string(stmt, ndx, val, false); - break; - } - case BindTypes.number: { - const m = (isInt32(val) - ? api.sqlite3_bind_int - /*It's illegal to bind a 64-bit int - from here*/ - : api.sqlite3_bind_double); - rc = m(stmt._pStmt, ndx, val); - break; - } - case BindTypes.boolean: - rc = api.sqlite3_bind_int(stmt._pStmt, ndx, val ? 1 : 0); - break; - case BindTypes.blob: { - if('string'===typeof val){ - rc = f._.string(stmt, ndx, val, true); - }else if(!isBindableTypedArray(val)){ - toss("Binding a value as a blob requires", - "that it be a string, Uint8Array, or Int8Array."); - }else if(1){ - /* _Hypothetically_ more efficient than the impl in the 'else' block. */ - const stack = api.wasm.stackSave(); - try{ - const pBlob = api.wasm.stackAlloc(val.byteLength || 1); - api.wasm.HEAP8().set(val.byteLength ? val : [0], pBlob) - rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, val.byteLength, - api.SQLITE_TRANSIENT); - }finally{ - api.wasm.stackRestore(stack); - } - }else{ - const pBlob = api.wasm.mallocFromTypedArray(val); - try{ - rc = api.sqlite3_bind_blob(stmt._pStmt, ndx, pBlob, val.byteLength, - api.SQLITE_TRANSIENT); - }finally{ - api.wasm._free(pBlob); - } - } - break; - } - default: - console.warn("Unsupported bind() argument type:",val); - toss("Unsupported bind() argument type."); - } - if(rc) stmt.db.checkRc(rc); - return stmt; - }; - - Stmt.prototype = { - /** - "Finalizes" this statement. This is a no-op if the - statement has already been finalizes. Returns - undefined. Most methods in this class will throw if called - after this is. - */ - finalize: function(){ - if(this._pStmt){ - affirmUnlocked(this,'finalize()'); - delete this.db._statements[this._pStmt]; - api.sqlite3_finalize(this._pStmt); - delete this.columnCount; - delete this.parameterCount; - delete this._pStmt; - delete this.db; - delete this._isLocked; - } - }, - /** Clears all bound values. Returns this object. - Throws if this statement has been finalized. */ - clearBindings: function(){ - affirmUnlocked(affirmStmtOpen(this), 'clearBindings()') - api.sqlite3_clear_bindings(this._pStmt); - this._mayGet = false; - return this; - }, - /** - Resets this statement so that it may be step()ed again - from the beginning. Returns this object. Throws if this - statement has been finalized. - - If passed a truthy argument then this.clearBindings() is - also called, otherwise any existing bindings, along with - any memory allocated for them, are retained. - */ - reset: function(alsoClearBinds){ - affirmUnlocked(this,'reset()'); - if(alsoClearBinds) this.clearBindings(); - api.sqlite3_reset(affirmStmtOpen(this)._pStmt); - this._mayGet = false; - return this; - }, - /** - Binds one or more values to its bindable parameters. It - accepts 1 or 2 arguments: - - If passed a single argument, it must be either an array, an - object, or a value of a bindable type (see below). - - If passed 2 arguments, the first one is the 1-based bind - index or bindable parameter name and the second one must be - a value of a bindable type. - - Bindable value types: - - - null is bound as NULL. - - - undefined as a standalone value is a no-op intended to - simplify certain client-side use cases: passing undefined - as a value to this function will not actually bind - anything and this function will skip confirmation that - binding is even legal. (Those semantics simplify certain - client-side uses.) Conversely, a value of undefined as an - array or object property when binding an array/object - (see below) is treated the same as null. - - - Numbers are bound as either doubles or integers: doubles - if they are larger than 32 bits, else double or int32, - depending on whether they have a fractional part. (It is, - as of this writing, illegal to call (from JS) a WASM - function which either takes or returns an int64.) - Booleans are bound as integer 0 or 1. It is not expected - the distinction of binding doubles which have no - fractional parts is integers is significant for the - majority of clients due to sqlite3's data typing - model. This API does not currently support the BigInt - type. - - - Strings are bound as strings (use bindAsBlob() to force - blob binding). - - - Uint8Array and Int8Array instances are bound as blobs. - (TODO: support binding other TypedArray types with larger - int sizes.) - - If passed an array, each element of the array is bound at - the parameter index equal to the array index plus 1 - (because arrays are 0-based but binding is 1-based). - - If passed an object, each object key is treated as a - bindable parameter name. The object keys _must_ match any - bindable parameter names, including any `$`, `@`, or `:` - prefix. Because `$` is a legal identifier chararacter in - JavaScript, that is the suggested prefix for bindable - parameters: `stmt.bind({$a: 1, $b: 2})`. - - It returns this object on success and throws on - error. Errors include: - - - Any bind index is out of range, a named bind parameter - does not match, or this statement has no bindable - parameters. - - - Any value to bind is of an unsupported type. - - - Passed no arguments or more than two. - - - The statement has been finalized. - */ - bind: function(/*[ndx,] arg*/){ - affirmStmtOpen(this); - let ndx, arg; - switch(arguments.length){ - case 1: ndx = 1; arg = arguments[0]; break; - case 2: ndx = arguments[0]; arg = arguments[1]; break; - default: toss("Invalid bind() arguments."); - } - if(undefined===arg){ - /* It might seem intuitive to bind undefined as NULL - but this approach simplifies certain client-side - uses when passing on arguments between 2+ levels of - functions. */ - return this; - }else if(!this.parameterCount){ - toss("This statement has no bindable parameters."); - } - this._mayGet = false; - if(null===arg){ - /* bind NULL */ - return bindOne(this, ndx, BindTypes.null, arg); - } - else if(Array.isArray(arg)){ - /* bind each entry by index */ - if(1!==arguments.length){ - toss("When binding an array, an index argument is not permitted."); - } - arg.forEach((v,i)=>bindOne(this, i+1, affirmSupportedBindType(v), v)); - return this; - } - else if('object'===typeof arg/*null was checked above*/ - && !isBindableTypedArray(arg)){ - /* Treat each property of arg as a named bound parameter. */ - if(1!==arguments.length){ - toss("When binding an object, an index argument is not permitted."); - } - Object.keys(arg) - .forEach(k=>bindOne(this, k, - affirmSupportedBindType(arg[k]), - arg[k])); - return this; - }else{ - return bindOne(this, ndx, affirmSupportedBindType(arg), arg); - } - toss("Should not reach this point."); - }, - /** - Special case of bind() which binds the given value using - the BLOB binding mechanism instead of the default selected - one for the value. The ndx may be a numbered or named bind - index. The value must be of type string, null/undefined - (both treated as null), or a TypedArray of a type supported - by the bind() API. - - If passed a single argument, a bind index of 1 is assumed. - */ - bindAsBlob: function(ndx,arg){ - affirmStmtOpen(this); - if(1===arguments.length){ - arg = ndx; - ndx = 1; - } - const t = affirmSupportedBindType(arg); - if(BindTypes.string !== t && BindTypes.blob !== t - && BindTypes.null !== t){ - toss("Invalid value type for bindAsBlob()"); - } - this._mayGet = false; - return bindOne(this, ndx, BindTypes.blob, arg); - }, - /** - Steps the statement one time. If the result indicates that - a row of data is available, true is returned. If no row of - data is available, false is returned. Throws on error. - */ - step: function(){ - affirmUnlocked(this, 'step()'); - const rc = api.sqlite3_step(affirmStmtOpen(this)._pStmt); - switch(rc){ - case api.SQLITE_DONE: return this._mayGet = false; - case api.SQLITE_ROW: return this._mayGet = true; - default: - this._mayGet = false; - console.warn("sqlite3_step() rc=",rc,"SQL =", - api.sqlite3_sql(this._pStmt)); - this.db.checkRc(rc); - }; - }, - /** - Fetches the value from the given 0-based column index of - the current data row, throwing if index is out of range. - - Requires that step() has just returned a truthy value, else - an exception is thrown. - - By default it will determine the data type of the result - automatically. If passed a second arugment, it must be one - of the enumeration values for sqlite3 types, which are - defined as members of the sqlite3 module: SQLITE_INTEGER, - SQLITE_FLOAT, SQLITE_TEXT, SQLITE_BLOB. Any other value, - except for undefined, will trigger an exception. Passing - undefined is the same as not passing a value. It is legal - to, e.g., fetch an integer value as a string, in which case - sqlite3 will convert the value to a string. - - If ndx is an array, this function behaves a differently: it - assigns the indexes of the array, from 0 to the number of - result columns, to the values of the corresponding column, - and returns that array. - - If ndx is a plain object, this function behaves even - differentlier: it assigns the properties of the object to - the values of their corresponding result columns. - - Blobs are returned as Uint8Array instances. - - Potential TODO: add type ID SQLITE_JSON, which fetches the - result as a string and passes it (if it's not null) to - JSON.parse(), returning the result of that. Until then, - getJSON() can be used for that. - */ - get: function(ndx,asType){ - if(!affirmStmtOpen(this)._mayGet){ - toss("Stmt.step() has not (recently) returned true."); - } - if(Array.isArray(ndx)){ - let i = 0; - while(i32bits */ - } - case api.SQLITE_FLOAT: - return api.sqlite3_column_double(this._pStmt, ndx); - case api.SQLITE_TEXT: - return api.sqlite3_column_text(this._pStmt, ndx); - case api.SQLITE_BLOB: { - const n = api.sqlite3_column_bytes(this._pStmt, ndx), - ptr = api.sqlite3_column_blob(this._pStmt, ndx), - rc = new Uint8Array(n), - heap = n ? api.wasm.HEAP8() : false; - for(let i = 0; i < n; ++i) rc[i] = heap[ptr + i]; - if(n && this.db._blobXfer instanceof Array){ - /* This is an optimization soley for the - Worker-based API. These values will be - transfered to the main thread directly - instead of being copied. */ - this.db._blobXfer.push(rc.buffer); - } - return rc; - } - default: toss("Don't know how to translate", - "type of result column #"+ndx+"."); - } - abort("Not reached."); - }, - /** Equivalent to get(ndx) but coerces the result to an - integer. */ - getInt: function(ndx){return this.get(ndx,api.SQLITE_INTEGER)}, - /** Equivalent to get(ndx) but coerces the result to a - float. */ - getFloat: function(ndx){return this.get(ndx,api.SQLITE_FLOAT)}, - /** Equivalent to get(ndx) but coerces the result to a - string. */ - getString: function(ndx){return this.get(ndx,api.SQLITE_TEXT)}, - /** Equivalent to get(ndx) but coerces the result to a - Uint8Array. */ - getBlob: function(ndx){return this.get(ndx,api.SQLITE_BLOB)}, - /** - A convenience wrapper around get() which fetches the value - as a string and then, if it is not null, passes it to - JSON.parse(), returning that result. Throws if parsing - fails. If the result is null, null is returned. An empty - string, on the other hand, will trigger an exception. - */ - getJSON: function(ndx){ - const s = this.get(ndx, api.SQLITE_STRING); - return null===s ? s : JSON.parse(s); - }, - /** - Returns the result column name of the given index, or - throws if index is out of bounds or this statement has been - finalized. This can be used without having run step() - first. - */ - getColumnName: function(ndx){ - return api.sqlite3_column_name( - affirmColIndex(affirmStmtOpen(this),ndx)._pStmt, ndx - ); - }, - /** - If this statement potentially has result columns, this - function returns an array of all such names. If passed an - array, it is used as the target and all names are appended - to it. Returns the target array. Throws if this statement - cannot have result columns. This object's columnCount member - holds the number of columns. - */ - getColumnNames: function(tgt){ - affirmColIndex(affirmStmtOpen(this),0); - if(!tgt) tgt = []; - for(let i = 0; i < this.columnCount; ++i){ - tgt.push(api.sqlite3_column_name(this._pStmt, i)); - } - return tgt; - }, - /** - If this statement has named bindable parameters and the - given name matches one, its 1-based bind index is - returned. If no match is found, 0 is returned. If it has no - bindable parameters, the undefined value is returned. - */ - getParamIndex: function(name){ - return (affirmStmtOpen(this).parameterCount - ? api.sqlite3_bind_parameter_index(this._pStmt, name) - : undefined); - } - }/*Stmt.prototype*/; - - /** OO binding's namespace. */ - const SQLite3 = { - version: { - lib: api.sqlite3_libversion(), - ooApi: "0.0.1" - }, - DB, - Stmt, - /** - Reports info about compile-time options. It has several - distinct uses: - - If optName is an array then it is expected to be a list of - compilation options and this function returns an object - which maps each such option to true or false, indicating - whether or not the given option was included in this - build. That object is returned. - - If optName is an object, its keys are expected to be - compilation options and this function sets each entry to - true or false. That object is returned. - - If passed no arguments then it returns an object mapping - all known compilation options to their compile-time values, - or boolean true if they are defined with no value. This - result, which is relatively expensive to compute, is cached - and returned for future no-argument calls. - - In all other cases it returns true if the given option was - active when when compiling the sqlite3 module, else false. - - Compile-time option names may optionally include their - "SQLITE_" prefix. When it returns an object of all options, - the prefix is elided. - */ - compileOptionUsed: function f(optName){ - if(!arguments.length){ - if(f._result) return f._result; - else if(!f._opt){ - f._rx = /^([^=]+)=(.+)/; - f._rxInt = /^-?\d+$/; - f._opt = function(opt, rv){ - const m = f._rx.exec(opt); - rv[0] = (m ? m[1] : opt); - rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; - }; - } - const rc = {}, ov = [0,0]; - let i = 0, k; - while((k = api.sqlite3_compileoption_get(i++))){ - f._opt(k,ov); - rc[ov[0]] = ov[1]; - } - return f._result = rc; - }else if(Array.isArray(optName)){ - const rc = {}; - optName.forEach((v)=>{ - rc[v] = api.sqlite3_compileoption_used(v); - }); - return rc; - }else if('object' === typeof optName){ - Object.keys(optName).forEach((k)=> { - optName[k] = api.sqlite3_compileoption_used(k); - }); - return optName; - } - return ( - 'string'===typeof optName - ) ? !!api.sqlite3_compileoption_used(optName) : false; - } - }/*SQLite3 object*/; - - namespace.sqlite3 = { - api: api, - SQLite3 - }; - - if(self === self.window){ - /* This is running in the main window thread, so we're done. */ - postMessage({type:'sqlite3-api',data:'loaded'}); - return; - } - /****************************************************************** - End of main window thread. What follows is only intended for use - in Worker threads. - ******************************************************************/ - - /** - UNDER CONSTRUCTION - - We need an API which can proxy the DB API via a Worker message - interface. The primary quirky factor in such an API is that we - cannot pass callback functions between the window thread and a - worker thread, so we have to receive all db results via - asynchronous message-passing. That requires an asychronous API - with a distinctly different shape that the main OO API. - - Certain important considerations here include: - - - Support only one db connection or multiple? The former is far - easier, but there's always going to be a user out there who - wants to juggle six database handles at once. Do we add that - complexity or tell such users to write their own code using - the provided lower-level APIs? - - - Fetching multiple results: do we pass them on as a series of - messages, with start/end messages on either end, or do we - collect all results and bundle them back in a single message? - The former is, generically speaking, more memory-efficient but - the latter far easier to implement in this environment. The - latter is untennable for large data sets. Despite a web page - hypothetically being a relatively limited environment, there - will always be those users who feel that they should/need to - be able to work with multi-hundred-meg (or larger) blobs, and - passing around arrays of those may quickly exhaust the JS - engine's memory. - - TODOs include, but are not limited to: - - - The ability to manage multiple DB handles. This can - potentially be done via a simple mapping of DB.filename or - DB._pDb (`sqlite3*` handle) to DB objects. The open() - interface would need to provide an ID (probably DB._pDb) back - to the user which can optionally be passed as an argument to - the other APIs (they'd default to the first-opened DB, for - ease of use). Client-side usability of this feature would - benefit from making another wrapper class (or a singleton) - available to the main thread, with that object proxying all(?) - communication with the worker. - - - Revisit how virtual files are managed. We currently delete DBs - from the virtual filesystem when we close them, for the sake - of saving memory (the VFS lives in RAM). Supporting multiple - DBs may require that we give up that habit. Similarly, fully - supporting ATTACH, where a user can upload multiple DBs and - ATTACH them, also requires the that we manage the VFS entries - better. As of this writing, ATTACH will fail fatally in the - fiddle app (but not the lower-level APIs) because it runs in - safe mode, where ATTACH is disabled. - */ - - /** - Helper for managing Worker-level state. - */ - const wState = { - db: undefined, - open: function(arg){ - if(!arg && this.db) return this.db; - else if(this.db) this.db.close(); - return this.db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg)); - }, - close: function(alsoUnlink){ - if(this.db){ - this.db.close(alsoUnlink); - this.db = undefined; - } - }, - affirmOpen: function(){ - return this.db || toss("DB is not opened."); - }, - post: function(type,data,xferList){ - if(xferList){ - self.postMessage({type, data},xferList); - xferList.length = 0; - }else{ - self.postMessage({type, data}); - } - } - }; - - /** - A level of "organizational abstraction" for the Worker - API. Each method in this object must map directly to a Worker - message type key. The onmessage() dispatcher attempts to - dispatch all inbound messages to a method of this object, - passing it the event.data part of the inbound event object. All - methods must return a plain Object containing any response - state, which the dispatcher may amend. All methods must throw - on error. - */ - const wMsgHandler = { - xfer: [/*Temp holder for "transferable" postMessage() state.*/], - /** - Proxy for DB.exec() which expects a single argument of type - string (SQL to execute) or an options object in the form - expected by exec(). The notable differences from exec() - include: - - - The default value for options.rowMode is 'array' because - the normal default cannot cross the window/Worker boundary. - - - A function-type options.callback property cannot cross - the window/Worker boundary, so is not useful here. If - options.callback is a string then it is assumed to be a - message type key, in which case a callback function will be - applied which posts each row result via: - - postMessage({type: thatKeyType, data: theRow}) - - And, at the end of the result set (whether or not any - result rows were produced), it will post an identical - message with data:null to alert the caller than the result - set is completed. - - The callback proxy must not recurse into this interface, or - results are undefined. (It hypothetically cannot recurse - because an exec() call will be tying up the Worker thread, - causing any recursion attempt to wait until the first - exec() is completed.) - - The response is the input options object (or a synthesized - one if passed only a string), noting that - options.resultRows and options.columnNames may be populated - by the call to exec(). - - This opens/creates the Worker's db if needed. - */ - exec: function(ev){ - const opt = ( - 'string'===typeof ev.data - ) ? {sql: ev.data} : (ev.data || {}); - if(!opt.rowMode){ - /* Since the default rowMode of 'stmt' is not useful - for the Worker interface, we'll default to - something else. */ - opt.rowMode = 'array'; - }else if('stmt'===opt.rowMode){ - toss("Invalid rowMode for exec(): stmt mode", - "does not work in the Worker API."); - } - const db = wState.open(); - if(opt.callback || opt.resultRows instanceof Array){ - // Part of a copy-avoidance optimization for blobs - db._blobXfer = this.xfer; - } - const callbackMsgType = opt.callback; - if('string' === typeof callbackMsgType){ - const that = this; - opt.callback = - (row)=>wState.post(callbackMsgType,row,this.xfer); - } - try { - db.exec(opt); - if(opt.callback instanceof Function){ - opt.callback = callbackMsgType; - wState.post(callbackMsgType, null); - } - }finally{ - delete db._blobXfer; - if('string'===typeof callbackMsgType){ - opt.callback = callbackMsgType; - } - } - return opt; - }/*exec()*/, - /** - Proxy for DB.exportBinaryImage(). Throws if the db has not - been opened. Response is an object: - - { - buffer: Uint8Array (db file contents), - filename: the current db filename, - mimetype: string - } - */ - export: function(ev){ - const db = wState.affirmOpen(); - const response = { - buffer: db.exportBinaryImage(), - filename: db.filename, - mimetype: 'application/x-sqlite3' - }; - this.xfer.push(response.buffer.buffer); - return response; - }/*export()*/, - /** - Proxy for the DB constructor. Expects to be passed a single - object or a falsy value to use defaults. The object may - have a filename property to name the db file (see the DB - constructor for peculiarities and transformations) and/or a - buffer property (a Uint8Array holding a complete database - file's contents). The response is an object: - - { - filename: db filename (possibly differing from the input) - } - - If the Worker's db is currently opened, this call closes it - before proceeding. - */ - open: function(ev){ - wState.close(/*true???*/); - const args = [], data = (ev.data || {}); - if(data.filename) args.push(data.filename); - if(data.buffer){ - args.push(data.buffer); - this.xfer.push(data.buffer.buffer); - } - const db = wState.open(args); - return {filename: db.filename}; - }, - /** - Proxy for DB.close(). If ev.data may either be a boolean or - an object with an `unlink` property. If that value is - truthy then the db file (if the db is currently open) will - be unlinked from the virtual filesystem, else it will be - kept intact. The response object is: - - {filename: db filename _if_ the db is is opened when this - is called, else the undefined value - } - */ - close: function(ev){ - const response = { - filename: wState.db && wState.db.filename - }; - if(wState.db){ - wState.close(!!(ev.data && 'object'===typeof ev.data) - ? ev.data.unlink : ev.data); - } - return response; - } - }/*wMsgHandler*/; - - /** - UNDER CONSTRUCTION! - - A subset of the DB API is accessible via Worker messages in the form: - - { type: apiCommand, - data: apiArguments } - - As a rule, these commands respond with a postMessage() of their - own in the same form, but will, if needed, transform the `data` - member to an object and may add state to it. The responses - always have an object-format `data` part. If the inbound `data` - is an object which has a `messageId` property, that property is - always mirrored in the result object, for use in client-side - dispatching of these asynchronous results. Exceptions thrown - during processing result in an `error`-type event with a - payload in the form: - - { - message: error string, - errorClass: class name of the error type, - input: ev.data, - [messageId: if set in the inbound message] - } - - The individual APIs are documented in the wMsgHandler object. - */ - self.onmessage = function(ev){ - ev = ev.data; - let response, evType = ev.type; - try { - if(wMsgHandler.hasOwnProperty(evType) && - wMsgHandler[evType] instanceof Function){ - response = wMsgHandler[evType](ev); - }else{ - toss("Unknown db worker message type:",ev.type); - } - }catch(err){ - evType = 'error'; - response = { - message: err.message, - errorClass: err.name, - input: ev - }; - } - if(!response.messageId && ev.data - && 'object'===typeof ev.data && ev.data.messageId){ - response.messageId = ev.data.messageId; - } - wState.post(evType, response, wMsgHandler.xfer); - }; - - postMessage({type:'sqlite3-api',data:'loaded'}); -})/*postRun.push(...)*/; diff --git a/ext/fiddle/sqlite3-worker.js b/ext/fiddle/sqlite3-worker.js deleted file mode 100644 index ed369ad35b..0000000000 --- a/ext/fiddle/sqlite3-worker.js +++ /dev/null @@ -1,44 +0,0 @@ -/* - 2022-05-23 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - This is a JS Worker file for the main sqlite3 api. It loads - sqlite3.js, initializes the module, and postMessage()'s a message - after the module is initialized: - - {type: 'sqlite3-api', data: 'ready'} - - This seemingly superfluous level of indirection is necessary when - loading sqlite3.js via a Worker. Loading sqlite3.js from the main - window thread elides the Worker-specific API. Instantiating a worker - with new Worker("sqlite.js") will not (cannot) call - sqlite3InitModule() to initialize the module due to a - timing/order-of-operations conflict (and that symbol is not exported - in a way that a Worker loading it that way can see it). Thus JS - code wanting to load the sqlite3 Worker-specific API needs to pass - _this_ file (or equivalent) to the Worker constructor and then - listen for an event in the form shown above in order to know when - the module has completed initialization. sqlite3.js will fire a - similar event, with data:'loaded' as the final step in its loading - process. Whether or not we _really_ need both 'loaded' and 'ready' - events is unclear, but they are currently separate events primarily - for the sake of clarity in the timing of when it's okay to use the - loaded module. At the time the 'loaded' event is fired, it's - possible (but unknown and unknowable) that the emscripten-generated - module-setup infrastructure still has work to do. Thus it is - hypothesized that client code is better off waiting for the 'ready' - even before using the API. -*/ -"use strict"; -importScripts('sqlite3.js'); -sqlite3InitModule().then(function(){ - setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'ready'}), 0); -}); diff --git a/ext/fiddle/testing1.js b/ext/fiddle/testing1.js deleted file mode 100644 index 552df5d6f7..0000000000 --- a/ext/fiddle/testing1.js +++ /dev/null @@ -1,239 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - A basic test script for sqlite3-api.js. This file must be run in - main JS thread and sqlite3.js must have been loaded before it. -*/ -(function(){ - const T = self.SqliteTestUtil; - const log = console.log.bind(console); - const debug = console.debug.bind(console); - - const assert = function(condition, text) { - if (!condition) { - throw new Error('Assertion failed' + (text ? ': ' + text : '')); - } - }; - - const test1 = function(db,sqlite3){ - const api = sqlite3.api; - log("Basic sanity tests..."); - T.assert(db._pDb). - assert(0===api.sqlite3_extended_result_codes(db._pDb,1)); - let st = db.prepare( - new TextEncoder('utf-8').encode("select 3 as a") - /* Testing handling of Uint8Array input */ - ); - //debug("statement =",st); - T.assert(st._pStmt) - .assert(!st._mayGet) - .assert('a' === st.getColumnName(0)) - .assert(st === db._statements[st._pStmt]) - .assert(1===st.columnCount) - .assert(0===st.parameterCount) - .mustThrow(()=>st.bind(1,null)) - .assert(true===st.step()) - .assert(3 === st.get(0)) - .mustThrow(()=>st.get(1)) - .mustThrow(()=>st.get(0,~api.SQLITE_INTEGER)) - .assert(3 === st.get(0,api.SQLITE_INTEGER)) - .assert(3 === st.getInt(0)) - .assert('3' === st.get(0,api.SQLITE_TEXT)) - .assert('3' === st.getString(0)) - .assert(3.0 === st.get(0,api.SQLITE_FLOAT)) - .assert(3.0 === st.getFloat(0)) - .assert(st.get(0,api.SQLITE_BLOB) instanceof Uint8Array) - .assert(1===st.get(0,api.SQLITE_BLOB).length) - .assert(st.getBlob(0) instanceof Uint8Array) - .assert(3 === st.get([])[0]) - .assert(3 === st.get({}).a) - .assert(3 === st.getJSON(0)) - .assert(st._mayGet) - .assert(false===st.step()) - .assert(!st._mayGet) - ; - let pId = st._pStmt; - st.finalize(); - T.assert(!st._pStmt) - .assert(!db._statements[pId]); - - let list = []; - db.exec({ - sql:['CREATE TABLE t(a,b);', - "INSERT INTO t(a,b) VALUES(1,2),(3,4),", - "(?,?),('blob',X'6869');" - ], - multi: true, - saveSql: list, - bind: [5,6] - /* Achtung: ^^^ bind support might be removed from multi-mode exec. */ - }); - T.assert(2 === list.length); - //debug("Exec'd SQL:", list); - - let blob = db.selectValue("select b from t where a='blob'"); - T.assert(blob instanceof Uint8Array). - assert(0x68===blob[0] && 0x69===blob[1]); - blob = null; - - let counter = 0, colNames = []; - list.length = 0; - db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ - rowMode: 'object', - resultRows: list, - columnNames: colNames, - callback: function(row,stmt){ - ++counter; - T.assert((row.a%2 && row.a<6) || 'blob'===row.a); - } - }); - T.assert(2 === colNames.length) - .assert('a' === colNames[0]) - .assert(4 === counter) - .assert(4 === list.length); - list.length = 0; - db.exec("SELECT a a, b b FROM t",{ - rowMode: 'array', - callback: function(row,stmt){ - ++counter; - T.assert(Array.isArray(row)) - .assert((0===row[1]%2 && row[1]<7) - || (row[1] instanceof Uint8Array)); - } - }); - T.assert(8 === counter); - }; - - const testUDF = function(db){ - log("Testing UDF..."); - db.createFunction("foo",function(a,b){return a+b}); - T.assert(7===db.selectValue("select foo(3,4)")). - assert(5===db.selectValue("select foo(3,?)",2)). - assert(5===db.selectValue("select foo(?,?2)",[1,4])). - assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); - db.createFunction("bar", { - arity: -1, - callback: function(){ - var rc = 0; - for(let i = 0; i < arguments.length; ++i) rc += arguments[i]; - return rc; - } - }).createFunction({ - name: "asis", - callback: (arg)=>arg - }); - - log("Testing DB::selectValue() w/ UDF..."); - T.assert(0===db.selectValue("select bar()")). - assert(1===db.selectValue("select bar(1)")). - assert(3===db.selectValue("select bar(1,2)")). - assert(-1===db.selectValue("select bar(1,2,-4)")). - assert('hi'===db.selectValue("select asis('hi')")); - - const eqApprox = function(v1,v2,factor=0.05){ - //debug('eqApprox',v1, v2); - return v1>=(v2-factor) && v1<=(v2+factor); - }; - - T.assert('hi' === db.selectValue("select ?",'hi')). - assert(null===db.selectValue("select null")). - assert(null === db.selectValue("select ?",null)). - assert(null === db.selectValue("select ?",[null])). - assert(null === db.selectValue("select $a",{$a:null})). - assert(eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). - assert(eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))) - ; - - log("Testing binding and UDF propagation of blobs..."); - let blobArg = new Uint8Array(2); - blobArg.set([0x68, 0x69], 0); - let blobRc = db.selectValue("select asis(?1)", blobArg); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length). - assert(0x68==blobRc[0] && 0x69==blobRc[1]); - blobRc = db.selectValue("select asis(X'6869')"); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length). - assert(0x68==blobRc[0] && 0x69==blobRc[1]); - - blobArg = new Int8Array(2); - blobArg.set([0x68, 0x69]); - //debug("blobArg=",blobArg); - blobRc = db.selectValue("select asis(?1)", blobArg); - T.assert(blobRc instanceof Uint8Array). - assert(2 === blobRc.length); - //debug("blobRc=",blobRc); - T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); - }; - - const testAttach = function(db){ - log("Testing ATTACH..."); - db.exec({ - sql:[ - "attach 'foo.db' as foo", - "create table foo.bar(a)", - "insert into foo.bar(a) values(1),(2),(3)" - ].join(';'), - multi: true - }); - T.assert(2===db.selectValue('select a from foo.bar where a>1 order by a')); - db.exec("detach foo"); - T.mustThrow(()=>db.exec("select * from foo.bar")); - }; - - const runTests = function(Module){ - const sqlite3 = Module.sqlite3; - const api = sqlite3.api; - const oo = sqlite3.SQLite3; - log("Loaded module:",api.sqlite3_libversion(), - api.sqlite3_sourceid()); - log("Build options:",oo.compileOptionUsed()); - log("api.wasm.HEAP8 size =",api.wasm.HEAP8().length); - log("wasmEnum",JSON.parse(Module.ccall('sqlite3_wasm_enum_json', 'string', []))); - [ /* Spot-check a handful of constants to make sure they got installed... */ - 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', - 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', - 'SQLITE_OPEN_CREATE' - ].forEach(function(k){ - T.assert('number' === typeof api[k]); - }); - [/* Spot-check a few of the WASM API methods. */ - '_free', '_malloc', 'addFunction', 'stackRestore' - ].forEach(function(k){ - T.assert(Module[k] instanceof Function). - assert(api.wasm[k] instanceof Function); - }); - - const db = new oo.DB(); - try { - log("DB:",db.filename); - [ - test1, testUDF, testAttach - ].forEach((f)=>{ - const t = T.counter; - f(db, sqlite3); - log("Test count:",T.counter - t); - }); - }finally{ - db.close(); - } - log("Total Test count:",T.counter); - }; - - sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){ - /** Use a timeout so that we are (hopefully) out from under - the module init stack when our setup gets run. Just on - principle, not because we _need_ to be. */ - //console.debug("theModule =",theModule); - setTimeout(()=>runTests(theModule), 0); - }); -})(); diff --git a/ext/fiddle/testing2.js b/ext/fiddle/testing2.js deleted file mode 100644 index 4bb21b95f0..0000000000 --- a/ext/fiddle/testing2.js +++ /dev/null @@ -1,235 +0,0 @@ -/* - 2022-05-22 - - The author disclaims copyright to this source code. In place of a - legal notice, here is a blessing: - - * May you do good and not evil. - * May you find forgiveness for yourself and forgive others. - * May you share freely, never taking more than you give. - - *********************************************************************** - - A basic test script for sqlite3-worker.js. -*/ -(function(){ - const T = self.SqliteTestUtil; - const SW = new Worker("sqlite3-worker.js"); - /** Posts a worker message as {type:type, data:data}. */ - const wMsg = function(type,data){ - SW.postMessage({type, data}); - return SW; - }; - const log = console.log.bind(console); - const warn = console.warn.bind(console); - const error = console.error.bind(console); - - SW.onerror = function(event){ - error("onerror",event); - }; - - /** - A queue for callbacks which are to be run in response to async - DB commands. See the notes in runTests() for why we need - this. The event-handling plumbing of this file requires that - any DB command which includes a `messageId` property also have - a queued callback entry, as the existence of that property in - response payloads is how it knows whether or not to shift an - entry off of the queue. - */ - const MsgHandlerQueue = { - queue: [], - id: 0, - push: function(type,callback){ - this.queue.push(callback); - return type + '-' + (++this.id); - }, - shift: function(){ - return this.queue.shift(); - } - }; - - const testCount = ()=>log("Total test count:",T.counter); - - const runOneTest = function(eventType, eventData, callback){ - T.assert(eventData && 'object'===typeof eventData); - /* ^^^ that is for the testing and messageId-related code, not - a hard requirement of all of the Worker-exposed APIs. */ - eventData.messageId = MsgHandlerQueue.push(eventType,function(ev){ - log("runOneTest",eventType,"result",ev.data); - if(callback instanceof Function){ - callback(ev); - testCount(); - } - }); - wMsg(eventType, eventData); - }; - - /** Methods which map directly to onmessage() event.type keys. - They get passed the inbound event.data. */ - const dbMsgHandler = { - open: function(ev){ - log("open result",ev.data); - }, - exec: function(ev){ - log("exec result",ev.data); - }, - export: function(ev){ - log("exec result",ev.data); - }, - error: function(ev){ - error("ERROR from the worker:",ev.data); - }, - resultRowTest1: function f(ev){ - if(undefined === f.counter) f.counter = 0; - if(ev.data) ++f.counter; - //log("exec() result row:",ev.data); - T.assert(null===ev.data || 'number' === typeof ev.data.b); - } - }; - - const runTests = function(){ - const mustNotReach = ()=>{ - throw new Error("This is not supposed to be reached."); - }; - /** - "The problem" now is that the test results are async. We - know, however, that the messages posted to the worker will - be processed in the order they are passed to it, so we can - create a queue of callbacks to handle them. The problem - with that approach is that it's not error-handling - friendly, in that an error can cause us to bypass a result - handler queue entry. We have to perform some extra - acrobatics to account for that. - */ - runOneTest('open', {filename:'testing2.sqlite3'}, function(ev){ - //log("open result",ev); - T.assert('testing2.sqlite3'===ev.data.filename) - .assert(ev.data.messageId); - }); - runOneTest('exec',{ - sql: ["create table t(a,b)", - "insert into t(a,b) values(1,2),(3,4),(5,6)" - ].join(';'), - multi: true, - resultRows: [], columnNames: [] - }, function(ev){ - ev = ev.data; - T.assert(0===ev.resultRows.length) - .assert(0===ev.columnNames.length); - }); - runOneTest('exec',{ - sql: 'select a a, b b from t order by a', - resultRows: [], columnNames: [], - }, function(ev){ - ev = ev.data; - T.assert(3===ev.resultRows.length) - .assert(1===ev.resultRows[0][0]) - .assert(6===ev.resultRows[2][1]) - .assert(2===ev.columnNames.length) - .assert('b'===ev.columnNames[1]); - }); - runOneTest('exec',{ - sql: 'select a a, b b from t order by a', - resultRows: [], columnNames: [], - rowMode: 'object' - }, function(ev){ - ev = ev.data; - T.assert(3===ev.resultRows.length) - .assert(1===ev.resultRows[0].a) - .assert(6===ev.resultRows[2].b) - }); - runOneTest('exec',{sql:'intentional_error'}, mustNotReach); - // Ensure that the message-handler queue survives ^^^ that error... - runOneTest('exec',{ - sql:'select 1', - resultRows: [], - //rowMode: 'array', // array is the default in the Worker interface - }, function(ev){ - ev = ev.data; - T.assert(1 === ev.resultRows.length) - .assert(1 === ev.resultRows[0][0]); - }); - runOneTest('exec',{ - sql: 'select a a, b b from t order by a', - callback: 'resultRowTest1', - rowMode: 'object' - }, function(ev){ - T.assert(3===dbMsgHandler.resultRowTest1.counter); - dbMsgHandler.resultRowTest1.counter = 0; - }); - runOneTest('exec',{sql: 'delete from t where a>3'}); - runOneTest('exec',{ - sql: 'select count(a) from t', - resultRows: [] - },function(ev){ - ev = ev.data; - T.assert(1===ev.resultRows.length) - .assert(2===ev.resultRows[0][0]); - }); - runOneTest('export',{}, function(ev){ - ev = ev.data; - T.assert('string' === typeof ev.filename) - .assert(ev.buffer instanceof Uint8Array) - .assert(ev.buffer.length > 1024) - .assert('application/x-sqlite3' === ev.mimetype); - }); - - /***** close() tests must come last. *****/ - runOneTest('close',{unlink:true},function(ev){ - ev = ev.data; - T.assert('string' === typeof ev.filename); - }); - runOneTest('close',{unlink:true},function(ev){ - ev = ev.data; - T.assert(undefined === ev.filename); - }); - }; - - SW.onmessage = function(ev){ - if(!ev.data || 'object'!==typeof ev.data){ - warn("Unknown sqlite3-worker message type:",ev); - return; - } - ev = ev.data/*expecting a nested object*/; - //log("main window onmessage:",ev); - if(ev.data && ev.data.messageId){ - /* We're expecting a queued-up callback handler. */ - const f = MsgHandlerQueue.shift(); - if('error'===ev.type){ - dbMsgHandler.error(ev); - return; - } - T.assert(f instanceof Function); - f(ev); - return; - } - switch(ev.type){ - case 'sqlite3-api': - switch(ev.data){ - case 'loaded': - log("Message:",ev); return; - case 'ready': - log("Message:",ev); - self.sqlite3TestModule.setStatus(null); - setTimeout(runTests, 0); - return; - default: - warn("Unknown sqlite3-api message type:",ev); - return; - } - default: - if(dbMsgHandler.hasOwnProperty(ev.type)){ - try{dbMsgHandler[ev.type](ev);} - catch(err){ - error("Exception while handling db result message", - ev,":",err); - } - return; - } - warn("Unknown sqlite3-api message type:",ev); - } - }; - - log("Init complete, but async init bits may still be running."); -})(); diff --git a/ext/fiddle/wasm_util.c b/ext/fiddle/wasm_util.c deleted file mode 100644 index a02063fc48..0000000000 --- a/ext/fiddle/wasm_util.c +++ /dev/null @@ -1,127 +0,0 @@ -#include "sqlite3.h" -#include /*atexit()*/ -/* -** 2022-06-25 -** -** The author disclaims copyright to this source code. In place of a -** legal notice, here is a blessing: -** -** * May you do good and not evil. -** * May you find forgiveness for yourself and forgive others. -** * May you share freely, never taking more than you give. -** -*********************************************************************** -** -** Utility functions for use with the emscripten/WASM bits. These -** functions ARE NOT part of the sqlite3 public API. They are strictly -** for internal use by the JS/WASM bindings. -** -** This file is intended to be WASM-compiled together with sqlite3.c, -** e.g.: -** -** emcc ... sqlite3.c wasm_util.c -*/ - -/** Result value of sqlite3_wasm_enum_json(). */ -static char * zWasmEnum = 0; -/* atexit() handler to clean up any WASM-related state. */ -static void sqlite3_wasm_cleanup(void){ - free(zWasmEnum); -} - -/* -** Returns a string containing a JSON-format "enum" of C-level -** constants intended to be imported into the JS environment. The JSON -** is initialized the first time this function is called and that -** result is reused for all future calls and cleaned up via atexit(). -** (If we didn't cache the result, it would be leaked by the JS glue -** code on each call during the WASM-to-JS conversion.) -** -** This function is NOT part of the sqlite3 public API. It is strictly -** for use by the JS/WASM bindings. -*/ -const char * sqlite3_wasm_enum_json(void){ - sqlite3_str * s; - if(zWasmEnum) return zWasmEnum; - s = sqlite3_str_new(0); - sqlite3_str_appendall(s, "{"); - -#define SD_(X,S,FINAL) \ - sqlite3_str_appendf(s, "\"%s\": %d%s", S, (int)X, (FINAL ? "}" : ", ")) -#define SD(X) SD_(X,#X,0) -#define SDFinal(X) SD_(X,#X,1) - - sqlite3_str_appendall(s,"\"resultCodes\": {"); - SD(SQLITE_OK); - SD(SQLITE_ERROR); - SD(SQLITE_INTERNAL); - SD(SQLITE_PERM); - SD(SQLITE_ABORT); - SD(SQLITE_BUSY); - SD(SQLITE_LOCKED); - SD(SQLITE_NOMEM); - SD(SQLITE_READONLY); - SD(SQLITE_INTERRUPT); - SD(SQLITE_IOERR); - SD(SQLITE_CORRUPT); - SD(SQLITE_NOTFOUND); - SD(SQLITE_FULL); - SD(SQLITE_CANTOPEN); - SD(SQLITE_PROTOCOL); - SD(SQLITE_EMPTY); - SD(SQLITE_SCHEMA); - SD(SQLITE_TOOBIG); - SD(SQLITE_CONSTRAINT); - SD(SQLITE_MISMATCH); - SD(SQLITE_MISUSE); - SD(SQLITE_NOLFS); - SD(SQLITE_AUTH); - SD(SQLITE_FORMAT); - SD(SQLITE_RANGE); - SD(SQLITE_NOTADB); - SD(SQLITE_NOTICE); - SD(SQLITE_WARNING); - SD(SQLITE_ROW); - SDFinal(SQLITE_DONE); - - sqlite3_str_appendall(s,",\"dataTypes\": {"); - SD(SQLITE_INTEGER); - SD(SQLITE_FLOAT); - SD(SQLITE_TEXT); - SD(SQLITE_BLOB); - SDFinal(SQLITE_NULL); - - sqlite3_str_appendf(s,",\"encodings\": {"); - SDFinal(SQLITE_UTF8); - - sqlite3_str_appendall(s,",\"blobFinalizers\": {"); - SD(SQLITE_STATIC); - SDFinal(SQLITE_TRANSIENT); - - sqlite3_str_appendall(s,",\"udfFlags\": {"); - SD(SQLITE_DETERMINISTIC); - SD(SQLITE_DIRECTONLY); - SDFinal(SQLITE_INNOCUOUS); - - sqlite3_str_appendall(s,",\"openFlags\": {"); - /* Noting that not all of these will have any effect in WASM-space. */ - SD(SQLITE_OPEN_READONLY); - SD(SQLITE_OPEN_READWRITE); - SD(SQLITE_OPEN_CREATE); - SD(SQLITE_OPEN_URI); - SD(SQLITE_OPEN_MEMORY); - SD(SQLITE_OPEN_NOMUTEX); - SD(SQLITE_OPEN_FULLMUTEX); - SD(SQLITE_OPEN_SHAREDCACHE); - SD(SQLITE_OPEN_PRIVATECACHE); - SD(SQLITE_OPEN_EXRESCODE); - SDFinal(SQLITE_OPEN_NOFOLLOW); - -#undef SD_ -#undef SD -#undef SDFinal - sqlite3_str_appendall(s, "}"); - zWasmEnum = sqlite3_str_finish(s); - atexit(sqlite3_wasm_cleanup); - return zWasmEnum; -} diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 99a1c5b031..ee8ade74a3 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -1,11 +1,269 @@ # This GNU makefile exists primarily to simplify/speed up development -# from emacs. It is not part of the canonical build process. -default: - $(MAKE) -C ../.. wasm -e emcc_opt=-O0 +# of the sqlite3 WASM components. It is not part of the canonical +# build process. +# +# Maintenance notes: the fiddle build is currently performed in the +# top-level ../../Makefile.in. It may be moved into this file at some +# point, as GNU Make has been deemed acceptable for the WASM-related +# components (whereas POSIX Make is required for the more conventional +# components). +SHELL := $(shell which bash 2>/dev/null) +all: + +.PHONY: fiddle +ifneq (,$(wildcard /home/stephan)) + fiddle_opt ?= -O0 +else + fiddle_opt = -Os +endif +fiddle: + $(MAKE) -C ../.. fiddle -e emcc_opt=$(fiddle_opt) clean: - $(MAKE) -C ../../ clean-wasm + $(MAKE) -C ../../ clean-fiddle + -rm -f $(CLEAN_FILES) +MAKEFILE := $(lastword $(MAKEFILE_LIST)) +dir.top := ../.. +# Reminder: some Emscripten flags require absolute paths +dir.wasm := $(patsubst %/,%,$(dir $(abspath $(MAKEFILE)))) +dir.api := api +dir.jacc := jaccwabyt +dir.common := common +CLEAN_FILES := *~ $(dir.jacc)/*~ $(dir.api)/*~ $(dir.common)/*~ + +SQLITE_OPT = \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_OMIT_DEPRECATED \ + -DSQLITE_OMIT_UTF16 \ + -DSQLITE_THREADSAFE=0 +#SQLITE_OPT += -DSQLITE_ENABLE_MEMSYS5 +$(dir.top)/sqlite3.c: + $(MAKE) -C $(dir.top) sqlite3.c + +# SQLITE_OMIT_LOAD_EXTENSION: if this is true, sqlite3_vfs::xDlOpen +# and friends may be NULL. + +emcc_opt ?= -O0 +.PHONY: release +release: + $(MAKE) 'emcc_opt=-Os -g3' +# ^^^^^ target-specific vars, e.g.: +# release: emcc_opt=... +# apparently only work for file targets, not PHONY targets? +# +# ^^^^ -O3, -Oz, -Os minify symbol names and there appears to be no +# way around that except to use -g3, but -g3 causes the binary file +# size to absolutely explode (approx. 5x larger). This minification +# utterly breaks the resulting module, making it unsable except as +# self-contained/self-referential-only code, as ALL of the exported +# symbols get minified names. +# +# However, we have an option for using -Oz or -Os: +# +# Build with (-Os -g3) or (-Oz -g3) then use wasm-strip, from the wabt +# tools package (https://github.com/WebAssembly/wabt), to strip the +# debugging symbols. That results in a small build with unmangled +# symbol names. -Oz gives ever-so-slightly better compression than +# -Os: not quite 1% in some completely unscientific tests. Runtime +# speed for the unit tests is all over the place either way so it's +# difficult to say whether -Os gives any speed benefit over -Oz. +######################################################################## + +# Emscripten SDK home dir and related binaries... +EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/src/emsdk $(HOME)/emsdk)) +emcc.bin ?= $(word 1,$(wildcard $(shell which emcc) $(EMSDK_HOME)/upstream/emscripten/emcc)) +ifeq (,$(emcc.bin)) + $(error Cannot find emcc.) +endif + +wasm-strip ?= $(shell which wasm-strip 2>/dev/null) +ifeq (,$(filter clean,$(MAKECMDGOALS))) +ifeq (,$(wasm-strip)) + $(info WARNING: *******************************************************************) + $(info WARNING: builds using -O3/-Os/-Oz will minify WASM-exported names,) + $(info WARNING: breaking _All The Things_. The workaround for that is to build) + $(info WARNING: with -g3 (which explodes the file size) and then strip the debug) + $(info WARNING: info after compilation, using wasm-strip, to shrink the wasm file.) + $(info WARNING: wasm-strip was not found in the PATH so we cannot strip those.) + $(info WARNING: If this build uses any optimization level higher than -O2 then) + $(info WARNING: the ***resulting WASM binary WILL NOT BE USABLE***.) + $(info WARNING: wasm-strip is part of the wabt package:) + $(info WARNING: https://github.com/WebAssembly/wabt) + $(info WARNING: on Ubuntu-like systems it can be installed with:) + $(info WARNING: sudo apt install wabt) + $(info WARNING: *******************************************************************) +endif +endif # 'make clean' check + +ifeq (release,$(filter release,$(MAKECMDGOALS))) + ifeq (,$(wasm-strip)) + $(error Cannot make release-quality binary because wasm-strip is not available. \ + See notes in the warning above) + endif +else + $(info Development build. Use '$(MAKE) release' for a smaller release build.) +endif + +EXPORTED_FUNCTIONS.api.in := $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api \ + $(dir.jacc)/jaccwabyt_test.exports + +EXPORTED_FUNCTIONS.api: $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) + cat $(EXPORTED_FUNCTIONS.api.in) > $@ +CLEAN_FILES += EXPORTED_FUNCTIONS.api + +sqlite3-api.jses := \ + $(dir.api)/sqlite3-api-prologue.js \ + $(dir.common)/whwasmutil.js \ + $(dir.jacc)/jaccwabyt.js \ + $(dir.api)/sqlite3-api-glue.js \ + $(dir.api)/sqlite3-api-oo1.js \ + $(dir.api)/sqlite3-api-worker.js \ + $(dir.api)/sqlite3-api-opfs.js \ + $(dir.api)/sqlite3-api-cleanup.js + +sqlite3-api.js := $(dir.api)/sqlite3-api.js +CLEAN_FILES += $(sqlite3-api.js) +$(sqlite3-api.js): $(sqlite3-api.jses) $(MAKEFILE) + @echo "Making $@..." + @for i in $(sqlite3-api.jses); do \ + echo "/* BEGIN FILE: $$i */"; \ + cat $$i; \ + echo "/* END FILE: $$i */"; \ + done > $@ + +post-js.js := $(dir.api)/post-js.js +CLEAN_FILES += $(post-js.js) +post-jses := \ + $(dir.api)/post-js-header.js \ + $(sqlite3-api.js) \ + $(dir.api)/post-js-footer.js + +$(post-js.js): $(post-jses) $(MAKEFILE) + @echo "Making $@..." + @for i in $(post-jses); do \ + echo "/* BEGIN FILE: $$i */"; \ + cat $$i; \ + echo "/* END FILE: $$i */"; \ + done > $@ + + +######################################################################## +# emcc flags for .c/.o/.wasm. +emcc.flags = +#emcc.flags += -v # _very_ loud but also informative about what it's doing + +######################################################################## +# emcc flags for .c/.o. +emcc.cflags := +emcc.cflags += -std=c99 -fPIC +# -------------^^^^^^^^ we currently need c99 for WASM-specific sqlite3 APIs. +emcc.cflags += -I. -I$(dir.top) # $(SQLITE_OPT) + +######################################################################## +# emcc flags specific to building the final .js/.wasm file... +emcc.jsflags := -fPIC +emcc.jsflags += --no-entry +emcc.jsflags += -sENVIRONMENT=web +emcc.jsflags += -sMODULARIZE +emcc.jsflags += -sSTRICT_JS +emcc.jsflags += -sDYNAMIC_EXECUTION=0 +emcc.jsflags += -sNO_POLYFILL +emcc.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.api +emcc.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory # wasmMemory==>for -sIMPORTED_MEMORY +emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 +emcc.jsflags += -sIMPORTED_MEMORY +#emcc.jsflags += -sINITIAL_MEMORY=13107200 +#emcc.jsflags += -sTOTAL_STACK=4194304 +emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule +emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr. +emcc.jsflags +=--post-js=$(post-js.js) +#emcc.jsflags += -sSTRICT # fails due to missing __syscall_...() +#emcc.jsflags += -sALLOW_UNIMPLEMENTED_SYSCALLS +#emcc.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API +#emcc.jsflags += -sABORTING_MALLOC +emcc.jsflags += -sALLOW_MEMORY_GROWTH +emcc.jsflags += -sALLOW_TABLE_GROWTH +emcc.jsflags += -Wno-limited-postlink-optimizations +# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. +#emcc.jsflags += -sMALLOC=emmalloc +#emcc.jsflags += -sMALLOC=dlmalloc # a good 8k larger than emmalloc +#emcc.jsflags += -sSTANDALONE_WASM # causes OOM errors, not sure why +#emcc.jsflags += --import=foo_bar +#emcc.jsflags += --no-gc-sections +# https://lld.llvm.org/WebAssembly.html +emcc.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0 +emcc.jsflags += -sLLD_REPORT_UNDEFINED +#emcc.jsflags += --allow-undefined +emcc.jsflags += --import-undefined +#emcc.jsflags += --unresolved-symbols=import-dynamic --experimental-pic +#emcc.jsflags += --experimental-pic --unresolved-symbols=ingore-all --import-undefined +#emcc.jsflags += --unresolved-symbols=ignore-all +enable_bigint ?= 1 +ifneq (0,$(enable_bigint)) +emcc.jsflags += -sWASM_BIGINT +endif +emcc.jsflags += -sMEMORY64=0 +# ^^^^ MEMORY64=1 fails to load, erroring with: +# invalid memory limits flags 0x5 +# (enable via --experimental-wasm-memory64) +# +# ^^^^ MEMORY64=2 builds and loads but dies when we do things like: +# +# new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n) +# +# because ptr is now a BigInt, so is invalid for passing to arguments +# which have strict must-be-a-number requirements. +######################################################################## + + +sqlite3.js := $(dir.api)/sqlite3.js +sqlite3.wasm := $(dir.api)/sqlite3.wasm +$(dir.api)/sqlite3-wasm.o: emcc.cflags += $(SQLITE_OPT) +$(dir.api)/sqlite3-wasm.o: $(dir.top)/sqlite3.c +$(dir.api)/wasm_util.o: emcc.cflags += $(SQLITE_OPT) +sqlite3.wasm.c := $(dir.api)/sqlite3-wasm.c \ + $(dir.jacc)/jaccwabyt_test.c +# ^^^ FIXME (how?): jaccwabyt_test.c is only needed for the test +# apps. However, we want to test the release builds with those apps, +# so we cannot simply elide that file in release builds. That +# component is critical to the VFS bindings so needs to be tested +# along with the core APIs. +define WASM_C_COMPILE +$(1).o := $$(subst .c,.o,$(1)) +sqlite3.wasm.obj += $$($(1).o) +$$($(1).o): $$(MAKEFILE) $(1) + $$(emcc.bin) $$(emcc_opt) $$(emcc.flags) $$(emcc.cflags) -c $(1) -o $$@ +CLEAN_FILES += $$($(1).o) +endef +$(foreach c,$(sqlite3.wasm.c),$(eval $(call WASM_C_COMPILE,$(c)))) +$(sqlite3.js): +$(sqlite3.js): $(MAKEFILE) $(sqlite3.wasm.obj) \ + EXPORTED_FUNCTIONS.api \ + $(post-js.js) + $(emcc.bin) -o $@ $(emcc_opt) $(emcc.flags) $(emcc.jsflags) $(sqlite3.wasm.obj) + chmod -x $(sqlite3.wasm) +ifneq (,$(wasm-strip)) + $(wasm-strip) $(sqlite3.wasm) +endif + @ls -la $@ $(sqlite3.wasm) + +CLEAN_FILES += $(sqlite3.js) $(sqlite3.wasm) +all: $(sqlite3.js) +# End main Emscripten-based module build +######################################################################## + + +######################################################################## # fiddle_remote is the remote destination for the fiddle app. It # must be a [user@]HOST:/path for rsync. # Note that the target "should probably" contain a symlink of @@ -18,12 +276,12 @@ else ifneq (,$(wildcard /home/drh)) #fiddle_remote = if appropriate, add that user@host:/path here endif endif - $(fiddle_files): default - push-fiddle: $(fiddle_files) @if [ x = "x$(fiddle_remote)" ]; then \ echo "fiddle_remote must be a [user@]HOST:/path for rsync"; \ exit 1; \ fi rsync -va fiddle/ $(fiddle_remote) +# end fiddle remote push +######################################################################## diff --git a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api similarity index 71% rename from ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api rename to ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index dae46f0359..8f103c7c0b 100644 --- a/ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -7,6 +7,7 @@ _sqlite3_bind_parameter_count _sqlite3_bind_parameter_index _sqlite3_bind_text _sqlite3_changes +_sqlite3_changes64 _sqlite3_clear_bindings _sqlite3_close_v2 _sqlite3_column_blob @@ -24,30 +25,48 @@ _sqlite3_compileoption_used _sqlite3_create_function_v2 _sqlite3_data_count _sqlite3_db_filename +_sqlite3_db_name _sqlite3_errmsg +_sqlite3_error_offset +_sqlite3_errstr _sqlite3_exec +_sqlite3_expanded_sql +_sqlite3_extended_errcode _sqlite3_extended_result_codes _sqlite3_finalize +_sqlite3_initialize _sqlite3_interrupt _sqlite3_libversion +_sqlite3_libversion_number _sqlite3_open _sqlite3_open_v2 _sqlite3_prepare_v2 -_sqlite3_prepare_v2 +_sqlite3_prepare_v3 _sqlite3_reset _sqlite3_result_blob _sqlite3_result_double _sqlite3_result_error +_sqlite3_result_error_code +_sqlite3_result_error_nomem +_sqlite3_result_error_toobig _sqlite3_result_int _sqlite3_result_null _sqlite3_result_text _sqlite3_sourceid _sqlite3_sql _sqlite3_step +_sqlite3_strglob +_sqlite3_strlike +_sqlite3_total_changes +_sqlite3_total_changes64 _sqlite3_value_blob _sqlite3_value_bytes _sqlite3_value_double _sqlite3_value_text _sqlite3_value_type +_sqlite3_vfs_find +_sqlite3_vfs_register +_sqlite3_wasm_db_error _sqlite3_wasm_enum_json +_malloc _free diff --git a/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api new file mode 100644 index 0000000000..aab1d8bd37 --- /dev/null +++ b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api @@ -0,0 +1,3 @@ +FS +wasmMemory + diff --git a/ext/wasm/api/post-js-footer.js b/ext/wasm/api/post-js-footer.js new file mode 100644 index 0000000000..ee470928ba --- /dev/null +++ b/ext/wasm/api/post-js-footer.js @@ -0,0 +1,3 @@ +/* The current function scope was opened via post-js-header.js, which + gets prepended to this at build-time. */ +})/*postRun.push(...)*/; diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js new file mode 100644 index 0000000000..1763188a21 --- /dev/null +++ b/ext/wasm/api/post-js-header.js @@ -0,0 +1,26 @@ +/** + post-js-header.js is to be prepended to other code to create + post-js.js for use with Emscripten's --post-js flag. This code + requires that it be running in that context. The Emscripten + environment must have been set up already but it will not have + loaded its WASM when the code in this file is run. The function it + installs will be run after the WASM module is loaded, at which + point the sqlite3 WASM API bits will be set up. +*/ +if(!Module.postRun) Module.postRun = []; +Module.postRun.push(function(Module/*the Emscripten-style module object*/){ + 'use strict'; + /* This function will contain: + + - post-js-header.js (this file) + - sqlite3-api-prologue.js => Bootstrapping bits to attach the rest to + - sqlite3-api-whwasmutil.js => Replacements for much of Emscripten's glue + - sqlite3-api-jaccwabyt.js => Jaccwabyt (C/JS struct binding) + - sqlite3-api-glue.js => glues previous parts together + - sqlite3-api-oo.js => SQLite3 OO API #1. + - sqlite3-api-worker.js => Worker-based API + - sqlite3-api-cleanup.js => final API cleanup + - post-js-footer.js => closes this postRun() function + + Whew! + */ diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js new file mode 100644 index 0000000000..a2f921a5d7 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -0,0 +1,44 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is the tail end of the sqlite3-api.js constellation, + intended to be appended after all other files so that it can clean + up any global systems temporarily used for setting up the API's + various subsystems. +*/ +'use strict'; +self.sqlite3.postInit.forEach( + self.importScripts/*global is a Worker*/ + ? function(f){ + /** We try/catch/report for the sake of failures which happen in + a Worker, as those exceptions can otherwise get completely + swallowed, leading to confusing downstream errors which have + nothing to do with this failure. */ + try{ f(self, self.sqlite3) } + catch(e){ + console.error("Error in postInit() function:",e); + throw e; + } + } + : (f)=>f(self, self.sqlite3) +); +delete self.sqlite3.postInit; +if(self.location && +self.location.port > 1024){ + console.warn("Installing sqlite3 bits as global S for dev-testing purposes."); + self.S = self.sqlite3; +} +/* Clean up temporary global-scope references to our APIs... */ +self.sqlite3.config.Module.sqlite3 = self.sqlite3 +/* ^^^^ Currently needed by test code and Worker API setup */; +delete self.sqlite3.capi.util /* arguable, but these are (currently) internal-use APIs */; +delete self.sqlite3 /* clean up our global-scope reference */; +//console.warn("Module.sqlite3 =",Module.sqlite3); diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js new file mode 100644 index 0000000000..e962c93b64 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -0,0 +1,211 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file glues together disparate pieces of JS which are loaded in + previous steps of the sqlite3-api.js bootstrapping process: + sqlite3-api-prologue.js, whwasmutil.js, and jaccwabyt.js. It + initializes the main API pieces so that the downstream components + (e.g. sqlite3-api-oo1.js) have all that they need. +*/ +(function(self){ + 'use strict'; + const toss = (...args)=>{throw new Error(args.join(' '))}; + + self.sqlite3 = self.sqlite3ApiBootstrap({ + Module: Module /* ==> Emscripten-style Module object. Currently + needs to be exposed here for test code. NOT part + of the public API. */, + exports: Module['asm'], + memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */, + bigIntEnabled: !!self.BigInt64Array, + allocExportName: 'malloc', + deallocExportName: 'free' + }); + delete self.sqlite3ApiBootstrap; + + const sqlite3 = self.sqlite3; + const capi = sqlite3.capi, wasm = capi.wasm, util = capi.util; + self.WhWasmUtilInstaller(capi.wasm); + delete self.WhWasmUtilInstaller; + + if(0){ + /* "The problem" is that the following isn't type-safe. + OTOH, nothing about WASM pointers is. */ + /** + Add the `.pointer` xWrap() signature entry to extend + the `pointer` arg handler to check for a `pointer` + property. This can be used to permit, e.g., passing + an SQLite3.DB instance to a C-style sqlite3_xxx function + which takes an `sqlite3*` argument. + */ + const oldP = wasm.xWrap.argAdapter('pointer'); + const adapter = function(v){ + if(v && 'object'===typeof v && v.constructor){ + const x = v.pointer; + if(Number.isInteger(x)) return x; + else toss("Invalid (object) type for pointer-type argument."); + } + return oldP(v); + }; + wasm.xWrap.argAdapter('.pointer', adapter); + } + + // WhWasmUtil.xWrap() bindings... + { + /** + Add some descriptive xWrap() aliases for '*' intended to + (A) initially improve readability/correctness of capi.signatures + and (B) eventually perhaps provide some sort of type-safety + in their conversions. + */ + const aPtr = wasm.xWrap.argAdapter('*'); + wasm.xWrap.argAdapter('sqlite3*', aPtr)('sqlite3_stmt*', aPtr); + + /** + Populate api object with sqlite3_...() by binding the "raw" wasm + exports into type-converting proxies using wasm.xWrap(). + */ + for(const e of wasm.bindingSignatures){ + capi[e[0]] = wasm.xWrap.apply(null, e); + } + + /* For functions which cannot work properly unless + wasm.bigIntEnabled is true, install a bogus impl which + throws if called when bigIntEnabled is false. */ + const fI64Disabled = function(fname){ + return ()=>toss(fname+"() disabled due to lack", + "of BigInt support in this build."); + }; + for(const e of wasm.bindingSignatures.int64){ + capi[e[0]] = wasm.bigIntEnabled + ? wasm.xWrap.apply(null, e) + : fI64Disabled(e[0]); + } + + if(wasm.exports.sqlite3_wasm_db_error){ + util.sqlite3_wasm_db_error = capi.wasm.xWrap( + 'sqlite3_wasm_db_error', 'int', 'sqlite3*', 'int', 'string' + ); + }else{ + util.sqlite3_wasm_db_error = function(pDb,errCode,msg){ + console.warn("sqlite3_wasm_db_error() is not exported.",arguments); + return errCode; + }; + } + + /** + When registering a VFS and its related components it may be + necessary to ensure that JS keeps a reference to them to keep + them from getting garbage collected. Simply pass each such value + to this function and a reference will be held to it for the life + of the app. + */ + capi.sqlite3_vfs_register.addReference = function f(...args){ + if(!f._) f._ = []; + f._.push(...args); + }; + + }/*xWrap() bindings*/; + + /** + Scope-local holder of the two impls of sqlite3_prepare_v2/v3(). + */ + const __prepare = Object.create(null); + /** + This binding expects a JS string as its 2nd argument and + null as its final argument. In order to compile multiple + statements from a single string, the "full" impl (see + below) must be used. + */ + __prepare.basic = wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "string", + "int"/*MUST always be negative*/, + "int", "**", + "**"/*MUST be 0 or null or undefined!*/]); + /** + Impl which requires that the 2nd argument be a pointer + to the SQL string, instead of being converted to a + string. This variant is necessary for cases where we + require a non-NULL value for the final argument + (exec()'ing multiple statements from one input + string). For simpler cases, where only the first + statement in the SQL string is required, the wrapper + named sqlite3_prepare_v2() is sufficient and easier to + use because it doesn't require dealing with pointers. + */ + __prepare.full = wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "*", "int", "int", + "**", "**"]); + + /* Documented in the api object's initializer. */ + capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ + /* 2022-07-08: xWrap() 'string' arg handling may be able do this + special-case handling for us. It needs to be tested. Or maybe + not: we always want to treat pzTail as null when passed a + non-pointer SQL string and the argument adapters don't have + enough state to know that. Maybe they could/should, by passing + the currently-collected args as an array as the 2nd arg to the + argument adapters? Or maybe we collect all args in an array, + pass that to an optional post-args-collected callback, and give + it a chance to manipulate the args before we pass them on? */ + if(util.isSQLableTypedArray(sql)) sql = util.typedArrayToString(sql); + switch(typeof sql){ + case 'string': return __prepare.basic(pDb, sql, -1, prepFlags, ppStmt, null); + case 'number': return __prepare.full(pDb, sql, sqlLen||-1, prepFlags, ppStmt, pzTail); + default: + return util.sqlite3_wasm_db_error( + pDb, capi.SQLITE_MISUSE, + "Invalid SQL argument type for sqlite3_prepare_v2/v3()." + ); + } + }; + + capi.sqlite3_prepare_v2 = + (pDb, sql, sqlLen, ppStmt, pzTail)=>capi.sqlite3_prepare_v3(pDb, sql, sqlLen, 0, ppStmt, pzTail); + + /** + Install JS<->C struct bindings for the non-opaque struct types we + need... */ + sqlite3.StructBinder = self.Jaccwabyt({ + heap: 0 ? wasm.memory : wasm.heap8u, + alloc: wasm.alloc, + dealloc: wasm.dealloc, + functionTable: wasm.functionTable, + bigIntEnabled: wasm.bigIntEnabled, + memberPrefix: '$' + }); + delete self.Jaccwabyt; + + {/* Import C-level constants and structs... */ + const cJson = wasm.xCall('sqlite3_wasm_enum_json'); + if(!cJson){ + toss("Maintenance required: increase sqlite3_wasm_enum_json()'s", + "static buffer size!"); + } + wasm.ctype = JSON.parse(wasm.cstringToJs(cJson)); + //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); + for(const t of ['access', 'blobFinalizers', 'dataTypes', + 'encodings', 'flock', 'ioCap', + 'openFlags', 'prepareFlags', 'resultCodes', + 'syncFlags', 'udfFlags', 'version' + ]){ + for(const [k,v] of Object.entries(wasm.ctype[t])){ + capi[k] = v; + } + } + /* Bind all registered C-side structs... */ + for(const s of wasm.ctype.structs){ + capi[s.name] = sqlite3.StructBinder(s); + } + } + +})(self); diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js new file mode 100644 index 0000000000..9e54733966 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -0,0 +1,1438 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file contains the so-called OO #1 API wrapper for the sqlite3 + WASM build. It requires that sqlite3-api-glue.js has already run + and it installs its deliverable as self.sqlite3.oo1. +*/ +(function(self){ + const toss = (...args)=>{throw new Error(args.join(' '))}; + + const sqlite3 = self.sqlite3 || toss("Missing main sqlite3 object."); + const capi = sqlite3.capi, util = capi.util; + /* What follows is colloquially known as "OO API #1". It is a + binding of the sqlite3 API which is designed to be run within + the same thread (main or worker) as the one in which the + sqlite3 WASM binding was initialized. This wrapper cannot use + the sqlite3 binding if, e.g., the wrapper is in the main thread + and the sqlite3 API is in a worker. */ + + /** + In order to keep clients from manipulating, perhaps + inadvertently, the underlying pointer values of DB and Stmt + instances, we'll gate access to them via the `pointer` property + accessor and store their real values in this map. Keys = DB/Stmt + objects, values = pointer values. This also unifies how those are + accessed, for potential use downstream via custom + capi.wasm.xWrap() function signatures which know how to extract + it. + */ + const __ptrMap = new WeakMap(); + /** + Map of DB instances to objects, each object being a map of UDF + names to wasm function _pointers_ added to that DB handle via + createFunction(). + */ + const __udfMap = new WeakMap(); + /** + Map of DB instances to objects, each object being a map of Stmt + wasm pointers to Stmt objects. + */ + const __stmtMap = new WeakMap(); + + /** If object opts has _its own_ property named p then that + property's value is returned, else dflt is returned. */ + const getOwnOption = (opts, p, dflt)=> + opts.hasOwnProperty(p) ? opts[p] : dflt; + + /** + An Error subclass specifically for reporting DB-level errors and + enabling clients to unambiguously identify such exceptions. + */ + class SQLite3Error extends Error { + constructor(...args){ + super(...args); + this.name = 'SQLite3Error'; + } + }; + const toss3 = (...args)=>{throw new SQLite3Error(args)}; + sqlite3.SQLite3Error = SQLite3Error; + + /** + The DB class provides a high-level OO wrapper around an sqlite3 + db handle. + + The given db filename must be resolvable using whatever + filesystem layer (virtual or otherwise) is set up for the default + sqlite3 VFS. + + Note that the special sqlite3 db names ":memory:" and "" + (temporary db) have their normal special meanings here and need + not resolve to real filenames, but "" uses an on-storage + temporary database and requires that the VFS support that. + + The db is currently opened with a fixed set of flags: + (SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXRESCODE). This API will change in the future + permit the caller to provide those flags via an additional + argument. + + For purposes of passing a DB instance to C-style sqlite3 + functions, its read-only `pointer` property holds its `sqlite3*` + pointer value. That property can also be used to check whether + this DB instance is still open. + */ + const DB = function ctor(fn=':memory:'){ + if('string'!==typeof fn){ + toss3("Invalid filename for DB constructor."); + } + const stack = capi.wasm.scopedAllocPush(); + let ptr; + try { + const ppDb = capi.wasm.scopedAllocPtr() /* output (sqlite3**) arg */; + const rc = capi.sqlite3_open_v2(fn, ppDb, capi.SQLITE_OPEN_READWRITE + | capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_EXRESCODE, null); + ptr = capi.wasm.getMemValue(ppDb, '*'); + ctor.checkRc(ptr, rc); + }catch(e){ + if(ptr) capi.sqlite3_close_v2(ptr); + throw e; + } + finally{capi.wasm.scopedAllocPop(stack);} + this.filename = fn; + __ptrMap.set(this, ptr); + __stmtMap.set(this, Object.create(null)); + __udfMap.set(this, Object.create(null)); + }; + + /** + Internal-use enum for mapping JS types to DB-bindable types. + These do not (and need not) line up with the SQLITE_type + values. All values in this enum must be truthy and distinct + but they need not be numbers. + */ + const BindTypes = { + null: 1, + number: 2, + string: 3, + boolean: 4, + blob: 5 + }; + BindTypes['undefined'] == BindTypes.null; + if(capi.wasm.bigIntEnabled){ + BindTypes.bigint = BindTypes.number; + } + + /** + This class wraps sqlite3_stmt. Calling this constructor + directly will trigger an exception. Use DB.prepare() to create + new instances. + + For purposes of passing a Stmt instance to C-style sqlite3 + functions, its read-only `pointer` property holds its `sqlite3_stmt*` + pointer value. + */ + const Stmt = function(){ + if(BindTypes!==arguments[2]){ + toss3("Do not call the Stmt constructor directly. Use DB.prepare()."); + } + this.db = arguments[0]; + __ptrMap.set(this, arguments[1]); + this.columnCount = capi.sqlite3_column_count(this.pointer); + this.parameterCount = capi.sqlite3_bind_parameter_count(this.pointer); + }; + + /** Throws if the given DB has been closed, else it is returned. */ + const affirmDbOpen = function(db){ + if(!db.pointer) toss3("DB has been closed."); + return db; + }; + + /** Throws if ndx is not an integer or if it is out of range + for stmt.columnCount, else returns stmt. + + Reminder: this will also fail after the statement is finalized + but the resulting error will be about an out-of-bounds column + index. + */ + const affirmColIndex = function(stmt,ndx){ + if((ndx !== (ndx|0)) || ndx<0 || ndx>=stmt.columnCount){ + toss3("Column index",ndx,"is out of range."); + } + return stmt; + }; + + /** + Expects to be passed (arguments) from DB.exec() and + DB.execMulti(). Does the argument processing/validation, throws + on error, and returns a new object on success: + + { sql: the SQL, opt: optionsObj, cbArg: function} + + cbArg is only set if the opt.callback is set, in which case + it's a function which expects to be passed the current Stmt + and returns the callback argument of the type indicated by + the input arguments. + */ + const parseExecArgs = function(args){ + const out = Object.create(null); + out.opt = Object.create(null); + switch(args.length){ + case 1: + if('string'===typeof args[0] || util.isSQLableTypedArray(args[0])){ + out.sql = args[0]; + }else if(args[0] && 'object'===typeof args[0]){ + out.opt = args[0]; + out.sql = out.opt.sql; + } + break; + case 2: + out.sql = args[0]; + out.opt = args[1]; + break; + default: toss3("Invalid argument count for exec()."); + }; + if(util.isSQLableTypedArray(out.sql)){ + out.sql = util.typedArrayToString(out.sql); + }else if(Array.isArray(out.sql)){ + out.sql = out.sql.join(''); + }else if('string'!==typeof out.sql){ + toss3("Missing SQL argument."); + } + if(out.opt.callback || out.opt.resultRows){ + switch((undefined===out.opt.rowMode) + ? 'stmt' : out.opt.rowMode) { + case 'object': out.cbArg = (stmt)=>stmt.get({}); break; + case 'array': out.cbArg = (stmt)=>stmt.get([]); break; + case 'stmt': + if(Array.isArray(out.opt.resultRows)){ + toss3("Invalid rowMode for resultRows array: must", + "be one of 'array', 'object',", + "or a result column number."); + } + out.cbArg = (stmt)=>stmt; + break; + default: + if(util.isInt32(out.opt.rowMode)){ + out.cbArg = (stmt)=>stmt.get(out.opt.rowMode); + break; + } + toss3("Invalid rowMode:",out.opt.rowMode); + } + } + return out; + }; + + /** + Expects to be given a DB instance or an `sqlite3*` pointer, and an + sqlite3 API result code. If the result code is not falsy, this + function throws an SQLite3Error with an error message from + sqlite3_errmsg(), using dbPtr as the db handle. Note that if it's + passed a non-error code like SQLITE_ROW or SQLITE_DONE, it will + still throw but the error string might be "Not an error." The + various non-0 non-error codes need to be checked for in client + code where they are expected. + */ + DB.checkRc = function(dbPtr, sqliteResultCode){ + if(sqliteResultCode){ + if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; + throw new SQLite3Error([ + "sqlite result code",sqliteResultCode+":", + capi.sqlite3_errmsg(dbPtr) || "Unknown db error." + ].join(' ')); + } + }; + + DB.prototype = { + /** + Finalizes all open statements and closes this database + connection. This is a no-op if the db has already been + closed. After calling close(), `this.pointer` will resolve to + `undefined`, so that can be used to check whether the db + instance is still opened. + */ + close: function(){ + if(this.pointer){ + const pDb = this.pointer; + let s; + const that = this; + Object.keys(__stmtMap.get(this)).forEach((k,s)=>{ + if(s && s.pointer) s.finalize(); + }); + Object.values(__udfMap.get(this)).forEach( + capi.wasm.uninstallFunction.bind(capi.wasm) + ); + __ptrMap.delete(this); + __stmtMap.delete(this); + __udfMap.delete(this); + capi.sqlite3_close_v2(pDb); + delete this.filename; + } + }, + /** + Returns the number of changes, as per sqlite3_changes() + (if the first argument is false) or sqlite3_total_changes() + (if it's true). If the 2nd argument is true, it uses + sqlite3_changes64() or sqlite3_total_changes64(), which + will trigger an exception if this build does not have + BigInt support enabled. + */ + changes: function(total=false,sixtyFour=false){ + const p = affirmDbOpen(this).pointer; + if(total){ + return sixtyFour + ? capi.sqlite3_total_changes64(p) + : capi.sqlite3_total_changes(p); + }else{ + return sixtyFour + ? capi.sqlite3_changes64(p) + : capi.sqlite3_changes(p); + } + }, + /** + Similar to this.filename but will return NULL for + special names like ":memory:". Not of much use until + we have filesystem support. Throws if the DB has + been closed. If passed an argument it then it will return + the filename of the ATTACHEd db with that name, else it assumes + a name of `main`. + */ + fileName: function(dbName){ + return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName||"main"); + }, + /** + Returns true if this db instance has a name which resolves to a + file. If the name is "" or ":memory:", it resolves to false. + Note that it is not aware of the peculiarities of URI-style + names and a URI-style name for a ":memory:" db will fool it. + */ + hasFilename: function(){ + const fn = this.filename; + if(!fn || ':memory'===fn) return false; + return true; + }, + /** + Returns the name of the given 0-based db number, as documented + for sqlite3_db_name(). + */ + dbName: function(dbNumber=0){ + return capi.sqlite3_db_name(affirmDbOpen(this).pointer, dbNumber); + }, + /** + Compiles the given SQL and returns a prepared Stmt. This is + the only way to create new Stmt objects. Throws on error. + + The given SQL must be a string, a Uint8Array holding SQL, or a + WASM pointer to memory holding the NUL-terminated SQL string. + If the SQL contains no statements, an SQLite3Error is thrown. + + Design note: the C API permits empty SQL, reporting it as a 0 + result code and a NULL stmt pointer. Supporting that case here + would cause extra work for all clients: any use of the Stmt API + on such a statement will necessarily throw, so clients would be + required to check `stmt.pointer` after calling `prepare()` in + order to determine whether the Stmt instance is empty or not. + Long-time practice (with other sqlite3 script bindings) + suggests that the empty-prepare case is sufficiently rare (and + useless) that supporting it here would simply hurt overall + usability. + */ + prepare: function(sql){ + affirmDbOpen(this); + const stack = capi.wasm.scopedAllocPush(); + let ppStmt, pStmt; + try{ + ppStmt = capi.wasm.scopedAllocPtr()/* output (sqlite3_stmt**) arg */; + DB.checkRc(this, capi.sqlite3_prepare_v2(this.pointer, sql, -1, ppStmt, null)); + pStmt = capi.wasm.getMemValue(ppStmt, '*'); + } + finally {capi.wasm.scopedAllocPop(stack)} + if(!pStmt) toss3("Cannot prepare empty SQL."); + const stmt = new Stmt(this, pStmt, BindTypes); + __stmtMap.get(this)[pStmt] = stmt; + return stmt; + }, + /** + This function works like execMulti(), and takes most of the + same arguments, but is more efficient (performs much less + work) when the input SQL is only a single statement. If + passed a multi-statement SQL, it only processes the first + one. + + This function supports the following additional options not + supported by execMulti(): + + - .multi: if true, this function acts as a proxy for + execMulti() and behaves identically to that function. + + - .columnNames: if this is an array and the query has + result columns, the array is passed to + Stmt.getColumnNames() to append the column names to it + (regardless of whether the query produces any result + rows). If the query has no result columns, this value is + unchanged. + + The following options to execMulti() are _not_ supported by + this method (they are simply ignored): + + - .saveSql + */ + exec: function(/*(sql [,optionsObj]) or (optionsObj)*/){ + affirmDbOpen(this); + const arg = parseExecArgs(arguments); + if(!arg.sql) return this; + else if(arg.opt.multi){ + return this.execMulti(arg, undefined, BindTypes); + } + const opt = arg.opt; + let stmt, rowTarget; + try { + if(Array.isArray(opt.resultRows)){ + rowTarget = opt.resultRows; + } + stmt = this.prepare(arg.sql); + if(stmt.columnCount && Array.isArray(opt.columnNames)){ + stmt.getColumnNames(opt.columnNames); + } + if(opt.bind) stmt.bind(opt.bind); + if(opt.callback || rowTarget){ + while(stmt.step()){ + const row = arg.cbArg(stmt); + if(rowTarget) rowTarget.push(row); + if(opt.callback){ + stmt._isLocked = true; + opt.callback(row, stmt); + stmt._isLocked = false; + } + } + }else{ + stmt.step(); + } + }finally{ + if(stmt){ + delete stmt._isLocked; + stmt.finalize(); + } + } + return this; + }/*exec()*/, + /** + Executes one or more SQL statements in the form of a single + string. Its arguments must be either (sql,optionsObject) or + (optionsObject). In the latter case, optionsObject.sql + must contain the SQL to execute. Returns this + object. Throws on error. + + If no SQL is provided, or a non-string is provided, an + exception is triggered. Empty SQL, on the other hand, is + simply a no-op. + + The optional options object may contain any of the following + properties: + + - .sql = the SQL to run (unless it's provided as the first + argument). This must be of type string, Uint8Array, or an + array of strings (in which case they're concatenated + together as-is, with no separator between elements, + before evaluation). + + - .bind = a single value valid as an argument for + Stmt.bind(). This is ONLY applied to the FIRST non-empty + statement in the SQL which has any bindable + parameters. (Empty statements are skipped entirely.) + + - .callback = a function which gets called for each row of + the FIRST statement in the SQL which has result + _columns_, but only if that statement has any result + _rows_. The second argument passed to the callback is + always the current Stmt object (so that the caller may + collect column names, or similar). The first argument + passed to the callback defaults to the current Stmt + object but may be changed with ... + + - .rowMode = either a string describing what type of argument + should be passed as the first argument to the callback or an + integer representing a result column index. A `rowMode` of + 'object' causes the results of `stmt.get({})` to be passed to + the `callback` and/or appended to `resultRows`. A value of + 'array' causes the results of `stmt.get([])` to be passed to + passed on. A value of 'stmt' is equivalent to the default, + passing the current Stmt to the callback (noting that it's + always passed as the 2nd argument), but this mode will trigger + an exception if `resultRows` is an array. If `rowMode` is an + integer, only the single value from that result column will be + passed on. Any other value for the option triggers an + exception. + + - .resultRows: if this is an array, it functions similarly to + the `callback` option: each row of the result set (if any) of + the FIRST first statement which has result _columns_ is + appended to the array in the format specified for the `rowMode` + option, with the exception that the only legal values for + `rowMode` in this case are 'array' or 'object', neither of + which is the default. It is legal to use both `resultRows` and + `callback`, but `resultRows` is likely much simpler to use for + small data sets and can be used over a WebWorker-style message + interface. execMulti() throws if `resultRows` is set and + `rowMode` is 'stmt' (which is the default!). + + - saveSql = an optional array. If set, the SQL of each + executed statement is appended to this array before the + statement is executed (but after it is prepared - we + don't have the string until after that). Empty SQL + statements are elided. + + See also the exec() method, which is a close cousin of this + one. + + ACHTUNG #1: The callback MUST NOT modify the Stmt + object. Calling any of the Stmt.get() variants, + Stmt.getColumnName(), or similar, is legal, but calling + step() or finalize() is not. Routines which are illegal + in this context will trigger an exception. + + ACHTUNG #2: The semantics of the `bind` and `callback` + options may well change or those options may be removed + altogether for this function (but retained for exec()). + Generally speaking, neither bind parameters nor a callback + are generically useful when executing multi-statement SQL. + */ + execMulti: function(/*(sql [,obj]) || (obj)*/){ + affirmDbOpen(this); + const wasm = capi.wasm; + const arg = (BindTypes===arguments[2] + /* ^^^ Being passed on from exec() */ + ? arguments[0] : parseExecArgs(arguments)); + if(!arg.sql) return this; + const opt = arg.opt; + const callback = opt.callback; + const resultRows = (Array.isArray(opt.resultRows) + ? opt.resultRows : undefined); + if(resultRows && 'stmt'===opt.rowMode){ + toss3("rowMode 'stmt' is not valid in combination", + "with a resultRows array."); + } + let rowMode = (((callback||resultRows) && (undefined!==opt.rowMode)) + ? opt.rowMode : undefined); + let stmt; + let bind = opt.bind; + const stack = wasm.scopedAllocPush(); + try{ + const isTA = util.isSQLableTypedArray(arg.sql) + /* Optimization: if the SQL is a TypedArray we can save some string + conversion costs. */; + /* Allocate the two output pointers (ppStmt, pzTail) and heap + space for the SQL (pSql). When prepare_v2() returns, pzTail + will point to somewhere in pSql. */ + let sqlByteLen = isTA ? arg.sql.byteLength : wasm.jstrlen(arg.sql); + const ppStmt = wasm.scopedAlloc(/* output (sqlite3_stmt**) arg and pzTail */ + (2 * wasm.ptrSizeof) + + (sqlByteLen + 1/* SQL + NUL */)); + const pzTail = ppStmt + wasm.ptrSizeof /* final arg to sqlite3_prepare_v2() */; + let pSql = pzTail + wasm.ptrSizeof; + const pSqlEnd = pSql + sqlByteLen; + if(isTA) wasm.heap8().set(arg.sql, pSql); + else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); + wasm.setMemValue(pSql + sqlByteLen, 0/*NUL terminator*/); + while(wasm.getMemValue(pSql, 'i8') + /* Maintenance reminder: ^^^^ _must_ be i8 or else we + will very likely cause an endless loop. What that's + doing is checking for a terminating NUL byte. If we + use i32 or similar then we read 4 bytes, read stuff + around the NUL terminator, and get stuck in and + endless loop at the end of the SQL, endlessly + re-preparing an empty statement. */ ){ + wasm.setMemValue(ppStmt, 0, wasm.ptrIR); + wasm.setMemValue(pzTail, 0, wasm.ptrIR); + DB.checkRc(this, capi.sqlite3_prepare_v2( + this.pointer, pSql, sqlByteLen, ppStmt, pzTail + )); + const pStmt = wasm.getMemValue(ppStmt, wasm.ptrIR); + pSql = wasm.getMemValue(pzTail, wasm.ptrIR); + sqlByteLen = pSqlEnd - pSql; + if(!pStmt) continue; + if(Array.isArray(opt.saveSql)){ + opt.saveSql.push(capi.sqlite3_sql(pStmt).trim()); + } + stmt = new Stmt(this, pStmt, BindTypes); + if(bind && stmt.parameterCount){ + stmt.bind(bind); + bind = null; + } + if(stmt.columnCount && undefined!==rowMode){ + /* Only forward SELECT results for the FIRST query + in the SQL which potentially has them. */ + while(stmt.step()){ + stmt._isLocked = true; + const row = arg.cbArg(stmt); + if(callback) callback(row, stmt); + if(resultRows) resultRows.push(row); + stmt._isLocked = false; + } + rowMode = undefined; + }else{ + // Do we need to while(stmt.step()){} here? + stmt.step(); + } + stmt.finalize(); + stmt = null; + } + }catch(e){ + console.warn("DB.execMulti() is propagating exception",opt,e); + throw e; + }finally{ + if(stmt){ + delete stmt._isLocked; + stmt.finalize(); + } + wasm.scopedAllocPop(stack); + } + return this; + }/*execMulti()*/, + /** + Creates a new scalar UDF (User-Defined Function) which is + accessible via SQL code. This function may be called in any + of the following forms: + + - (name, function) + - (name, function, optionsObject) + - (name, optionsObject) + - (optionsObject) + + In the final two cases, the function must be defined as the + 'callback' property of the options object. In the final + case, the function's name must be the 'name' property. + + This can only be used to create scalar functions, not + aggregate or window functions. UDFs cannot be removed from + a DB handle after they're added. + + On success, returns this object. Throws on error. + + When called from SQL, arguments to the UDF, and its result, + will be converted between JS and SQL with as much fidelity + as is feasible, triggering an exception if a type + conversion cannot be determined. Some freedom is afforded + to numeric conversions due to friction between the JS and C + worlds: integers which are larger than 32 bits will be + treated as doubles, as JS does not support 64-bit integers + and it is (as of this writing) illegal to use WASM + functions which take or return 64-bit integers from JS. + + The optional options object may contain flags to modify how + the function is defined: + + - .arity: the number of arguments which SQL calls to this + function expect or require. The default value is the + callback's length property (i.e. the number of declared + parameters it has). A value of -1 means that the function + is variadic and may accept any number of arguments, up to + sqlite3's compile-time limits. sqlite3 will enforce the + argument count if is zero or greater. + + The following properties correspond to flags documented at: + + https://sqlite.org/c3ref/create_function.html + + - .deterministic = SQLITE_DETERMINISTIC + - .directOnly = SQLITE_DIRECTONLY + - .innocuous = SQLITE_INNOCUOUS + + Maintenance reminder: the ability to add new + WASM-accessible functions to the runtime requires that the + WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` + flag. + */ + createFunction: function f(name, callback,opt){ + switch(arguments.length){ + case 1: /* (optionsObject) */ + opt = name; + name = opt.name; + callback = opt.callback; + break; + case 2: /* (name, callback|optionsObject) */ + if(!(callback instanceof Function)){ + opt = callback; + callback = opt.callback; + } + break; + default: break; + } + if(!opt) opt = {}; + if(!(callback instanceof Function)){ + toss3("Invalid arguments: expecting a callback function."); + }else if('string' !== typeof name){ + toss3("Invalid arguments: missing function name."); + } + if(!f._extractArgs){ + /* Static init */ + f._extractArgs = function(argc, pArgv){ + let i, pVal, valType, arg; + const tgt = []; + for(i = 0; i < argc; ++i){ + pVal = capi.wasm.getMemValue(pArgv + (capi.wasm.ptrSizeof * i), + capi.wasm.ptrIR); + /** + Curiously: despite ostensibly requiring 8-byte + alignment, the pArgv array is parcelled into chunks of + 4 bytes (1 pointer each). The values those point to + have 8-byte alignment but the individual argv entries + do not. + */ + valType = capi.sqlite3_value_type(pVal); + switch(valType){ + case capi.SQLITE_INTEGER: + case capi.SQLITE_FLOAT: + arg = capi.sqlite3_value_double(pVal); + break; + case capi.SQLITE_TEXT: + arg = capi.sqlite3_value_text(pVal); + break; + case capi.SQLITE_BLOB:{ + const n = capi.sqlite3_value_bytes(pVal); + const pBlob = capi.sqlite3_value_blob(pVal); + arg = new Uint8Array(n); + let i; + const heap = n ? capi.wasm.heap8() : false; + for(i = 0; i < n; ++i) arg[i] = heap[pBlob+i]; + break; + } + case capi.SQLITE_NULL: + arg = null; break; + default: + toss3("Unhandled sqlite3_value_type()",valType, + "is possibly indicative of incorrect", + "pointer size assumption."); + } + tgt.push(arg); + } + return tgt; + }/*_extractArgs()*/; + f._setResult = function(pCx, val){ + switch(typeof val) { + case 'boolean': + capi.sqlite3_result_int(pCx, val ? 1 : 0); + break; + case 'number': { + (util.isInt32(val) + ? capi.sqlite3_result_int + : capi.sqlite3_result_double)(pCx, val); + break; + } + case 'string': + capi.sqlite3_result_text(pCx, val, -1, capi.SQLITE_TRANSIENT); + break; + case 'object': + if(null===val) { + capi.sqlite3_result_null(pCx); + break; + }else if(util.isBindableTypedArray(val)){ + const pBlob = capi.wasm.mallocFromTypedArray(val); + capi.sqlite3_result_blob(pCx, pBlob, val.byteLength, + capi.SQLITE_TRANSIENT); + capi.wasm.dealloc(pBlob); + break; + } + // else fall through + default: + toss3("Don't not how to handle this UDF result value:",val); + }; + }/*_setResult()*/; + }/*static init*/ + const wrapper = function(pCx, argc, pArgv){ + try{ + f._setResult(pCx, callback.apply(null, f._extractArgs(argc, pArgv))); + }catch(e){ + if(e instanceof capi.WasmAllocError){ + capi.sqlite3_result_error_nomem(pCx); + }else{ + capi.sqlite3_result_error(pCx, e.message, -1); + } + } + }; + const pUdf = capi.wasm.installFunction(wrapper, "v(iii)"); + let fFlags = 0 /*flags for sqlite3_create_function_v2()*/; + if(getOwnOption(opt, 'deterministic')) fFlags |= capi.SQLITE_DETERMINISTIC; + if(getOwnOption(opt, 'directOnly')) fFlags |= capi.SQLITE_DIRECTONLY; + if(getOwnOption(opt, 'innocuous')) fFlags |= capi.SQLITE_INNOCUOUS; + name = name.toLowerCase(); + try { + DB.checkRc(this, capi.sqlite3_create_function_v2( + this.pointer, name, + (opt.hasOwnProperty('arity') ? +opt.arity : callback.length), + capi.SQLITE_UTF8 | fFlags, null/*pApp*/, pUdf, + null/*xStep*/, null/*xFinal*/, null/*xDestroy*/)); + }catch(e){ + capi.wasm.uninstallFunction(pUdf); + throw e; + } + const udfMap = __udfMap.get(this); + if(udfMap[name]){ + try{capi.wasm.uninstallFunction(udfMap[name])} + catch(e){/*ignore*/} + } + udfMap[name] = pUdf; + return this; + }/*createFunction()*/, + /** + Prepares the given SQL, step()s it one time, and returns + the value of the first result column. If it has no results, + undefined is returned. + + If passed a second argument, it is treated like an argument + to Stmt.bind(), so may be any type supported by that + function. Passing the undefined value is the same as passing + no value, which is useful when... + + If passed a 3rd argument, it is expected to be one of the + SQLITE_{typename} constants. Passing the undefined value is + the same as not passing a value. + + Throws on error (e.g. malformedSQL). + */ + selectValue: function(sql,bind,asType){ + let stmt, rc; + try { + stmt = this.prepare(sql).bind(bind); + if(stmt.step()) rc = stmt.get(0,asType); + }finally{ + if(stmt) stmt.finalize(); + } + return rc; + }, + + /** + Returns the number of currently-opened Stmt handles for this db + handle, or 0 if this DB instance is closed. + */ + openStatementCount: function(){ + return this.pointer ? Object.keys(__stmtMap.get(this)).length : 0; + }, + + /** + This function currently does nothing and always throws. It + WILL BE REMOVED pending other refactoring, to eliminate a hard + dependency on Emscripten. This feature will be moved into a + higher-level API or a runtime-configurable feature. + + That said, what its replacement should eventually do is... + + Exports a copy of this db's file as a Uint8Array and + returns it. It is technically not legal to call this while + any prepared statement are currently active because, + depending on the platform, it might not be legal to read + the db while a statement is locking it. Throws if this db + is not open or has any opened statements. + + The resulting buffer can be passed to this class's + constructor to restore the DB. + + Maintenance reminder: the corresponding sql.js impl of this + feature closes the current db, finalizing any active + statements and (seemingly unnecessarily) destroys any UDFs, + copies the file, and then re-opens it (without restoring + the UDFs). Those gymnastics are not necessary on the tested + platform but might be necessary on others. Because of that + eventuality, this interface currently enforces that no + statements are active when this is run. It will throw if + any are. + */ + exportBinaryImage: function(){ + toss3("exportBinaryImage() is slated for removal for portability reasons."); + /*********************** + The following is currently kept only for reference when + porting to some other layer, noting that we may well not be + able to implement this, at this level, when using the OPFS + VFS because of its exclusive locking policy. + + affirmDbOpen(this); + if(this.openStatementCount()>0){ + toss3("Cannot export with prepared statements active!", + "finalize() all statements and try again."); + } + return MODCFG.FS.readFile(this.filename, {encoding:"binary"}); + ***********************/ + } + }/*DB.prototype*/; + + + /** Throws if the given Stmt has been finalized, else stmt is + returned. */ + const affirmStmtOpen = function(stmt){ + if(!stmt.pointer) toss3("Stmt has been closed."); + return stmt; + }; + + /** Returns an opaque truthy value from the BindTypes + enum if v's type is a valid bindable type, else + returns a falsy value. As a special case, a value of + undefined is treated as a bind type of null. */ + const isSupportedBindType = function(v){ + let t = BindTypes[(null===v||undefined===v) ? 'null' : typeof v]; + switch(t){ + case BindTypes.boolean: + case BindTypes.null: + case BindTypes.number: + case BindTypes.string: + return t; + case BindTypes.bigint: + if(capi.wasm.bigIntEnabled) return t; + /* else fall through */ + default: + //console.log("isSupportedBindType",t,v); + return util.isBindableTypedArray(v) ? BindTypes.blob : undefined; + } + }; + + /** + If isSupportedBindType(v) returns a truthy value, this + function returns that value, else it throws. + */ + const affirmSupportedBindType = function(v){ + //console.log('affirmSupportedBindType',v); + return isSupportedBindType(v) || toss3("Unsupported bind() argument type:",typeof v); + }; + + /** + If key is a number and within range of stmt's bound parameter + count, key is returned. + + If key is not a number then it is checked against named + parameters. If a match is found, its index is returned. + + Else it throws. + */ + const affirmParamIndex = function(stmt,key){ + const n = ('number'===typeof key) + ? key : capi.sqlite3_bind_parameter_index(stmt.pointer, key); + if(0===n || !util.isInt32(n)){ + toss3("Invalid bind() parameter name: "+key); + } + else if(n<1 || n>stmt.parameterCount) toss3("Bind index",key,"is out of range."); + return n; + }; + + /** + If stmt._isLocked is truthy, this throws an exception + complaining that the 2nd argument (an operation name, + e.g. "bind()") is not legal while the statement is "locked". + Locking happens before an exec()-like callback is passed a + statement, to ensure that the callback does not mutate or + finalize the statement. If it does not throw, it returns stmt. + */ + const affirmUnlocked = function(stmt,currentOpName){ + if(stmt._isLocked){ + toss3("Operation is illegal when statement is locked:",currentOpName); + } + return stmt; + }; + + /** + Binds a single bound parameter value on the given stmt at the + given index (numeric or named) using the given bindType (see + the BindTypes enum) and value. Throws on error. Returns stmt on + success. + */ + const bindOne = function f(stmt,ndx,bindType,val){ + affirmUnlocked(stmt, 'bind()'); + if(!f._){ + if(capi.wasm.bigIntEnabled){ + f._maxInt = BigInt("0x7fffffffffffffff"); + f._minInt = ~f._maxInt; + } + /* Reminder: when not in BigInt mode, it's impossible for + JS to represent a number out of the range we can bind, + so we have no range checking. */ + f._ = { + string: function(stmt, ndx, val, asBlob){ + if(1){ + /* _Hypothetically_ more efficient than the impl in the 'else' block. */ + const stack = capi.wasm.scopedAllocPush(); + try{ + const n = capi.wasm.jstrlen(val); + const pStr = capi.wasm.scopedAlloc(n); + capi.wasm.jstrcpy(val, capi.wasm.heap8u(), pStr, n, false); + const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; + return f(stmt.pointer, ndx, pStr, n, capi.SQLITE_TRANSIENT); + }finally{ + capi.wasm.scopedAllocPop(stack); + } + }else{ + const bytes = capi.wasm.jstrToUintArray(val,false); + const pStr = capi.wasm.alloc(bytes.length || 1); + capi.wasm.heap8u().set(bytes.length ? bytes : [0], pStr); + try{ + const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; + return f(stmt.pointer, ndx, pStr, bytes.length, capi.SQLITE_TRANSIENT); + }finally{ + capi.wasm.dealloc(pStr); + } + } + } + }; + } + affirmSupportedBindType(val); + ndx = affirmParamIndex(stmt,ndx); + let rc = 0; + switch((null===val || undefined===val) ? BindTypes.null : bindType){ + case BindTypes.null: + rc = capi.sqlite3_bind_null(stmt.pointer, ndx); + break; + case BindTypes.string: + rc = f._.string(stmt, ndx, val, false); + break; + case BindTypes.number: { + let m; + if(util.isInt32(val)) m = capi.sqlite3_bind_int; + else if(capi.wasm.bigIntEnabled && ('bigint'===typeof val)){ + if(valf._maxInt){ + toss3("BigInt value is out of range for int64: "+val); + } + m = capi.sqlite3_bind_int64; + }else if(Number.isInteger(val)){ + m = capi.sqlite3_bind_int64; + }else{ + m = capi.sqlite3_bind_double; + } + rc = m(stmt.pointer, ndx, val); + break; + } + case BindTypes.boolean: + rc = capi.sqlite3_bind_int(stmt.pointer, ndx, val ? 1 : 0); + break; + case BindTypes.blob: { + if('string'===typeof val){ + rc = f._.string(stmt, ndx, val, true); + }else if(!util.isBindableTypedArray(val)){ + toss3("Binding a value as a blob requires", + "that it be a string, Uint8Array, or Int8Array."); + }else if(1){ + /* _Hypothetically_ more efficient than the impl in the 'else' block. */ + const stack = capi.wasm.scopedAllocPush(); + try{ + const pBlob = capi.wasm.scopedAlloc(val.byteLength || 1); + capi.wasm.heap8().set(val.byteLength ? val : [0], pBlob) + rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, + capi.SQLITE_TRANSIENT); + }finally{ + capi.wasm.scopedAllocPop(stack); + } + }else{ + const pBlob = capi.wasm.mallocFromTypedArray(val); + try{ + rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, + capi.SQLITE_TRANSIENT); + }finally{ + capi.wasm.dealloc(pBlob); + } + } + break; + } + default: + console.warn("Unsupported bind() argument type:",val); + toss3("Unsupported bind() argument type: "+(typeof val)); + } + if(rc) checkDbRc(stmt.db.pointer, rc); + return stmt; + }; + + Stmt.prototype = { + /** + "Finalizes" this statement. This is a no-op if the + statement has already been finalizes. Returns + undefined. Most methods in this class will throw if called + after this is. + */ + finalize: function(){ + if(this.pointer){ + affirmUnlocked(this,'finalize()'); + delete __stmtMap.get(this.db)[this.pointer]; + capi.sqlite3_finalize(this.pointer); + __ptrMap.delete(this); + delete this.columnCount; + delete this.parameterCount; + delete this.db; + delete this._isLocked; + } + }, + /** Clears all bound values. Returns this object. + Throws if this statement has been finalized. */ + clearBindings: function(){ + affirmUnlocked(affirmStmtOpen(this), 'clearBindings()') + capi.sqlite3_clear_bindings(this.pointer); + this._mayGet = false; + return this; + }, + /** + Resets this statement so that it may be step()ed again + from the beginning. Returns this object. Throws if this + statement has been finalized. + + If passed a truthy argument then this.clearBindings() is + also called, otherwise any existing bindings, along with + any memory allocated for them, are retained. + */ + reset: function(alsoClearBinds){ + affirmUnlocked(this,'reset()'); + if(alsoClearBinds) this.clearBindings(); + capi.sqlite3_reset(affirmStmtOpen(this).pointer); + this._mayGet = false; + return this; + }, + /** + Binds one or more values to its bindable parameters. It + accepts 1 or 2 arguments: + + If passed a single argument, it must be either an array, an + object, or a value of a bindable type (see below). + + If passed 2 arguments, the first one is the 1-based bind + index or bindable parameter name and the second one must be + a value of a bindable type. + + Bindable value types: + + - null is bound as NULL. + + - undefined as a standalone value is a no-op intended to + simplify certain client-side use cases: passing undefined + as a value to this function will not actually bind + anything and this function will skip confirmation that + binding is even legal. (Those semantics simplify certain + client-side uses.) Conversely, a value of undefined as an + array or object property when binding an array/object + (see below) is treated the same as null. + + - Numbers are bound as either doubles or integers: doubles + if they are larger than 32 bits, else double or int32, + depending on whether they have a fractional part. (It is, + as of this writing, illegal to call (from JS) a WASM + function which either takes or returns an int64.) + Booleans are bound as integer 0 or 1. It is not expected + the distinction of binding doubles which have no + fractional parts is integers is significant for the + majority of clients due to sqlite3's data typing + model. If capi.wasm.bigIntEnabled is true then this + routine will bind BigInt values as 64-bit integers. + + - Strings are bound as strings (use bindAsBlob() to force + blob binding). + + - Uint8Array and Int8Array instances are bound as blobs. + (TODO: binding the other TypedArray types.) + + If passed an array, each element of the array is bound at + the parameter index equal to the array index plus 1 + (because arrays are 0-based but binding is 1-based). + + If passed an object, each object key is treated as a + bindable parameter name. The object keys _must_ match any + bindable parameter names, including any `$`, `@`, or `:` + prefix. Because `$` is a legal identifier chararacter in + JavaScript, that is the suggested prefix for bindable + parameters: `stmt.bind({$a: 1, $b: 2})`. + + It returns this object on success and throws on + error. Errors include: + + - Any bind index is out of range, a named bind parameter + does not match, or this statement has no bindable + parameters. + + - Any value to bind is of an unsupported type. + + - Passed no arguments or more than two. + + - The statement has been finalized. + */ + bind: function(/*[ndx,] arg*/){ + affirmStmtOpen(this); + let ndx, arg; + switch(arguments.length){ + case 1: ndx = 1; arg = arguments[0]; break; + case 2: ndx = arguments[0]; arg = arguments[1]; break; + default: toss3("Invalid bind() arguments."); + } + if(undefined===arg){ + /* It might seem intuitive to bind undefined as NULL + but this approach simplifies certain client-side + uses when passing on arguments between 2+ levels of + functions. */ + return this; + }else if(!this.parameterCount){ + toss3("This statement has no bindable parameters."); + } + this._mayGet = false; + if(null===arg){ + /* bind NULL */ + return bindOne(this, ndx, BindTypes.null, arg); + } + else if(Array.isArray(arg)){ + /* bind each entry by index */ + if(1!==arguments.length){ + toss3("When binding an array, an index argument is not permitted."); + } + arg.forEach((v,i)=>bindOne(this, i+1, affirmSupportedBindType(v), v)); + return this; + } + else if('object'===typeof arg/*null was checked above*/ + && !util.isBindableTypedArray(arg)){ + /* Treat each property of arg as a named bound parameter. */ + if(1!==arguments.length){ + toss3("When binding an object, an index argument is not permitted."); + } + Object.keys(arg) + .forEach(k=>bindOne(this, k, + affirmSupportedBindType(arg[k]), + arg[k])); + return this; + }else{ + return bindOne(this, ndx, affirmSupportedBindType(arg), arg); + } + toss3("Should not reach this point."); + }, + /** + Special case of bind() which binds the given value using the + BLOB binding mechanism instead of the default selected one for + the value. The ndx may be a numbered or named bind index. The + value must be of type string, null/undefined (both get treated + as null), or a TypedArray of a type supported by the bind() + API. + + If passed a single argument, a bind index of 1 is assumed and + the first argument is the value. + */ + bindAsBlob: function(ndx,arg){ + affirmStmtOpen(this); + if(1===arguments.length){ + arg = ndx; + ndx = 1; + } + const t = affirmSupportedBindType(arg); + if(BindTypes.string !== t && BindTypes.blob !== t + && BindTypes.null !== t){ + toss3("Invalid value type for bindAsBlob()"); + } + bindOne(this, ndx, BindTypes.blob, arg); + this._mayGet = false; + return this; + }, + /** + Steps the statement one time. If the result indicates that + a row of data is available, true is returned. If no row of + data is available, false is returned. Throws on error. + */ + step: function(){ + affirmUnlocked(this, 'step()'); + const rc = capi.sqlite3_step(affirmStmtOpen(this).pointer); + switch(rc){ + case capi.SQLITE_DONE: return this._mayGet = false; + case capi.SQLITE_ROW: return this._mayGet = true; + default: + this._mayGet = false; + console.warn("sqlite3_step() rc=",rc,"SQL =", + capi.sqlite3_sql(this.pointer)); + checkDbRc(this.db.pointer, rc); + }; + }, + /** + Fetches the value from the given 0-based column index of + the current data row, throwing if index is out of range. + + Requires that step() has just returned a truthy value, else + an exception is thrown. + + By default it will determine the data type of the result + automatically. If passed a second arugment, it must be one + of the enumeration values for sqlite3 types, which are + defined as members of the sqlite3 module: SQLITE_INTEGER, + SQLITE_FLOAT, SQLITE_TEXT, SQLITE_BLOB. Any other value, + except for undefined, will trigger an exception. Passing + undefined is the same as not passing a value. It is legal + to, e.g., fetch an integer value as a string, in which case + sqlite3 will convert the value to a string. + + If ndx is an array, this function behaves a differently: it + assigns the indexes of the array, from 0 to the number of + result columns, to the values of the corresponding column, + and returns that array. + + If ndx is a plain object, this function behaves even + differentlier: it assigns the properties of the object to + the values of their corresponding result columns. + + Blobs are returned as Uint8Array instances. + + Potential TODO: add type ID SQLITE_JSON, which fetches the + result as a string and passes it (if it's not null) to + JSON.parse(), returning the result of that. Until then, + getJSON() can be used for that. + */ + get: function(ndx,asType){ + if(!affirmStmtOpen(this)._mayGet){ + toss3("Stmt.step() has not (recently) returned true."); + } + if(Array.isArray(ndx)){ + let i = 0; + while(i=Number.MIN_SAFE_INTEGER && rc<=Number.MAX_SAFE_INTEGER){ + /* Coerce "normal" number ranges to normal number values, + and only return BigInt-type values for numbers out of this + range. */ + return Number(rc).valueOf(); + } + return rc; + }else{ + const rc = capi.sqlite3_column_double(this.pointer, ndx); + if(rc>Number.MAX_SAFE_INTEGER || rctoss3("The pointer property is read-only.") + } + Object.defineProperty(Stmt.prototype, 'pointer', prop); + Object.defineProperty(DB.prototype, 'pointer', prop); + } + + /** The OO API's public namespace. */ + sqlite3.oo1 = { + version: { + lib: capi.sqlite3_libversion(), + ooApi: "0.1" + }, + DB, + Stmt + }/*SQLite3 object*/; +})(self); diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js new file mode 100644 index 0000000000..a04029e302 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -0,0 +1,394 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file contains extensions to the sqlite3 WASM API related to the + Origin-Private FileSystem (OPFS). It is intended to be appended to + the main JS deliverable somewhere after sqlite3-api-glue.js and + before sqlite3-api-cleanup.js. + + Significant notes and limitations: + + - As of this writing, OPFS is still very much in flux and only + available in bleeding-edge versions of Chrome (v102+, noting that + that number will increase as the OPFS API matures). + + - The _synchronous_ family of OPFS features (which is what this API + requires) are only available in non-shared Worker threads. This + file tries to detect that case and becomes a no-op if those + features do not seem to be available. +*/ + +// FileSystemHandle +// FileSystemDirectoryHandle +// FileSystemFileHandle +// FileSystemFileHandle.prototype.createSyncAccessHandle +self.sqlite3.postInit.push(function(self, sqlite3){ + const warn = console.warn.bind(console); + if(!self.importScripts || !self.FileSystemFileHandle + || !self.FileSystemFileHandle.prototype.createSyncAccessHandle){ + warn("OPFS not found or its sync API is not available in this environment."); + return; + }else if(!sqlite3.capi.wasm.bigIntEnabled){ + error("OPFS requires BigInt support but sqlite3.capi.wasm.bigIntEnabled is false."); + return; + } + //warn('self.FileSystemFileHandle =',self.FileSystemFileHandle); + //warn('self.FileSystemFileHandle.prototype =',self.FileSystemFileHandle.prototype); + const toss = (...args)=>{throw new Error(args.join(' '))}; + /* This is a web worker, so init the worker-based API. */ + const capi = sqlite3.capi, + wasm = capi.wasm; + const sqlite3_vfs = capi.sqlite3_vfs + || toss("Missing sqlite3.capi.sqlite3_vfs object."); + const sqlite3_file = capi.sqlite3_file + || toss("Missing sqlite3.capi.sqlite3_file object."); + const sqlite3_io_methods = capi.sqlite3_io_methods + || toss("Missing sqlite3.capi.sqlite3_io_methods object."); + const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder."); + const error = console.error.bind(console), + debug = console.debug.bind(console), + log = console.log.bind(console); + warn("UNDER CONSTRUCTION: setting up OPFS VFS..."); + + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. Though we cannot currently handle + that case, the hope is to eventually be able to. */; + const oVfs = new sqlite3_vfs(); + const oIom = new sqlite3_io_methods(); + oVfs.$iVersion = 2/*yes, two*/; + oVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + oVfs.$mxPathname = 1024/*sure, why not?*/; + oVfs.$zName = wasm.allocCString("opfs"); + oVfs.ondispose = [ + '$zName', oVfs.$zName, + 'cleanup dVfs', ()=>(dVfs ? dVfs.dispose() : null) + ]; + if(dVfs){ + oVfs.$xSleep = dVfs.$xSleep; + oVfs.$xRandomness = dVfs.$xRandomness; + } + // All C-side memory of oVfs is zeroed out, but just to be explicit: + oVfs.$xDlOpen = oVfs.$xDlError = oVfs.$xDlSym = oVfs.$xDlClose = null; + + /** + Pedantic sidebar about oVfs.ondispose: the entries in that array + are items to clean up when oVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown order + of the wasm engine and the JS one are undefined so there is no + guaranty that the oVfs instance would be available in one + environment or the other when sqlite3_os_end() is called (_if_ it + gets called at all in a wasm build, which is undefined). + */ + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error (e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc.). + + Returns a proxy for this function which is bound to tgt and takes + 2 args (name,func). That function returns the same thing, + permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + */ + const installMethod = function callee(tgt, name, func){ + if(!(tgt instanceof StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + //log("installMethod",tgt, name, sigN); + const fProxy = 1 + // We can remove this proxy middle-man once the VFS is working + ? callee.argcProxy(func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; + + /** + Map of sqlite3_file pointers to OPFS handles. + */ + const __opfsHandles = Object.create(null); + + const randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(''); + }; + + //const rootDir = await navigator.storage.getDirectory(); + + //////////////////////////////////////////////////////////////////////// + // Set up OPFS VFS methods... + let inst = installMethod(oVfs); + inst('xOpen', function(pVfs, zName, pFile, flags, pOutFlags){ + const f = new sqlite3_file(pFile); + f.$pMethods = oIom.pointer; + __opfsHandles[pFile] = f; + f.opfsHandle = null /* TODO */; + if(capi.SQLITE_OPEN_DELETEONCLOSE){ + f.deleteOnClose = true; + } + f.filename = zName ? wasm.cstringToJs(zName) : randomFilename(); + error("OPFS sqlite3_vfs::xOpen is not yet full implemented."); + return capi.SQLITE_IOERR; + }) + ('xFullPathname', function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return i SQLITE_DEFAULT_SECTOR_SIZE */; + //}) + + const rc = capi.sqlite3_vfs_register(oVfs.pointer, 0); + if(rc){ + oVfs.dispose(); + toss("sqlite3_vfs_register(OPFS) failed with rc",rc); + } + capi.sqlite3_vfs_register.addReference(oVfs, oIom); + warn("End of (very incomplete) OPFS setup.", oVfs); + //oVfs.dispose()/*only because we can't yet do anything with it*/; +}); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js new file mode 100644 index 0000000000..60ed61477e --- /dev/null +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -0,0 +1,593 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is intended to be combined at build-time with other + related code, most notably a header and footer which wraps this whole + file into an Emscripten Module.postRun() handler which has a parameter + named "Module" (the Emscripten Module object). The exact requirements, + conventions, and build process are very much under construction and + will be (re)documented once they've stopped fluctuating so much. + + Specific goals of this project: + + - Except where noted in the non-goals, provide a more-or-less + feature-complete wrapper to the sqlite3 C API, insofar as WASM + feature parity with C allows for. In fact, provide at least 3 + APIs... + + 1) Bind a low-level sqlite3 API which is as close to the native + one as feasible in terms of usage. + + 2) A higher-level API, more akin to sql.js and node.js-style + implementations. This one speaks directly to the low-level + API. This API must be used from the same thread as the + low-level API. + + 3) A second higher-level API which speaks to the previous APIs via + worker messages. This one is intended for use in the main + thread, with the lower-level APIs installed in a Worker thread, + and talking to them via Worker messages. Because Workers are + asynchronouns and have only a single message channel, some + acrobatics are needed here to feed async work results back to + the client (as we cannot simply pass around callbacks between + the main and Worker threads). + + - Insofar as possible, support client-side storage using JS + filesystem APIs. As of this writing, such things are still very + much TODO. Initial testing with using IndexedDB as backing storage + showed it to work reasonably well, but it's also too easy to + corrupt by using a web page in two browser tabs because IndexedDB + lacks the locking features needed to support that. + + Specific non-goals of this project: + + - As WASM is a web-centric technology and UTF-8 is the King of + Encodings in that realm, there are no currently plans to support + the UTF16-related sqlite3 APIs. They would add a complication to + the bindings for no appreciable benefit. Though web-related + implementation details take priority, the lower-level WASM module + "should" work in non-web WASM environments. + + - Supporting old or niche-market platforms. WASM is built for a + modern web and requires modern platforms. + + - Though scalar User-Defined Functions (UDFs) may be created in + JavaScript, there are currently no plans to add support for + aggregate and window functions. + + Attribution: + + This project is endebted to the work of sql.js: + + https://github.com/sql-js/sql.js + + sql.js was an essential stepping stone in this code's development as + it demonstrated how to handle some of the WASM-related voodoo (like + handling pointers-to-pointers and adding JS implementations of + C-bound callback functions). These APIs have a considerably + different shape than sql.js's, however. +*/ + +/** + This global symbol is is only a temporary measure: the JS-side + post-processing will remove that object from the global scope when + setup is complete. We require it there temporarily in order to glue + disparate parts together during the loading of the API (which spans + several components). + + This function requires a configuration object intended to abstract + away details specific to any given WASM environment, primarily so + that it can be used without any _direct_ dependency on Emscripten. + (That said, OO API #1 requires, as of this writing, Emscripten's + virtual filesystem API. Baby steps.) +*/ +self.sqlite3ApiBootstrap = function(config){ + 'use strict'; + + /** Throws a new Error, the message of which is the concatenation + all args with a space between each. */ + const toss = (...args)=>{throw new Error(args.join(' '))}; + + /** + Returns true if n is a 32-bit (signed) integer, else + false. This is used for determining when we need to switch to + double-type DB operations for integer values in order to keep + more precision. + */ + const isInt32 = function(n){ + return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/) + && !!(n===(n|0) && n<=2147483647 && n>=-2147483648); + }; + + /** Returns v if v appears to be a TypedArray, else false. */ + const isTypedArray = (v)=>{ + return (v && v.constructor && isInt32(v.constructor.BYTES_PER_ELEMENT)) ? v : false; + }; + + /** + Returns true if v appears to be one of our bind()-able + TypedArray types: Uint8Array or Int8Array. Support for + TypedArrays with element sizes >1 is TODO. + */ + const isBindableTypedArray = (v)=>{ + return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + }; + + /** + Returns true if v appears to be one of the TypedArray types + which is legal for holding SQL code (as opposed to binary blobs). + + Currently this is the same as isBindableTypedArray() but it + seems likely that we'll eventually want to add Uint32Array + and friends to the isBindableTypedArray() list but not to the + isSQLableTypedArray() list. + */ + const isSQLableTypedArray = (v)=>{ + return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + }; + + /** Returns true if isBindableTypedArray(v) does, else throws with a message + that v is not a supported TypedArray value. */ + const affirmBindableTypedArray = (v)=>{ + return isBindableTypedArray(v) + || toss("Value is not of a supported TypedArray type."); + }; + + const utf8Decoder = new TextDecoder('utf-8'); + const typedArrayToString = (str)=>utf8Decoder.decode(str); + + /** + An Error subclass specifically for reporting Wasm-level malloc() + failure and enabling clients to unambiguously identify such + exceptions. + */ + class WasmAllocError extends Error { + constructor(...args){ + super(...args); + this.name = 'WasmAllocError'; + } + }; + + /** + The main sqlite3 binding API gets installed into this object, + mimicking the C API as closely as we can. The numerous members + names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as + possible, identically to the C-native counterparts, as documented at: + + https://www.sqlite.org/c3ref/intro.html + + A very few exceptions require an additional level of proxy + function or may otherwise require special attention in the WASM + environment, and all such cases are document here. Those not + documented here are installed as 1-to-1 proxies for their C-side + counterparts. + */ + const capi = { + /** + An Error subclass which is thrown by this object's alloc() method + on OOM. + */ + WasmAllocError: WasmAllocError, + /** + The API's one single point of access to the WASM-side memory + allocator. Works like malloc(3) (and is likely bound to + malloc()) but throws an WasmAllocError if allocation fails. It is + important that any code which might pass through the sqlite3 C + API NOT throw and must instead return SQLITE_NOMEM (or + equivalent, depending on the context). + + That said, very few cases in the API can result in + client-defined functions propagating exceptions via the C-style + API. Most notably, this applies ot User-defined SQL Functions + (UDFs) registered via sqlite3_create_function_v2(). For that + specific case it is recommended that all UDF creation be + funneled through a utility function and that a wrapper function + be added around the UDF which catches any exception and sets + the error state to OOM. (The overall complexity of registering + UDFs essentially requires a helper for doing so!) + */ + alloc: undefined/*installed later*/, + /** + The API's one single point of access to the WASM-side memory + deallocator. Works like free(3) (and is likely bound to + free()). + */ + dealloc: undefined/*installed later*/, + /** + When using sqlite3_open_v2() it is important to keep the following + in mind: + + https://www.sqlite.org/c3ref/open.html + + - The flags for use with its 3rd argument are installed in this + object using the C-cide names, e.g. SQLITE_OPEN_CREATE. + + - If the combination of flags passed to it are invalid, + behavior is undefined. Thus is is never okay to call this + with fewer than 3 arguments, as JS will default the + missing arguments to `undefined`, which will result in a + flag value of 0. Most of the available SQLITE_OPEN_xxx + flags are meaningless in the WASM build, e.g. the mutext- + and cache-related flags, but they are retained in this + API for consistency's sake. + + - The final argument to this function specifies the VFS to + use, which is largely (but not entirely!) meaningless in + the WASM environment. It should always be null or + undefined, and it is safe to elide that argument when + calling this function. + */ + sqlite3_open_v2: function(filename,dbPtrPtr,flags,vfsStr){}/*installed later*/, + /** + The sqlite3_prepare_v3() binding handles two different uses + with differing JS/WASM semantics: + + 1) sqlite3_prepare_v3(pDb, sqlString, -1, prepFlags, ppStmt [, null]) + + 2) sqlite3_prepare_v3(pDb, sqlPointer, sqlByteLen, prepFlags, ppStmt, sqlPointerToPointer) + + Note that the SQL length argument (the 3rd argument) must, for + usage (1), always be negative because it must be a byte length + and that value is expensive to calculate from JS (where only + the character length of strings is readily available). It is + retained in this API's interface for code/documentation + compatibility reasons but is currently _always_ ignored. With + usage (2), the 3rd argument is used as-is but is is still + critical that the C-style input string (2nd argument) be + terminated with a 0 byte. + + In usage (1), the 2nd argument must be of type string, + Uint8Array, or Int8Array (either of which is assumed to + hold SQL). If it is, this function assumes case (1) and + calls the underyling C function with the equivalent of: + + (pDb, sqlAsString, -1, prepFlags, ppStmt, null) + + The pzTail argument is ignored in this case because its result + is meaningless when a string-type value is passed through + (because the string goes through another level of internal + conversion for WASM's sake and the result pointer would refer + to that transient conversion's memory, not the passed-in + string). + + If the sql argument is not a string, it must be a _pointer_ to + a NUL-terminated string which was allocated in the WASM memory + (e.g. using cwapi.wasm.alloc() or equivalent). In that case, + the final argument may be 0/null/undefined or must be a pointer + to which the "tail" of the compiled SQL is written, as + documented for the C-side sqlite3_prepare_v3(). In case (2), + the underlying C function is called with the equivalent of: + + (pDb, sqlAsPointer, (sqlByteLen||-1), prepFlags, ppStmt, pzTail) + + It returns its result and compiled statement as documented in + the C API. Fetching the output pointers (5th and 6th + parameters) requires using capi.wasm.getMemValue() (or + equivalent) and the pzTail will point to an address relative to + the sqlAsPointer value. + + If passed an invalid 2nd argument type, this function will + return SQLITE_MISUSE but will unfortunately be able to return + any additional error information because we have no way to set + the db's error state such that this function could return a + non-0 integer and the client could call sqlite3_errcode() or + sqlite3_errmsg() to fetch it. See the RFE at: + + https://sqlite.org/forum/forumpost/f9eb79b11aefd4fc81d + + The alternative would be to throw an exception for that case, + but that would be in strong constrast to the rest of the + C-level API and seems likely to cause more confusion. + + Side-note: in the C API the function does not fail if provided + an empty string but its result output pointer will be NULL. + */ + sqlite3_prepare_v3: function(dbPtr, sql, sqlByteLen, prepFlags, + stmtPtrPtr, strPtrPtr){}/*installed later*/, + + /** + Equivalent to calling sqlite3_prapare_v3() with 0 as its 4th argument. + */ + sqlite3_prepare_v2: function(dbPtr, sql, sqlByteLen, stmtPtrPtr, + strPtrPtr){}/*installed later*/, + + /** + Various internal-use utilities are added here as needed. They + are bound to an object only so that we have access to them in + the differently-scoped steps of the API bootstrapping + process. At the end of the API setup process, this object gets + removed. + */ + util:{ + isInt32, isTypedArray, isBindableTypedArray, isSQLableTypedArray, + affirmBindableTypedArray, typedArrayToString + }, + + /** + Holds state which are specific to the WASM-related + infrastructure and glue code. It is not expected that client + code will normally need these, but they're exposed here in case + it does. These APIs are _not_ to be considered an + official/stable part of the sqlite3 WASM API. They may change + as the developers' experience suggests appropriate changes. + + Note that a number of members of this object are injected + dynamically after the api object is fully constructed, so + not all are documented inline here. + */ + wasm: { + //^^^ TODO?: move wasm from sqlite3.capi.wasm to sqlite3.wasm + /** + Emscripten APIs have a deep-seated assumption that all pointers + are 32 bits. We'll remain optimistic that that won't always be + the case and will use this constant in places where we might + otherwise use a hard-coded 4. + */ + ptrSizeof: config.wasmPtrSizeof || 4, + /** + The WASM IR (Intermediate Representation) value for + pointer-type values. It MUST refer to a value type of the + size described by this.ptrSizeof _or_ it may be any value + which ends in '*', which Emscripten's glue code internally + translates to i32. + */ + ptrIR: config.wasmPtrIR || "i32", + /** + True if BigInt support was enabled via (e.g.) the + Emscripten -sWASM_BIGINT flag, else false. When + enabled, certain 64-bit sqlite3 APIs are enabled which + are not otherwise enabled due to JS/WASM int64 + impedence mismatches. + */ + bigIntEnabled: !!config.bigIntEnabled, + /** + The symbols exported by the WASM environment. + */ + exports: config.exports + || toss("Missing API config.exports (WASM module exports)."), + + /** + When Emscripten compiles with `-sIMPORT_MEMORY`, it + initalizes the heap and imports it into wasm, as opposed to + the other way around. In this case, the memory is not + available via this.exports.memory. + */ + memory: config.memory || config.exports['memory'] + || toss("API config object requires a WebAssembly.Memory object", + "in either config.exports.memory (exported)", + "or config.memory (imported)."), + /* Many more wasm-related APIs get installed later on. */ + }/*wasm*/ + }/*capi*/; + + /** + capi.wasm.alloc()'s srcTypedArray.byteLength bytes, + populates them with the values from the source + TypedArray, and returns the pointer to that memory. The + returned pointer must eventually be passed to + capi.wasm.dealloc() to clean it up. + + As a special case, to avoid further special cases where + this is used, if srcTypedArray.byteLength is 0, it + allocates a single byte and sets it to the value + 0. Even in such cases, calls must behave as if the + allocated memory has exactly srcTypedArray.byteLength + bytes. + + ACHTUNG: this currently only works for Uint8Array and + Int8Array types and will throw if srcTypedArray is of + any other type. + */ + capi.wasm.mallocFromTypedArray = function(srcTypedArray){ + affirmBindableTypedArray(srcTypedArray); + const pRet = this.alloc(srcTypedArray.byteLength || 1); + this.heapForSize(srcTypedArray.constructor).set(srcTypedArray.byteLength ? srcTypedArray : [0], pRet); + return pRet; + }.bind(capi.wasm); + + const keyAlloc = config.allocExportName || 'malloc', + keyDealloc = config.deallocExportName || 'free'; + for(const key of [keyAlloc, keyDealloc]){ + const f = capi.wasm.exports[key]; + if(!(f instanceof Function)) toss("Missing required exports[",key,"] function."); + } + capi.wasm.alloc = function(n){ + const m = this.exports[keyAlloc](n); + if(!m) throw new WasmAllocError("Failed to allocate "+n+" bytes."); + return m; + }.bind(capi.wasm) + capi.wasm.dealloc = (m)=>capi.wasm.exports[keyDealloc](m); + + /** + Reports info about compile-time options using + sqlite_compileoption_get() and sqlite3_compileoption_used(). It + has several distinct uses: + + If optName is an array then it is expected to be a list of + compilation options and this function returns an object + which maps each such option to true or false, indicating + whether or not the given option was included in this + build. That object is returned. + + If optName is an object, its keys are expected to be compilation + options and this function sets each entry to true or false, + indicating whether the compilation option was used or not. That + object is returned. + + If passed no arguments then it returns an object mapping + all known compilation options to their compile-time values, + or boolean true if they are defined with no value. This + result, which is relatively expensive to compute, is cached + and returned for future no-argument calls. + + In all other cases it returns true if the given option was + active when when compiling the sqlite3 module, else false. + + Compile-time option names may optionally include their + "SQLITE_" prefix. When it returns an object of all options, + the prefix is elided. + */ + capi.wasm.compileOptionUsed = function f(optName){ + if(!arguments.length){ + if(f._result) return f._result; + else if(!f._opt){ + f._rx = /^([^=]+)=(.+)/; + f._rxInt = /^-?\d+$/; + f._opt = function(opt, rv){ + const m = f._rx.exec(opt); + rv[0] = (m ? m[1] : opt); + rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; + }; + } + const rc = {}, ov = [0,0]; + let i = 0, k; + while((k = capi.sqlite3_compileoption_get(i++))){ + f._opt(k,ov); + rc[ov[0]] = ov[1]; + } + return f._result = rc; + }else if(Array.isArray(optName)){ + const rc = {}; + optName.forEach((v)=>{ + rc[v] = capi.sqlite3_compileoption_used(v); + }); + return rc; + }else if('object' === typeof optName){ + Object.keys(optName).forEach((k)=> { + optName[k] = capi.sqlite3_compileoption_used(k); + }); + return optName; + } + return ( + 'string'===typeof optName + ) ? !!capi.sqlite3_compileoption_used(optName) : false; + }/*compileOptionUsed()*/; + + capi.wasm.bindingSignatures = [ + /** + Signatures for the WASM-exported C-side functions. Each entry + is an array with 2+ elements: + + ["c-side name", + "result type" (capi.wasm.xWrap() syntax), + [arg types in xWrap() syntax] + // ^^^ this needn't strictly be an array: it can be subsequent + // elements instead: [x,y,z] is equivalent to x,y,z + ] + */ + // Please keep these sorted by function name! + ["sqlite3_bind_blob","int", "sqlite3_stmt*", "int", "*", "int", "*"], + ["sqlite3_bind_double","int", "sqlite3_stmt*", "int", "f64"], + ["sqlite3_bind_int","int", "sqlite3_stmt*", "int", "int"], + ["sqlite3_bind_null",undefined, "sqlite3_stmt*", "int"], + ["sqlite3_bind_parameter_count", "int", "sqlite3_stmt*"], + ["sqlite3_bind_parameter_index","int", "sqlite3_stmt*", "string"], + ["sqlite3_bind_text","int", "sqlite3_stmt*", "int", "string", "int", "int"], + ["sqlite3_close_v2", "int", "sqlite3*"], + ["sqlite3_changes", "int", "sqlite3*"], + ["sqlite3_clear_bindings","int", "sqlite3_stmt*"], + ["sqlite3_column_blob","*", "sqlite3_stmt*", "int"], + ["sqlite3_column_bytes","int", "sqlite3_stmt*", "int"], + ["sqlite3_column_count", "int", "sqlite3_stmt*"], + ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], + ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], + ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_text","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], + ["sqlite3_compileoption_get", "string", "int"], + ["sqlite3_compileoption_used", "int", "string"], + ["sqlite3_create_function_v2", "int", + "sqlite3*", "string", "int", "int", "*", "*", "*", "*", "*"], + ["sqlite3_data_count", "int", "sqlite3_stmt*"], + ["sqlite3_db_filename", "string", "sqlite3*", "string"], + ["sqlite3_db_name", "string", "sqlite3*", "int"], + ["sqlite3_errmsg", "string", "sqlite3*"], + ["sqlite3_error_offset", "int", "sqlite3*"], + ["sqlite3_errstr", "string", "int"], + //["sqlite3_exec", "int", "sqlite3*", "string", "*", "*", "**"], + // ^^^ TODO: we need a wrapper to support passing a function pointer or a function + // for the callback. + ["sqlite3_expanded_sql", "string", "sqlite3_stmt*"], + ["sqlite3_extended_errcode", "int", "sqlite3*"], + ["sqlite3_extended_result_codes", "int", "sqlite3*", "int"], + ["sqlite3_finalize", "int", "sqlite3_stmt*"], + ["sqlite3_initialize", undefined], + ["sqlite3_interrupt", undefined, "sqlite3*" + /* ^^^ we cannot actually currently support this because JS is + single-threaded and we don't have a portable way to access a DB + from 2 SharedWorkers concurrently. */], + ["sqlite3_libversion", "string"], + ["sqlite3_libversion_number", "int"], + ["sqlite3_open", "int", "string", "*"], + ["sqlite3_open_v2", "int", "string", "*", "int", "string"], + /* sqlite3_prepare_v2() and sqlite3_prepare_v3() are handled + separately due to us requiring two different sets of semantics + for those, depending on how their SQL argument is provided. */ + ["sqlite3_reset", "int", "sqlite3_stmt*"], + ["sqlite3_result_blob",undefined, "*", "*", "int", "*"], + ["sqlite3_result_double",undefined, "*", "f64"], + ["sqlite3_result_error",undefined, "*", "string", "int"], + ["sqlite3_result_error_code", undefined, "*", "int"], + ["sqlite3_result_error_nomem", undefined, "*"], + ["sqlite3_result_error_toobig", undefined, "*"], + ["sqlite3_result_int",undefined, "*", "int"], + ["sqlite3_result_null",undefined, "*"], + ["sqlite3_result_text",undefined, "*", "string", "int", "*"], + ["sqlite3_sourceid", "string"], + ["sqlite3_sql", "string", "sqlite3_stmt*"], + ["sqlite3_step", "int", "sqlite3_stmt*"], + ["sqlite3_strglob", "int", "string","string"], + ["sqlite3_strlike", "int", "string","string","int"], + ["sqlite3_total_changes", "int", "sqlite3*"], + ["sqlite3_value_blob", "*", "*"], + ["sqlite3_value_bytes","int", "*"], + ["sqlite3_value_double","f64", "*"], + ["sqlite3_value_text", "string", "*"], + ["sqlite3_value_type", "int", "*"], + ["sqlite3_vfs_find", "*", "string"], + ["sqlite3_vfs_register", "int", "*", "int"] + ]/*capi.wasm.bindingSignatures*/; + + if(false && capi.wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ + /* ^^^ "the problem" is that this is an option feature and the + build-time function-export list does not currently take + optional features into account. */ + capi.wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); + } + + /** + Functions which require BigInt (int64) support are separated from + the others because we need to conditionally bind them or apply + dummy impls, depending on the capabilities of the environment. + */ + capi.wasm.bindingSignatures.int64 = [ + ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], + ["sqlite3_changes64","i64", ["sqlite3*"]], + ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]], + ["sqlite3_total_changes64", "i64", ["sqlite3*"]] + ]; + + /* The remainder of the API will be set up in later steps. */ + return { + capi, + postInit: [ + /* some pieces of the API may install functions into this array, + and each such function will be called, passed (self,sqlite3), + at the very end of the API load/init process, where self is + the current global object and sqlite3 is the object returned + from sqlite3ApiBootstrap(). This array will be removed at the + end of the API setup process. */], + /** Config is needed downstream for gluing pieces together. It + will be removed at the end of the API setup process. */ + config + }; +}/*sqlite3ApiBootstrap()*/; diff --git a/ext/wasm/api/sqlite3-api-worker.js b/ext/wasm/api/sqlite3-api-worker.js new file mode 100644 index 0000000000..1d13d4ed6b --- /dev/null +++ b/ext/wasm/api/sqlite3-api-worker.js @@ -0,0 +1,421 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements a Worker-based wrapper around SQLite3 OO API + #1. + + In order to permit this API to be loaded in worker threads without + automatically registering onmessage handlers, initializing the + worker API requires calling initWorkerAPI(). If this function + is called from a non-worker thread then it throws an exception. + + When initialized, it installs message listeners to receive messages + from the main thread and then it posts a message in the form: + + ``` + {type:'sqlite3-api',data:'worker-ready'} + ``` + + This file requires that the core C-style sqlite3 API and OO API #1 + have been loaded and that self.sqlite3 contains both, + as documented for those APIs. +*/ +self.sqlite3.initWorkerAPI = function(){ + 'use strict'; + /** + UNDER CONSTRUCTION + + We need an API which can proxy the DB API via a Worker message + interface. The primary quirky factor in such an API is that we + cannot pass callback functions between the window thread and a + worker thread, so we have to receive all db results via + asynchronous message-passing. That requires an asychronous API + with a distinctly different shape that the main OO API. + + Certain important considerations here include: + + - Support only one db connection or multiple? The former is far + easier, but there's always going to be a user out there who wants + to juggle six database handles at once. Do we add that complexity + or tell such users to write their own code using the provided + lower-level APIs? + + - Fetching multiple results: do we pass them on as a series of + messages, with start/end messages on either end, or do we collect + all results and bundle them back in a single message? The former + is, generically speaking, more memory-efficient but the latter + far easier to implement in this environment. The latter is + untennable for large data sets. Despite a web page hypothetically + being a relatively limited environment, there will always be + those users who feel that they should/need to be able to work + with multi-hundred-meg (or larger) blobs, and passing around + arrays of those may quickly exhaust the JS engine's memory. + + TODOs include, but are not limited to: + + - The ability to manage multiple DB handles. This can + potentially be done via a simple mapping of DB.filename or + DB.pointer (`sqlite3*` handle) to DB objects. The open() + interface would need to provide an ID (probably DB.pointer) back + to the user which can optionally be passed as an argument to + the other APIs (they'd default to the first-opened DB, for + ease of use). Client-side usability of this feature would + benefit from making another wrapper class (or a singleton) + available to the main thread, with that object proxying all(?) + communication with the worker. + + - Revisit how virtual files are managed. We currently delete DBs + from the virtual filesystem when we close them, for the sake of + saving memory (the VFS lives in RAM). Supporting multiple DBs may + require that we give up that habit. Similarly, fully supporting + ATTACH, where a user can upload multiple DBs and ATTACH them, + also requires the that we manage the VFS entries better. + */ + const toss = (...args)=>{throw new Error(args.join(' '))}; + if('function' !== typeof importScripts){ + toss("Cannot initalize the sqlite3 worker API in the main thread."); + } + /* This is a web worker, so init the worker-based API. */ + const self = this.self; + const sqlite3 = this.sqlite3 || toss("Missing self.sqlite3 object."); + const SQLite3 = sqlite3.oo1 || toss("Missing self.sqlite3.oo1 OO API."); + const DB = SQLite3.DB; + + /** + Returns the app-wide unique ID for the given db, creating one if + needed. + */ + const getDbId = function(db){ + let id = wState.idMap.get(db); + if(id) return id; + id = 'db#'+(++wState.idSeq)+'@'+db.pointer; + /** ^^^ can't simply use db.pointer b/c closing/opening may re-use + the same address, which could map pending messages to a wrong + instance. */ + wState.idMap.set(db, id); + return id; + }; + + /** + Helper for managing Worker-level state. + */ + const wState = { + defaultDb: undefined, + idSeq: 0, + idMap: new WeakMap, + open: function(arg){ + // TODO: if arg is a filename, look for a db in this.dbs with the + // same filename and close/reopen it (or just pass it back as is?). + if(!arg && this.defaultDb) return this.defaultDb; + //???if(this.defaultDb) this.defaultDb.close(); + let db; + db = (Array.isArray(arg) ? new DB(...arg) : new DB(arg)); + this.dbs[getDbId(db)] = db; + if(!this.defaultDb) this.defaultDb = db; + return db; + }, + close: function(db,alsoUnlink){ + if(db){ + delete this.dbs[getDbId(db)]; + db.close(alsoUnlink); + if(db===this.defaultDb) this.defaultDb = undefined; + } + }, + post: function(type,data,xferList){ + if(xferList){ + self.postMessage({type, data},xferList); + xferList.length = 0; + }else{ + self.postMessage({type, data}); + } + }, + /** Map of DB IDs to DBs. */ + dbs: Object.create(null), + getDb: function(id,require=true){ + return this.dbs[id] + || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); + } + }; + + /** Throws if the given db is falsy or not opened. */ + const affirmDbOpen = function(db = wState.defaultDb){ + return (db && db.pointer) ? db : toss("DB is not opened."); + }; + + /** Extract dbId from the given message payload. */ + const getMsgDb = function(msgData,affirmExists=true){ + const db = wState.getDb(msgData.dbId,false) || wState.defaultDb; + return affirmExists ? affirmDbOpen(db) : db; + }; + + const getDefaultDbId = function(){ + return wState.defaultDb && getDbId(wState.defaultDb); + }; + + /** + A level of "organizational abstraction" for the Worker + API. Each method in this object must map directly to a Worker + message type key. The onmessage() dispatcher attempts to + dispatch all inbound messages to a method of this object, + passing it the event.data part of the inbound event object. All + methods must return a plain Object containing any response + state, which the dispatcher may amend. All methods must throw + on error. + */ + const wMsgHandler = { + xfer: [/*Temp holder for "transferable" postMessage() state.*/], + /** + Proxy for DB.exec() which expects a single argument of type + string (SQL to execute) or an options object in the form + expected by exec(). The notable differences from exec() + include: + + - The default value for options.rowMode is 'array' because + the normal default cannot cross the window/Worker boundary. + + - A function-type options.callback property cannot cross + the window/Worker boundary, so is not useful here. If + options.callback is a string then it is assumed to be a + message type key, in which case a callback function will be + applied which posts each row result via: + + postMessage({type: thatKeyType, data: theRow}) + + And, at the end of the result set (whether or not any + result rows were produced), it will post an identical + message with data:null to alert the caller than the result + set is completed. + + The callback proxy must not recurse into this interface, or + results are undefined. (It hypothetically cannot recurse + because an exec() call will be tying up the Worker thread, + causing any recursion attempt to wait until the first + exec() is completed.) + + The response is the input options object (or a synthesized + one if passed only a string), noting that + options.resultRows and options.columnNames may be populated + by the call to exec(). + + This opens/creates the Worker's db if needed. + */ + exec: function(ev){ + const opt = ( + 'string'===typeof ev.data + ) ? {sql: ev.data} : (ev.data || Object.create(null)); + if(undefined===opt.rowMode){ + /* Since the default rowMode of 'stmt' is not useful + for the Worker interface, we'll default to + something else. */ + opt.rowMode = 'array'; + }else if('stmt'===opt.rowMode){ + toss("Invalid rowMode for exec(): stmt mode", + "does not work in the Worker API."); + } + const db = getMsgDb(ev); + if(opt.callback || Array.isArray(opt.resultRows)){ + // Part of a copy-avoidance optimization for blobs + db._blobXfer = this.xfer; + } + const callbackMsgType = opt.callback; + if('string' === typeof callbackMsgType){ + /* Treat this as a worker message type and post each + row as a message of that type. */ + const that = this; + opt.callback = + (row)=>wState.post(callbackMsgType,row,this.xfer); + } + try { + db.exec(opt); + if(opt.callback instanceof Function){ + opt.callback = callbackMsgType; + wState.post(callbackMsgType, null); + } + }/*catch(e){ + console.warn("Worker is propagating:",e);throw e; + }*/finally{ + delete db._blobXfer; + if(opt.callback){ + opt.callback = callbackMsgType; + } + } + return opt; + }/*exec()*/, + /** + TO(re)DO, once we can abstract away access to the + JS environment's virtual filesystem. Currently this + always throws. + + Response is (should be) an object: + + { + buffer: Uint8Array (db file contents), + filename: the current db filename, + mimetype: 'application/x-sqlite3' + } + + TODO is to determine how/whether this feature can support + exports of ":memory:" and "" (temp file) DBs. The latter is + ostensibly easy because the file is (potentially) on disk, but + the former does not have a structure which maps directly to a + db file image. + */ + export: function(ev){ + toss("export() requires reimplementing for portability reasons."); + /**const db = getMsgDb(ev); + const response = { + buffer: db.exportBinaryImage(), + filename: db.filename, + mimetype: 'application/x-sqlite3' + }; + this.xfer.push(response.buffer.buffer); + return response;**/ + }/*export()*/, + /** + Proxy for the DB constructor. Expects to be passed a single + object or a falsy value to use defaults. The object may + have a filename property to name the db file (see the DB + constructor for peculiarities and transformations) and/or a + buffer property (a Uint8Array holding a complete database + file's contents). The response is an object: + + { + filename: db filename (possibly differing from the input), + + id: an opaque ID value intended for future distinction + between multiple db handles. Messages including a specific + ID will use the DB for that ID. + + } + + If the Worker's db is currently opened, this call closes it + before proceeding. + */ + open: function(ev){ + wState.close(/*true???*/); + const args = [], data = (ev.data || {}); + if(data.simulateError){ + toss("Throwing because of open.simulateError flag."); + } + if(data.filename) args.push(data.filename); + if(data.buffer){ + args.push(data.buffer); + this.xfer.push(data.buffer.buffer); + } + const db = wState.open(args); + return { + filename: db.filename, + dbId: getDbId(db) + }; + }, + /** + Proxy for DB.close(). If ev.data may either be a boolean or + an object with an `unlink` property. If that value is + truthy then the db file (if the db is currently open) will + be unlinked from the virtual filesystem, else it will be + kept intact. The response object is: + + { + filename: db filename _if_ the db is opened when this + is called, else the undefined value + } + */ + close: function(ev){ + const db = getMsgDb(ev,false); + const response = { + filename: db && db.filename + }; + if(db){ + wState.close(db, !!((ev.data && 'object'===typeof ev.data) + ? ev.data.unlink : ev.data)); + } + return response; + }, + toss: function(ev){ + toss("Testing worker exception"); + } + }/*wMsgHandler*/; + + /** + UNDER CONSTRUCTION! + + A subset of the DB API is accessible via Worker messages in the + form: + + { type: apiCommand, + dbId: optional DB ID value (not currently used!) + data: apiArguments + } + + As a rule, these commands respond with a postMessage() of their + own in the same form, but will, if needed, transform the `data` + member to an object and may add state to it. The responses + always have an object-format `data` part. If the inbound `data` + is an object which has a `messageId` property, that property is + always mirrored in the result object, for use in client-side + dispatching of these asynchronous results. Exceptions thrown + during processing result in an `error`-type event with a + payload in the form: + + { + message: error string, + errorClass: class name of the error type, + dbId: DB handle ID, + input: ev.data, + [messageId: if set in the inbound message] + } + + The individual APIs are documented in the wMsgHandler object. + */ + self.onmessage = function(ev){ + ev = ev.data; + let response, dbId = ev.dbId, evType = ev.type; + const arrivalTime = performance.now(); + try { + if(wMsgHandler.hasOwnProperty(evType) && + wMsgHandler[evType] instanceof Function){ + response = wMsgHandler[evType](ev); + }else{ + toss("Unknown db worker message type:",ev.type); + } + }catch(err){ + evType = 'error'; + response = { + message: err.message, + errorClass: err.name, + input: ev + }; + if(err.stack){ + response.stack = ('string'===typeof err.stack) + ? err.stack.split('\n') : err.stack; + } + if(0) console.warn("Worker is propagating an exception to main thread.", + "Reporting it _here_ for the stack trace:",err,response); + } + if(!response.messageId && ev.data + && 'object'===typeof ev.data && ev.data.messageId){ + response.messageId = ev.data.messageId; + } + if(!dbId){ + dbId = response.dbId/*from 'open' cmd*/ + || getDefaultDbId(); + } + if(!response.dbId) response.dbId = dbId; + // Timing info is primarily for use in testing this API. It's not part of + // the public API. arrivalTime = when the worker got the message. + response.workerReceivedTime = arrivalTime; + response.workerRespondTime = performance.now(); + response.departureTime = ev.departureTime; + wState.post(evType, response, wMsgHandler.xfer); + }; + setTimeout(()=>self.postMessage({type:'sqlite3-api',data:'worker-ready'}), 0); +}.bind({self, sqlite3: self.sqlite3}); diff --git a/ext/wasm/api/sqlite3-wasi.h b/ext/wasm/api/sqlite3-wasi.h new file mode 100644 index 0000000000..096f45dfec --- /dev/null +++ b/ext/wasm/api/sqlite3-wasi.h @@ -0,0 +1,69 @@ +/** + Dummy function stubs to get sqlite3.c compiling with + wasi-sdk. This requires, in addition: + + -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_GETPID + + -lwasi-emulated-getpid +*/ +typedef unsigned mode_t; +int fchmod(int fd, mode_t mode); +int fchmod(int fd, mode_t mode){ + return (fd && mode) ? 0 : 0; +} +typedef unsigned uid_t; +typedef uid_t gid_t; +int fchown(int fd, uid_t owner, gid_t group); +int fchown(int fd, uid_t owner, gid_t group){ + return (fd && owner && group) ? 0 : 0; +} +uid_t geteuid(void); +uid_t geteuid(void){return 0;} +#if !defined(F_WRLCK) +enum { +F_WRLCK, +F_RDLCK, +F_GETLK, +F_SETLK, +F_UNLCK +}; +#endif + +#undef HAVE_PREAD + +#include +#define WASM__KEEP __attribute__((used)) + +#if 0 +/** + wasi-sdk cannot build sqlite3's default VFS without at least the following + functions. They are apparently syscalls which clients have to implement or + otherwise obtain. + + https://github.com/WebAssembly/WASI/blob/main/phases/snapshot/docs.md +*/ +environ_get +environ_sizes_get +clock_time_get +fd_close +fd_fdstat_get +fd_fdstat_set_flags +fd_filestat_get +fd_filestat_set_size +fd_pread +fd_prestat_get +fd_prestat_dir_name +fd_read +fd_seek +fd_sync +fd_write +path_create_directory +path_filestat_get +path_filestat_set_times +path_open +path_readlink +path_remove_directory +path_unlink_file +poll_oneoff +proc_exit +#endif diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c new file mode 100644 index 0000000000..c27dad56ff --- /dev/null +++ b/ext/wasm/api/sqlite3-wasm.c @@ -0,0 +1,413 @@ +#include "sqlite3.c" + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** For purposes of certain hand-crafted C/Wasm function bindings, we +** need a way of reporting errors which is consistent with the rest of +** the C API. To that end, this internal-use-only function is a thin +** proxy around sqlite3ErrorWithMessage(). The intent is that it only +** be used from Wasm bindings such as sqlite3_prepare_v2/v3(), and +** definitely not from client code. +** +** Returns err_code. +*/ +int sqlite3_wasm_db_error(sqlite3*db, int err_code, + const char *zMsg){ + if(0!=zMsg){ + const int nMsg = sqlite3Strlen30(zMsg); + sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); + }else{ + sqlite3ErrorWithMsg(db, err_code, NULL); + } + return err_code; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. Unlike the +** rest of the sqlite3 API, this part requires C99 for snprintf() and +** variadic macros. +** +** Returns a string containing a JSON-format "enum" of C-level +** constants intended to be imported into the JS environment. The JSON +** is initialized the first time this function is called and that +** result is reused for all future calls. +** +** If this function returns NULL then it means that the internal +** buffer is not large enough for the generated JSON. In debug builds +** that will trigger an assert(). +*/ +const char * sqlite3_wasm_enum_json(void){ + static char strBuf[1024 * 8] = {0} /* where the JSON goes */; + int n = 0, childCount = 0, structCount = 0 + /* output counters for figuring out where commas go */; + char * pos = &strBuf[1] /* skip first byte for now to help protect + ** against a small race condition */; + char const * const zEnd = pos + sizeof(strBuf) /* one-past-the-end */; + if(strBuf[0]) return strBuf; + /* Leave strBuf[0] at 0 until the end to help guard against a tiny + ** race condition. If this is called twice concurrently, they might + ** end up both writing to strBuf, but they'll both write the same + ** thing, so that's okay. If we set byte 0 up front then the 2nd + ** instance might return and use the string before the 1st instance + ** is done filling it. */ + +/* Core output macros... */ +#define lenCheck assert(pos < zEnd - 128 \ + && "sqlite3_wasm_enum_json() buffer is too small."); \ + if(pos >= zEnd - 128) return 0 +#define outf(format,...) \ + pos += snprintf(pos, ((size_t)(zEnd - pos)), format, __VA_ARGS__); \ + lenCheck +#define out(TXT) outf("%s",TXT) +#define CloseBrace(LEVEL) \ + assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck + +/* Macros for emitting maps of integer- and string-type macros to +** their values. */ +#define DefGroup(KEY) n = 0; \ + outf("%s\"" #KEY "\": {",(childCount++ ? "," : "")); +#define DefInt(KEY) \ + outf("%s\"%s\": %d", (n++ ? ", " : ""), #KEY, (int)KEY) +#define DefStr(KEY) \ + outf("%s\"%s\": \"%s\"", (n++ ? ", " : ""), #KEY, KEY) +#define _DefGroup CloseBrace(1) + + DefGroup(version) { + DefInt(SQLITE_VERSION_NUMBER); + DefStr(SQLITE_VERSION); + DefStr(SQLITE_SOURCE_ID); + } _DefGroup; + + DefGroup(resultCodes) { + DefInt(SQLITE_OK); + DefInt(SQLITE_ERROR); + DefInt(SQLITE_INTERNAL); + DefInt(SQLITE_PERM); + DefInt(SQLITE_ABORT); + DefInt(SQLITE_BUSY); + DefInt(SQLITE_LOCKED); + DefInt(SQLITE_NOMEM); + DefInt(SQLITE_READONLY); + DefInt(SQLITE_INTERRUPT); + DefInt(SQLITE_IOERR); + DefInt(SQLITE_CORRUPT); + DefInt(SQLITE_NOTFOUND); + DefInt(SQLITE_FULL); + DefInt(SQLITE_CANTOPEN); + DefInt(SQLITE_PROTOCOL); + DefInt(SQLITE_EMPTY); + DefInt(SQLITE_SCHEMA); + DefInt(SQLITE_TOOBIG); + DefInt(SQLITE_CONSTRAINT); + DefInt(SQLITE_MISMATCH); + DefInt(SQLITE_MISUSE); + DefInt(SQLITE_NOLFS); + DefInt(SQLITE_AUTH); + DefInt(SQLITE_FORMAT); + DefInt(SQLITE_RANGE); + DefInt(SQLITE_NOTADB); + DefInt(SQLITE_NOTICE); + DefInt(SQLITE_WARNING); + DefInt(SQLITE_ROW); + DefInt(SQLITE_DONE); + + // Extended Result Codes + DefInt(SQLITE_ERROR_MISSING_COLLSEQ); + DefInt(SQLITE_ERROR_RETRY); + DefInt(SQLITE_ERROR_SNAPSHOT); + DefInt(SQLITE_IOERR_READ); + DefInt(SQLITE_IOERR_SHORT_READ); + DefInt(SQLITE_IOERR_WRITE); + DefInt(SQLITE_IOERR_FSYNC); + DefInt(SQLITE_IOERR_DIR_FSYNC); + DefInt(SQLITE_IOERR_TRUNCATE); + DefInt(SQLITE_IOERR_FSTAT); + DefInt(SQLITE_IOERR_UNLOCK); + DefInt(SQLITE_IOERR_RDLOCK); + DefInt(SQLITE_IOERR_DELETE); + DefInt(SQLITE_IOERR_BLOCKED); + DefInt(SQLITE_IOERR_NOMEM); + DefInt(SQLITE_IOERR_ACCESS); + DefInt(SQLITE_IOERR_CHECKRESERVEDLOCK); + DefInt(SQLITE_IOERR_LOCK); + DefInt(SQLITE_IOERR_CLOSE); + DefInt(SQLITE_IOERR_DIR_CLOSE); + DefInt(SQLITE_IOERR_SHMOPEN); + DefInt(SQLITE_IOERR_SHMSIZE); + DefInt(SQLITE_IOERR_SHMLOCK); + DefInt(SQLITE_IOERR_SHMMAP); + DefInt(SQLITE_IOERR_SEEK); + DefInt(SQLITE_IOERR_DELETE_NOENT); + DefInt(SQLITE_IOERR_MMAP); + DefInt(SQLITE_IOERR_GETTEMPPATH); + DefInt(SQLITE_IOERR_CONVPATH); + DefInt(SQLITE_IOERR_VNODE); + DefInt(SQLITE_IOERR_AUTH); + DefInt(SQLITE_IOERR_BEGIN_ATOMIC); + DefInt(SQLITE_IOERR_COMMIT_ATOMIC); + DefInt(SQLITE_IOERR_ROLLBACK_ATOMIC); + DefInt(SQLITE_IOERR_DATA); + DefInt(SQLITE_IOERR_CORRUPTFS); + DefInt(SQLITE_LOCKED_SHAREDCACHE); + DefInt(SQLITE_LOCKED_VTAB); + DefInt(SQLITE_BUSY_RECOVERY); + DefInt(SQLITE_BUSY_SNAPSHOT); + DefInt(SQLITE_BUSY_TIMEOUT); + DefInt(SQLITE_CANTOPEN_NOTEMPDIR); + DefInt(SQLITE_CANTOPEN_ISDIR); + DefInt(SQLITE_CANTOPEN_FULLPATH); + DefInt(SQLITE_CANTOPEN_CONVPATH); + //DefInt(SQLITE_CANTOPEN_DIRTYWAL)/*docs say not used*/; + DefInt(SQLITE_CANTOPEN_SYMLINK); + DefInt(SQLITE_CORRUPT_VTAB); + DefInt(SQLITE_CORRUPT_SEQUENCE); + DefInt(SQLITE_CORRUPT_INDEX); + DefInt(SQLITE_READONLY_RECOVERY); + DefInt(SQLITE_READONLY_CANTLOCK); + DefInt(SQLITE_READONLY_ROLLBACK); + DefInt(SQLITE_READONLY_DBMOVED); + DefInt(SQLITE_READONLY_CANTINIT); + DefInt(SQLITE_READONLY_DIRECTORY); + DefInt(SQLITE_ABORT_ROLLBACK); + DefInt(SQLITE_CONSTRAINT_CHECK); + DefInt(SQLITE_CONSTRAINT_COMMITHOOK); + DefInt(SQLITE_CONSTRAINT_FOREIGNKEY); + DefInt(SQLITE_CONSTRAINT_FUNCTION); + DefInt(SQLITE_CONSTRAINT_NOTNULL); + DefInt(SQLITE_CONSTRAINT_PRIMARYKEY); + DefInt(SQLITE_CONSTRAINT_TRIGGER); + DefInt(SQLITE_CONSTRAINT_UNIQUE); + DefInt(SQLITE_CONSTRAINT_VTAB); + DefInt(SQLITE_CONSTRAINT_ROWID); + DefInt(SQLITE_CONSTRAINT_PINNED); + DefInt(SQLITE_CONSTRAINT_DATATYPE); + DefInt(SQLITE_NOTICE_RECOVER_WAL); + DefInt(SQLITE_NOTICE_RECOVER_ROLLBACK); + DefInt(SQLITE_WARNING_AUTOINDEX); + DefInt(SQLITE_AUTH_USER); + DefInt(SQLITE_OK_LOAD_PERMANENTLY); + //DefInt(SQLITE_OK_SYMLINK) /* internal use only */; + } _DefGroup; + + DefGroup(dataTypes) { + DefInt(SQLITE_INTEGER); + DefInt(SQLITE_FLOAT); + DefInt(SQLITE_TEXT); + DefInt(SQLITE_BLOB); + DefInt(SQLITE_NULL); + } _DefGroup; + + DefGroup(encodings) { + /* Noting that the wasm binding only aims to support UTF-8. */ + DefInt(SQLITE_UTF8); + DefInt(SQLITE_UTF16LE); + DefInt(SQLITE_UTF16BE); + DefInt(SQLITE_UTF16); + /*deprecated DefInt(SQLITE_ANY); */ + DefInt(SQLITE_UTF16_ALIGNED); + } _DefGroup; + + DefGroup(blobFinalizers) { + /* SQLITE_STATIC/TRANSIENT need to be handled explicitly as + ** integers to avoid casting-related warnings. */ + out("\"SQLITE_STATIC\":0, " + "\"SQLITE_TRANSIENT\":-1"); + } _DefGroup; + + DefGroup(udfFlags) { + DefInt(SQLITE_DETERMINISTIC); + DefInt(SQLITE_DIRECTONLY); + DefInt(SQLITE_INNOCUOUS); + } _DefGroup; + + DefGroup(openFlags) { + /* Noting that not all of these will have any effect in WASM-space. */ + DefInt(SQLITE_OPEN_READONLY); + DefInt(SQLITE_OPEN_READWRITE); + DefInt(SQLITE_OPEN_CREATE); + DefInt(SQLITE_OPEN_URI); + DefInt(SQLITE_OPEN_MEMORY); + DefInt(SQLITE_OPEN_NOMUTEX); + DefInt(SQLITE_OPEN_FULLMUTEX); + DefInt(SQLITE_OPEN_SHAREDCACHE); + DefInt(SQLITE_OPEN_PRIVATECACHE); + DefInt(SQLITE_OPEN_EXRESCODE); + DefInt(SQLITE_OPEN_NOFOLLOW); + /* OPEN flags for use with VFSes... */ + DefInt(SQLITE_OPEN_MAIN_DB); + DefInt(SQLITE_OPEN_MAIN_JOURNAL); + DefInt(SQLITE_OPEN_TEMP_DB); + DefInt(SQLITE_OPEN_TEMP_JOURNAL); + DefInt(SQLITE_OPEN_TRANSIENT_DB); + DefInt(SQLITE_OPEN_SUBJOURNAL); + DefInt(SQLITE_OPEN_SUPER_JOURNAL); + DefInt(SQLITE_OPEN_WAL); + DefInt(SQLITE_OPEN_DELETEONCLOSE); + DefInt(SQLITE_OPEN_EXCLUSIVE); + } _DefGroup; + + DefGroup(syncFlags) { + DefInt(SQLITE_SYNC_NORMAL); + DefInt(SQLITE_SYNC_FULL); + DefInt(SQLITE_SYNC_DATAONLY); + } _DefGroup; + + DefGroup(prepareFlags) { + DefInt(SQLITE_PREPARE_PERSISTENT); + DefInt(SQLITE_PREPARE_NORMALIZE); + DefInt(SQLITE_PREPARE_NO_VTAB); + } _DefGroup; + + DefGroup(flock) { + DefInt(SQLITE_LOCK_NONE); + DefInt(SQLITE_LOCK_SHARED); + DefInt(SQLITE_LOCK_RESERVED); + DefInt(SQLITE_LOCK_PENDING); + DefInt(SQLITE_LOCK_EXCLUSIVE); + } _DefGroup; + + DefGroup(ioCap) { + DefInt(SQLITE_IOCAP_ATOMIC); + DefInt(SQLITE_IOCAP_ATOMIC512); + DefInt(SQLITE_IOCAP_ATOMIC1K); + DefInt(SQLITE_IOCAP_ATOMIC2K); + DefInt(SQLITE_IOCAP_ATOMIC4K); + DefInt(SQLITE_IOCAP_ATOMIC8K); + DefInt(SQLITE_IOCAP_ATOMIC16K); + DefInt(SQLITE_IOCAP_ATOMIC32K); + DefInt(SQLITE_IOCAP_ATOMIC64K); + DefInt(SQLITE_IOCAP_SAFE_APPEND); + DefInt(SQLITE_IOCAP_SEQUENTIAL); + DefInt(SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN); + DefInt(SQLITE_IOCAP_POWERSAFE_OVERWRITE); + DefInt(SQLITE_IOCAP_IMMUTABLE); + DefInt(SQLITE_IOCAP_BATCH_ATOMIC); + } _DefGroup; + + DefGroup(access){ + DefInt(SQLITE_ACCESS_EXISTS); + DefInt(SQLITE_ACCESS_READWRITE); + DefInt(SQLITE_ACCESS_READ)/*docs say this is unused*/; + } _DefGroup; + +#undef DefGroup +#undef DefStr +#undef DefInt +#undef _DefGroup + + /* + ** Emit an array of "StructBinder" struct descripions, which look + ** like: + ** + ** { + ** "name": "MyStruct", + ** "sizeof": 16, + ** "members": { + ** "member1": {"offset": 0,"sizeof": 4,"signature": "i"}, + ** "member2": {"offset": 4,"sizeof": 4,"signature": "p"}, + ** "member3": {"offset": 8,"sizeof": 8,"signature": "j"} + ** } + ** } + ** + ** Detailed documentation for those bits are in an external + ** file (StackBinder.md, as of this writing). + */ + + /** Macros for emitting StructBinder description. */ +#define StructBinder__(TYPE) \ + n = 0; \ + outf("%s{", (structCount++ ? ", " : "")); \ + out("\"name\": \"" # TYPE "\","); \ + outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ + out(",\"members\": {"); +#define StructBinder_(T) StructBinder__(T) + /** ^^^ indirection needed to expand CurrentStruct */ +#define StructBinder StructBinder_(CurrentStruct) +#define _StructBinder CloseBrace(2) +#define M(MEMBER,SIG) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG) + + structCount = 0; + out(", \"structs\": ["); { + +#define CurrentStruct sqlite3_vfs + StructBinder { + M(iVersion,"i"); + M(szOsFile,"i"); + M(mxPathname,"i"); + M(pNext,"p"); + M(zName,"s"); + M(pAppData,"p"); + M(xOpen,"i(pppip)"); + M(xDelete,"i(ppi)"); + M(xAccess,"i(ppip)"); + M(xFullPathname,"i(ppip)"); + M(xDlOpen,"p(pp)"); + M(xDlError,"p(pip)"); + M(xDlSym,"p()"); + M(xDlClose,"v(pp)"); + M(xRandomness,"i(pip)"); + M(xSleep,"i(pi)"); + M(xCurrentTime,"i(pp)"); + M(xGetLastError,"i(pip)"); + M(xCurrentTimeInt64,"i(pp)"); + M(xSetSystemCall,"i(ppp)"); + M(xGetSystemCall,"p(pp)"); + M(xNextSystemCall,"p(pp)"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_io_methods + StructBinder { + M(iVersion,"i"); + M(xClose,"i(p)"); + M(xRead,"i(ppij)"); + M(xWrite,"i(ppij)"); + M(xTruncate,"i(pj)"); + M(xSync,"i(pi)"); + M(xFileSize,"i(pp)"); + M(xLock,"i(pi)"); + M(xUnlock,"i(pi)"); + M(xCheckReservedLock,"i(pp)"); + M(xFileControl,"i(pip)"); + M(xSectorSize,"i(p)"); + M(xDeviceCharacteristics,"i(p)"); + M(xShmMap,"i(piiip)"); + M(xShmLock,"i(piii)"); + M(xShmBarrier,"v(p)"); + M(xShmUnmap,"i(pi)"); + M(xFetch,"i(pjip)"); + M(xUnfetch,"i(pjp)"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_file + StructBinder { + M(pMethods,"P"); + } _StructBinder; +#undef CurrentStruct + + } out( "]"/*structs*/); + + out("}"/*top-level object*/); + *pos = 0; + strBuf[0] = '{'/*end of the race-condition workaround*/; + return strBuf; +#undef StructBinder +#undef StructBinder_ +#undef StructBinder__ +#undef M +#undef _StructBinder +#undef CloseBrace +#undef out +#undef outf +#undef lenCheck +} diff --git a/ext/wasm/api/sqlite3-worker.js b/ext/wasm/api/sqlite3-worker.js new file mode 100644 index 0000000000..48797de8ab --- /dev/null +++ b/ext/wasm/api/sqlite3-worker.js @@ -0,0 +1,31 @@ +/* + 2022-05-23 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This is a JS Worker file for the main sqlite3 api. It loads + sqlite3.js, initializes the module, and postMessage()'s a message + after the module is initialized: + + {type: 'sqlite3-api', data: 'worker-ready'} + + This seemingly superfluous level of indirection is necessary when + loading sqlite3.js via a Worker. Instantiating a worker with new + Worker("sqlite.js") will not (cannot) call sqlite3InitModule() to + initialize the module due to a timing/order-of-operations conflict + (and that symbol is not exported in a way that a Worker loading it + that way can see it). Thus JS code wanting to load the sqlite3 + Worker-specific API needs to pass _this_ file (or equivalent) to the + Worker constructor and then listen for an event in the form shown + above in order to know when the module has completed initialization. +*/ +"use strict"; +importScripts('sqlite3.js'); +sqlite3InitModule().then((EmscriptenModule)=>EmscriptenModule.sqlite3.initWorkerAPI()); diff --git a/ext/wasm/common/SqliteTestUtil.js b/ext/wasm/common/SqliteTestUtil.js new file mode 100644 index 0000000000..c7c99240e6 --- /dev/null +++ b/ext/wasm/common/SqliteTestUtil.js @@ -0,0 +1,173 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file contains bootstrapping code used by various test scripts + which live in this file's directory. +*/ +'use strict'; +(function(self){ + /* querySelectorAll() proxy */ + const EAll = function(/*[element=document,] cssSelector*/){ + return (arguments.length>1 ? arguments[0] : document) + .querySelectorAll(arguments[arguments.length-1]); + }; + /* querySelector() proxy */ + const E = function(/*[element=document,] cssSelector*/){ + return (arguments.length>1 ? arguments[0] : document) + .querySelector(arguments[arguments.length-1]); + }; + + /** + Helpers for writing sqlite3-specific tests. + */ + self.SqliteTestUtil = { + /** Running total of the number of tests run via + this API. */ + counter: 0, + /** + If expr is a function, it is called and its result + is returned, coerced to a bool, else expr, coerced to + a bool, is returned. + */ + toBool: function(expr){ + return (expr instanceof Function) ? !!expr() : !!expr; + }, + /** abort() if expr is false. If expr is a function, it + is called and its result is evaluated. + */ + assert: function f(expr, msg){ + if(!f._){ + f._ = ('undefined'===typeof abort + ? (msg)=>{throw new Error(msg)} + : abort); + } + ++this.counter; + if(!this.toBool(expr)){ + f._(msg || "Assertion failed."); + } + return this; + }, + /** Identical to assert() but throws instead of calling + abort(). */ + affirm: function(expr, msg){ + ++this.counter; + if(!this.toBool(expr)) throw new Error(msg || "Affirmation failed."); + return this; + }, + /** Calls f() and squelches any exception it throws. If it + does not throw, this function throws. */ + mustThrow: function(f, msg){ + ++this.counter; + let err; + try{ f(); } catch(e){err=e;} + if(!err) throw new Error(msg || "Expected exception."); + return this; + }, + /** + Works like mustThrow() but expects filter to be a regex, + function, or string to match/filter the resulting exception + against. If f() does not throw, this test fails and an Error is + thrown. If filter is a regex, the test passes if + filter.test(error.message) passes. If it's a function, the test + passes if filter(error) returns truthy. If it's a string, the + test passes if the filter matches the exception message + precisely. In all other cases the test fails, throwing an + Error. + + If it throws, msg is used as the error report unless it's falsy, + in which case a default is used. + */ + mustThrowMatching: function(f, filter, msg){ + ++this.counter; + let err; + try{ f(); } catch(e){err=e;} + if(!err) throw new Error(msg || "Expected exception."); + let pass = false; + if(filter instanceof RegExp) pass = filter.test(err.message); + else if(filter instanceof Function) pass = filter(err); + else if('string' === typeof filter) pass = (err.message === filter); + if(!pass){ + throw new Error(msg || ("Filter rejected this exception: "+err.message)); + } + return this; + }, + /** Throws if expr is truthy or expr is a function and expr() + returns truthy. */ + throwIf: function(expr, msg){ + ++this.counter; + if(this.toBool(expr)) throw new Error(msg || "throwIf() failed"); + return this; + }, + /** Throws if expr is falsy or expr is a function and expr() + returns falsy. */ + throwUnless: function(expr, msg){ + ++this.counter; + if(!this.toBool(expr)) throw new Error(msg || "throwUnless() failed"); + return this; + } + }; + + + /** + This is a module object for use with the emscripten-installed + sqlite3InitModule() factory function. + */ + self.sqlite3TestModule = { + postRun: [ + /* function(theModule){...} */ + ], + //onRuntimeInitialized: function(){}, + /* Proxy for C-side stdout output. */ + print: function(){ + console.log.apply(console, Array.prototype.slice.call(arguments)); + }, + /* Proxy for C-side stderr output. */ + printErr: function(){ + console.error.apply(console, Array.prototype.slice.call(arguments)); + }, + /** + Called by the module init bits to report loading + progress. It gets passed an empty argument when loading is + done (after onRuntimeInitialized() and any this.postRun + callbacks have been run). + */ + setStatus: function f(text){ + if(!f.last){ + f.last = { text: '', step: 0 }; + f.ui = { + status: E('#module-status'), + progress: E('#module-progress'), + spinner: E('#module-spinner') + }; + } + if(text === f.last.text) return; + f.last.text = text; + if(f.ui.progress){ + f.ui.progress.value = f.last.step; + f.ui.progress.max = f.last.step + 1; + } + ++f.last.step; + if(text) { + f.ui.status.classList.remove('hidden'); + f.ui.status.innerText = text; + }else{ + if(f.ui.progress){ + f.ui.progress.remove(); + f.ui.spinner.remove(); + delete f.ui.progress; + delete f.ui.spinner; + } + f.ui.status.classList.add('hidden'); + } + } + }; +})(self/*window or worker*/); diff --git a/ext/wasm/common/emscripten.css b/ext/wasm/common/emscripten.css new file mode 100644 index 0000000000..7e3dc811d0 --- /dev/null +++ b/ext/wasm/common/emscripten.css @@ -0,0 +1,24 @@ +/* emcscript-related styling, used during the module load/intialization processes... */ +.emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; } +div.emscripten { text-align: center; } +div.emscripten_border { border: 1px solid black; } +#module-spinner { overflow: visible; } +#module-spinner > * { + margin-top: 1em; +} +.spinner { + height: 50px; + width: 50px; + margin: 0px auto; + animation: rotation 0.8s linear infinite; + border-left: 10px solid rgb(0,150,240); + border-right: 10px solid rgb(0,150,240); + border-bottom: 10px solid rgb(0,150,240); + border-top: 10px solid rgb(100,0,200); + border-radius: 100%; + background-color: rgb(200,100,250); +} +@keyframes rotation { + from {transform: rotate(0deg);} + to {transform: rotate(360deg);} +} diff --git a/ext/fiddle/testing.css b/ext/wasm/common/testing.css similarity index 93% rename from ext/fiddle/testing.css rename to ext/wasm/common/testing.css index f87dbd2cf1..09c570f48a 100644 --- a/ext/fiddle/testing.css +++ b/ext/wasm/common/testing.css @@ -29,3 +29,4 @@ span.labeled-input { color: red; background-color: yellow; } +#test-output { font-family: monospace } diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js new file mode 100644 index 0000000000..5a1d425caf --- /dev/null +++ b/ext/wasm/common/whwasmutil.js @@ -0,0 +1,1548 @@ +/** + 2022-07-08 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + The whwasmutil is developed in conjunction with the Jaccwabyt + project: + + https://fossil.wanderinghorse.net/r/jaccwabyt + + Maintenance reminder: If you're reading this in a tree other than + the Jaccwabyt tree, note that this copy may be replaced with + upstream copies of that one from time to time. Thus the code + installed by this function "should not" be edited outside of that + project, else it risks getting overwritten. +*/ +/** + This function is intended to simplify porting around various bits + of WASM-related utility code from project to project. + + The primary goal of this code is to replace, where possible, + Emscripten-generated glue code with equivalent utility code which + can be used in arbitrary WASM environments built with toolchains + other than Emscripten. As of this writing, this code is capable of + acting as a replacement for Emscripten's generated glue code + _except_ that the latter installs handlers for Emscripten-provided + APIs such as its "FS" (virtual filesystem) API. Loading of such + things still requires using Emscripten's glue, but the post-load + utility APIs provided by this code are still usable as replacements + for their sub-optimally-documented Emscripten counterparts. + + Intended usage: + + ``` + self.WhWasmUtilInstaller(appObject); + delete self.WhWasmUtilInstaller; + ``` + + Its global-scope symbol is intended only to provide an easy way to + make it available to 3rd-party scripts and "should" be deleted + after calling it. That symbols is _not_ used within the library. + + Forewarning: this API explicitly targets only browser + environments. If a given non-browser environment has the + capabilities needed for a given feature (e.g. TextEncoder), great, + but it does not go out of its way to account for them and does not + provide compatibility crutches for them. + + It currently offers alternatives to the following + Emscripten-generated APIs: + + - OPTIONALLY memory allocation, but how this gets imported is + environment-specific. Most of the following features only work + if allocation is available. + + - WASM-exported "indirect function table" access and + manipulation. e.g. creating new WASM-side functions using JS + functions, analog to Emscripten's addFunction() and + removeFunction() but slightly different. + + - Get/set specific heap memory values, analog to Emscripten's + getValue() and setValue(). + + - String length counting in UTF-8 bytes (C-style and JS strings). + + - JS string to C-string conversion and vice versa, analog to + Emscripten's stringToUTF8Array() and friends, but with slighter + different interfaces. + + - JS string to Uint8Array conversion, noting that browsers actually + already have this built in via TextEncoder. + + - "Scoped" allocation, such that allocations made inside of a given + explicit scope will be automatically cleaned up when the scope is + closed. This is fundamentally similar to Emscripten's + stackAlloc() and friends but uses the heap instead of the stack + because access to the stack requires C code. + + - Create JS wrappers for WASM functions, analog to Emscripten's + ccall() and cwrap() functions, except that the automatic + conversions for function arguments and return values can be + easily customized by the client by assigning custom function + signature type names to conversion functions. Essentially, + it's ccall() and cwrap() on steroids. + + How to install... + + Passing an object to this function will install the functionality + into that object. Afterwards, client code "should" delete the global + symbol. + + This code requires that the target object have the following + properties, noting that they needn't be available until the first + time one of the installed APIs is used (as opposed to when this + function is called) except where explicitly noted: + + - `exports` must be a property of the target object OR a property + of `target.instance` (a WebAssembly.Module instance) and it must + contain the symbols exported by the WASM module associated with + this code. In an Enscripten environment it must be set to + `Module['asm']`. The exports object must contain a minimum of the + following symbols: + + - `memory`: a WebAssembly.Memory object representing the WASM + memory. _Alternately_, the `memory` property can be set on the + target instance, in particular if the WASM heap memory is + initialized in JS an _imported_ into WASM, as opposed to being + initialized in WASM and exported to JS. + + - `__indirect_function_table`: the WebAssembly.Table object which + holds WASM-exported functions. This API does not strictly + require that the table be able to grow but it will throw if its + `installFunction()` is called and the table cannot grow. + + In order to simplify downstream usage, if `target.exports` is not + set when this is called then a property access interceptor + (read-only, configurable, enumerable) gets installed as `exports` + which resolves to `target.instance.exports`, noting that the latter + property need not exist until the first time `target.exports` is + accessed. + + Some APIs _optionally_ make use of the `bigIntEnabled` property of + the target object. It "should" be set to true if the WASM + environment is compiled with BigInt support, else it must be + false. If it is false, certain BigInt-related features will trigger + an exception if invoked. This property, if not set when this is + called, will get a default value of true only if the BigInt64Array + constructor is available, else it will default to false. + + Some optional APIs require that the target have the following + methods: + + - 'alloc()` must behave like C's `malloc()`, allocating N bytes of + memory and returning its pointer. In Emscripten this is + conventionally made available via `Module['_malloc']`. This API + requires that the alloc routine throw on allocation error, as + opposed to returning null or 0. + + - 'dealloc()` must behave like C's `free()`, accepting either a + pointer returned from its allocation counterpart or the values + null/0 (for which it must be a no-op). allocating N bytes of + memory and returning its pointer. In Emscripten this is + conventionally made available via `Module['_free']`. + + APIs which require allocation routines are explicitly documented as + such and/or have "alloc" in their names. + + This code is developed and maintained in conjunction with the + Jaccwabyt project: + + https://fossil.wanderinghorse.net/r/jaccwabbyt + + More specifically: + + https://fossil.wanderinghorse.net/r/jaccwabbyt/file/common/whwasmutil.js +*/ +self.WhWasmUtilInstaller = function(target){ + 'use strict'; + if(undefined===target.bigIntEnabled){ + target.bigIntEnabled = !!self['BigInt64Array']; + } + + /** Throws a new Error, the message of which is the concatenation of + all args with a space between each. */ + const toss = (...args)=>{throw new Error(args.join(' '))}; + + if(!target.exports){ + Object.defineProperty(target, 'exports', { + enumerable: true, configurable: true, + get: ()=>(target.instance && target.instance.exports) + }); + } + + /********* + alloc()/dealloc() auto-install... + + This would be convenient but it can also cause us to pick up + malloc() even when the client code is using a different exported + allocator (who, me?), which is bad. malloc() may be exported even + if we're not explicitly using it and overriding the malloc() + function, linking ours first, is not always feasible when using a + malloc() proxy, as it can lead to recursion and stack overflow + (who, me?). So... we really need the downstream code to set up + target.alloc/dealloc() itself. + ******/ + /****** + if(target.exports){ + //Maybe auto-install alloc()/dealloc()... + if(!target.alloc && target.exports.malloc){ + target.alloc = function(n){ + const m = this(n); + return m || toss("Allocation of",n,"byte(s) failed."); + }.bind(target.exports.malloc); + } + + if(!target.dealloc && target.exports.free){ + target.dealloc = function(ptr){ + if(ptr) this(ptr); + }.bind(target.exports.free); + } + }*******/ + + /** + Pointers in WASM are currently assumed to be 32-bit, but someday + that will certainly change. + */ + const ptrIR = target.pointerIR || 'i32'; + const ptrSizeof = ('i32'===ptrIR ? 4 + : ('i64'===ptrIR + ? 8 : toss("Unhandled ptrSizeof:",ptrIR))); + /** Stores various cached state. */ + const cache = Object.create(null); + /** Previously-recorded size of cache.memory.buffer, noted so that + we can recreate the view objects if the heap grows. */ + cache.heapSize = 0; + /** WebAssembly.Memory object extracted from target.memory or + target.exports.memory the first time heapWrappers() is + called. */ + cache.memory = null; + /** uninstallFunction() puts table indexes in here for reuse and + installFunction() extracts them. */ + cache.freeFuncIndexes = []; + /** + Used by scopedAlloc() and friends. + */ + cache.scopedAlloc = []; + + cache.utf8Decoder = new TextDecoder(); + cache.utf8Encoder = new TextEncoder('utf-8'); + + /** + If (cache.heapSize !== cache.memory.buffer.byteLength), i.e. if + the heap has grown since the last call, updates cache.HEAPxyz. + Returns the cache object. + */ + const heapWrappers = function(){ + if(!cache.memory){ + cache.memory = (target.memory instanceof WebAssembly.Memory) + ? target.memory : target.exports.memory; + }else if(cache.heapSize === cache.memory.buffer.byteLength){ + return cache; + } + // heap is newly-acquired or has been resized.... + const b = cache.memory.buffer; + cache.HEAP8 = new Int8Array(b); cache.HEAP8U = new Uint8Array(b); + cache.HEAP16 = new Int16Array(b); cache.HEAP16U = new Uint16Array(b); + cache.HEAP32 = new Int32Array(b); cache.HEAP32U = new Uint32Array(b); + if(target.bigIntEnabled){ + cache.HEAP64 = new BigInt64Array(b); cache.HEAP64U = new BigUint64Array(b); + } + cache.HEAP32F = new Float32Array(b); cache.HEAP64F = new Float64Array(b); + cache.heapSize = b.byteLength; + return cache; + }; + + /** Convenience equivalent of this.heapForSize(8,false). */ + target.heap8 = ()=>heapWrappers().HEAP8; + + /** Convenience equivalent of this.heapForSize(8,true). */ + target.heap8u = ()=>heapWrappers().HEAP8U; + + /** Convenience equivalent of this.heapForSize(16,false). */ + target.heap16 = ()=>heapWrappers().HEAP16; + + /** Convenience equivalent of this.heapForSize(16,true). */ + target.heap16u = ()=>heapWrappers().HEAP16U; + + /** Convenience equivalent of this.heapForSize(32,false). */ + target.heap32 = ()=>heapWrappers().HEAP32; + + /** Convenience equivalent of this.heapForSize(32,true). */ + target.heap32u = ()=>heapWrappers().HEAP32U; + + /** + Requires n to be one of: + + - integer 8, 16, or 32. + - A integer-type TypedArray constructor: Int8Array, Int16Array, + Int32Array, or their Uint counterparts. + + If this.bigIntEnabled is true, it also accepts the value 64 or a + BigInt64Array/BigUint64Array, else it throws if passed 64 or one + of those constructors. + + Returns an integer-based TypedArray view of the WASM heap + memory buffer associated with the given block size. If passed + an integer as the first argument and unsigned is truthy then + the "U" (unsigned) variant of that view is returned, else the + signed variant is returned. If passed a TypedArray value, the + 2nd argument is ignores. Note that Float32Array and + Float64Array views are not supported by this function. + + Note that growth of the heap will invalidate any references to + this heap, so do not hold a reference longer than needed and do + not use a reference after any operation which may + allocate. Instead, re-fetch the reference by calling this + function again. + + Throws if passed an invalid n. + + Pedantic side note: the name "heap" is a bit of a misnomer. In an + Emscripten environment, the memory managed via the stack + allocation API is in the same Memory object as the heap (which + makes sense because otherwise arbitrary pointer X would be + ambiguous: is it in the heap or the stack?). + */ + target.heapForSize = function(n,unsigned = false){ + let ctor; + const c = (cache.memory && cache.heapSize === cache.memory.buffer.byteLength) + ? cache : heapWrappers(); + switch(n){ + case Int8Array: return c.HEAP8; case Uint8Array: return c.HEAP8U; + case Int16Array: return c.HEAP16; case Uint16Array: return c.HEAP16U; + case Int32Array: return c.HEAP32; case Uint32Array: return c.HEAP32U; + case 8: return unsigned ? c.HEAP8U : c.HEAP8; + case 16: return unsigned ? c.HEAP16U : c.HEAP16; + case 32: return unsigned ? c.HEAP32U : c.HEAP32; + case 64: + if(c.HEAP64) return unsigned ? c.HEAP64U : c.HEAP64; + break; + default: + if(this.bigIntEnabled){ + if(n===self['BigUint64Array']) return c.HEAP64U; + else if(n===self['BigInt64Array']) return c.HEAP64; + break; + } + } + toss("Invalid heapForSize() size: expecting 8, 16, 32,", + "or (if BigInt is enabled) 64."); + }.bind(target); + + /** + Returns the WASM-exported "indirect function table." + */ + target.functionTable = function(){ + return target.exports.__indirect_function_table; + /** -----------------^^^^^ "seems" to be a standardized export name. + From Emscripten release notes from 2020-09-10: + - Use `__indirect_function_table` as the import name for the + table, which is what LLVM does. + */ + }.bind(target); + + /** + Given a function pointer, returns the WASM function table entry + if found, else returns a falsy value. + */ + target.functionEntry = function(fptr){ + const ft = this.functionTable(); + return fptr < ft.length ? ft.get(fptr) : undefined; + }.bind(target); + + /** + Creates a WASM function which wraps the given JS function and + returns the JS binding of that WASM function. The signature + argument must be the Jaccwabyt-format or Emscripten + addFunction()-format function signature string. In short: in may + have one of the following formats: + + - Emscripten: `x...`, where the first x is a letter representing + the result type and subsequent letters represent the argument + types. See below. + + - Jaccwabyt: `x(...)` where `x` is the letter representing the + result type and letters in the parens (if any) represent the + argument types. See below. + + Supported letters: + + - `i` = int32 + - `p` = int32 ("pointer") + - `j` = int64 + - `f` = float32 + - `d` = float64 + - `v` = void, only legal for use as the result type + + It throws if an invalid signature letter is used. + + Jaccwabyt-format signatures support some additional letters which + have no special meaning here but (in this context) act as aliases + for other letters: + + - `s`, `P`: same as `p` + + Sidebar: this code is developed together with Jaccwabyt, thus the + support for its signature format. + */ + target.jsFuncToWasm = function f(func, sig){ + /** Attribution: adapted up from Emscripten-generated glue code, + refactored primarily for efficiency's sake, eliminating + call-local functions and superfluous temporary arrays. */ + if(!f._){/*static init...*/ + f._ = { + // Map of signature letters to type IR values + sigTypes: Object.create(null), + // Map of type IR values to WASM type code values + typeCodes: Object.create(null), + /** Encodes n, which must be <2^14 (16384), into target array + tgt, as a little-endian value, using the given method + ('push' or 'unshift'). */ + uleb128Encode: function(tgt, method, n){ + if(n<128) tgt[method](n); + else tgt[method]( (n % 128) | 128, n>>7); + }, + /** Intentionally-lax pattern for Jaccwabyt-format function + pointer signatures, the intent of which is simply to + distinguish them from Emscripten-format signatures. The + downstream checks are less lax. */ + rxJSig: /^(\w)\((\w*)\)$/, + /** Returns the parameter-value part of the given signature + string. */ + sigParams: function(sig){ + const m = f._.rxJSig.exec(sig); + return m ? m[2] : sig.substr(1); + }, + /** Returns the IR value for the given letter or throws + if the letter is invalid. */ + letterType: (x)=>f._.sigTypes[x] || toss("Invalid signature letter:",x), + /** Returns an object describing the result type and parameter + type(s) of the given function signature, or throws if the + signature is invalid. */ + /******** // only valid for use with the WebAssembly.Function ctor, which + // is not yet documented on MDN. + sigToWasm: function(sig){ + const rc = {parameters:[], results: []}; + if('v'!==sig[0]) rc.results.push(f._.letterType(sig[0])); + for(const x of f._.sigParams(sig)){ + rc.parameters.push(f._.letterType(x)); + } + return rc; + },************/ + /** Pushes the WASM data type code for the given signature + letter to the given target array. Throws if letter is + invalid. */ + pushSigType: (dest, letter)=>dest.push(f._.typeCodes[f._.letterType(letter)]) + }; + f._.sigTypes.i = f._.sigTypes.p = f._.sigTypes.P = f._.sigTypes.s = 'i32'; + f._.sigTypes.j = 'i64'; f._.sigTypes.f = 'f32'; f._.sigTypes.d = 'f64'; + f._.typeCodes['i32'] = 0x7f; f._.typeCodes['i64'] = 0x7e; + f._.typeCodes['f32'] = 0x7d; f._.typeCodes['f64'] = 0x7c; + }/*static init*/ + const sigParams = f._.sigParams(sig); + const wasmCode = [0x01/*count: 1*/, 0x60/*function*/]; + f._.uleb128Encode(wasmCode, 'push', sigParams.length); + for(const x of sigParams) f._.pushSigType(wasmCode, x); + if('v'===sig[0]) wasmCode.push(0); + else{ + wasmCode.push(1); + f._.pushSigType(wasmCode, sig[0]); + } + f._.uleb128Encode(wasmCode, 'unshift', wasmCode.length)/* type section length */; + wasmCode.unshift( + 0x00, 0x61, 0x73, 0x6d, /* magic: "\0asm" */ + 0x01, 0x00, 0x00, 0x00, /* version: 1 */ + 0x01 /* type section code */ + ); + wasmCode.push( + /* import section: */ 0x02, 0x07, + /* (import "e" "f" (func 0 (type 0))): */ + 0x01, 0x01, 0x65, 0x01, 0x66, 0x00, 0x00, + /* export section: */ 0x07, 0x05, + /* (export "f" (func 0 (type 0))): */ + 0x01, 0x01, 0x66, 0x00, 0x00 + ); + return (new WebAssembly.Instance( + new WebAssembly.Module(new Uint8Array(wasmCode)), { + e: { f: func } + })).exports['f']; + }/*jsFuncToWasm()*/; + + /** + Expects a JS function and signature, exactly as for + this.jsFuncToWasm(). It uses that function to create a + WASM-exported function, installs that function to the next + available slot of this.functionTable(), and returns the + function's index in that table (which acts as a pointer to that + function). The returned pointer can be passed to + removeFunction() to uninstall it and free up the table slot for + reuse. + + As a special case, if the passed-in function is a WASM-exported + function then the signature argument is ignored and func is + installed as-is, without requiring re-compilation/re-wrapping. + + This function will propagate an exception if + WebAssembly.Table.grow() throws or this.jsFuncToWasm() throws. + The former case can happen in an Emscripten-compiled + environment when building without Emscripten's + `-sALLOW_TABLE_GROWTH` flag. + + Sidebar: this function differs from Emscripten's addFunction() + _primarily_ in that it does not share that function's + undocumented behavior of reusing a function if it's passed to + addFunction() more than once, which leads to removeFunction() + breaking clients which do not take care to avoid that case: + + https://github.com/emscripten-core/emscripten/issues/17323 + */ + target.installFunction = function f(func, sig){ + const ft = this.functionTable(); + const oldLen = ft.length; + let ptr; + while(cache.freeFuncIndexes.length){ + ptr = cache.freeFuncIndexes.pop(); + if(ft.get(ptr)){ /* Table was modified via a different API */ + ptr = null; + continue; + }else{ + break; + } + } + if(!ptr){ + ptr = oldLen; + ft.grow(1); + } + try{ + /*this will only work if func is a WASM-exported function*/ + ft.set(ptr, func); + return ptr; + }catch(e){ + if(!(e instanceof TypeError)){ + if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); + throw e; + } + } + // It's not a WASM-exported function, so compile one... + try { + ft.set(ptr, this.jsFuncToWasm(func, sig)); + }catch(e){ + if(ptr===oldLen) cache.freeFuncIndexes.push(oldLen); + throw e; + } + return ptr; + }.bind(target); + + /** + Requires a pointer value previously returned from + this.installFunction(). Removes that function from the WASM + function table, marks its table slot as free for re-use, and + returns that function. It is illegal to call this before + installFunction() has been called and results are undefined if + ptr was not returned by that function. The returned function + may be passed back to installFunction() to reinstall it. + */ + target.uninstallFunction = function(ptr){ + const fi = cache.freeFuncIndexes; + const ft = this.functionTable(); + fi.push(ptr); + const rc = ft.get(ptr); + ft.set(ptr, null); + return rc; + }.bind(target); + + /** + Given a WASM heap memory address and a data type name in the form + (i8, i16, i32, i64, float (or f32), double (or f64)), this + fetches the numeric value from that address and returns it as a + number or, for the case of type='i64', a BigInt (noting that that + type triggers an exception if this.bigIntEnabled is + falsy). Throws if given an invalid type. + + As a special case, if type ends with a `*`, it is considered to + be a pointer type and is treated as the WASM numeric type + appropriate for the pointer size (`i32`). + + While likely not obvious, this routine and its setMemValue() + counterpart are how pointer-to-value _output_ parameters + in WASM-compiled C code can be interacted with: + + ``` + const ptr = alloc(4); + setMemValue(ptr, 0, 'i32'); // clear the ptr's value + aCFuncWithOutputPtrToInt32Arg( ptr ); // e.g. void foo(int *x); + const result = getMemValue(ptr, 'i32'); // fetch ptr's value + dealloc(ptr); + ``` + + scopedAlloc() and friends can be used to make handling of + `ptr` safe against leaks in the case of an exception: + + ``` + let result; + const scope = scopedAllocPush(); + try{ + const ptr = scopedAlloc(4); + setMemValue(ptr, 0, 'i32'); + aCFuncWithOutputPtrArg( ptr ); + result = getMemValue(ptr, 'i32'); + }finally{ + scopedAllocPop(scope); + } + ``` + + As a rule setMemValue() must be called to set (typically zero + out) the pointer's value, else it will contain an essentially + random value. + + See: setMemValue() + */ + target.getMemValue = function(ptr, type='i8'){ + if(type.endsWith('*')) type = ptrIR; + const c = (cache.memory && cache.heapSize === cache.memory.buffer.byteLength) + ? cache : heapWrappers(); + switch(type){ + case 'i1': + case 'i8': return c.HEAP8[ptr>>0]; + case 'i16': return c.HEAP16[ptr>>1]; + case 'i32': return c.HEAP32[ptr>>2]; + case 'i64': + if(this.bigIntEnabled) return BigInt(c.HEAP64[ptr>>3]); + break; + case 'float': case 'f32': return c.HEAP32F[ptr>>2]; + case 'double': case 'f64': return Number(c.HEAP64F[ptr>>3]); + default: break; + } + toss('Invalid type for getMemValue():',type); + }.bind(target); + + /** + The counterpart of getMemValue(), this sets a numeric value at + the given WASM heap address, using the type to define how many + bytes are written. Throws if given an invalid type. See + getMemValue() for details about the type argument. If the 3rd + argument ends with `*` then it is treated as a pointer type and + this function behaves as if the 3rd argument were `i32`. + + This function returns itself. + */ + target.setMemValue = function f(ptr, value, type='i8'){ + if (type.endsWith('*')) type = ptrIR; + const c = (cache.memory && cache.heapSize === cache.memory.buffer.byteLength) + ? cache : heapWrappers(); + switch (type) { + case 'i1': + case 'i8': c.HEAP8[ptr>>0] = value; return f; + case 'i16': c.HEAP16[ptr>>1] = value; return f; + case 'i32': c.HEAP32[ptr>>2] = value; return f; + case 'i64': + if(c.HEAP64){ + c.HEAP64[ptr>>3] = BigInt(value); + return f; + } + break; + case 'float': case 'f32': c.HEAP32F[ptr>>2] = value; return f; + case 'double': case 'f64': c.HEAP64F[ptr>>3] = value; return f; + } + toss('Invalid type for setMemValue(): ' + type); + }; + + /** + Expects ptr to be a pointer into the WASM heap memory which + refers to a NUL-terminated C-style string encoded as UTF-8. + Returns the length, in bytes, of the string, as for `strlen(3)`. + As a special case, if !ptr then it it returns `null`. Throws if + ptr is out of range for target.heap8u(). + */ + target.cstrlen = function(ptr){ + if(!ptr) return null; + const h = heapWrappers().HEAP8U; + let pos = ptr; + for( ; h[pos] !== 0; ++pos ){} + return pos - ptr; + }; + + /** + Expects ptr to be a pointer into the WASM heap memory which + refers to a NUL-terminated C-style string encoded as UTF-8. This + function counts its byte length using cstrlen() then returns a + JS-format string representing its contents. As a special case, if + ptr is falsy, `null` is returned. + */ + target.cstringToJs = function(ptr){ + const n = this.cstrlen(ptr); + if(null===n) return n; + return n + ? cache.utf8Decoder.decode( + new Uint8Array(heapWrappers().HEAP8U.buffer, ptr, n) + ) : ""; + }.bind(target); + + /** + Given a JS string, this function returns its UTF-8 length in + bytes. Returns null if str is not a string. + */ + target.jstrlen = function(str){ + /** Attribution: derived from Emscripten's lengthBytesUTF8() */ + if('string'!==typeof str) return null; + const n = str.length; + let len = 0; + for(let i = 0; i < n; ++i){ + let u = str.charCodeAt(i); + if(u>=0xd800 && u<=0xdfff){ + u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt(++i) & 0x3FF); + } + if(u<=0x7f) ++len; + else if(u<=0x7ff) len += 2; + else if(u<=0xffff) len += 3; + else len += 4; + } + return len; + }; + + /** + Encodes the given JS string as UTF8 into the given TypedArray + tgt, starting at the given offset and writing, at most, maxBytes + bytes (including the NUL terminator if addNul is true, else no + NUL is added). If it writes any bytes at all and addNul is true, + it always NUL-terminates the output, even if doing so means that + the NUL byte is all that it writes. + + If maxBytes is negative (the default) then it is treated as the + remaining length of tgt, starting at the given offset. + + If writing the last character would surpass the maxBytes count + because the character is multi-byte, that character will not be + written (as opposed to writing a truncated multi-byte character). + This can lead to it writing as many as 3 fewer bytes than + maxBytes specifies. + + Returns the number of bytes written to the target, _including_ + the NUL terminator (if any). If it returns 0, it wrote nothing at + all, which can happen if: + + - str is empty and addNul is false. + - offset < 0. + - maxBytes == 0. + - maxBytes is less than the byte length of a multi-byte str[0]. + + Throws if tgt is not an Int8Array or Uint8Array. + + Design notes: + + - In C's strcpy(), the destination pointer is the first + argument. That is not the case here primarily because the 3rd+ + arguments are all referring to the destination, so it seems to + make sense to have them grouped with it. + + - Emscripten's counterpart of this function (stringToUTF8Array()) + returns the number of bytes written sans NUL terminator. That + is, however, ambiguous: str.length===0 or maxBytes===(0 or 1) + all cause 0 to be returned. + */ + target.jstrcpy = function(jstr, tgt, offset = 0, maxBytes = -1, addNul = true){ + /** Attribution: the encoding bits are taken from Emscripten's + stringToUTF8Array(). */ + if(!tgt || (!(tgt instanceof Int8Array) && !(tgt instanceof Uint8Array))){ + toss("jstrcpy() target must be an Int8Array or Uint8Array."); + } + if(maxBytes<0) maxBytes = tgt.length - offset; + if(!(maxBytes>0) || !(offset>=0)) return 0; + let i = 0, max = jstr.length; + const begin = offset, end = offset + maxBytes - (addNul ? 1 : 0); + for(; i < max && offset < end; ++i){ + let u = jstr.charCodeAt(i); + if(u>=0xd800 && u<=0xdfff){ + u = 0x10000 + ((u & 0x3FF) << 10) | (jstr.charCodeAt(++i) & 0x3FF); + } + if(u<=0x7f){ + if(offset >= end) break; + tgt[offset++] = u; + }else if(u<=0x7ff){ + if(offset + 1 >= end) break; + tgt[offset++] = 0xC0 | (u >> 6); + tgt[offset++] = 0x80 | (u & 0x3f); + }else if(u<=0xffff){ + if(offset + 2 >= end) break; + tgt[offset++] = 0xe0 | (u >> 12); + tgt[offset++] = 0x80 | ((u >> 6) & 0x3f); + tgt[offset++] = 0x80 | (u & 0x3f); + }else{ + if(offset + 3 >= end) break; + tgt[offset++] = 0xf0 | (u >> 18); + tgt[offset++] = 0x80 | ((u >> 12) & 0x3f); + tgt[offset++] = 0x80 | ((u >> 6) & 0x3f); + tgt[offset++] = 0x80 | (u & 0x3f); + } + } + if(addNul) tgt[offset++] = 0; + return offset - begin; + }; + + /** + Works similarly to C's strncpy(), copying, at most, n bytes (not + characters) from srcPtr to tgtPtr. It copies until n bytes have + been copied or a 0 byte is reached in src. _Unlike_ strncpy(), it + returns the number of bytes it assigns in tgtPtr, _including_ the + NUL byte (if any). If n is reached before a NUL byte in srcPtr, + tgtPtr will _not_ be NULL-terminated. If a NUL byte is reached + before n bytes are copied, tgtPtr will be NUL-terminated. + + If n is negative, cstrlen(srcPtr)+1 is used to calculate it, the + +1 being for the NUL byte. + + Throws if tgtPtr or srcPtr are falsy. Results are undefined if: + + - either is not a pointer into the WASM heap or + + - srcPtr is not NUL-terminated AND n is less than srcPtr's + logical length. + + ACHTUNG: it is possible to copy partial multi-byte characters + this way, and converting such strings back to JS strings will + have undefined results. + */ + target.cstrncpy = function(tgtPtr, srcPtr, n){ + if(!tgtPtr || !srcPtr) toss("cstrncpy() does not accept NULL strings."); + if(n<0) n = this.cstrlen(strPtr)+1; + else if(!(n>0)) return 0; + const heap = this.heap8u(); + let i = 0, ch; + for(; i < n && (ch = heap[srcPtr+i]); ++i){ + heap[tgtPtr+i] = ch; + } + if(i{ + return cache.utf8Encoder.encode(addNul ? (str+"\0") : str); + // Or the hard way... + /** Attribution: derived from Emscripten's stringToUTF8Array() */ + //const a = [], max = str.length; + //let i = 0, pos = 0; + //for(; i < max; ++i){ + // let u = str.charCodeAt(i); + // if(u>=0xd800 && u<=0xdfff){ + // u = 0x10000 + ((u & 0x3FF) << 10) | (str.charCodeAt(++i) & 0x3FF); + // } + // if(u<=0x7f) a[pos++] = u; + // else if(u<=0x7ff){ + // a[pos++] = 0xC0 | (u >> 6); + // a[pos++] = 0x80 | (u & 63); + // }else if(u<=0xffff){ + // a[pos++] = 0xe0 | (u >> 12); + // a[pos++] = 0x80 | ((u >> 6) & 63); + // a[pos++] = 0x80 | (u & 63); + // }else{ + // a[pos++] = 0xf0 | (u >> 18); + // a[pos++] = 0x80 | ((u >> 12) & 63); + // a[pos++] = 0x80 | ((u >> 6) & 63); + // a[pos++] = 0x80 | (u & 63); + // } + // } + // return new Uint8Array(a); + }; + + const __affirmAlloc = (obj,funcName)=>{ + if(!(obj.alloc instanceof Function) || + !(obj.dealloc instanceof Function)){ + toss("Object is missing alloc() and/or dealloc() function(s)", + "required by",funcName+"()."); + } + }; + + const __allocCStr = function(jstr, returnWithLength, allocator, funcName){ + __affirmAlloc(this, funcName); + if('string'!==typeof jstr) return null; + const n = this.jstrlen(jstr), + ptr = allocator(n+1); + this.jstrcpy(jstr, this.heap8u(), ptr, n+1, true); + return returnWithLength ? [ptr, n] : ptr; + }.bind(target); + + /** + Uses target.alloc() to allocate enough memory for jstrlen(jstr)+1 + bytes of memory, copies jstr to that memory using jstrcpy(), + NUL-terminates it, and returns the pointer to that C-string. + Ownership of the pointer is transfered to the caller, who must + eventually pass the pointer to dealloc() to free it. + + If passed a truthy 2nd argument then its return semantics change: + it returns [ptr,n], where ptr is the C-string's pointer and n is + its cstrlen(). + + Throws if `target.alloc` or `target.dealloc` are not functions. + */ + target.allocCString = + (jstr, returnWithLength=false)=>__allocCStr(jstr, returnWithLength, + target.alloc, 'allocCString()'); + + /** + Starts an "allocation scope." All allocations made using + scopedAlloc() are recorded in this scope and are freed when the + value returned from this function is passed to + scopedAllocPop(). + + This family of functions requires that the API's object have both + `alloc()` and `dealloc()` methods, else this function will throw. + + Intended usage: + + ``` + const scope = scopedAllocPush(); + try { + const ptr1 = scopedAlloc(100); + const ptr2 = scopedAlloc(200); + const ptr3 = scopedAlloc(300); + ... + // Note that only allocations made via scopedAlloc() + // are managed by this allocation scope. + }finally{ + scopedAllocPop(scope); + } + ``` + + The value returned by this function must be treated as opaque by + the caller, suitable _only_ for passing to scopedAllocPop(). + Its type and value are not part of this function's API and may + change in any given version of this code. + + `scopedAlloc.level` can be used to determine how many scoped + alloc levels are currently active. + */ + target.scopedAllocPush = function(){ + __affirmAlloc(this, 'scopedAllocPush'); + const a = []; + cache.scopedAlloc.push(a); + return a; + }.bind(target); + + /** + Cleans up all allocations made using scopedAlloc() in the context + of the given opaque state object, which must be a value returned + by scopedAllocPush(). See that function for an example of how to + use this function. + + Though scoped allocations are managed like a stack, this API + behaves properly if allocation scopes are popped in an order + other than the order they were pushed. + + If called with no arguments, it pops the most recent + scopedAllocPush() result: + + ``` + scopedAllocPush(); + try{ ... } finally { scopedAllocPop(); } + ``` + + It's generally recommended that it be passed an explicit argument + to help ensure that push/push are used in matching pairs, but in + trivial code that may be a non-issue. + */ + target.scopedAllocPop = function(state){ + __affirmAlloc(this, 'scopedAllocPop'); + const n = arguments.length + ? cache.scopedAlloc.indexOf(state) + : cache.scopedAlloc.length-1; + if(n<0) toss("Invalid state object for scopedAllocPop()."); + if(0===arguments.length) state = cache.scopedAlloc[n]; + cache.scopedAlloc.splice(n,1); + for(let p; (p = state.pop()); ) this.dealloc(p); + }.bind(target); + + /** + Allocates n bytes of memory using this.alloc() and records that + fact in the state for the most recent call of scopedAllocPush(). + Ownership of the memory is given to scopedAllocPop(), which + will clean it up when it is called. The memory _must not_ be + passed to this.dealloc(). Throws if this API object is missing + the required `alloc()` or `dealloc()` functions or no scoped + alloc is active. + + See scopedAllocPush() for an example of how to use this function. + + The `level` property of this function can be queried to query how + many scoped allocation levels are currently active. + + See also: scopedAllocPtr(), scopedAllocCString() + */ + target.scopedAlloc = function(n){ + if(!cache.scopedAlloc.length){ + toss("No scopedAllocPush() scope is active."); + } + const p = this.alloc(n); + cache.scopedAlloc[cache.scopedAlloc.length-1].push(p); + return p; + }.bind(target); + + Object.defineProperty(target.scopedAlloc, 'level', { + configurable: false, enumerable: false, + get: ()=>cache.scopedAlloc.length, + set: ()=>toss("The 'active' property is read-only.") + }); + + /** + Works identically to allocCString() except that it allocates the + memory using scopedAlloc(). + + Will throw if no scopedAllocPush() call is active. + */ + target.scopedAllocCString = + (jstr, returnWithLength=false)=>__allocCStr(jstr, returnWithLength, + target.scopedAlloc, 'scopedAllocCString()'); + + /** + Wraps function call func() in a scopedAllocPush() and + scopedAllocPop() block, such that all calls to scopedAlloc() and + friends from within that call will have their memory freed + automatically when func() returns. If func throws or propagates + an exception, the scope is still popped, otherwise it returns the + result of calling func(). + */ + target.scopedAllocCall = function(func){ + this.scopedAllocPush(); + try{ return func() } finally{ this.scopedAllocPop() } + }.bind(target); + + /** Internal impl for allocPtr() and scopedAllocPtr(). */ + const __allocPtr = function(howMany, method){ + __affirmAlloc(this, method); + let m = this[method](howMany * ptrSizeof); + this.setMemValue(m, 0, ptrIR) + if(1===howMany){ + return m; + } + const a = [m]; + for(let i = 1; i < howMany; ++i){ + m += ptrSizeof; + a[i] = m; + this.setMemValue(m, 0, ptrIR); + } + return a; + }.bind(target); + + /** + Allocates a single chunk of memory capable of holding `howMany` + pointers and zeroes them out. If `howMany` is 1 then the memory + chunk is returned directly, else an array of pointer addresses is + returned, which can optionally be used with "destructuring + assignment" like this: + + ``` + const [p1, p2, p3] = allocPtr(3); + ``` + + ACHTUNG: when freeing the memory, pass only the _first_ result + value to dealloc(). The others are part of the same memory chunk + and must not be freed separately. + */ + target.allocPtr = (howMany=1)=>__allocPtr(howMany, 'alloc'); + + /** + Identical to allocPtr() except that it allocates using scopedAlloc() + instead of alloc(). + */ + target.scopedAllocPtr = (howMany=1)=>__allocPtr(howMany, 'scopedAlloc'); + + /** + If target.exports[name] exists, it is returned, else an + exception is thrown. + */ + target.xGet = function(name){ + return target.exports[name] || toss("Cannot find exported symbol:",name); + }; + + const __argcMismatch = + (f,n)=>toss(f+"() requires",n,"argument(s)."); + + /** + Looks up a WASM-exported function named fname from + target.exports. If found, it is called, passed all remaining + arguments, and its return value is returned to xCall's caller. If + not found, an exception is thrown. This function does no + conversion of argument or return types, but see xWrap() + and xCallWrapped() for variants which do. + + As a special case, if passed only 1 argument after the name and + that argument in an Array, that array's entries become the + function arguments. (This is not an ambiguous case because it's + not legal to pass an Array object to a WASM function.) + */ + target.xCall = function(fname, ...args){ + const f = this.xGet(fname); + if(!(f instanceof Function)) toss("Exported symbol",fname,"is not a function."); + if(f.length!==args.length) __argcMismatch(fname,f.length) + /* This is arguably over-pedantic but we want to help clients keep + from shooting themselves in the foot when calling C APIs. */; + return (2===arguments.length && Array.isArray(arguments[1])) + ? f.apply(null, arguments[1]) + : f.apply(null, args); + }.bind(target); + + /** + State for use with xWrap() + */ + cache.xWrap = Object.create(null); + const xcv = cache.xWrap.convert = Object.create(null); + /** Map of type names to argument conversion functions. */ + cache.xWrap.convert.arg = Object.create(null); + /** Map of type names to return result conversion functions. */ + cache.xWrap.convert.result = Object.create(null); + + xcv.arg.i64 = (i)=>BigInt(i); + xcv.arg.i32 = (i)=>(i | 0); + xcv.arg.i16 = (i)=>((i | 0) & 0xFFFF); + xcv.arg.i8 = (i)=>((i | 0) & 0xFF); + xcv.arg.f32 = xcv.arg.float = (i)=>Number(i).valueOf(); + xcv.arg.f64 = xcv.arg.double = xcv.arg.f32; + xcv.arg.int = xcv.arg.i32; + xcv.result['*'] = xcv.result['pointer'] = xcv.arg[ptrIR]; + + for(const t of ['i8', 'i16', 'i32', 'int', 'i64', + 'f32', 'float', 'f64', 'double']){ + xcv.arg[t+'*'] = xcv.result[t+'*'] = xcv.arg[ptrIR] + xcv.result[t] = xcv.arg[t] || toss("Missing arg converter:",t); + } + xcv.arg['**'] = xcv.arg[ptrIR]; + + /** + In order for args of type string to work in various contexts in + the sqlite3 API, we need to pass them on as, variably, a C-string + or a pointer value. Thus for ARGs of type 'string' and + '*'/'pointer' we behave differently depending on whether the + argument is a string or not: + + - If v is a string, scopeAlloc() a new C-string from it and return + that temp string's pointer. + + - Else return the value from the arg adaptor defined for ptrIR. + + TODO? Permit an Int8Array/Uint8Array and convert it to a string? + Would that be too much magic concentrated in one place, ready to + backfire? + */ + xcv.arg.string = xcv.arg['pointer'] = xcv.arg['*'] = function(v){ + if('string'===typeof v) return target.scopedAllocCString(v); + return v ? xcv.arg[ptrIR](v) : null; + }; + xcv.result.string = (i)=>target.cstringToJs(i); + xcv.result['string:free'] = function(i){ + try { return i ? target.cstringToJs(i) : null } + finally{ target.dealloc(i) } + }; + xcv.result.json = (i)=>JSON.parse(target.cstringToJs(i)); + xcv.result['json:free'] = function(i){ + try{ return i ? JSON.parse(target.cstringToJs(i)) : null } + finally{ target.dealloc(i) } + } + xcv.result['void'] = (v)=>undefined; + xcv.result['null'] = (v)=>v; + + if(0){ + /*** + This idea can't currently work because we don't know the + signature for the func and don't have a way for the user to + convey it. To do this we likely need to be able to match + arg/result handlers by a regex, but that would incur an O(N) + cost as we check the regex one at a time. Another use case for + such a thing would be pseudotypes like "int:-1" to say that + the value will always be treated like -1 (which has a useful + case in the sqlite3 bindings). + */ + xcv.arg['func-ptr'] = function(v){ + if(!(v instanceof Function)) return xcv.arg[ptrIR]; + const f = this.jsFuncToWasm(v, WHAT_SIGNATURE); + }.bind(target); + } + + const __xArgAdapter = + (t)=>xcv.arg[t] || toss("Argument adapter not found:",t); + + const __xResultAdapter = + (t)=>xcv.result[t] || toss("Result adapter not found:",t); + + cache.xWrap.convertArg = (t,v)=>__xArgAdapter(t)(v); + cache.xWrap.convertResult = + (t,v)=>(null===t ? v : (t ? __xResultAdapter(t)(v) : undefined)); + + /** + Creates a wrapper for the WASM-exported function fname. Uses + xGet() to fetch the exported function (which throws on + error) and returns either that function or a wrapper for that + function which converts the JS-side argument types into WASM-side + types and converts the result type. If the function takes no + arguments and resultType is `null` then the function is returned + as-is, else a wrapper is created for it to adapt its arguments + and result value, as described below. + + (If you're familiar with Emscripten's ccall() and cwrap(), this + function is essentially cwrap() on steroids.) + + This function's arguments are: + + - fname: the exported function's name. xGet() is used to fetch + this, so will throw if no exported function is found with that + name. + + - resultType: the name of the result type. A literal `null` means + to return the original function's value as-is (mnemonic: there + is "null" conversion going on). Literal `undefined` or the + string `"void"` mean to ignore the function's result and return + `undefined`. Aside from those two special cases, it may be one + of the values described below or any mapping installed by the + client using xWrap.resultAdapter(). + + If passed 3 arguments and the final one is an array, that array + must contain a list of type names (see below) for adapting the + arguments from JS to WASM. If passed 2 arguments, more than 3, + or the 3rd is not an array, all arguments after the 2nd (if any) + are treated as type names. i.e.: + + ``` + xWrap('funcname', 'i32', 'string', 'f64'); + // is equivalent to: + xWrap('funcname', 'i32', ['string', 'f64']); + ``` + + Type names are symbolic names which map the arguments to an + adapter function to convert, if needed, the value before passing + it on to WASM or to convert a return result from WASM. The list + of built-in names: + + - `i8`, `i16`, `i32` (args and results): all integer conversions + which convert their argument to an integer and truncate it to + the given bit length. + + - `N*` (args): a type name in the form `N*`, where N is a numeric + type name, is treated the same as WASM pointer. + + - `*` and `pointer` (args): have multple semantics. They + behave exactly as described below for `string` args. + + - `*` and `pointer` (results): are aliases for the current + WASM pointer numeric type. + + - `**` (args): is simply a descriptive alias for the WASM pointer + type. It's primarily intended to mark output-pointer arguments. + + - `i64` (args and results): passes the value to BigInt() to + convert it to an int64. + + - `f32` (`float`), `f64` (`double`) (args and results): pass + their argument to Number(). i.e. the adaptor does not currently + distinguish between the two types of floating-point numbers. + + Non-numeric conversions include: + + - `string` (args): has two different semantics in order to + accommodate various uses of certain C APIs (e.g. output-style + strings)... + + - If the arg is a string, it creates a _temporary_ C-string to + pass to the exported function, cleaning it up before the + wrapper returns. If a long-lived C-string pointer is + required, that requires client-side code to create the + string, then pass its pointer to the function. + + - Else the arg is assumed to be a pointer to a string the + client has already allocated and it's passed on as + a WASM pointer. + + - `string` (results): treats the result value as a const C-string, + copies it to a JS string, and returns that JS string. + + - `string:free` (results): treats the result value as a non-const + C-string, ownership of which has just been transfered to the + caller. It copies the C-string to a JS string, frees the + C-string, and returns the JS string. If such a result value is + NULL, the JS result is `null`. + + - `json` (results): treats the result as a const C-string and + returns the result of passing the converted-to-JS string to + JSON.parse(). Returns `null` if the C-string is a NULL pointer. + + - `json:free` (results): works exactly like `string:free` but + returns the same thing as the `json` adapter. + + The type names for results and arguments are validated when + xWrap() is called and any unknown names will trigger an + exception. + + Clients may map their own result and argument adapters using + xWrap.resultAdapter() and xWrap.argAdaptor(), noting that not all + type conversions are valid for both arguments _and_ result types + as they often have different memory ownership requirements. + + TODOs: + + - Figure out how/whether we can (semi-)transparently handle + pointer-type _output_ arguments. Those currently require + explicit handling by allocating pointers, assigning them before + the call using setMemValue(), and fetching them with + getMemValue() after the call. We may be able to automate some + or all of that. + + - Figure out whether it makes sense to extend the arg adapter + interface such that each arg adapter gets an array containing + the results of the previous arguments in the current call. That + might allow some interesting type-conversion feature. Use case: + handling of the final argument to sqlite3_prepare_v2() depends + on the type (pointer vs JS string) of its 2nd + argument. Currently that distinction requires hand-writing a + wrapper for that function. That case is unusual enough that + abstracting it into this API (and taking on the associated + costs) may well not make good sense. + */ + target.xWrap = function(fname, resultType, ...argTypes){ + if(3===arguments.length && Array.isArray(arguments[2])){ + argTypes = arguments[2]; + } + const xf = this.xGet(fname); + if(argTypes.length!==xf.length) __argcMismatch(fname, xf.length) + if((null===resultType) && 0===xf.length){ + /* Func taking no args with an as-is return. We don't need a wrapper. */ + return xf; + } + /*Verify the arg type conversions are valid...*/; + if(undefined!==resultType && null!==resultType) __xResultAdapter(resultType); + argTypes.forEach(__xArgAdapter) + if(0===xf.length){ + // No args to convert, so we can create a simpler wrapper... + return function(){ + return (arguments.length + ? __argcMismatch(fname, xf.length) + : cache.xWrap.convertResult(resultType, xf.call(null))); + }; + } + return function(...args){ + if(args.length!==xf.length) __argcMismatch(fname, xf.length); + const scope = this.scopedAllocPush(); + try{ + const rc = xf.apply(null,args.map((v,i)=>cache.xWrap.convertArg(argTypes[i], v))); + return cache.xWrap.convertResult(resultType, rc); + }finally{ + this.scopedAllocPop(scope); + } + }.bind(this); + }.bind(target)/*xWrap()*/; + + /** Internal impl for xWrap.resultAdapter() and argAdaptor(). */ + const __xAdapter = function(func, argc, typeName, adapter, modeName, xcvPart){ + if('string'===typeof typeName){ + if(1===argc) return xcvPart[typeName]; + else if(2===argc){ + if(!adapter){ + delete xcvPart[typeName]; + return func; + }else if(!(adapter instanceof Function)){ + toss(modeName,"requires a function argument."); + } + xcvPart[typeName] = adapter; + return func; + } + } + toss("Invalid arguments to",modeName); + }; + + /** + Gets, sets, or removes a result value adapter for use with + xWrap(). If passed only 1 argument, the adapter function for the + given type name is returned. If the second argument is explicit + falsy (as opposed to defaulted), the adapter named by the first + argument is removed. If the 2nd argument is not falsy, it must be + a function which takes one value and returns a value appropriate + for the given type name. The adapter may throw if its argument is + not of a type it can work with. This function throws for invalid + arguments. + + Example: + + ``` + xWrap.resultAdapter('twice',(v)=>v+v); + ``` + + xWrap.resultAdapter() MUST NOT use the scopedAlloc() family of + APIs to allocate a result value. xWrap()-generated wrappers run + in the context of scopedAllocPush() so that argument adapters can + easily convert, e.g., to C-strings, and have them cleaned up + automatically before the wrapper returns to the caller. Likewise, + if a _result_ adapter uses scoped allocation, the result will be + freed before because they would be freed before the wrapper + returns, leading to chaos and undefined behavior. + + Except when called as a getter, this function returns itself. + */ + target.xWrap.resultAdapter = function f(typeName, adapter){ + return __xAdapter(f, arguments.length, typeName, adapter, + 'resultAdaptor()', xcv.result); + }; + + /** + Functions identically to xWrap.resultAdapter() but applies to + call argument conversions instead of result value conversions. + + xWrap()-generated wrappers perform argument conversion in the + context of a scopedAllocPush(), so any memory allocation + performed by argument adapters really, really, really should be + made using the scopedAlloc() family of functions unless + specifically necessary. For example: + + ``` + xWrap.argAdapter('my-string', function(v){ + return ('string'===typeof v) + ? myWasmObj.scopedAllocCString(v) : null; + }; + ``` + + Contrariwise, xWrap.resultAdapter() must _not_ use scopedAlloc() + to allocate its results because they would be freed before the + xWrap()-created wrapper returns. + + Note that it is perfectly legitimate to use these adapters to + perform argument validation, as opposed (or in addition) to + conversion. + */ + target.xWrap.argAdapter = function f(typeName, adapter){ + return __xAdapter(f, arguments.length, typeName, adapter, + 'argAdaptor()', xcv.arg); + }; + + /** + Functions like xCall() but performs argument and result type + conversions as for xWrap(). The first argument is the name of the + exported function to call. The 2nd its the name of its result + type, as documented for xWrap(). The 3rd is an array of argument + type name, as documented for xWrap() (use a falsy value or an + empty array for nullary functions). The 4th+ arguments are + arguments for the call, with the special case that if the 4th + argument is an array, it is used as the arguments for the call + (again, falsy or an empty array for nullary functions). Returns + the converted result of the call. + + This is just a thin wrapp around xWrap(). If the given function + is to be called more than once, it's more efficient to use + xWrap() to create a wrapper, then to call that wrapper as many + times as needed. For one-shot calls, however, this variant is + arguably more efficient because it will hypothetically free the + wrapper function quickly. + */ + target.xCallWrapped = function(fname, resultType, argTypes, ...args){ + if(Array.isArray(arguments[3])) args = arguments[3]; + return this.xWrap(fname, resultType, argTypes||[]).apply(null, args||[]); + }.bind(target); + + return target; +}; + +/** + yawl (Yet Another Wasm Loader) provides very basic wasm loader. + It requires a config object: + + - `uri`: required URI of the WASM file to load. + + - `onload(loadResult,config)`: optional callback. The first + argument is the result object from + WebAssembly.instanitate[Streaming](). The 2nd is the config + object passed to this function. Described in more detail below. + + - `imports`: optional imports object for + WebAssembly.instantiate[Streaming](). The default is am empty set + of imports. If the module requires any imports, this object + must include them. + + - `wasmUtilTarget`: optional object suitable for passing to + WhWasmUtilInstaller(). If set, it gets passed to that function + after the promise resolves. This function sets several properties + on it before passing it on to that function (which sets many + more): + + - `module`, `instance`: the properties from the + instantiate[Streaming]() result. + + - If `instance.exports.memory` is _not_ set then it requires that + `config.imports.env.memory` be set (else it throws), and + assigns that to `target.memory`. + + - If `wasmUtilTarget.alloc` is not set and + `instance.exports.malloc` is, it installs + `wasmUtilTarget.alloc()` and `wasmUtilTarget.dealloc()` + wrappers for the exports `malloc` and `free` functions. + + It returns a function which, when called, initiates loading of the + module and returns a Promise. When that Promise resolves, it calls + the `config.onload` callback (if set) and passes it + `(loadResult,config)`, where `loadResult` is the result of + WebAssembly.instantiate[Streaming](): an object in the form: + + ``` + { + module: a WebAssembly.Module, + instance: a WebAssembly.Instance + } + ``` + + (Note that the initial `then()` attached to the promise gets only + that object, and not the `config` one.) + + Error handling is up to the caller, who may attach a `catch()` call + to the promise. +*/ +self.WhWasmUtilInstaller.yawl = function(config){ + const wfetch = ()=>fetch(config.uri, {credentials: 'same-origin'}); + const wui = this; + const finalThen = function(arg){ + //log("finalThen()",arg); + if(config.wasmUtilTarget){ + const toss = (...args)=>{throw new Error(args.join(' '))}; + const tgt = config.wasmUtilTarget; + tgt.module = arg.module; + tgt.instance = arg.instance; + //tgt.exports = tgt.instance.exports; + if(!tgt.instance.exports.memory){ + /** + WhWasmUtilInstaller requires either tgt.exports.memory + (exported from WASM) or tgt.memory (JS-provided memory + imported into WASM). + */ + tgt.memory = (config.imports && config.imports.env + && config.imports.env.memory) + || toss("Missing 'memory' object!"); + } + if(!tgt.alloc && arg.instance.exports.malloc){ + tgt.alloc = function(n){ + return this(n) || toss("Allocation of",n,"bytes failed."); + }.bind(arg.instance.exports.malloc); + tgt.dealloc = function(m){this(m)}.bind(arg.instance.exports.free); + } + wui(tgt); + } + if(config.onload) config.onload(arg,config); + return arg /* for any then() handler attached to + yetAnotherWasmLoader()'s return value */; + }; + const loadWasm = WebAssembly.instantiateStreaming + ? function loadWasmStreaming(){ + return WebAssembly.instantiateStreaming(wfetch(), config.imports||{}) + .then(finalThen); + } + : function loadWasmOldSchool(){ // Safari < v15 + return wfetch() + .then(response => response.arrayBuffer()) + .then(bytes => WebAssembly.instantiate(bytes, config.imports||{})) + .then(finalThen); + }; + return loadWasm; +}.bind(self.WhWasmUtilInstaller)/*yawl()*/; diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js new file mode 100644 index 0000000000..a018658579 --- /dev/null +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -0,0 +1,737 @@ +/** + 2022-06-30 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + The Jaccwabyt API is documented in detail in an external file. + + Project home: https://fossil.wanderinghorse.net/r/jaccwabyt + +*/ +'use strict'; +self.Jaccwabyt = function StructBinderFactory(config){ +/* ^^^^ it is recommended that clients move that object into wherever + they'd like to have it and delete the self-held copy ("self" being + the global window or worker object). This API does not require the + global reference - it is simply installed as a convenience for + connecting these bits to other co-developed code before it gets + removed from the global namespace. +*/ + + /** Throws a new Error, the message of which is the concatenation + all args with a space between each. */ + const toss = (...args)=>{throw new Error(args.join(' '))}; + + /** + Implementing function bindings revealed significant + shortcomings in Emscripten's addFunction()/removeFunction() + interfaces: + + https://github.com/emscripten-core/emscripten/issues/17323 + + Until those are resolved, or a suitable replacement can be + implemented, our function-binding API will be more limited + and/or clumsier to use than initially hoped. + */ + if(!(config.heap instanceof WebAssembly.Memory) + && !(config.heap instanceof Function)){ + toss("config.heap must be WebAssembly.Memory instance or a function."); + } + ['alloc','dealloc'].forEach(function(k){ + (config[k] instanceof Function) || + toss("Config option '"+k+"' must be a function."); + }); + const SBF = StructBinderFactory; + const heap = (config.heap instanceof Function) + ? config.heap : (()=>new Uint8Array(config.heap.buffer)), + alloc = config.alloc, + dealloc = config.dealloc, + log = config.log || console.log.bind(console), + memberPrefix = (config.memberPrefix || ""), + memberSuffix = (config.memberSuffix || ""), + bigIntEnabled = (undefined===config.bigIntEnabled + ? !!self['BigInt64Array'] : !!config.bigIntEnabled), + BigInt = self['BigInt'], + BigInt64Array = self['BigInt64Array'], + /* Undocumented (on purpose) config options: */ + functionTable = config.functionTable/*EXPERIMENTAL, undocumented*/, + ptrSizeof = config.ptrSizeof || 4, + ptrIR = config.ptrIR || 'i32' + ; + + if(!SBF.debugFlags){ + SBF.__makeDebugFlags = function(deriveFrom=null){ + /* This is disgustingly overengineered. :/ */ + if(deriveFrom && deriveFrom.__flags) deriveFrom = deriveFrom.__flags; + const f = function f(flags){ + if(0===arguments.length){ + return f.__flags; + } + if(flags<0){ + delete f.__flags.getter; delete f.__flags.setter; + delete f.__flags.alloc; delete f.__flags.dealloc; + }else{ + f.__flags.getter = 0!==(0x01 & flags); + f.__flags.setter = 0!==(0x02 & flags); + f.__flags.alloc = 0!==(0x04 & flags); + f.__flags.dealloc = 0!==(0x08 & flags); + } + return f._flags; + }; + Object.defineProperty(f,'__flags', { + iterable: false, writable: false, + value: Object.create(deriveFrom) + }); + if(!deriveFrom) f(0); + return f; + }; + SBF.debugFlags = SBF.__makeDebugFlags(); + }/*static init*/ + + const isLittleEndian = (function() { + const buffer = new ArrayBuffer(2); + new DataView(buffer).setInt16(0, 256, true /* littleEndian */); + // Int16Array uses the platform's endianness. + return new Int16Array(buffer)[0] === 256; + })(); + /** + Some terms used in the internal docs: + + StructType: a struct-wrapping class generated by this + framework. + DEF: struct description object. + SIG: struct member signature string. + */ + + /** True if SIG s looks like a function signature, else + false. */ + const isFuncSig = (s)=>'('===s[1]; + /** True if SIG s is-a pointer signature. */ + const isPtrSig = (s)=>'p'===s || 'P'===s; + const isAutoPtrSig = (s)=>'P'===s /*EXPERIMENTAL*/; + const sigLetter = (s)=>isFuncSig(s) ? 'p' : s[0]; + /** Returns the WASM IR form of the Emscripten-conventional letter + at SIG s[0]. Throws for an unknown SIG. */ + const sigIR = function(s){ + switch(sigLetter(s)){ + case 'i': return 'i32'; + case 'p': case 'P': case 's': return ptrIR; + case 'j': return 'i64'; + case 'f': return 'float'; + case 'd': return 'double'; + } + toss("Unhandled signature IR:",s); + }; + /** Returns the sizeof value for the given SIG. Throws for an + unknown SIG. */ + const sigSizeof = function(s){ + switch(sigLetter(s)){ + case 'i': return 4; + case 'p': case 'P': case 's': return ptrSizeof; + case 'j': return 8; + case 'f': return 4 /* C-side floats, not JS-side */; + case 'd': return 8; + } + toss("Unhandled signature sizeof:",s); + }; + const affirmBigIntArray = BigInt64Array + ? ()=>true : ()=>toss('BigInt64Array is not available.'); + /** Returns the (signed) TypedArray associated with the type + described by the given SIG. Throws for an unknown SIG. */ + /********** + const sigTypedArray = function(s){ + switch(sigIR(s)) { + case 'i32': return Int32Array; + case 'i64': return affirmBigIntArray() && BigInt64Array; + case 'float': return Float32Array; + case 'double': return Float64Array; + } + toss("Unhandled signature TypedArray:",s); + }; + **************/ + /** Returns the name of a DataView getter method corresponding + to the given SIG. */ + const sigDVGetter = function(s){ + switch(sigLetter(s)) { + case 'p': case 'P': case 's': { + switch(ptrSizeof){ + case 4: return 'getInt32'; + case 8: return affirmBigIntArray() && 'getBigInt64'; + } + break; + } + case 'i': return 'getInt32'; + case 'j': return affirmBigIntArray() && 'getBigInt64'; + case 'f': return 'getFloat32'; + case 'd': return 'getFloat64'; + } + toss("Unhandled DataView getter for signature:",s); + }; + /** Returns the name of a DataView setter method corresponding + to the given SIG. */ + const sigDVSetter = function(s){ + switch(sigLetter(s)){ + case 'p': case 'P': case 's': { + switch(ptrSizeof){ + case 4: return 'setInt32'; + case 8: return affirmBigIntArray() && 'setBigInt64'; + } + break; + } + case 'i': return 'setInt32'; + case 'j': return affirmBigIntArray() && 'setBigInt64'; + case 'f': return 'setFloat32'; + case 'd': return 'setFloat64'; + } + toss("Unhandled DataView setter for signature:",s); + }; + /** + Returns either Number of BigInt, depending on the given + SIG. This constructor is used in property setters to coerce + the being-set value to the correct size. + */ + const sigDVSetWrapper = function(s){ + switch(sigLetter(s)) { + case 'i': case 'f': case 'd': return Number; + case 'j': return affirmBigIntArray() && BigInt; + case 'p': case 'P': case 's': + switch(ptrSizeof){ + case 4: return Number; + case 8: return affirmBigIntArray() && BigInt; + } + break; + } + toss("Unhandled DataView set wrapper for signature:",s); + }; + + const sPropName = (s,k)=>s+'::'+k; + + const __propThrowOnSet = function(structName,propName){ + return ()=>toss(sPropName(structName,propName),"is read-only."); + }; + + /** + When C code passes a pointer of a bound struct to back into + a JS function via a function pointer struct member, it + arrives in JS as a number (pointer). + StructType.instanceForPointer(ptr) can be used to get the + instance associated with that pointer, and __ptrBacklinks + holds that mapping. WeakMap keys must be objects, so we + cannot use a weak map to map pointers to instances. We use + the StructType constructor as the WeakMap key, mapped to a + plain, prototype-less Object which maps the pointers to + struct instances. That arrangement gives us a + per-StructType type-safe way to resolve pointers. + */ + const __ptrBacklinks = new WeakMap(); + /** + Similar to __ptrBacklinks but is scoped at the StructBinder + level and holds pointer-to-object mappings for all struct + instances created by any struct from any StructFactory + which this specific StructBinder has created. The intention + of this is to help implement more transparent handling of + pointer-type property resolution. + */ + const __ptrBacklinksGlobal = Object.create(null); + + /** + In order to completely hide StructBinder-bound struct + pointers from JS code, we store them in a scope-local + WeakMap which maps the struct-bound objects to their WASM + pointers. The pointers are accessible via + boundObject.pointer, which is gated behind an accessor + function, but are not exposed anywhere else in the + object. The main intention of that is to make it impossible + for stale copies to be made. + */ + const __instancePointerMap = new WeakMap(); + + /** Property name for the pointer-is-external marker. */ + const xPtrPropName = '(pointer-is-external)'; + + /** Frees the obj.pointer memory and clears the pointer + property. */ + const __freeStruct = function(ctor, obj, m){ + if(!m) m = __instancePointerMap.get(obj); + if(m) { + if(obj.ondispose instanceof Function){ + try{obj.ondispose()} + catch(e){ + /*do not rethrow: destructors must not throw*/ + console.warn("ondispose() for",ctor.structName,'@', + m,'threw. NOT propagating it.',e); + } + }else if(Array.isArray(obj.ondispose)){ + obj.ondispose.forEach(function(x){ + try{ + if(x instanceof Function) x.call(obj); + else if('number' === typeof x) dealloc(x); + // else ignore. Strings are permitted to annotate entries + // to assist in debugging. + }catch(e){ + console.warn("ondispose() for",ctor.structName,'@', + m,'threw. NOT propagating it.',e); + } + }); + } + delete obj.ondispose; + delete __ptrBacklinks.get(ctor)[m]; + delete __ptrBacklinksGlobal[m]; + __instancePointerMap.delete(obj); + if(ctor.debugFlags.__flags.dealloc){ + log("debug.dealloc:",(obj[xPtrPropName]?"EXTERNAL":""), + ctor.structName,"instance:", + ctor.structInfo.sizeof,"bytes @"+m); + } + if(!obj[xPtrPropName]) dealloc(m); + } + }; + + /** Returns a skeleton for a read-only property accessor wrapping + value v. */ + const rop = (v)=>{return {configurable: false, writable: false, + iterable: false, value: v}}; + + /** Allocates obj's memory buffer based on the size defined in + DEF.sizeof. */ + const __allocStruct = function(ctor, obj, m){ + let fill = !m; + if(m) Object.defineProperty(obj, xPtrPropName, rop(m)); + else{ + m = alloc(ctor.structInfo.sizeof); + if(!m) toss("Allocation of",ctor.structName,"structure failed."); + } + try { + if(ctor.debugFlags.__flags.alloc){ + log("debug.alloc:",(fill?"":"EXTERNAL"), + ctor.structName,"instance:", + ctor.structInfo.sizeof,"bytes @"+m); + } + if(fill) heap().fill(0, m, m + ctor.structInfo.sizeof); + __instancePointerMap.set(obj, m); + __ptrBacklinks.get(ctor)[m] = obj; + __ptrBacklinksGlobal[m] = obj; + }catch(e){ + __freeStruct(ctor, obj, m); + throw e; + } + }; + /** Gets installed as the memoryDump() method of all structs. */ + const __memoryDump = function(){ + const p = this.pointer; + return p + ? new Uint8Array(heap().slice(p, p+this.structInfo.sizeof)) + : null; + }; + + const __memberKey = (k)=>memberPrefix + k + memberSuffix; + const __memberKeyProp = rop(__memberKey); + + /** + Looks up a struct member in structInfo.members. Throws if found + if tossIfNotFound is true, else returns undefined if not + found. The given name may be either the name of the + structInfo.members key (faster) or the key as modified by the + memberPrefix/memberSuffix settings. + */ + const __lookupMember = function(structInfo, memberName, tossIfNotFound=true){ + let m = structInfo.members[memberName]; + if(!m && (memberPrefix || memberSuffix)){ + // Check for a match on members[X].key + for(const v of Object.values(structInfo.members)){ + if(v.key===memberName){ m = v; break; } + } + if(!m && tossIfNotFound){ + toss(sPropName(structInfo.name,memberName),'is not a mapped struct member.'); + } + } + return m; + }; + + /** + Uses __lookupMember(obj.structInfo,memberName) to find a member, + throwing if not found. Returns its signature, either in this + framework's native format or in Emscripten format. + */ + const __memberSignature = function f(obj,memberName,emscriptenFormat=false){ + if(!f._) f._ = (x)=>x.replace(/[^vipPsjrd]/g,'').replace(/[pPs]/g,'i'); + const m = __lookupMember(obj.structInfo, memberName, true); + return emscriptenFormat ? f._(m.signature) : m.signature; + }; + + /** + Returns the instanceForPointer() impl for the given + StructType constructor. + */ + const __instanceBacklinkFactory = function(ctor){ + const b = Object.create(null); + __ptrBacklinks.set(ctor, b); + return (ptr)=>b[ptr]; + }; + + const __ptrPropDescriptor = { + configurable: false, enumerable: false, + get: function(){return __instancePointerMap.get(this)}, + set: ()=>toss("Cannot assign the 'pointer' property of a struct.") + // Reminder: leaving `set` undefined makes assignments + // to the property _silently_ do nothing. Current unit tests + // rely on it throwing, though. + }; + + /** Impl of X.memberKeys() for StructType and struct ctors. */ + const __structMemberKeys = rop(function(){ + const a = []; + Object.keys(this.structInfo.members).forEach((k)=>a.push(this.memberKey(k))); + return a; + }); + + const __utf8Decoder = new TextDecoder('utf-8'); + const __utf8Encoder = new TextEncoder(); + + /** + Uses __lookupMember() to find the given obj.structInfo key. + Returns that member if it is a string, else returns false. If the + member is not found, throws if tossIfNotFound is true, else + returns false. + */ + const __memberIsString = function(obj,memberName, tossIfNotFound=false){ + const m = __lookupMember(obj.structInfo, memberName, tossIfNotFound); + return (m && 1===m.signature.length && 's'===m.signature[0]) ? m : false; + }; + + /** + Given a member description object, throws if member.signature is + not valid for assigning to or interpretation as a C-style string. + It optimistically assumes that any signature of (i,p,s) is + C-string compatible. + */ + const __affirmCStringSignature = function(member){ + if('s'===member.signature) return; + toss("Invalid member type signature for C-string value:", + JSON.stringify(member)); + }; + + /** + Looks up the given member in obj.structInfo. If it has a + signature of 's' then it is assumed to be a C-style UTF-8 string + and a decoded copy of the string at its address is returned. If + the signature is of any other type, it throws. If an s-type + member's address is 0, `null` is returned. + */ + const __memberToJsString = function f(obj,memberName){ + const m = __lookupMember(obj.structInfo, memberName, true); + __affirmCStringSignature(m); + const addr = obj[m.key]; + //log("addr =",addr,memberName,"m =",m); + if(!addr) return null; + let pos = addr; + const mem = heap(); + for( ; mem[pos]!==0; ++pos ) { + //log("mem[",pos,"]",mem[pos]); + }; + //log("addr =",addr,"pos =",pos); + if(addr===pos) return ""; + return __utf8Decoder.decode(new Uint8Array(mem.buffer, addr, pos-addr)); + }; + + /** + Adds value v to obj.ondispose, creating ondispose, + or converting it to an array, if needed. + */ + const __addOnDispose = function(obj, v){ + if(obj.ondispose){ + if(obj.ondispose instanceof Function){ + obj.ondispose = [obj.ondispose]; + }/*else assume it's an array*/ + }else{ + obj.ondispose = []; + } + obj.ondispose.push(v); + }; + + /** + Allocates a new UTF-8-encoded, NUL-terminated copy of the given + JS string and returns its address relative to heap(). If + allocation returns 0 this function throws. Ownership of the + memory is transfered to the caller, who must eventually pass it + to the configured dealloc() function. + */ + const __allocCString = function(str){ + const u = __utf8Encoder.encode(str); + const mem = alloc(u.length+1); + if(!mem) toss("Allocation error while duplicating string:",str); + const h = heap(); + let i = 0; + for( ; i < u.length; ++i ) h[mem + i] = u[i]; + h[mem + u.length] = 0; + //log("allocCString @",mem," =",u); + return mem; + }; + + /** + Sets the given struct member of obj to a dynamically-allocated, + UTF-8-encoded, NUL-terminated copy of str. It is up to the caller + to free any prior memory, if appropriate. The newly-allocated + string is added to obj.ondispose so will be freed when the object + is disposed. + */ + const __setMemberCString = function(obj, memberName, str){ + const m = __lookupMember(obj.structInfo, memberName, true); + __affirmCStringSignature(m); + /* Potential TODO: if obj.ondispose contains obj[m.key] then + dealloc that value and clear that ondispose entry */ + const mem = __allocCString(str); + obj[m.key] = mem; + __addOnDispose(obj, mem); + return obj; + }; + + /** + Prototype for all StructFactory instances (the constructors + returned from StructBinder). + */ + const StructType = function ctor(structName, structInfo){ + if(arguments[2]!==rop){ + toss("Do not call the StructType constructor", + "from client-level code."); + } + Object.defineProperties(this,{ + //isA: rop((v)=>v instanceof ctor), + structName: rop(structName), + structInfo: rop(structInfo) + }); + }; + + /** + Properties inherited by struct-type-specific StructType instances + and (indirectly) concrete struct-type instances. + */ + StructType.prototype = Object.create(null, { + dispose: rop(function(){__freeStruct(this.constructor, this)}), + lookupMember: rop(function(memberName, tossIfNotFound=true){ + return __lookupMember(this.structInfo, memberName, tossIfNotFound); + }), + memberToJsString: rop(function(memberName){ + return __memberToJsString(this, memberName); + }), + memberIsString: rop(function(memberName, tossIfNotFound=true){ + return __memberIsString(this, memberName, tossIfNotFound); + }), + memberKey: __memberKeyProp, + memberKeys: __structMemberKeys, + memberSignature: rop(function(memberName, emscriptenFormat=false){ + return __memberSignature(this, memberName, emscriptenFormat); + }), + memoryDump: rop(__memoryDump), + pointer: __ptrPropDescriptor, + setMemberCString: rop(function(memberName, str){ + return __setMemberCString(this, memberName, str); + }) + }); + + /** + "Static" properties for StructType. + */ + Object.defineProperties(StructType, { + allocCString: rop(__allocCString), + instanceForPointer: rop((ptr)=>__ptrBacklinksGlobal[ptr]), + isA: rop((v)=>v instanceof StructType), + hasExternalPointer: rop((v)=>(v instanceof StructType) && !!v[xPtrPropName]), + memberKey: __memberKeyProp + }); + + const isNumericValue = (v)=>Number.isFinite(v) || (v instanceof (BigInt || Number)); + + /** + Pass this a StructBinder-generated prototype, and the struct + member description object. It will define property accessors for + proto[memberKey] which read from/write to memory in + this.pointer. It modifies descr to make certain downstream + operations much simpler. + */ + const makeMemberWrapper = function f(ctor,name, descr){ + if(!f._){ + /*cache all available getters/setters/set-wrappers for + direct reuse in each accessor function. */ + f._ = {getters: {}, setters: {}, sw:{}}; + const a = ['i','p','P','s','f','d','v()']; + if(bigIntEnabled) a.push('j'); + a.forEach(function(v){ + //const ir = sigIR(v); + f._.getters[v] = sigDVGetter(v) /* DataView[MethodName] values for GETTERS */; + f._.setters[v] = sigDVSetter(v) /* DataView[MethodName] values for SETTERS */; + f._.sw[v] = sigDVSetWrapper(v) /* BigInt or Number ctor to wrap around values + for conversion */; + }); + const rxSig1 = /^[ipPsjfd]$/, + rxSig2 = /^[vipPsjfd]\([ipPsjfd]*\)$/; + f.sigCheck = function(obj, name, key,sig){ + if(Object.prototype.hasOwnProperty.call(obj, key)){ + toss(obj.structName,'already has a property named',key+'.'); + } + rxSig1.test(sig) || rxSig2.test(sig) + || toss("Malformed signature for", + sPropName(obj.structName,name)+":",sig); + }; + } + const key = ctor.memberKey(name); + f.sigCheck(ctor.prototype, name, key, descr.signature); + descr.key = key; + descr.name = name; + const sizeOf = sigSizeof(descr.signature); + const sigGlyph = sigLetter(descr.signature); + const xPropName = sPropName(ctor.prototype.structName,key); + const dbg = ctor.prototype.debugFlags.__flags; + /* + TODO?: set prototype of descr to an object which can set/fetch + its prefered representation, e.g. conversion to string or mapped + function. Advantage: we can avoid doing that via if/else if/else + in the get/set methods. + */ + const prop = Object.create(null); + prop.configurable = false; + prop.enumerable = false; + prop.get = function(){ + if(dbg.getter){ + log("debug.getter:",f._.getters[sigGlyph],"for", sigIR(sigGlyph), + xPropName,'@', this.pointer,'+',descr.offset,'sz',sizeOf); + } + let rc = ( + new DataView(heap().buffer, this.pointer + descr.offset, sizeOf) + )[f._.getters[sigGlyph]](0, isLittleEndian); + if(dbg.getter) log("debug.getter:",xPropName,"result =",rc); + if(rc && isAutoPtrSig(descr.signature)){ + rc = StructType.instanceForPointer(rc) || rc; + if(dbg.getter) log("debug.getter:",xPropName,"resolved =",rc); + } + return rc; + }; + if(descr.readOnly){ + prop.set = __propThrowOnSet(ctor.prototype.structName,key); + }else{ + prop.set = function(v){ + if(dbg.setter){ + log("debug.setter:",f._.setters[sigGlyph],"for", sigIR(sigGlyph), + xPropName,'@', this.pointer,'+',descr.offset,'sz',sizeOf, v); + } + if(!this.pointer){ + toss("Cannot set struct property on disposed instance."); + } + if(null===v) v = 0; + else while(!isNumericValue(v)){ + if(isAutoPtrSig(descr.signature) && (v instanceof StructType)){ + // It's a struct instance: let's store its pointer value! + v = v.pointer || 0; + if(dbg.setter) log("debug.setter:",xPropName,"resolved to",v); + break; + } + toss("Invalid value for pointer-type",xPropName+'.'); + } + ( + new DataView(heap().buffer, this.pointer + descr.offset, sizeOf) + )[f._.setters[sigGlyph]](0, f._.sw[sigGlyph](v), isLittleEndian); + }; + } + Object.defineProperty(ctor.prototype, key, prop); + }/*makeMemberWrapper*/; + + /** + The main factory function which will be returned to the + caller. + */ + const StructBinder = function StructBinder(structName, structInfo){ + if(1===arguments.length){ + structInfo = structName; + structName = structInfo.name; + }else if(!structInfo.name){ + structInfo.name = structName; + } + if(!structName) toss("Struct name is required."); + let lastMember = false; + Object.keys(structInfo.members).forEach((k)=>{ + const m = structInfo.members[k]; + if(!m.sizeof) toss(structName,"member",k,"is missing sizeof."); + else if(0!==(m.sizeof%4)){ + toss(structName,"member",k,"sizeof is not aligned."); + } + else if(0!==(m.offset%4)){ + toss(structName,"member",k,"offset is not aligned."); + } + if(!lastMember || lastMember.offset < m.offset) lastMember = m; + }); + if(!lastMember) toss("No member property descriptions found."); + else if(structInfo.sizeof < lastMember.offset+lastMember.sizeof){ + toss("Invalid struct config:",structName, + "max member offset ("+lastMember.offset+") ", + "extends past end of struct (sizeof="+structInfo.sizeof+")."); + } + const debugFlags = rop(SBF.__makeDebugFlags(StructBinder.debugFlags)); + /** Constructor for the StructCtor. */ + const StructCtor = function StructCtor(externalMemory){ + if(!(this instanceof StructCtor)){ + toss("The",structName,"constructor may only be called via 'new'."); + }else if(arguments.length){ + if(externalMemory!==(externalMemory|0) || externalMemory<=0){ + toss("Invalid pointer value for",structName,"constructor."); + } + __allocStruct(StructCtor, this, externalMemory); + }else{ + __allocStruct(StructCtor, this); + } + }; + Object.defineProperties(StructCtor,{ + debugFlags: debugFlags, + disposeAll: rop(function(){ + const map = __ptrBacklinks.get(StructCtor); + Object.keys(map).forEach(function(ptr){ + const b = map[ptr]; + if(b) __freeStruct(StructCtor, b, ptr); + }); + __ptrBacklinks.set(StructCtor, Object.create(null)); + return StructCtor; + }), + instanceForPointer: rop(__instanceBacklinkFactory(StructCtor)), + isA: rop((v)=>v instanceof StructCtor), + memberKey: __memberKeyProp, + memberKeys: __structMemberKeys, + resolveToInstance: rop(function(v, throwIfNot=false){ + if(!(v instanceof StructCtor)){ + v = Number.isSafeInteger(v) + ? StructCtor.instanceForPointer(v) : undefined; + } + if(!v && throwIfNot) toss("Value is-not-a",StructCtor.structName); + return v; + }), + methodInfoForKey: rop(function(mKey){ + }), + structInfo: rop(structInfo), + structName: rop(structName) + }); + StructCtor.prototype = new StructType(structName, structInfo, rop); + Object.defineProperties(StructCtor.prototype,{ + debugFlags: debugFlags, + constructor: rop(StructCtor) + /*if we assign StructCtor.prototype and don't do + this then StructCtor!==instance.constructor!*/ + }); + Object.keys(structInfo.members).forEach( + (name)=>makeMemberWrapper(StructCtor, name, structInfo.members[name]) + ); + return StructCtor; + }; + StructBinder.instanceForPointer = StructType.instanceForPointer; + StructBinder.StructType = StructType; + StructBinder.config = config; + StructBinder.allocCString = __allocCString; + if(!StructBinder.debugFlags){ + StructBinder.debugFlags = SBF.__makeDebugFlags(SBF.debugFlags); + } + return StructBinder; +}/*StructBinderFactory*/; diff --git a/ext/wasm/jaccwabyt/jaccwabyt.md b/ext/wasm/jaccwabyt/jaccwabyt.md new file mode 100644 index 0000000000..2bb39e636f --- /dev/null +++ b/ext/wasm/jaccwabyt/jaccwabyt.md @@ -0,0 +1,1078 @@ +Jaccwabyt 🐇 +============================================================ + +**Jaccwabyt**: _JavaScript ⇄ C Struct Communication via WASM Byte +Arrays_ + + +Welcome to Jaccwabyt, a JavaScript API which creates bindings for +WASM-compiled C structs, defining them in such a way that changes to +their state in JS are visible in C/WASM, and vice versa, permitting +two-way interchange of struct state with very little user-side +friction. + +(If that means nothing to you, neither will the rest of this page!) + +**Browser compatibility**: this library requires a _recent_ browser +and makes no attempt whatsoever to accommodate "older" or +lesser-capable ones, where "recent," _very roughly_, means released in +mid-2018 or later, with late 2021 releases required for some optional +features in some browsers (e.g. [BigInt64Array][] in Safari). It also +relies on a couple non-standard, but widespread, features, namely +[TextEncoder][] and [TextDecoder][]. It is developed primarily on +Firefox and Chrome on Linux and all claims of Safari compatibility +are based solely on feature compatibility tables provided at +[MDN][]. + +**Formalities:** + +- Author: [Stephan Beal][sgb] +- License: Public Domain +- Project Home: + + +Table of Contents +============================================================ + +- [Overview](#overview) + - [Architecture](#architecture) +- [Creating and Binding Structs](#creating-binding) + - [Step 1: Configure Jaccwabyt](#step-1) + - [Step 2: Struct Description](#step-2) + - [`P` vs `p`](#step-2-pvsp) + - [Step 3: Binding a Struct](#step-3) + - [Step 4: Creating, Using, and Destroying Instances](#step-4) +- APIs + - [Struct Binder Factory](#api-binderfactory) + - [Struct Binder](#api-structbinder) + - [Struct Type](#api-structtype) + - [Struct Constructors](#api-structctor) + - [Struct Protypes](#api-structprototype) + - [Struct Instances](#api-structinstance) +- Appendices + - [Appendix A: Limitations, TODOs, etc.](#appendix-a) + - [Appendix D: Debug Info](#appendix-d) + - [Appendix G: Generating Struct Descriptions](#appendix-g) + + +Overview +============================================================ + +Management summary: this JavaScript-only framework provides limited +two-way bindings between C structs and JavaScript objects, such that +changes to the struct in one environment are visible in the other. + +Details... + +It works by creating JavaScript proxies for C structs. Reads and +writes of the JS-side members are marshaled through a flat byte array +allocated from the WASM heap. As that heap is shared with the C-side +code, and the memory block is written using the same approach C does, +that byte array can be used to access and manipulate a given struct +instance from both JS and C. + +Motivating use case: this API was initially developed as an +experiment to determine whether it would be feasible to implement, +completely in JS, custom "VFS" and "virtual table" objects for the +WASM build of [sqlite3][]. Doing so was going to require some form of +two-way binding of several structs. Once the proof of concept was +demonstrated, a rabbit hole appeared and _down we went_... It has +since grown beyond its humble proof-of-concept origins and is believed +to be a useful (or at least interesting) tool for mixed JS/C +applications. + +Portability notes: + +- These docs sometimes use [Emscripten][] as a point of reference + because it is the most widespread WASM toolchain, but this code is + specifically designed to be usable in arbitrary WASM environments. + It abstracts away a few Emscripten-specific features into + configurable options. Similarly, the build tree requires Emscripten + but Jaccwabyt does not have any hard Emscripten dependencies. +- This code is encapsulated into a single JavaScript function. It + should be trivial to copy/paste into arbitrary WASM/JS-using + projects. +- The source tree includes C code, but only for testing and + demonstration purposes. It is not part of the core distributable. + + +Architecture +------------------------------------------------------------ + + + +```pikchr +BSBF: box rad 0.3*boxht "StructBinderFactory" fit fill lightblue +BSB: box same "StructBinder" fit at 0.75 e of 0.7 s of BSBF.c +BST: box same "StructType" fit at 1.5 e of BSBF +BSC: box same "Struct" "Ctor" fit at 1.5 s of BST +BSI: box same "Struct" "Instances" fit at 1 right of BSB.e +BC: box same at 0.25 right of 1.6 e of BST "C Structs" fit fill lightgrey + +arrow -> from BSBF.s to BSB.w "Generates" aligned above +arrow -> from BSB.n to BST.sw "Contains" aligned above +arrow -> from BSB.s to BSC.nw "Generates" aligned below +arrow -> from BSC.ne to BSI.s "Constructs" aligned below +arrow <- from BST.se to BSI.n "Inherits" aligned above +arrow <-> from BSI.e to BC.s dotted "Shared" aligned above "Memory" aligned below +arrow -> from BST.e to BC.w dotted "Mirrors Struct" aligned above "Model From" aligned below +arrow -> from BST.s to BSC.n "Prototype of" aligned above +``` + +Its major classes and functions are: + +- **[StructBinderFactory][StructBinderFactory]** is a factory function which + accepts a configuration object to customize it for a given WASM + environment. A client will typically call this only one time, with + an appropriate configuration, to generate a single... +- **[StructBinder][]** is a factory function which converts an + arbitrary number struct descriptions into... +- **[StructTypes][StructCtors]** are constructors, one per struct + description, which inherit from + **[`StructBinder.StructType`][StructType]** and are used to instantiate... +- **[Struct instances][StructInstance]** are objects representing + individual instances of generated struct types. + +An app may have any number of StructBinders, but will typically +need only one. Each StructBinder is effectively a separate +namespace for struct creation. + + + +Creating and Binding Structs +============================================================ + +From the amount of documentation provided, it may seem that +creating and using struct bindings is a daunting task, but it +essentially boils down to: + +1. [Confire Jaccwabyt for your WASM environment](#step-1). This is a + one-time task per project and results is a factory function which + can create new struct bindings. +2. [Create a JSON-format description of your C structs](#step-2). This is + required once for each struct and required updating if the C + structs change. +3. [Feed (2) to the function generated by (1)](#step-3) to create JS + constuctor functions for each struct. This is done at runtime, as + opposed to during a build-process step, and can be set up in such a + way that it does not require any maintenace after its initial + setup. +4. [Create and use instances of those structs](#step-4). + +Detailed instructions for each of those steps follows... + + +Step 1: Configure Jaccwabyt for the Environment +------------------------------------------------------------ + +Jaccwabyt's highest-level API is a single function. It creates a +factory for processing struct descriptions, but does not process any +descriptions itself. This level of abstraction exist primarily so that +the struct-specific factories can be configured for a given WASM +environment. Its usage looks like: + +> +```javascript +const MyBinder = StructBinderFactory({ + // These config options are all required: + heap: WebAssembly.Memory instance or a function which returns + a Uint8Array or Int8Array view of the WASM memory, + alloc: function(howMuchMemory){...}, + dealloc: function(pointerToFree){...} +}); +``` + +It also offers a number of other settings, but all are optional except +for the ones shown above. Those three config options abstract away +details which are specific to a given WASM environment. They provide +the WASM "heap" memory (a byte array), the memory allocator, and the +deallocator. In a conventional Emscripten setup, that config might +simply look like: + +> +```javascript +{ + heap: Module['asm']['memory'], + //Or: + // heap: ()=>Module['HEAP8'], + alloc: (n)=>Module['_malloc'](n), + dealloc: (m)=>Module['_free'](m) +} +``` + +The StructBinder factory function returns a function which can then be +used to create bindings for our structs. + + + +Step 2: Create a Struct Description +------------------------------------------------------------ + +The primary input for this framework is a JSON-compatible construct +which describes a struct we want to bind. For example, given this C +struct: + +> +```c +// C-side: +struct Foo { + int member1; + void * member2; + int64_t member3; +}; +``` + +Its JSON description looks like: + +> +```json +{ + "name": "Foo", + "sizeof": 16, + "members": { + "member1": {"offset": 0,"sizeof": 4,"signature": "i"}, + "member2": {"offset": 4,"sizeof": 4,"signature": "p"}, + "member3": {"offset": 8,"sizeof": 8,"signature": "j"} + } +} +``` + +These data _must_ match up with the C-side definition of the struct +(if any). See [Appendix G][appendix-g] for one way to easily generate +these from C code. + +Each entry in the `members` object maps the member's name to +its low-level layout: + +- `offset`: the byte offset from the start of the struct, as reported + by C's `offsetof()` feature. +- `sizeof`: as reported by C's `sizeof()`. +- `signature`: described below. +- `readOnly`: optional. If set to true, the binding layer will + throw if JS code tries to set that property. + +The order of the `members` entries is not important: their memory +layout is determined by their `offset` and `sizeof` members. The +`name` property is technically optional, but one of the steps in the +binding process requires that either it be passed an explicit name or +there be one in the struct description. The names of the `members` +entries need not match their C counterparts. Project conventions may +call for giving them different names in the JS side and the +[StructBinderFactory][] can be configured to automatically add a +prefix and/or suffix to their names. + +Nested structs are as-yet unsupported by this tool. + +Struct member "signatures" describe the data types of the members and +are an extended variant of the format used by Emscripten's +`addFunction()`. A signature for a non-function-pointer member, or +function pointer member which is to be modelled as an opaque pointer, +is a single letter. A signature for a function pointer may also be +modelled as a series of letters describing the call signature. The +supported letters are: + +- **`v`** = `void` (only used as return type for function pointer members) +- **`i`** = `int32` (4 bytes) +- **`j`** = `int64` (8 bytes) is only really usable if this code is built + with BigInt support (e.g. using the Emscripten `-sWASM_BIGINT` build + flag). Without that, this API may throw when encountering the `j` + signature entry. +- **`f`** = `float` (4 bytes) +- **`d`** = `double` (8 bytes) +- **`p`** = `int32` (but see below!) +- **`P`** = Like `p` but with extra handling. Described below. +- **`s`** = like `int32` but is a _hint_ that it's a pointer to a string + so that _some_ (very limited) contexts may treat it as such, noting + such algorithms must, for lack of information to the contrary, + assume both that the encoding is UTF-8 and that the pointer's member + is NUL-terminated. If that is _not_ the case for a given string + member, do not use `s`: use `i` or `p` instead and do any string + handling yourself. + +Noting that: + +- All of these types are numeric. Attempting to set any struct-bound + property to a non-numeric value will trigger an exception except in + cases explicitly noted otherwise. + +> Sidebar: Emscripten's public docs do not mention `p`, but their +generated code includes `p` as an alias for `i`, presumably to mean +"pointer". Though `i` is legal for pointer types in the signature, `p` +is more descriptive, so this framework encourages the use of `p` for +pointer-type members. Using `p` for pointers also helps future-proof +the signatures against the eventuality that WASM eventually supports +64-bit pointers. Note that sometimes `p` really means +pointer-to-pointer, but the Emscripten JS/WASM glue does not offer +that level of expressiveness in these signatures. We simply have to be +aware of when we need to deal with pointers and pointers-to-pointers +in JS code. + +> Trivia: this API treates `p` as distinctly different from `i` in +some contexts, so its use is encouraged for pointer types. + +Signatures in the form `x(...)` denote function-pointer members and +`x` denotes non-function members. Functions with no arguments use the +form `x()`. For function-type signatures, the strings are formulated +such that they can be passed to Emscripten's `addFunction()` after +stripping out the `(` and `)` characters. For good measure, to match +the public Emscripten docs, `p` should also be replaced with `i`. In +JavaScript that might look like: + +> +``` +signature.replace(/[^vipPsjfd]/g,'').replace(/[pPs]/g,'i'); +``` + + +### `P` vs `p` in Method Signatures + +*This support is experimental and subject to change.* + +The method signature letter `p` means "pointer," which, in WASM, means +"integer." `p` is treated as an integer for most contexts, while still +also being a separate type (analog to how pointers in C are just a +special use of unsigned numbers). A capital `P` changes the semantics +of plain member pointers (but not, as of this writing, function +pointer members) as follows: + +- When a `P`-type member is **fetched** via `myStruct.x` and its value is + a non-0 integer, [`StructBinder.instanceForPointer()`][StructBinder] + is used to try to map that pointer to a struct instance. If a match + is found, the "get" operation returns that instance instead of the + integer. If no match is found, it behaves exactly as for `p`, returning + the integer value. +- When a `P`-type member is **set** via `myStruct.x=y`, if + [`(y instanceof StructType)`][StructType] then the value of `y.pointer` is + stored in `myStruct.x`. If `y` is neither a number nor + a [StructType][], an exception is triggered (regardless of whether + `p` or `P` is used). + + + +Step 3: Binding the Struct +------------------------------------------------------------ + +We can now use the results of steps 1 and 2: + +> +```javascript +const MyStruct = MyBinder(myStructDescription); +``` + +That creates a new constructor function, `MyStruct`, which can be used +to instantiate new instances. The binder will throw if it encounters +any problems. + +That's all there is to it. + +> Sidebar: that function may modify the struct description object +and/or its sub-objects, or may even replace sub-objects, in order to +simplify certain later operations. If that is not desired, then feed +it a copy of the original, e.g. by passing it +`JSON.parse(JSON.stringify(structDefinition))`. + + +Step 4: Creating, Using, and Destroying Struct Instances +------------------------------------------------------------ + +Now that we have our constructor... + +> +```javascript +const my = new MyStruct(); +``` + +It is important to understand that creating a new instance allocates +memory on the WASM heap. We must not simply rely on garbage collection +to clean up the instances because doing so will not free up the WASM +heap memory. The correct way to free up that memory is to use the +object's `dispose()` method. Alternately, there is a "nuclear option": +`MyBinder.disposeAll()` will free the memory allocated for _all_ +instances which have not been manually disposed. + +The following usage pattern offers one way to easily ensure proper +cleanup of struct instances: + + +> +```javascript +const my = new MyStruct(); +try { + console.log(my.member1, my.member2, my.member3); + my.member1 = 12; + assert(12 === my.member1); + /* ^^^ it may seem silly to test that, but recall that assigning that + property encodes the value into a byte array in heap memory, not + a normal JS property. Similarly, fetching the property decodes it + from the byte array. */ + // Pass the struct to C code which takes a MyStruct pointer: + aCFunction( my.pointer ); + // Type-safely check if a pointer returned from C is a MyStruct: + const x = MyStruct.instanceForPointer( anotherCFunction() ); + // If it is a MyStruct, x now refers to that object. Note, however, + // that this only works for instances created in JS, as the + // pointer mapping only exists in JS space. +} finally { + my.dispose(); +} +``` + +> Sidebar: the `finally` block will be run no matter how the `try` +exits, whether it runs to completion, propagates an exception, or uses +flow-control keywords like `return` or `break`. It is perfectly legal +to use `try`/`finally` without a `catch`, and doing so is an ideal +match for the memory management requirements of Jaccwaby-bound struct +instances. + +Now that we have struct instances, there are a number of things we +can do with them, as covered in the rest of this document. + + + +API Reference +============================================================ + + +API: Binder Factory +------------------------------------------------------------ + +This is the top-most function of the API, from which all other +functions and types are generated. The binder factory's signature is: + +> +``` +Function StructBinderFactory(object configOptions); +``` + +It returns a function which these docs refer to as a [StructBinder][] +(covered in the next section). It throws on error. + +The binder factory supports the following options in its +configuration object argument: + + +- `heap` + Must be either a `WebAssembly.Memory` instance representing the WASM + heap memory OR a function which returns an Int8Array or Uint8Array + view of the WASM heap. In the latter case the function should, if + appropriate for the environment, account for the heap being able to + grow. Jaccwabyt uses this property in such a way that it "should" be + okay for the WASM heap to grow at runtime (that case is, however, + untested). + +- `alloc` + Must be a function semantically compatible with Emscripten's + `Module._malloc()`. That is, it is passed the number of bytes to + allocate and it returns a pointer. On allocation failure it may + either return 0 or throw an exception. This API will throw an + exception if allocation fails or will propagate whatever exception + the allocator throws. The allocator _must_ use the same heap as the + `heap` config option. + +- `dealloc` + Must be a function semantically compatible with Emscripten's + `Module._free()`. That is, it takes a pointer returned from + `alloc()` and releases that memory. It must never throw and must + accept a value of 0/null to mean "do nothing" (noting that 0 is + _technically_ a legal memory address in WASM, but that seems like a + design flaw). + +- `bigIntEnabled` (bool=true if BigInt64Array is available, else false) + If true, the WASM bits this code is used with must have been + compiled with int64 support (e.g. using Emscripten's `-sWASM_BIGINT` + flag). If that's not the case, this flag should be set to false. If + it's enabled, BigInt support is assumed to work and certain extra + features are enabled. Trying to use features which requires BigInt + when it is disabled (e.g. using 64-bit integer types) will trigger + an exception. + +- `memberPrefix` and `memberSuffix` (string="") + If set, struct-defined properties get bound to JS with this string + as a prefix resp. suffix. This can be used to avoid symbol name + collisions between the struct-side members and the JS-side ones + and/or to make more explicit which object-level properties belong to + the struct mapping and which to the JS side. This does not modify + the values in the struct description objects, just the property + names through which they are accessed via property access operations + and the various a [StructInstance][] APIs (noting that the latter + tend to permit both the original names and the names as modified by + these settings). + +- `log` + Optional function used for debugging output. By default + `console.log` is used but by default no debug output is generated. + This API assumes that the function will space-separate each argument + (like `console.log` does). See [Appendix D](#appendix-d) for info + about enabling debugging output. + + + +API: Struct Binder +------------------------------------------------------------ + +Struct Binders are factories which are created by the +[StructBinderFactory][]. A given Struct Binder can process any number +of distinct structs. In a typical setup, an app will have ony one +shared Binder Factory and one Struct Binder. Struct Binders which are +created via different [StructBinderFactory][] calls are unrelated to each +other, sharing no state except, perhaps, indirectly via +[StructBinderFactory][] configuration (e.g. the memory heap). + +These factories have two call signatures: + +> +```javascript +Function StructBinder([string structName,] object structDescription) +``` + +If the struct description argument has a `name` property then the name +argument is optional, otherwise it is required. + +The returned object is a constructor for instances of the struct +described by its argument(s), each of which derives from +a separate [StructType][] instance. + +The Struct Binder has the following members: + +- `allocCString(str)` + Allocates a new UTF-8-encoded, NUL-terminated copy of the given JS + string and returns its address relative to `config.heap()`. If + allocation returns 0 this function throws. Ownership of the memory + is transfered to the caller, who must eventually pass it to the + configured `config.dealloc()` function. + +- `config` + The configuration object passed to the [StructBinderFactory][], + primarily for accessing the memory (de)allocator and memory. Modifying + any of its "significant" configuration values may have undefined + results. + +- `instanceForPointer(pointer)` + Given a pointer value relative to `config.memory`, if that pointer + resolves to a struct of _any type_ generated via the same Struct + Binder, this returns the struct instance associated with it, or + `undefined` if no struct object is mapped to that pointer. This + differs from the struct-type-specific member of the same name in + that this one is not "type-safe": it does not know the type of the + returned object (if any) and may return a struct of any + [StructType][] for which this Struct Binder has created a + constructor. It cannot return instances created via a different + [StructBinderFactory][] because each factory can hypothetically have + a different memory heap. + + + +API: Struct Type +------------------------------------------------------------ + +The StructType class is a property of the [StructBinder][] function. + +Each constructor created by a [StructBinder][] inherits from _its own +instance_ of the StructType class, which contains state specific to +that struct type (e.g. the struct name and description metadata). +StructTypes which are created via different [StructBinder][] instances +are unrelated to each other, sharing no state except [StructBinderFactory][] +config options. + +The StructType constructor cannot be called from client code. It is +only called by the [StructBinder][]-generated +[constructors][StructCtors]. The `StructBinder.StructType` object +has the following "static" properties (^Which are accessible from +individual instances via `theInstance.constructor`.): + +- `allocCString(str)` + Identical to the [StructBinder][] method of the same name. + +- `hasExternalPointer(object)` + Returns true if the given object's `pointer` member refers to an + "external" object. That is the case when a pointer is passed to a + [struct's constructor][StructCtors]. If true, the memory is owned by + someone other than the object and must outlive the object. + +- `instanceForPointer(pointer)` + Works identically to the [StructBinder][] method of the same name. + +- `isA(value)` + Returns true if its argument is a StructType instance _from the same + [StructBinder][]_ as this StructType. + +- `memberKey(string)` + Returns the given string wrapped in the configured `memberPrefix` + and `memberSuffix` values. e.g. if passed `"x"` and `memberPrefix` + is `"$"` then it returns `"$x"`. This does not verify that the + property is actually a struct a member, it simply transforms the + given string. TODO(?): add a 2nd parameter indicating whether it + should validate that it's a known member name. + +The base StructType prototype has the following members, all of which +are inherited by [struct instances](#api-structinstance) and may only +legally be called on concrete struct instances unless noted otherwise: + +- `dispose()` + Frees, if appropriate, the WASM-allocated memory which is allocated + by the constructor. If this is not called before the JS engine + cleans up the object, a leak in the WASM heap memory pool will result. + When `dispose()` is called, if the object has a property named `ondispose` + then it is treated as follows: + - If it is a function, it is called with the struct object as its `this`. + That method must not throw - if it does, the exception will be + ignored. + - If it is an array, it may contain functions, pointers, and/or JS + strings. If an entry is a function, it is called as described + above. If it's a number, it's assumed to be a pointer and is + passed to the `dealloc()` function configured for the parent + [StructBinder][]. If it's a JS string, it's assumed to be a + helpful description of the next entry in the list and is simply + ignored. Strings are supported primarily for use as debugging + information. + - Some struct APIs will manipulate the `ondispose` member, creating + it as an array or converting it from a function to array as + needed. + +- `lookupMember(memberName,throwIfNotFound=true)` + Given the name of a mapped struct member, it returns the member + description object. If not found, it either throws (if the 2nd + argument is true) or returns `undefined` (if the second argument is + false). The first argument may be either the member name as it is + mapped in the struct description or that same name with the + configured `memberPrefix` and `memberSuffix` applied, noting that + the lookup in the former case is faster.\ + This method may be called directly on the prototype, without a + struct instance. + +- `memberToJsString(memberName)` + Uses `this.lookupMember(memberName,true)` to look up the given + member. If its signature is `s` then it is assumed to refer to a + NUL-terminated, UTF-8-encoded string and its memory is decoded as + such. If its signature is not one of those then an exception is + thrown. If its address is 0, `null` is returned. See also: + `setMemberCString()`. + +- `memberIsString(memberName [,throwIfNotFound=true])` + Uses `this.lookupMember(memberName,throwIfNotFound)` to look up the + given member. Returns the member description object if the member + has a signature of `s`, else returns false. If the given member is + not found, it throws if the 2nd argument is true, else it returns + false. + +- `memberKey(string)` + Works identically to `StructBinder.StructType.memberKey()`. + +- `memberKeys()` + Returns an array of the names of the properties of this object + which refer to C-side struct counterparts. + +- `memberSignature(memberName [,emscriptenFormat=false])` + Returns the signature for a given a member property, either in this + framework's format or, if passed a truthy 2nd argument, in a format + suitable for the 2nd argument to Emscripten's `addFunction()`. + Throws if the first argument does not resolve to a struct-bound + member name. The member name is resolved using `this.lookupMember()` + and throws if the member is found mapped. + +- `memoryDump()` + Returns a Uint8Array which contains the current state of this + object's raw memory buffer. Potentially useful for debugging, but + not much else. Note that the memory is necessarily, for + compatibility with C, written in the host platform's endianness and + is thus not useful as a persistent/portable serialization format. + +- `setMemberCString(memberName,str)` + Uses `StructType.allocCString()` to allocate a new C-style string, + assign it to the given member, and add the new string to this + object's `ondispose` list for cleanup when `this.dispose()` is + called. This function throws if `lookupMember()` fails for the given + member name, if allocation of the string fails, or if the member has + a signature value of anything other than `s`. Returns `this`. + *Achtung*: calling this repeatedly will not immediately free the + previous values because this code cannot know whether they are in + use in other places, namely C. Instead, each time this is called, + the prior value is retained in the `ondispose` list for cleanup when + the struct is disposed of. Because of the complexities and general + uncertainties of memory ownership and lifetime in such + constellations, it is recommended that the use of C-string members + from JS be kept to a minimum or that the relationship be one-way: + let C manage the strings and only fetch them from JS using, e.g., + `memberToJsString()`. + + + +API: Struct Constructors +------------------------------------------------------------ + +Struct constructors (the functions returned from [StructBinder][]) +are used for, intuitively enough, creating new instances of a given +struct type: + +> +``` +const x = new MyStruct; +``` + +Normally they should be passed no arguments, but they optionally +accept a single argument: a WASM heap pointer address of memory +which the object will use for storage. It does _not_ take over +ownership of that memory and that memory must be valid at +for least as long as this struct instance. This is used, for example, +to proxy static/shared C-side instances: + +> +``` +const x = new MyStruct( someCFuncWhichReturnsAMyStructPointer() ); +... +x.dispose(); // does NOT free the memory +``` + +The JS-side construct does not own the memory in that case and has no +way of knowing when the C-side struct is destroyed. Results are +specifically undefined if the JS-side struct is used after the C-side +struct's member is freed. + +> Potential TODO: add a way of passing ownership of the C-side struct +to the JS-side object. e.g. maybe simply pass `true` as the second +argument to tell the constructor to take over ownership. Currently the +pointer can be taken over using something like +`myStruct.ondispose=[myStruct.pointer]` immediately after creation. + +These constructors have the following "static" members: + +- `disposeAll()` + For each instance of this struct, the equivalent of its `dispose()` + method is called. This frees all WASM-allocated memory associated + with _all_ instances and clears the `instanceForPointer()` + mappings. Returns `this`. + +- `instanceForPointer(pointer)` + Given a pointer value (accessible via the `pointer` property of all + struct instances) which ostensibly refers to an instance of this + class, this returns the instance associated with it, or `undefined` + if no object _of this specific struct type_ is mapped to that + pointer. When C-side code calls back into JS code and passes a + pointer to an object, this function can be used to type-safely + "cast" that pointer back to its original object. + +- `isA(value)` + Returns true if its argument was created by this constructor. + +- `memberKey(string)` + Works exactly as documented for [StructType][]. + +- `memberKeys(string)` + Works exactly as documented for [StructType][]. + +- `resolveToInstance(value [,throwIfNot=false])` + Works like `instanceForPointer()` but accepts either an instance + of this struct type or a pointer which resolves to one. + It returns an instance of this struct type on success. + By default it returns a falsy value if its argument is not, + or does not resolve to, an instance of this struct type, + but if passed a truthy second argument then it will throw + instead. + +- `structInfo` + The structure description passed to [StructBinder][] when this + constructor was generated. + +- `structName` + The structure name passed to [StructBinder][] when this constructor + was generated. + + + +API: Struct Prototypes +------------------------------------------------------------ + +The prototypes of structs created via [the constructors described in +the previous section][StructCtors] are each a struct-type-specific +instance of [StructType][] and add the following struct-type-specific +properties to the mix: + +- `structInfo` + The struct description metadata, as it was given to the + [StructBinder][] which created this class. + +- `structName` + The name of the struct, as it was given to the [StructBinder][] which + created this class. + + +API: Struct Instances +------------------------------------------------------------------------ + +Instances of structs created via [the constructors described +above][StructCtors] each have the following instance-specific state in +common: + +- `pointer` + A read-only numeric property which is the "pointer" returned by the + configured allocator when this object is constructed. After + `dispose()` (inherited from [StructType][]) is called, this property + has the `undefined` value. When passing instances of this struct to + C-bound code, `pointer` is the value which must be passed in place + of a C-side struct pointer. When calling C-side code which takes a + pointer to a struct of this type, simply pass it `myStruct.pointer`. + + +Appendices +============================================================ + + +Appendix A: Limitations, TODOs, and Non-TODOs +------------------------------------------------------------ + +- This library only supports the basic set of member types supported + by WASM: numbers (which includes pointers). Nested structs are not + handled except that a member may be a _pointer_ to such a + struct. Whether or not it ever will depends entirely on whether its + developer ever needs that support. Conversion of strings between + JS and C requires infrastructure specific to each WASM environment + and is not directly supported by this library. + +- Binding functions to struct instances, such that C can see and call + JS-defined functions, is not as transparent as it really could be, + due to [shortcomings in the Emscripten + `addFunction()`/`removeFunction()` + interfaces](https://github.com/emscripten-core/emscripten/issues/17323). Until + a replacement for that API can be written, this support will be + quite limited. It _is_ possible to bind a JS-defined function to a + C-side function pointer and call that function from C. What's + missing is easier-to-use/more transparent support for doing so. + - In the meantime, a [standalone + subproject](/file/common/whwasmutil.js) of Jaccwabyt provides such a + binding mechanism, but integrating it directly with Jaccwabyt would + not only more than double its size but somehow feels inappropriate, so + experimentation is in order for how to offer that capability via + completely optional [StructBinderFactory][] config options. + +- It "might be interesting" to move access of the C-bound members into + a sub-object. e.g., from JS they might be accessed via + `myStructInstance.s.structMember`. The main advantage is that it would + eliminate any potential confusion about which members are part of + the C struct and which exist purely in JS. "The problem" with that + is that it requires internally mapping the `s` member back to the + object which contains it, which makes the whole thing more costly + and adds one more moving part which can break. Even so, it's + something to try out one rainy day. Maybe even make it optional and + make the `s` name configurable via the [StructBinderFactory][] + options. (Over-engineering is an arguably bad habit of mine.) + +- It "might be interesting" to offer (de)serialization support. It + would be very limited, e.g. we can't serialize arbitrary pointers in + any meaningful way, but "might" be useful for structs which contain + only numeric or C-string state. As it is, it's easy enough for + client code to write wrappers for that and handle the members in + ways appropriate to their apps. Any impl provided in this library + would have the shortcoming that it may inadvertently serialize + pointers (since they're just integers), resulting in potential chaos + after deserialization. Perhaps the struct description can be + extended to tag specific members as serializable and how to + serialize them. + + +Appendix D: Debug Info +------------------------------------------------------------ + +The [StructBinderFactory][], [StructBinder][], and [StructType][] classes +all have the following "unsupported" method intended primarily +to assist in their own development, as opposed to being for use in +client code: + +- `debugFlags(flags)` (integer) + An "unsupported" debugging option which may change or be removed at + any time. Its argument is a set of flags to enable/disable certain + debug/tracing output for property accessors: 0x01 for getters, 0x02 + for setters, 0x04 for allocations, 0x08 for deallocations. Pass 0 to + disable all flags and pass a negative value to _completely_ clear + all flags. The latter has the side effect of telling the flags to be + inherited from the next-higher-up class in the hierarchy, with + [StructBinderFactory][] being top-most, followed by [StructBinder][], then + [StructType][]. + + + +Appendix G: Generating Struct Descriptions From C +------------------------------------------------------------ + +Struct definitions are _ideally_ generated from WASM-compiled C, as +opposed to simply guessing the sizeofs and offsets, so that the sizeof +and offset information can be collected using C's `sizeof()` and +`offsetof()` features (noting that struct padding may impact offsets +in ways which might not be immediately obvious, so writing them by +hand is _most certainly not recommended_). + +How exactly the desciption is generated is necessarily +project-dependent. It's tempting say, "oh, that's easy! We'll just +write it by hand!" but that would be folly. The struct sizes and byte +offsets into the struct _must_ be precisely how C-side code sees the +struct or the runtime results are completely undefined. + +The approach used in developing and testing _this_ software is... + +Below is a complete copy/pastable example of how we can use a small +set of macros to generate struct descriptions from C99 or later into +static string memory. Simply add such a file to your WASM build, +arrange for its function to be exported[^export-func], and call it +from JS (noting that it requires environment-specific JS glue to +convert the returned pointer to a JS-side string). Use `JSON.parse()` +to process it, then feed the included struct descriptions into the +binder factory at your leisure. + +------------------------------------------------------------ + +```c +#include /* memset() */ +#include /* offsetof() */ +#include /* snprintf() */ +#include /* int64_t */ +#include + +struct ExampleStruct { + int v4; + void * ppV; + int64_t v8; + void (*xFunc)(void*); +}; +typedef struct ExampleStruct ExampleStruct; + +const char * wasm__ctype_json(void){ + static char strBuf[512 * 8] = {0} + /* Static buffer which must be sized large enough for + our JSON. The string-generation macros try very + hard to assert() if this buffer is too small. */; + int n = 0, structCount = 0 /* counters for the macros */; + char * pos = &strBuf[1] + /* Write-position cursor. Skip the first byte for now to help + protect against a small race condition */; + char const * const zEnd = pos + sizeof(strBuf) + /* one-past-the-end cursor (virtual EOF) */; + if(strBuf[0]) return strBuf; // Was set up in a previous call. + + //////////////////////////////////////////////////////////////////// + // First we need to build up our macro framework... + + //////////////////////////////////////////////////////////////////// + // Core output-generating macros... +#define lenCheck assert(pos < zEnd - 100) +#define outf(format,...) \ + pos += snprintf(pos, ((size_t)(zEnd - pos)), format, __VA_ARGS__); \ + lenCheck +#define out(TXT) outf("%s",TXT) +#define CloseBrace(LEVEL) \ + assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck + + //////////////////////////////////////////////////////////////////// + // Macros for emiting StructBinders... +#define StructBinder__(TYPE) \ + n = 0; \ + outf("%s{", (structCount++ ? ", " : "")); \ + out("\"name\": \"" # TYPE "\","); \ + outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ + out(",\"members\": {"); +#define StructBinder_(T) StructBinder__(T) +// ^^^ extra indirection needed to expand CurrentStruct +#define StructBinder StructBinder_(CurrentStruct) +#define _StructBinder CloseBrace(2) +#define M(MEMBER,SIG) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG) + // End of macros. + //////////////////////////////////////////////////////////////////// + + //////////////////////////////////////////////////////////////////// + // With that out of the way, we can do what we came here to do. + out("\"structs\": ["); { + +// For each struct description, do... +#define CurrentStruct ExampleStruct + StructBinder { + M(v4,"i"); + M(ppV,"p"); + M(v8,"j"); + M(xFunc,"v(p)"); + } _StructBinder; +#undef CurrentStruct + + } out( "]"/*structs*/); + //////////////////////////////////////////////////////////////////// + // Done! Finalize the output... + out("}"/*top-level wrapper*/); + *pos = 0; + strBuf[0] = '{'/*end of the race-condition workaround*/; + return strBuf; + +// If this file will ever be concatenated or #included with others, +// it's good practice to clean up our macros: +#undef StructBinder +#undef StructBinder_ +#undef StructBinder__ +#undef M +#undef _StructBinder +#undef CloseBrace +#undef out +#undef outf +#undef lenCheck +} +``` + +------------------------------------------------------------ + + + +[sqlite3]: https://sqlite.org +[emscripten]: https://emscripten.org +[sgb]: https://wanderinghorse.net/home/stephan/ +[appendix-g]: #appendix-g +[StructBinderFactory]: #api-binderfactory +[StructCtors]: #api-structctor +[StructType]: #api-structtype +[StructBinder]: #api-structbinder +[StructInstance]: #api-structinstance +[^export-func]: In Emscripten, add its name, prefixed with `_`, to the + project's `EXPORT_FUNCTIONS` list. +[BigInt64Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt64Array +[TextDecoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextDecoder +[TextEncoder]: https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder +[MDN]: https://developer.mozilla.org/docs/Web/API diff --git a/ext/wasm/jaccwabyt/jaccwabyt_test.c b/ext/wasm/jaccwabyt/jaccwabyt_test.c new file mode 100644 index 0000000000..7e2db394c6 --- /dev/null +++ b/ext/wasm/jaccwabyt/jaccwabyt_test.c @@ -0,0 +1,178 @@ +#include +#include /* memset() */ +#include /* offsetof() */ +#include /* snprintf() */ +#include /* int64_t */ +/*#include */ /* malloc/free(), needed for emscripten exports. */ +extern void * malloc(size_t); +extern void free(void *); + +/* +** 2022-06-25 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +** +*********************************************************************** +** +** Utility functions for use with the emscripten/WASM bits. These +** functions ARE NOT part of the sqlite3 public API. They are strictly +** for internal use by the JS/WASM bindings. +** +** This file is intended to be WASM-compiled together with sqlite3.c, +** e.g.: +** +** emcc ... sqlite3.c wasm_util.c +*/ + +/* +** Experimenting with output parameters. +*/ +int jaccwabyt_test_intptr(int * p){ + if(1==((int)p)%3){ + /* kludge to get emscripten to export malloc() and free() */; + free(malloc(0)); + } + return *p = *p * 2; +} +int64_t jaccwabyt_test_int64_max(void){ + return (int64_t)0x7fffffffffffffff; +} +int64_t jaccwabyt_test_int64_min(void){ + return ~jaccwabyt_test_int64_max(); +} +int64_t jaccwabyt_test_int64_times2(int64_t x){ + return x * 2; +} + +void jaccwabyt_test_int64_minmax(int64_t * min, int64_t *max){ + *max = jaccwabyt_test_int64_max(); + *min = jaccwabyt_test_int64_min(); + /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ +} +int64_t jaccwabyt_test_int64ptr(int64_t * p){ + /*printf("jaccwabyt_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ + return *p = *p * 2; +} + +void jaccwabyt_test_stack_overflow(int recurse){ + if(recurse) jaccwabyt_test_stack_overflow(recurse); +} + +struct WasmTestStruct { + int v4; + void * ppV; + const char * cstr; + int64_t v8; + void (*xFunc)(void*); +}; +typedef struct WasmTestStruct WasmTestStruct; +void jaccwabyt_test_struct(WasmTestStruct * s){ + if(s){ + s->v4 *= 2; + s->v8 = s->v4 * 2; + s->ppV = s; + s->cstr = __FILE__; + if(s->xFunc) s->xFunc(s); + } + return; +} + +/** For testing the 'string-free' whwasmutil.xWrap() conversion. */ +char * jaccwabyt_test_str_hello(int fail){ + char * s = fail ? 0 : (char *)malloc(6); + if(s){ + memcpy(s, "hello", 5); + s[5] = 0; + } + return s; +} + +/* +** Returns a NUL-terminated string containing a JSON-format metadata +** regarding C structs, for use with the StructBinder API. The +** returned memory is static and is only written to the first time +** this is called. +*/ +const char * jaccwabyt_test_ctype_json(void){ + static char strBuf[1024 * 8] = {0}; + int n = 0, structCount = 0, groupCount = 0; + char * pos = &strBuf[1] /* skip first byte for now to help protect + against a small race condition */; + char const * const zEnd = pos + sizeof(strBuf); + if(strBuf[0]) return strBuf; + /* Leave first strBuf[0] at 0 until the end to help guard against a + tiny race condition. If this is called twice concurrently, they + might end up both writing to strBuf, but they'll both write the + same thing, so that's okay. If we set byte 0 up front then the + 2nd instance might return a partially-populated string. */ + + //////////////////////////////////////////////////////////////////// + // First we need to build up our macro framework... + //////////////////////////////////////////////////////////////////// + // Core output macros... +#define lenCheck assert(pos < zEnd - 100) +#define outf(format,...) \ + pos += snprintf(pos, ((size_t)(zEnd - pos)), format, __VA_ARGS__); \ + lenCheck +#define out(TXT) outf("%s",TXT) +#define CloseBrace(LEVEL) \ + assert(LEVEL<5); memset(pos, '}', LEVEL); pos+=LEVEL; lenCheck + + //////////////////////////////////////////////////////////////////// + // Macros for emitting StructBinder descriptions... +#define StructBinder__(TYPE) \ + n = 0; \ + outf("%s{", (structCount++ ? ", " : "")); \ + out("\"name\": \"" # TYPE "\","); \ + outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ + out(",\"members\": {"); +#define StructBinder_(T) StructBinder__(T) +// ^^^ indirection needed to expand CurrentStruct +#define StructBinder StructBinder_(CurrentStruct) +#define _StructBinder CloseBrace(2) +#define M(MEMBER,SIG) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG) + // End of macros + //////////////////////////////////////////////////////////////////// + + out("\"structs\": ["); { + +#define CurrentStruct WasmTestStruct + StructBinder { + M(v4,"i"); + M(cstr,"s"); + M(ppV,"p"); + M(v8,"j"); + M(xFunc,"v(p)"); + } _StructBinder; +#undef CurrentStruct + + } out( "]"/*structs*/); + out("}"/*top-level object*/); + *pos = 0; + strBuf[0] = '{'/*end of the race-condition workaround*/; + return strBuf; +#undef DefGroup +#undef Def +#undef _DefGroup +#undef StructBinder +#undef StructBinder_ +#undef StructBinder__ +#undef M +#undef _StructBinder +#undef CurrentStruct +#undef CloseBrace +#undef out +#undef outf +#undef lenCheck +} diff --git a/ext/wasm/jaccwabyt/jaccwabyt_test.exports b/ext/wasm/jaccwabyt/jaccwabyt_test.exports new file mode 100644 index 0000000000..b6182207b5 --- /dev/null +++ b/ext/wasm/jaccwabyt/jaccwabyt_test.exports @@ -0,0 +1,10 @@ +_jaccwabyt_test_intptr +_jaccwabyt_test_int64ptr +_jaccwabyt_test_int64_max +_jaccwabyt_test_int64_min +_jaccwabyt_test_int64_minmax +_jaccwabyt_test_int64_times2 +_jaccwabyt_test_struct +_jaccwabyt_test_ctype_json +_jaccwabyt_test_stack_overflow +_jaccwabyt_test_str_hello diff --git a/ext/fiddle/testing1.html b/ext/wasm/testing1.html similarity index 76% rename from ext/fiddle/testing1.html rename to ext/wasm/testing1.html index bf22f30ff3..0c64470221 100644 --- a/ext/fiddle/testing1.html +++ b/ext/wasm/testing1.html @@ -4,10 +4,9 @@ - - + + sqlite3-api.js tests -
sqlite3-api.js tests
@@ -25,9 +24,11 @@
-
Everything on this page happens in the dev console.
- - +
Most stuff on this page happens in the dev console.
+
+
+ + diff --git a/ext/wasm/testing1.js b/ext/wasm/testing1.js new file mode 100644 index 0000000000..e4b0882644 --- /dev/null +++ b/ext/wasm/testing1.js @@ -0,0 +1,1088 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3-api.js. This file must be run in + main JS thread and sqlite3.js must have been loaded before it. +*/ +'use strict'; +(function(){ + const T = self.SqliteTestUtil; + const toss = function(...args){throw new Error(args.join(' '))}; + const debug = console.debug.bind(console); + const eOutput = document.querySelector('#test-output'); + const log = console.log.bind(console) + const logHtml = function(...args){ + log.apply(this, args); + const ln = document.createElement('div'); + ln.append(document.createTextNode(args.join(' '))); + eOutput.append(ln); + }; + + const eqApprox = function(v1,v2,factor=0.05){ + //debug('eqApprox',v1, v2); + return v1>=(v2-factor) && v1<=(v2+factor); + }; + + const testBasicSanity = function(db,sqlite3){ + const capi = sqlite3.capi; + log("Basic sanity tests..."); + T.assert(Number.isInteger(db.pointer)). + mustThrowMatching(()=>db.pointer=1, /read-only/). + assert(0===capi.sqlite3_extended_result_codes(db.pointer,1)). + assert('main'===db.dbName(0)); + let pId; + let st = db.prepare( + new TextEncoder('utf-8').encode("select 3 as a") + /* Testing handling of Uint8Array input */ + ); + //debug("statement =",st); + try { + T.assert(Number.isInteger(st.pointer)) + .mustThrowMatching(()=>st.pointer=1, /read-only/) + .assert(1===db.openStatementCount()) + .assert(!st._mayGet) + .assert('a' === st.getColumnName(0)) + .assert(1===st.columnCount) + .assert(0===st.parameterCount) + .mustThrow(()=>st.bind(1,null)) + .assert(true===st.step()) + .assert(3 === st.get(0)) + .mustThrow(()=>st.get(1)) + .mustThrow(()=>st.get(0,~capi.SQLITE_INTEGER)) + .assert(3 === st.get(0,capi.SQLITE_INTEGER)) + .assert(3 === st.getInt(0)) + .assert('3' === st.get(0,capi.SQLITE_TEXT)) + .assert('3' === st.getString(0)) + .assert(3.0 === st.get(0,capi.SQLITE_FLOAT)) + .assert(3.0 === st.getFloat(0)) + .assert(3 === st.get({}).a) + .assert(3 === st.get([])[0]) + .assert(3 === st.getJSON(0)) + .assert(st.get(0,capi.SQLITE_BLOB) instanceof Uint8Array) + .assert(1===st.get(0,capi.SQLITE_BLOB).length) + .assert(st.getBlob(0) instanceof Uint8Array) + .assert('3'.charCodeAt(0) === st.getBlob(0)[0]) + .assert(st._mayGet) + .assert(false===st.step()) + .assert(!st._mayGet) + ; + pId = st.pointer; + T.assert(0===capi.sqlite3_strglob("*.txt", "foo.txt")). + assert(0!==capi.sqlite3_strglob("*.txt", "foo.xtx")). + assert(0===capi.sqlite3_strlike("%.txt", "foo.txt", 0)). + assert(0!==capi.sqlite3_strlike("%.txt", "foo.xtx", 0)); + }finally{ + st.finalize(); + } + T.assert(!st.pointer) + .assert(0===db.openStatementCount()); + let list = []; + db.exec({ + sql:['CREATE TABLE t(a,b);', + "INSERT INTO t(a,b) VALUES(1,2),(3,4),", + "(?,?),('blob',X'6869')"/*intentionally missing semicolon to test for + off-by-one bug in string-to-WASM conversion*/], + multi: true, + saveSql: list, + bind: [5,6] + }); + //debug("Exec'd SQL:", list); + T.assert(2 === list.length) + .assert('string'===typeof list[1]) + .assert(4===db.changes()); + if(capi.wasm.bigIntEnabled){ + T.assert(4n===db.changes(false,true)); + } + let blob = db.selectValue("select b from t where a='blob'"); + T.assert(blob instanceof Uint8Array). + assert(0x68===blob[0] && 0x69===blob[1]); + blob = null; + + let counter = 0, colNames = []; + list.length = 0; + db.exec(new TextEncoder('utf-8').encode("SELECT a a, b b FROM t"),{ + rowMode: 'object', + resultRows: list, + columnNames: colNames, + callback: function(row,stmt){ + ++counter; + T.assert((row.a%2 && row.a<6) || 'blob'===row.a); + } + }); + T.assert(2 === colNames.length) + .assert('a' === colNames[0]) + .assert(4 === counter) + .assert(4 === list.length); + list.length = 0; + db.exec("SELECT a a, b b FROM t",{ + rowMode: 'array', + callback: function(row,stmt){ + ++counter; + T.assert(Array.isArray(row)) + .assert((0===row[1]%2 && row[1]<7) + || (row[1] instanceof Uint8Array)); + } + }); + T.assert(8 === counter); + T.assert(Number.MIN_SAFE_INTEGER === + db.selectValue("SELECT "+Number.MIN_SAFE_INTEGER)). + assert(Number.MAX_SAFE_INTEGER === + db.selectValue("SELECT "+Number.MAX_SAFE_INTEGER)); + if(capi.wasm.bigIntEnabled){ + const mI = capi.wasm.xCall('jaccwabyt_test_int64_max'); + const b = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(b === db.selectValue("SELECT "+b)). + assert(b === db.selectValue("SELECT ?", b)). + assert(mI == db.selectValue("SELECT $x", {$x:mI})); + }else{ + /* Curiously, the JS spec seems to be off by one with the definitions + of MIN/MAX_SAFE_INTEGER: + + https://github.com/emscripten-core/emscripten/issues/17391 */ + T.mustThrow(()=>db.selectValue("SELECT "+(Number.MAX_SAFE_INTEGER+1))). + mustThrow(()=>db.selectValue("SELECT "+(Number.MIN_SAFE_INTEGER-1))); + } + + st = db.prepare("update t set b=:b where a='blob'"); + try { + const ndx = st.getParamIndex(':b'); + T.assert(1===ndx); + st.bindAsBlob(ndx, "ima blob").reset(true); + } finally { + st.finalize(); + } + + try { + throw new capi.WasmAllocError; + }catch(e){ + T.assert(e instanceof Error) + .assert(e instanceof capi.WasmAllocError); + } + + try { + db.prepare("/*empty SQL*/"); + toss("Must not be reached."); + }catch(e){ + T.assert(e instanceof sqlite3.SQLite3Error) + .assert(0==e.message.indexOf('Cannot prepare empty')); + } + + T.assert(capi.sqlite3_errstr(capi.SQLITE_IOERR_ACCESS).indexOf("I/O")>=0). + assert(capi.sqlite3_errstr(capi.SQLITE_CORRUPT).indexOf('malformed')>0). + assert(capi.sqlite3_errstr(capi.SQLITE_OK) === 'not an error'); + + // Custom db error message handling via sqlite3_prepare_v2/v3() + if(capi.wasm.exports.sqlite3_wasm_db_error){ + log("Testing custom error message via prepare_v3()..."); + let rc = capi.sqlite3_prepare_v3(db.pointer, [/*invalid*/], -1, 0, null, null); + T.assert(capi.SQLITE_MISUSE === rc) + .assert(0 === capi.sqlite3_errmsg(db.pointer).indexOf("Invalid SQL")); + log("errmsg =",capi.sqlite3_errmsg(db.pointer)); + } + }/*testBasicSanity()*/; + + const testUDF = function(db){ + db.createFunction("foo",function(a,b){return a+b}); + T.assert(7===db.selectValue("select foo(3,4)")). + assert(5===db.selectValue("select foo(3,?)",2)). + assert(5===db.selectValue("select foo(?,?2)",[1,4])). + assert(5===db.selectValue("select foo($a,$b)",{$a:0,$b:5})); + db.createFunction("bar", { + arity: -1, + callback: function(){ + var rc = 0; + for(let i = 0; i < arguments.length; ++i) rc += arguments[i]; + return rc; + } + }).createFunction({ + name: "asis", + callback: (arg)=>arg + }); + + //log("Testing DB::selectValue() w/ UDF..."); + T.assert(0===db.selectValue("select bar()")). + assert(1===db.selectValue("select bar(1)")). + assert(3===db.selectValue("select bar(1,2)")). + assert(-1===db.selectValue("select bar(1,2,-4)")). + assert('hi'===db.selectValue("select asis('hi')")); + + T.assert('hi' === db.selectValue("select ?",'hi')). + assert(null===db.selectValue("select null")). + assert(null === db.selectValue("select ?",null)). + assert(null === db.selectValue("select ?",[null])). + assert(null === db.selectValue("select $a",{$a:null})). + assert(eqApprox(3.1,db.selectValue("select 3.0 + 0.1"))). + assert(eqApprox(1.3,db.selectValue("select asis(1 + 0.3)"))) + ; + + //log("Testing binding and UDF propagation of blobs..."); + let blobArg = new Uint8Array(2); + blobArg.set([0x68, 0x69], 0); + let blobRc = db.selectValue("select asis(?1)", blobArg); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + blobRc = db.selectValue("select asis(X'6869')"); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length). + assert(0x68==blobRc[0] && 0x69==blobRc[1]); + + blobArg = new Int8Array(2); + blobArg.set([0x68, 0x69]); + //debug("blobArg=",blobArg); + blobRc = db.selectValue("select asis(?1)", blobArg); + T.assert(blobRc instanceof Uint8Array). + assert(2 === blobRc.length); + //debug("blobRc=",blobRc); + T.assert(0x68==blobRc[0] && 0x69==blobRc[1]); + }; + + const testAttach = function(db){ + const resultRows = []; + db.exec({ + sql:new TextEncoder('utf-8').encode([ + // ^^^ testing string-vs-typedarray handling in execMulti() + "attach 'foo.db' as foo;", + "create table foo.bar(a);", + "insert into foo.bar(a) values(1),(2),(3);", + "select a from foo.bar order by a;" + ].join('')), + multi: true, + rowMode: 0, + resultRows + }); + T.assert(3===resultRows.length) + .assert(2===resultRows[1]); + T.assert(2===db.selectValue('select a from foo.bar where a>1 order by a')); + db.exec("detach foo"); + T.mustThrow(()=>db.exec("select * from foo.bar")); + }; + + const testIntPtr = function(db,S,Module){ + const w = S.capi.wasm; + const stack = w.scopedAllocPush(); + let ptrInt; + const origValue = 512; + const ptrValType = 'i32'; + try{ + ptrInt = w.scopedAlloc(4); + w.setMemValue(ptrInt,origValue, ptrValType); + const cf = w.xGet('jaccwabyt_test_intptr'); + const oldPtrInt = ptrInt; + //log('ptrInt',ptrInt); + //log('getMemValue(ptrInt)',w.getMemValue(ptrInt)); + T.assert(origValue === w.getMemValue(ptrInt, ptrValType)); + const rc = cf(ptrInt); + //log('cf(ptrInt)',rc); + //log('ptrInt',ptrInt); + //log('getMemValue(ptrInt)',w.getMemValue(ptrInt,ptrValType)); + T.assert(2*origValue === rc). + assert(rc === w.getMemValue(ptrInt,ptrValType)). + assert(oldPtrInt === ptrInt); + const pi64 = w.scopedAlloc(8)/*ptr to 64-bit integer*/; + const o64 = 0x010203040506/*>32-bit integer*/; + const ptrType64 = 'i64'; + if(w.bigIntEnabled){ + log("BigInt support is enabled..."); + w.setMemValue(pi64, o64, ptrType64); + //log("pi64 =",pi64, "o64 = 0x",o64.toString(16), o64); + const v64 = ()=>w.getMemValue(pi64,ptrType64) + //log("getMemValue(pi64)",v64()); + T.assert(v64() == o64); + //T.assert(o64 === w.getMemValue(pi64, ptrType64)); + const cf64w = w.xGet('jaccwabyt_test_int64ptr'); + cf64w(pi64); + //log("getMemValue(pi64)",v64()); + T.assert(v64() == BigInt(2 * o64)); + cf64w(pi64); + T.assert(v64() == BigInt(4 * o64)); + + const biTimes2 = w.xGet('jaccwabyt_test_int64_times2'); + T.assert(BigInt(2 * o64) === + biTimes2(BigInt(o64)/*explicit conv. required to avoid TypeError + in the call :/ */)); + + const pMin = w.scopedAlloc(16); + const pMax = pMin + 8; + const g64 = (p)=>w.getMemValue(p,ptrType64); + w.setMemValue(pMin, 0, ptrType64); + w.setMemValue(pMax, 0, ptrType64); + const minMaxI64 = [ + w.xCall('jaccwabyt_test_int64_min'), + w.xCall('jaccwabyt_test_int64_max') + ]; + T.assert(minMaxI64[0] < BigInt(Number.MIN_SAFE_INTEGER)). + assert(minMaxI64[1] > BigInt(Number.MAX_SAFE_INTEGER)); + //log("int64_min/max() =",minMaxI64, typeof minMaxI64[0]); + w.xCall('jaccwabyt_test_int64_minmax', pMin, pMax); + T.assert(g64(pMin) === minMaxI64[0], "int64 mismatch"). + assert(g64(pMax) === minMaxI64[1], "int64 mismatch") + /* ^^^ that will fail, as of this writing, due to + mismatched getMemValue()/setMemValue() impls in the + Emscripten-generated glue. We install a + replacement getMemValue() in sqlite3-api.js to work + around that bug: + + https://github.com/emscripten-core/emscripten/issues/17322 + */; + //log("pMin",g64(pMin), "pMax",g64(pMax)); + w.setMemValue(pMin, minMaxI64[0], ptrType64); + T.assert(g64(pMin) === minMaxI64[0]). + assert(minMaxI64[0] === db.selectValue("select ?",g64(pMin))). + assert(minMaxI64[1] === db.selectValue("select ?",g64(pMax))); + const rxRange = /out of range for int64/; + T.mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[0] - BigInt(1))}, + rxRange). + mustThrowMatching(()=>{db.prepare("select ?").bind(minMaxI64[1] + BigInt(1))}, + (e)=>rxRange.test(e.message)); + }else{ + log("No BigInt support. Skipping related tests."); + log("\"The problem\" here is that we can manipulate, at the byte level,", + "heap memory to set 64-bit values, but we can't get those values", + "back into JS because of the lack of 64-bit number support."); + } + }finally{ + const x = w.scopedAlloc(1), y = w.scopedAlloc(1), z = w.scopedAlloc(1); + //log("x=",x,"y=",y,"z=",z); // just looking at the alignment + w.scopedAllocPop(stack); + } + }/*testIntPtr()*/; + + const testStructStuff = function(db,S,M){ + const W = S.capi.wasm, C = S; + /** Maintenance reminder: the rest of this function is copy/pasted + from the upstream jaccwabyt tests. */ + log("Jaccwabyt tests..."); + const MyStructDef = { + sizeof: 16, + members: { + p4: {offset: 0, sizeof: 4, signature: "i"}, + pP: {offset: 4, sizeof: 4, signature: "P"}, + ro: {offset: 8, sizeof: 4, signature: "i", readOnly: true}, + cstr: {offset: 12, sizeof: 4, signature: "s"} + } + }; + if(W.bigIntEnabled){ + const m = MyStructDef; + m.members.p8 = {offset: m.sizeof, sizeof: 8, signature: "j"}; + m.sizeof += m.members.p8.sizeof; + } + const StructType = C.StructBinder.StructType; + const K = C.StructBinder('my_struct',MyStructDef); + T.mustThrowMatching(()=>K(), /via 'new'/). + mustThrowMatching(()=>new K('hi'), /^Invalid pointer/); + const k1 = new K(), k2 = new K(); + try { + T.assert(k1.constructor === K). + assert(K.isA(k1)). + assert(k1 instanceof K). + assert(K.prototype.lookupMember('p4').key === '$p4'). + assert(K.prototype.lookupMember('$p4').name === 'p4'). + mustThrowMatching(()=>K.prototype.lookupMember('nope'), /not a mapped/). + assert(undefined === K.prototype.lookupMember('nope',false)). + assert(k1 instanceof StructType). + assert(StructType.isA(k1)). + assert(K.resolveToInstance(k1.pointer)===k1). + mustThrowMatching(()=>K.resolveToInstance(null,true), /is-not-a my_struct/). + assert(k1 === StructType.instanceForPointer(k1.pointer)). + mustThrowMatching(()=>k1.$ro = 1, /read-only/); + Object.keys(MyStructDef.members).forEach(function(key){ + key = K.memberKey(key); + T.assert(0 == k1[key], + "Expecting allocation to zero the memory "+ + "for "+key+" but got: "+k1[key]+ + " from "+k1.memoryDump()); + }); + T.assert('number' === typeof k1.pointer). + mustThrowMatching(()=>k1.pointer = 1, /pointer/). + assert(K.instanceForPointer(k1.pointer) === k1); + k1.$p4 = 1; k1.$pP = 2; + T.assert(1 === k1.$p4).assert(2 === k1.$pP); + if(MyStructDef.members.$p8){ + k1.$p8 = 1/*must not throw despite not being a BigInt*/; + k1.$p8 = BigInt(Number.MAX_SAFE_INTEGER * 2); + T.assert(BigInt(2 * Number.MAX_SAFE_INTEGER) === k1.$p8); + } + T.assert(!k1.ondispose); + k1.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(k1.ondispose)). + assert(k1.ondispose[0] === k1.$cstr). + assert('number' === typeof k1.$cstr). + assert('A C-string.' === k1.memberToJsString('cstr')); + k1.$pP = k2; + T.assert(k1.$pP === k2); + k1.$pP = null/*null is special-cased to 0.*/; + T.assert(0===k1.$pP); + let ptr = k1.pointer; + k1.dispose(); + T.assert(undefined === k1.pointer). + assert(undefined === K.instanceForPointer(ptr)). + mustThrowMatching(()=>{k1.$pP=1}, /disposed instance/); + const k3 = new K(); + ptr = k3.pointer; + T.assert(k3 === K.instanceForPointer(ptr)); + K.disposeAll(); + T.assert(ptr). + assert(undefined === k2.pointer). + assert(undefined === k3.pointer). + assert(undefined === K.instanceForPointer(ptr)); + }finally{ + k1.dispose(); + k2.dispose(); + } + + if(!W.bigIntEnabled){ + log("Skipping WasmTestStruct tests: BigInt not enabled."); + return; + } + + const ctype = W.xCallWrapped('jaccwabyt_test_ctype_json', 'json'); + log("Struct descriptions:",ctype.structs); + const WTStructDesc = + ctype.structs.filter((e)=>'WasmTestStruct'===e.name)[0]; + const autoResolvePtr = true /* EXPERIMENTAL */; + if(autoResolvePtr){ + WTStructDesc.members.ppV.signature = 'P'; + } + const WTStruct = C.StructBinder(WTStructDesc); + log(WTStruct.structName, WTStruct.structInfo); + const wts = new WTStruct(); + log("WTStruct.prototype keys:",Object.keys(WTStruct.prototype)); + try{ + T.assert(wts.constructor === WTStruct). + assert(WTStruct.memberKeys().indexOf('$ppV')>=0). + assert(wts.memberKeys().indexOf('$v8')>=0). + assert(!K.isA(wts)). + assert(WTStruct.isA(wts)). + assert(wts instanceof WTStruct). + assert(wts instanceof StructType). + assert(StructType.isA(wts)). + assert(wts === StructType.instanceForPointer(wts.pointer)); + T.assert(wts.pointer>0).assert(0===wts.$v4).assert(0n===wts.$v8). + assert(0===wts.$ppV).assert(0===wts.$xFunc). + assert(WTStruct.instanceForPointer(wts.pointer) === wts); + const testFunc = + W.xGet('jaccwabyt_test_struct'/*name gets mangled in -O3 builds!*/); + let counter = 0; + log("wts.pointer =",wts.pointer); + const wtsFunc = function(arg){ + log("This from a JS function called from C, "+ + "which itself was called from JS. arg =",arg); + ++counter; + T.assert(WTStruct.instanceForPointer(arg) === wts); + if(3===counter){ + toss("Testing exception propagation."); + } + } + wts.$v4 = 10; wts.$v8 = 20; + wts.$xFunc = W.installFunction(wtsFunc, wts.memberSignature('xFunc')) + /* ^^^ compiles wtsFunc to WASM and returns its new function pointer */; + T.assert(0===counter).assert(10 === wts.$v4).assert(20n === wts.$v8) + .assert(0 === wts.$ppV).assert('number' === typeof wts.$xFunc) + .assert(0 === wts.$cstr) + .assert(wts.memberIsString('$cstr')) + .assert(!wts.memberIsString('$v4')) + .assert(null === wts.memberToJsString('$cstr')) + .assert(W.functionEntry(wts.$xFunc) instanceof Function); + /* It might seem silly to assert that the values match + what we just set, but recall that all of those property + reads and writes are, via property interceptors, + actually marshaling their data to/from a raw memory + buffer, so merely reading them back is actually part of + testing the struct-wrapping API. */ + + testFunc(wts.pointer); + log("wts.pointer, wts.$ppV",wts.pointer, wts.$ppV); + T.assert(1===counter).assert(20 === wts.$v4).assert(40n === wts.$v8) + .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)) + .assert('string' === typeof wts.memberToJsString('cstr')) + .assert(wts.memberToJsString('cstr') === wts.memberToJsString('$cstr')) + .mustThrowMatching(()=>wts.memberToJsString('xFunc'), + /Invalid member type signature for C-string/) + ; + testFunc(wts.pointer); + T.assert(2===counter).assert(40 === wts.$v4).assert(80n === wts.$v8) + .assert(autoResolvePtr ? (wts.$ppV === wts) : (wts.$ppV === wts.pointer)); + /** The 3rd call to wtsFunc throw from JS, which is called + from C, which is called from JS. Let's ensure that + that exception propagates back here... */ + T.mustThrowMatching(()=>testFunc(wts.pointer),/^Testing/); + W.uninstallFunction(wts.$xFunc); + wts.$xFunc = 0; + if(autoResolvePtr){ + wts.$ppV = 0; + T.assert(!wts.$ppV); + WTStruct.debugFlags(0x03); + wts.$ppV = wts; + T.assert(wts === wts.$ppV) + WTStruct.debugFlags(0); + } + wts.setMemberCString('cstr', "A C-string."); + T.assert(Array.isArray(wts.ondispose)). + assert(wts.ondispose[0] === wts.$cstr). + assert('A C-string.' === wts.memberToJsString('cstr')); + const ptr = wts.pointer; + wts.dispose(); + T.assert(ptr).assert(undefined === wts.pointer). + assert(undefined === WTStruct.instanceForPointer(ptr)) + }finally{ + wts.dispose(); + } + }/*testStructStuff()*/; + + const testSqliteStructs = function(db,sqlite3,M){ + log("Tinkering with sqlite3_io_methods..."); + // https://www.sqlite.org/c3ref/vfs.html + // https://www.sqlite.org/c3ref/io_methods.html + const capi = sqlite3.capi, W = capi.wasm; + const sqlite3_io_methods = capi.sqlite3_io_methods, + sqlite3_vfs = capi.sqlite3_vfs, + sqlite3_file = capi.sqlite3_file; + log("struct sqlite3_file", sqlite3_file.memberKeys()); + log("struct sqlite3_vfs", sqlite3_vfs.memberKeys()); + log("struct sqlite3_io_methods", sqlite3_io_methods.memberKeys()); + + const installMethod = function callee(tgt, name, func){ + if(1===arguments.length){ + return (n,f)=>callee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.ondisposeRemoveFunc = function(){ + if(this.__ondispose){ + const who = this; + this.__ondispose.forEach( + (v)=>{ + if('number'===typeof v){ + try{capi.wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + }else{/*wasm function wrapper property*/ + delete who[v]; + } + } + ); + delete this.__ondispose; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name), + memKey = tgt.memberKey(name); + //log("installMethod",tgt, name, sigN); + if(!tgt.__ondispose){ + T.assert(undefined === tgt.ondispose); + tgt.ondispose = [callee.ondisposeRemoveFunc]; + tgt.__ondispose = []; + } + const fProxy = callee.argcProxy(func, sigN); + const pFunc = capi.wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + /** + ACHTUNG: function pointer IDs are from a different pool than + allocation IDs, starting at 1 and incrementing in steps of 1, + so if we set tgt[memKey] to those values, we'd very likely + later misinterpret them as plain old pointer addresses unless + unless we use some silly heuristic like "all values <5k are + presumably function pointers," or actually perform a function + lookup on every pointer to first see if it's a function. That + would likely work just fine, but would be kludgy. + + It turns out that "all values less than X are functions" is + essentially how it works in wasm: a function pointer is + reported to the client as its index into the + __indirect_function_table. + + So... once jaccwabyt can be told how to access the + function table, it could consider all pointer values less + than that table's size to be functions. As "real" pointer + values start much, much higher than the function table size, + that would likely work reasonably well. e.g. the object + pointer address for sqlite3's default VFS is (in this local + setup) 65104, whereas the function table has fewer than 600 + entries. + */ + const wrapperKey = '$'+memKey; + tgt[wrapperKey] = fProxy; + tgt.__ondispose.push(pFunc, wrapperKey); + //log("tgt.__ondispose =",tgt.__ondispose); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; + + const installIOMethods = function instm(iom){ + (iom instanceof capi.sqlite3_io_methods) || toss("Invalid argument type."); + if(!instm._requireFileArg){ + instm._requireFileArg = function(arg,methodName){ + arg = capi.sqlite3_file.resolveToInstance(arg); + if(!arg){ + err("sqlite3_io_methods::xClose() was passed a non-sqlite3_file."); + } + return arg; + }; + instm._methods = { + // https://sqlite.org/c3ref/io_methods.html + xClose: /*i(P)*/function(f){ + /* int (*xClose)(sqlite3_file*) */ + log("xClose(",f,")"); + if(!(f = instm._requireFileArg(f,'xClose'))) return capi.SQLITE_MISUSE; + f.dispose(/*noting that f has externally-owned memory*/); + return 0; + }, + xRead: /*i(Ppij)*/function(f,dest,n,offset){ + /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */ + log("xRead(",arguments,")"); + if(!(f = instm._requireFileArg(f))) return capi.SQLITE_MISUSE; + capi.wasm.heap8().fill(0, dest + offset, n); + return 0; + }, + xWrite: /*i(Ppij)*/function(f,dest,n,offset){ + /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */ + log("xWrite(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xWrite'))) return capi.SQLITE_MISUSE; + return 0; + }, + xTruncate: /*i(Pj)*/function(f){ + /* int (*xTruncate)(sqlite3_file*, sqlite3_int64 size) */ + log("xTruncate(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xTruncate'))) return capi.SQLITE_MISUSE; + return 0; + }, + xSync: /*i(Pi)*/function(f){ + /* int (*xSync)(sqlite3_file*, int flags) */ + log("xSync(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xSync'))) return capi.SQLITE_MISUSE; + return 0; + }, + xFileSize: /*i(Pp)*/function(f,pSz){ + /* int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize) */ + log("xFileSize(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xFileSize'))) return capi.SQLITE_MISUSE; + capi.wasm.setMemValue(pSz, 0/*file size*/); + return 0; + }, + xLock: /*i(Pi)*/function(f){ + /* int (*xLock)(sqlite3_file*, int) */ + log("xLock(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xLock'))) return capi.SQLITE_MISUSE; + return 0; + }, + xUnlock: /*i(Pi)*/function(f){ + /* int (*xUnlock)(sqlite3_file*, int) */ + log("xUnlock(",arguments,")"); + if(!(f=instm._requireFileArg(f,'xUnlock'))) return capi.SQLITE_MISUSE; + return 0; + }, + xCheckReservedLock: /*i(Pp)*/function(){ + /* int (*xCheckReservedLock)(sqlite3_file*, int *pResOut) */ + log("xCheckReservedLock(",arguments,")"); + return 0; + }, + xFileControl: /*i(Pip)*/function(){ + /* int (*xFileControl)(sqlite3_file*, int op, void *pArg) */ + log("xFileControl(",arguments,")"); + return capi.SQLITE_NOTFOUND; + }, + xSectorSize: /*i(P)*/function(){ + /* int (*xSectorSize)(sqlite3_file*) */ + log("xSectorSize(",arguments,")"); + return 0/*???*/; + }, + xDeviceCharacteristics:/*i(P)*/function(){ + /* int (*xDeviceCharacteristics)(sqlite3_file*) */ + log("xDeviceCharacteristics(",arguments,")"); + return 0; + } + }; + }/*static init*/ + iom.$iVersion = 1; + Object.keys(instm._methods).forEach( + (k)=>installMethod(iom, k, instm._methods[k]) + ); + }/*installIOMethods()*/; + + const iom = new sqlite3_io_methods, sfile = new sqlite3_file; + const err = console.error.bind(console); + try { + const IOM = sqlite3_io_methods, S3F = sqlite3_file; + //log("iom proto",iom,iom.constructor.prototype); + //log("sfile",sfile,sfile.constructor.prototype); + T.assert(0===sfile.$pMethods).assert(iom.pointer > 0); + //log("iom",iom); + /** Some of the following tests require that pMethods has a + signature of "P", as opposed to "p". */ + sfile.$pMethods = iom; + T.assert(iom === sfile.$pMethods); + sfile.$pMethods = iom.pointer; + T.assert(iom === sfile.$pMethods) + .assert(IOM.resolveToInstance(iom)) + .assert(undefined ===IOM.resolveToInstance(sfile)) + .mustThrow(()=>IOM.resolveToInstance(0,true)) + .assert(S3F.resolveToInstance(sfile.pointer)) + .assert(undefined===S3F.resolveToInstance(iom)); + T.assert(0===iom.$iVersion); + installIOMethods(iom); + T.assert(1===iom.$iVersion); + //log("iom.__ondispose",iom.__ondispose); + T.assert(Array.isArray(iom.__ondispose)).assert(iom.__ondispose.length>10); + }finally{ + iom.dispose(); + T.assert(undefined === iom.__ondispose); + } + + const dVfs = new sqlite3_vfs(capi.sqlite3_vfs_find(null)); + try { + const SB = sqlite3.StructBinder; + T.assert(dVfs instanceof SB.StructType) + .assert(dVfs.pointer) + .assert('sqlite3_vfs' === dVfs.structName) + .assert(!!dVfs.structInfo) + .assert(SB.StructType.hasExternalPointer(dVfs)) + .assert(3===dVfs.$iVersion) + .assert('number'===typeof dVfs.$zName) + .assert('number'===typeof dVfs.$xSleep) + .assert(capi.wasm.functionEntry(dVfs.$xOpen)) + .assert(dVfs.memberIsString('zName')) + .assert(dVfs.memberIsString('$zName')) + .assert(!dVfs.memberIsString('pAppData')) + .mustThrowMatching(()=>dVfs.memberToJsString('xSleep'), + /Invalid member type signature for C-string/) + .mustThrowMatching(()=>dVfs.memberSignature('nope'), /nope is not a mapped/) + .assert('string' === typeof dVfs.memberToJsString('zName')) + .assert(dVfs.memberToJsString('zName')===dVfs.memberToJsString('$zName')) + ; + log("Default VFS: @",dVfs.pointer); + Object.keys(sqlite3_vfs.structInfo.members).forEach(function(mname){ + const mk = sqlite3_vfs.memberKey(mname), mbr = sqlite3_vfs.structInfo.members[mname], + addr = dVfs[mk], prefix = 'defaultVfs.'+mname; + if(1===mbr.signature.length){ + let sep = '?', val = undefined; + switch(mbr.signature[0]){ + // TODO: move this into an accessor, e.g. getPreferredValue(member) + case 'i': case 'j': case 'f': case 'd': sep = '='; val = dVfs[mk]; break + case 'p': case 'P': sep = '@'; val = dVfs[mk]; break; + case 's': sep = '='; + //val = capi.wasm.UTF8ToString(addr); + val = dVfs.memberToJsString(mname); + break; + } + log(prefix, sep, val); + } + else{ + log(prefix," = funcptr @",addr, capi.wasm.functionEntry(addr)); + } + }); + }finally{ + dVfs.dispose(); + T.assert(undefined===dVfs.pointer); + } + }/*testSqliteStructs()*/; + + const testWasmUtil = function(DB,S){ + const w = S.capi.wasm; + /** + Maintenance reminder: the rest of this function is part of the + upstream Jaccwabyt tree. + */ + const chr = (x)=>x.charCodeAt(0); + log("heap getters..."); + { + const li = [8, 16, 32]; + if(w.bigIntEnabled) li.push(64); + for(const n of li){ + const bpe = n/8; + const s = w.heapForSize(n,false); + T.assert(bpe===s.BYTES_PER_ELEMENT). + assert(w.heapForSize(s.constructor) === s); + const u = w.heapForSize(n,true); + T.assert(bpe===u.BYTES_PER_ELEMENT). + assert(s!==u). + assert(w.heapForSize(u.constructor) === u); + } + } + + log("jstrlen()..."); + { + T.assert(3 === w.jstrlen("abc")).assert(4 === w.jstrlen("äbc")); + } + + log("jstrcpy()..."); + { + const fillChar = 10; + let ua = new Uint8Array(8), rc, + refill = ()=>ua.fill(fillChar); + refill(); + rc = w.jstrcpy("hello", ua); + T.assert(6===rc).assert(0===ua[5]).assert(chr('o')===ua[4]); + refill(); + ua[5] = chr('!'); + rc = w.jstrcpy("HELLO", ua, 0, -1, false); + T.assert(5===rc).assert(chr('!')===ua[5]).assert(chr('O')===ua[4]); + refill(); + rc = w.jstrcpy("the end", ua, 4); + //log("rc,ua",rc,ua); + T.assert(4===rc).assert(0===ua[7]). + assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); + refill(); + rc = w.jstrcpy("the end", ua, 4, -1, false); + T.assert(4===rc).assert(chr(' ')===ua[7]). + assert(chr('e')===ua[6]).assert(chr('t')===ua[4]); + refill(); + rc = w.jstrcpy("", ua, 0, 1, true); + //log("rc,ua",rc,ua); + T.assert(1===rc).assert(0===ua[0]); + refill(); + rc = w.jstrcpy("x", ua, 0, 1, true); + //log("rc,ua",rc,ua); + T.assert(1===rc).assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 1, true); + T.assert(1===rc, 'Must not write partial multi-byte char.') + .assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 2, true); + T.assert(1===rc, 'Must not write partial multi-byte char.') + .assert(0===ua[0]); + refill(); + rc = w.jstrcpy('äbä', ua, 0, 2, false); + T.assert(2===rc).assert(fillChar!==ua[1]).assert(fillChar===ua[2]); + }/*jstrcpy()*/ + + log("cstrncpy()..."); + { + w.scopedAllocPush(); + try { + let cStr = w.scopedAllocCString("hello"); + const n = w.cstrlen(cStr); + let cpy = w.scopedAlloc(n+10); + let rc = w.cstrncpy(cpy, cStr, n+10); + T.assert(n+1 === rc). + assert("hello" === w.cstringToJs(cpy)). + assert(chr('o') === w.getMemValue(cpy+n-1)). + assert(0 === w.getMemValue(cpy+n)); + let cStr2 = w.scopedAllocCString("HI!!!"); + rc = w.cstrncpy(cpy, cStr2, 3); + T.assert(3===rc). + assert("HI!lo" === w.cstringToJs(cpy)). + assert(chr('!') === w.getMemValue(cpy+2)). + assert(chr('l') === w.getMemValue(cpy+3)); + }finally{ + w.scopedAllocPop(); + } + } + + log("jstrToUintArray()..."); + { + let a = w.jstrToUintArray("hello", false); + T.assert(5===a.byteLength).assert(chr('o')===a[4]); + a = w.jstrToUintArray("hello", true); + T.assert(6===a.byteLength).assert(chr('o')===a[4]).assert(0===a[5]); + a = w.jstrToUintArray("äbä", false); + T.assert(5===a.byteLength).assert(chr('b')===a[2]); + a = w.jstrToUintArray("äbä", true); + T.assert(6===a.byteLength).assert(chr('b')===a[2]).assert(0===a[5]); + } + + log("allocCString()..."); + { + const cstr = w.allocCString("hällo, world"); + const n = w.cstrlen(cstr); + T.assert(13 === n) + .assert(0===w.getMemValue(cstr+n)) + .assert(chr('d')===w.getMemValue(cstr+n-1)); + } + + log("scopedAlloc() and friends..."); + { + const alloc = w.alloc, dealloc = w.dealloc; + w.alloc = w.dealloc = null; + T.assert(!w.scopedAlloc.level) + .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) + .mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); + w.alloc = alloc; + T.mustThrowMatching(()=>w.scopedAllocPush(), /missing alloc/); + w.dealloc = dealloc; + T.mustThrowMatching(()=>w.scopedAllocPop(), /^Invalid state/) + .mustThrowMatching(()=>w.scopedAlloc(1), /^No scopedAllocPush/) + .mustThrowMatching(()=>w.scopedAlloc.level=0, /read-only/); + const asc = w.scopedAllocPush(); + let asc2; + try { + const p1 = w.scopedAlloc(16), + p2 = w.scopedAlloc(16); + T.assert(1===w.scopedAlloc.level) + .assert(Number.isFinite(p1)) + .assert(Number.isFinite(p2)) + .assert(asc[0] === p1) + .assert(asc[1]===p2); + asc2 = w.scopedAllocPush(); + const p3 = w.scopedAlloc(16); + T.assert(2===w.scopedAlloc.level) + .assert(Number.isFinite(p3)) + .assert(2===asc.length) + .assert(p3===asc2[0]); + + const [z1, z2, z3] = w.scopedAllocPtr(3); + T.assert('number'===typeof z1).assert(z2>z1).assert(z3>z2) + .assert(0===w.getMemValue(z1,'i32'), 'allocPtr() must zero the targets') + .assert(0===w.getMemValue(z3,'i32')); + }finally{ + // Pop them in "incorrect" order to make sure they behave: + w.scopedAllocPop(asc); + T.assert(0===asc.length); + T.mustThrowMatching(()=>w.scopedAllocPop(asc), + /^Invalid state object/); + if(asc2){ + T.assert(2===asc2.length,'Should be p3 and z1'); + w.scopedAllocPop(asc2); + T.assert(0===asc2.length); + T.mustThrowMatching(()=>w.scopedAllocPop(asc2), + /^Invalid state object/); + } + } + T.assert(0===w.scopedAlloc.level); + w.scopedAllocCall(function(){ + T.assert(1===w.scopedAlloc.level); + const [cstr, n] = w.scopedAllocCString("hello, world", true); + T.assert(12 === n) + .assert(0===w.getMemValue(cstr+n)) + .assert(chr('d')===w.getMemValue(cstr+n-1)); + }); + }/*scopedAlloc()*/ + + log("xCall()..."); + { + const pJson = w.xCall('jaccwabyt_test_ctype_json'); + T.assert(Number.isFinite(pJson)).assert(w.cstrlen(pJson)>300); + } + + log("xWrap()..."); + { + //int jaccwabyt_test_intptr(int * p); + //int64_t jaccwabyt_test_int64_max(void) + //int64_t jaccwabyt_test_int64_min(void) + //int64_t jaccwabyt_test_int64_times2(int64_t x) + //void jaccwabyt_test_int64_minmax(int64_t * min, int64_t *max) + //int64_t jaccwabyt_test_int64ptr(int64_t * p) + //const char * jaccwabyt_test_ctype_json(void) + T.mustThrowMatching(()=>w.xWrap('jaccwabyt_test_ctype_json',null,'i32'), + /requires 0 arg/). + assert(w.xWrap.resultAdapter('i32') instanceof Function). + assert(w.xWrap.argAdapter('i32') instanceof Function); + let fw = w.xWrap('jaccwabyt_test_ctype_json','string'); + T.mustThrowMatching(()=>fw(1), /requires 0 arg/); + let rc = fw(); + T.assert('string'===typeof rc).assert(rc.length>300); + rc = w.xCallWrapped('jaccwabyt_test_ctype_json','*'); + T.assert(rc>0 && Number.isFinite(rc)); + rc = w.xCallWrapped('jaccwabyt_test_ctype_json','string'); + T.assert('string'===typeof rc).assert(rc.length>300); + fw = w.xWrap('jaccwabyt_test_str_hello', 'string:free',['i32']); + rc = fw(0); + T.assert('hello'===rc); + rc = fw(1); + T.assert(null===rc); + + w.xWrap.resultAdapter('thrice', (v)=>3n*BigInt(v)); + w.xWrap.argAdapter('twice', (v)=>2n*BigInt(v)); + fw = w.xWrap('jaccwabyt_test_int64_times2','thrice','twice'); + rc = fw(1); + T.assert(12n===rc); + + w.scopedAllocCall(function(){ + let pI1 = w.scopedAlloc(8), pI2 = pI1+4; + w.setMemValue(pI1, 0,'*')(pI2, 0, '*'); + let f = w.xWrap('jaccwabyt_test_int64_minmax',undefined,['i64*','i64*']); + let r1 = w.getMemValue(pI1, 'i64'), r2 = w.getMemValue(pI2, 'i64'); + T.assert(!Number.isSafeInteger(r1)).assert(!Number.isSafeInteger(r2)); + }); + } + }/*testWasmUtil()*/; + + const runTests = function(Module){ + //log("Module",Module); + const sqlite3 = Module.sqlite3, + capi = sqlite3.capi, + oo = sqlite3.oo1, + wasm = capi.wasm; + log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); + log("Build options:",wasm.compileOptionUsed()); + + if(1){ + /* Let's grab those last few lines of test coverage for + sqlite3-api.js... */ + const rc = wasm.compileOptionUsed(['COMPILER']); + T.assert(1 === rc.COMPILER); + const obj = {COMPILER:undefined}; + wasm.compileOptionUsed(obj); + T.assert(1 === obj.COMPILER); + } + log("WASM heap size =",wasm.heap8().length); + //log("capi.wasm.exports.__indirect_function_table",capi.wasm.exports.__indirect_function_table); + + const wasmCtypes = wasm.ctype; + //log("wasmCtypes",wasmCtypes); + T.assert(wasmCtypes.structs[0].name==='sqlite3_vfs'). + assert(wasmCtypes.structs[0].members.szOsFile.sizeof>=4). + assert(wasmCtypes.structs[1/*sqlite3_io_methods*/ + ].members.xFileSize.offset>0); + //log(wasmCtypes.structs[0].name,"members",wasmCtypes.structs[0].members); + [ /* Spot-check a handful of constants to make sure they got installed... */ + 'SQLITE_SCHEMA','SQLITE_NULL','SQLITE_UTF8', + 'SQLITE_STATIC', 'SQLITE_DIRECTONLY', + 'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE' + ].forEach(function(k){ + T.assert('number' === typeof capi[k]); + }); + [/* Spot-check a few of the WASM API methods. */ + 'alloc', 'dealloc', 'installFunction' + ].forEach(function(k){ + T.assert(capi.wasm[k] instanceof Function); + }); + + const db = new oo.DB(':memory:'), startTime = performance.now(); + try { + log("DB filename:",db.filename,db.fileName()); + const banner1 = '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', + banner2 = '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'; + [ + testWasmUtil, testBasicSanity, testUDF, + testAttach, testIntPtr, testStructStuff, + testSqliteStructs + ].forEach((f)=>{ + const t = T.counter, n = performance.now(); + logHtml(banner1,"Running",f.name+"()..."); + f(db, sqlite3, Module); + logHtml(banner2,f.name+"():",T.counter - t,'tests in',(performance.now() - n),"ms"); + }); + }finally{ + db.close(); + } + logHtml("Total Test count:",T.counter,"in",(performance.now() - startTime),"ms"); + log('capi.wasm.exports',capi.wasm.exports); + }; + + sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){ + /** Use a timeout so that we are (hopefully) out from under + the module init stack when our setup gets run. Just on + principle, not because we _need_ to be. */ + //console.debug("theModule =",theModule); + //setTimeout(()=>runTests(theModule), 0); + // ^^^ Chrome warns: "VIOLATION: setTimeout() handler took A WHOLE 50ms!" + self._MODULE = theModule /* this is only to facilitate testing from the console */ + runTests(theModule); + }); +})(); diff --git a/ext/fiddle/testing2.html b/ext/wasm/testing2.html similarity index 79% rename from ext/fiddle/testing2.html rename to ext/wasm/testing2.html index b773c4aa48..739c7f66be 100644 --- a/ext/fiddle/testing2.html +++ b/ext/wasm/testing2.html @@ -4,10 +4,9 @@ - - + + sqlite3-worker.js tests -
sqlite3-worker.js tests
@@ -25,8 +24,10 @@
-
Everything on this page happens in the dev console.
- +
Most stuff on this page happens in the dev console.
+
+
+ diff --git a/ext/wasm/testing2.js b/ext/wasm/testing2.js new file mode 100644 index 0000000000..3a279513f8 --- /dev/null +++ b/ext/wasm/testing2.js @@ -0,0 +1,340 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A basic test script for sqlite3-worker.js. +*/ +'use strict'; +(function(){ + const T = self.SqliteTestUtil; + const SW = new Worker("api/sqlite3-worker.js"); + const DbState = { + id: undefined + }; + const eOutput = document.querySelector('#test-output'); + const log = console.log.bind(console) + const logHtml = function(cssClass,...args){ + log.apply(this, args); + const ln = document.createElement('div'); + if(cssClass) ln.classList.add(cssClass); + ln.append(document.createTextNode(args.join(' '))); + eOutput.append(ln); + }; + const warn = console.warn.bind(console); + const error = console.error.bind(console); + const toss = (...args)=>{throw new Error(args.join(' '))}; + /** Posts a worker message as {type:type, data:data}. */ + const wMsg = function(type,data){ + log("Posting message to worker dbId="+(DbState.id||'default')+':',data); + SW.postMessage({ + type, + dbId: DbState.id, + data, + departureTime: performance.now() + }); + return SW; + }; + + SW.onerror = function(event){ + error("onerror",event); + }; + + let startTime; + + /** + A queue for callbacks which are to be run in response to async + DB commands. See the notes in runTests() for why we need + this. The event-handling plumbing of this file requires that + any DB command which includes a `messageId` property also have + a queued callback entry, as the existence of that property in + response payloads is how it knows whether or not to shift an + entry off of the queue. + */ + const MsgHandlerQueue = { + queue: [], + id: 0, + push: function(type,callback){ + this.queue.push(callback); + return type + '-' + (++this.id); + }, + shift: function(){ + return this.queue.shift(); + } + }; + + const testCount = ()=>{ + logHtml("","Total test count:",T.counter+". Total time =",(performance.now() - startTime),"ms"); + }; + + const logEventResult = function(evd){ + logHtml(evd.errorClass ? 'error' : '', + "runOneTest",evd.messageId,"Worker time =", + (evd.workerRespondTime - evd.workerReceivedTime),"ms.", + "Round-trip event time =", + (performance.now() - evd.departureTime),"ms.", + (evd.errorClass ? evd.message : "") + ); + }; + + const runOneTest = function(eventType, eventData, callback){ + T.assert(eventData && 'object'===typeof eventData); + /* ^^^ that is for the testing and messageId-related code, not + a hard requirement of all of the Worker-exposed APIs. */ + eventData.messageId = MsgHandlerQueue.push(eventType,function(ev){ + logEventResult(ev.data); + if(callback instanceof Function){ + callback(ev); + testCount(); + } + }); + wMsg(eventType, eventData); + }; + + /** Methods which map directly to onmessage() event.type keys. + They get passed the inbound event.data. */ + const dbMsgHandler = { + open: function(ev){ + DbState.id = ev.dbId; + log("open result",ev.data); + }, + exec: function(ev){ + log("exec result",ev.data); + }, + export: function(ev){ + log("export result",ev.data); + }, + error: function(ev){ + error("ERROR from the worker:",ev.data); + logEventResult(ev.data); + }, + resultRowTest1: function f(ev){ + if(undefined === f.counter) f.counter = 0; + if(ev.data) ++f.counter; + //log("exec() result row:",ev.data); + T.assert(null===ev.data || 'number' === typeof ev.data.b); + } + }; + + /** + "The problem" now is that the test results are async. We + know, however, that the messages posted to the worker will + be processed in the order they are passed to it, so we can + create a queue of callbacks to handle them. The problem + with that approach is that it's not error-handling + friendly, in that an error can cause us to bypass a result + handler queue entry. We have to perform some extra + acrobatics to account for that. + + Problem #2 is that we cannot simply start posting events: we + first have to post an 'open' event, wait for it to respond, and + collect its db ID before continuing. If we don't wait, we may + well fire off 10+ messages before the open actually responds. + */ + const runTests2 = function(){ + const mustNotReach = ()=>{ + throw new Error("This is not supposed to be reached."); + }; + runOneTest('exec',{ + sql: ["create table t(a,b)", + "insert into t(a,b) values(1,2),(3,4),(5,6)" + ].join(';'), + multi: true, + resultRows: [], columnNames: [] + }, function(ev){ + ev = ev.data; + T.assert(0===ev.resultRows.length) + .assert(0===ev.columnNames.length); + }); + runOneTest('exec',{ + sql: 'select a a, b b from t order by a', + resultRows: [], columnNames: [], + }, function(ev){ + ev = ev.data; + T.assert(3===ev.resultRows.length) + .assert(1===ev.resultRows[0][0]) + .assert(6===ev.resultRows[2][1]) + .assert(2===ev.columnNames.length) + .assert('b'===ev.columnNames[1]); + }); + runOneTest('exec',{ + sql: 'select a a, b b from t order by a', + resultRows: [], columnNames: [], + rowMode: 'object' + }, function(ev){ + ev = ev.data; + T.assert(3===ev.resultRows.length) + .assert(1===ev.resultRows[0].a) + .assert(6===ev.resultRows[2].b) + }); + runOneTest('exec',{sql:'intentional_error'}, mustNotReach); + // Ensure that the message-handler queue survives ^^^ that error... + runOneTest('exec',{ + sql:'select 1', + resultRows: [], + //rowMode: 'array', // array is the default in the Worker interface + }, function(ev){ + ev = ev.data; + T.assert(1 === ev.resultRows.length) + .assert(1 === ev.resultRows[0][0]); + }); + runOneTest('exec',{ + sql: 'select a a, b b from t order by a', + callback: 'resultRowTest1', + rowMode: 'object' + }, function(ev){ + T.assert(3===dbMsgHandler.resultRowTest1.counter); + dbMsgHandler.resultRowTest1.counter = 0; + }); + runOneTest('exec',{ + multi: true, + sql:[ + 'pragma foreign_keys=0;', + // ^^^ arbitrary query with no result columns + 'select a, b from t order by a desc; select a from t;' + // multi-exec only honors results from the first + // statement with result columns (regardless of whether) + // it has any rows). + ], + rowMode: 1, + resultRows: [] + },function(ev){ + const rows = ev.data.resultRows; + T.assert(3===rows.length). + assert(6===rows[0]); + }); + runOneTest('exec',{sql: 'delete from t where a>3'}); + runOneTest('exec',{ + sql: 'select count(a) from t', + resultRows: [] + },function(ev){ + ev = ev.data; + T.assert(1===ev.resultRows.length) + .assert(2===ev.resultRows[0][0]); + }); + if(0){ + // export requires reimpl. for portability reasons. + runOneTest('export',{}, function(ev){ + ev = ev.data; + T.assert('string' === typeof ev.filename) + .assert(ev.buffer instanceof Uint8Array) + .assert(ev.buffer.length > 1024) + .assert('application/x-sqlite3' === ev.mimetype); + }); + } + /***** close() tests must come last. *****/ + runOneTest('close',{unlink:true},function(ev){ + ev = ev.data; + T.assert('string' === typeof ev.filename); + }); + runOneTest('close',{unlink:true},function(ev){ + ev = ev.data; + T.assert(undefined === ev.filename); + }); + }; + + const runTests = function(){ + /** + Design decision time: all remaining tests depend on the 'open' + command having succeeded. In order to support multiple DBs, the + upcoming commands ostensibly have to know the ID of the DB they + want to talk to. We have two choices: + + 1) We run 'open' and wait for its response, which contains the + db id. + + 2) We have the Worker automatically use the current "default + db" (the one which was most recently opened) if no db id is + provided in the message. When we do this, the main thread may + well fire off _all_ of the test messages before the 'open' + actually responds, but because the messages are handled on a + FIFO basis, those after the initial 'open' will pick up the + "default" db. However, if the open fails, then all pending + messages (until next next 'open', at least) except for 'close' + will fail and we have no way of cancelling them once they've + been posted to the worker. + + We currently do (2) because (A) it's certainly the most + client-friendly thing to do and (B) it seems likely that most + apps using this API will only have a single db to work with so + won't need to juggle multiple DB ids. If we revert to (1) then + the following call to runTests2() needs to be moved into the + callback function of the runOneTest() check for the 'open' + command. Note, also, that using approach (2) does not keep the + user from instead using approach (1), noting that doing so + requires explicit handling of the 'open' message to account for + it. + */ + const waitForOpen = 1, + simulateOpenError = 0 /* if true, the remaining tests will + all barf if waitForOpen is + false. */; + logHtml('', + "Sending 'open' message and",(waitForOpen ? "" : "NOT ")+ + "waiting for its response before continuing."); + startTime = performance.now(); + runOneTest('open', { + filename:'testing2.sqlite3', + simulateError: simulateOpenError + }, function(ev){ + //log("open result",ev); + T.assert('testing2.sqlite3'===ev.data.filename) + .assert(ev.data.dbId) + .assert(ev.data.messageId); + DbState.id = ev.data.dbId; + if(waitForOpen) setTimeout(runTests2, 0); + }); + if(!waitForOpen) runTests2(); + }; + + SW.onmessage = function(ev){ + if(!ev.data || 'object'!==typeof ev.data){ + warn("Unknown sqlite3-worker message type:",ev); + return; + } + ev = ev.data/*expecting a nested object*/; + //log("main window onmessage:",ev); + if(ev.data && ev.data.messageId){ + /* We're expecting a queued-up callback handler. */ + const f = MsgHandlerQueue.shift(); + if('error'===ev.type){ + dbMsgHandler.error(ev); + return; + } + T.assert(f instanceof Function); + f(ev); + return; + } + switch(ev.type){ + case 'sqlite3-api': + switch(ev.data){ + case 'worker-ready': + log("Message:",ev); + self.sqlite3TestModule.setStatus(null); + runTests(); + return; + default: + warn("Unknown sqlite3-api message type:",ev); + return; + } + default: + if(dbMsgHandler.hasOwnProperty(ev.type)){ + try{dbMsgHandler[ev.type](ev);} + catch(err){ + error("Exception while handling db result message", + ev,":",err); + } + return; + } + warn("Unknown sqlite3-api message type:",ev); + } + }; + log("Init complete, but async init bits may still be running."); +})(); diff --git a/manifest b/manifest index 913da59c3c..8e9f19a064 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C wasm/fiddle\srefactoring\spart\s1\sof\sN:\smove\sfiddle\sapp\sfrom\sext/fiddle\sto\sext/wasm/fiddle,\swhich\sonly\scontains\sfiles\sintended\sto\sbe\spushed\sto\sthe\slive\ssite.\sDisabled\sbuild\sof\sthe\snon-fiddle\swasm\sparts,\spending\sa\slater\sstep\sof\sthe\srefactoring. -D 2022-08-10T09:36:10.232 +C wasm\srefactoring\spart\s2\sof\s(apparently)\s2:\smoved\sext/fiddle/...\sinto\sext/wasm\sand\srestructured\sthe\score\sAPI-related\sparts\sof\sthe\sJS/WASM\sconsiderably. +D 2022-08-10T11:26:08.660 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in a77d419b19eb2f806109ae2d0b81abb39a3a8659b00e528da7e27bd95c7e29fd +F Makefile.in 06385361460c98e2f2fafc4ec698d726b296d9b89b214d610531a6b4341c8279 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -55,16 +55,6 @@ F ext/expert/expert1.test 3c642a4e7bbb14f21ddab595436fb465a4733f47a0fe5b2855e1d5 F ext/expert/sqlite3expert.c 6ca30d73b9ed75bd56d6e0d7f2c962d2affaa72c505458619d0ff5d9cdfac204 F ext/expert/sqlite3expert.h ca81efc2679a92373a13a3e76a6138d0310e32be53d6c3bfaedabd158ea8969b F ext/expert/test_expert.c d56c194b769bdc90cf829a14c9ecbc1edca9c850b837a4d0b13be14095c32a72 -F ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api 356c356931b58eccf68367120f304db43ab6c2ef2f62f17f12f5a99737b43c38 -F ext/fiddle/SqliteTestUtil.js 2e87d424b12674476bdf8139934dcacc3ff8a7a5f5ff4392ba5e5a8d8cee9fbd -F ext/fiddle/sqlite3-api.js 5a6cc120f3eeaab65e49bcdab234e83d83c67440e04bd97191bdc004ac0cda35 -F ext/fiddle/sqlite3-worker.js 50b7a9ce14c8fae0af965e35605fe12cafb79c1e01e99216d8110d8b02fbf4b5 -F ext/fiddle/testing.css 750572dded671d2cf142bbcb27af5542522ac08db128245d0b9fe410aa1d7f2a -F ext/fiddle/testing1.html ea1f3be727f78e420007f823912c1a03b337ecbb8e79449abc2244ad4fe15d9a -F ext/fiddle/testing1.js fbeac92a5ac22668a54fd358ffc75c275d83e505e770aa484045614cb2a6cf44 -F ext/fiddle/testing2.html 9063b2430ade2fe9da4e711addd1b51a2741cf0c7ebf6926472a5e5dd63c0bc4 -F ext/fiddle/testing2.js 7b45b4e7fddbd51dbaf89b6722c02758051b34bac5a98c11b569a7e7572f88ee -F ext/fiddle/wasm_util.c 5944e38a93d3a436a47dd7c69059844697b6a5e44499656baa6da0ffe7fe23d5 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea @@ -482,14 +472,39 @@ F ext/session/test_session.c f433f68a8a8c64b0f5bc74dc725078f12483301ad4ae8375205 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb -F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 w ext/fiddle/EXPORTED_FUNCTIONS.fiddle -F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 w ext/fiddle/EXPORTED_RUNTIME_METHODS -F ext/wasm/GNUmakefile c71257754d3f69ed19308e91c2829be98532aa27ba1feaefe53d2bf17c047dc8 w ext/fiddle/Makefile -F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 w ext/fiddle/index.md -F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f w ext/fiddle/emscripten.css -F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae w ext/fiddle/fiddle-worker.js -F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 w ext/fiddle/fiddle.html -F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 w ext/fiddle/fiddle.js +F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3 +F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02 +F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3b1ae2b1 +F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331 +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33 w ext/fiddle/EXPORTED_FUNCTIONS.sqlite3-api +F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 +F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c +F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a4a06cb776c003880b +F ext/wasm/api/sqlite3-api-cleanup.js 149fd63a0400cd1d69548887ffde2ed89c13283384a63c2e9fcfc695e38a9e11 +F ext/wasm/api/sqlite3-api-glue.js 82c09f49c69984009ba5af2b628e67cc26c5dd203d383cd3091d40dab4e6514b +F ext/wasm/api/sqlite3-api-oo1.js e9612cb704c0563c5d71ed2a8dccd95bf6394fa4de3115d1b978dc269c49ab02 +F ext/wasm/api/sqlite3-api-opfs.js 0a4aa8993ae54c58a7925f547ff496bd09af2e3ef80f93168edec218ab96a182 +F ext/wasm/api/sqlite3-api-prologue.js 0fb0703d2d8ac89fa2d4dd8f9726b0ea226b8708ac34e5b482df046e147de0eb w ext/fiddle/sqlite3-api.js +F ext/wasm/api/sqlite3-api-worker.js cae932a89e48730cd850ab280963a65a96cb8b4c58bacd54ba961991a3c32f51 w ext/fiddle/sqlite3-worker.js +F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 +F ext/wasm/api/sqlite3-wasm.c 2d3e6dea54ecaa58bbb2f74a051fa65bfcf6f5e5fc96fd62f1763443f8cd36e4 +F ext/wasm/api/sqlite3-worker.js 1325ca8d40129a82531902a3a077b795db2eeaee81746e5a0c811a04b415fa7f +F ext/wasm/common/SqliteTestUtil.js e41a1406f18da9224523fad0c48885caf995b56956a5b9852909c0989e687e90 w ext/fiddle/SqliteTestUtil.js +F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f +F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0 w ext/fiddle/testing.css +F ext/wasm/common/whwasmutil.js 3d9deda1be718e2b10e2b6b474ba6ba857d905be314201ae5b3df5eef79f66aa +F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f +F ext/wasm/fiddle/fiddle-worker.js 88bc2193a6cb6a3f04d8911bed50a4401fe6f277de7a71ba833865ab64a1b4ae +F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 +F ext/wasm/fiddle/fiddle.js 812f9954cc7c4b191884ad171f36fcf2d0112d0a7ecfdf6087896833a0c079a8 +F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9de2a985f3c21e7082b2 +F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 +F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f +F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19 +F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231 w ext/fiddle/testing1.html +F ext/wasm/testing1.js aef553114aada187eef125f5361fd1e58bf5e8e97acfa65c10cb41dd60295daa w ext/fiddle/testing1.js +F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8 w ext/fiddle/testing2.html +F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e42238ebc w ext/fiddle/testing2.js F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1983,8 +1998,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c3a3cb0103126210692bbeb703e7b8793974042e1fc2473be6d0a0d9b07d5770 -R 7933790875a6b3f54b530ce22fe4d8f0 +P fb4eb93080288b60815be14afd7ddbbca470ce363fa3735352ea9a558fef583e +R 1fc860ea2e3b64c464acaae9c6ad06c1 U stephan -Z f8f037574e62bc4aeafe6c26802ca165 +Z 47854f455ec6bca85db3d0b3dd61eec0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1031463cb5..e6efbffcca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb4eb93080288b60815be14afd7ddbbca470ce363fa3735352ea9a558fef583e \ No newline at end of file +27f9da4eaaff39d1d58e9ffef7ddccf1e41b3726914f754b920e3e1fb572cba6 \ No newline at end of file