From 85ef4e434415ac8e674f4f92480a38cd7c3a135e Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 3 Dec 2022 11:16:55 +0000 Subject: [PATCH] JavaScript: add sqlite3.wasm.realloc(), sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE, and related tests. FossilOrigin-Name: eeb84ba5de1152ef0f42105b8b285fdee9f5ad58281e60a4e0c8b1d6de1dead8 --- ext/wasm/GNUmakefile | 2 +- ext/wasm/api/sqlite3-api-glue.js | 1 + ext/wasm/api/sqlite3-api-prologue.js | 92 +++++++++++++++++++--------- ext/wasm/api/sqlite3-wasm.c | 10 +++ ext/wasm/tester1-worker.html | 2 +- ext/wasm/tester1.c-pp.js | 32 ++++++++++ manifest | 22 +++---- manifest.uuid | 2 +- 8 files changed, 121 insertions(+), 42 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 71c508f7c2..968d4f440c 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -489,7 +489,7 @@ emcc.jsflags += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY)) emcc.jsflags += $(emcc.environment) emcc.jsflags += -sSTACK_SIZE=1MB -# ^^^ ACHTUNG: emsdk 3.1.27 reduced the default stack size from 4MB to +# ^^^ ACHTUNG: emsdk 3.1.27 reduced the default stack size from 5MB to # a mere 64KB, which leads to silent memory corruption via the kvvfs # VFS, which requires twice that for its xRead() and xWrite() methods. ######################################################################## diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index e60baeb7f3..ac1e9fd6d6 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -605,6 +605,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); for(const t of ['access', 'blobFinalizers', 'dataTypes', 'encodings', 'fcntl', 'flock', 'ioCap', + 'limits', 'openFlags', 'prepareFlags', 'resultCodes', 'serialize', 'syncFlags', 'trace', 'udfFlags', 'version' diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index a99065663d..b6ff94dfe4 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -75,6 +75,10 @@ the `free(3)`-compatible routine for the WASM environment. Defaults to `"sqlite3_free"`. + - `reallocExportName`: the name of the function, in `exports`, of + the `realloc(3)`-compatible routine for the WASM + environment. Defaults to `"sqlite3_realloc"`. + - `wasmfsOpfsDir`[^1]: if the environment supports persistent storage using OPFS-over-WASMFS , this directory names the "mount point" for that directory. It must be prefixed by `/` and may @@ -110,6 +114,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( })(), allocExportName: 'sqlite3_malloc', deallocExportName: 'sqlite3_free', + reallocExportName: 'sqlite3_realloc', wasmfsOpfsDir: '/opfs' }, apiConfig || {}); @@ -284,12 +289,14 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( }; /** - Returns true if v appears to be one of our bind()-able - TypedArray types: Uint8Array or Int8Array. Support for - TypedArrays with element sizes >1 is TODO. + Returns true if v appears to be one of our bind()-able TypedArray + types: Uint8Array or Int8Array. Support for TypedArrays with + element sizes >1 is a potential TODO just waiting on a use case + to justify them. */ const isBindableTypedArray = (v)=>{ - return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + return v && (v instanceof Uint8Array || v instanceof Int8Array); + //v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); }; /** @@ -302,7 +309,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( isSQLableTypedArray() list. */ const isSQLableTypedArray = (v)=>{ - return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + return v && (v instanceof Uint8Array || v instanceof Int8Array); + //v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); }; /** Returns true if isBindableTypedArray(v) does, else throws with a message @@ -664,12 +672,12 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( "or config.memory (imported)."), /** - The API's one single point of access to the WASM-side memory - allocator. Works like malloc(3) (and is likely bound to - malloc()) but throws an WasmAllocError if allocation fails. It is - important that any code which might pass through the sqlite3 C - API NOT throw and must instead return SQLITE_NOMEM (or - equivalent, depending on the context). + The API's primary point of access to the WASM-side memory + allocator. Works like sqlite3_malloc() but throws a + WasmAllocError if allocation fails. It is important that any + code which might pass through the sqlite3 C API NOT throw and + must instead return SQLITE_NOMEM (or equivalent, depending on + the context). Very few cases in the sqlite3 JS APIs can result in client-defined functions propagating exceptions via the C-style @@ -681,7 +689,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( catch exceptions and convert them to appropriate error codes. For cases where non-throwing allocation is required, use - sqlite3.wasm.alloc.impl(), which is direct binding of the + this.alloc.impl(), which is direct binding of the underlying C-level allocator. Design note: this function is not named "malloc" primarily @@ -692,9 +700,27 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( alloc: undefined/*installed later*/, /** - The API's one single point of access to the WASM-side memory - deallocator. Works like free(3) (and is likely bound to - free()). + Rarely necessary in JS code, this routine works like + sqlite3_realloc(M,N), where M is either NULL or a pointer + obtained from this function or this.alloc() and N is the number + of bytes to reallocate the block to. Returns a pointer to the + reallocated block or 0 if allocation fails. + + If M is NULL and N is positive, this behaves like + this.alloc(N). If N is 0, it behaves like this.dealloc(). + Results are undefined if N is negative (sqlite3_realloc() + treats that as 0, but if this code is built with a different + allocator it may misbehave with negative values). + + Like this.alloc.impl(), this.realloc.impl() is a direct binding + to the underlying realloc() implementation which does not throw + exceptions, instead returning 0 on allocation error. + */ + realloc: undefined/*installed later*/, + + /** + The API's primary point of access to the WASM-side memory + deallocator. Works like sqlite3_free(). Design note: this function is not named "free" for the same reason that this.alloc() is not called this.malloc(). @@ -731,20 +757,30 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( return pRet; }; - const keyAlloc = config.allocExportName, - keyDealloc = config.deallocExportName; - for(const key of [keyAlloc, keyDealloc]){ - const f = wasm.exports[key]; - if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); - } + { + // Set up allocators... + const keyAlloc = config.allocExportName, + keyDealloc = config.deallocExportName, + keyRealloc = config.reallocExportName; + for(const key of [keyAlloc, keyDealloc, keyRealloc]){ + const f = wasm.exports[key]; + if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); + } - wasm.alloc = function f(n){ - const m = f.impl(n); - if(!m) throw new WasmAllocError("Failed to allocate",n," bytes."); - return m; - }; - wasm.alloc.impl = wasm.exports[keyAlloc]; - wasm.dealloc = wasm.exports[keyDealloc]; + wasm.alloc = function f(n){ + const m = f.impl(n); + if(!m) throw new WasmAllocError("Failed to allocate",n," bytes."); + return m; + }; + wasm.alloc.impl = wasm.exports[keyAlloc]; + wasm.realloc = function f(m,n){ + const m2 = f.impl(m,n); + if(n && !m2) throw new WasmAllocError("Failed to reallocate",n," bytes."); + return n ? m2 : 0; + }; + wasm.realloc.impl = wasm.exports[keyRealloc]; + wasm.dealloc = wasm.exports[keyDealloc]; + } /** Reports info about compile-time options using diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 9acc8020e5..cbc4499760 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -112,6 +112,12 @@ # define SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION #endif +/**********************************************************************/ +/* SQLITE_M... */ +#ifndef SQLITE_MAX_ALLOCATION_SIZE +# define SQLITE_MAX_ALLOCATION_SIZE 0x1fffffff +#endif + /**********************************************************************/ /* SQLITE_O... */ #ifndef SQLITE_OMIT_DEPRECATED @@ -497,6 +503,10 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_IOCAP_BATCH_ATOMIC); } _DefGroup; + DefGroup(limits) { + DefInt(SQLITE_MAX_ALLOCATION_SIZE); + } _DefGroup; + DefGroup(openFlags) { /* Noting that not all of these will have any effect in ** WASM-space. */ diff --git a/ext/wasm/tester1-worker.html b/ext/wasm/tester1-worker.html index ee03874df3..eba3fdeb4e 100644 --- a/ext/wasm/tester1-worker.html +++ b/ext/wasm/tester1-worker.html @@ -45,7 +45,7 @@ if(urlParams.has('esm')){ logHtml('warning',"Attempting to run an ES6 Worker Module, "+ "which is not supported by all browsers! "+ - "e.g. Firefox (as of 2022-11) cannot do this."); + "e.g. Firefox (as of 2022-12) cannot do this."); workerArgs.push("tester1.mjs",{type:"module"}); document.querySelectorAll('title,#color-target').forEach((e)=>{ e.innerText = "sqlite3 tester #1: ES6 Worker Module"; diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 1db8499ef6..949d2365c4 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -424,6 +424,38 @@ self.sqlite3InitModule = sqlite3InitModule; } } + // alloc(), realloc(), allocFromTypedArray() + { + let m = w.alloc(14); + let m2 = w.realloc(m, 16); + T.assert(m === m2/* because of alignment */); + T.assert(0 === w.realloc(m, 0)); + m = m2 = 0; + + // Check allocation limits and allocator's responses... + T.assert('number' === typeof sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE); + const tooMuch = sqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE + 1, + isAllocErr = (e)=>e instanceof sqlite3.WasmAllocError; + T.mustThrowMatching(()=>w.alloc(tooMuch), isAllocErr) + .assert(0 === w.alloc.impl(tooMuch)) + .mustThrowMatching(()=>w.realloc(0, tooMuch), isAllocErr) + .assert(0 === w.realloc.impl(0, tooMuch)); + + // Check allocFromTypedArray()... + const byteList = [11,22,33] + const u = new Uint8Array(byteList); + m = w.allocFromTypedArray(u); + for(let i = 0; i < u.length; ++i){ + T.assert(u[i] === byteList[i]) + .assert(u[i] === w.getMemValue(m + i, 'i8')); + } + w.dealloc(m); + T.mustThrowMatching( + ()=>w.allocFromTypedArray(1), + 'Value is not of a supported TypedArray type.' + ); + } + // isPtr32() { const ip = w.isPtr32; diff --git a/manifest b/manifest index da738ab08b..696930b4d7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm\sbuild:\srename\sthe\spath\sto\sthe\swasm\sdocs\scheckout,\sfor\sclarity\sand\sconsistency. -D 2022-12-03T03:06:16.365 +C JavaScript:\sadd\ssqlite3.wasm.realloc(),\ssqlite3.capi.SQLITE_MAX_ALLOCATION_SIZE,\sand\srelated\stests. +D 2022-12-03T11:16:55.292 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 004f3662abc5588d0e460893ad8f0fd94c970957159b6bba97087bac27d78007 +F ext/wasm/GNUmakefile bfa47f169468ca9db031105b0e336db29a88e93c3abd217d0bbb2b8731fa5413 F ext/wasm/README-dist.txt 2d670b426fc7c613b90a7d2f2b05b433088fe65181abead970980f0a4a75ea20 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b4d68c97d14944b48d55e06aa44f544a6f56a7fa2bcb6f9e030936a5b2a9479a @@ -503,16 +503,16 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js b88499dc303c21fc3f55f2c364a0f814f587b60a95784303881169f9e91c1d5f F ext/wasm/api/sqlite3-api-cleanup.js 680d5ccfff54459db136a49b2199d9f879c8405d9c99af1dda0cc5e7c29056f4 -F ext/wasm/api/sqlite3-api-glue.js b528207ba43f7740d1ade623f3f6b08a49f44ce7e9126915b78e1818c2466d8e +F ext/wasm/api/sqlite3-api-glue.js 6fe39964605fda3b699f69365eed565b5172d29cab2c49bc057a43f9a93f9f36 F ext/wasm/api/sqlite3-api-oo1.js 91a7d7b9203fb0f031e6ba380a644a7f871e1798b388de399c01ed4087bac9e0 -F ext/wasm/api/sqlite3-api-prologue.js 42d6b316b542cf8e086f2f272460deb72dff184f1438a3377383cab99b08070b +F ext/wasm/api/sqlite3-api-prologue.js 3d9550021269fd97636595ea2d2a1ec3ce00866f0a5d3b5fab94d4583afdafe0 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3 F ext/wasm/api/sqlite3-opfs-async-proxy.js f79dd8d98ef3e0b55c10bb2bee7a3840fa967318e1f577c156aafc34664271d1 F ext/wasm/api/sqlite3-vfs-helper.js 4ad4faf02e1524bf0296be8452c00b5708dce6faf649468d0377e26a0b299263 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 29d6487a26b2fb6a471cde52c37ffee7c27ed6a91914b308c247e0706f454ffb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 733bc939f93caef0df0b3ebfea14cbd528da580fdef1a35b1f69c2b3e044c7b7 +F ext/wasm/api/sqlite3-wasm.c 69c2c1bf555dd25596137bf282d721657d5c49243061e0cb420375203107adcd F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b F ext/wasm/api/sqlite3-worker1.js 1e54ea3d540161bcfb2100368a2fc0cad871a207b8336afee1c445715851ec54 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -553,9 +553,9 @@ F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d826 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555e685bce3da8c3f F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d962f15e73bf2ac -F ext/wasm/tester1-worker.html ead6bdcc6cca221deb0dc9855a56f376351dbf2294fd7978cd1609b3a56b245b +F ext/wasm/tester1-worker.html 29b1d87f7d51f70d61645719fee657f3787fe939bb695f27034c75404e8f1e6f F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399 -F ext/wasm/tester1.c-pp.js e73a91eba4b59aaadd98f383c00a5101dbbbc52d937fff3162fc4761986f4a88 +F ext/wasm/tester1.c-pp.js d25cea43933bf86590aab63038a6a0b6e7002ffba7e85d8df2720b7a69f85690 F ext/wasm/tests/opfs/concurrency/index.html 86d8ac435074d1e7007b91105f4897f368c165e8cecb6a9aa3d81f5cf5dcbe70 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2065,8 +2065,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 13eb1abd06f55fb88fc7f7be6149fd94b12057d9c38cc6b97bec3940e7e01f04 -R f5c92ee96049664c2a17edeac52c505f +P b820db32365b2ca8e2397fd6ea85883e4555ffd82948e248a0f98415b7328349 +R c074a7899efe7412c63b603c13447040 U stephan -Z a6b554242e7c212e38bd6fdb1ff7082c +Z 037a89bfec293517ee42832df5b904a5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 65b7f302f2..ba0da199e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b820db32365b2ca8e2397fd6ea85883e4555ffd82948e248a0f98415b7328349 \ No newline at end of file +eeb84ba5de1152ef0f42105b8b285fdee9f5ad58281e60a4e0c8b1d6de1dead8 \ No newline at end of file