From 50c61825fc020542477719787ac9a2838fa429f3 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 14 Jul 2023 21:17:29 +0000 Subject: [PATCH 01/56] Initial sketches for an alternate OPFS VFS which uses a pool of pre-opened SyncAccessHandles to bypass the need for a dedicated I/O worker and the COOP/COEP HTTP response headers. Currently completely non-functional. FossilOrigin-Name: a93de9f2a553a3a4edd1b361dd6f465a1b0b5b51f7bb8ede432067aedcfefda4 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 310 +++++++++++++++++++++++ manifest | 14 +- manifest.uuid | 2 +- 3 files changed, 320 insertions(+), 6 deletions(-) create mode 100644 ext/wasm/api/sqlite3-vfs-opfs-sahpool.js diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js new file mode 100644 index 0000000000..0832b32bbf --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -0,0 +1,310 @@ +/* + 2023-07-14 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + INCOMPLETE! WORK IN PROGRESS! + + This file holds an experimental sqlite3_vfs backed by OPFS storage + which uses a different implementation strategy than the "opfs" + VFS. This one is a port of Roy Hashimoto's OPFS SyncAccessHandle + pool: + + https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/AccessHandlePoolVFS.js + + As described at: + + https://github.com/rhashimoto/wa-sqlite/discussions/67 + + with Roy's explicit permission to permit us to port his to our + infrastructure rather than having to clean-room reverse-engineer it: + + https://sqlite.org/forum/forumpost/e140d84e71 + + Primary differences from the original "opfs" VFS include: + + - This one avoids the need for a sub-worker to synchronize + communication between the synchronous C API and the only-partly + synchronous OPFS API. + + - It does so by opening a fixed number of OPFS files at + library-level initialization time, obtaining SyncAccessHandles to + each, and manipulating those handles via the synchronous sqlite3_vfs + interface. + + - Because of that, this one lacks all library-level concurrency + support. + + - Also because of that, it does not require the SharedArrayBuffer, + so can function without the COOP/COEP HTTP response headers. + + - It can hypothetically support Safari 16.4+, whereas the "opfs" + VFS requires v17 due to a bug in 16.x which makes it incompatible + with that VFS. + + - This VFS requires the "semi-fully-sync" FileSystemSyncAccessHandle + (hereafter "SAH") APIs released with Chrome v108. There is + unfortunately no known programmatic way to determine whether a given + API is from that release or newer without actually calling it and + checking whether one of the "fully-sync" functions returns a Promise + (in which case it's the older version). (Reminder to self: when + opening up the initial pool of files, we can close() the first one + we open and see if close() returns a Promise. If it does, it's the + older version so fail VFS initialization. If it doesn't, re-open it.) + +*/ +'use strict'; +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +const installOpfsVfs = async function(sqlite3){ + const pToss = (...args)=>Promise.reject(new Error(args.join(' '))); + if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ + return pToss("Missing required OPFS APIs."); + } + const thePromise = new Promise(function(promiseResolve, promiseReject_){ + const verbosity = 3; + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const logImpl = (level,...args)=>{ + if(verbosity>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const toss = sqlite3.util.toss; + const capi = sqlite3.capi; + const wasm = sqlite3.wasm; + const opfsIoMethods = new capi.sqlite3_io_methods(); + const opfsVfs = new capi.sqlite3_vfs() + .addOnDispose(()=>opfsIoMethods.dispose()); + const promiseReject = (err)=>{ + opfsVfs.dispose(); + return promiseReject_(err); + }; + + // Config opts for the VFS... + const SECTOR_SIZE = 4096; + const HEADER_MAX_PATH_SIZE = 512; + const HEADER_FLAGS_SIZE = 4; + const HEADER_DIGEST_SIZE = 8; + const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE; + const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE; + const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; + const HEADER_OFFSET_DATA = SECTOR_SIZE; + const DEFAULT_CAPACITY = 6; + /* Bitmask of file types which may persist across sessions. + SQLITE_OPEN_xyz types not listed here may be inadvertently + left in OPFS but are treated as transient by this VFS and + they will be cleaned up during VFS init. */ + const PERSISTENT_FILE_TYPES = + capi.SQLITE_OPEN_MAIN_DB | + capi.SQLITE_OPEN_MAIN_JOURNAL | + capi.SQLITE_OPEN_SUPER_JOURNAL | + capi.SQLITE_OPEN_WAL /* noting that WAL support is + unavailable in the WASM build.*/; + const pDVfs = capi.sqlite3_vfs_find(null)/*default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. */; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; + opfsVfs.$zName = wasm.allocCString("opfs-sahpool"); + opfsVfs.addOnDispose( + '$zName', opfsVfs.$zName, + 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) + ); + + const VState = Object.assign(Object.create(null),{ + /* OPFS dir in which VFS metadata is stored. */ + vfsDir: ".sqlite3-sahpool", + dirHandle: undefined, + /* Maps OPFS access handles to their opaque file names. */ + mapAH2Name: new Map(), + mapPath2AH: new Map(), + availableAH: new Set(), + mapId2File: new Map(), + getCapacity: function(){return this.mapAH2Name.size}, + getFileCount: function(){return this.mapPath2AH.size}, + addCapacity: async function(n){ + for(let i = 0; i < n; ++i){ + const name = Math.random().toString(36).replace('0.',''); + const h = await this.dirHandle.getFileName(name, {create:true}); + const ah = await h.createSyncAccessHandle(); + this.mapAH2Name(ah,name); + this.setAssociatedPath(ah, '', 0); + } + }, + setAssociatedPath: function(accessHandle, path, flags){ + // TODO + }, + releaseAccessHandles: function(){ + for(const ah of this.mapAH2Name.keys()) ah.close(); + this.mapAH2Name.clear(); + this.mapPath2AH.clear(); + this.availableAH.clear(); + }, + acquireAccessHandles: async function(){ + // TODO + }, + reset: async function(){ + await this.isReady; + let h = await navigator.storage.getDirectory(); + for(const d of this.vfsDir.split('/')){ + if(d){ + h = await h.getDirectoryHandle(d,{create:true}); + } + } + this.dirHandle = h; + this.releaseAccessHandles(); + await this.acquireAccessHandles(); + } + // much more TODO + })/*VState*/; + + // Much, much more TODO... + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = { + xCheckReservedLock: function(pFile,pOut){ + return 0; + }, + xClose: function(pFile){ + let rc = 0; + return rc; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + let rc = 0; + return rc; + }, + xLock: function(pFile,lockType){ + let rc = capi.SQLITE_IOERR_LOCK; + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + let rc = capi.SQLITE_IOERR_READ; + return rc; + }, + xSync: function(pFile,flags){ + let rc = capi.SQLITE_IOERR_FSYNC; + return rc; + }, + xTruncate: function(pFile,sz64){ + let rc = capi.SQLITE_IOERR_TRUNCATE; + return rc; + }, + xUnlock: function(pFile,lockType){ + let rc = capi.SQLITE_IOERR_UNLOCK; + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + let rc = capi.SQLITE_IOERR_WRITE; + return rc; + } + }/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + const rc = capi.SQLITE_ERROR; + return rc; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + const rc = capi.SQLITE_ERROR; + return rc; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + const i = wasm.cstrncpy(pOut, zName, nOut); + return i{ + if(0===VState.getCapacity())[ + await VState.addCapacity(DEFAULT_CAPACITY); + } + promiseResolve(sqlite3); + }).catch(promiseReject); + })/*thePromise*/; + return thePromise; +}/*installOpfsVfs()*/; + +globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + return installOpfsVfs(sqlite3).catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install opfs-sahpool sqlite3_vfs:", + e.message); + }); +}/*sqlite3ApiBootstrap.initializersAsync*/); +}/*sqlite3ApiBootstrap.initializers*/); diff --git a/manifest b/manifest index 918e28c08b..28f5f9852d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\scleanups\sin\sthe\sOPFS\sVFS. -D 2023-07-14T21:06:00.870 +C Initial\ssketches\sfor\san\salternate\sOPFS\sVFS\swhich\suses\sa\spool\sof\spre-opened\sSyncAccessHandles\sto\sbypass\sthe\sneed\sfor\sa\sdedicated\sI/O\sworker\sand\sthe\sCOOP/COEP\sHTTP\sresponse\sheaders.\sCurrently\scompletely\snon-functional. +D 2023-07-14T21:17:29.056 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,6 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js d0bc04c29983e967e37a91ca4e849beae6db6c883a6612da9717ca22a24119d1 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2043,8 +2044,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 816b503f093c4e6d92d0eb2f9fbd841acd01cc9bc89ee58d961b56c64f71406a -R 6597235c929ad6ed0b90d6716177ff1b +P 984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e +R 81377ab74a003e9bec4077c3a186be8e +T *branch * opfs-sahpool +T *sym-opfs-sahpool * +T -sym-trunk * Cancelled\sby\sbranch. U stephan -Z b7b5463364c5290e11704a96855173c7 +Z bc905b63d521aded8c5f64b8ee091932 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d9fb0905d5..a6b6be909d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e \ No newline at end of file +a93de9f2a553a3a4edd1b361dd6f465a1b0b5b51f7bb8ede432067aedcfefda4 \ No newline at end of file From 92bf6fd4f63c8cccf6a0e455fe2b8276f95cddab Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 14 Jul 2023 21:48:45 +0000 Subject: [PATCH 02/56] Integrate the opfs-sahpool VFS into the JS build and get it loading (but it's still far from functional). FossilOrigin-Name: ef96e6b586825a2b3ed011174309cba8ce1031876c86dc59ed87ab9bbc64f57f --- ext/wasm/GNUmakefile | 5 ++++- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 19 ++++++++++++++----- ext/wasm/tester1.c-pp.js | 2 +- manifest | 19 ++++++++----------- manifest.uuid | 2 +- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index e9e25be894..9a05ca5455 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -51,6 +51,7 @@ MAKEFILE := $(lastword $(MAKEFILE_LIST)) CLEAN_FILES := DISTCLEAN_FILES := ./--dummy-- default: all +#default: quick release: oz # JS_BUILD_MODES exists solely to reduce repetition in documentation # below. @@ -375,6 +376,7 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js sqlite3-api.jses += $(dir.api)/sqlite3-v-helper.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.js sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # SOAP.js is an external API file which is part of our distribution @@ -754,7 +756,7 @@ $(5): $(4) $$(MAKEFILE) $$(sqlite3-wasm.cfiles) $$(EXPORTED_FUNCTIONS.api) $$(pr esac; \ ls -la $$$$dotwasm $$@ all: $(5) -quick: $(5) +#quick: $(5) CLEAN_FILES += $(4) $(5) endef # ^^^ /SETUP_LIB_BUILD_MODE @@ -959,6 +961,7 @@ tester1: tester1.js tester1.mjs tester1.html tester1-esm.html # Note that we do not include $(sqlite3-bundler-friendly.mjs) in this # because bundlers are client-specific. all quick: tester1 +quick: $(sqlite3.js) ######################################################################## # Convenience rules to rebuild with various -Ox levels. Much diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 0832b32bbf..bb91d34643 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -79,7 +79,7 @@ const installOpfsVfs = async function(sqlite3){ sqlite3.config.log ]; const logImpl = (level,...args)=>{ - if(verbosity>level) loggers[level]("OPFS syncer:",...args); + if(verbosity>level) loggers[level]("opfs-sahpool:",...args); }; const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); @@ -117,13 +117,14 @@ const installOpfsVfs = async function(sqlite3){ unavailable in the WASM build.*/; const pDVfs = capi.sqlite3_vfs_find(null)/*default VFS*/; const dVfs = pDVfs - ? new sqlite3_vfs(pDVfs) + ? new capi.sqlite3_vfs(pDVfs) : null /* dVfs will be null when sqlite3 is built with SQLITE_OS_OTHER. */; opfsVfs.$iVersion = 2/*yes, two*/; opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; opfsVfs.$zName = wasm.allocCString("opfs-sahpool"); + log('opfsVfs.$zName =',opfsVfs.$zName); opfsVfs.addOnDispose( '$zName', opfsVfs.$zName, 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) @@ -143,9 +144,9 @@ const installOpfsVfs = async function(sqlite3){ addCapacity: async function(n){ for(let i = 0; i < n; ++i){ const name = Math.random().toString(36).replace('0.',''); - const h = await this.dirHandle.getFileName(name, {create:true}); + const h = await this.dirHandle.getFileHandle(name, {create:true}); const ah = await h.createSyncAccessHandle(); - this.mapAH2Name(ah,name); + this.mapAH2Name.set(ah,name); this.setAssociatedPath(ah, '', 0); } }, @@ -280,21 +281,29 @@ const installOpfsVfs = async function(sqlite3){ return i; }; } + if(!opfsVfs.$xSleep){ + vfsSyncWrappers.xSleep = function(pVfs,ms){ + return 0; + }; + } try{ + log("vfs list:",capi.sqlite3_js_vfs_list()); sqlite3.vfs.installVfs({ io: {struct: opfsIoMethods, methods: ioSyncWrappers}, vfs: {struct: opfsVfs, methods: vfsSyncWrappers} }); + log("vfs list:",capi.sqlite3_js_vfs_list()); }catch(e){ promiseReject(e); return; } VState.isReady = VState.reset().then(async ()=>{ - if(0===VState.getCapacity())[ + if(0===VState.getCapacity()){ await VState.addCapacity(DEFAULT_CAPACITY); } + log("opfs-sahpool VFS initialized."); promiseResolve(sqlite3); }).catch(promiseReject); })/*thePromise*/; diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 8b71198669..9af2856022 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1288,7 +1288,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if(1){ const vfsList = capi.sqlite3_js_vfs_list(); T.assert(vfsList.length>1); - //log("vfsList =",vfsList); + log("vfsList =",vfsList); wasm.scopedAllocCall(()=>{ const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v); for(const v of vfsList){ diff --git a/manifest b/manifest index 28f5f9852d..3cc3488cea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\ssketches\sfor\san\salternate\sOPFS\sVFS\swhich\suses\sa\spool\sof\spre-opened\sSyncAccessHandles\sto\sbypass\sthe\sneed\sfor\sa\sdedicated\sI/O\sworker\sand\sthe\sCOOP/COEP\sHTTP\sresponse\sheaders.\sCurrently\scompletely\snon-functional. -D 2023-07-14T21:17:29.056 +C Integrate\sthe\sopfs-sahpool\sVFS\sinto\sthe\sJS\sbuild\sand\sget\sit\sloading\s(but\sit's\sstill\sfar\sfrom\sfunctional). +D 2023-07-14T21:48:45.896 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 74e351ff45b4061cfed8df237d301819a04182ae304a99118883b064baa25fc2 +F ext/wasm/GNUmakefile b3d23a5f85dd9baff57c525a8fbd8f8ba2cdb41f2988fc93fb5e7746a276d8ab F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js d0bc04c29983e967e37a91ca4e849beae6db6c883a6612da9717ca22a24119d1 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js ce281cec8035a501ad5c5fd8826a2ed2e4c598833df9b80f49b8806b7d6a84ad F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 4420eb97b6b4fc79e4e156b4b8010dd9f373365f4230dd76d823fb04ce28ffde +F ext/wasm/tester1.c-pp.js c021334b946f657251b2cc2cde11b2ff6c4f059fd5a0f4f07ea2d7564409c2cd F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,11 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e -R 81377ab74a003e9bec4077c3a186be8e -T *branch * opfs-sahpool -T *sym-opfs-sahpool * -T -sym-trunk * Cancelled\sby\sbranch. +P a93de9f2a553a3a4edd1b361dd6f465a1b0b5b51f7bb8ede432067aedcfefda4 +R 810cac7ef97d10423572712b3e6ecbc3 U stephan -Z bc905b63d521aded8c5f64b8ee091932 +Z 472fa2e9a25e0392397949c3036372fe # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a6b6be909d..1ff535fff1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a93de9f2a553a3a4edd1b361dd6f465a1b0b5b51f7bb8ede432067aedcfefda4 \ No newline at end of file +ef96e6b586825a2b3ed011174309cba8ce1031876c86dc59ed87ab9bbc64f57f \ No newline at end of file From bee3213145d88f19ea9767a86fef40fd1aa1b557 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 14 Jul 2023 23:02:58 +0000 Subject: [PATCH 03/56] Switch opfs-sahpool to use deterministic backing-store file names. Delay VFS registration until after the pool's files are all opened. Fail vfs init if the client's OPFS API has an async FileSystemSyncAccessHandle.close() method (indicating that it's outdated). FossilOrigin-Name: 7c6697ededee9a64962ac6fd78934c6d6e39258b9558a03c1a6c02bf3be1759e --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 42 ++++++++++++++---------- manifest | 12 +++---- manifest.uuid | 2 +- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index bb91d34643..410986ff27 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -130,9 +130,13 @@ const installOpfsVfs = async function(sqlite3){ 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) ); - const VState = Object.assign(Object.create(null),{ + const getFilename = function(ndx){ + return 'sahpool-'+('00'+ndx).substr(-3); + } + + const SAHPool = Object.assign(Object.create(null),{ /* OPFS dir in which VFS metadata is stored. */ - vfsDir: ".sqlite3-sahpool", + vfsDir: ".sqlite3-opfs-sahpool", dirHandle: undefined, /* Maps OPFS access handles to their opaque file names. */ mapAH2Name: new Map(), @@ -142,10 +146,20 @@ const installOpfsVfs = async function(sqlite3){ getCapacity: function(){return this.mapAH2Name.size}, getFileCount: function(){return this.mapPath2AH.size}, addCapacity: async function(n){ - for(let i = 0; i < n; ++i){ - const name = Math.random().toString(36).replace('0.',''); + const cap = this.getCapacity(); + for(let i = cap; i < cap+n; ++i){ + const name = getFilename(i); const h = await this.dirHandle.getFileHandle(name, {create:true}); - const ah = await h.createSyncAccessHandle(); + let ah = await h.createSyncAccessHandle(); + if(0===i){ + /* Ensure that this client has the "all-synchronous" + OPFS API and fail if they don't. */ + if(undefined !== ah.close()){ + toss("OPFS API is too old for opfs-sahpool:", + "it has an async close() method."); + } + ah = await h.createSyncAccessHandle(); + } this.mapAH2Name.set(ah,name); this.setAssociatedPath(ah, '', 0); } @@ -175,8 +189,8 @@ const installOpfsVfs = async function(sqlite3){ await this.acquireAccessHandles(); } // much more TODO - })/*VState*/; - + })/*SAHPool*/; + sqlite3.SAHPool = SAHPool/*only for testing*/; // Much, much more TODO... /** Impls for the sqlite3_io_methods methods. Maintenance reminder: @@ -287,22 +301,16 @@ const installOpfsVfs = async function(sqlite3){ }; } - try{ + SAHPool.isReady = SAHPool.reset().then(async ()=>{ + if(0===SAHPool.getCapacity()){ + await SAHPool.addCapacity(DEFAULT_CAPACITY); + } log("vfs list:",capi.sqlite3_js_vfs_list()); sqlite3.vfs.installVfs({ io: {struct: opfsIoMethods, methods: ioSyncWrappers}, vfs: {struct: opfsVfs, methods: vfsSyncWrappers} }); log("vfs list:",capi.sqlite3_js_vfs_list()); - }catch(e){ - promiseReject(e); - return; - } - - VState.isReady = VState.reset().then(async ()=>{ - if(0===VState.getCapacity()){ - await VState.addCapacity(DEFAULT_CAPACITY); - } log("opfs-sahpool VFS initialized."); promiseResolve(sqlite3); }).catch(promiseReject); diff --git a/manifest b/manifest index 3cc3488cea..0e36e7381d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Integrate\sthe\sopfs-sahpool\sVFS\sinto\sthe\sJS\sbuild\sand\sget\sit\sloading\s(but\sit's\sstill\sfar\sfrom\sfunctional). -D 2023-07-14T21:48:45.896 +C Switch\sopfs-sahpool\sto\suse\sdeterministic\sbacking-store\sfile\snames.\sDelay\sVFS\sregistration\suntil\safter\sthe\spool's\sfiles\sare\sall\sopened.\sFail\svfs\sinit\sif\sthe\sclient's\sOPFS\sAPI\shas\san\sasync\sFileSystemSyncAccessHandle.close()\smethod\s(indicating\sthat\sit's\soutdated). +D 2023-07-14T23:02:58.852 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js ce281cec8035a501ad5c5fd8826a2ed2e4c598833df9b80f49b8806b7d6a84ad +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js a54fee5e4112dce167bbcddd496f7817846cc269f859237460a835fe693b9c39 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a93de9f2a553a3a4edd1b361dd6f465a1b0b5b51f7bb8ede432067aedcfefda4 -R 810cac7ef97d10423572712b3e6ecbc3 +P ef96e6b586825a2b3ed011174309cba8ce1031876c86dc59ed87ab9bbc64f57f +R 1c797ce49382346884871d0f3bb17cf9 U stephan -Z 472fa2e9a25e0392397949c3036372fe +Z de96eab25d9db47795591a43ac723ba9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1ff535fff1..34a7aa0e8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef96e6b586825a2b3ed011174309cba8ce1031876c86dc59ed87ab9bbc64f57f \ No newline at end of file +7c6697ededee9a64962ac6fd78934c6d6e39258b9558a03c1a6c02bf3be1759e \ No newline at end of file From 100bc4429a32381b61e8058c559caba5fd33d0c0 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 15 Jul 2023 01:02:38 +0000 Subject: [PATCH 04/56] More work on porting the sahpool bits. Revert to random backing-store names because it works better with the capacity-reduction algorithm. FossilOrigin-Name: b4e005fd38b06b8d2f2317b955b93807e80a6a18db5f06d7747978d3bfa11411 --- ext/wasm/GNUmakefile | 4 +- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 287 +++++++++++++++++++---- ext/wasm/tester1.c-pp.js | 2 +- manifest | 16 +- manifest.uuid | 2 +- 5 files changed, 255 insertions(+), 56 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 9a05ca5455..2b2aeb5e04 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -46,12 +46,12 @@ # $(eval), or at least centralize the setup of the numerous vars # related to each build variant $(JS_BUILD_MODES). # +#default: all +default: quick SHELL := $(shell which bash 2>/dev/null) MAKEFILE := $(lastword $(MAKEFILE_LIST)) CLEAN_FILES := DISTCLEAN_FILES := ./--dummy-- -default: all -#default: quick release: oz # JS_BUILD_MODES exists solely to reduce repetition in documentation # below. diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 410986ff27..6bfa5770cb 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -63,15 +63,14 @@ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const installOpfsVfs = async function(sqlite3){ - const pToss = (...args)=>Promise.reject(new Error(args.join(' '))); if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || !globalThis.FileSystemFileHandle || !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || !navigator?.storage?.getDirectory){ - return pToss("Missing required OPFS APIs."); + return Promise.reject(new Error("Missing required OPFS APIs.")); } - const thePromise = new Promise(function(promiseResolve, promiseReject_){ + return new Promise(async function(promiseResolve, promiseReject_){ const verbosity = 3; const loggers = [ sqlite3.config.error, @@ -130,10 +129,13 @@ const installOpfsVfs = async function(sqlite3){ 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) ); - const getFilename = function(ndx){ - return 'sahpool-'+('00'+ndx).substr(-3); - } + const getFilename = false + ? (ndx)=>'sahpool-'+('00'+ndx).substr(-3) + : ()=>Math.random().toString(36).slice(2) + /** + All state for the VFS. + */ const SAHPool = Object.assign(Object.create(null),{ /* OPFS dir in which VFS metadata is stored. */ vfsDir: ".sqlite3-opfs-sahpool", @@ -148,24 +150,33 @@ const installOpfsVfs = async function(sqlite3){ addCapacity: async function(n){ const cap = this.getCapacity(); for(let i = cap; i < cap+n; ++i){ - const name = getFilename(i); + const name = getFilename(i) + /* Reminder: because of how removeCapacity() works, we + really want random names. At this point in the dev + process that fills up the OPFS with randomly-named files + each time the page is reloaded, so delay the return to + random names until we've gotten far enough to eliminate + that problem. */; const h = await this.dirHandle.getFileHandle(name, {create:true}); - let ah = await h.createSyncAccessHandle(); - if(0===i){ - /* Ensure that this client has the "all-synchronous" - OPFS API and fail if they don't. */ - if(undefined !== ah.close()){ - toss("OPFS API is too old for opfs-sahpool:", - "it has an async close() method."); - } - ah = await h.createSyncAccessHandle(); - } + const ah = await h.createSyncAccessHandle(); this.mapAH2Name.set(ah,name); this.setAssociatedPath(ah, '', 0); } }, - setAssociatedPath: function(accessHandle, path, flags){ - // TODO + removeCapacity: async function(n){ + let nRm = 0; + for(const ah of Array.from(this.availableAH)){ + if(nRm === n || this.getFileCount() === this.getCapacity()){ + break; + } + const name = this.mapAH2Name.get(ah); + ah.close(); + await this.dirHandle.removeEntry(name); + this.mapAH2Name.delete(ah); + this.availableAH.delete(ah); + ++nRm; + } + return nRm; }, releaseAccessHandles: function(){ for(const ah of this.mapAH2Name.keys()) ah.close(); @@ -174,7 +185,98 @@ const installOpfsVfs = async function(sqlite3){ this.availableAH.clear(); }, acquireAccessHandles: async function(){ - // TODO + const files = []; + for await (const [name,h] of this.dirHandle){ + if('file'===h.kind){ + files.push([name,h]); + } + } + await Promise.all(files.map(async ([name,h])=>{ + const ah = await h.createSyncAccessHandle() + /*TODO: clean up and fail vfs init on error*/; + this.mapAH2Name.set(ah, name); + const path = this.getAssociatedPath(ah); + if(path){ + this.mapPath2AH.set(path, ah); + }else{ + this.availableAH.add(ah); + } + })); + }, + gapBody: new Uint8Array(HEADER_CORPUS_SIZE), + textDecoder: new TextDecoder(), + getAssociatedPath: function(sah){ + const body = this.gapBody; + sah.read(body, {at: 0}); + // Delete any unexpected files left over by previous + // untimely errors... + const dv = new DataView(body.buffer, body.byteOffset); + const flags = dv.getUint32(HEADER_OFFSET_FLAGS); + if(body[0] && + ((flags & capi.SQLITE_OPEN_DELETEONCLOSE) || + (flags & PERSISTENT_FILE_TYPES)===0)){ + warn(`Removing file with unexpected flags ${flags.toString(16)}`); + this.setAssociatedPath(sah, '', 0); + return ''; + } + + const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); + sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); + const compDigest = this.computeDigest(body); + if(fileDigest.every((v,i) => v===compDigest[i])){ + // Valid digest + const pathBytes = body.findIndex((v)=>0===v); + if(0===pathBytes){ + // This file is unassociated, so ensure that it's empty + // to avoid leaving stale db data laying around. + sah.truncate(HEADER_OFFSET_DATA); + } + return this.textDecoder.decode(body.subarray(0,pathBytes)); + }else{ + // Invalid digest + warn('Disassociating file with bad digest.'); + this.setAssociatedPath(sah, '', 0); + return ''; + } + }, + textEncoder: new TextEncoder(), + setAssociatedPath: function(sah, path, flags){ + const body = this.gapBody; + const enc = this.textEncoder.encodeInto(path, body); + if(HEADER_MAX_PATH_SIZE <= enc.written){ + toss("Path too long:",path); + } + + const dv = new DataView(body.buffer, body.byteOffset); + dv.setUint32(HEADER_OFFSET_FLAGS, flags); + + const digest = this.computeDigest(body); + sah.write(body, {at: 0}); + sah.write(digest, {at: HEADER_OFFSET_DIGEST}); + sah.flush(); + + if(path){ + this.mapPath2AH.set(path, sah); + this.availableAH.delete(sah); + }else{ + // This is not a persistent file, so eliminate the contents. + sah.truncate(HEADER_OFFSET_DATA); + this.mapPath2AH.delete(path); + this.availableAH.add(sah); + } + }, + computeDigest: function(byteArray){ + if(!byteArray[0]){ + // Deleted file + return new Uint32Array([0xfecc5f80, 0xaccec037]); + } + let h1 = 0xdeadbeef; + let h2 = 0x41c6ce57; + for(const v of byteArray){ + h1 = 31 * h1 + (v * 307); + h2 = 31 * h2 + (v * 307); + } + return new Uint32Array([h1>>>0, h2>>>0]); }, reset: async function(){ await this.isReady; @@ -187,8 +289,20 @@ const installOpfsVfs = async function(sqlite3){ this.dirHandle = h; this.releaseAccessHandles(); await this.acquireAccessHandles(); + }, + getPath: function(arg) { + if(wasm.isPtr(arg)) arg = wasm.cstrToJs(arg); + return ((arg instanceof URL) + ? arg + : new URL(arg, 'file://localhost/')).pathname; + }, + deletePath: function(path) { + const sah = this.mapPath2AH.get(path); + if(sah) { + // Un-associate the SQLite path from the OPFS file. + this.setAssociatedPath(sah, '', 0); + } } - // much more TODO })/*SAHPool*/; sqlite3.SAHPool = SAHPool/*only for testing*/; // Much, much more TODO... @@ -201,8 +315,21 @@ const installOpfsVfs = async function(sqlite3){ return 0; }, xClose: function(pFile){ - let rc = 0; - return rc; + const file = SAHPool.mapId2File.get(pFile); + if(file) { + try{ + log(`xClose ${file.path}`); + file.sah.flush(); + SAHPool.mapId2File.delete(pFIle); + if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ + SAHPool.deletePath(file.path); + } + }catch(e){ + error("xClose() failed:",e.message); + return capi.SQLITE_IOERR; + } + } + return 0; }, xDeviceCharacteristics: function(pFile){ return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; @@ -211,32 +338,74 @@ const installOpfsVfs = async function(sqlite3){ return capi.SQLITE_NOTFOUND; }, xFileSize: function(pFile,pSz64){ - let rc = 0; - return rc; + const file = SAHPool.mapId2File(pFile); + const size = file.sah.getSize() - HEADER_OFFSET_DATA; + //log(`xFileSize ${file.path} ${size}`); + wasm.poke64(pSz64, BigInt(size)); + return 0; }, xLock: function(pFile,lockType){ - let rc = capi.SQLITE_IOERR_LOCK; + let rc = capi.SQLITE_IOERR; return rc; }, xRead: function(pFile,pDest,n,offset64){ - let rc = capi.SQLITE_IOERR_READ; - return rc; + const file = SAHPool.mapId2File.get(pFile); + log(`xRead ${file.path} ${n} ${offset64}`); + try { + const nRead = file.sah.read( + pDest, {at: HEADER_OFFSET_DATA + offset64} + ); + if(nRead < n){ + wasm.heap8u().fill(0, pDest + nRead, pDest + n); + return capi.SQLITE_IOERR_SHORT_READ; + } + return 0; + }catch(e){ + error("xRead() failed:",e.message); + return capi.SQLITE_IOERR; + } + }, + xSectorSize: function(pFile){ + return SECTOR_SIZE; }, xSync: function(pFile,flags){ - let rc = capi.SQLITE_IOERR_FSYNC; - return rc; + const file = SAHPool.mapId2File.get(pFile); + //log(`xSync ${file.path} ${flags}`); + try{ + file.sah.flush(); + return 0; + }catch(e){ + error("xSync() failed:",e.message); + return capi.SQLITE_IOERR; + } }, xTruncate: function(pFile,sz64){ - let rc = capi.SQLITE_IOERR_TRUNCATE; - return rc; - }, - xUnlock: function(pFile,lockType){ - let rc = capi.SQLITE_IOERR_UNLOCK; - return rc; + const file = SAHPool.mapId2File.get(pFile); + //log(`xTruncate ${file.path} ${iSize}`); + try{ + file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); + return 0; + }catch(e){ + error("xTruncate() failed:",e.message); + return capi.SQLITE_IOERR; + } }, + /**xUnlock: function(pFile,lockType){ + return capi.SQLITE_IOERR; + },*/ xWrite: function(pFile,pSrc,n,offset64){ - let rc = capi.SQLITE_IOERR_WRITE; - return rc; + const file = SAHPool.mapId2File(pFile); + //log(`xWrite ${file.path} ${n} ${offset64}`); + try{ + const nBytes = file.sah.write( + pSrc, { at: HEADER_OFFSET_DATA + Number(offset64) } + ); + return nBytes === n ? 0 : capi.SQLITE_IOERR; + return 0; + }catch(e){ + error("xWrite() failed:",e.message); + return capi.SQLITE_IOERR; + } } }/*ioSyncWrappers*/; @@ -246,8 +415,9 @@ const installOpfsVfs = async function(sqlite3){ */ const vfsSyncWrappers = { xAccess: function(pVfs,zName,flags,pOut){ - const rc = capi.SQLITE_ERROR; - return rc; + const name = this.getPath(zName); + wasm.poke32(pOut, SAHPool.mapPath2AH.has(name) ? 1 : 0); + return 0; }, xCurrentTime: function(pVfs,pOut){ wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), @@ -260,8 +430,13 @@ const installOpfsVfs = async function(sqlite3){ return 0; }, xDelete: function(pVfs, zName, doSyncDir){ - const rc = capi.SQLITE_ERROR; - return rc; + try{ + SAHPool.deletePath(SAHPool.getPath(zName)); + return 0; + }catch(e){ + error("Error xDelete()ing file:",e.message); + return capi.SQLITE_IOERR_DELETE; + } }, xFullPathname: function(pVfs,zName,nOut,pOut){ const i = wasm.cstrncpy(pOut, zName, nOut); @@ -301,6 +476,31 @@ const installOpfsVfs = async function(sqlite3){ }; } + /** + Ensure that the client has a "fully-sync" SAH impl, + else reject the promise. Returns true on success, + else false. + */ + const affirmHasSyncAPI = async function(){ + try { + const dh = await navigator.storage.getDirectory(); + const fn = '.opfs-sahpool-sync-check-'+Math.random().toString(36).slice(2); + const fh = await dh.getFileHandle(fn, { create: true }); + const ah = await fh.createSyncAccessHandle(); + const close = ah.close(); + await close; + await dh.removeEntry(fn); + if(close?.then){ + toss("The local OPFS API is too old for opfs-sahpool:", + "it has an async FileSystemSyncAccessHandle.close() method."); + } + return true; + }catch(e){ + promiseReject(e); + return false; + } + }; + if(!(await affirmHasSyncAPI())) return; SAHPool.isReady = SAHPool.reset().then(async ()=>{ if(0===SAHPool.getCapacity()){ await SAHPool.addCapacity(DEFAULT_CAPACITY); @@ -314,8 +514,7 @@ const installOpfsVfs = async function(sqlite3){ log("opfs-sahpool VFS initialized."); promiseResolve(sqlite3); }).catch(promiseReject); - })/*thePromise*/; - return thePromise; + })/*return Promise*/; }/*installOpfsVfs()*/; globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 9af2856022..8b71198669 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1288,7 +1288,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if(1){ const vfsList = capi.sqlite3_js_vfs_list(); T.assert(vfsList.length>1); - log("vfsList =",vfsList); + //log("vfsList =",vfsList); wasm.scopedAllocCall(()=>{ const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v); for(const v of vfsList){ diff --git a/manifest b/manifest index 0e36e7381d..d8a87ebc0b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Switch\sopfs-sahpool\sto\suse\sdeterministic\sbacking-store\sfile\snames.\sDelay\sVFS\sregistration\suntil\safter\sthe\spool's\sfiles\sare\sall\sopened.\sFail\svfs\sinit\sif\sthe\sclient's\sOPFS\sAPI\shas\san\sasync\sFileSystemSyncAccessHandle.close()\smethod\s(indicating\sthat\sit's\soutdated). -D 2023-07-14T23:02:58.852 +C More\swork\son\sporting\sthe\ssahpool\sbits.\sRevert\sto\srandom\sbacking-store\snames\sbecause\sit\sworks\sbetter\swith\sthe\scapacity-reduction\salgorithm. +D 2023-07-15T01:02:38.317 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile b3d23a5f85dd9baff57c525a8fbd8f8ba2cdb41f2988fc93fb5e7746a276d8ab +F ext/wasm/GNUmakefile 1712a1fbf615dea49dfdc9cf9057c268df3a6887930aec8d9249f02a2059f2b2 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js a54fee5e4112dce167bbcddd496f7817846cc269f859237460a835fe693b9c39 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 54beb56c4cc403c4bc9cf38704186e330b38fe6f24a86516a21d9d6239769b05 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js c021334b946f657251b2cc2cde11b2ff6c4f059fd5a0f4f07ea2d7564409c2cd +F ext/wasm/tester1.c-pp.js 4420eb97b6b4fc79e4e156b4b8010dd9f373365f4230dd76d823fb04ce28ffde F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ef96e6b586825a2b3ed011174309cba8ce1031876c86dc59ed87ab9bbc64f57f -R 1c797ce49382346884871d0f3bb17cf9 +P 7c6697ededee9a64962ac6fd78934c6d6e39258b9558a03c1a6c02bf3be1759e +R ce448f7649c77f7ac682168b6f7b78d5 U stephan -Z de96eab25d9db47795591a43ac723ba9 +Z 9bbeb2300d5ae5fa66a5962c456126f8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 34a7aa0e8c..04ee2beedf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c6697ededee9a64962ac6fd78934c6d6e39258b9558a03c1a6c02bf3be1759e \ No newline at end of file +b4e005fd38b06b8d2f2317b955b93807e80a6a18db5f06d7747978d3bfa11411 \ No newline at end of file From 88af76f62dc833141e3d8b927b2ec5eaa387f4f4 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 15 Jul 2023 11:23:57 +0000 Subject: [PATCH 05/56] opfs-sahpool VFS now seems to work, in that it runs fine (and blazingly fast) in speedtest1. Add sqlite3.config options for the high-level configurable parts of opfs-sahpool. FossilOrigin-Name: 5d92d5f4d8ac4cfa37ba473e5cc861628b783bbf1ae4d138bcae8b9d6cc6e798 --- ext/wasm/GNUmakefile | 4 +- ext/wasm/api/sqlite3-api-prologue.js | 30 +- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 367 +++++++++++++++++------ ext/wasm/index.html | 4 + manifest | 18 +- manifest.uuid | 2 +- 6 files changed, 308 insertions(+), 117 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 2b2aeb5e04..f8df213797 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -46,8 +46,8 @@ # $(eval), or at least centralize the setup of the numerous vars # related to each build variant $(JS_BUILD_MODES). # -#default: all -default: quick +default: all +#default: quick SHELL := $(shell which bash 2>/dev/null) MAKEFILE := $(lastword $(MAKEFILE_LIST)) CLEAN_FILES := diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index f3d0a7c408..fb085e299c 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -88,9 +88,24 @@ can be replaced with (e.g.) empty functions to squelch all such output. - - `wasmfsOpfsDir`[^1]: As of 2022-12-17, this feature does not - currently work due to incompatible Emscripten-side changes made - in the WASMFS+OPFS combination. This option is currently ignored. + - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the OPFS-backed + filesystem in WASMFS-capable builds. + + - `opfs-sahpool.dir`[^1]: Specifies the OPFS directory name in + which to store metadata for the `"opfs-sahpool"` sqlite3_vfs. + Changing this name will effectively orphan any databases stored + under previous names. The default is unspecified but descriptive. + This option may contain multiple path elements, + e.g. "foo/bar/baz", and they are created automatically. In + practice there should be no driving need to change this. + + - `opfs-sahpool.defaultCapacity`[^1]: Specifies the default + capacity of the `"opfs-sahpool"` VFS. This should not be set + unduly high because the VFS has to open (and keep open) a file + for each entry in the pool. This setting only has an effect when + the pool is initially empty. It does not have any effect if a + pool already exists. + [^1] = This property may optionally be a function, in which case this function calls that function to fetch the value, @@ -125,11 +140,11 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( log: console.log.bind(console), wasmfsOpfsDir: '/opfs', /** - useStdAlloc is just for testing an allocator discrepancy. The + useStdAlloc is just for testing allocator discrepancies. The docs guarantee that this is false in the canonical builds. For 99% of purposes it doesn't matter which allocators we use, but - it becomes significant with, e.g., sqlite3_deserialize() - and certain wasm.xWrap.resultAdapter()s. + it becomes significant with, e.g., sqlite3_deserialize() and + certain wasm.xWrap.resultAdapter()s. */ useStdAlloc: false }, apiConfig || {}); @@ -143,7 +158,8 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( [ // If any of these config options are functions, replace them with // the result of calling that function... - 'exports', 'memory', 'wasmfsOpfsDir' + 'exports', 'memory', 'wasmfsOpfsDir', + 'opfs-sahpool.dir', 'opfs-sahpool.defaultCapacity' ].forEach((k)=>{ if('function' === typeof config[k]){ config[k] = config[k](); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 6bfa5770cb..352a54039a 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -10,12 +10,9 @@ *********************************************************************** - INCOMPLETE! WORK IN PROGRESS! - - This file holds an experimental sqlite3_vfs backed by OPFS storage - which uses a different implementation strategy than the "opfs" - VFS. This one is a port of Roy Hashimoto's OPFS SyncAccessHandle - pool: + This file holds a sqlite3_vfs backed by OPFS storage which uses a + different implementation strategy than the "opfs" VFS. This one is a + port of Roy Hashimoto's OPFS SyncAccessHandle pool: https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/AccessHandlePoolVFS.js @@ -28,16 +25,17 @@ https://sqlite.org/forum/forumpost/e140d84e71 - Primary differences from the original "opfs" VFS include: + Primary differences from the "opfs" VFS include: - This one avoids the need for a sub-worker to synchronize - communication between the synchronous C API and the only-partly - synchronous OPFS API. + communication between the synchronous C API and the + only-partly-synchronous OPFS API. - It does so by opening a fixed number of OPFS files at library-level initialization time, obtaining SyncAccessHandles to each, and manipulating those handles via the synchronous sqlite3_vfs - interface. + interface. If it cannot open them (e.g. they are already opened by + another tab) then the VFS will not be installed. - Because of that, this one lacks all library-level concurrency support. @@ -45,20 +43,13 @@ - Also because of that, it does not require the SharedArrayBuffer, so can function without the COOP/COEP HTTP response headers. - - It can hypothetically support Safari 16.4+, whereas the "opfs" - VFS requires v17 due to a bug in 16.x which makes it incompatible - with that VFS. + - It can hypothetically support Safari 16.4+, whereas the "opfs" VFS + requires v17 due to a subworker/storage bug in 16.x which makes it + incompatible with that VFS. - This VFS requires the "semi-fully-sync" FileSystemSyncAccessHandle - (hereafter "SAH") APIs released with Chrome v108. There is - unfortunately no known programmatic way to determine whether a given - API is from that release or newer without actually calling it and - checking whether one of the "fully-sync" functions returns a Promise - (in which case it's the older version). (Reminder to self: when - opening up the initial pool of files, we can close() the first one - we open and see if close() returns a Promise. If it does, it's the - older version so fail VFS initialization. If it doesn't, re-open it.) - + (hereafter "SAH") APIs released with Chrome v108. If that API + is not detected, the VFS is not registered. */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ @@ -71,7 +62,7 @@ const installOpfsVfs = async function(sqlite3){ return Promise.reject(new Error("Missing required OPFS APIs.")); } return new Promise(async function(promiseResolve, promiseReject_){ - const verbosity = 3; + const verbosity = 2; const loggers = [ sqlite3.config.error, sqlite3.config.warn, @@ -103,7 +94,8 @@ const installOpfsVfs = async function(sqlite3){ const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE; const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; const HEADER_OFFSET_DATA = SECTOR_SIZE; - const DEFAULT_CAPACITY = 6; + const DEFAULT_CAPACITY = + sqlite3.config['opfs-sahpool.defaultCapacity'] || 6; /* Bitmask of file types which may persist across sessions. SQLITE_OPEN_xyz types not listed here may be inadvertently left in OPFS but are treated as transient by this VFS and @@ -114,7 +106,9 @@ const installOpfsVfs = async function(sqlite3){ capi.SQLITE_OPEN_SUPER_JOURNAL | capi.SQLITE_OPEN_WAL /* noting that WAL support is unavailable in the WASM build.*/; - const pDVfs = capi.sqlite3_vfs_find(null)/*default VFS*/; + /* We fetch the default VFS so that we can inherit some + methods from it. */ + const pDVfs = capi.sqlite3_vfs_find(null); const dVfs = pDVfs ? new capi.sqlite3_vfs(pDVfs) : null /* dVfs will be null when sqlite3 is built with @@ -122,68 +116,91 @@ const installOpfsVfs = async function(sqlite3){ opfsVfs.$iVersion = 2/*yes, two*/; opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; - opfsVfs.$zName = wasm.allocCString("opfs-sahpool"); - log('opfsVfs.$zName =',opfsVfs.$zName); opfsVfs.addOnDispose( - '$zName', opfsVfs.$zName, - 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) + opfsVfs.$zName = wasm.allocCString("opfs-sahpool"), + ()=>(dVfs ? dVfs.dispose() : null) ); - const getFilename = false - ? (ndx)=>'sahpool-'+('00'+ndx).substr(-3) - : ()=>Math.random().toString(36).slice(2) + /** + Returns short a string of random alphanumeric characters + suitable for use as a random filename. + */ + const getRandomName = ()=>Math.random().toString(36).slice(2); /** All state for the VFS. */ const SAHPool = Object.assign(Object.create(null),{ /* OPFS dir in which VFS metadata is stored. */ - vfsDir: ".sqlite3-opfs-sahpool", + vfsDir: sqlite3.config['vfs:opfs-sahpool:dir'] + || ".sqlite3-opfs-sahpool", + /* Directory handle to this.vfsDir. */ dirHandle: undefined, - /* Maps OPFS access handles to their opaque file names. */ - mapAH2Name: new Map(), - mapPath2AH: new Map(), - availableAH: new Set(), - mapId2File: new Map(), - getCapacity: function(){return this.mapAH2Name.size}, - getFileCount: function(){return this.mapPath2AH.size}, + /* Maps SAHs to their opaque file names. */ + mapSAHToName: new Map(), + /* Maps client-side file names to SAHs. */ + mapPathToSAH: new Map(), + /* Set of currently-unused SAHs. */ + availableSAH: new Set(), + /* Maps (sqlite3_file*) to xOpen's file objects. */ + mapIdToFile: new Map(), + /* Current pool capacity. */ + getCapacity: function(){return this.mapSAHToName.size}, + /* Current number of in-use files from pool. */ + getFileCount: function(){return this.mapPathToSAH.size}, + /** + Adds n files to the pool's capacity. This change is + persistent across settings. Returns a Promise which resolves + to the new capacity. + */ addCapacity: async function(n){ const cap = this.getCapacity(); for(let i = cap; i < cap+n; ++i){ - const name = getFilename(i) - /* Reminder: because of how removeCapacity() works, we - really want random names. At this point in the dev - process that fills up the OPFS with randomly-named files - each time the page is reloaded, so delay the return to - random names until we've gotten far enough to eliminate - that problem. */; + const name = getRandomName(); const h = await this.dirHandle.getFileHandle(name, {create:true}); const ah = await h.createSyncAccessHandle(); - this.mapAH2Name.set(ah,name); + this.mapSAHToName.set(ah,name); this.setAssociatedPath(ah, '', 0); } + return i; }, + /** + Removes n entries from the pool's current capacity + if possible. It can only remove currently-unallocated + files. Returns a Promise resolving to the number of + removed files. + */ removeCapacity: async function(n){ let nRm = 0; - for(const ah of Array.from(this.availableAH)){ + for(const ah of Array.from(this.availableSAH)){ if(nRm === n || this.getFileCount() === this.getCapacity()){ break; } - const name = this.mapAH2Name.get(ah); + const name = this.mapSAHToName.get(ah); ah.close(); await this.dirHandle.removeEntry(name); - this.mapAH2Name.delete(ah); - this.availableAH.delete(ah); + this.mapSAHToName.delete(ah); + this.availableSAH.delete(ah); ++nRm; } return nRm; }, + /** + Releases all currently-opened SAHs. + */ releaseAccessHandles: function(){ - for(const ah of this.mapAH2Name.keys()) ah.close(); - this.mapAH2Name.clear(); - this.mapPath2AH.clear(); - this.availableAH.clear(); + for(const ah of this.mapSAHToName.keys()) ah.close(); + this.mapSAHToName.clear(); + this.mapPathToSAH.clear(); + this.availableSAH.clear(); }, + /** + Opens all files under this.vfsDir/this.dirHandle and acquires + a SAH for each. returns a Promise which resolves to no value + but completes once all SAHs are acquired. If acquiring an SAH + throws, SAHPool.$error will contain the corresponding + exception. + */ acquireAccessHandles: async function(){ const files = []; for await (const [name,h] of this.dirHandle){ @@ -192,21 +209,35 @@ const installOpfsVfs = async function(sqlite3){ } } await Promise.all(files.map(async ([name,h])=>{ - const ah = await h.createSyncAccessHandle() + try{ + const ah = await h.createSyncAccessHandle() /*TODO: clean up and fail vfs init on error*/; - this.mapAH2Name.set(ah, name); - const path = this.getAssociatedPath(ah); - if(path){ - this.mapPath2AH.set(path, ah); - }else{ - this.availableAH.add(ah); + this.mapSAHToName.set(ah, name); + const path = this.getAssociatedPath(ah); + if(path){ + this.mapPathToSAH.set(path, ah); + }else{ + this.availableSAH.add(ah); + } + }catch(e){ + SAHPool.storeErr(e); + throw e; } })); }, - gapBody: new Uint8Array(HEADER_CORPUS_SIZE), + /** Buffer used by [sg]etAssociatedPath(). */ + apBody: new Uint8Array(HEADER_CORPUS_SIZE), textDecoder: new TextDecoder(), + textEncoder: new TextEncoder(), + /** + Given an SAH, returns the client-specified name of + that file by extracting it from the SAH's header. + + On error, it disassociates SAH from the pool and + returns an empty string. + */ getAssociatedPath: function(sah){ - const body = this.gapBody; + const body = this.apBody; sah.read(body, {at: 0}); // Delete any unexpected files left over by previous // untimely errors... @@ -227,8 +258,8 @@ const installOpfsVfs = async function(sqlite3){ // Valid digest const pathBytes = body.findIndex((v)=>0===v); if(0===pathBytes){ - // This file is unassociated, so ensure that it's empty - // to avoid leaving stale db data laying around. + // This file is unassociated, so truncate it to avoid + // leaving stale db data laying around. sah.truncate(HEADER_OFFSET_DATA); } return this.textDecoder.decode(body.subarray(0,pathBytes)); @@ -239,9 +270,12 @@ const installOpfsVfs = async function(sqlite3){ return ''; } }, - textEncoder: new TextEncoder(), + /** + Stores the given client-defined path and SQLITE_OPEN_xyz + flags into the given SAH. + */ setAssociatedPath: function(sah, path, flags){ - const body = this.gapBody; + const body = this.apBody; const enc = this.textEncoder.encodeInto(path, body); if(HEADER_MAX_PATH_SIZE <= enc.written){ toss("Path too long:",path); @@ -256,15 +290,19 @@ const installOpfsVfs = async function(sqlite3){ sah.flush(); if(path){ - this.mapPath2AH.set(path, sah); - this.availableAH.delete(sah); + this.mapPathToSAH.set(path, sah); + this.availableSAH.delete(sah); }else{ // This is not a persistent file, so eliminate the contents. sah.truncate(HEADER_OFFSET_DATA); - this.mapPath2AH.delete(path); - this.availableAH.add(sah); + this.mapPathToSAH.delete(path); + this.availableSAH.add(sah); } }, + /** + Computes a digest for the given byte array and + returns it as a two-element Uint32Array. + */ computeDigest: function(byteArray){ if(!byteArray[0]){ // Deleted file @@ -290,41 +328,71 @@ const installOpfsVfs = async function(sqlite3){ this.releaseAccessHandles(); await this.acquireAccessHandles(); }, + /** + Returns the pathname part of the given argument, + which may be any of: + + - a URL object + - A JS string representing a file name + - Wasm C-string representing a file name + */ getPath: function(arg) { if(wasm.isPtr(arg)) arg = wasm.cstrToJs(arg); return ((arg instanceof URL) ? arg : new URL(arg, 'file://localhost/')).pathname; }, + /** + Removes the association of the given client-specified file + name (JS string) from the pool. + */ deletePath: function(path) { - const sah = this.mapPath2AH.get(path); + const sah = this.mapPathToSAH.get(path); if(sah) { // Un-associate the SQLite path from the OPFS file. this.setAssociatedPath(sah, '', 0); } + }, + /** + Sets e as this object's current error. Pass a falsy + (or no) value to clear it. + */ + storeErr: function(e){ + return this.$error = e; + }, + /** + Pops this object's Error object and returns + it (a falsy value if no error is set). + */ + popErr: function(e){ + const rc = this.$error; + this.$error = undefined; + return rc; } })/*SAHPool*/; sqlite3.SAHPool = SAHPool/*only for testing*/; - // Much, much more TODO... /** Impls for the sqlite3_io_methods methods. Maintenance reminder: members are in alphabetical order to simplify finding them. */ const ioSyncWrappers = { xCheckReservedLock: function(pFile,pOut){ + SAHPool.storeErr(); return 0; }, xClose: function(pFile){ - const file = SAHPool.mapId2File.get(pFile); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); if(file) { try{ log(`xClose ${file.path}`); file.sah.flush(); - SAHPool.mapId2File.delete(pFIle); + SAHPool.mapIdToFile.delete(pFIle); if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ SAHPool.deletePath(file.path); } }catch(e){ + SAHPool.storeErr(e); error("xClose() failed:",e.message); return capi.SQLITE_IOERR; } @@ -338,18 +406,20 @@ const installOpfsVfs = async function(sqlite3){ return capi.SQLITE_NOTFOUND; }, xFileSize: function(pFile,pSz64){ - const file = SAHPool.mapId2File(pFile); + const file = SAHPool.mapIdToFile(pFile); const size = file.sah.getSize() - HEADER_OFFSET_DATA; //log(`xFileSize ${file.path} ${size}`); wasm.poke64(pSz64, BigInt(size)); return 0; }, xLock: function(pFile,lockType){ + SAHPool.storeErr(); let rc = capi.SQLITE_IOERR; return rc; }, xRead: function(pFile,pDest,n,offset64){ - const file = SAHPool.mapId2File.get(pFile); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); log(`xRead ${file.path} ${n} ${offset64}`); try { const nRead = file.sah.read( @@ -361,6 +431,7 @@ const installOpfsVfs = async function(sqlite3){ } return 0; }catch(e){ + SAHPool.storeErr(e); error("xRead() failed:",e.message); return capi.SQLITE_IOERR; } @@ -369,23 +440,27 @@ const installOpfsVfs = async function(sqlite3){ return SECTOR_SIZE; }, xSync: function(pFile,flags){ - const file = SAHPool.mapId2File.get(pFile); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); //log(`xSync ${file.path} ${flags}`); try{ file.sah.flush(); return 0; }catch(e){ + SAHPool.storeErr(e); error("xSync() failed:",e.message); return capi.SQLITE_IOERR; } }, xTruncate: function(pFile,sz64){ - const file = SAHPool.mapId2File.get(pFile); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); //log(`xTruncate ${file.path} ${iSize}`); try{ file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); return 0; }catch(e){ + SAHPool.storeErr(e); error("xTruncate() failed:",e.message); return capi.SQLITE_IOERR; } @@ -394,15 +469,16 @@ const installOpfsVfs = async function(sqlite3){ return capi.SQLITE_IOERR; },*/ xWrite: function(pFile,pSrc,n,offset64){ - const file = SAHPool.mapId2File(pFile); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile(pFile); //log(`xWrite ${file.path} ${n} ${offset64}`); try{ const nBytes = file.sah.write( pSrc, { at: HEADER_OFFSET_DATA + Number(offset64) } ); return nBytes === n ? 0 : capi.SQLITE_IOERR; - return 0; }catch(e){ + SAHPool.storeErr(e); error("xWrite() failed:",e.message); return capi.SQLITE_IOERR; } @@ -415,8 +491,13 @@ const installOpfsVfs = async function(sqlite3){ */ const vfsSyncWrappers = { xAccess: function(pVfs,zName,flags,pOut){ - const name = this.getPath(zName); - wasm.poke32(pOut, SAHPool.mapPath2AH.has(name) ? 1 : 0); + SAHPool.storeErr(); + try{ + const name = this.getPath(zName); + wasm.poke32(pOut, SAHPool.mapPathToSAH.has(name) ? 1 : 0); + }catch(e){ + /*ignored*/; + } return 0; }, xCurrentTime: function(pVfs,pOut){ @@ -430,10 +511,12 @@ const installOpfsVfs = async function(sqlite3){ return 0; }, xDelete: function(pVfs, zName, doSyncDir){ + SAHPool.storeErr(); try{ SAHPool.deletePath(SAHPool.getPath(zName)); return 0; }catch(e){ + SAHPool.storeErr(e); error("Error xDelete()ing file:",e.message); return capi.SQLITE_IOERR_DELETE; } @@ -443,15 +526,53 @@ const installOpfsVfs = async function(sqlite3){ return i nOut) wasm.poke8(pOut + nOut - 1, 0); + }catch(e){ + return capi.SQLITE_NOMEM; + }finally{ + wasm.scopedAllocPop(scope); + } + } return 0; }, //xSleep is optionally defined below xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ - let rc = capi.SQLITE_ERROR; - return rc; + try{ + // First try to open a path that already exists in the file system. + const path = (zName && wasm.peek8(zName)) + ? SAHPool.getPath(name) + : getRandomName(); + let ah = SAHPool.mapPathToSAH.get(path); + if(!ah && (flags & capi.SQLITE_OPEN_CREATE)) { + // File not found so try to create it. + if(SAHPool.getFileCount() < SAHPool.getCapacity()) { + // Choose an unassociated OPFS file from the pool. + ah = SAHPool.availableSAH.keys()[0]; + SAHPool.setAssociatedPath(ah, path, flags); + }else{ + // File pool is full. + toss('SAH pool is full. Cannot create file',path); + } + } + if(!ah){ + toss('file not found:',path); + } + // Subsequent methods are only passed the file pointer, so + // map the relevant info we need to that pointer. + const file = { path, flags, ah }; + SAHPool.mapIdToFile.set(pFile, file); + wasm.poke32(pOutFlags, flags); + return 0; + }catch(e){ + SAHPool.storeErr(e); + return capi.SQLITE_CANTOPEN; + } }/*xOpen()*/ }/*vfsSyncWrappers*/; @@ -481,10 +602,10 @@ const installOpfsVfs = async function(sqlite3){ else reject the promise. Returns true on success, else false. */ - const affirmHasSyncAPI = async function(){ + if(!(async ()=>{ try { const dh = await navigator.storage.getDirectory(); - const fn = '.opfs-sahpool-sync-check-'+Math.random().toString(36).slice(2); + const fn = '.opfs-sahpool-sync-check-'+getRandomName(); const fh = await dh.getFileHandle(fn, { create: true }); const ah = await fh.createSyncAccessHandle(); const close = ah.close(); @@ -499,19 +620,69 @@ const installOpfsVfs = async function(sqlite3){ promiseReject(e); return false; } - }; - if(!(await affirmHasSyncAPI())) return; + })()){ + return; + } + SAHPool.isReady = SAHPool.reset().then(async ()=>{ + if(SAHPool.$error){ + throw SAHPool.$error; + } if(0===SAHPool.getCapacity()){ await SAHPool.addCapacity(DEFAULT_CAPACITY); } - log("vfs list:",capi.sqlite3_js_vfs_list()); + //log("vfs list:",capi.sqlite3_js_vfs_list()); sqlite3.vfs.installVfs({ io: {struct: opfsIoMethods, methods: ioSyncWrappers}, vfs: {struct: opfsVfs, methods: vfsSyncWrappers} }); - log("vfs list:",capi.sqlite3_js_vfs_list()); - log("opfs-sahpool VFS initialized."); + //log("vfs list:",capi.sqlite3_js_vfs_list()); + if(sqlite3.oo1){ + const OpfsSAHPoolDb = function(...args){ + const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = opfsVfs.$zName; + sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + }; + OpfsSAHPoolDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + OpfsSAHPoolDb.addPoolCapacity = async (n)=>SAHPool.addCapacity(n); + OpfsSAHPoolDb.removePoolCapacity = async (n)=>SAHPool.removeCapacity(n); + OpfsSAHPoolDb.getPoolCapacity = ()=>SAHPool.getCapacity(); + OpfsSAHPoolDb.getPoolUsage = ()=>SAHPool.getFileCount(); + sqlite3.oo1.OpfsSAHPoolDb = OpfsSAHPoolDb; + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( + opfsVfs.pointer, + function(oo1Db, sqlite3){ + sqlite3.capi.sqlite3_exec(oo1Db, [ + /* As of July 2023, the PERSIST journal mode on OPFS is + somewhat slower than DELETE or TRUNCATE (it was faster + before Chrome version 108 or 109). TRUNCATE and DELETE + have very similar performance on OPFS. + + Roy Hashimoto notes that TRUNCATE and PERSIST modes may + decrease OPFS concurrency because multiple connections + can open the journal file in those modes: + + https://github.com/rhashimoto/wa-sqlite/issues/68 + + Given that, and the fact that testing has not revealed + any appreciable difference between performance of + TRUNCATE and DELETE modes on OPFS, we currently (as of + 2023-07-13) default to DELETE mode. + */ + "pragma journal_mode=DELETE;", + /* + OPFS benefits hugely from cache on moderate/large + speedtest1 --size 50 and --size 100 workloads. We + currently rely on setting a non-default cache size when + building sqlite3.wasm. If that policy changes, the cache + can be set here. + */ + "pragma cache_size=-16384;" + ], 0, 0, 0); + } + ); + }/*extend sqlite3.oo1*/ + log("VFS initialized."); promiseResolve(sqlite3); }).catch(promiseReject); })/*return Promise*/; @@ -520,7 +691,7 @@ const installOpfsVfs = async function(sqlite3){ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ return installOpfsVfs(sqlite3).catch((e)=>{ sqlite3.config.warn("Ignoring inability to install opfs-sahpool sqlite3_vfs:", - e.message); + e.message, e); }); }/*sqlite3ApiBootstrap.initializersAsync*/); }/*sqlite3ApiBootstrap.initializers*/); diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 2c4bc4eb87..d91dca6cb4 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -96,6 +96,10 @@
  • speedtest1-worker: an interactive Worker-thread variant of speedtest1.
  • speedtest1-worker?vfs=opfs: speedtest1-worker with the OPFS VFS preselected and configured for a moderate workload.
    • +
  • speedtest1-worker?vfs=opfs-sahpool: + speedtest1-worker with the OPFS-SAHPOOL VFS preselected + and configured for a moderate workload. +
  • The obligatory "misc." category... diff --git a/manifest b/manifest index d8a87ebc0b..bb0cf325b1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\swork\son\sporting\sthe\ssahpool\sbits.\sRevert\sto\srandom\sbacking-store\snames\sbecause\sit\sworks\sbetter\swith\sthe\scapacity-reduction\salgorithm. -D 2023-07-15T01:02:38.317 +C opfs-sahpool\sVFS\snow\sseems\sto\swork,\sin\sthat\sit\sruns\sfine\s(and\sblazingly\sfast)\sin\sspeedtest1.\sAdd\ssqlite3.config\soptions\sfor\sthe\shigh-level\sconfigurable\sparts\sof\sopfs-sahpool. +D 2023-07-15T11:23:57.597 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 1712a1fbf615dea49dfdc9cf9057c268df3a6887930aec8d9249f02a2059f2b2 +F ext/wasm/GNUmakefile e38d5835d86f8103c2165b42bb49a2558dfcced1129763e935b7c829fe0452d9 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -497,12 +497,12 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 -F ext/wasm/api/sqlite3-api-prologue.js 7d1c1ef59b9dcc42ad3a9cec9da972c42e29316a270cd126e7f660509b09027b +F ext/wasm/api/sqlite3-api-prologue.js 5dcb5d2d74269545073eec197614b86bd28950132b5fe4de67c10a8a0d5524b2 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 54beb56c4cc403c4bc9cf38704186e330b38fe6f24a86516a21d9d6239769b05 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 6335869943166626710080460b9ce871f2dcd11a5d01078a5430044dc08c9955 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -531,7 +531,7 @@ F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa0 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e -F ext/wasm/index.html b768e8659b4fe311912e54d42906449d51c0f84b7f036cca47ec1f93bf3f91de +F ext/wasm/index.html b4e55de741be9fb7656445ea55085f703a784aebde620e1c4852fa21c1ac1c5b F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7c6697ededee9a64962ac6fd78934c6d6e39258b9558a03c1a6c02bf3be1759e -R ce448f7649c77f7ac682168b6f7b78d5 +P b4e005fd38b06b8d2f2317b955b93807e80a6a18db5f06d7747978d3bfa11411 +R da5634bacf6dceb36474c55e5f7adbe7 U stephan -Z 9bbeb2300d5ae5fa66a5962c456126f8 +Z c74716fca4644a5bd074f3e9b97acc41 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 04ee2beedf..e61abf0822 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4e005fd38b06b8d2f2317b955b93807e80a6a18db5f06d7747978d3bfa11411 \ No newline at end of file +5d92d5f4d8ac4cfa37ba473e5cc861628b783bbf1ae4d138bcae8b9d6cc6e798 \ No newline at end of file From bb65feb8691862c6f4c57e6b314850005c27201f Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 15 Jul 2023 11:37:34 +0000 Subject: [PATCH 06/56] Rename opfs-sahpool removeCapacity() to reduceCapacity(). FossilOrigin-Name: fff68e9f25a57045e9d636b02ffa073cf1b984b2587d4fce10f6e35c9988469c --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 352a54039a..acff9ea207 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -170,7 +170,7 @@ const installOpfsVfs = async function(sqlite3){ files. Returns a Promise resolving to the number of removed files. */ - removeCapacity: async function(n){ + reduceCapacity: async function(n){ let nRm = 0; for(const ah of Array.from(this.availableSAH)){ if(nRm === n || this.getFileCount() === this.getCapacity()){ @@ -645,7 +645,7 @@ const installOpfsVfs = async function(sqlite3){ }; OpfsSAHPoolDb.prototype = Object.create(sqlite3.oo1.DB.prototype); OpfsSAHPoolDb.addPoolCapacity = async (n)=>SAHPool.addCapacity(n); - OpfsSAHPoolDb.removePoolCapacity = async (n)=>SAHPool.removeCapacity(n); + OpfsSAHPoolDb.reducePoolCapacity = async (n)=>SAHPool.reduceCapacity(n); OpfsSAHPoolDb.getPoolCapacity = ()=>SAHPool.getCapacity(); OpfsSAHPoolDb.getPoolUsage = ()=>SAHPool.getFileCount(); sqlite3.oo1.OpfsSAHPoolDb = OpfsSAHPoolDb; diff --git a/manifest b/manifest index bb0cf325b1..0a36f3c117 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C opfs-sahpool\sVFS\snow\sseems\sto\swork,\sin\sthat\sit\sruns\sfine\s(and\sblazingly\sfast)\sin\sspeedtest1.\sAdd\ssqlite3.config\soptions\sfor\sthe\shigh-level\sconfigurable\sparts\sof\sopfs-sahpool. -D 2023-07-15T11:23:57.597 +C Rename\sopfs-sahpool\sremoveCapacity()\sto\sreduceCapacity(). +D 2023-07-15T11:37:34.991 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 6335869943166626710080460b9ce871f2dcd11a5d01078a5430044dc08c9955 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js ad6ec4e87f47152a871a23bf90b64709094bf04e8ee76671fc6cedd1ce45086d F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b4e005fd38b06b8d2f2317b955b93807e80a6a18db5f06d7747978d3bfa11411 -R da5634bacf6dceb36474c55e5f7adbe7 +P 5d92d5f4d8ac4cfa37ba473e5cc861628b783bbf1ae4d138bcae8b9d6cc6e798 +R e8317151b75d57cc72a3267fda8db77e U stephan -Z c74716fca4644a5bd074f3e9b97acc41 +Z 8ef36dab6f79c3b7f36e55e6d2cf608f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e61abf0822..25bf326e8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d92d5f4d8ac4cfa37ba473e5cc861628b783bbf1ae4d138bcae8b9d6cc6e798 \ No newline at end of file +fff68e9f25a57045e9d636b02ffa073cf1b984b2587d4fce10f6e35c9988469c \ No newline at end of file From eadabc65132e65f45a280720e9fd64cccc0a22c2 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 15 Jul 2023 16:30:46 +0000 Subject: [PATCH 07/56] speedtest1 JS: only add --memdb flag by default if no --vfs is provided. FossilOrigin-Name: 676ffe6280c1ce787b04d0cdb4a0664229c6125c601af4b18d1bfa125aac3675 --- ext/wasm/speedtest1-worker.html | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index 3d0fda86ac..b2acf159b6 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -267,7 +267,7 @@ if(urlParams.has('flags')){ preselectedFlags.push(...urlParams.get('flags').split(',')); } - if('opfs'!==urlParams.get('vfs')){ + if(!urlParams.get('vfs')){ preselectedFlags.push('--memdb'); } Object.keys(flags).sort().forEach(function(f){ diff --git a/manifest b/manifest index 0a36f3c117..9a3fc29be1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\sopfs-sahpool\sremoveCapacity()\sto\sreduceCapacity(). -D 2023-07-15T11:37:34.991 +C speedtest1\sJS:\sonly\sadd\s--memdb\sflag\sby\sdefault\sif\sno\s--vfs\sis\sprovided. +D 2023-07-15T16:30:46.383 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -539,7 +539,7 @@ F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe -F ext/wasm/speedtest1-worker.html 97c2bf5f8534091ce718de05801090d5a80c3f13575996f095ba23638e1bdca0 +F ext/wasm/speedtest1-worker.html bbcf1e7fd79541040c1a7ca2ebf1cb7793ddaf9900d6bde1784148f11b807c34 F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d92d5f4d8ac4cfa37ba473e5cc861628b783bbf1ae4d138bcae8b9d6cc6e798 -R e8317151b75d57cc72a3267fda8db77e +P fff68e9f25a57045e9d636b02ffa073cf1b984b2587d4fce10f6e35c9988469c +R 30945b34df9134e0f98668ba08cfc13f U stephan -Z 8ef36dab6f79c3b7f36e55e6d2cf608f +Z 32f9fd4e9e8f1a70cce170b39e2a4458 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 25bf326e8e..dc9bec08cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fff68e9f25a57045e9d636b02ffa073cf1b984b2587d4fce10f6e35c9988469c \ No newline at end of file +676ffe6280c1ce787b04d0cdb4a0664229c6125c601af4b18d1bfa125aac3675 \ No newline at end of file From 38d1db9b79ac1f89d2e97cb35d0de3df330c1f32 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 15 Jul 2023 19:08:58 +0000 Subject: [PATCH 08/56] Correct opfs-sahpool VFS after the pebkac involving the previous speedtest1 runs. Make that VFS explicitly opt-in to avoid certain unfortunate locking situations. FossilOrigin-Name: 41bf1fe31f2f3d0daa2bac25dc57262a4b90f22fed6fa97e4e92467c32ae02dc --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 1260 +++++++++++----------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 1 + ext/wasm/speedtest1-worker.js | 39 +- manifest | 16 +- manifest.uuid | 2 +- 5 files changed, 698 insertions(+), 620 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index acff9ea207..57d674e9fc 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -53,645 +53,689 @@ */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ -const installOpfsVfs = async function(sqlite3){ +const toss = sqlite3.util.toss; +let vfsRegisterResult = undefined; +/** + installOpfsSAHPoolVfs() asynchronously initializes the + OPFS SyncAccessHandle Pool VFS. It returns a Promise + which either resolves to the sqlite3 object or rejects + with an Error value. + + Initialization of this VFS is not automatic because its + registration requires that it lock all resources it + will potentially use, even if client code does not want + to use them. That, in turn, can lead to locking errors + when, for example, one page in a given origin has loaded + this VFS but does not use it, then another page in that + origin tries to use the VFS. If the VFS were automatically + registered, the second page would fail to load the VFS + due to OPFS locking errors. + + On calls after the first this function immediately returns a + resolved or rejected Promise. If called while the first call is + still pending resolution, a rejected promise with a descriptive + error is returned. +*/ +sqlite3.installOpfsSAHPoolVfs = async function(){ + if(sqlite3===vfsRegisterResult) return Promise.resolve(sqlite3); + else if(undefined!==vfsRegisterResult){ + return Promise.reject(vfsRegisterResult); + } if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || !globalThis.FileSystemFileHandle || !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || !navigator?.storage?.getDirectory){ - return Promise.reject(new Error("Missing required OPFS APIs.")); + return Promise.reject(vfsRegisterResult = new Error("Missing required OPFS APIs.")); } - return new Promise(async function(promiseResolve, promiseReject_){ - const verbosity = 2; - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(verbosity>level) loggers[level]("opfs-sahpool:",...args); - }; - const log = (...args)=>logImpl(2, ...args); - const warn = (...args)=>logImpl(1, ...args); - const error = (...args)=>logImpl(0, ...args); - const toss = sqlite3.util.toss; - const capi = sqlite3.capi; - const wasm = sqlite3.wasm; - const opfsIoMethods = new capi.sqlite3_io_methods(); - const opfsVfs = new capi.sqlite3_vfs() - .addOnDispose(()=>opfsIoMethods.dispose()); - const promiseReject = (err)=>{ - opfsVfs.dispose(); - return promiseReject_(err); - }; + vfsRegisterResult = new Error("VFS initialization still underway."); + const verbosity = 2 /*3+ == everything*/; + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const logImpl = (level,...args)=>{ + if(verbosity>level) loggers[level]("opfs-sahpool:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const capi = sqlite3.capi; + const wasm = sqlite3.wasm; + const opfsIoMethods = new capi.sqlite3_io_methods(); + const opfsVfs = new capi.sqlite3_vfs() + .addOnDispose(()=>opfsIoMethods.dispose()); + const promiseReject = (err)=>{ + error("rejecting promise:",err); + //opfsVfs.dispose(); + vfsRegisterResult = err; + return Promise.reject(err); + }; + const promiseResolve = + ()=>Promise.resolve(vfsRegisterResult = sqlite3); + // Config opts for the VFS... + const SECTOR_SIZE = 4096; + const HEADER_MAX_PATH_SIZE = 512; + const HEADER_FLAGS_SIZE = 4; + const HEADER_DIGEST_SIZE = 8; + const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE; + const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE; + const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; + const HEADER_OFFSET_DATA = SECTOR_SIZE; + const DEFAULT_CAPACITY = + sqlite3.config['opfs-sahpool.defaultCapacity'] || 6; + /* Bitmask of file types which may persist across sessions. + SQLITE_OPEN_xyz types not listed here may be inadvertently + left in OPFS but are treated as transient by this VFS and + they will be cleaned up during VFS init. */ + const PERSISTENT_FILE_TYPES = + capi.SQLITE_OPEN_MAIN_DB | + capi.SQLITE_OPEN_MAIN_JOURNAL | + capi.SQLITE_OPEN_SUPER_JOURNAL | + capi.SQLITE_OPEN_WAL /* noting that WAL support is + unavailable in the WASM build.*/; + /* We fetch the default VFS so that we can inherit some + methods from it. */ + const pDVfs = capi.sqlite3_vfs_find(null); + const dVfs = pDVfs + ? new capi.sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. */; + opfsIoMethods.$iVersion = 1; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; + opfsVfs.addOnDispose( + opfsVfs.$zName = wasm.allocCString("opfs-sahpool"), + ()=>(dVfs ? dVfs.dispose() : null) + ); - // Config opts for the VFS... - const SECTOR_SIZE = 4096; - const HEADER_MAX_PATH_SIZE = 512; - const HEADER_FLAGS_SIZE = 4; - const HEADER_DIGEST_SIZE = 8; - const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE; - const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE; - const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; - const HEADER_OFFSET_DATA = SECTOR_SIZE; - const DEFAULT_CAPACITY = - sqlite3.config['opfs-sahpool.defaultCapacity'] || 6; - /* Bitmask of file types which may persist across sessions. - SQLITE_OPEN_xyz types not listed here may be inadvertently - left in OPFS but are treated as transient by this VFS and - they will be cleaned up during VFS init. */ - const PERSISTENT_FILE_TYPES = - capi.SQLITE_OPEN_MAIN_DB | - capi.SQLITE_OPEN_MAIN_JOURNAL | - capi.SQLITE_OPEN_SUPER_JOURNAL | - capi.SQLITE_OPEN_WAL /* noting that WAL support is - unavailable in the WASM build.*/; - /* We fetch the default VFS so that we can inherit some - methods from it. */ - const pDVfs = capi.sqlite3_vfs_find(null); - const dVfs = pDVfs - ? new capi.sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; - opfsVfs.addOnDispose( - opfsVfs.$zName = wasm.allocCString("opfs-sahpool"), - ()=>(dVfs ? dVfs.dispose() : null) - ); + /** + Returns short a string of random alphanumeric characters + suitable for use as a random filename. + */ + const getRandomName = ()=>Math.random().toString(36).slice(2); + /** + All state for the VFS. + */ + const SAHPool = Object.assign(Object.create(null),{ + /* OPFS dir in which VFS metadata is stored. */ + vfsDir: sqlite3.config['vfs:opfs-sahpool:dir'] + || ".sqlite3-opfs-sahpool", + /* Directory handle to this.vfsDir. */ + dirHandle: undefined, + /* Maps SAHs to their opaque file names. */ + mapSAHToName: new Map(), + /* Maps client-side file names to SAHs. */ + mapPathToSAH: new Map(), + /* Set of currently-unused SAHs. */ + availableSAH: new Set(), + /* Maps (sqlite3_file*) to xOpen's file objects. */ + mapIdToFile: new Map(), + /* Current pool capacity. */ + getCapacity: function(){return this.mapSAHToName.size}, + /* Current number of in-use files from pool. */ + getFileCount: function(){return this.mapPathToSAH.size}, /** - Returns short a string of random alphanumeric characters - suitable for use as a random filename. + Adds n files to the pool's capacity. This change is + persistent across settings. Returns a Promise which resolves + to the new capacity. */ - const getRandomName = ()=>Math.random().toString(36).slice(2); - + addCapacity: async function(n){ + const cap = this.getCapacity(); + for(let i = cap; i < cap+n; ++i){ + const name = getRandomName(); + const h = await this.dirHandle.getFileHandle(name, {create:true}); + const ah = await h.createSyncAccessHandle(); + this.mapSAHToName.set(ah,name); + this.setAssociatedPath(ah, '', 0); + } + return i; + }, /** - All state for the VFS. + Removes n entries from the pool's current capacity + if possible. It can only remove currently-unallocated + files. Returns a Promise resolving to the number of + removed files. */ - const SAHPool = Object.assign(Object.create(null),{ - /* OPFS dir in which VFS metadata is stored. */ - vfsDir: sqlite3.config['vfs:opfs-sahpool:dir'] - || ".sqlite3-opfs-sahpool", - /* Directory handle to this.vfsDir. */ - dirHandle: undefined, - /* Maps SAHs to their opaque file names. */ - mapSAHToName: new Map(), - /* Maps client-side file names to SAHs. */ - mapPathToSAH: new Map(), - /* Set of currently-unused SAHs. */ - availableSAH: new Set(), - /* Maps (sqlite3_file*) to xOpen's file objects. */ - mapIdToFile: new Map(), - /* Current pool capacity. */ - getCapacity: function(){return this.mapSAHToName.size}, - /* Current number of in-use files from pool. */ - getFileCount: function(){return this.mapPathToSAH.size}, - /** - Adds n files to the pool's capacity. This change is - persistent across settings. Returns a Promise which resolves - to the new capacity. - */ - addCapacity: async function(n){ - const cap = this.getCapacity(); - for(let i = cap; i < cap+n; ++i){ - const name = getRandomName(); - const h = await this.dirHandle.getFileHandle(name, {create:true}); - const ah = await h.createSyncAccessHandle(); - this.mapSAHToName.set(ah,name); - this.setAssociatedPath(ah, '', 0); + reduceCapacity: async function(n){ + let nRm = 0; + for(const ah of Array.from(this.availableSAH)){ + if(nRm === n || this.getFileCount() === this.getCapacity()){ + break; } - return i; - }, - /** - Removes n entries from the pool's current capacity - if possible. It can only remove currently-unallocated - files. Returns a Promise resolving to the number of - removed files. - */ - reduceCapacity: async function(n){ - let nRm = 0; - for(const ah of Array.from(this.availableSAH)){ - if(nRm === n || this.getFileCount() === this.getCapacity()){ - break; + const name = this.mapSAHToName.get(ah); + ah.close(); + await this.dirHandle.removeEntry(name); + this.mapSAHToName.delete(ah); + this.availableSAH.delete(ah); + ++nRm; + } + return nRm; + }, + /** + Releases all currently-opened SAHs. + */ + releaseAccessHandles: function(){ + for(const ah of this.mapSAHToName.keys()) ah.close(); + this.mapSAHToName.clear(); + this.mapPathToSAH.clear(); + this.availableSAH.clear(); + }, + /** + Opens all files under this.vfsDir/this.dirHandle and acquires + a SAH for each. returns a Promise which resolves to no value + but completes once all SAHs are acquired. If acquiring an SAH + throws, SAHPool.$error will contain the corresponding + exception. + */ + acquireAccessHandles: async function(){ + const files = []; + for await (const [name,h] of this.dirHandle){ + if('file'===h.kind){ + files.push([name,h]); + } + } + await Promise.all(files.map(async ([name,h])=>{ + try{ + const ah = await h.createSyncAccessHandle() + /*TODO: clean up and fail vfs init on error*/; + this.mapSAHToName.set(ah, name); + const path = this.getAssociatedPath(ah); + if(path){ + this.mapPathToSAH.set(path, ah); + }else{ + this.availableSAH.add(ah); } - const name = this.mapSAHToName.get(ah); - ah.close(); - await this.dirHandle.removeEntry(name); - this.mapSAHToName.delete(ah); - this.availableSAH.delete(ah); - ++nRm; + }catch(e){ + SAHPool.storeErr(e); + throw e; } - return nRm; - }, - /** - Releases all currently-opened SAHs. - */ - releaseAccessHandles: function(){ - for(const ah of this.mapSAHToName.keys()) ah.close(); - this.mapSAHToName.clear(); - this.mapPathToSAH.clear(); - this.availableSAH.clear(); - }, - /** - Opens all files under this.vfsDir/this.dirHandle and acquires - a SAH for each. returns a Promise which resolves to no value - but completes once all SAHs are acquired. If acquiring an SAH - throws, SAHPool.$error will contain the corresponding - exception. - */ - acquireAccessHandles: async function(){ - const files = []; - for await (const [name,h] of this.dirHandle){ - if('file'===h.kind){ - files.push([name,h]); - } - } - await Promise.all(files.map(async ([name,h])=>{ - try{ - const ah = await h.createSyncAccessHandle() - /*TODO: clean up and fail vfs init on error*/; - this.mapSAHToName.set(ah, name); - const path = this.getAssociatedPath(ah); - if(path){ - this.mapPathToSAH.set(path, ah); - }else{ - this.availableSAH.add(ah); - } - }catch(e){ - SAHPool.storeErr(e); - throw e; - } - })); - }, - /** Buffer used by [sg]etAssociatedPath(). */ - apBody: new Uint8Array(HEADER_CORPUS_SIZE), - textDecoder: new TextDecoder(), - textEncoder: new TextEncoder(), - /** - Given an SAH, returns the client-specified name of - that file by extracting it from the SAH's header. + })); + }, + /** Buffer used by [sg]etAssociatedPath(). */ + apBody: new Uint8Array(HEADER_CORPUS_SIZE), + textDecoder: new TextDecoder(), + textEncoder: new TextEncoder(), + /** + Given an SAH, returns the client-specified name of + that file by extracting it from the SAH's header. - On error, it disassociates SAH from the pool and - returns an empty string. - */ - getAssociatedPath: function(sah){ - const body = this.apBody; - sah.read(body, {at: 0}); - // Delete any unexpected files left over by previous - // untimely errors... - const dv = new DataView(body.buffer, body.byteOffset); - const flags = dv.getUint32(HEADER_OFFSET_FLAGS); - if(body[0] && - ((flags & capi.SQLITE_OPEN_DELETEONCLOSE) || - (flags & PERSISTENT_FILE_TYPES)===0)){ - warn(`Removing file with unexpected flags ${flags.toString(16)}`); - this.setAssociatedPath(sah, '', 0); - return ''; - } + On error, it disassociates SAH from the pool and + returns an empty string. + */ + getAssociatedPath: function(sah){ + const body = this.apBody; + sah.read(body, {at: 0}); + // Delete any unexpected files left over by previous + // untimely errors... + const dv = new DataView(body.buffer, body.byteOffset); + const flags = dv.getUint32(HEADER_OFFSET_FLAGS); + if(body[0] && + ((flags & capi.SQLITE_OPEN_DELETEONCLOSE) || + (flags & PERSISTENT_FILE_TYPES)===0)){ + warn(`Removing file with unexpected flags ${flags.toString(16)}`); + this.setAssociatedPath(sah, '', 0); + return ''; + } - const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); - sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); - const compDigest = this.computeDigest(body); - if(fileDigest.every((v,i) => v===compDigest[i])){ - // Valid digest - const pathBytes = body.findIndex((v)=>0===v); - if(0===pathBytes){ - // This file is unassociated, so truncate it to avoid - // leaving stale db data laying around. - sah.truncate(HEADER_OFFSET_DATA); - } - return this.textDecoder.decode(body.subarray(0,pathBytes)); - }else{ - // Invalid digest - warn('Disassociating file with bad digest.'); - this.setAssociatedPath(sah, '', 0); - return ''; - } - }, - /** - Stores the given client-defined path and SQLITE_OPEN_xyz - flags into the given SAH. - */ - setAssociatedPath: function(sah, path, flags){ - const body = this.apBody; - const enc = this.textEncoder.encodeInto(path, body); - if(HEADER_MAX_PATH_SIZE <= enc.written){ - toss("Path too long:",path); - } - - const dv = new DataView(body.buffer, body.byteOffset); - dv.setUint32(HEADER_OFFSET_FLAGS, flags); - - const digest = this.computeDigest(body); - sah.write(body, {at: 0}); - sah.write(digest, {at: HEADER_OFFSET_DIGEST}); - sah.flush(); - - if(path){ - this.mapPathToSAH.set(path, sah); - this.availableSAH.delete(sah); - }else{ - // This is not a persistent file, so eliminate the contents. + const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); + sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); + const compDigest = this.computeDigest(body); + if(fileDigest.every((v,i) => v===compDigest[i])){ + // Valid digest + const pathBytes = body.findIndex((v)=>0===v); + if(0===pathBytes){ + // This file is unassociated, so truncate it to avoid + // leaving stale db data laying around. sah.truncate(HEADER_OFFSET_DATA); - this.mapPathToSAH.delete(path); - this.availableSAH.add(sah); } - }, - /** - Computes a digest for the given byte array and - returns it as a two-element Uint32Array. - */ - computeDigest: function(byteArray){ - if(!byteArray[0]){ - // Deleted file - return new Uint32Array([0xfecc5f80, 0xaccec037]); - } - let h1 = 0xdeadbeef; - let h2 = 0x41c6ce57; - for(const v of byteArray){ - h1 = 31 * h1 + (v * 307); - h2 = 31 * h2 + (v * 307); - } - return new Uint32Array([h1>>>0, h2>>>0]); - }, - reset: async function(){ - await this.isReady; - let h = await navigator.storage.getDirectory(); - for(const d of this.vfsDir.split('/')){ - if(d){ - h = await h.getDirectoryHandle(d,{create:true}); - } - } - this.dirHandle = h; - this.releaseAccessHandles(); - await this.acquireAccessHandles(); - }, - /** - Returns the pathname part of the given argument, - which may be any of: - - - a URL object - - A JS string representing a file name - - Wasm C-string representing a file name - */ - getPath: function(arg) { - if(wasm.isPtr(arg)) arg = wasm.cstrToJs(arg); - return ((arg instanceof URL) - ? arg - : new URL(arg, 'file://localhost/')).pathname; - }, - /** - Removes the association of the given client-specified file - name (JS string) from the pool. - */ - deletePath: function(path) { - const sah = this.mapPathToSAH.get(path); - if(sah) { - // Un-associate the SQLite path from the OPFS file. - this.setAssociatedPath(sah, '', 0); - } - }, - /** - Sets e as this object's current error. Pass a falsy - (or no) value to clear it. - */ - storeErr: function(e){ - return this.$error = e; - }, - /** - Pops this object's Error object and returns - it (a falsy value if no error is set). - */ - popErr: function(e){ - const rc = this.$error; - this.$error = undefined; - return rc; + return this.textDecoder.decode(body.subarray(0,pathBytes)); + }else{ + // Invalid digest + warn('Disassociating file with bad digest.'); + this.setAssociatedPath(sah, '', 0); + return ''; } - })/*SAHPool*/; - sqlite3.SAHPool = SAHPool/*only for testing*/; + }, /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. + Stores the given client-defined path and SQLITE_OPEN_xyz + flags into the given SAH. */ - const ioSyncWrappers = { - xCheckReservedLock: function(pFile,pOut){ - SAHPool.storeErr(); - return 0; - }, - xClose: function(pFile){ - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - if(file) { - try{ - log(`xClose ${file.path}`); - file.sah.flush(); - SAHPool.mapIdToFile.delete(pFIle); - if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ - SAHPool.deletePath(file.path); - } - }catch(e){ - SAHPool.storeErr(e); - error("xClose() failed:",e.message); - return capi.SQLITE_IOERR; - } + setAssociatedPath: function(sah, path, flags){ + const body = this.apBody; + const enc = this.textEncoder.encodeInto(path, body); + if(HEADER_MAX_PATH_SIZE <= enc.written){ + toss("Path too long:",path); + } + + const dv = new DataView(body.buffer, body.byteOffset); + dv.setUint32(HEADER_OFFSET_FLAGS, flags); + + const digest = this.computeDigest(body); + sah.write(body, {at: 0}); + sah.write(digest, {at: HEADER_OFFSET_DIGEST}); + sah.flush(); + + if(path){ + this.mapPathToSAH.set(path, sah); + this.availableSAH.delete(sah); + }else{ + // This is not a persistent file, so eliminate the contents. + sah.truncate(HEADER_OFFSET_DATA); + this.mapPathToSAH.delete(path); + this.availableSAH.add(sah); + } + }, + /** + Computes a digest for the given byte array and + returns it as a two-element Uint32Array. + */ + computeDigest: function(byteArray){ + if(!byteArray[0]){ + // Deleted file + return new Uint32Array([0xfecc5f80, 0xaccec037]); + } + let h1 = 0xdeadbeef; + let h2 = 0x41c6ce57; + for(const v of byteArray){ + h1 = 31 * h1 + (v * 307); + h2 = 31 * h2 + (v * 307); + } + return new Uint32Array([h1>>>0, h2>>>0]); + }, + reset: async function(){ + await this.isReady; + let h = await navigator.storage.getDirectory(); + for(const d of this.vfsDir.split('/')){ + if(d){ + h = await h.getDirectoryHandle(d,{create:true}); } - return 0; - }, - xDeviceCharacteristics: function(pFile){ - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - const file = SAHPool.mapIdToFile(pFile); - const size = file.sah.getSize() - HEADER_OFFSET_DATA; - //log(`xFileSize ${file.path} ${size}`); - wasm.poke64(pSz64, BigInt(size)); - return 0; - }, - xLock: function(pFile,lockType){ - SAHPool.storeErr(); - let rc = capi.SQLITE_IOERR; - return rc; - }, - xRead: function(pFile,pDest,n,offset64){ - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - log(`xRead ${file.path} ${n} ${offset64}`); - try { - const nRead = file.sah.read( - pDest, {at: HEADER_OFFSET_DATA + offset64} - ); - if(nRead < n){ - wasm.heap8u().fill(0, pDest + nRead, pDest + n); - return capi.SQLITE_IOERR_SHORT_READ; - } - return 0; - }catch(e){ - SAHPool.storeErr(e); - error("xRead() failed:",e.message); - return capi.SQLITE_IOERR; - } - }, - xSectorSize: function(pFile){ - return SECTOR_SIZE; - }, - xSync: function(pFile,flags){ - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - //log(`xSync ${file.path} ${flags}`); + } + this.dirHandle = h; + this.releaseAccessHandles(); + await this.acquireAccessHandles(); + }, + /** + Returns the pathname part of the given argument, + which may be any of: + + - a URL object + - A JS string representing a file name + - Wasm C-string representing a file name + */ + getPath: function(arg) { + if(wasm.isPtr(arg)) arg = wasm.cstrToJs(arg); + return ((arg instanceof URL) + ? arg + : new URL(arg, 'file://localhost/')).pathname; + }, + /** + Removes the association of the given client-specified file + name (JS string) from the pool. + */ + deletePath: function(path) { + const sah = this.mapPathToSAH.get(path); + if(sah) { + // Un-associate the SQLite path from the OPFS file. + this.setAssociatedPath(sah, '', 0); + } + }, + /** + Sets e as this object's current error. Pass a falsy + (or no) value to clear it. + */ + storeErr: function(e){ + if(e) error(e); + return this.$error = e; + }, + /** + Pops this object's Error object and returns + it (a falsy value if no error is set). + */ + popErr: function(){ + const rc = this.$error; + this.$error = undefined; + return rc; + } + })/*SAHPool*/; + sqlite3.SAHPool = SAHPool/*only for testing*/; + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = { + xCheckReservedLock: function(pFile,pOut){ + log('xCheckReservedLock'); + SAHPool.storeErr(); + wasm.poke32(pOut, 1); + return 0; + }, + xClose: function(pFile){ + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + if(file) { try{ + log(`xClose ${file}`); + if(file.sq3File) file.sq3File.dispose(); file.sah.flush(); - return 0; + SAHPool.mapIdToFile.delete(pFile); + if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ + SAHPool.deletePath(file.path); + } }catch(e){ SAHPool.storeErr(e); - error("xSync() failed:",e.message); - return capi.SQLITE_IOERR; - } - }, - xTruncate: function(pFile,sz64){ - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - //log(`xTruncate ${file.path} ${iSize}`); - try{ - file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); - return 0; - }catch(e){ - SAHPool.storeErr(e); - error("xTruncate() failed:",e.message); - return capi.SQLITE_IOERR; - } - }, - /**xUnlock: function(pFile,lockType){ - return capi.SQLITE_IOERR; - },*/ - xWrite: function(pFile,pSrc,n,offset64){ - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile(pFile); - //log(`xWrite ${file.path} ${n} ${offset64}`); - try{ - const nBytes = file.sah.write( - pSrc, { at: HEADER_OFFSET_DATA + Number(offset64) } - ); - return nBytes === n ? 0 : capi.SQLITE_IOERR; - }catch(e){ - SAHPool.storeErr(e); - error("xWrite() failed:",e.message); return capi.SQLITE_IOERR; } } - }/*ioSyncWrappers*/; - - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsSyncWrappers = { - xAccess: function(pVfs,zName,flags,pOut){ - SAHPool.storeErr(); - try{ - const name = this.getPath(zName); - wasm.poke32(pOut, SAHPool.mapPathToSAH.has(name) ? 1 : 0); - }catch(e){ - /*ignored*/; - } - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - SAHPool.storeErr(); - try{ - SAHPool.deletePath(SAHPool.getPath(zName)); - return 0; - }catch(e){ - SAHPool.storeErr(e); - error("Error xDelete()ing file:",e.message); - return capi.SQLITE_IOERR_DELETE; - } - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - const i = wasm.cstrncpy(pOut, zName, nOut); - return i nOut) wasm.poke8(pOut + nOut - 1, 0); - }catch(e){ - return capi.SQLITE_NOMEM; - }finally{ - wasm.scopedAllocPop(scope); - } - } - return 0; - }, - //xSleep is optionally defined below - xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ - try{ - // First try to open a path that already exists in the file system. - const path = (zName && wasm.peek8(zName)) - ? SAHPool.getPath(name) - : getRandomName(); - let ah = SAHPool.mapPathToSAH.get(path); - if(!ah && (flags & capi.SQLITE_OPEN_CREATE)) { - // File not found so try to create it. - if(SAHPool.getFileCount() < SAHPool.getCapacity()) { - // Choose an unassociated OPFS file from the pool. - ah = SAHPool.availableSAH.keys()[0]; - SAHPool.setAssociatedPath(ah, path, flags); - }else{ - // File pool is full. - toss('SAH pool is full. Cannot create file',path); - } - } - if(!ah){ - toss('file not found:',path); - } - // Subsequent methods are only passed the file pointer, so - // map the relevant info we need to that pointer. - const file = { path, flags, ah }; - SAHPool.mapIdToFile.set(pFile, file); - wasm.poke32(pOutFlags, flags); - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_CANTOPEN; - } - }/*xOpen()*/ - }/*vfsSyncWrappers*/; - - if(dVfs){ - /* Inherit certain VFS members from the default VFS, - if available. */ - opfsVfs.$xRandomness = dVfs.$xRandomness; - opfsVfs.$xSleep = dVfs.$xSleep; - } - if(!opfsVfs.$xRandomness){ - /* If the default VFS has no xRandomness(), add a basic JS impl... */ - vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ - const heap = wasm.heap8u(); - let i = 0; - for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; - return i; - }; - } - if(!opfsVfs.$xSleep){ - vfsSyncWrappers.xSleep = function(pVfs,ms){ - return 0; - }; - } - - /** - Ensure that the client has a "fully-sync" SAH impl, - else reject the promise. Returns true on success, - else false. - */ - if(!(async ()=>{ + return 0; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + log(`xFileSize`); + const file = SAHPool.mapIdToFile.get(pFile); + const size = file.sah.getSize() - HEADER_OFFSET_DATA; + //log(`xFileSize ${file.path} ${size}`); + wasm.poke64(pSz64, BigInt(size)); + return 0; + }, + xLock: function(pFile,lockType){ + log(`xLock ${lockType}`); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + file.lockType = lockType; + return 0; + }, + xRead: function(pFile,pDest,n,offset64){ + log(`xRead ${n}@${offset64}`); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + log(`xRead ${file.path} ${n} ${offset64}`); try { - const dh = await navigator.storage.getDirectory(); - const fn = '.opfs-sahpool-sync-check-'+getRandomName(); - const fh = await dh.getFileHandle(fn, { create: true }); - const ah = await fh.createSyncAccessHandle(); - const close = ah.close(); - await close; - await dh.removeEntry(fn); - if(close?.then){ - toss("The local OPFS API is too old for opfs-sahpool:", - "it has an async FileSystemSyncAccessHandle.close() method."); - } - return true; - }catch(e){ - promiseReject(e); - return false; - } - })()){ - return; - } - - SAHPool.isReady = SAHPool.reset().then(async ()=>{ - if(SAHPool.$error){ - throw SAHPool.$error; - } - if(0===SAHPool.getCapacity()){ - await SAHPool.addCapacity(DEFAULT_CAPACITY); - } - //log("vfs list:",capi.sqlite3_js_vfs_list()); - sqlite3.vfs.installVfs({ - io: {struct: opfsIoMethods, methods: ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: vfsSyncWrappers} - }); - //log("vfs list:",capi.sqlite3_js_vfs_list()); - if(sqlite3.oo1){ - const OpfsSAHPoolDb = function(...args){ - const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; - sqlite3.oo1.DB.dbCtorHelper.call(this, opt); - }; - OpfsSAHPoolDb.prototype = Object.create(sqlite3.oo1.DB.prototype); - OpfsSAHPoolDb.addPoolCapacity = async (n)=>SAHPool.addCapacity(n); - OpfsSAHPoolDb.reducePoolCapacity = async (n)=>SAHPool.reduceCapacity(n); - OpfsSAHPoolDb.getPoolCapacity = ()=>SAHPool.getCapacity(); - OpfsSAHPoolDb.getPoolUsage = ()=>SAHPool.getFileCount(); - sqlite3.oo1.OpfsSAHPoolDb = OpfsSAHPoolDb; - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( - opfsVfs.pointer, - function(oo1Db, sqlite3){ - sqlite3.capi.sqlite3_exec(oo1Db, [ - /* As of July 2023, the PERSIST journal mode on OPFS is - somewhat slower than DELETE or TRUNCATE (it was faster - before Chrome version 108 or 109). TRUNCATE and DELETE - have very similar performance on OPFS. - - Roy Hashimoto notes that TRUNCATE and PERSIST modes may - decrease OPFS concurrency because multiple connections - can open the journal file in those modes: - - https://github.com/rhashimoto/wa-sqlite/issues/68 - - Given that, and the fact that testing has not revealed - any appreciable difference between performance of - TRUNCATE and DELETE modes on OPFS, we currently (as of - 2023-07-13) default to DELETE mode. - */ - "pragma journal_mode=DELETE;", - /* - OPFS benefits hugely from cache on moderate/large - speedtest1 --size 50 and --size 100 workloads. We - currently rely on setting a non-default cache size when - building sqlite3.wasm. If that policy changes, the cache - can be set here. - */ - "pragma cache_size=-16384;" - ], 0, 0, 0); - } + const nRead = file.sah.read( + wasm.heap8u().subarray(pDest, pDest+n), + {at: HEADER_OFFSET_DATA + Number(offset64)} ); - }/*extend sqlite3.oo1*/ - log("VFS initialized."); - promiseResolve(sqlite3); - }).catch(promiseReject); - })/*return Promise*/; -}/*installOpfsVfs()*/; + if(nRead < n){ + wasm.heap8u().fill(0, pDest + nRead, pDest + n); + return capi.SQLITE_IOERR_SHORT_READ; + } + return 0; + }catch(e){ + SAHPool.storeErr(e); + error("xRead() failed:",e.message); + return capi.SQLITE_IOERR; + } + }, + /*xSectorSize: function(pFile){ + return SECTOR_SIZE; + },*/ + xSync: function(pFile,flags){ + log(`xSync ${flags}`); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + //log(`xSync ${file.path} ${flags}`); + try{ + file.sah.flush(); + return 0; + }catch(e){ + SAHPool.storeErr(e); + error("xSync() failed:",e.message); + return capi.SQLITE_IOERR; + } + }, + xTruncate: function(pFile,sz64){ + log(`xTruncate ${sz64}`); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + //log(`xTruncate ${file.path} ${iSize}`); + try{ + file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); + return 0; + }catch(e){ + SAHPool.storeErr(e); + error("xTruncate() failed:",e.message); + return capi.SQLITE_IOERR; + } + }, + xUnlock: function(pFile,lockType){ + log('xUnlock'); + const file = SAHPool.mapIdToFile.get(pFile); + file.lockType = lockType; + return 0; + }, + xWrite: function(pFile,pSrc,n,offset64){ + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + log(`xWrite ${file.path} ${n} ${offset64}`); + try{ + const nBytes = file.sah.write( + wasm.heap8u().subarray(pSrc, pSrc+n), + { at: HEADER_OFFSET_DATA + Number(offset64) } + ); + return nBytes === n ? 0 : capi.SQLITE_IOERR; + }catch(e){ + SAHPool.storeErr(e); + error("xWrite() failed:",e.message); + return capi.SQLITE_IOERR; + } + } + }/*ioSyncWrappers*/; -globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ - return installOpfsVfs(sqlite3).catch((e)=>{ - sqlite3.config.warn("Ignoring inability to install opfs-sahpool sqlite3_vfs:", - e.message, e); - }); -}/*sqlite3ApiBootstrap.initializersAsync*/); + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + log(`xAccess ${wasm.cstrToJs(zName)}`); + SAHPool.storeErr(); + try{ + const name = this.getPath(zName); + wasm.poke32(pOut, SAHPool.mapPathToSAH.has(name) ? 1 : 0); + }catch(e){ + /*ignored*/; + } + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + log(`xDelete ${wasm.cstrToJs(zName)}`); + SAHPool.storeErr(); + try{ + SAHPool.deletePath(SAHPool.getPath(zName)); + return 0; + }catch(e){ + SAHPool.storeErr(e); + error("Error xDelete()ing file:",e.message); + return capi.SQLITE_IOERR_DELETE; + } + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + log(`xFullPathname ${wasm.cstrToJs(zName)}`); + const i = wasm.cstrncpy(pOut, zName, nOut); + return i nOut) wasm.poke8(pOut + nOut - 1, 0); + }catch(e){ + return capi.SQLITE_NOMEM; + }finally{ + wasm.scopedAllocPop(scope); + } + } + return 0; + }, + //xSleep is optionally defined below + xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ + log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`); + try{ + // First try to open a path that already exists in the file system. + const path = (zName && wasm.peek8(zName)) + ? SAHPool.getPath(zName) + : getRandomName(); + let sah = SAHPool.mapPathToSAH.get(path); + if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { + // File not found so try to create it. + if(SAHPool.getFileCount() < SAHPool.getCapacity()) { + // Choose an unassociated OPFS file from the pool. + [sah] = SAHPool.availableSAH.keys(); + SAHPool.setAssociatedPath(sah, path, flags); + }else{ + // File pool is full. + toss('SAH pool is full. Cannot create file',path); + } + } + if(!sah){ + toss('file not found:',path); + } + // Subsequent methods are only passed the file pointer, so + // map the relevant info we need to that pointer. + const file = {path, flags, sah}; + SAHPool.mapIdToFile.set(pFile, file); + wasm.poke32(pOutFlags, flags); + file.sq3File = new capi.sqlite3_file(pFile); + file.sq3File.$pMethods = opfsIoMethods.pointer; + file.lockType = capi.SQLITE_LOCK_NONE; + return 0; + }catch(e){ + SAHPool.storeErr(e); + return capi.SQLITE_CANTOPEN; + } + }/*xOpen()*/ + }/*vfsSyncWrappers*/; + + if(dVfs){ + /* Inherit certain VFS members from the default VFS, + if available. */ + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; + } + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + vfsSyncWrappers.xSleep = (pVfs,ms)=>0; + } + + /** + Ensure that the client has a "fully-sync" SAH impl, + else reject the promise. Returns true on success, + else a value intended to be returned via the containing + function's Promise result. + */ + const apiVersionCheck = await (async ()=>{ + try { + const dh = await navigator.storage.getDirectory(); + const fn = '.opfs-sahpool-sync-check-'+getRandomName(); + const fh = await dh.getFileHandle(fn, { create: true }); + const ah = await fh.createSyncAccessHandle(); + const close = ah.close(); + await close; + await dh.removeEntry(fn); + if(close?.then){ + toss("The local OPFS API is too old for opfs-sahpool:", + "it has an async FileSystemSyncAccessHandle.close() method."); + } + return true; + }catch(e){ + return e; + } + })(); + if(true!==apiVersionCheck){ + return promiseReject(apiVersionCheck); + } + return SAHPool.isReady = SAHPool.reset().then(async ()=>{ + if(SAHPool.$error){ + throw SAHPool.$error; + } + if(0===SAHPool.getCapacity()){ + await SAHPool.addCapacity(DEFAULT_CAPACITY); + } + //log("vfs list:",capi.sqlite3_js_vfs_list()); + sqlite3.vfs.installVfs({ + io: {struct: opfsIoMethods, methods: ioSyncWrappers}, + vfs: {struct: opfsVfs, methods: vfsSyncWrappers}, + applyArgcCheck: true + }); + log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods); + log("vfs list:",capi.sqlite3_js_vfs_list()); + if(sqlite3.oo1){ + const OpfsSAHPoolDb = function(...args){ + const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = opfsVfs.$zName; + sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + }; + OpfsSAHPoolDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + OpfsSAHPoolDb.addPoolCapacity = async (n)=>SAHPool.addCapacity(n); + OpfsSAHPoolDb.reducePoolCapacity = async (n)=>SAHPool.reduceCapacity(n); + OpfsSAHPoolDb.getPoolCapacity = ()=>SAHPool.getCapacity(); + OpfsSAHPoolDb.getPoolUsage = ()=>SAHPool.getFileCount(); + sqlite3.oo1.OpfsSAHPoolDb = OpfsSAHPoolDb; + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( + opfsVfs.pointer, + function(oo1Db, sqlite3){ + sqlite3.capi.sqlite3_exec(oo1Db, [ + /* As of July 2023, the PERSIST journal mode on OPFS is + somewhat slower than DELETE or TRUNCATE (it was faster + before Chrome version 108 or 109). TRUNCATE and DELETE + have very similar performance on OPFS. + + Roy Hashimoto notes that TRUNCATE and PERSIST modes may + decrease OPFS concurrency because multiple connections + can open the journal file in those modes: + + https://github.com/rhashimoto/wa-sqlite/issues/68 + + Given that, and the fact that testing has not revealed + any appreciable difference between performance of + TRUNCATE and DELETE modes on OPFS, we currently (as of + 2023-07-13) default to DELETE mode. + */ + "pragma journal_mode=DELETE;", + /* + OPFS benefits hugely from cache on moderate/large + speedtest1 --size 50 and --size 100 workloads. We + currently rely on setting a non-default cache size when + building sqlite3.wasm. If that policy changes, the cache + can be set here. + */ + "pragma cache_size=-16384;" + ], 0, 0, 0); + } + ); + }/*extend sqlite3.oo1*/ + log("VFS initialized."); + return promiseResolve(); + }).catch(promiseReject); +}/*installOpfsSAHPoolVfs()*/; }/*sqlite3ApiBootstrap.initializers*/); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 40c6090bf9..c7a752441f 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -236,6 +236,7 @@ const installOpfsVfs = function callee(options){ ? new sqlite3_vfs(pDVfs) : null /* dVfs will be null when sqlite3 is built with SQLITE_OS_OTHER. */; + opfsIoMethods.$iVersion = 1; opfsVfs.$iVersion = 2/*yes, two*/; opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; opfsVfs.$mxPathname = 1024/*sure, why not?*/; diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index c61cab9190..89ab2149ef 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -71,20 +71,48 @@ self.onmessage = function(msg){ msg = msg.data; switch(msg.type){ - case 'run': runSpeedtest(msg.data || []); break; + case 'run': + try { + runSpeedtest(msg.data || []); + }catch(e){ + mPost('error',e); + } + break; default: logErr("Unhandled worker message type:",msg.type); break; } }; + const sahpSanityChecks = function(sqlite3){ + log("Attempting OpfsSAHPoolDb sanity checks..."); + const db = new sqlite3.oo1.OpfsSAHPoolDb('opfs-sahpoool.db'); + const fn = db.filename; + db.exec([ + 'create table t(a);', + 'insert into t(a) values(1),(2),(3);' + ]); + db.close(); + sqlite3.wasm.sqlite3_wasm_vfs_unlink(sqlite3_vfs_find("opfs-sahpool"), fn); + log("SAH sanity checks done."); + }; + const EmscriptenModule = { print: log, printErr: logErr, setStatus: (text)=>mPost('load-status',text) }; - self.sqlite3InitModule(EmscriptenModule).then((sqlite3)=>{ - const S = sqlite3; + log("Initializing speedtest1 module..."); + self.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ + const S = globalThis.S = sqlite3; + log("Loaded speedtest1 module. Setting up..."); + if(S.installOpfsSAHPoolVfs){ + await S.installOpfsSAHPoolVfs().then(()=>{ + log("Loaded SAHPool."); + }).catch(e=>{ + logErr("Error setting up SAHPool:",e.message); + }); + } App.vfsUnlink = function(pDb, fname){ const pVfs = S.wasm.sqlite3_wasm_db_vfs(pDb, 0); if(pVfs) S.wasm.sqlite3_wasm_vfs_unlink(pVfs, fname||0); @@ -95,5 +123,10 @@ //else log("Using transient storage."); mPost('ready',true); log("Registered VFSes:", ...S.capi.sqlite3_js_vfs_list()); + if(0 && S.installOpfsSAHPoolVfs){ + sahpSanityChecks(S); + } + }).catch(e=>{ + logErr(e); }); })(); diff --git a/manifest b/manifest index 9a3fc29be1..c5fd720553 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C speedtest1\sJS:\sonly\sadd\s--memdb\sflag\sby\sdefault\sif\sno\s--vfs\sis\sprovided. -D 2023-07-15T16:30:46.383 +C Correct\sopfs-sahpool\sVFS\safter\sthe\spebkac\sinvolving\sthe\sprevious\sspeedtest1\sruns.\sMake\sthat\sVFS\sexplicitly\sopt-in\sto\savoid\scertain\sunfortunate\slocking\ssituations. +D 2023-07-15T19:08:58.138 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,8 +502,8 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js ad6ec4e87f47152a871a23bf90b64709094bf04e8ee76671fc6cedd1ce45086d -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 83388ead4bfc489bee008298ab51948ccb75227795ce8d1634f2eec8e02548f1 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js a5c3195203e6085d7aa89fae4b84cf3f3eec4ff4f928c6d0e5d3ef8b14cbc1c0 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html bbcf1e7fd79541040c1a7ca2ebf1cb7793ddaf9900d6bde1784148f11b807c34 -F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1 +F ext/wasm/speedtest1-worker.js 4de92e4e6718b8bd1cdecb75af62739d1115fa66656a700b0b51822c848948f5 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fff68e9f25a57045e9d636b02ffa073cf1b984b2587d4fce10f6e35c9988469c -R 30945b34df9134e0f98668ba08cfc13f +P 676ffe6280c1ce787b04d0cdb4a0664229c6125c601af4b18d1bfa125aac3675 +R bca4913f68935c8abed9e461aac753fd U stephan -Z 32f9fd4e9e8f1a70cce170b39e2a4458 +Z 2546a1c8fd9c0ca0c4fd392086704b47 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dc9bec08cd..9ae594ffea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -676ffe6280c1ce787b04d0cdb4a0664229c6125c601af4b18d1bfa125aac3675 \ No newline at end of file +41bf1fe31f2f3d0daa2bac25dc57262a4b90f22fed6fa97e4e92467c32ae02dc \ No newline at end of file From 28d46cce0bb2bc9643e394634d8841be9239fe07 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 15 Jul 2023 21:08:48 +0000 Subject: [PATCH 09/56] Minor cleanups in the opfs-sahpool VFS. FossilOrigin-Name: 279e09070918dab7b60c39179ebb7eb931ca6bd4e589b414f436740499a2f910 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 21 ++++++++------------- ext/wasm/speedtest1-worker.js | 6 ++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 57d674e9fc..15fc687877 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -162,7 +162,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ */ const SAHPool = Object.assign(Object.create(null),{ /* OPFS dir in which VFS metadata is stored. */ - vfsDir: sqlite3.config['vfs:opfs-sahpool:dir'] + vfsDir: sqlite3.config['opfs-sahpool.dir'] || ".sqlite3-opfs-sahpool", /* Directory handle to this.vfsDir. */ dirHandle: undefined, @@ -241,7 +241,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ await Promise.all(files.map(async ([name,h])=>{ try{ const ah = await h.createSyncAccessHandle() - /*TODO: clean up and fail vfs init on error*/; this.mapSAHToName.set(ah, name); const path = this.getAssociatedPath(ah); if(path){ @@ -251,6 +250,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ } }catch(e){ SAHPool.storeErr(e); + this.releaseAccessHandles(); throw e; } })); @@ -334,10 +334,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ returns it as a two-element Uint32Array. */ computeDigest: function(byteArray){ - if(!byteArray[0]){ - // Deleted file - return new Uint32Array([0xfecc5f80, 0xaccec037]); - } let h1 = 0xdeadbeef; let h2 = 0x41c6ce57; for(const v of byteArray){ @@ -346,6 +342,10 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ } return new Uint32Array([h1>>>0, h2>>>0]); }, + /** + Re-initializes the state of the SAH pool, + releasing and re-acquiring all handles. + */ reset: async function(){ await this.isReady; let h = await navigator.storage.getDirectory(); @@ -470,13 +470,12 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ return 0; }catch(e){ SAHPool.storeErr(e); - error("xRead() failed:",e.message); return capi.SQLITE_IOERR; } }, - /*xSectorSize: function(pFile){ + xSectorSize: function(pFile){ return SECTOR_SIZE; - },*/ + }, xSync: function(pFile,flags){ log(`xSync ${flags}`); SAHPool.storeErr(); @@ -487,7 +486,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ return 0; }catch(e){ SAHPool.storeErr(e); - error("xSync() failed:",e.message); return capi.SQLITE_IOERR; } }, @@ -501,7 +499,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ return 0; }catch(e){ SAHPool.storeErr(e); - error("xTruncate() failed:",e.message); return capi.SQLITE_IOERR; } }, @@ -523,7 +520,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ return nBytes === n ? 0 : capi.SQLITE_IOERR; }catch(e){ SAHPool.storeErr(e); - error("xWrite() failed:",e.message); return capi.SQLITE_IOERR; } } @@ -563,7 +559,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ return 0; }catch(e){ SAHPool.storeErr(e); - error("Error xDelete()ing file:",e.message); return capi.SQLITE_IOERR_DELETE; } }, diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index 89ab2149ef..17169bc553 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -107,10 +107,8 @@ const S = globalThis.S = sqlite3; log("Loaded speedtest1 module. Setting up..."); if(S.installOpfsSAHPoolVfs){ - await S.installOpfsSAHPoolVfs().then(()=>{ - log("Loaded SAHPool."); - }).catch(e=>{ - logErr("Error setting up SAHPool:",e.message); + await S.installOpfsSAHPoolVfs().catch(e=>{ + logErr("Error setting up opfs-sahpool:",e.message); }); } App.vfsUnlink = function(pDb, fname){ diff --git a/manifest b/manifest index c5fd720553..524f846225 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\sopfs-sahpool\sVFS\safter\sthe\spebkac\sinvolving\sthe\sprevious\sspeedtest1\sruns.\sMake\sthat\sVFS\sexplicitly\sopt-in\sto\savoid\scertain\sunfortunate\slocking\ssituations. -D 2023-07-15T19:08:58.138 +C Minor\scleanups\sin\sthe\sopfs-sahpool\sVFS. +D 2023-07-15T21:08:48.986 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 83388ead4bfc489bee008298ab51948ccb75227795ce8d1634f2eec8e02548f1 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 0032097c168c8fe7e753abc5b35e65323116d04b0dbaaa97176604660b7bb98c F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js a5c3195203e6085d7aa89fae4b84cf3f3eec4ff4f928c6d0e5d3ef8b14cbc1c0 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html bbcf1e7fd79541040c1a7ca2ebf1cb7793ddaf9900d6bde1784148f11b807c34 -F ext/wasm/speedtest1-worker.js 4de92e4e6718b8bd1cdecb75af62739d1115fa66656a700b0b51822c848948f5 +F ext/wasm/speedtest1-worker.js 554b0985f791758e40ff2b1a04b771e315ab84b4e26b4b8a1c7a5ba968086c45 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 676ffe6280c1ce787b04d0cdb4a0664229c6125c601af4b18d1bfa125aac3675 -R bca4913f68935c8abed9e461aac753fd +P 41bf1fe31f2f3d0daa2bac25dc57262a4b90f22fed6fa97e4e92467c32ae02dc +R a6cacf00d5cb9e3eccf543ab6eefecd8 U stephan -Z 2546a1c8fd9c0ca0c4fd392086704b47 +Z d51af2a044894ea09e404a3df277218e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9ae594ffea..3be8e1f2dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41bf1fe31f2f3d0daa2bac25dc57262a4b90f22fed6fa97e4e92467c32ae02dc \ No newline at end of file +279e09070918dab7b60c39179ebb7eb931ca6bd4e589b414f436740499a2f910 \ No newline at end of file From d0ae50411f36b52e358343e7fd1095ed663513b7 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 10:02:41 +0000 Subject: [PATCH 10/56] Redefine what the opfs-sahpool installation promise resolves to. Fix addCapacity(). Add utility methods to import/export files. FossilOrigin-Name: 809c6f4de3653ad7a7751af45a7a0d6cb20c3ee3be80c69833c729242227d970 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 135 +++++++++++++++-------- ext/wasm/speedtest1-worker.js | 5 +- manifest | 14 +-- manifest.uuid | 2 +- 4 files changed, 99 insertions(+), 57 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 15fc687877..0551523101 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -56,10 +56,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = sqlite3.util.toss; let vfsRegisterResult = undefined; /** - installOpfsSAHPoolVfs() asynchronously initializes the - OPFS SyncAccessHandle Pool VFS. It returns a Promise - which either resolves to the sqlite3 object or rejects - with an Error value. + installOpfsSAHPoolVfs() asynchronously initializes the OPFS + SyncAccessHandle Pool VFS. It returns a Promise which either + resolves to a utility object described below or rejects with an + Error value. Initialization of this VFS is not automatic because its registration requires that it lock all resources it @@ -75,6 +75,12 @@ let vfsRegisterResult = undefined; resolved or rejected Promise. If called while the first call is still pending resolution, a rejected promise with a descriptive error is returned. + + On success, the resulting Promise resolves to a utility object + which can be used to query and manipulate the pool. Its API is... + + TODO + */ sqlite3.installOpfsSAHPoolVfs = async function(){ if(sqlite3===vfsRegisterResult) return Promise.resolve(sqlite3); @@ -112,8 +118,11 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ vfsRegisterResult = err; return Promise.reject(err); }; + /** The PoolUtil object will be the result of the + resolved Promise. */ + const PoolUtil = Object.create(null); const promiseResolve = - ()=>Promise.resolve(vfsRegisterResult = sqlite3); + ()=>Promise.resolve(vfsRegisterResult = PoolUtil); // Config opts for the VFS... const SECTOR_SIZE = 4096; const HEADER_MAX_PATH_SIZE = 512; @@ -184,15 +193,14 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ to the new capacity. */ addCapacity: async function(n){ - const cap = this.getCapacity(); - for(let i = cap; i < cap+n; ++i){ + for(let i = 0; i < n; ++i){ const name = getRandomName(); const h = await this.dirHandle.getFileHandle(name, {create:true}); const ah = await h.createSyncAccessHandle(); this.mapSAHToName.set(ah,name); this.setAssociatedPath(ah, '', 0); } - return i; + return this.getCapacity(); }, /** Removes n entries from the pool's current capacity @@ -399,14 +407,19 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ const rc = this.$error; this.$error = undefined; return rc; + }, + nextAvailableSAH: function(){ + const [rc] = this.availableSAH.keys(); + return rc; } + })/*SAHPool*/; - sqlite3.SAHPool = SAHPool/*only for testing*/; + //sqlite3.SAHPool = SAHPool/*only for testing*/; /** Impls for the sqlite3_io_methods methods. Maintenance reminder: members are in alphabetical order to simplify finding them. */ - const ioSyncWrappers = { + const ioMethods = { xCheckReservedLock: function(pFile,pOut){ log('xCheckReservedLock'); SAHPool.storeErr(); @@ -523,13 +536,13 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ return capi.SQLITE_IOERR; } } - }/*ioSyncWrappers*/; + }/*ioMethods*/; /** Impls for the sqlite3_vfs methods. Maintenance reminder: members are in alphabetical order to simplify finding them. */ - const vfsSyncWrappers = { + const vfsMethods = { xAccess: function(pVfs,zName,flags,pOut){ log(`xAccess ${wasm.cstrToJs(zName)}`); SAHPool.storeErr(); @@ -597,7 +610,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ // File not found so try to create it. if(SAHPool.getFileCount() < SAHPool.getCapacity()) { // Choose an unassociated OPFS file from the pool. - [sah] = SAHPool.availableSAH.keys(); + sah = SAHPool.nextAvailableSAH(); SAHPool.setAssociatedPath(sah, path, flags); }else{ // File pool is full. @@ -621,7 +634,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ return capi.SQLITE_CANTOPEN; } }/*xOpen()*/ - }/*vfsSyncWrappers*/; + }/*vfsMethods*/; if(dVfs){ /* Inherit certain VFS members from the default VFS, @@ -631,7 +644,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ } if(!opfsVfs.$xRandomness){ /* If the default VFS has no xRandomness(), add a basic JS impl... */ - vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + vfsMethods.xRandomness = function(pVfs, nOut, pOut){ const heap = wasm.heap8u(); let i = 0; for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; @@ -639,7 +652,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ }; } if(!opfsVfs.$xSleep){ - vfsSyncWrappers.xSleep = (pVfs,ms)=>0; + vfsMethods.xSleep = (pVfs,ms)=>0; } /** @@ -669,6 +682,58 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ if(true!==apiVersionCheck){ return promiseReject(apiVersionCheck); } + + PoolUtil.$SAHPool = SAHPool/* ONLY for testing and debugging */; + PoolUtil.addCapacity = async (n)=>SAHPool.addCapacity(n); + PoolUtil.reduceCapacity = async (n)=>SAHPool.reduceCapacity(n); + PoolUtil.getCapacity = SAHPool.getCapacity.bind(SAHPool); + PoolUtil.getActiveFileCount = SAHPool.getFileCount.bind(SAHPool); + /** + Synchronously reads the contents of the given file into a + Uint8Array and returns it. This will throw if the given name is + not currently in active use or on I/O error. + */ + PoolUtil.exportFile = function(name){ + const sah = SAHPool.mapPathToSAH.get(name) || toss("File not found:",name); + const n = sah.getSize() - HEADER_OFFSET_DATA; + const b = new Uint8Array(n>=0 ? n : 0); + if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); + return b; + }; + + /** + The counterpart of exportFile(), this imports the contents of an + SQLite database, provided as a byte array, under the given name, + overwriting any existing content. Throws if the pool has no + available file slots, on I/O error, or if the input does not + appear to be a database. In the latter case, only a cursory + examination is made. + + Note that this routine is _only_ for importing database files, + not arbitrary files, the reason being that this VFS will + automatically clean up any non-database files so importing them + is pointless. + + Returns undefined. + */ + PoolUtil.importDb = function(name, bytes){ + const n = bytes.byteLength; + if(n<512 || n%512!=0){ + toss("Byte array size is invalid for an SQLite db."); + } + const header = "SQLite format 3"; + for(let i = 0; i < header.length; ++i){ + if( header.charCodeAt(i) !== bytes[i] ){ + toss("Input does not contain an SQLite database header."); + } + } + const sah = SAHPool.mapPathToSAH.get(name) + || SAHPool.nextAvailableSAH() + || toss("No available handles to import to."); + sah.write(bytes, {at: HEADER_OFFSET_DATA}); + SAHPool.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); + }; + return SAHPool.isReady = SAHPool.reset().then(async ()=>{ if(SAHPool.$error){ throw SAHPool.$error; @@ -678,12 +743,11 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ } //log("vfs list:",capi.sqlite3_js_vfs_list()); sqlite3.vfs.installVfs({ - io: {struct: opfsIoMethods, methods: ioSyncWrappers}, - vfs: {struct: opfsVfs, methods: vfsSyncWrappers}, - applyArgcCheck: true + io: {struct: opfsIoMethods, methods: ioMethods}, + vfs: {struct: opfsVfs, methods: vfsMethods} }); - log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods); - log("vfs list:",capi.sqlite3_js_vfs_list()); + //log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods); + //log("vfs list:",capi.sqlite3_js_vfs_list()); if(sqlite3.oo1){ const OpfsSAHPoolDb = function(...args){ const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); @@ -691,39 +755,14 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; OpfsSAHPoolDb.prototype = Object.create(sqlite3.oo1.DB.prototype); - OpfsSAHPoolDb.addPoolCapacity = async (n)=>SAHPool.addCapacity(n); - OpfsSAHPoolDb.reducePoolCapacity = async (n)=>SAHPool.reduceCapacity(n); - OpfsSAHPoolDb.getPoolCapacity = ()=>SAHPool.getCapacity(); - OpfsSAHPoolDb.getPoolUsage = ()=>SAHPool.getFileCount(); + OpfsSAHPoolDb.PoolUtil; sqlite3.oo1.OpfsSAHPoolDb = OpfsSAHPoolDb; sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, function(oo1Db, sqlite3){ sqlite3.capi.sqlite3_exec(oo1Db, [ - /* As of July 2023, the PERSIST journal mode on OPFS is - somewhat slower than DELETE or TRUNCATE (it was faster - before Chrome version 108 or 109). TRUNCATE and DELETE - have very similar performance on OPFS. - - Roy Hashimoto notes that TRUNCATE and PERSIST modes may - decrease OPFS concurrency because multiple connections - can open the journal file in those modes: - - https://github.com/rhashimoto/wa-sqlite/issues/68 - - Given that, and the fact that testing has not revealed - any appreciable difference between performance of - TRUNCATE and DELETE modes on OPFS, we currently (as of - 2023-07-13) default to DELETE mode. - */ + /* See notes in sqlite3-vfs-opfs.js */ "pragma journal_mode=DELETE;", - /* - OPFS benefits hugely from cache on moderate/large - speedtest1 --size 50 and --size 100 workloads. We - currently rely on setting a non-default cache size when - building sqlite3.wasm. If that policy changes, the cache - can be set here. - */ "pragma cache_size=-16384;" ], 0, 0, 0); } diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index 17169bc553..6cd0c1af00 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -107,7 +107,10 @@ const S = globalThis.S = sqlite3; log("Loaded speedtest1 module. Setting up..."); if(S.installOpfsSAHPoolVfs){ - await S.installOpfsSAHPoolVfs().catch(e=>{ + await S.installOpfsSAHPoolVfs().then(P=>{ + S.SAHPoolUtil = P; + //return P.addCapacity(5).then(log("pool capacity:",P.getCapacity()));; + }).catch(e=>{ logErr("Error setting up opfs-sahpool:",e.message); }); } diff --git a/manifest b/manifest index 524f846225..78b72b420f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scleanups\sin\sthe\sopfs-sahpool\sVFS. -D 2023-07-15T21:08:48.986 +C Redefine\swhat\sthe\sopfs-sahpool\sinstallation\spromise\sresolves\sto.\sFix\saddCapacity().\sAdd\sutility\smethods\sto\simport/export\sfiles. +D 2023-07-16T10:02:41.870 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 0032097c168c8fe7e753abc5b35e65323116d04b0dbaaa97176604660b7bb98c +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 0e982cc4f1b9ed4786086d1115e740b7efd628de5cbaf16caf3a71913f91241b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js a5c3195203e6085d7aa89fae4b84cf3f3eec4ff4f928c6d0e5d3ef8b14cbc1c0 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html bbcf1e7fd79541040c1a7ca2ebf1cb7793ddaf9900d6bde1784148f11b807c34 -F ext/wasm/speedtest1-worker.js 554b0985f791758e40ff2b1a04b771e315ab84b4e26b4b8a1c7a5ba968086c45 +F ext/wasm/speedtest1-worker.js faa4a06ec21921aaa0e0b672a94b56037da837e16732bdd6545b99f1cadbb32e F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 41bf1fe31f2f3d0daa2bac25dc57262a4b90f22fed6fa97e4e92467c32ae02dc -R a6cacf00d5cb9e3eccf543ab6eefecd8 +P 279e09070918dab7b60c39179ebb7eb931ca6bd4e589b414f436740499a2f910 +R 3700fe28ac7e053b813d4612e5083eb3 U stephan -Z d51af2a044894ea09e404a3df277218e +Z 2285d7e58406d05c66733e55ed60721b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3be8e1f2dd..3cd58eacd0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -279e09070918dab7b60c39179ebb7eb931ca6bd4e589b414f436740499a2f910 \ No newline at end of file +809c6f4de3653ad7a7751af45a7a0d6cb20c3ee3be80c69833c729242227d970 \ No newline at end of file From 8449dacbd4204dd3144919cd83ae872ca77f2b46 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 11:49:18 +0000 Subject: [PATCH 11/56] Micro-optimizations in opfs-sahpool. FossilOrigin-Name: 52f23db948ae9694df69c00177b85cb569e9b211350a4a2dbf249e7cd8de700c --- ext/wasm/api/sqlite3-opfs-async-proxy.js | 3 +++ ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 31 ++++++++++++------------ manifest | 14 +++++------ manifest.uuid | 2 +- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 179a816358..cafd296c61 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -35,6 +35,9 @@ https://developer.chrome.com/blog/sync-methods-for-accesshandles/ + Firefox v111 and Safari 16.4, both released in March 2023, also + include this. + We cannot change to the sync forms at this point without breaking clients who use Chrome v104-ish or higher. truncate(), getSize(), flush(), and close() are now (as of v108) synchronous. Calling them diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 0551523101..163544f915 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -275,32 +275,33 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ returns an empty string. */ getAssociatedPath: function(sah){ - const body = this.apBody; - sah.read(body, {at: 0}); + sah.read(this.apBody, {at: 0}); // Delete any unexpected files left over by previous // untimely errors... - const dv = new DataView(body.buffer, body.byteOffset); - const flags = dv.getUint32(HEADER_OFFSET_FLAGS); - if(body[0] && + const flags = this.dvBody.getUint32(HEADER_OFFSET_FLAGS); + if(this.apBody[0] && ((flags & capi.SQLITE_OPEN_DELETEONCLOSE) || (flags & PERSISTENT_FILE_TYPES)===0)){ - warn(`Removing file with unexpected flags ${flags.toString(16)}`); + warn(`Removing file with unexpected flags ${flags.toString(16)}`, + this.apBody); this.setAssociatedPath(sah, '', 0); return ''; } const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); - const compDigest = this.computeDigest(body); + const compDigest = this.computeDigest(this.apBody); if(fileDigest.every((v,i) => v===compDigest[i])){ // Valid digest - const pathBytes = body.findIndex((v)=>0===v); + const pathBytes = this.apBody.findIndex((v)=>0===v); if(0===pathBytes){ // This file is unassociated, so truncate it to avoid // leaving stale db data laying around. sah.truncate(HEADER_OFFSET_DATA); } - return this.textDecoder.decode(body.subarray(0,pathBytes)); + return pathBytes + ? this.textDecoder.decode(this.apBody.subarray(0,pathBytes)) + : ''; }else{ // Invalid digest warn('Disassociating file with bad digest.'); @@ -313,17 +314,15 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ flags into the given SAH. */ setAssociatedPath: function(sah, path, flags){ - const body = this.apBody; - const enc = this.textEncoder.encodeInto(path, body); + const enc = this.textEncoder.encodeInto(path || '\0', this.apBody); if(HEADER_MAX_PATH_SIZE <= enc.written){ toss("Path too long:",path); } - const dv = new DataView(body.buffer, body.byteOffset); - dv.setUint32(HEADER_OFFSET_FLAGS, flags); + this.dvBody.setUint32(HEADER_OFFSET_FLAGS, flags); - const digest = this.computeDigest(body); - sah.write(body, {at: 0}); + const digest = this.computeDigest(this.apBody); + sah.write(this.apBody, {at: 0}); sah.write(digest, {at: HEADER_OFFSET_DIGEST}); sah.flush(); @@ -414,6 +413,8 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ } })/*SAHPool*/; + SAHPool.dvBody = + new DataView(SAHPool.apBody.buffer, SAHPool.apBody.byteOffset); //sqlite3.SAHPool = SAHPool/*only for testing*/; /** Impls for the sqlite3_io_methods methods. Maintenance reminder: diff --git a/manifest b/manifest index 78b72b420f..a17a7271a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Redefine\swhat\sthe\sopfs-sahpool\sinstallation\spromise\sresolves\sto.\sFix\saddCapacity().\sAdd\sutility\smethods\sto\simport/export\sfiles. -D 2023-07-16T10:02:41.870 +C Micro-optimizations\sin\sopfs-sahpool. +D 2023-07-16T11:49:18.318 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -500,9 +500,9 @@ F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f2 F ext/wasm/api/sqlite3-api-prologue.js 5dcb5d2d74269545073eec197614b86bd28950132b5fe4de67c10a8a0d5524b2 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 +F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 0e982cc4f1b9ed4786086d1115e740b7efd628de5cbaf16caf3a71913f91241b +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js ba799928d945e35c97696005c4911c32f38ba4290168a1fcf0109aea6a2ba19b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js a5c3195203e6085d7aa89fae4b84cf3f3eec4ff4f928c6d0e5d3ef8b14cbc1c0 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 279e09070918dab7b60c39179ebb7eb931ca6bd4e589b414f436740499a2f910 -R 3700fe28ac7e053b813d4612e5083eb3 +P 809c6f4de3653ad7a7751af45a7a0d6cb20c3ee3be80c69833c729242227d970 +R 025c645f0a348eef018e7d822fdbac9d U stephan -Z 2285d7e58406d05c66733e55ed60721b +Z a785870b274d18ca6df0df66a08dd418 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3cd58eacd0..771b243369 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -809c6f4de3653ad7a7751af45a7a0d6cb20c3ee3be80c69833c729242227d970 \ No newline at end of file +52f23db948ae9694df69c00177b85cb569e9b211350a4a2dbf249e7cd8de700c \ No newline at end of file From aa1b2a80eef92233a4c2d096ab0a09588c106ea2 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 12:02:37 +0000 Subject: [PATCH 12/56] During "opfs" VFS init, check for URL param opfs-disable and, if set, do not install the VFS. Added per forum suggestion to provide a way to help rule out misinteraction between the "opfs" and "opfs-sahpool" VFSes. FossilOrigin-Name: 29905b7a75b73e32125bf9116033cae7235a135b668a3b783a3d8dcb0bc80374 --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 12 ++++++++---- ext/wasm/speedtest1-worker.html | 3 ++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index c7a752441f..35b7b88650 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -101,6 +101,10 @@ const installOpfsVfs = function callee(options){ options = Object.create(null); } const urlParams = new URL(globalThis.location.href).searchParams; + if(urlParams.has('opfs-disable')){ + //sqlite3.config.warn('Explicitly not installing "opfs" VFS due to opfs-disable flag.'); + return Promise.resolve(sqlite3); + } if(undefined===options.verbose){ options.verbose = urlParams.has('opfs-verbose') ? (+urlParams.get('opfs-verbose') || 2) : 1; @@ -200,9 +204,9 @@ const installOpfsVfs = function callee(options){ opfsVfs.dispose(); return promiseReject_(err); }; - const promiseResolve = (value)=>{ + const promiseResolve = ()=>{ promiseWasRejected = false; - return promiseResolve_(value); + return promiseResolve_(sqlite3); }; const W = //#if target=es6-bundler-friendly @@ -1322,10 +1326,10 @@ const installOpfsVfs = function callee(options){ sqlite3.opfs = opfsUtil; opfsUtil.rootDirectory = d; log("End of OPFS sqlite3_vfs setup.", opfsVfs); - promiseResolve(sqlite3); + promiseResolve(); }).catch(promiseReject); }else{ - promiseResolve(sqlite3); + promiseResolve(); } }catch(e){ error(e); diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index b2acf159b6..3adf8f2ee9 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -171,7 +171,8 @@ const urlParams = new URL(self.location.href).searchParams; const W = new Worker( "speedtest1-worker.js?sqlite3.dir=jswasm"+ - (urlParams.has('opfs-verbose') ? '&opfs-verbose' : '') + (urlParams.has('opfs-verbose') ? '&opfs-verbose' : '')+ + (urlParams.has('opfs-disable') ? '&opfs-disable' : '') ); const mPost = function(msgType,payload){ W.postMessage({type: msgType, data: payload}); diff --git a/manifest b/manifest index a17a7271a1..3cf5646d7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Micro-optimizations\sin\sopfs-sahpool. -D 2023-07-16T11:49:18.318 +C During\s"opfs"\sVFS\sinit,\scheck\sfor\sURL\sparam\sopfs-disable\sand,\sif\sset,\sdo\snot\sinstall\sthe\sVFS.\sAdded\sper\sforum\ssuggestion\sto\sprovide\sa\sway\sto\shelp\srule\sout\smisinteraction\sbetween\sthe\s"opfs"\sand\s"opfs-sahpool"\sVFSes. +D 2023-07-16T12:02:37.445 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,7 +503,7 @@ F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d299 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js ba799928d945e35c97696005c4911c32f38ba4290168a1fcf0109aea6a2ba19b -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js a5c3195203e6085d7aa89fae4b84cf3f3eec4ff4f928c6d0e5d3ef8b14cbc1c0 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 @@ -539,7 +539,7 @@ F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe -F ext/wasm/speedtest1-worker.html bbcf1e7fd79541040c1a7ca2ebf1cb7793ddaf9900d6bde1784148f11b807c34 +F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 F ext/wasm/speedtest1-worker.js faa4a06ec21921aaa0e0b672a94b56037da837e16732bdd6545b99f1cadbb32e F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 809c6f4de3653ad7a7751af45a7a0d6cb20c3ee3be80c69833c729242227d970 -R 025c645f0a348eef018e7d822fdbac9d +P 52f23db948ae9694df69c00177b85cb569e9b211350a4a2dbf249e7cd8de700c +R e0c8363b212a784e3aba23833aa919e5 U stephan -Z a785870b274d18ca6df0df66a08dd418 +Z 6a5be95b2a13778a907aeca05fd3977d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 771b243369..24fb7ae1c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52f23db948ae9694df69c00177b85cb569e9b211350a4a2dbf249e7cd8de700c \ No newline at end of file +29905b7a75b73e32125bf9116033cae7235a135b668a3b783a3d8dcb0bc80374 \ No newline at end of file From d62c4645417ff33f7efc588a4c8005273a12fb55 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 14:07:59 +0000 Subject: [PATCH 13/56] speedtest1.js: only install opfs-sahpool if it's provided via --vfs flag, to avoid locking errors in concurrent speedtest1 tabs with other VFSes. Add opfs-sahpool reserveMinimumCapacity(). FossilOrigin-Name: aa94c8abfbdfc4c7b36554c4b3ea90a5065e7e3f4294c64c8cbf688b4688300d --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 6 +++++ ext/wasm/speedtest1-worker.js | 28 +++++++++++------------- manifest | 14 ++++++------ manifest.uuid | 2 +- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 163544f915..d40581aba3 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -689,6 +689,12 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ PoolUtil.reduceCapacity = async (n)=>SAHPool.reduceCapacity(n); PoolUtil.getCapacity = SAHPool.getCapacity.bind(SAHPool); PoolUtil.getActiveFileCount = SAHPool.getFileCount.bind(SAHPool); + /** If capacity is < min, increase capacity to min, else do + nothing. Resolves to the new capacity. */ + PoolUtil.reserveMinimumCapacity = async (min)=>{ + const c = SAHPool.getCapacity(); + return (c < min) ? SAHPool.addCapacity(min - c) : c; + }; /** Synchronously reads the contents of the given file into a Uint8Array and returns it. This will throw if the given name is diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index 6cd0c1af00..c2bf37b831 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -48,7 +48,7 @@ const log = (...args)=>logMsg('stdout',args); const logErr = (...args)=>logMsg('stderr',args); - const runSpeedtest = function(cliFlagsArray){ + const runSpeedtest = async function(cliFlagsArray){ const scope = App.wasm.scopedAllocPush(); const dbFile = App.pDir+"/speedtest1.sqlite3"; try{ @@ -57,6 +57,15 @@ ]; App.logBuffer.length = 0; mPost('run-start', [...argv]); + if(App.sqlite3.installOpfsSAHPoolVfs + && !App.sqlite3.$SAHPoolUtil + && cliFlagsArray.indexOf('opfs-sahpool')>=0){ + log("Installing opfs-sahpool..."); + await App.sqlite3.installOpfsSAHPoolVfs().then(PoolUtil=>{ + log("opfs-sahpool successfully installed."); + App.sqlite3.$SAHPoolUtil = PoolUtil; + }); + } App.wasm.xCall('wasm_main', argv.length, App.wasm.scopedAllocMainArgv(argv)); }catch(e){ @@ -72,11 +81,8 @@ msg = msg.data; switch(msg.type){ case 'run': - try { - runSpeedtest(msg.data || []); - }catch(e){ - mPost('error',e); - } + runSpeedtest(msg.data || []) + .catch(e=>mPost('error',e)); break; default: logErr("Unhandled worker message type:",msg.type); @@ -104,16 +110,8 @@ }; log("Initializing speedtest1 module..."); self.sqlite3InitModule(EmscriptenModule).then(async (sqlite3)=>{ - const S = globalThis.S = sqlite3; + const S = globalThis.S = App.sqlite3 = sqlite3; log("Loaded speedtest1 module. Setting up..."); - if(S.installOpfsSAHPoolVfs){ - await S.installOpfsSAHPoolVfs().then(P=>{ - S.SAHPoolUtil = P; - //return P.addCapacity(5).then(log("pool capacity:",P.getCapacity()));; - }).catch(e=>{ - logErr("Error setting up opfs-sahpool:",e.message); - }); - } App.vfsUnlink = function(pDb, fname){ const pVfs = S.wasm.sqlite3_wasm_db_vfs(pDb, 0); if(pVfs) S.wasm.sqlite3_wasm_vfs_unlink(pVfs, fname||0); diff --git a/manifest b/manifest index 3cf5646d7c..d2e88ef2ac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C During\s"opfs"\sVFS\sinit,\scheck\sfor\sURL\sparam\sopfs-disable\sand,\sif\sset,\sdo\snot\sinstall\sthe\sVFS.\sAdded\sper\sforum\ssuggestion\sto\sprovide\sa\sway\sto\shelp\srule\sout\smisinteraction\sbetween\sthe\s"opfs"\sand\s"opfs-sahpool"\sVFSes. -D 2023-07-16T12:02:37.445 +C speedtest1.js:\sonly\sinstall\sopfs-sahpool\sif\sit's\sprovided\svia\s--vfs\sflag,\sto\savoid\slocking\serrors\sin\sconcurrent\sspeedtest1\stabs\swith\sother\sVFSes.\sAdd\sopfs-sahpool\sreserveMinimumCapacity(). +D 2023-07-16T14:07:59.930 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js ba799928d945e35c97696005c4911c32f38ba4290168a1fcf0109aea6a2ba19b +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 5ffed44d7bac1b4038e1505ffc7ab63e82726a97a64193ddbd5b414722f0808b F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 -F ext/wasm/speedtest1-worker.js faa4a06ec21921aaa0e0b672a94b56037da837e16732bdd6545b99f1cadbb32e +F ext/wasm/speedtest1-worker.js cda2f6cf0a6b864d82e51b9e4dfd1dfb0c4024987c5d94a81cc587e07acc9be4 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 52f23db948ae9694df69c00177b85cb569e9b211350a4a2dbf249e7cd8de700c -R e0c8363b212a784e3aba23833aa919e5 +P 29905b7a75b73e32125bf9116033cae7235a135b668a3b783a3d8dcb0bc80374 +R 04e7987eb127f55eddff193be36455e6 U stephan -Z 6a5be95b2a13778a907aeca05fd3977d +Z 53c8cb4a4900e0bba3854b3011d70f79 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 24fb7ae1c3..c9034b92fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29905b7a75b73e32125bf9116033cae7235a135b668a3b783a3d8dcb0bc80374 \ No newline at end of file +aa94c8abfbdfc4c7b36554c4b3ea90a5065e7e3f4294c64c8cbf688b4688300d \ No newline at end of file From da6a42a92143723e0c6c4554e7fec19c9a15cb03 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 16:52:09 +0000 Subject: [PATCH 14/56] Move SAH pool configuration options from the library-level config to a config passed to the VFS install routine. Extend and document the PoolUtil object. FossilOrigin-Name: d2ed99556fa1f40994c1c6bd90d1d5733bebc824b1ebfabe978fae9e18948437 --- ext/wasm/api/sqlite3-api-prologue.js | 18 +- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 207 ++++++++++++++++++----- ext/wasm/speedtest1-worker.js | 9 +- manifest | 16 +- manifest.uuid | 2 +- 5 files changed, 184 insertions(+), 68 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index fb085e299c..ac3253670f 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -91,21 +91,6 @@ - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the OPFS-backed filesystem in WASMFS-capable builds. - - `opfs-sahpool.dir`[^1]: Specifies the OPFS directory name in - which to store metadata for the `"opfs-sahpool"` sqlite3_vfs. - Changing this name will effectively orphan any databases stored - under previous names. The default is unspecified but descriptive. - This option may contain multiple path elements, - e.g. "foo/bar/baz", and they are created automatically. In - practice there should be no driving need to change this. - - - `opfs-sahpool.defaultCapacity`[^1]: Specifies the default - capacity of the `"opfs-sahpool"` VFS. This should not be set - unduly high because the VFS has to open (and keep open) a file - for each entry in the pool. This setting only has an effect when - the pool is initially empty. It does not have any effect if a - pool already exists. - [^1] = This property may optionally be a function, in which case this function calls that function to fetch the value, @@ -158,8 +143,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( [ // If any of these config options are functions, replace them with // the result of calling that function... - 'exports', 'memory', 'wasmfsOpfsDir', - 'opfs-sahpool.dir', 'opfs-sahpool.defaultCapacity' + 'exports', 'memory', 'wasmfsOpfsDir' ].forEach((k)=>{ if('function' === typeof config[k]){ config[k] = config[k](); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index d40581aba3..a19589aac9 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -55,11 +55,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = sqlite3.util.toss; let vfsRegisterResult = undefined; +/** The PoolUtil object will be the result of the + resolved Promise. */ +const PoolUtil = Object.create(null); +let isPromiseReady; + /** installOpfsSAHPoolVfs() asynchronously initializes the OPFS - SyncAccessHandle Pool VFS. It returns a Promise which either - resolves to a utility object described below or rejects with an - Error value. + SyncAccessHandle (a.k.a. SAH) Pool VFS. It returns a Promise which + either resolves to a utility object described below or rejects with + an Error value. Initialization of this VFS is not automatic because its registration requires that it lock all resources it @@ -72,18 +77,113 @@ let vfsRegisterResult = undefined; due to OPFS locking errors. On calls after the first this function immediately returns a - resolved or rejected Promise. If called while the first call is - still pending resolution, a rejected promise with a descriptive - error is returned. + pending, resolved, or rejected Promise, depending on the state + of the first call's Promise. On success, the resulting Promise resolves to a utility object - which can be used to query and manipulate the pool. Its API is... + which can be used to query and manipulate the pool. Its API is + described at the end of these docs. - TODO + This function accepts an options object to configure certain + parts but it is only acknowledged for the very first call and + ignored for all subsequent calls. + + The options, in alphabetical order: + + - `clearOnInit`: if truthy, as each SAH is acquired during + initalization of the VFS, its contents and filename name mapping + are removed, leaving the VFS's storage in a pristine state. + + - `defaultCapacity`: Specifies the default capacity of the + VFS. This should not be set unduly high because the VFS has to + open (and keep open) a file for each entry in the pool. This + setting only has an effect when the pool is initially empty. It + does not have any effect if a pool already exists. + + - `directory`: Specifies the OPFS directory name in which to store + metadata for the `"opfs-sahpool"` sqlite3_vfs. Only 1 instance + of this VFS can be installed per JavaScript engine, and any two + engines with the same storage directory name will collide with + each other, leading to locking errors and the inability to + register the VFS in the second and subsequent engine. Using a + different directory name for each application enables different + engines in the same HTTP origin to co-exist, but their data are + invisible to each other. Changing this name will effectively + orphan any databases stored under previous names. The default is + unspecified but descriptive. This option may contain multiple + path elements, e.g. "foo/bar/baz", and they are created + automatically. In practice there should be no driving need to + change this. + + + API for the utility object passed on by this function's Promise, in + alphabetical order... + +- [async] addCapacity(n) + + Adds `n` entries to the current pool. This change is persistent + across sessions so should not be called automatically at each app + startup (but see `reserveMinimumCapacity()`). Its returned Promise + resolves to the new capacity. Because this operation is necessarily + asynchronous, the C-level VFS API cannot call this on its own as + needed. + +- byteArray exportFile(name) + + Synchronously reads the contents of the given file into a Uint8Array + and returns it. This will throw if the given name is not currently + in active use or on I/O error. + +- number getCapacity() + + Returns the number of files currently contained + in the SAH pool. The default capacity is only large enough for one + or two databases and their associated temp files. + +- number getActiveFileCount() + + Returns the number of files from the pool currently in use. + +- importDb(name, byteArray) + + Imports the contents of an SQLite database, provided as a byte + array, under the given name, overwriting any existing + content. Throws if the pool has no available file slots, on I/O + error, or if the input does not appear to be a database. In the + latter case, only a cursory examination is made. Note that this + routine is _only_ for importing database files, not arbitrary files, + the reason being that this VFS will automatically clean up any + non-database files so importing them is pointless. + +- [async] number reduceCapacity(n) + + Removes up to `n` entries from the pool, with the caveat that it can + only remove currently-unused entries. It returns a Promise which + resolves to the number of entries actually removed. + +- [async] number reserveMinimumCapacity(min) + + If the current capacity is less than `min`, the capacity is + increased to `min`, else this returns with no side effects. The + resulting Promise resolves to the new capacity. + +- boolean unlink(filename) + + If a virtual file exists with the given name, disassociates it from + the pool and returns true, else returns false without side + effects. Results are undefined if the file is currently in active + use. + +- [async] wipeFiles() + + Clears all client-defined state of all SAHs and makes all of them + available for re-use by the pool. Results are undefined if any such + handles are currently in use, e.g. by an sqlite3 db. */ -sqlite3.installOpfsSAHPoolVfs = async function(){ - if(sqlite3===vfsRegisterResult) return Promise.resolve(sqlite3); +sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ + if(PoolUtil===vfsRegisterResult) return Promise.resolve(PoolUtil); + else if(isPromiseReady) return isPromiseReady; else if(undefined!==vfsRegisterResult){ return Promise.reject(vfsRegisterResult); } @@ -94,7 +194,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ !navigator?.storage?.getDirectory){ return Promise.reject(vfsRegisterResult = new Error("Missing required OPFS APIs.")); } - vfsRegisterResult = new Error("VFS initialization still underway."); + vfsRegisterResult = new Error("opfs-sahpool initialization still underway."); const verbosity = 2 /*3+ == everything*/; const loggers = [ sqlite3.config.error, @@ -118,9 +218,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ vfsRegisterResult = err; return Promise.reject(err); }; - /** The PoolUtil object will be the result of the - resolved Promise. */ - const PoolUtil = Object.create(null); const promiseResolve = ()=>Promise.resolve(vfsRegisterResult = PoolUtil); // Config opts for the VFS... @@ -133,7 +230,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; const HEADER_OFFSET_DATA = SECTOR_SIZE; const DEFAULT_CAPACITY = - sqlite3.config['opfs-sahpool.defaultCapacity'] || 6; + options.defaultCapacity || 6; /* Bitmask of file types which may persist across sessions. SQLITE_OPEN_xyz types not listed here may be inadvertently left in OPFS but are treated as transient by this VFS and @@ -171,14 +268,13 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ */ const SAHPool = Object.assign(Object.create(null),{ /* OPFS dir in which VFS metadata is stored. */ - vfsDir: sqlite3.config['opfs-sahpool.dir'] - || ".sqlite3-opfs-sahpool", + vfsDir: options.directory || ".sqlite3-opfs-sahpool", /* Directory handle to this.vfsDir. */ dirHandle: undefined, /* Maps SAHs to their opaque file names. */ mapSAHToName: new Map(), /* Maps client-side file names to SAHs. */ - mapPathToSAH: new Map(), + mapFilenameToSAH: new Map(), /* Set of currently-unused SAHs. */ availableSAH: new Set(), /* Maps (sqlite3_file*) to xOpen's file objects. */ @@ -186,7 +282,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ /* Current pool capacity. */ getCapacity: function(){return this.mapSAHToName.size}, /* Current number of in-use files from pool. */ - getFileCount: function(){return this.mapPathToSAH.size}, + getFileCount: function(){return this.mapFilenameToSAH.size}, /** Adds n files to the pool's capacity. This change is persistent across settings. Returns a Promise which resolves @@ -229,7 +325,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ releaseAccessHandles: function(){ for(const ah of this.mapSAHToName.keys()) ah.close(); this.mapSAHToName.clear(); - this.mapPathToSAH.clear(); + this.mapFilenameToSAH.clear(); this.availableSAH.clear(); }, /** @@ -238,8 +334,13 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ but completes once all SAHs are acquired. If acquiring an SAH throws, SAHPool.$error will contain the corresponding exception. + + + If clearFiles is true, the client-stored state of each file is + cleared when its handle is acquired, including its name, flags, + and any data stored after the metadata block. */ - acquireAccessHandles: async function(){ + acquireAccessHandles: async function(clearFiles){ const files = []; for await (const [name,h] of this.dirHandle){ if('file'===h.kind){ @@ -250,11 +351,16 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ try{ const ah = await h.createSyncAccessHandle() this.mapSAHToName.set(ah, name); - const path = this.getAssociatedPath(ah); - if(path){ - this.mapPathToSAH.set(path, ah); + if(clearFiles){ + ah.truncate(HEADER_OFFSET_DATA); + this.setAssociatedPath(ah, '', 0); }else{ - this.availableSAH.add(ah); + const path = this.getAssociatedPath(ah); + if(path){ + this.mapFilenameToSAH.set(path, ah); + }else{ + this.availableSAH.add(ah); + } } }catch(e){ SAHPool.storeErr(e); @@ -327,12 +433,12 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ sah.flush(); if(path){ - this.mapPathToSAH.set(path, sah); + this.mapFilenameToSAH.set(path, sah); this.availableSAH.delete(sah); }else{ // This is not a persistent file, so eliminate the contents. sah.truncate(HEADER_OFFSET_DATA); - this.mapPathToSAH.delete(path); + this.mapFilenameToSAH.delete(path); this.availableSAH.add(sah); } }, @@ -352,9 +458,12 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ /** Re-initializes the state of the SAH pool, releasing and re-acquiring all handles. + + See acquireAccessHandles() for the specifics of the clearFiles + argument. */ - reset: async function(){ - await this.isReady; + reset: async function(clearFiles){ + await isPromiseReady; let h = await navigator.storage.getDirectory(); for(const d of this.vfsDir.split('/')){ if(d){ @@ -363,7 +472,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ } this.dirHandle = h; this.releaseAccessHandles(); - await this.acquireAccessHandles(); + await this.acquireAccessHandles(clearFiles); }, /** Returns the pathname part of the given argument, @@ -381,14 +490,17 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ }, /** Removes the association of the given client-specified file - name (JS string) from the pool. + name (JS string) from the pool. Returns true if a mapping + is found, else false. */ deletePath: function(path) { - const sah = this.mapPathToSAH.get(path); + const sah = this.mapFilenameToSAH.get(path); if(sah) { - // Un-associate the SQLite path from the OPFS file. + // Un-associate the name from the SAH. + this.mapFilenameToSAH.delete(path); this.setAssociatedPath(sah, '', 0); } + return !!sah; }, /** Sets e as this object's current error. Pass a falsy @@ -549,7 +661,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ SAHPool.storeErr(); try{ const name = this.getPath(zName); - wasm.poke32(pOut, SAHPool.mapPathToSAH.has(name) ? 1 : 0); + wasm.poke32(pOut, SAHPool.mapFilenameToSAH.has(name) ? 1 : 0); }catch(e){ /*ignored*/; } @@ -606,7 +718,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ const path = (zName && wasm.peek8(zName)) ? SAHPool.getPath(zName) : getRandomName(); - let sah = SAHPool.mapPathToSAH.get(path); + let sah = SAHPool.mapFilenameToSAH.get(path); if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { // File not found so try to create it. if(SAHPool.getFileCount() < SAHPool.getCapacity()) { @@ -701,7 +813,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ not currently in active use or on I/O error. */ PoolUtil.exportFile = function(name){ - const sah = SAHPool.mapPathToSAH.get(name) || toss("File not found:",name); + const sah = SAHPool.mapFilenameToSAH.get(name) || toss("File not found:",name); const n = sah.getSize() - HEADER_OFFSET_DATA; const b = new Uint8Array(n>=0 ? n : 0); if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); @@ -734,14 +846,33 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ toss("Input does not contain an SQLite database header."); } } - const sah = SAHPool.mapPathToSAH.get(name) + const sah = SAHPool.mapFilenameToSAH.get(name) || SAHPool.nextAvailableSAH() || toss("No available handles to import to."); sah.write(bytes, {at: HEADER_OFFSET_DATA}); SAHPool.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); }; + /** + Clears all client-defined state of all SAHs and makes all of them + available for re-use by the pool. Results are undefined if any + such handles are currently in use, e.g. by an sqlite3 db. + */ + PoolUtil.wipeFiles = async ()=>SAHPool.reset(true); - return SAHPool.isReady = SAHPool.reset().then(async ()=>{ + /** + If a virtual file exists with the given name, disassociates it + from the pool and returns true, else returns false without side + effects. + */ + PoolUtil.unlink = (filename)=>SAHPool.deletePath(filename); + + /** + PoolUtil TODOs: + + - function to wipe out all traces of the VFS from storage. + */ + + return isPromiseReady = SAHPool.reset(!!options.clearOnInit).then(async ()=>{ if(SAHPool.$error){ throw SAHPool.$error; } diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index c2bf37b831..61af26b23e 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -61,7 +61,11 @@ && !App.sqlite3.$SAHPoolUtil && cliFlagsArray.indexOf('opfs-sahpool')>=0){ log("Installing opfs-sahpool..."); - await App.sqlite3.installOpfsSAHPoolVfs().then(PoolUtil=>{ + await App.sqlite3.installOpfsSAHPoolVfs({ + directory: '.speedtest1-sahpool', + defaultCapacity: 3, + clearOnInit: true + }).then(PoolUtil=>{ log("opfs-sahpool successfully installed."); App.sqlite3.$SAHPoolUtil = PoolUtil; }); @@ -122,9 +126,6 @@ //else log("Using transient storage."); mPost('ready',true); log("Registered VFSes:", ...S.capi.sqlite3_js_vfs_list()); - if(0 && S.installOpfsSAHPoolVfs){ - sahpSanityChecks(S); - } }).catch(e=>{ logErr(e); }); diff --git a/manifest b/manifest index d2e88ef2ac..1e2c3ff671 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C speedtest1.js:\sonly\sinstall\sopfs-sahpool\sif\sit's\sprovided\svia\s--vfs\sflag,\sto\savoid\slocking\serrors\sin\sconcurrent\sspeedtest1\stabs\swith\sother\sVFSes.\sAdd\sopfs-sahpool\sreserveMinimumCapacity(). -D 2023-07-16T14:07:59.930 +C Move\sSAH\spool\sconfiguration\soptions\sfrom\sthe\slibrary-level\sconfig\sto\sa\sconfig\spassed\sto\sthe\sVFS\sinstall\sroutine.\sExtend\sand\sdocument\sthe\sPoolUtil\sobject. +D 2023-07-16T16:52:09.106 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -497,12 +497,12 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 -F ext/wasm/api/sqlite3-api-prologue.js 5dcb5d2d74269545073eec197614b86bd28950132b5fe4de67c10a8a0d5524b2 +F ext/wasm/api/sqlite3-api-prologue.js f68e87edc049793c4ed46b0ec8f3a3d8013eeb3fd56481029dda916d4d5fa3a3 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 5ffed44d7bac1b4038e1505ffc7ab63e82726a97a64193ddbd5b414722f0808b +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js c19ccfc2995c0dcae00f13fe1be6fa436a39a3d629b6bf4208965ea78a50cab3 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 -F ext/wasm/speedtest1-worker.js cda2f6cf0a6b864d82e51b9e4dfd1dfb0c4024987c5d94a81cc587e07acc9be4 +F ext/wasm/speedtest1-worker.js 41fdc91878d3481b198bba771f073aad8837063ea2a23a0e9a278a54634f8ffe F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 29905b7a75b73e32125bf9116033cae7235a135b668a3b783a3d8dcb0bc80374 -R 04e7987eb127f55eddff193be36455e6 +P aa94c8abfbdfc4c7b36554c4b3ea90a5065e7e3f4294c64c8cbf688b4688300d +R 76883f55e00ff0c4af4f43f15f164d03 U stephan -Z 53c8cb4a4900e0bba3854b3011d70f79 +Z fd9b47fd6c1916432a0f6dc613a90b88 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c9034b92fd..9dc2139fb9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa94c8abfbdfc4c7b36554c4b3ea90a5065e7e3f4294c64c8cbf688b4688300d \ No newline at end of file +d2ed99556fa1f40994c1c6bd90d1d5733bebc824b1ebfabe978fae9e18948437 \ No newline at end of file From 24873723f4b21e9e47505880398842c926269002 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 17:51:43 +0000 Subject: [PATCH 15/56] Add a way to remove the opfs-sahpool's persistent state from OPFS or unlink() an individual file. Doc cleanups. FossilOrigin-Name: 80982daac3c098033dbc249bb7a17ef84ae218d2d789f8644e7f4af18b553d24 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 100 +++++++++++++---------- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 62 insertions(+), 52 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index a19589aac9..d7c5b198e0 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -101,7 +101,7 @@ let isPromiseReady; does not have any effect if a pool already exists. - `directory`: Specifies the OPFS directory name in which to store - metadata for the `"opfs-sahpool"` sqlite3_vfs. Only 1 instance + metadata for the `"opfs-sahpool"` sqlite3_vfs. Only one instance of this VFS can be installed per JavaScript engine, and any two engines with the same storage directory name will collide with each other, leading to locking errors and the inability to @@ -113,13 +113,16 @@ let isPromiseReady; unspecified but descriptive. This option may contain multiple path elements, e.g. "foo/bar/baz", and they are created automatically. In practice there should be no driving need to - change this. + change this. ACHTUNG: all files in this directory are assumed to + be managed by the VFS. Do not place other files in that + directory, as they may be deleted or otherwise modified by the + VFS. - API for the utility object passed on by this function's Promise, in - alphabetical order... + The API for the utility object passed on by this function's + Promise, in alphabetical order... -- [async] addCapacity(n) +- [async] number addCapacity(n) Adds `n` entries to the current pool. This change is persistent across sessions so should not be called automatically at each app @@ -144,7 +147,7 @@ let isPromiseReady; Returns the number of files from the pool currently in use. -- importDb(name, byteArray) +- void importDb(name, byteArray) Imports the contents of an SQLite database, provided as a byte array, under the given name, overwriting any existing @@ -161,6 +164,25 @@ let isPromiseReady; only remove currently-unused entries. It returns a Promise which resolves to the number of entries actually removed. +- [async] boolean removeVfs() + + Unregisters the opfs-sahpool VFS and removes its directory + from OPFS. After calling this, the VFS may no longer be used + and there is no way to re-add it aside from reloading the + current JavaScript context. + + Results are undefined if a database is currently in use with this + VFS. + + The returned Promise resolves to true if it performed the removal + and false if the VFS was not installed. + + If the VFS has a multi-level directory, e.g. "/foo/bar/baz", _only_ + the bottom-most level is removed because this VFS cannot know for + certain whether the higher-level directories contain data which + should be removed. + + - [async] number reserveMinimumCapacity(min) If the current capacity is less than `min`, the capacity is @@ -174,7 +196,7 @@ let isPromiseReady; effects. Results are undefined if the file is currently in active use. -- [async] wipeFiles() +- [async] void wipeFiles() Clears all client-defined state of all SAHs and makes all of them available for re-use by the pool. Results are undefined if any such @@ -271,6 +293,9 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ vfsDir: options.directory || ".sqlite3-opfs-sahpool", /* Directory handle to this.vfsDir. */ dirHandle: undefined, + /* Directory handle to this.dirHandle's parent dir. Needed + for a VFS-wipe op. */ + parentDirHandle: undefined, /* Maps SAHs to their opaque file names. */ mapSAHToName: new Map(), /* Maps client-side file names to SAHs. */ @@ -465,12 +490,15 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ reset: async function(clearFiles){ await isPromiseReady; let h = await navigator.storage.getDirectory(); + let prev, prevName; for(const d of this.vfsDir.split('/')){ if(d){ + prev = h; h = await h.getDirectoryHandle(d,{create:true}); } } this.dirHandle = h; + this.parentDirHandle = prev; this.releaseAccessHandles(); await this.acquireAccessHandles(clearFiles); }, @@ -801,17 +829,11 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ PoolUtil.reduceCapacity = async (n)=>SAHPool.reduceCapacity(n); PoolUtil.getCapacity = SAHPool.getCapacity.bind(SAHPool); PoolUtil.getActiveFileCount = SAHPool.getFileCount.bind(SAHPool); - /** If capacity is < min, increase capacity to min, else do - nothing. Resolves to the new capacity. */ PoolUtil.reserveMinimumCapacity = async (min)=>{ const c = SAHPool.getCapacity(); return (c < min) ? SAHPool.addCapacity(min - c) : c; }; - /** - Synchronously reads the contents of the given file into a - Uint8Array and returns it. This will throw if the given name is - not currently in active use or on I/O error. - */ + PoolUtil.exportFile = function(name){ const sah = SAHPool.mapFilenameToSAH.get(name) || toss("File not found:",name); const n = sah.getSize() - HEADER_OFFSET_DATA; @@ -820,21 +842,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ return b; }; - /** - The counterpart of exportFile(), this imports the contents of an - SQLite database, provided as a byte array, under the given name, - overwriting any existing content. Throws if the pool has no - available file slots, on I/O error, or if the input does not - appear to be a database. In the latter case, only a cursory - examination is made. - - Note that this routine is _only_ for importing database files, - not arbitrary files, the reason being that this VFS will - automatically clean up any non-database files so importing them - is pointless. - - Returns undefined. - */ PoolUtil.importDb = function(name, bytes){ const n = bytes.byteLength; if(n<512 || n%512!=0){ @@ -852,25 +859,28 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ sah.write(bytes, {at: HEADER_OFFSET_DATA}); SAHPool.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); }; - /** - Clears all client-defined state of all SAHs and makes all of them - available for re-use by the pool. Results are undefined if any - such handles are currently in use, e.g. by an sqlite3 db. - */ - PoolUtil.wipeFiles = async ()=>SAHPool.reset(true); - /** - If a virtual file exists with the given name, disassociates it - from the pool and returns true, else returns false without side - effects. - */ + PoolUtil.wipeFiles = async ()=>SAHPool.reset(true); PoolUtil.unlink = (filename)=>SAHPool.deletePath(filename); - /** - PoolUtil TODOs: - - - function to wipe out all traces of the VFS from storage. - */ + PoolUtil.removeVfs = async function(){ + if(!opfsVfs.pointer) return false; + capi.sqlite3_vfs_unregister(opfsVfs.pointer); + opfsVfs.dispose(); + try{ + SAHPool.releaseAccessHandles(); + if(SAHPool.parentDirHandle){ + await SAHPool.parentDirHandle.removeEntry( + SAHPool.dirHandle.name, {recursive: true} + ); + SAHPool.dirHandle = SAHPool.parentDirHandle = undefined; + } + }catch(e){ + warn("removeVfs() failed:",e); + /*but otherwise ignored*/ + } + return true; + }; return isPromiseReady = SAHPool.reset(!!options.clearOnInit).then(async ()=>{ if(SAHPool.$error){ diff --git a/manifest b/manifest index 1e2c3ff671..c9f239f8bb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sSAH\spool\sconfiguration\soptions\sfrom\sthe\slibrary-level\sconfig\sto\sa\sconfig\spassed\sto\sthe\sVFS\sinstall\sroutine.\sExtend\sand\sdocument\sthe\sPoolUtil\sobject. -D 2023-07-16T16:52:09.106 +C Add\sa\sway\sto\sremove\sthe\sopfs-sahpool's\spersistent\sstate\sfrom\sOPFS\sor\sunlink()\san\sindividual\sfile.\sDoc\scleanups. +D 2023-07-16T17:51:43.068 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js c19ccfc2995c0dcae00f13fe1be6fa436a39a3d629b6bf4208965ea78a50cab3 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js fa2f08cb8a2f6ed4e87241872a005d68376330f688560cf214fb46d4bb8d4e77 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P aa94c8abfbdfc4c7b36554c4b3ea90a5065e7e3f4294c64c8cbf688b4688300d -R 76883f55e00ff0c4af4f43f15f164d03 +P d2ed99556fa1f40994c1c6bd90d1d5733bebc824b1ebfabe978fae9e18948437 +R 8239157f9b8eb58dfb7c9f17d6e6ac86 U stephan -Z fd9b47fd6c1916432a0f6dc613a90b88 +Z e36f1591c077277bbe10a5f210ace350 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9dc2139fb9..25f5cba9c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2ed99556fa1f40994c1c6bd90d1d5733bebc824b1ebfabe978fae9e18948437 \ No newline at end of file +80982daac3c098033dbc249bb7a17ef84ae218d2d789f8644e7f4af18b553d24 \ No newline at end of file From a34f27daed6c05aaeded2ab919ae2fd72e24f415 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 19:20:45 +0000 Subject: [PATCH 16/56] Doc tweaks only, no code changes. FossilOrigin-Name: 044c28dffd45f7c4484686995edd4a1b92151450743968e7d0f662b5c850aa6b --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 7 ++++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index d7c5b198e0..a6bb05dcd2 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -90,9 +90,10 @@ let isPromiseReady; The options, in alphabetical order: - - `clearOnInit`: if truthy, as each SAH is acquired during - initalization of the VFS, its contents and filename name mapping - are removed, leaving the VFS's storage in a pristine state. + - `clearOnInit`: if truthy, contents and filename mapping are + removed from each SAH it is acquired during initalization of the + VFS, leaving the VFS's storage in a pristine state. Use this only + for databases which need not survive a page reload. - `defaultCapacity`: Specifies the default capacity of the VFS. This should not be set unduly high because the VFS has to diff --git a/manifest b/manifest index c9f239f8bb..a81fec1ac7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sway\sto\sremove\sthe\sopfs-sahpool's\spersistent\sstate\sfrom\sOPFS\sor\sunlink()\san\sindividual\sfile.\sDoc\scleanups. -D 2023-07-16T17:51:43.068 +C Doc\stweaks\sonly,\sno\scode\schanges. +D 2023-07-16T19:20:45.816 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js fa2f08cb8a2f6ed4e87241872a005d68376330f688560cf214fb46d4bb8d4e77 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 62a20546a380861fc6a7ab429b2c48440006e310b755882544b4c89cabaf0086 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d2ed99556fa1f40994c1c6bd90d1d5733bebc824b1ebfabe978fae9e18948437 -R 8239157f9b8eb58dfb7c9f17d6e6ac86 +P 80982daac3c098033dbc249bb7a17ef84ae218d2d789f8644e7f4af18b553d24 +R 6e687dfe648eade37ad557e4da56c423 U stephan -Z e36f1591c077277bbe10a5f210ace350 +Z 673837ecb501bfdd398983832825b7ae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 25f5cba9c5..0b2bd6dcd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -80982daac3c098033dbc249bb7a17ef84ae218d2d789f8644e7f4af18b553d24 \ No newline at end of file +044c28dffd45f7c4484686995edd4a1b92151450743968e7d0f662b5c850aa6b \ No newline at end of file From d703d25a35d10cc87d1840952700eb24a7289c76 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 19:29:50 +0000 Subject: [PATCH 17/56] Correct a missing assignment. Minor doc additions. FossilOrigin-Name: 1c4957d0ef23ff14d4f7bfb33a809dd92712ee9faf77b6052f823eb55de15cf6 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 7 +++++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index a6bb05dcd2..d8cf1a4d32 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -136,7 +136,10 @@ let isPromiseReady; Synchronously reads the contents of the given file into a Uint8Array and returns it. This will throw if the given name is not currently - in active use or on I/O error. + in active use or on I/O error. Note that the given name is _not_ + visible directly in OPFS (or, if it is, it's not from this VFS). The + reason for that is that this VFS manages name-to-file mappings in + a roundabout way in order to maintain its list of SAHs. - number getCapacity() @@ -904,7 +907,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ sqlite3.oo1.DB.dbCtorHelper.call(this, opt); }; OpfsSAHPoolDb.prototype = Object.create(sqlite3.oo1.DB.prototype); - OpfsSAHPoolDb.PoolUtil; + OpfsSAHPoolDb.PoolUtil = PoolUtil; sqlite3.oo1.OpfsSAHPoolDb = OpfsSAHPoolDb; sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, diff --git a/manifest b/manifest index a81fec1ac7..2002f2945f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Doc\stweaks\sonly,\sno\scode\schanges. -D 2023-07-16T19:20:45.816 +C Correct\sa\smissing\sassignment.\sMinor\sdoc\sadditions. +D 2023-07-16T19:29:50.521 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 62a20546a380861fc6a7ab429b2c48440006e310b755882544b4c89cabaf0086 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b35db2985fa9af223df18541a8854b5bada4ffaca3775302dc9955ff522e0281 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 80982daac3c098033dbc249bb7a17ef84ae218d2d789f8644e7f4af18b553d24 -R 6e687dfe648eade37ad557e4da56c423 +P 044c28dffd45f7c4484686995edd4a1b92151450743968e7d0f662b5c850aa6b +R bc433c86128be68279e883b7dab0d27c U stephan -Z 673837ecb501bfdd398983832825b7ae +Z 26de4c22ee5034492563d8a4ae215d10 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0b2bd6dcd3..b3a38db896 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -044c28dffd45f7c4484686995edd4a1b92151450743968e7d0f662b5c850aa6b \ No newline at end of file +1c4957d0ef23ff14d4f7bfb33a809dd92712ee9faf77b6052f823eb55de15cf6 \ No newline at end of file From 1ff2903785638cfc2fa862dedfef7455925fc834 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 20:20:03 +0000 Subject: [PATCH 18/56] Minor doc tweaks. FossilOrigin-Name: 0cb0d7a9e73e48406f146ebebf211311fd0a0b14b285f0af9e2b645c3d2f172a --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 10 +++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index d8cf1a4d32..8f0eb4be79 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -170,10 +170,10 @@ let isPromiseReady; - [async] boolean removeVfs() - Unregisters the opfs-sahpool VFS and removes its directory - from OPFS. After calling this, the VFS may no longer be used - and there is no way to re-add it aside from reloading the - current JavaScript context. + Unregisters the opfs-sahpool VFS and removes its directory from OPFS + (which means that _all client content_ is removed). After calling + this, the VFS may no longer be used and there is no way to re-add it + aside from reloading the current JavaScript context. Results are undefined if a database is currently in use with this VFS. @@ -182,7 +182,7 @@ let isPromiseReady; and false if the VFS was not installed. If the VFS has a multi-level directory, e.g. "/foo/bar/baz", _only_ - the bottom-most level is removed because this VFS cannot know for + the bottom-most directory is removed because this VFS cannot know for certain whether the higher-level directories contain data which should be removed. diff --git a/manifest b/manifest index 2002f2945f..da3ddc0344 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\sa\smissing\sassignment.\sMinor\sdoc\sadditions. -D 2023-07-16T19:29:50.521 +C Minor\sdoc\stweaks. +D 2023-07-16T20:20:03.760 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b35db2985fa9af223df18541a8854b5bada4ffaca3775302dc9955ff522e0281 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js c1d6244ebc3a12bc6c2f6b8056113b30176d82d4e69b59adf80ec1fd527ae13c F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 044c28dffd45f7c4484686995edd4a1b92151450743968e7d0f662b5c850aa6b -R bc433c86128be68279e883b7dab0d27c +P 1c4957d0ef23ff14d4f7bfb33a809dd92712ee9faf77b6052f823eb55de15cf6 +R 002fe055ac6d7277ae09c712f214aa59 U stephan -Z 26de4c22ee5034492563d8a4ae215d10 +Z a09c40b2c5acf5c48483c73a0a048183 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b3a38db896..678592856a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c4957d0ef23ff14d4f7bfb33a809dd92712ee9faf77b6052f823eb55de15cf6 \ No newline at end of file +0cb0d7a9e73e48406f146ebebf211311fd0a0b14b285f0af9e2b645c3d2f172a \ No newline at end of file From 8d67d501e31ae83fc6facf08bc443547bcd266ab Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 20:42:19 +0000 Subject: [PATCH 19/56] Add sqlite3-vfs-opfs-sahpool.js to ext/wasm/api/README.md. FossilOrigin-Name: 58d05201042804c682a417495f13ab47a535e353da6cc307f9d726a289e71ca2 --- ext/wasm/api/README.md | 15 +++++++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index 6618666ae1..be53ac25aa 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -83,15 +83,18 @@ browser client: helpers for use by downstream code which creates `sqlite3_vfs` and `sqlite3_module` implementations. - **`sqlite3-vfs-opfs.c-pp.js`**\ - is an sqlite3 VFS implementation which supports Google Chrome's - Origin-Private FileSystem (OPFS) as a storage layer to provide - persistent storage for database files in a browser. It requires... + is an sqlite3 VFS implementation which supports the Origin-Private + FileSystem (OPFS) as a storage layer to provide persistent storage + for database files in a browser. It requires... - **`sqlite3-opfs-async-proxy.js`**\ is the asynchronous backend part of the OPFS proxy. It speaks directly to the (async) OPFS API and channels those results back to its synchronous counterpart. This file, because it must be started in its own Worker, is not part of the amalgamation. -- **`api/sqlite3-api-cleanup.js`**\ +- **`sqlite3-vfs-opfs-sahpool.js`**\ + is another sqlite3 VFS supporting the OPFS, but uses a completely + different approach that the above-listed one. +- **`sqlite3-api-cleanup.js`**\ The previous files do not immediately extend the library. Instead they add callback functions to be called during its bootstrapping. Some also temporarily create global objects in order @@ -152,8 +155,8 @@ Preprocessing of Source Files ------------------------------------------------------------------------ Certain files in the build require preprocessing to filter in/out -parts which differ between vanilla JS builds and ES6 Module -(a.k.a. esm) builds. The preprocessor application itself is in +parts which differ between vanilla JS, ES6 Modules, and node.js +builds. The preprocessor application itself is in [`c-pp.c`](/file/ext/wasm/c-pp.c) and the complete technical details of such preprocessing are maintained in [`GNUMakefile`](/file/ext/wasm/GNUmakefile). diff --git a/manifest b/manifest index da3ddc0344..4107397c2a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sdoc\stweaks. -D 2023-07-16T20:20:03.760 +C Add\ssqlite3-vfs-opfs-sahpool.js\sto\sext/wasm/api/README.md. +D 2023-07-16T20:42:19.745 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -488,7 +488,7 @@ F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md 77a2f1f2fc60a35def7455dffc8d3f2c56385d6ac5c6cecc60fa938252ea2c54 +F ext/wasm/api/README.md f6cec6b0ce122cdff9440b30a3132dea3665b5b7baace910b43cbccdaaa376b9 F ext/wasm/api/extern-post-js.c-pp.js 116749b7e55b7519129de06d3d353e19df68cfb24b12204aa4dc30c9a83023fe F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1c4957d0ef23ff14d4f7bfb33a809dd92712ee9faf77b6052f823eb55de15cf6 -R 002fe055ac6d7277ae09c712f214aa59 +P 0cb0d7a9e73e48406f146ebebf211311fd0a0b14b285f0af9e2b645c3d2f172a +R c6942b2b515fb79cb584a67745d1ba97 U stephan -Z a09c40b2c5acf5c48483c73a0a048183 +Z 4418670e816c5df9f151e2a496afef18 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 678592856a..1bca9d4d26 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cb0d7a9e73e48406f146ebebf211311fd0a0b14b285f0af9e2b645c3d2f172a \ No newline at end of file +58d05201042804c682a417495f13ab47a535e353da6cc307f9d726a289e71ca2 \ No newline at end of file From 039081123cae1fecfc925dd6dc2f18e17cac3944 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 21:07:01 +0000 Subject: [PATCH 20/56] Remove an extraneous JS script import. FossilOrigin-Name: 5a205b25912f3feea594a2161a4b8f3955bd29163b39cb0e621b3abdb81fc24d --- ext/wasm/speedtest1-worker.js | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index 61af26b23e..451ba458a6 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -5,7 +5,7 @@ if(urlParams.has('sqlite3.dir')){ speedtestJs = urlParams.get('sqlite3.dir') + '/' + speedtestJs; } - importScripts('common/whwasmutil.js', speedtestJs); + importScripts(speedtestJs); /** If this environment contains OPFS, this function initializes it and returns the name of the dir on which OPFS is mounted, else it returns diff --git a/manifest b/manifest index 4107397c2a..1ecacb5913 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssqlite3-vfs-opfs-sahpool.js\sto\sext/wasm/api/README.md. -D 2023-07-16T20:42:19.745 +C Remove\san\sextraneous\sJS\sscript\simport. +D 2023-07-16T21:07:01.718 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 -F ext/wasm/speedtest1-worker.js 41fdc91878d3481b198bba771f073aad8837063ea2a23a0e9a278a54634f8ffe +F ext/wasm/speedtest1-worker.js 6ddd3c8fab3b0be6534af6ba2c8a419cda723ada9eea661668f76a142d77e4c9 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cb0d7a9e73e48406f146ebebf211311fd0a0b14b285f0af9e2b645c3d2f172a -R c6942b2b515fb79cb584a67745d1ba97 +P 58d05201042804c682a417495f13ab47a535e353da6cc307f9d726a289e71ca2 +R adf708d1e44ca386b9dfb73d1670c35b U stephan -Z 4418670e816c5df9f151e2a496afef18 +Z 20b4588b247bbe89f1dcafce271a338a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1bca9d4d26..b688b017e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58d05201042804c682a417495f13ab47a535e353da6cc307f9d726a289e71ca2 \ No newline at end of file +5a205b25912f3feea594a2161a4b8f3955bd29163b39cb0e621b3abdb81fc24d \ No newline at end of file From 2bf2ff605fffe0a501d83b076780a01178fb8de8 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 17 Jul 2023 07:13:44 +0000 Subject: [PATCH 21/56] Change wasm build's -DSQLITE_TEMP_STORE=3 to -DSQLITE_TEMP_STORE=2, primarily so that clients can optionally shift temp files from memory to OPFS storage. FossilOrigin-Name: 764430e804332b8f7b2f63a3c0c8acf8f6cc92534858d89de2d310938c1c0d27 --- ext/wasm/api/sqlite3-wasm.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index dbe594dc3d..94b16d7500 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -151,7 +151,7 @@ /**********************************************************************/ /* SQLITE_T... */ #ifndef SQLITE_TEMP_STORE -# define SQLITE_TEMP_STORE 3 +# define SQLITE_TEMP_STORE 2 #endif #ifndef SQLITE_THREADSAFE # define SQLITE_THREADSAFE 0 diff --git a/manifest b/manifest index 1ecacb5913..42905a8b0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sextraneous\sJS\sscript\simport. -D 2023-07-16T21:07:01.718 +C Change\swasm\sbuild's\s-DSQLITE_TEMP_STORE=3\sto\s-DSQLITE_TEMP_STORE=2,\sprimarily\sso\sthat\sclients\scan\soptionally\sshift\stemp\sfiles\sfrom\smemory\sto\sOPFS\sstorage. +D 2023-07-17T07:13:44.233 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -504,7 +504,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js c1d6244ebc3a12bc6c2f6b8056113b30176d82d4e69b59adf80ec1fd527ae13c F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 -F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda +F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 58d05201042804c682a417495f13ab47a535e353da6cc307f9d726a289e71ca2 -R adf708d1e44ca386b9dfb73d1670c35b +P 5a205b25912f3feea594a2161a4b8f3955bd29163b39cb0e621b3abdb81fc24d +R 20740462df16b4bb4e845e220cae17b6 U stephan -Z 20b4588b247bbe89f1dcafce271a338a +Z 01e26e5888d09bb4530f76d3fff6e86b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b688b017e1..6a8a558193 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a205b25912f3feea594a2161a4b8f3955bd29163b39cb0e621b3abdb81fc24d \ No newline at end of file +764430e804332b8f7b2f63a3c0c8acf8f6cc92534858d89de2d310938c1c0d27 \ No newline at end of file From ed640943c4306908c5e0f7a09fb80e8b4e1e0aea Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 17 Jul 2023 07:14:12 +0000 Subject: [PATCH 22/56] sahpool vfs: rename defaultCapacity to initialCapacity, per feedback. FossilOrigin-Name: 2c9c95d087c7db2a975ffae15af5ade621411c493145b889f0c9157a55c4952a --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 5 ++--- ext/wasm/speedtest1-worker.js | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 8f0eb4be79..56af5d402b 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -95,7 +95,7 @@ let isPromiseReady; VFS, leaving the VFS's storage in a pristine state. Use this only for databases which need not survive a page reload. - - `defaultCapacity`: Specifies the default capacity of the + - `initialCapacity`: Specifies the default capacity of the VFS. This should not be set unduly high because the VFS has to open (and keep open) a file for each entry in the pool. This setting only has an effect when the pool is initially empty. It @@ -256,7 +256,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; const HEADER_OFFSET_DATA = SECTOR_SIZE; const DEFAULT_CAPACITY = - options.defaultCapacity || 6; + options.initialCapacity || 6; /* Bitmask of file types which may persist across sessions. SQLITE_OPEN_xyz types not listed here may be inadvertently left in OPFS but are treated as transient by this VFS and @@ -467,7 +467,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ }else{ // This is not a persistent file, so eliminate the contents. sah.truncate(HEADER_OFFSET_DATA); - this.mapFilenameToSAH.delete(path); this.availableSAH.add(sah); } }, diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index 451ba458a6..e223084887 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -63,7 +63,7 @@ log("Installing opfs-sahpool..."); await App.sqlite3.installOpfsSAHPoolVfs({ directory: '.speedtest1-sahpool', - defaultCapacity: 3, + initialCapacity: 3, clearOnInit: true }).then(PoolUtil=>{ log("opfs-sahpool successfully installed."); diff --git a/manifest b/manifest index 42905a8b0f..576508f68b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\swasm\sbuild's\s-DSQLITE_TEMP_STORE=3\sto\s-DSQLITE_TEMP_STORE=2,\sprimarily\sso\sthat\sclients\scan\soptionally\sshift\stemp\sfiles\sfrom\smemory\sto\sOPFS\sstorage. -D 2023-07-17T07:13:44.233 +C sahpool\svfs:\srename\sdefaultCapacity\sto\sinitialCapacity,\sper\sfeedback. +D 2023-07-17T07:14:12.157 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js c1d6244ebc3a12bc6c2f6b8056113b30176d82d4e69b59adf80ec1fd527ae13c +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js ff3495c0140787fc0232081582a6126f22b7b2e7894bee83061e8bf9eea2ef7f F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 -F ext/wasm/speedtest1-worker.js 6ddd3c8fab3b0be6534af6ba2c8a419cda723ada9eea661668f76a142d77e4c9 +F ext/wasm/speedtest1-worker.js 4f18caa941ed89d42af46c598e7f7fe31cecac853e0b370d235adcb19ce8cbee F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a205b25912f3feea594a2161a4b8f3955bd29163b39cb0e621b3abdb81fc24d -R 20740462df16b4bb4e845e220cae17b6 +P 764430e804332b8f7b2f63a3c0c8acf8f6cc92534858d89de2d310938c1c0d27 +R 753478ab53ba7376ae32d6678ec6019e U stephan -Z 01e26e5888d09bb4530f76d3fff6e86b +Z e2f8b2590695ded5aa3bfa89876a6a8d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6a8a558193..f69aadc4af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -764430e804332b8f7b2f63a3c0c8acf8f6cc92534858d89de2d310938c1c0d27 \ No newline at end of file +2c9c95d087c7db2a975ffae15af5ade621411c493145b889f0c9157a55c4952a \ No newline at end of file From 99d4a2db46ddcfc5c6b44477a78a8f3a2772d143 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 17 Jul 2023 07:43:04 +0000 Subject: [PATCH 23/56] Another -DSQLITE_TEMP_STORE=3 to 2 change for the wasm build. FossilOrigin-Name: 5ad8f9987c11b5db67edd69a3471ff388140d6268d5d3f5a05dec6bb6d92ac05 --- ext/wasm/GNUmakefile | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index f8df213797..486d1396d8 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -183,7 +183,7 @@ SQLITE_OPT = \ -DSQLITE_OMIT_SHARED_CACHE \ -DSQLITE_OMIT_WAL \ -DSQLITE_THREADSAFE=0 \ - -DSQLITE_TEMP_STORE=3 \ + -DSQLITE_TEMP_STORE=2 \ -DSQLITE_OS_KV_OPTIONAL=1 \ '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ -DSQLITE_USE_URI=1 \ diff --git a/manifest b/manifest index 576508f68b..e205dc4824 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C sahpool\svfs:\srename\sdefaultCapacity\sto\sinitialCapacity,\sper\sfeedback. -D 2023-07-17T07:14:12.157 +C Another\s-DSQLITE_TEMP_STORE=3\sto\s2\schange\sfor\sthe\swasm\sbuild. +D 2023-07-17T07:43:04.661 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile e38d5835d86f8103c2165b42bb49a2558dfcced1129763e935b7c829fe0452d9 +F ext/wasm/GNUmakefile b425091409491f0f9dca77f7a41143530f8b7d37abfb3ba59f1d50f4cc85d02f F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 764430e804332b8f7b2f63a3c0c8acf8f6cc92534858d89de2d310938c1c0d27 -R 753478ab53ba7376ae32d6678ec6019e +P 2c9c95d087c7db2a975ffae15af5ade621411c493145b889f0c9157a55c4952a +R f9198174ee4df68f95986dfc1d71ac1a U stephan -Z e2f8b2590695ded5aa3bfa89876a6a8d +Z 5f625ecf88ca9a37df1d1dfeecd67369 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f69aadc4af..5528ba7f19 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c9c95d087c7db2a975ffae15af5ade621411c493145b889f0c9157a55c4952a \ No newline at end of file +5ad8f9987c11b5db67edd69a3471ff388140d6268d5d3f5a05dec6bb6d92ac05 \ No newline at end of file From dec4cea24ceb78a363d4e4dd50ff58143ddd5f51 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 18 Jul 2023 12:09:16 +0000 Subject: [PATCH 24/56] Refactor opfs-sahpool to support multiple instances, each with a separate VFS name and directory. FossilOrigin-Name: d036eaf6ac60c576428db40f015733c5d5425f7d613194fd8d9d4d98659077c4 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 790 ++++++++++++----------- ext/wasm/speedtest1-worker.js | 18 +- manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 430 insertions(+), 394 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 56af5d402b..acba1d2d15 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -54,11 +54,33 @@ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = sqlite3.util.toss; -let vfsRegisterResult = undefined; -/** The PoolUtil object will be the result of the - resolved Promise. */ -const PoolUtil = Object.create(null); -let isPromiseReady; +const isPromiseReady = Object.create(null); +const capi = sqlite3.capi; +// Config opts for the VFS... +const SECTOR_SIZE = 4096; +const HEADER_MAX_PATH_SIZE = 512; +const HEADER_FLAGS_SIZE = 4; +const HEADER_DIGEST_SIZE = 8; +const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE; +const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE; +const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; +const HEADER_OFFSET_DATA = SECTOR_SIZE; +/* Bitmask of file types which may persist across sessions. + SQLITE_OPEN_xyz types not listed here may be inadvertently + left in OPFS but are treated as transient by this VFS and + they will be cleaned up during VFS init. */ +const PERSISTENT_FILE_TYPES = + capi.SQLITE_OPEN_MAIN_DB | + capi.SQLITE_OPEN_MAIN_JOURNAL | + capi.SQLITE_OPEN_SUPER_JOURNAL | + capi.SQLITE_OPEN_WAL /* noting that WAL support is + unavailable in the WASM build.*/; + +/** + Returns short a string of random alphanumeric characters + suitable for use as a random filename. +*/ +const getRandomName = ()=>Math.random().toString(36).slice(2); /** installOpfsSAHPoolVfs() asynchronously initializes the OPFS @@ -76,9 +98,14 @@ let isPromiseReady; registered, the second page would fail to load the VFS due to OPFS locking errors. - On calls after the first this function immediately returns a - pending, resolved, or rejected Promise, depending on the state - of the first call's Promise. + If this function is called more than once with a given "name" + option (see below), it will return the same Promise, with one + exception: if called twice in immediate succession, the first will + not yet have had time to create its (cached) return Promise and the + second call will attempt to re-initialize the VFS, failing. (How to + resolve that race is unclear.) Calls for different names will + return different Promises which resolve to independent objects and + refer to different VFS registrations. On success, the resulting Promise resolves to a utility object which can be used to query and manipulate the pool. Its API is @@ -90,34 +117,48 @@ let isPromiseReady; The options, in alphabetical order: - - `clearOnInit`: if truthy, contents and filename mapping are - removed from each SAH it is acquired during initalization of the - VFS, leaving the VFS's storage in a pristine state. Use this only - for databases which need not survive a page reload. + - `clearOnInit`: (default=false) if truthy, contents and filename + mapping are removed from each SAH it is acquired during + initalization of the VFS, leaving the VFS's storage in a pristine + state. Use this only for databases which need not survive a page + reload. - - `initialCapacity`: Specifies the default capacity of the - VFS. This should not be set unduly high because the VFS has to - open (and keep open) a file for each entry in the pool. This + - `initialCapacity`: (default=6) Specifies the default capacity of + the VFS. This should not be set unduly high because the VFS has + to open (and keep open) a file for each entry in the pool. This setting only has an effect when the pool is initially empty. It does not have any effect if a pool already exists. - - `directory`: Specifies the OPFS directory name in which to store - metadata for the `"opfs-sahpool"` sqlite3_vfs. Only one instance - of this VFS can be installed per JavaScript engine, and any two - engines with the same storage directory name will collide with - each other, leading to locking errors and the inability to - register the VFS in the second and subsequent engine. Using a - different directory name for each application enables different - engines in the same HTTP origin to co-exist, but their data are - invisible to each other. Changing this name will effectively - orphan any databases stored under previous names. The default is - unspecified but descriptive. This option may contain multiple - path elements, e.g. "foo/bar/baz", and they are created - automatically. In practice there should be no driving need to - change this. ACHTUNG: all files in this directory are assumed to - be managed by the VFS. Do not place other files in that - directory, as they may be deleted or otherwise modified by the - VFS. + - `directory`: (default="."+`name`) Specifies the OPFS directory + name in which to store metadata for the `"opfs-sahpool"` + sqlite3_vfs. Only one instance of this VFS can be installed per + JavaScript engine, and any two engines with the same storage + directory name will collide with each other, leading to locking + errors and the inability to register the VFS in the second and + subsequent engine. Using a different directory name for each + application enables different engines in the same HTTP origin to + co-exist, but their data are invisible to each other. Changing + this name will effectively orphan any databases stored under + previous names. The default is unspecified but descriptive. This + option may contain multiple path elements, e.g. "foo/bar/baz", + and they are created automatically. In practice there should be + no driving need to change this. ACHTUNG: all files in this + directory are assumed to be managed by the VFS. Do not place + other files in that directory, as they may be deleted or + otherwise modified by the VFS. + + - `name`: (default="opfs-sahpool") sets the name to register this + VFS under. Normally this should not be changed, but it is + possible to register this VFS under multiple names so long as + each has its own separate directory to work from. The storage for + each is invisible to all others. The name must be a string + compatible with `sqlite3_vfs_register()` and friends and suitable + for use in URI-style database file names. + + Achtung: if a custom `name` is provided, a custom `directory` + must also be provided if any other instance is registered with + the default directory. If no directory is explicitly provided + then a directory name is synthesized from the `name` option. The API for the utility object passed on by this function's @@ -186,7 +227,6 @@ let isPromiseReady; certain whether the higher-level directories contain data which should be removed. - - [async] number reserveMinimumCapacity(min) If the current capacity is less than `min`, the capacity is @@ -200,6 +240,10 @@ let isPromiseReady; effects. Results are undefined if the file is currently in active use. +- string vfsName + + The SQLite VFS name under which this pool's VFS is registered. + - [async] void wipeFiles() Clears all client-defined state of all SAHs and makes all of them @@ -208,65 +252,49 @@ let isPromiseReady; */ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ - if(PoolUtil===vfsRegisterResult) return Promise.resolve(PoolUtil); - else if(isPromiseReady) return isPromiseReady; - else if(undefined!==vfsRegisterResult){ - return Promise.reject(vfsRegisterResult); +/** The PoolUtil object will be the result of the + resolved Promise. */ + const PoolUtil = Object.create(null); + const vfsName = PoolUtil.vfsName = options.name || "opfs-sahpool"; + if(isPromiseReady[vfsName]){ + //console.warn("Returning same OpfsSAHPool result",vfsName,isPromiseReady[vfsName]); + return isPromiseReady[vfsName]; } if(!globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle || !globalThis.FileSystemFileHandle || !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || !navigator?.storage?.getDirectory){ - return Promise.reject(vfsRegisterResult = new Error("Missing required OPFS APIs.")); + return (isPromiseReady[vfsName] = Promise.reject(new Error("Missing required OPFS APIs."))); } - vfsRegisterResult = new Error("opfs-sahpool initialization still underway."); - const verbosity = 2 /*3+ == everything*/; + const verbosity = options.verbosity + || 2 /*3+ == everything, 2 == warnings+errors, 1 == errors only*/; const loggers = [ sqlite3.config.error, sqlite3.config.warn, sqlite3.config.log ]; const logImpl = (level,...args)=>{ - if(verbosity>level) loggers[level]("opfs-sahpool:",...args); + if(verbosity>level) loggers[level](vfsName+":",...args); }; const log = (...args)=>logImpl(2, ...args); const warn = (...args)=>logImpl(1, ...args); const error = (...args)=>logImpl(0, ...args); - const capi = sqlite3.capi; const wasm = sqlite3.wasm; const opfsIoMethods = new capi.sqlite3_io_methods(); const opfsVfs = new capi.sqlite3_vfs() .addOnDispose(()=>opfsIoMethods.dispose()); + const promiseReject = (err)=>{ error("rejecting promise:",err); //opfsVfs.dispose(); - vfsRegisterResult = err; - return Promise.reject(err); + return isPromiseReady[vfsName] = Promise.reject(err); }; - const promiseResolve = - ()=>Promise.resolve(vfsRegisterResult = PoolUtil); - // Config opts for the VFS... - const SECTOR_SIZE = 4096; - const HEADER_MAX_PATH_SIZE = 512; - const HEADER_FLAGS_SIZE = 4; - const HEADER_DIGEST_SIZE = 8; - const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE; - const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE; - const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; - const HEADER_OFFSET_DATA = SECTOR_SIZE; - const DEFAULT_CAPACITY = - options.initialCapacity || 6; - /* Bitmask of file types which may persist across sessions. - SQLITE_OPEN_xyz types not listed here may be inadvertently - left in OPFS but are treated as transient by this VFS and - they will be cleaned up during VFS init. */ - const PERSISTENT_FILE_TYPES = - capi.SQLITE_OPEN_MAIN_DB | - capi.SQLITE_OPEN_MAIN_JOURNAL | - capi.SQLITE_OPEN_SUPER_JOURNAL | - capi.SQLITE_OPEN_WAL /* noting that WAL support is - unavailable in the WASM build.*/; + if( sqlite3.capi.sqlite3_vfs_find(vfsName)){ + return promiseReject(new Error("VFS name is already registered:", + vfsName)); + } + /* We fetch the default VFS so that we can inherit some methods from it. */ const pDVfs = capi.sqlite3_vfs_find(null); @@ -279,22 +307,16 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; opfsVfs.addOnDispose( - opfsVfs.$zName = wasm.allocCString("opfs-sahpool"), + opfsVfs.$zName = wasm.allocCString(vfsName), ()=>(dVfs ? dVfs.dispose() : null) ); - /** - Returns short a string of random alphanumeric characters - suitable for use as a random filename. - */ - const getRandomName = ()=>Math.random().toString(36).slice(2); - /** All state for the VFS. */ const SAHPool = Object.assign(Object.create(null),{ /* OPFS dir in which VFS metadata is stored. */ - vfsDir: options.directory || ".sqlite3-opfs-sahpool", + vfsDir: options.directory || ("."+vfsName), /* Directory handle to this.vfsDir. */ dirHandle: undefined, /* Directory handle to this.dirHandle's parent dir. Needed @@ -491,7 +513,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ argument. */ reset: async function(clearFiles){ - await isPromiseReady; + await isPromiseReady[vfsName]; let h = await navigator.storage.getDirectory(); let prev, prevName; for(const d of this.vfsDir.split('/')){ @@ -559,245 +581,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ SAHPool.dvBody = new DataView(SAHPool.apBody.buffer, SAHPool.apBody.byteOffset); //sqlite3.SAHPool = SAHPool/*only for testing*/; - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioMethods = { - xCheckReservedLock: function(pFile,pOut){ - log('xCheckReservedLock'); - SAHPool.storeErr(); - wasm.poke32(pOut, 1); - return 0; - }, - xClose: function(pFile){ - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - if(file) { - try{ - log(`xClose ${file}`); - if(file.sq3File) file.sq3File.dispose(); - file.sah.flush(); - SAHPool.mapIdToFile.delete(pFile); - if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ - SAHPool.deletePath(file.path); - } - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - } - return 0; - }, - xDeviceCharacteristics: function(pFile){ - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - log(`xFileSize`); - const file = SAHPool.mapIdToFile.get(pFile); - const size = file.sah.getSize() - HEADER_OFFSET_DATA; - //log(`xFileSize ${file.path} ${size}`); - wasm.poke64(pSz64, BigInt(size)); - return 0; - }, - xLock: function(pFile,lockType){ - log(`xLock ${lockType}`); - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - file.lockType = lockType; - return 0; - }, - xRead: function(pFile,pDest,n,offset64){ - log(`xRead ${n}@${offset64}`); - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - log(`xRead ${file.path} ${n} ${offset64}`); - try { - const nRead = file.sah.read( - wasm.heap8u().subarray(pDest, pDest+n), - {at: HEADER_OFFSET_DATA + Number(offset64)} - ); - if(nRead < n){ - wasm.heap8u().fill(0, pDest + nRead, pDest + n); - return capi.SQLITE_IOERR_SHORT_READ; - } - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - }, - xSectorSize: function(pFile){ - return SECTOR_SIZE; - }, - xSync: function(pFile,flags){ - log(`xSync ${flags}`); - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - //log(`xSync ${file.path} ${flags}`); - try{ - file.sah.flush(); - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - }, - xTruncate: function(pFile,sz64){ - log(`xTruncate ${sz64}`); - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - //log(`xTruncate ${file.path} ${iSize}`); - try{ - file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - }, - xUnlock: function(pFile,lockType){ - log('xUnlock'); - const file = SAHPool.mapIdToFile.get(pFile); - file.lockType = lockType; - return 0; - }, - xWrite: function(pFile,pSrc,n,offset64){ - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - log(`xWrite ${file.path} ${n} ${offset64}`); - try{ - const nBytes = file.sah.write( - wasm.heap8u().subarray(pSrc, pSrc+n), - { at: HEADER_OFFSET_DATA + Number(offset64) } - ); - return nBytes === n ? 0 : capi.SQLITE_IOERR; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - } - }/*ioMethods*/; - - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsMethods = { - xAccess: function(pVfs,zName,flags,pOut){ - log(`xAccess ${wasm.cstrToJs(zName)}`); - SAHPool.storeErr(); - try{ - const name = this.getPath(zName); - wasm.poke32(pOut, SAHPool.mapFilenameToSAH.has(name) ? 1 : 0); - }catch(e){ - /*ignored*/; - } - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - log(`xDelete ${wasm.cstrToJs(zName)}`); - SAHPool.storeErr(); - try{ - SAHPool.deletePath(SAHPool.getPath(zName)); - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR_DELETE; - } - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - log(`xFullPathname ${wasm.cstrToJs(zName)}`); - const i = wasm.cstrncpy(pOut, zName, nOut); - return i nOut) wasm.poke8(pOut + nOut - 1, 0); - }catch(e){ - return capi.SQLITE_NOMEM; - }finally{ - wasm.scopedAllocPop(scope); - } - } - return 0; - }, - //xSleep is optionally defined below - xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ - log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`); - try{ - // First try to open a path that already exists in the file system. - const path = (zName && wasm.peek8(zName)) - ? SAHPool.getPath(zName) - : getRandomName(); - let sah = SAHPool.mapFilenameToSAH.get(path); - if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { - // File not found so try to create it. - if(SAHPool.getFileCount() < SAHPool.getCapacity()) { - // Choose an unassociated OPFS file from the pool. - sah = SAHPool.nextAvailableSAH(); - SAHPool.setAssociatedPath(sah, path, flags); - }else{ - // File pool is full. - toss('SAH pool is full. Cannot create file',path); - } - } - if(!sah){ - toss('file not found:',path); - } - // Subsequent methods are only passed the file pointer, so - // map the relevant info we need to that pointer. - const file = {path, flags, sah}; - SAHPool.mapIdToFile.set(pFile, file); - wasm.poke32(pOutFlags, flags); - file.sq3File = new capi.sqlite3_file(pFile); - file.sq3File.$pMethods = opfsIoMethods.pointer; - file.lockType = capi.SQLITE_LOCK_NONE; - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_CANTOPEN; - } - }/*xOpen()*/ - }/*vfsMethods*/; - - if(dVfs){ - /* Inherit certain VFS members from the default VFS, - if available. */ - opfsVfs.$xRandomness = dVfs.$xRandomness; - opfsVfs.$xSleep = dVfs.$xSleep; - } - if(!opfsVfs.$xRandomness){ - /* If the default VFS has no xRandomness(), add a basic JS impl... */ - vfsMethods.xRandomness = function(pVfs, nOut, pOut){ - const heap = wasm.heap8u(); - let i = 0; - for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; - return i; - }; - } - if(!opfsVfs.$xSleep){ - vfsMethods.xSleep = (pVfs,ms)=>0; - } /** Ensure that the client has a "fully-sync" SAH impl, @@ -827,71 +610,311 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ return promiseReject(apiVersionCheck); } - PoolUtil.$SAHPool = SAHPool/* ONLY for testing and debugging */; - PoolUtil.addCapacity = async (n)=>SAHPool.addCapacity(n); - PoolUtil.reduceCapacity = async (n)=>SAHPool.reduceCapacity(n); - PoolUtil.getCapacity = SAHPool.getCapacity.bind(SAHPool); - PoolUtil.getActiveFileCount = SAHPool.getFileCount.bind(SAHPool); - PoolUtil.reserveMinimumCapacity = async (min)=>{ - const c = SAHPool.getCapacity(); - return (c < min) ? SAHPool.addCapacity(min - c) : c; - }; - - PoolUtil.exportFile = function(name){ - const sah = SAHPool.mapFilenameToSAH.get(name) || toss("File not found:",name); - const n = sah.getSize() - HEADER_OFFSET_DATA; - const b = new Uint8Array(n>=0 ? n : 0); - if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); - return b; - }; - - PoolUtil.importDb = function(name, bytes){ - const n = bytes.byteLength; - if(n<512 || n%512!=0){ - toss("Byte array size is invalid for an SQLite db."); - } - const header = "SQLite format 3"; - for(let i = 0; i < header.length; ++i){ - if( header.charCodeAt(i) !== bytes[i] ){ - toss("Input does not contain an SQLite database header."); - } - } - const sah = SAHPool.mapFilenameToSAH.get(name) - || SAHPool.nextAvailableSAH() - || toss("No available handles to import to."); - sah.write(bytes, {at: HEADER_OFFSET_DATA}); - SAHPool.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); - }; - - PoolUtil.wipeFiles = async ()=>SAHPool.reset(true); - PoolUtil.unlink = (filename)=>SAHPool.deletePath(filename); - - PoolUtil.removeVfs = async function(){ - if(!opfsVfs.pointer) return false; - capi.sqlite3_vfs_unregister(opfsVfs.pointer); - opfsVfs.dispose(); - try{ - SAHPool.releaseAccessHandles(); - if(SAHPool.parentDirHandle){ - await SAHPool.parentDirHandle.removeEntry( - SAHPool.dirHandle.name, {recursive: true} - ); - SAHPool.dirHandle = SAHPool.parentDirHandle = undefined; - } - }catch(e){ - warn("removeVfs() failed:",e); - /*but otherwise ignored*/ - } - return true; - }; - - return isPromiseReady = SAHPool.reset(!!options.clearOnInit).then(async ()=>{ + return isPromiseReady[vfsName] = SAHPool.reset(!!options.clearOnInit).then(async ()=>{ if(SAHPool.$error){ throw SAHPool.$error; } if(0===SAHPool.getCapacity()){ - await SAHPool.addCapacity(DEFAULT_CAPACITY); + await SAHPool.addCapacity(options.initialCapacity || 6); } + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioMethods = { + xCheckReservedLock: function(pFile,pOut){ + log('xCheckReservedLock'); + SAHPool.storeErr(); + wasm.poke32(pOut, 1); + return 0; + }, + xClose: function(pFile){ + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + if(file) { + try{ + log(`xClose ${file.path}`); + if(file.sq3File) file.sq3File.dispose(); + file.sah.flush(); + SAHPool.mapIdToFile.delete(pFile); + if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ + SAHPool.deletePath(file.path); + } + }catch(e){ + SAHPool.storeErr(e); + return capi.SQLITE_IOERR; + } + } + return 0; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + log(`xFileSize`); + const file = SAHPool.mapIdToFile.get(pFile); + const size = file.sah.getSize() - HEADER_OFFSET_DATA; + //log(`xFileSize ${file.path} ${size}`); + wasm.poke64(pSz64, BigInt(size)); + return 0; + }, + xLock: function(pFile,lockType){ + log(`xLock ${lockType}`); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + file.lockType = lockType; + return 0; + }, + xRead: function(pFile,pDest,n,offset64){ + log(`xRead ${n}@${offset64}`); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + log(`xRead ${file.path} ${n} ${offset64}`); + try { + const nRead = file.sah.read( + wasm.heap8u().subarray(pDest, pDest+n), + {at: HEADER_OFFSET_DATA + Number(offset64)} + ); + if(nRead < n){ + wasm.heap8u().fill(0, pDest + nRead, pDest + n); + return capi.SQLITE_IOERR_SHORT_READ; + } + return 0; + }catch(e){ + SAHPool.storeErr(e); + return capi.SQLITE_IOERR; + } + }, + xSectorSize: function(pFile){ + return SECTOR_SIZE; + }, + xSync: function(pFile,flags){ + log(`xSync ${flags}`); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + //log(`xSync ${file.path} ${flags}`); + try{ + file.sah.flush(); + return 0; + }catch(e){ + SAHPool.storeErr(e); + return capi.SQLITE_IOERR; + } + }, + xTruncate: function(pFile,sz64){ + log(`xTruncate ${sz64}`); + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + //log(`xTruncate ${file.path} ${iSize}`); + try{ + file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); + return 0; + }catch(e){ + SAHPool.storeErr(e); + return capi.SQLITE_IOERR; + } + }, + xUnlock: function(pFile,lockType){ + log('xUnlock'); + const file = SAHPool.mapIdToFile.get(pFile); + file.lockType = lockType; + return 0; + }, + xWrite: function(pFile,pSrc,n,offset64){ + SAHPool.storeErr(); + const file = SAHPool.mapIdToFile.get(pFile); + log(`xWrite ${file.path} ${n} ${offset64}`); + try{ + const nBytes = file.sah.write( + wasm.heap8u().subarray(pSrc, pSrc+n), + { at: HEADER_OFFSET_DATA + Number(offset64) } + ); + return nBytes === n ? 0 : capi.SQLITE_IOERR; + }catch(e){ + SAHPool.storeErr(e); + return capi.SQLITE_IOERR; + } + } + }/*ioMethods*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsMethods = { + xAccess: function(pVfs,zName,flags,pOut){ + log(`xAccess ${wasm.cstrToJs(zName)}`); + SAHPool.storeErr(); + try{ + const name = this.getPath(zName); + wasm.poke32(pOut, SAHPool.mapFilenameToSAH.has(name) ? 1 : 0); + }catch(e){ + /*ignored*/; + } + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + log(`xDelete ${wasm.cstrToJs(zName)}`); + SAHPool.storeErr(); + try{ + SAHPool.deletePath(SAHPool.getPath(zName)); + return 0; + }catch(e){ + SAHPool.storeErr(e); + return capi.SQLITE_IOERR_DELETE; + } + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + log(`xFullPathname ${wasm.cstrToJs(zName)}`); + const i = wasm.cstrncpy(pOut, zName, nOut); + return i nOut) wasm.poke8(pOut + nOut - 1, 0); + }catch(e){ + return capi.SQLITE_NOMEM; + }finally{ + wasm.scopedAllocPop(scope); + } + } + return 0; + }, + //xSleep is optionally defined below + xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ + log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`); + try{ + // First try to open a path that already exists in the file system. + const path = (zName && wasm.peek8(zName)) + ? SAHPool.getPath(zName) + : getRandomName(); + let sah = SAHPool.mapFilenameToSAH.get(path); + if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { + // File not found so try to create it. + if(SAHPool.getFileCount() < SAHPool.getCapacity()) { + // Choose an unassociated OPFS file from the pool. + sah = SAHPool.nextAvailableSAH(); + SAHPool.setAssociatedPath(sah, path, flags); + }else{ + // File pool is full. + toss('SAH pool is full. Cannot create file',path); + } + } + if(!sah){ + toss('file not found:',path); + } + // Subsequent methods are only passed the file pointer, so + // map the relevant info we need to that pointer. + const file = {path, flags, sah}; + SAHPool.mapIdToFile.set(pFile, file); + wasm.poke32(pOutFlags, flags); + file.sq3File = new capi.sqlite3_file(pFile); + file.sq3File.$pMethods = opfsIoMethods.pointer; + file.lockType = capi.SQLITE_LOCK_NONE; + return 0; + }catch(e){ + SAHPool.storeErr(e); + return capi.SQLITE_CANTOPEN; + } + }/*xOpen()*/ + }/*vfsMethods*/; + + if(dVfs){ + /* Inherit certain VFS members from the default VFS, + if available. */ + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; + } + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + vfsMethods.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + vfsMethods.xSleep = (pVfs,ms)=>0; + } + + PoolUtil.$SAHPool = SAHPool/* ONLY for testing and debugging */; + PoolUtil.addCapacity = async (n)=>SAHPool.addCapacity(n); + PoolUtil.reduceCapacity = async (n)=>SAHPool.reduceCapacity(n); + PoolUtil.getCapacity = SAHPool.getCapacity.bind(SAHPool); + PoolUtil.getActiveFileCount = SAHPool.getFileCount.bind(SAHPool); + PoolUtil.reserveMinimumCapacity = async (min)=>{ + const c = SAHPool.getCapacity(); + return (c < min) ? SAHPool.addCapacity(min - c) : c; + }; + + PoolUtil.exportFile = function(name){ + const sah = SAHPool.mapFilenameToSAH.get(name) || toss("File not found:",name); + const n = sah.getSize() - HEADER_OFFSET_DATA; + const b = new Uint8Array(n>=0 ? n : 0); + if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); + return b; + }; + + PoolUtil.importDb = function(name, bytes){ + const n = bytes.byteLength; + if(n<512 || n%512!=0){ + toss("Byte array size is invalid for an SQLite db."); + } + const header = "SQLite format 3"; + for(let i = 0; i < header.length; ++i){ + if( header.charCodeAt(i) !== bytes[i] ){ + toss("Input does not contain an SQLite database header."); + } + } + const sah = SAHPool.mapFilenameToSAH.get(name) + || SAHPool.nextAvailableSAH() + || toss("No available handles to import to."); + sah.write(bytes, {at: HEADER_OFFSET_DATA}); + SAHPool.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); + }; + + PoolUtil.wipeFiles = async ()=>SAHPool.reset(true); + PoolUtil.unlink = (filename)=>SAHPool.deletePath(filename); + + PoolUtil.removeVfs = async function(){ + if(!opfsVfs.pointer) return false; + capi.sqlite3_vfs_unregister(opfsVfs.pointer); + opfsVfs.dispose(); + try{ + SAHPool.releaseAccessHandles(); + if(SAHPool.parentDirHandle){ + await SAHPool.parentDirHandle.removeEntry( + SAHPool.dirHandle.name, {recursive: true} + ); + SAHPool.dirHandle = SAHPool.parentDirHandle = undefined; + } + }catch(e){ + warn("removeVfs() failed:",e); + /*but otherwise ignored*/ + } + return true; + }; + //log("vfs list:",capi.sqlite3_js_vfs_list()); sqlite3.vfs.installVfs({ io: {struct: opfsIoMethods, methods: ioMethods}, @@ -900,15 +923,20 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ //log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods); //log("vfs list:",capi.sqlite3_js_vfs_list()); if(sqlite3.oo1){ + const oo1 = sqlite3.oo1; const OpfsSAHPoolDb = function(...args){ - const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + const opt = oo1.DB.dbCtorHelper.normalizeArgs(...args); opt.vfs = opfsVfs.$zName; - sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + oo1.DB.dbCtorHelper.call(this, opt); }; - OpfsSAHPoolDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + OpfsSAHPoolDb.prototype = Object.create(oo1.DB.prototype); OpfsSAHPoolDb.PoolUtil = PoolUtil; - sqlite3.oo1.OpfsSAHPoolDb = OpfsSAHPoolDb; - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( + if(!oo1.OpfsSAHPoolDb){ + oo1.OpfsSAHPoolDb = Object.create(null); + oo1.OpfsSAHPoolDb.default = OpfsSAHPoolDb; + } + oo1.OpfsSAHPoolDb[vfsName] = OpfsSAHPoolDb; + oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, function(oo1Db, sqlite3){ sqlite3.capi.sqlite3_exec(oo1Db, [ @@ -920,7 +948,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ ); }/*extend sqlite3.oo1*/ log("VFS initialized."); - return promiseResolve(); + return PoolUtil; }).catch(promiseReject); }/*installOpfsSAHPoolVfs()*/; }/*sqlite3ApiBootstrap.initializers*/); diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index e223084887..dedc320292 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -56,18 +56,26 @@ "speedtest1.wasm", ...cliFlagsArray, dbFile ]; App.logBuffer.length = 0; + const ndxSahPool = argv.indexOf('opfs-sahpool'); + const realSahName = 'opfs-sahpool-speedtest1'; + if(ndxSahPool>0){ + argv[ndxSahPool] = realSahName; + log("Updated argv for opfs-sahpool: --vfs",realSahName); + } mPost('run-start', [...argv]); if(App.sqlite3.installOpfsSAHPoolVfs && !App.sqlite3.$SAHPoolUtil - && cliFlagsArray.indexOf('opfs-sahpool')>=0){ - log("Installing opfs-sahpool..."); + && ndxSahPool>0){ + log("Installing opfs-sahpool as",realSahName,"..."); await App.sqlite3.installOpfsSAHPoolVfs({ - directory: '.speedtest1-sahpool', + name: realSahName, initialCapacity: 3, - clearOnInit: true + clearOnInit: true, + verbosity: 2 }).then(PoolUtil=>{ - log("opfs-sahpool successfully installed."); + log("opfs-sahpool successfully installed as",realSahName); App.sqlite3.$SAHPoolUtil = PoolUtil; + //console.log("sqlite3.oo1.OpfsSAHPoolDb =", App.sqlite3.oo1.OpfsSAHPoolDb); }); } App.wasm.xCall('wasm_main', argv.length, diff --git a/manifest b/manifest index e205dc4824..aa2738ca4b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\s-DSQLITE_TEMP_STORE=3\sto\s2\schange\sfor\sthe\swasm\sbuild. -D 2023-07-17T07:43:04.661 +C Refactor\sopfs-sahpool\sto\ssupport\smultiple\sinstances,\seach\swith\sa\sseparate\sVFS\sname\sand\sdirectory. +D 2023-07-18T12:09:16.750 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js ff3495c0140787fc0232081582a6126f22b7b2e7894bee83061e8bf9eea2ef7f +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js a3307deb47d7d7a9a6e202a20b19252fa12fbeb60aeee11008ee0358a7137286 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 -F ext/wasm/speedtest1-worker.js 4f18caa941ed89d42af46c598e7f7fe31cecac853e0b370d235adcb19ce8cbee +F ext/wasm/speedtest1-worker.js 315d26198c46be7c85e26fda15d80ef882424276abde25ffd8b026fb02a35d8c F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2c9c95d087c7db2a975ffae15af5ade621411c493145b889f0c9157a55c4952a -R f9198174ee4df68f95986dfc1d71ac1a +P 5ad8f9987c11b5db67edd69a3471ff388140d6268d5d3f5a05dec6bb6d92ac05 +R a597bda217a200863fd7fdd91e2ff569 U stephan -Z 5f625ecf88ca9a37df1d1dfeecd67369 +Z 61da1feaecfdf3b26fae912b9de308b1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5528ba7f19..30ff3a470b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ad8f9987c11b5db67edd69a3471ff388140d6268d5d3f5a05dec6bb6d92ac05 \ No newline at end of file +d036eaf6ac60c576428db40f015733c5d5425f7d613194fd8d9d4d98659077c4 \ No newline at end of file From d69e6e6efb8a9204b332a188f05d05734f18f40d Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 18 Jul 2023 16:24:51 +0000 Subject: [PATCH 25/56] Major restructuring of the opfs-sahpool bits to better support multiple instances per app (each sandboxed from each other). Eliminate the pesky promise resolution race condition when two such instances are loaded in parallel. FossilOrigin-Name: 95e5fa498f71708caeb3394636c4853530a8b2d54406e503f32750732d6815d5 --- ext/wasm/api/sqlite3-v-helper.js | 2 +- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 1372 ++++++++++++---------- manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 732 insertions(+), 658 deletions(-) diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index 80ab7c5b04..cc9747aa5b 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -608,7 +608,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ This is to facilitate creation of those methods inline in the passed-in object without requiring the client to explicitly get a reference to one of them in order to assign it to the other - one. + one. The `catchExceptions`-installed handlers will account for identical references to the above functions and will install the diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index acba1d2d15..5ab58bef58 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -52,294 +52,105 @@ is not detected, the VFS is not registered. */ 'use strict'; -globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ -const toss = sqlite3.util.toss; -const isPromiseReady = Object.create(null); -const capi = sqlite3.capi; -// Config opts for the VFS... -const SECTOR_SIZE = 4096; -const HEADER_MAX_PATH_SIZE = 512; -const HEADER_FLAGS_SIZE = 4; -const HEADER_DIGEST_SIZE = 8; -const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE; -const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE; -const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; -const HEADER_OFFSET_DATA = SECTOR_SIZE; -/* Bitmask of file types which may persist across sessions. - SQLITE_OPEN_xyz types not listed here may be inadvertently - left in OPFS but are treated as transient by this VFS and - they will be cleaned up during VFS init. */ -const PERSISTENT_FILE_TYPES = - capi.SQLITE_OPEN_MAIN_DB | - capi.SQLITE_OPEN_MAIN_JOURNAL | - capi.SQLITE_OPEN_SUPER_JOURNAL | - capi.SQLITE_OPEN_WAL /* noting that WAL support is - unavailable in the WASM build.*/; - -/** - Returns short a string of random alphanumeric characters - suitable for use as a random filename. -*/ -const getRandomName = ()=>Math.random().toString(36).slice(2); - -/** - installOpfsSAHPoolVfs() asynchronously initializes the OPFS - SyncAccessHandle (a.k.a. SAH) Pool VFS. It returns a Promise which - either resolves to a utility object described below or rejects with - an Error value. - - Initialization of this VFS is not automatic because its - registration requires that it lock all resources it - will potentially use, even if client code does not want - to use them. That, in turn, can lead to locking errors - when, for example, one page in a given origin has loaded - this VFS but does not use it, then another page in that - origin tries to use the VFS. If the VFS were automatically - registered, the second page would fail to load the VFS - due to OPFS locking errors. - - If this function is called more than once with a given "name" - option (see below), it will return the same Promise, with one - exception: if called twice in immediate succession, the first will - not yet have had time to create its (cached) return Promise and the - second call will attempt to re-initialize the VFS, failing. (How to - resolve that race is unclear.) Calls for different names will - return different Promises which resolve to independent objects and - refer to different VFS registrations. - - On success, the resulting Promise resolves to a utility object - which can be used to query and manipulate the pool. Its API is - described at the end of these docs. - - This function accepts an options object to configure certain - parts but it is only acknowledged for the very first call and - ignored for all subsequent calls. - - The options, in alphabetical order: - - - `clearOnInit`: (default=false) if truthy, contents and filename - mapping are removed from each SAH it is acquired during - initalization of the VFS, leaving the VFS's storage in a pristine - state. Use this only for databases which need not survive a page - reload. - - - `initialCapacity`: (default=6) Specifies the default capacity of - the VFS. This should not be set unduly high because the VFS has - to open (and keep open) a file for each entry in the pool. This - setting only has an effect when the pool is initially empty. It - does not have any effect if a pool already exists. - - - `directory`: (default="."+`name`) Specifies the OPFS directory - name in which to store metadata for the `"opfs-sahpool"` - sqlite3_vfs. Only one instance of this VFS can be installed per - JavaScript engine, and any two engines with the same storage - directory name will collide with each other, leading to locking - errors and the inability to register the VFS in the second and - subsequent engine. Using a different directory name for each - application enables different engines in the same HTTP origin to - co-exist, but their data are invisible to each other. Changing - this name will effectively orphan any databases stored under - previous names. The default is unspecified but descriptive. This - option may contain multiple path elements, e.g. "foo/bar/baz", - and they are created automatically. In practice there should be - no driving need to change this. ACHTUNG: all files in this - directory are assumed to be managed by the VFS. Do not place - other files in that directory, as they may be deleted or - otherwise modified by the VFS. - - - `name`: (default="opfs-sahpool") sets the name to register this - VFS under. Normally this should not be changed, but it is - possible to register this VFS under multiple names so long as - each has its own separate directory to work from. The storage for - each is invisible to all others. The name must be a string - compatible with `sqlite3_vfs_register()` and friends and suitable - for use in URI-style database file names. - - Achtung: if a custom `name` is provided, a custom `directory` - must also be provided if any other instance is registered with - the default directory. If no directory is explicitly provided - then a directory name is synthesized from the `name` option. - - - The API for the utility object passed on by this function's - Promise, in alphabetical order... - -- [async] number addCapacity(n) - - Adds `n` entries to the current pool. This change is persistent - across sessions so should not be called automatically at each app - startup (but see `reserveMinimumCapacity()`). Its returned Promise - resolves to the new capacity. Because this operation is necessarily - asynchronous, the C-level VFS API cannot call this on its own as - needed. - -- byteArray exportFile(name) - - Synchronously reads the contents of the given file into a Uint8Array - and returns it. This will throw if the given name is not currently - in active use or on I/O error. Note that the given name is _not_ - visible directly in OPFS (or, if it is, it's not from this VFS). The - reason for that is that this VFS manages name-to-file mappings in - a roundabout way in order to maintain its list of SAHs. - -- number getCapacity() - - Returns the number of files currently contained - in the SAH pool. The default capacity is only large enough for one - or two databases and their associated temp files. - -- number getActiveFileCount() - - Returns the number of files from the pool currently in use. - -- void importDb(name, byteArray) - - Imports the contents of an SQLite database, provided as a byte - array, under the given name, overwriting any existing - content. Throws if the pool has no available file slots, on I/O - error, or if the input does not appear to be a database. In the - latter case, only a cursory examination is made. Note that this - routine is _only_ for importing database files, not arbitrary files, - the reason being that this VFS will automatically clean up any - non-database files so importing them is pointless. - -- [async] number reduceCapacity(n) - - Removes up to `n` entries from the pool, with the caveat that it can - only remove currently-unused entries. It returns a Promise which - resolves to the number of entries actually removed. - -- [async] boolean removeVfs() - - Unregisters the opfs-sahpool VFS and removes its directory from OPFS - (which means that _all client content_ is removed). After calling - this, the VFS may no longer be used and there is no way to re-add it - aside from reloading the current JavaScript context. - - Results are undefined if a database is currently in use with this - VFS. - - The returned Promise resolves to true if it performed the removal - and false if the VFS was not installed. - - If the VFS has a multi-level directory, e.g. "/foo/bar/baz", _only_ - the bottom-most directory is removed because this VFS cannot know for - certain whether the higher-level directories contain data which - should be removed. - -- [async] number reserveMinimumCapacity(min) - - If the current capacity is less than `min`, the capacity is - increased to `min`, else this returns with no side effects. The - resulting Promise resolves to the new capacity. - -- boolean unlink(filename) - - If a virtual file exists with the given name, disassociates it from - the pool and returns true, else returns false without side - effects. Results are undefined if the file is currently in active - use. - -- string vfsName - - The SQLite VFS name under which this pool's VFS is registered. - -- [async] void wipeFiles() - - Clears all client-defined state of all SAHs and makes all of them - available for re-use by the pool. Results are undefined if any such - handles are currently in use, e.g. by an sqlite3 db. - -*/ -sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ -/** The PoolUtil object will be the result of the - resolved Promise. */ - const PoolUtil = Object.create(null); - const vfsName = PoolUtil.vfsName = options.name || "opfs-sahpool"; - if(isPromiseReady[vfsName]){ - //console.warn("Returning same OpfsSAHPool result",vfsName,isPromiseReady[vfsName]); - return isPromiseReady[vfsName]; - } - if(!globalThis.FileSystemHandle || - !globalThis.FileSystemDirectoryHandle || - !globalThis.FileSystemFileHandle || - !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator?.storage?.getDirectory){ - return (isPromiseReady[vfsName] = Promise.reject(new Error("Missing required OPFS APIs."))); - } - const verbosity = options.verbosity - || 2 /*3+ == everything, 2 == warnings+errors, 1 == errors only*/; - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(verbosity>level) loggers[level](vfsName+":",...args); - }; - const log = (...args)=>logImpl(2, ...args); - const warn = (...args)=>logImpl(1, ...args); - const error = (...args)=>logImpl(0, ...args); +globalThis.sqlite3ApiBootstrap.initializersAsync.push(async function(sqlite3){ + const toss = sqlite3.util.toss; + const toss3 = sqlite3.util.toss3; + const initPromises = Object.create(null); + const capi = sqlite3.capi; const wasm = sqlite3.wasm; - const opfsIoMethods = new capi.sqlite3_io_methods(); - const opfsVfs = new capi.sqlite3_vfs() - .addOnDispose(()=>opfsIoMethods.dispose()); - - const promiseReject = (err)=>{ - error("rejecting promise:",err); - //opfsVfs.dispose(); - return isPromiseReady[vfsName] = Promise.reject(err); - }; - if( sqlite3.capi.sqlite3_vfs_find(vfsName)){ - return promiseReject(new Error("VFS name is already registered:", - vfsName)); - } - - /* We fetch the default VFS so that we can inherit some - methods from it. */ - const pDVfs = capi.sqlite3_vfs_find(null); - const dVfs = pDVfs - ? new capi.sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; - opfsIoMethods.$iVersion = 1; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; - opfsVfs.addOnDispose( - opfsVfs.$zName = wasm.allocCString(vfsName), - ()=>(dVfs ? dVfs.dispose() : null) - ); + // Config opts for the VFS... + const SECTOR_SIZE = 4096; + const HEADER_MAX_PATH_SIZE = 512; + const HEADER_FLAGS_SIZE = 4; + const HEADER_DIGEST_SIZE = 8; + const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE; + const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE; + const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; + const HEADER_OFFSET_DATA = SECTOR_SIZE; + /* Bitmask of file types which may persist across sessions. + SQLITE_OPEN_xyz types not listed here may be inadvertently + left in OPFS but are treated as transient by this VFS and + they will be cleaned up during VFS init. */ + const PERSISTENT_FILE_TYPES = + capi.SQLITE_OPEN_MAIN_DB | + capi.SQLITE_OPEN_MAIN_JOURNAL | + capi.SQLITE_OPEN_SUPER_JOURNAL | + capi.SQLITE_OPEN_WAL /* noting that WAL support is + unavailable in the WASM build.*/; /** - All state for the VFS. + Returns short a string of random alphanumeric characters + suitable for use as a random filename. */ - const SAHPool = Object.assign(Object.create(null),{ + const getRandomName = ()=>Math.random().toString(36).slice(2); + + const textDecoder = new TextDecoder(); + const textEncoder = new TextEncoder(); + + const optionDefaults = Object.assign(Object.create(null),{ + name: 'opfs-sahpool', + directory: undefined, + initialCapacity: 6, + clearOnInit: false, + verbosity: 2 /*3+ == everything, 2 == warnings+errors, 1 == errors only*/ + }); + + /** + Class for managing OPFS-related state for the + OPFS SharedAccessHandle Pool sqlite3_vfs. + */ + class OpfsSAHPool { /* OPFS dir in which VFS metadata is stored. */ - vfsDir: options.directory || ("."+vfsName), + vfsDir; /* Directory handle to this.vfsDir. */ - dirHandle: undefined, + dirHandle; /* Directory handle to this.dirHandle's parent dir. Needed for a VFS-wipe op. */ - parentDirHandle: undefined, + parentDirHandle; /* Maps SAHs to their opaque file names. */ - mapSAHToName: new Map(), + mapSAHToName = new Map(); /* Maps client-side file names to SAHs. */ - mapFilenameToSAH: new Map(), + mapFilenameToSAH = new Map(); /* Set of currently-unused SAHs. */ - availableSAH: new Set(), + availableSAH = new Set(); /* Maps (sqlite3_file*) to xOpen's file objects. */ - mapIdToFile: new Map(), + mapIdToFile = new Map(); + + /** Buffer used by [sg]etAssociatedPath(). */ + apBody = new Uint8Array(HEADER_CORPUS_SIZE); + + constructor(vfsObject, options = Object.create(null)){ + this.vfsName = options.name || optionDefaults.name; + if( sqlite3.capi.sqlite3_vfs_find(this.vfsName)){ + toss3("VFS name is already registered:", this.vfsName); + } + this.cVfs = vfsObject; + this.vfsDir = options.directory || ("."+this.vfsName); + this.dvBody = + new DataView(this.apBody.buffer, this.apBody.byteOffset); + this.isReady = this + .reset(!!(options.clearOnInit ?? optionDefaults.clearOnInit)) + .then(()=>{ + if(this.$error) throw this.$error; + return this.getCapacity() + ? Promise.resolve(undefined) + : this.addCapacity(options.initialCapacity + || optionDefaults.initialCapacity); + }); + } + /* Current pool capacity. */ - getCapacity: function(){return this.mapSAHToName.size}, + getCapacity(){return this.mapSAHToName.size} + /* Current number of in-use files from pool. */ - getFileCount: function(){return this.mapFilenameToSAH.size}, + getFileCount(){return this.mapFilenameToSAH.size} + /** Adds n files to the pool's capacity. This change is persistent across settings. Returns a Promise which resolves to the new capacity. */ - addCapacity: async function(n){ + async addCapacity(n){ for(let i = 0; i < n; ++i){ const name = getRandomName(); const h = await this.dirHandle.getFileHandle(name, {create:true}); @@ -348,14 +159,14 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ this.setAssociatedPath(ah, '', 0); } return this.getCapacity(); - }, + } /** Removes n entries from the pool's current capacity if possible. It can only remove currently-unallocated files. Returns a Promise resolving to the number of removed files. */ - reduceCapacity: async function(n){ + async reduceCapacity(n){ let nRm = 0; for(const ah of Array.from(this.availableSAH)){ if(nRm === n || this.getFileCount() === this.getCapacity()){ @@ -369,16 +180,18 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ ++nRm; } return nRm; - }, + } + /** Releases all currently-opened SAHs. */ - releaseAccessHandles: function(){ + releaseAccessHandles(){ for(const ah of this.mapSAHToName.keys()) ah.close(); this.mapSAHToName.clear(); this.mapFilenameToSAH.clear(); this.availableSAH.clear(); - }, + } + /** Opens all files under this.vfsDir/this.dirHandle and acquires a SAH for each. returns a Promise which resolves to no value @@ -386,19 +199,18 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ throws, SAHPool.$error will contain the corresponding exception. - If clearFiles is true, the client-stored state of each file is cleared when its handle is acquired, including its name, flags, and any data stored after the metadata block. */ - acquireAccessHandles: async function(clearFiles){ + async acquireAccessHandles(clearFiles){ const files = []; for await (const [name,h] of this.dirHandle){ if('file'===h.kind){ files.push([name,h]); } } - await Promise.all(files.map(async ([name,h])=>{ + return Promise.all(files.map(async([name,h])=>{ try{ const ah = await h.createSyncAccessHandle() this.mapSAHToName.set(ah, name); @@ -414,16 +226,13 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ } } }catch(e){ - SAHPool.storeErr(e); + this.storeErr(e); this.releaseAccessHandles(); throw e; } })); - }, - /** Buffer used by [sg]etAssociatedPath(). */ - apBody: new Uint8Array(HEADER_CORPUS_SIZE), - textDecoder: new TextDecoder(), - textEncoder: new TextEncoder(), + } + /** Given an SAH, returns the client-specified name of that file by extracting it from the SAH's header. @@ -431,7 +240,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ On error, it disassociates SAH from the pool and returns an empty string. */ - getAssociatedPath: function(sah){ + getAssociatedPath(sah){ sah.read(this.apBody, {at: 0}); // Delete any unexpected files left over by previous // untimely errors... @@ -440,7 +249,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ ((flags & capi.SQLITE_OPEN_DELETEONCLOSE) || (flags & PERSISTENT_FILE_TYPES)===0)){ warn(`Removing file with unexpected flags ${flags.toString(16)}`, - this.apBody); + this.apBody); this.setAssociatedPath(sah, '', 0); return ''; } @@ -457,7 +266,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ sah.truncate(HEADER_OFFSET_DATA); } return pathBytes - ? this.textDecoder.decode(this.apBody.subarray(0,pathBytes)) + ? textDecoder.decode(this.apBody.subarray(0,pathBytes)) : ''; }else{ // Invalid digest @@ -465,13 +274,16 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ this.setAssociatedPath(sah, '', 0); return ''; } - }, + } + /** - Stores the given client-defined path and SQLITE_OPEN_xyz - flags into the given SAH. + Stores the given client-defined path and SQLITE_OPEN_xyz flags + into the given SAH. If path is an empty string then the file is + disassociated from the pool but its previous name is preserved + in the metadata. */ - setAssociatedPath: function(sah, path, flags){ - const enc = this.textEncoder.encodeInto(path || '\0', this.apBody); + setAssociatedPath(sah, path, flags){ + const enc = textEncoder.encodeInto(path, this.apBody); if(HEADER_MAX_PATH_SIZE <= enc.written){ toss("Path too long:",path); } @@ -491,12 +303,13 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ sah.truncate(HEADER_OFFSET_DATA); this.availableSAH.add(sah); } - }, + } + /** Computes a digest for the given byte array and returns it as a two-element Uint32Array. */ - computeDigest: function(byteArray){ + computeDigest(byteArray){ let h1 = 0xdeadbeef; let h2 = 0x41c6ce57; for(const v of byteArray){ @@ -504,7 +317,8 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ h2 = 31 * h2 + (v * 307); } return new Uint32Array([h1>>>0, h2>>>0]); - }, + } + /** Re-initializes the state of the SAH pool, releasing and re-acquiring all handles. @@ -512,8 +326,8 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ See acquireAccessHandles() for the specifics of the clearFiles argument. */ - reset: async function(clearFiles){ - await isPromiseReady[vfsName]; + async reset(clearFiles){ + await this.isReady; let h = await navigator.storage.getDirectory(); let prev, prevName; for(const d of this.vfsDir.split('/')){ @@ -525,8 +339,9 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ this.dirHandle = h; this.parentDirHandle = prev; this.releaseAccessHandles(); - await this.acquireAccessHandles(clearFiles); - }, + return this.acquireAccessHandles(clearFiles); + } + /** Returns the pathname part of the given argument, which may be any of: @@ -535,18 +350,19 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ - A JS string representing a file name - Wasm C-string representing a file name */ - getPath: function(arg) { + getPath(arg) { if(wasm.isPtr(arg)) arg = wasm.cstrToJs(arg); return ((arg instanceof URL) ? arg : new URL(arg, 'file://localhost/')).pathname; - }, + } + /** Removes the association of the given client-specified file name (JS string) from the pool. Returns true if a mapping is found, else false. */ - deletePath: function(path) { + deletePath(path) { const sah = this.mapFilenameToSAH.get(path); if(sah) { // Un-associate the name from the SAH. @@ -554,328 +370,76 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ this.setAssociatedPath(sah, '', 0); } return !!sah; - }, + } + /** Sets e as this object's current error. Pass a falsy (or no) value to clear it. */ - storeErr: function(e){ - if(e) error(e); - return this.$error = e; - }, + storeErr(e){return this.$error = e;} /** Pops this object's Error object and returns it (a falsy value if no error is set). */ - popErr: function(){ + popErr(){ const rc = this.$error; this.$error = undefined; return rc; - }, - nextAvailableSAH: function(){ + } + + /** + Returns the next available SAH without removing + it from the set. + */ + nextAvailableSAH(){ const [rc] = this.availableSAH.keys(); return rc; } - })/*SAHPool*/; - SAHPool.dvBody = - new DataView(SAHPool.apBody.buffer, SAHPool.apBody.byteOffset); - //sqlite3.SAHPool = SAHPool/*only for testing*/; + }/*class OpfsSAHPool*/; + /** - Ensure that the client has a "fully-sync" SAH impl, - else reject the promise. Returns true on success, - else a value intended to be returned via the containing - function's Promise result. + A SAHPoolUtil instance is exposed to clients in order to + manipulate an OpfsSAHPool object without directly exposing that + object and allowing for some semantic changes compared to that + class. */ - const apiVersionCheck = await (async ()=>{ - try { - const dh = await navigator.storage.getDirectory(); - const fn = '.opfs-sahpool-sync-check-'+getRandomName(); - const fh = await dh.getFileHandle(fn, { create: true }); - const ah = await fh.createSyncAccessHandle(); - const close = ah.close(); - await close; - await dh.removeEntry(fn); - if(close?.then){ - toss("The local OPFS API is too old for opfs-sahpool:", - "it has an async FileSystemSyncAccessHandle.close() method."); - } - return true; - }catch(e){ - return e; - } - })(); - if(true!==apiVersionCheck){ - return promiseReject(apiVersionCheck); - } + class SAHPoolUtil { - return isPromiseReady[vfsName] = SAHPool.reset(!!options.clearOnInit).then(async ()=>{ - if(SAHPool.$error){ - throw SAHPool.$error; - } - if(0===SAHPool.getCapacity()){ - await SAHPool.addCapacity(options.initialCapacity || 6); - } - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioMethods = { - xCheckReservedLock: function(pFile,pOut){ - log('xCheckReservedLock'); - SAHPool.storeErr(); - wasm.poke32(pOut, 1); - return 0; - }, - xClose: function(pFile){ - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - if(file) { - try{ - log(`xClose ${file.path}`); - if(file.sq3File) file.sq3File.dispose(); - file.sah.flush(); - SAHPool.mapIdToFile.delete(pFile); - if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ - SAHPool.deletePath(file.path); - } - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - } - return 0; - }, - xDeviceCharacteristics: function(pFile){ - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - log(`xFileSize`); - const file = SAHPool.mapIdToFile.get(pFile); - const size = file.sah.getSize() - HEADER_OFFSET_DATA; - //log(`xFileSize ${file.path} ${size}`); - wasm.poke64(pSz64, BigInt(size)); - return 0; - }, - xLock: function(pFile,lockType){ - log(`xLock ${lockType}`); - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - file.lockType = lockType; - return 0; - }, - xRead: function(pFile,pDest,n,offset64){ - log(`xRead ${n}@${offset64}`); - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - log(`xRead ${file.path} ${n} ${offset64}`); - try { - const nRead = file.sah.read( - wasm.heap8u().subarray(pDest, pDest+n), - {at: HEADER_OFFSET_DATA + Number(offset64)} - ); - if(nRead < n){ - wasm.heap8u().fill(0, pDest + nRead, pDest + n); - return capi.SQLITE_IOERR_SHORT_READ; - } - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - }, - xSectorSize: function(pFile){ - return SECTOR_SIZE; - }, - xSync: function(pFile,flags){ - log(`xSync ${flags}`); - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - //log(`xSync ${file.path} ${flags}`); - try{ - file.sah.flush(); - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - }, - xTruncate: function(pFile,sz64){ - log(`xTruncate ${sz64}`); - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - //log(`xTruncate ${file.path} ${iSize}`); - try{ - file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - }, - xUnlock: function(pFile,lockType){ - log('xUnlock'); - const file = SAHPool.mapIdToFile.get(pFile); - file.lockType = lockType; - return 0; - }, - xWrite: function(pFile,pSrc,n,offset64){ - SAHPool.storeErr(); - const file = SAHPool.mapIdToFile.get(pFile); - log(`xWrite ${file.path} ${n} ${offset64}`); - try{ - const nBytes = file.sah.write( - wasm.heap8u().subarray(pSrc, pSrc+n), - { at: HEADER_OFFSET_DATA + Number(offset64) } - ); - return nBytes === n ? 0 : capi.SQLITE_IOERR; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR; - } - } - }/*ioMethods*/; - - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsMethods = { - xAccess: function(pVfs,zName,flags,pOut){ - log(`xAccess ${wasm.cstrToJs(zName)}`); - SAHPool.storeErr(); - try{ - const name = this.getPath(zName); - wasm.poke32(pOut, SAHPool.mapFilenameToSAH.has(name) ? 1 : 0); - }catch(e){ - /*ignored*/; - } - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - log(`xDelete ${wasm.cstrToJs(zName)}`); - SAHPool.storeErr(); - try{ - SAHPool.deletePath(SAHPool.getPath(zName)); - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_IOERR_DELETE; - } - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - log(`xFullPathname ${wasm.cstrToJs(zName)}`); - const i = wasm.cstrncpy(pOut, zName, nOut); - return i nOut) wasm.poke8(pOut + nOut - 1, 0); - }catch(e){ - return capi.SQLITE_NOMEM; - }finally{ - wasm.scopedAllocPop(scope); - } - } - return 0; - }, - //xSleep is optionally defined below - xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ - log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`); - try{ - // First try to open a path that already exists in the file system. - const path = (zName && wasm.peek8(zName)) - ? SAHPool.getPath(zName) - : getRandomName(); - let sah = SAHPool.mapFilenameToSAH.get(path); - if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { - // File not found so try to create it. - if(SAHPool.getFileCount() < SAHPool.getCapacity()) { - // Choose an unassociated OPFS file from the pool. - sah = SAHPool.nextAvailableSAH(); - SAHPool.setAssociatedPath(sah, path, flags); - }else{ - // File pool is full. - toss('SAH pool is full. Cannot create file',path); - } - } - if(!sah){ - toss('file not found:',path); - } - // Subsequent methods are only passed the file pointer, so - // map the relevant info we need to that pointer. - const file = {path, flags, sah}; - SAHPool.mapIdToFile.set(pFile, file); - wasm.poke32(pOutFlags, flags); - file.sq3File = new capi.sqlite3_file(pFile); - file.sq3File.$pMethods = opfsIoMethods.pointer; - file.lockType = capi.SQLITE_LOCK_NONE; - return 0; - }catch(e){ - SAHPool.storeErr(e); - return capi.SQLITE_CANTOPEN; - } - }/*xOpen()*/ - }/*vfsMethods*/; - - if(dVfs){ - /* Inherit certain VFS members from the default VFS, - if available. */ - opfsVfs.$xRandomness = dVfs.$xRandomness; - opfsVfs.$xSleep = dVfs.$xSleep; - } - if(!opfsVfs.$xRandomness){ - /* If the default VFS has no xRandomness(), add a basic JS impl... */ - vfsMethods.xRandomness = function(pVfs, nOut, pOut){ - const heap = wasm.heap8u(); - let i = 0; - for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; - return i; - }; - } - if(!opfsVfs.$xSleep){ - vfsMethods.xSleep = (pVfs,ms)=>0; + constructor(sahPool){ + /* TODO: move the this-to-sahPool mapping into an external + WeakMap so as to not expose it to downstream clients. */ + this.$p = sahPool; + this.vfsName = sahPool.vfsName; } - PoolUtil.$SAHPool = SAHPool/* ONLY for testing and debugging */; - PoolUtil.addCapacity = async (n)=>SAHPool.addCapacity(n); - PoolUtil.reduceCapacity = async (n)=>SAHPool.reduceCapacity(n); - PoolUtil.getCapacity = SAHPool.getCapacity.bind(SAHPool); - PoolUtil.getActiveFileCount = SAHPool.getFileCount.bind(SAHPool); - PoolUtil.reserveMinimumCapacity = async (min)=>{ - const c = SAHPool.getCapacity(); - return (c < min) ? SAHPool.addCapacity(min - c) : c; - }; + addCapacity = async function(n){ + return this.$p.addCapacity(n); + } + reduceCapacity = async function(n){ + return this.$p.reduceCapacity(n); + } + getCapacity = function(){ + return this.$p.getCapacity(this.$p); + } + getActiveFileCount = function(){ + return this.$p.getFileCount(); + } + reserveMinimumCapacity = async function(min){ + const c = this.$p.getCapacity(); + return (c < min) ? this.$p.addCapacity(min - c) : c; + } - PoolUtil.exportFile = function(name){ - const sah = SAHPool.mapFilenameToSAH.get(name) || toss("File not found:",name); + exportFile = function(name){ + const sah = this.$p.mapFilenameToSAH.get(name) || toss("File not found:",name); const n = sah.getSize() - HEADER_OFFSET_DATA; const b = new Uint8Array(n>=0 ? n : 0); if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); return b; - }; + } - PoolUtil.importDb = function(name, bytes){ + importDb = function(name, bytes){ const n = bytes.byteLength; if(n<512 || n%512!=0){ toss("Byte array size is invalid for an SQLite db."); @@ -886,69 +450,579 @@ sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ toss("Input does not contain an SQLite database header."); } } - const sah = SAHPool.mapFilenameToSAH.get(name) - || SAHPool.nextAvailableSAH() + const sah = this.$p.mapFilenameToSAH.get(name) + || this.$p.nextAvailableSAH() || toss("No available handles to import to."); sah.write(bytes, {at: HEADER_OFFSET_DATA}); - SAHPool.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); - }; + this.$p.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); + } - PoolUtil.wipeFiles = async ()=>SAHPool.reset(true); - PoolUtil.unlink = (filename)=>SAHPool.deletePath(filename); + wipeFiles = async function(){ + return this.$p.reset(true); + } - PoolUtil.removeVfs = async function(){ - if(!opfsVfs.pointer) return false; - capi.sqlite3_vfs_unregister(opfsVfs.pointer); - opfsVfs.dispose(); + unlink = function(filename){ + return this.$p.deletePath(filename); + } + + removeVfs = async function(){ + if(!this.$p.cVfs.pointer) return false; + capi.sqlite3_vfs_unregister(this.$p.cVfs.pointer); + this.$p.cVfs.dispose(); try{ - SAHPool.releaseAccessHandles(); - if(SAHPool.parentDirHandle){ - await SAHPool.parentDirHandle.removeEntry( - SAHPool.dirHandle.name, {recursive: true} + this.$p.releaseAccessHandles(); + if(this.$p.parentDirHandle){ + await this.$p.parentDirHandle.removeEntry( + this.$p.dirHandle.name, {recursive: true} ); - SAHPool.dirHandle = SAHPool.parentDirHandle = undefined; + this.$p.dirHandle = this.$p.parentDirHandle = undefined; } }catch(e){ - warn("removeVfs() failed:",e); - /*but otherwise ignored*/ + console.error(this.$p.vfsName,"removeVfs() failed:",e); + /*otherwise ignored - there is no recovery strategy*/ } return true; + } + + }/* class SAHPoolUtil */; + + /** + Ensure that the client has a "fully-sync" SAH impl, + else reject the promise. Returns true on success, + throws on error. + */ + const apiVersionCheck = async ()=>{ + const dh = await navigator.storage.getDirectory(); + const fn = '.opfs-sahpool-sync-check-'+getRandomName(); + const fh = await dh.getFileHandle(fn, { create: true }); + const ah = await fh.createSyncAccessHandle(); + const close = ah.close(); + await close; + await dh.removeEntry(fn); + if(close?.then){ + toss("The local OPFS API is too old for opfs-sahpool:", + "it has an async FileSystemSyncAccessHandle.close() method."); + } + return true; + }; + + /** Only for testing a rejection case. */ + let instanceCounter = 0; + + /** + installOpfsSAHPoolVfs() asynchronously initializes the OPFS + SyncAccessHandle (a.k.a. SAH) Pool VFS. It returns a Promise which + either resolves to a utility object described below or rejects with + an Error value. + + Initialization of this VFS is not automatic because its + registration requires that it lock all resources it + will potentially use, even if client code does not want + to use them. That, in turn, can lead to locking errors + when, for example, one page in a given origin has loaded + this VFS but does not use it, then another page in that + origin tries to use the VFS. If the VFS were automatically + registered, the second page would fail to load the VFS + due to OPFS locking errors. + + If this function is called more than once with a given "name" + option (see below), it will return the same Promise. Calls for + different names will return different Promises which resolve to + independent objects and refer to different VFS registrations. + + On success, the resulting Promise resolves to a utility object + which can be used to query and manipulate the pool. Its API is + described at the end of these docs. + + This function accepts an options object to configure certain + parts but it is only acknowledged for the very first call and + ignored for all subsequent calls. + + The options, in alphabetical order: + + - `clearOnInit`: (default=false) if truthy, contents and filename + mapping are removed from each SAH it is acquired during + initalization of the VFS, leaving the VFS's storage in a pristine + state. Use this only for databases which need not survive a page + reload. + + - `initialCapacity`: (default=6) Specifies the default capacity of + the VFS. This should not be set unduly high because the VFS has + to open (and keep open) a file for each entry in the pool. This + setting only has an effect when the pool is initially empty. It + does not have any effect if a pool already exists. + + - `directory`: (default="."+`name`) Specifies the OPFS directory + name in which to store metadata for the `"opfs-sahpool"` + sqlite3_vfs. Only one instance of this VFS can be installed per + JavaScript engine, and any two engines with the same storage + directory name will collide with each other, leading to locking + errors and the inability to register the VFS in the second and + subsequent engine. Using a different directory name for each + application enables different engines in the same HTTP origin to + co-exist, but their data are invisible to each other. Changing + this name will effectively orphan any databases stored under + previous names. The default is unspecified but descriptive. This + option may contain multiple path elements, e.g. "foo/bar/baz", + and they are created automatically. In practice there should be + no driving need to change this. ACHTUNG: all files in this + directory are assumed to be managed by the VFS. Do not place + other files in that directory, as they may be deleted or + otherwise modified by the VFS. + + - `name`: (default="opfs-sahpool") sets the name to register this + VFS under. Normally this should not be changed, but it is + possible to register this VFS under multiple names so long as + each has its own separate directory to work from. The storage for + each is invisible to all others. The name must be a string + compatible with `sqlite3_vfs_register()` and friends and suitable + for use in URI-style database file names. + + Achtung: if a custom `name` is provided, a custom `directory` + must also be provided if any other instance is registered with + the default directory. If no directory is explicitly provided + then a directory name is synthesized from the `name` option. + + + The API for the utility object passed on by this function's + Promise, in alphabetical order... + + - [async] number addCapacity(n) + + Adds `n` entries to the current pool. This change is persistent + across sessions so should not be called automatically at each app + startup (but see `reserveMinimumCapacity()`). Its returned Promise + resolves to the new capacity. Because this operation is necessarily + asynchronous, the C-level VFS API cannot call this on its own as + needed. + + - byteArray exportFile(name) + + Synchronously reads the contents of the given file into a Uint8Array + and returns it. This will throw if the given name is not currently + in active use or on I/O error. Note that the given name is _not_ + visible directly in OPFS (or, if it is, it's not from this VFS). The + reason for that is that this VFS manages name-to-file mappings in + a roundabout way in order to maintain its list of SAHs. + + - number getCapacity() + + Returns the number of files currently contained + in the SAH pool. The default capacity is only large enough for one + or two databases and their associated temp files. + + - number getActiveFileCount() + + Returns the number of files from the pool currently in use. + + - void importDb(name, byteArray) + + Imports the contents of an SQLite database, provided as a byte + array, under the given name, overwriting any existing + content. Throws if the pool has no available file slots, on I/O + error, or if the input does not appear to be a database. In the + latter case, only a cursory examination is made. Note that this + routine is _only_ for importing database files, not arbitrary files, + the reason being that this VFS will automatically clean up any + non-database files so importing them is pointless. + + - [async] number reduceCapacity(n) + + Removes up to `n` entries from the pool, with the caveat that it can + only remove currently-unused entries. It returns a Promise which + resolves to the number of entries actually removed. + + - [async] boolean removeVfs() + + Unregisters the opfs-sahpool VFS and removes its directory from OPFS + (which means that _all client content_ is removed). After calling + this, the VFS may no longer be used and there is no way to re-add it + aside from reloading the current JavaScript context. + + Results are undefined if a database is currently in use with this + VFS. + + The returned Promise resolves to true if it performed the removal + and false if the VFS was not installed. + + If the VFS has a multi-level directory, e.g. "/foo/bar/baz", _only_ + the bottom-most directory is removed because this VFS cannot know for + certain whether the higher-level directories contain data which + should be removed. + + - [async] number reserveMinimumCapacity(min) + + If the current capacity is less than `min`, the capacity is + increased to `min`, else this returns with no side effects. The + resulting Promise resolves to the new capacity. + + - boolean unlink(filename) + + If a virtual file exists with the given name, disassociates it from + the pool and returns true, else returns false without side + effects. Results are undefined if the file is currently in active + use. + + - string vfsName + + The SQLite VFS name under which this pool's VFS is registered. + + - [async] void wipeFiles() + + Clears all client-defined state of all SAHs and makes all of them + available for re-use by the pool. Results are undefined if any such + handles are currently in use, e.g. by an sqlite3 db. + */ + sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ + const vfsName = options.name || optionDefaults.name; + if(0 && 2===++instanceCounter){ + throw new Error("Just testing rejection."); + } + if(initPromises[vfsName]){ + //console.warn("Returning same OpfsSAHPool result",vfsName,initPromises[vfsName]); + return initPromises[vfsName]; + } + if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ + return (initPromises[vfsName] = Promise.reject(new Error("Missing required OPFS APIs."))); + } + const verbosity = options.verbosity ?? optionDefaults.verbosity; + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const logImpl = (level,...args)=>{ + if(verbosity>level) loggers[level](vfsName+":",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const opfsIoMethods = new capi.sqlite3_io_methods(); + const opfsVfs = new capi.sqlite3_vfs() + .addOnDispose(()=>opfsIoMethods.dispose()); + + const promiseReject = (err)=>{ + error("rejecting promise:",err); + opfsVfs.dispose(); + initPromises[vfsName] = Promise.reject(err); + throw err; }; - //log("vfs list:",capi.sqlite3_js_vfs_list()); - sqlite3.vfs.installVfs({ - io: {struct: opfsIoMethods, methods: ioMethods}, - vfs: {struct: opfsVfs, methods: vfsMethods} - }); - //log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods); - //log("vfs list:",capi.sqlite3_js_vfs_list()); - if(sqlite3.oo1){ - const oo1 = sqlite3.oo1; - const OpfsSAHPoolDb = function(...args){ - const opt = oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; - oo1.DB.dbCtorHelper.call(this, opt); - }; - OpfsSAHPoolDb.prototype = Object.create(oo1.DB.prototype); - OpfsSAHPoolDb.PoolUtil = PoolUtil; - if(!oo1.OpfsSAHPoolDb){ - oo1.OpfsSAHPoolDb = Object.create(null); - oo1.OpfsSAHPoolDb.default = OpfsSAHPoolDb; - } - oo1.OpfsSAHPoolDb[vfsName] = OpfsSAHPoolDb; - oo1.DB.dbCtorHelper.setVfsPostOpenSql( - opfsVfs.pointer, - function(oo1Db, sqlite3){ - sqlite3.capi.sqlite3_exec(oo1Db, [ - /* See notes in sqlite3-vfs-opfs.js */ - "pragma journal_mode=DELETE;", - "pragma cache_size=-16384;" - ], 0, 0, 0); + /* We fetch the default VFS so that we can inherit some + methods from it. */ + const pDVfs = capi.sqlite3_vfs_find(null); + const dVfs = pDVfs + ? new capi.sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. */; + opfsIoMethods.$iVersion = 1; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; + opfsVfs.addOnDispose( + opfsVfs.$zName = wasm.allocCString(vfsName), + ()=>(dVfs ? dVfs.dispose() : null) + ); + + /** + Maintenance reminder: the order of ASYNC ops in this function + is significant. We need to have them all chained at the very + end in order to be able to catch a race condition where + installOpfsSAHPoolVfs() is called twice in rapid succession, + e.g.: + + installOpfsSAHPoolVfs().then(console.warn.bind(console)); + installOpfsSAHPoolVfs().then(console.warn.bind(console)); + + If the timing of the async calls is not "just right" then that + second call can end up triggering the init a second time and chaos + ensues. + */ + return initPromises[vfsName] = apiVersionCheck().then(async function(){ + const thePool = new OpfsSAHPool(opfsVfs, options); + return thePool.isReady.then(async()=>{ + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioMethods = { + xCheckReservedLock: function(pFile,pOut){ + log('xCheckReservedLock'); + thePool.storeErr(); + wasm.poke32(pOut, 1); + return 0; + }, + xClose: function(pFile){ + thePool.storeErr(); + const file = thePool.mapIdToFile.get(pFile); + if(file) { + try{ + log(`xClose ${file.path}`); + if(file.sq3File) file.sq3File.dispose(); + file.sah.flush(); + thePool.mapIdToFile.delete(pFile); + if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ + thePool.deletePath(file.path); + } + }catch(e){ + thePool.storeErr(e); + return capi.SQLITE_IOERR; + } + } + return 0; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + log(`xFileSize`); + const file = thePool.mapIdToFile.get(pFile); + const size = file.sah.getSize() - HEADER_OFFSET_DATA; + //log(`xFileSize ${file.path} ${size}`); + wasm.poke64(pSz64, BigInt(size)); + return 0; + }, + xLock: function(pFile,lockType){ + log(`xLock ${lockType}`); + thePool.storeErr(); + const file = thePool.mapIdToFile.get(pFile); + file.lockType = lockType; + return 0; + }, + xRead: function(pFile,pDest,n,offset64){ + log(`xRead ${n}@${offset64}`); + thePool.storeErr(); + const file = thePool.mapIdToFile.get(pFile); + log(`xRead ${file.path} ${n} ${offset64}`); + try { + const nRead = file.sah.read( + wasm.heap8u().subarray(pDest, pDest+n), + {at: HEADER_OFFSET_DATA + Number(offset64)} + ); + if(nRead < n){ + wasm.heap8u().fill(0, pDest + nRead, pDest + n); + return capi.SQLITE_IOERR_SHORT_READ; + } + return 0; + }catch(e){ + thePool.storeErr(e); + return capi.SQLITE_IOERR; + } + }, + xSectorSize: function(pFile){ + return SECTOR_SIZE; + }, + xSync: function(pFile,flags){ + log(`xSync ${flags}`); + thePool.storeErr(); + const file = thePool.mapIdToFile.get(pFile); + //log(`xSync ${file.path} ${flags}`); + try{ + file.sah.flush(); + return 0; + }catch(e){ + thePool.storeErr(e); + return capi.SQLITE_IOERR; + } + }, + xTruncate: function(pFile,sz64){ + log(`xTruncate ${sz64}`); + thePool.storeErr(); + const file = thePool.mapIdToFile.get(pFile); + //log(`xTruncate ${file.path} ${iSize}`); + try{ + file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); + return 0; + }catch(e){ + thePool.storeErr(e); + return capi.SQLITE_IOERR; + } + }, + xUnlock: function(pFile,lockType){ + log('xUnlock'); + const file = thePool.mapIdToFile.get(pFile); + file.lockType = lockType; + return 0; + }, + xWrite: function(pFile,pSrc,n,offset64){ + thePool.storeErr(); + const file = thePool.mapIdToFile.get(pFile); + log(`xWrite ${file.path} ${n} ${offset64}`); + try{ + const nBytes = file.sah.write( + wasm.heap8u().subarray(pSrc, pSrc+n), + { at: HEADER_OFFSET_DATA + Number(offset64) } + ); + return nBytes === n ? 0 : capi.SQLITE_IOERR; + }catch(e){ + thePool.storeErr(e); + return capi.SQLITE_IOERR; + } + } + }/*ioMethods*/; + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsMethods = { + xAccess: function(pVfs,zName,flags,pOut){ + log(`xAccess ${wasm.cstrToJs(zName)}`); + thePool.storeErr(); + try{ + const name = this.getPath(zName); + wasm.poke32(pOut, thePool.mapFilenameToSAH.has(name) ? 1 : 0); + }catch(e){ + /*ignored*/; + } + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + log(`xDelete ${wasm.cstrToJs(zName)}`); + thePool.storeErr(); + try{ + thePool.deletePath(thePool.getPath(zName)); + return 0; + }catch(e){ + thePool.storeErr(e); + return capi.SQLITE_IOERR_DELETE; + } + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + log(`xFullPathname ${wasm.cstrToJs(zName)}`); + const i = wasm.cstrncpy(pOut, zName, nOut); + return i nOut) wasm.poke8(pOut + nOut - 1, 0); + }catch(e){ + return capi.SQLITE_NOMEM; + }finally{ + wasm.scopedAllocPop(scope); + } + } + return 0; + }, + //xSleep is optionally defined below + xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ + log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`); + try{ + // First try to open a path that already exists in the file system. + const path = (zName && wasm.peek8(zName)) + ? thePool.getPath(zName) + : getRandomName(); + let sah = thePool.mapFilenameToSAH.get(path); + if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { + // File not found so try to create it. + if(thePool.getFileCount() < thePool.getCapacity()) { + // Choose an unassociated OPFS file from the pool. + sah = thePool.nextAvailableSAH(); + thePool.setAssociatedPath(sah, path, flags); + }else{ + // File pool is full. + toss('SAH pool is full. Cannot create file',path); + } + } + if(!sah){ + toss('file not found:',path); + } + // Subsequent methods are only passed the file pointer, so + // map the relevant info we need to that pointer. + const file = {path, flags, sah}; + thePool.mapIdToFile.set(pFile, file); + wasm.poke32(pOutFlags, flags); + file.sq3File = new capi.sqlite3_file(pFile); + file.sq3File.$pMethods = opfsIoMethods.pointer; + file.lockType = capi.SQLITE_LOCK_NONE; + return 0; + }catch(e){ + thePool.storeErr(e); + return capi.SQLITE_CANTOPEN; + } + }/*xOpen()*/ + }/*vfsMethods*/; + + if(dVfs){ + /* Inherit certain VFS members from the default VFS, + if available. */ + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; } - ); - }/*extend sqlite3.oo1*/ - log("VFS initialized."); - return PoolUtil; - }).catch(promiseReject); -}/*installOpfsSAHPoolVfs()*/; -}/*sqlite3ApiBootstrap.initializers*/); + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + vfsMethods.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + vfsMethods.xSleep = (pVfs,ms)=>0; + } + + /** The poolUtil object will be the result of the + resolved Promise. */ + const poolUtil = new SAHPoolUtil(thePool); + + //log("vfs list:",capi.sqlite3_js_vfs_list()); + sqlite3.vfs.installVfs({ + io: {struct: opfsIoMethods, methods: ioMethods}, + vfs: {struct: opfsVfs, methods: vfsMethods} + }); + //log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods); + //log("vfs list:",capi.sqlite3_js_vfs_list()); + if(sqlite3.oo1){ + const oo1 = sqlite3.oo1; + const OpfsthePoolDb = function(...args){ + const opt = oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = opfsVfs.$zName; + oo1.DB.dbCtorHelper.call(this, opt); + }; + OpfsthePoolDb.prototype = Object.create(oo1.DB.prototype); + OpfsthePoolDb.PoolUtil = poolUtil; + if(!oo1.OpfsthePoolDb){ + oo1.OpfsthePoolDb = Object.create(null); + oo1.OpfsthePoolDb.default = OpfsthePoolDb; + } + oo1.OpfsthePoolDb[vfsName] = OpfsthePoolDb; + oo1.DB.dbCtorHelper.setVfsPostOpenSql( + opfsVfs.pointer, + function(oo1Db, sqlite3){ + sqlite3.capi.sqlite3_exec(oo1Db, [ + /* See notes in sqlite3-vfs-opfs.js */ + "pragma journal_mode=DELETE;", + "pragma cache_size=-16384;" + ], 0, 0, 0); + } + ); + }/*extend sqlite3.oo1*/ + log("VFS initialized."); + return poolUtil; + }); + }).catch(promiseReject); + }/*installOpfsSAHPoolVfs()*/; +}/*sqlite3ApiBootstrap.initializersAsync*/); diff --git a/manifest b/manifest index aa2738ca4b..5daeb35e5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sopfs-sahpool\sto\ssupport\smultiple\sinstances,\seach\swith\sa\sseparate\sVFS\sname\sand\sdirectory. -D 2023-07-18T12:09:16.750 +C Major\srestructuring\sof\sthe\sopfs-sahpool\sbits\sto\sbetter\ssupport\smultiple\sinstances\sper\sapp\s(each\ssandboxed\sfrom\seach\sother).\sEliminate\sthe\spesky\spromise\sresolution\srace\scondition\swhen\stwo\ssuch\sinstances\sare\sloaded\sin\sparallel. +D 2023-07-18T16:24:51.703 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,8 +501,8 @@ F ext/wasm/api/sqlite3-api-prologue.js f68e87edc049793c4ed46b0ec8f3a3d8013eeb3fd F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 -F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js a3307deb47d7d7a9a6e202a20b19252fa12fbeb60aeee11008ee0358a7137286 +F ext/wasm/api/sqlite3-v-helper.js fc9ed95433d943a65905d16b7ed51515ddb6667d2a2c5a711c7ce33b29d3be31 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js fc6d12298919652eacc6b51138011277be2598d60fbcb086049967621db74e2c F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5ad8f9987c11b5db67edd69a3471ff388140d6268d5d3f5a05dec6bb6d92ac05 -R a597bda217a200863fd7fdd91e2ff569 +P d036eaf6ac60c576428db40f015733c5d5425f7d613194fd8d9d4d98659077c4 +R f91cd88307b7706bc464075e9268cd3a U stephan -Z 61da1feaecfdf3b26fae912b9de308b1 +Z 4affd9934809bcbe1d4716ea36d23534 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 30ff3a470b..cb42f1710f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d036eaf6ac60c576428db40f015733c5d5425f7d613194fd8d9d4d98659077c4 \ No newline at end of file +95e5fa498f71708caeb3394636c4853530a8b2d54406e503f32750732d6815d5 \ No newline at end of file From b0dd9d427faac25681ba2cd4ba45b11fb7071ef5 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 18 Jul 2023 18:52:41 +0000 Subject: [PATCH 26/56] Further refactoring of opfs-sahpool and start integrating it into tester1.c-pp.js. FossilOrigin-Name: 91c789234963b660ae900f0738906b28a477993709e286d8125b2f4d6101601c --- ext/wasm/api/extern-post-js.c-pp.js | 2 +- ext/wasm/api/sqlite3-api-prologue.js | 38 ++++++----- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 84 ++++++++++++------------ ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 2 +- ext/wasm/tester1.c-pp.js | 81 ++++++++++++++++++++--- manifest | 20 +++--- manifest.uuid | 2 +- 7 files changed, 149 insertions(+), 80 deletions(-) diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index fac00370dd..bd5225d119 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -59,7 +59,7 @@ const toExportForESM = initModuleState.sqlite3Dir = li.join('/') + '/'; } - globalThis.sqlite3InitModule = function ff(...args){ + globalThis.sqlite3InitModule = async function ff(...args){ //console.warn("Using replaced sqlite3InitModule()",globalThis.location); return originalInit(...args).then((EmscriptenModule)=>{ if('undefined'!==typeof WorkerGlobalScope && diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index ac3253670f..22ec87ff31 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1876,26 +1876,28 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( then it must be called by client-level code, which must not use the library until the returned promise resolves. - Bug: if called while a prior call is still resolving, the 2nd - call will resolve prematurely, before the 1st call has finished - resolving. The current build setup precludes that possibility, - so it's only a hypothetical problem if/when this function - ever needs to be invoked by clients. + If called multiple times it will return the same promise on + subsequent calls. The current build setup precludes that + possibility, so it's only a hypothetical problem if/when this + function ever needs to be invoked by clients. In Emscripten-based builds, this function is called automatically and deleted from this object. */ - asyncPostInit: async function(){ + asyncPostInit: function ff(){ + if(ff.ready instanceof Promise) return ff.ready; let lip = sqlite3ApiBootstrap.initializersAsync; delete sqlite3ApiBootstrap.initializersAsync; - if(!lip || !lip.length) return Promise.resolve(sqlite3); + if(!lip || !lip.length){ + return ff.ready = Promise.resolve(sqlite3); + } lip = lip.map((f)=>{ - const p = (f instanceof Promise) ? f : f(sqlite3); - return p.catch((e)=>{ - console.error("an async sqlite3 initializer failed:",e); - throw e; - }); + return (f instanceof Promise) ? f : f(sqlite3); }); + const catcher = (e)=>{ + config.error("an async sqlite3 initializer failed:",e); + throw e; + }; const postInit = ()=>{ if(!sqlite3.__isUnderTest){ /* Delete references to internal-only APIs which are used by @@ -1911,16 +1913,16 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( return sqlite3; }; if(1){ - /* Run all initializers in sequence. The advantage is that it - allows us to have post-init cleanup defined outside of this - routine at the end of the list and have it run at a - well-defined time. */ + /* Run all initializers in the sequence they were added. The + advantage is that it allows us to have post-init cleanup + defined outside of this routine at the end of the list and + have it run at a well-defined time. */ let p = lip.shift(); while(lip.length) p = p.then(lip.shift()); - return p.then(postInit); + return ff.ready = p.then(postInit).catch(catcher); }else{ /* Run them in an arbitrary order. */ - return Promise.all(lip).then(postInit); + return ff.ready = Promise.all(lip).then(postInit).catch(catcher); } }, /** diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 5ab58bef58..1f026c7cc6 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -52,7 +52,7 @@ is not detected, the VFS is not registered. */ 'use strict'; -globalThis.sqlite3ApiBootstrap.initializersAsync.push(async function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = sqlite3.util.toss; const toss3 = sqlite3.util.toss3; const initPromises = Object.create(null); @@ -400,46 +400,47 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async function(sqlite3){ /** - A SAHPoolUtil instance is exposed to clients in order to - manipulate an OpfsSAHPool object without directly exposing that + A SAHPoolUtil instance is exposed to clients in order to manipulate an OpfsSAHPool object without directly exposing that object and allowing for some semantic changes compared to that class. + + Class docs are in the client-level docs for installOpfsSAHPoolVfs(). */ class SAHPoolUtil { + /* This object's associated OpfsSAHPool. */ + #p; constructor(sahPool){ - /* TODO: move the this-to-sahPool mapping into an external - WeakMap so as to not expose it to downstream clients. */ - this.$p = sahPool; + this.#p = sahPool; this.vfsName = sahPool.vfsName; } - addCapacity = async function(n){ - return this.$p.addCapacity(n); + async addCapacity(n){ + return this.#p.addCapacity(n); } - reduceCapacity = async function(n){ - return this.$p.reduceCapacity(n); + async reduceCapacity(n){ + return this.#p.reduceCapacity(n); } - getCapacity = function(){ - return this.$p.getCapacity(this.$p); + getCapacity(){ + return this.#p.getCapacity(this.#p); } - getActiveFileCount = function(){ - return this.$p.getFileCount(); + getActiveFileCount(){ + return this.#p.getFileCount(); } - reserveMinimumCapacity = async function(min){ - const c = this.$p.getCapacity(); - return (c < min) ? this.$p.addCapacity(min - c) : c; + async reserveMinimumCapacity(min){ + const c = this.#p.getCapacity(); + return (c < min) ? this.#p.addCapacity(min - c) : c; } - exportFile = function(name){ - const sah = this.$p.mapFilenameToSAH.get(name) || toss("File not found:",name); + exportFile(name){ + const sah = this.#p.mapFilenameToSAH.get(name) || toss("File not found:",name); const n = sah.getSize() - HEADER_OFFSET_DATA; const b = new Uint8Array(n>=0 ? n : 0); if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); return b; } - importDb = function(name, bytes){ + importDb(name, bytes){ const n = bytes.byteLength; if(n<512 || n%512!=0){ toss("Byte array size is invalid for an SQLite db."); @@ -450,35 +451,33 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async function(sqlite3){ toss("Input does not contain an SQLite database header."); } } - const sah = this.$p.mapFilenameToSAH.get(name) - || this.$p.nextAvailableSAH() + const sah = this.#p.mapFilenameToSAH.get(name) + || this.#p.nextAvailableSAH() || toss("No available handles to import to."); sah.write(bytes, {at: HEADER_OFFSET_DATA}); - this.$p.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); + this.#p.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); } - wipeFiles = async function(){ - return this.$p.reset(true); + async wipeFiles(){return this.#p.reset(true)} + + unlink(filename){ + return this.#p.deletePath(filename); } - unlink = function(filename){ - return this.$p.deletePath(filename); - } - - removeVfs = async function(){ - if(!this.$p.cVfs.pointer) return false; - capi.sqlite3_vfs_unregister(this.$p.cVfs.pointer); - this.$p.cVfs.dispose(); + async removeVfs(){ + if(!this.#p.cVfs.pointer) return false; + capi.sqlite3_vfs_unregister(this.#p.cVfs.pointer); + this.#p.cVfs.dispose(); try{ - this.$p.releaseAccessHandles(); - if(this.$p.parentDirHandle){ - await this.$p.parentDirHandle.removeEntry( - this.$p.dirHandle.name, {recursive: true} + this.#p.releaseAccessHandles(); + if(this.#p.parentDirHandle){ + await this.#p.parentDirHandle.removeEntry( + this.#p.dirHandle.name, {recursive: true} ); - this.$p.dirHandle = this.$p.parentDirHandle = undefined; + this.#p.dirHandle = this.#p.parentDirHandle = undefined; } }catch(e){ - console.error(this.$p.vfsName,"removeVfs() failed:",e); + sqlite3.config.error(this.#p.vfsName,"removeVfs() failed:",e); /*otherwise ignored - there is no recovery strategy*/ } return true; @@ -679,7 +678,7 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async function(sqlite3){ throw new Error("Just testing rejection."); } if(initPromises[vfsName]){ - //console.warn("Returning same OpfsSAHPool result",vfsName,initPromises[vfsName]); + console.warn("Returning same OpfsSAHPool result",options,vfsName,initPromises[vfsName]); return initPromises[vfsName]; } if(!globalThis.FileSystemHandle || @@ -743,6 +742,9 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async function(sqlite3){ ensues. */ return initPromises[vfsName] = apiVersionCheck().then(async function(){ + if(options.$testThrowInInit){ + throw options.$testThrowInInit; + } const thePool = new OpfsSAHPool(opfsVfs, options); return thePool.isReady.then(async()=>{ /** @@ -1025,4 +1027,4 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async function(sqlite3){ }); }).catch(promiseReject); }/*installOpfsSAHPoolVfs()*/; -}/*sqlite3ApiBootstrap.initializersAsync*/); +}/*sqlite3ApiBootstrap.initializers*/); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 35b7b88650..7d313a2635 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -23,7 +23,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs which accept a VFS. It is intended to be called via - sqlite3ApiBootstrap.initializersAsync or an equivalent mechanism. + sqlite3ApiBootstrap.initializers or an equivalent mechanism. The installed VFS uses the Origin-Private FileSystem API for all file storage. On error it is rejected with an exception diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 8b71198669..ed0e5c3c7b 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -65,6 +65,14 @@ globalThis.sqlite3InitModule = sqlite3InitModule; const haveWasmCTests = ()=>{ return !!wasm.exports.sqlite3_wasm_test_intptr; }; + const hasOpfs = ()=>{ + return globalThis.FileSystemHandle + && globalThis.FileSystemDirectoryHandle + && globalThis.FileSystemFileHandle + && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle + && navigator?.storage?.getDirectory; + }; + { const mapToString = (v)=>{ switch(typeof v){ @@ -277,7 +285,14 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } } const tc = TestUtil.counter, now = performance.now(); - await t.test.call(groupState, sqlite3); + let rc = t.test.call(groupState, sqlite3); + /*if(rc instanceof Promise){ + rc = rc.catch((e)=>{ + error("Test failure:",e); + throw e; + }); + }*/ + await rc; const then = performance.now(); runtime += then - now; logClass('faded',indent, indent, @@ -339,6 +354,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.g = T.addGroup; T.t = T.addTest; let capi, wasm/*assigned after module init*/; + const sahPoolConfig = { + name: 'opfs-sahpool-tester1', + clearOnInit: true, + initialCapacity: 3 + }; //////////////////////////////////////////////////////////////////////// // End of infrastructure setup. Now define the tests... //////////////////////////////////////////////////////////////////////// @@ -1288,7 +1308,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; if(1){ const vfsList = capi.sqlite3_js_vfs_list(); T.assert(vfsList.length>1); - //log("vfsList =",vfsList); wasm.scopedAllocCall(()=>{ const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v); for(const v of vfsList){ @@ -2617,8 +2636,8 @@ globalThis.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// T.g('OPFS: Origin-Private File System', - (sqlite3)=>(sqlite3.opfs - ? true : "requires Worker thread in a compatible browser")) + (sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs") + || 'requires "opfs" VFS')) .t({ name: 'OPFS db sanity checks', test: async function(sqlite3){ @@ -2737,6 +2756,48 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }/*OPFS util sanity checks*/) ;/* end OPFS tests */ + //////////////////////////////////////////////////////////////////////// + T.g('OPFS SyncAccessHandle Pool VFS', + (sqlite3)=>(hasOpfs() || "requires OPFS APIs")) + .t({ + name: 'SAH sanity checks', + test: async function(sqlite3){ + T.assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) + .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) < 0) + const inst = sqlite3.installOpfsSAHPoolVfs, + catcher = (e)=>{ + error("Cannot load SAH pool VFS.", + "This might not be a problem,", + "depending on the environment."); + return false; + }; + let u1, u2; + const P1 = inst(sahPoolConfig).then(u=>u1 = u).catch(catcher), + P2 = inst(sahPoolConfig).then(u=>u2 = u).catch(catcher); + await Promise.all([P1, P2]); + if(!P1) return; + T.assert(u1 === u2) + .assert(sahPoolConfig.name === u1.vfsName) + .assert(sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) + .assert(u1.getCapacity() === sahPoolConfig.initialCapacity) + .assert(5 === (await u2.addCapacity(2))) + .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) >= 0) + .assert(true === await u2.removeVfs()) + .assert(false === await u1.removeVfs()) + .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)); + + let cErr, u3; + const conf2 = JSON.parse(JSON.stringify(sahPoolConfig)); + conf2.$testThrowInInit = new Error("Testing throwing during init."); + conf2.name = sahPoolConfig.name+'-err'; + const P3 = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e); + T.assert(P3 === conf2.$testThrowInInit) + .assert(cErr === P3) + .assert(undefined === u3) + .assert(!sqlite3.capi.sqlite3_vfs_find(conf2.name)); + } + }/*OPFS SAH Pool sanity checks*/) + //////////////////////////////////////////////////////////////////////// T.g('Hook APIs') .t({ @@ -2942,8 +3003,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert( capi.sqlite3session_enable(pSession, -1) > 0 ) .assert(undefined === db1.selectValue('select a from t where rowid=2')); }else{ - warn("sqlite3session_enable() tests disabled due to unexpected results.", - "(Possibly a tester misunderstanding, as opposed to a bug.)"); + warn("sqlite3session_enable() tests are currently disabled."); } let db1Count = db1.selectValue("select count(*) from t"); T.assert( db1Count === (testSessionEnable ? 2 : 3) ); @@ -3088,11 +3148,15 @@ globalThis.sqlite3InitModule = sqlite3InitModule; globalThis.sqlite3InitModule({ print: log, printErr: error - }).then(function(sqlite3){ - //console.log('sqlite3 =',sqlite3); + }).then(async function(sqlite3){ log("Done initializing WASM/JS bits. Running tests..."); sqlite3.config.warn("Installing sqlite3 bits as global S for local dev/test purposes."); globalThis.S = sqlite3; + /*await sqlite3.installOpfsSAHPoolVfs(sahPoolConfig) + .then((u)=>log("Loaded",u.vfsName,"VFS")) + .catch(e=>{ + log("Cannot install OpfsSAHPool.",e); + });*/ capi = sqlite3.capi; wasm = sqlite3.wasm; log("sqlite3 version:",capi.sqlite3_libversion(), @@ -3107,6 +3171,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }else{ logClass('warning',"sqlite3_wasm_test_...() APIs unavailable."); } + log("registered vfs list =",capi.sqlite3_js_vfs_list()); TestUtil.runTests(sqlite3); }); })(self); diff --git a/manifest b/manifest index 5daeb35e5e..df9ec90511 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Major\srestructuring\sof\sthe\sopfs-sahpool\sbits\sto\sbetter\ssupport\smultiple\sinstances\sper\sapp\s(each\ssandboxed\sfrom\seach\sother).\sEliminate\sthe\spesky\spromise\sresolution\srace\scondition\swhen\stwo\ssuch\sinstances\sare\sloaded\sin\sparallel. -D 2023-07-18T16:24:51.703 +C Further\srefactoring\sof\sopfs-sahpool\sand\sstart\sintegrating\sit\sinto\stester1.c-pp.js. +D 2023-07-18T18:52:41.004 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -489,7 +489,7 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md f6cec6b0ce122cdff9440b30a3132dea3665b5b7baace910b43cbccdaaa376b9 -F ext/wasm/api/extern-post-js.c-pp.js 116749b7e55b7519129de06d3d353e19df68cfb24b12204aa4dc30c9a83023fe +F ext/wasm/api/extern-post-js.c-pp.js 80f288131f9f4486a66e79dbf42d4402dc23e3cb4ef605377ae69f0545a6b8e6 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 @@ -497,13 +497,13 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 -F ext/wasm/api/sqlite3-api-prologue.js f68e87edc049793c4ed46b0ec8f3a3d8013eeb3fd56481029dda916d4d5fa3a3 +F ext/wasm/api/sqlite3-api-prologue.js d747cbb379e13881c9edf39dce019cbbbae860c456ababe9d550b4b27666a1c9 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js fc9ed95433d943a65905d16b7ed51515ddb6667d2a2c5a711c7ce33b29d3be31 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js fc6d12298919652eacc6b51138011277be2598d60fbcb086049967621db74e2c -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js cd4d26fd8bf6c6b5bcb1666e8447c5388eb10ebfee57684e3658ee995f5a9645 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4946af0d6fbd395aa39966562ca85900664605a5f0cc10fff50146dee527812c F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 4420eb97b6b4fc79e4e156b4b8010dd9f373365f4230dd76d823fb04ce28ffde +F ext/wasm/tester1.c-pp.js 76607c9821e7854c061121b89337ca244f683b3a16c5415d078d361a8ec9e22f F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d036eaf6ac60c576428db40f015733c5d5425f7d613194fd8d9d4d98659077c4 -R f91cd88307b7706bc464075e9268cd3a +P 95e5fa498f71708caeb3394636c4853530a8b2d54406e503f32750732d6815d5 +R b629bc29f8d41e1b5f3ab58dda955102 U stephan -Z 4affd9934809bcbe1d4716ea36d23534 +Z c2f2230ed00c70ba745095d2d4043967 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cb42f1710f..51e938832b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95e5fa498f71708caeb3394636c4853530a8b2d54406e503f32750732d6815d5 \ No newline at end of file +91c789234963b660ae900f0738906b28a477993709e286d8125b2f4d6101601c \ No newline at end of file From 0649a1a05d9ae4e67cf7484ff7cbee06901b39c5 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 18 Jul 2023 19:27:11 +0000 Subject: [PATCH 27/56] Fix a shadowed var in opfs-sahpool and add more tests. FossilOrigin-Name: 9c8a73aff0f291e0c18072372e0d8961d3a05910489598d0d342227d99871954 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 14 +++++++------- ext/wasm/tester1.c-pp.js | 23 +++++++++++++++++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 1f026c7cc6..fc088c419d 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -999,18 +999,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ //log("vfs list:",capi.sqlite3_js_vfs_list()); if(sqlite3.oo1){ const oo1 = sqlite3.oo1; - const OpfsthePoolDb = function(...args){ + const OpfsSAHPoolDb = function(...args){ const opt = oo1.DB.dbCtorHelper.normalizeArgs(...args); opt.vfs = opfsVfs.$zName; oo1.DB.dbCtorHelper.call(this, opt); }; - OpfsthePoolDb.prototype = Object.create(oo1.DB.prototype); - OpfsthePoolDb.PoolUtil = poolUtil; - if(!oo1.OpfsthePoolDb){ - oo1.OpfsthePoolDb = Object.create(null); - oo1.OpfsthePoolDb.default = OpfsthePoolDb; + OpfsSAHPoolDb.prototype = Object.create(oo1.DB.prototype); + OpfsSAHPoolDb.PoolUtil = poolUtil; + if(!oo1.OpfsSAHPool){ + oo1.OpfsSAHPool = Object.create(null); + oo1.OpfsSAHPool.default = OpfsSAHPoolDb; } - oo1.OpfsthePoolDb[vfsName] = OpfsthePoolDb; + oo1.OpfsSAHPool[vfsName] = OpfsSAHPoolDb; oo1.DB.dbCtorHelper.setVfsPostOpenSql( opfsVfs.pointer, function(oo1Db, sqlite3){ diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index ed0e5c3c7b..e63d8440c6 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -2779,10 +2779,25 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(u1 === u2) .assert(sahPoolConfig.name === u1.vfsName) .assert(sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) - .assert(u1.getCapacity() === sahPoolConfig.initialCapacity) - .assert(5 === (await u2.addCapacity(2))) - .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) >= 0) - .assert(true === await u2.removeVfs()) + .assert(u1.getCapacity() >= sahPoolConfig.initialCapacity + /* If a test fails before we get to nuke the VFS, we + can have more than the initial capacity on the next + run. */) + .assert(u1.getCapacity() + 2 === (await u2.addCapacity(2))) + .assert(sqlite3.oo1.OpfsSAHPool.default instanceof Function) + .assert(sqlite3.oo1.OpfsSAHPool.default === + sqlite3.oo1.OpfsSAHPool[sahPoolConfig.name]) + .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) >= 0); + + const db = new sqlite3.oo1.OpfsSAHPool.default("foo.db"); + db.exec([ + 'create table t(a);', + 'insert into t(a) values(1),(2),(3)' + ]); + T.assert(3 === db.selectValue('select count(*) from t')); + db.close(); + + T.assert(true === await u2.removeVfs()) .assert(false === await u1.removeVfs()) .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)); diff --git a/manifest b/manifest index df9ec90511..a4d6dad250 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\srefactoring\sof\sopfs-sahpool\sand\sstart\sintegrating\sit\sinto\stester1.c-pp.js. -D 2023-07-18T18:52:41.004 +C Fix\sa\sshadowed\svar\sin\sopfs-sahpool\sand\sadd\smore\stests. +D 2023-07-18T19:27:11.899 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js fc9ed95433d943a65905d16b7ed51515ddb6667d2a2c5a711c7ce33b29d3be31 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js cd4d26fd8bf6c6b5bcb1666e8447c5388eb10ebfee57684e3658ee995f5a9645 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 4490d1b6a123d6d99fe5896e801de9dc8f81fa292a78e462ff10ebb5e9a10992 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4946af0d6fbd395aa39966562ca85900664605a5f0cc10fff50146dee527812c F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 76607c9821e7854c061121b89337ca244f683b3a16c5415d078d361a8ec9e22f +F ext/wasm/tester1.c-pp.js f26ed573b2b6948891c6f5c25e7018f3a4245b183532cfa386b74869634bd08e F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 95e5fa498f71708caeb3394636c4853530a8b2d54406e503f32750732d6815d5 -R b629bc29f8d41e1b5f3ab58dda955102 +P 91c789234963b660ae900f0738906b28a477993709e286d8125b2f4d6101601c +R 7463e6a2312d04dbe1f49055415382e0 U stephan -Z c2f2230ed00c70ba745095d2d4043967 +Z 6742837371ae9c474a1547585ca5d4c8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 51e938832b..208135c324 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91c789234963b660ae900f0738906b28a477993709e286d8125b2f4d6101601c \ No newline at end of file +9c8a73aff0f291e0c18072372e0d8961d3a05910489598d0d342227d99871954 \ No newline at end of file From aed5abcc0fcef79627966ceebb631952f29a1006 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 18 Jul 2023 19:47:19 +0000 Subject: [PATCH 28/56] More opfs-sahpool tests. FossilOrigin-Name: 60713fa9c4627ef17e0b8778eee37913d2b930c5a06414721a00af30e1395090 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 13 ++++--------- ext/wasm/tester1.c-pp.js | 16 +++++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index fc088c419d..7547c01a8e 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -160,12 +160,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } return this.getCapacity(); } - /** - Removes n entries from the pool's current capacity - if possible. It can only remove currently-unallocated - files. Returns a Promise resolving to the number of - removed files. - */ async reduceCapacity(n){ let nRm = 0; for(const ah of Array.from(this.availableSAH)){ @@ -424,7 +418,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ getCapacity(){ return this.#p.getCapacity(this.#p); } - getActiveFileCount(){ + getFileCount(){ return this.#p.getFileCount(); } async reserveMinimumCapacity(min){ @@ -610,9 +604,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ in the SAH pool. The default capacity is only large enough for one or two databases and their associated temp files. - - number getActiveFileCount() + - number getFileCount() - Returns the number of files from the pool currently in use. + Returns the number of files from the pool currently allocated to + slots. This is not the same as the files being "opened". - void importDb(name, byteArray) diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index e63d8440c6..27a8118e8d 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -2784,18 +2784,32 @@ globalThis.sqlite3InitModule = sqlite3InitModule; can have more than the initial capacity on the next run. */) .assert(u1.getCapacity() + 2 === (await u2.addCapacity(2))) + .assert(2 === (await u2.reduceCapacity(2))) .assert(sqlite3.oo1.OpfsSAHPool.default instanceof Function) .assert(sqlite3.oo1.OpfsSAHPool.default === sqlite3.oo1.OpfsSAHPool[sahPoolConfig.name]) .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) >= 0); - const db = new sqlite3.oo1.OpfsSAHPool.default("foo.db"); + T.assert(0 === u1.getFileCount()); + const DbCtor = sqlite3.oo1.OpfsSAHPool.default; + const dbName = '/foo.db'; + let db = new DbCtor(dbName); + T.assert(1 === u1.getFileCount()); db.exec([ 'create table t(a);', 'insert into t(a) values(1),(2),(3)' ]); + T.assert(1 === u1.getFileCount()); T.assert(3 === db.selectValue('select count(*) from t')); db.close(); + T.assert(1 === u1.getFileCount()); + db = new DbCtor(dbName); + T.assert(1 === u1.getFileCount()); + db.close(); + T.assert(1 === u1.getFileCount()) + .assert(true === u1.unlink(dbName)) + .assert(false === u1.unlink(dbName)) + .assert(0 === u1.getFileCount()); T.assert(true === await u2.removeVfs()) .assert(false === await u1.removeVfs()) diff --git a/manifest b/manifest index a4d6dad250..e7ddd646d8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sshadowed\svar\sin\sopfs-sahpool\sand\sadd\smore\stests. -D 2023-07-18T19:27:11.899 +C More\sopfs-sahpool\stests. +D 2023-07-18T19:47:19.982 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js fc9ed95433d943a65905d16b7ed51515ddb6667d2a2c5a711c7ce33b29d3be31 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 4490d1b6a123d6d99fe5896e801de9dc8f81fa292a78e462ff10ebb5e9a10992 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js a33b88beb55e0e19a8f9c3d37620e9aa58a0a32cacf7c398195d3d045d414815 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4946af0d6fbd395aa39966562ca85900664605a5f0cc10fff50146dee527812c F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js f26ed573b2b6948891c6f5c25e7018f3a4245b183532cfa386b74869634bd08e +F ext/wasm/tester1.c-pp.js f625ce01681956cfd895a834d8aeb68d59595200441628ac2e41adf48788d8c9 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 91c789234963b660ae900f0738906b28a477993709e286d8125b2f4d6101601c -R 7463e6a2312d04dbe1f49055415382e0 +P 9c8a73aff0f291e0c18072372e0d8961d3a05910489598d0d342227d99871954 +R 17462841136511130de69ca7a00ea9cc U stephan -Z 6742837371ae9c474a1547585ca5d4c8 +Z c3c355ff206bb421832084a0cbc9cd73 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 208135c324..a80d58ffe9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c8a73aff0f291e0c18072372e0d8961d3a05910489598d0d342227d99871954 \ No newline at end of file +60713fa9c4627ef17e0b8778eee37913d2b930c5a06414721a00af30e1395090 \ No newline at end of file From 6a9be2cbc72bb81208227a1130b9b447817a0979 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 18 Jul 2023 20:41:09 +0000 Subject: [PATCH 29/56] Improved ".wheretrace" for the VIEWSCAN optimization. FossilOrigin-Name: 27057ea76b5f72c73fb6f16094736685019643c665b49fd0bb8d60a812ce2338 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1303d23154..2333353d42 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smemory\sleak\sin\sfts5\sintroduced\sby\s[def41225]. -D 2023-07-18T16:29:34.926 +C Improved\s".wheretrace"\sfor\sthe\sVIEWSCAN\soptimization. +D 2023-07-18T20:41:09.317 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -722,7 +722,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c dd843f619ac60d5dadab7109cf402432ba74dde0c301505fd1c202add07659e3 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 2dc708cf8b6a691fb79f16bbc46567497ee6f991043318d421e294b2da114d93 +F src/where.c e551d13d4aff9176690368ae1b3acff10ede1f451652c75e9dfc9bb2d85d5970 F src/whereInt.h c7d19902863beadec1d04e66aca39c0bcd60b74f05f0eaa7422c7005dfc5d51a F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P def41225b2f10b0294ab506f28bc87873688d4030f7056839298ff18e058ece5 -R e9300e0dc9583f0f9cd6aefb60fd68d9 -U dan -Z 5551f44d2ee1d451fc303b92968039b5 +P 4dcad2db743fdb9ef72871ca5a4d1384f76cb697161b0f5110e2670a83a18e8a +R 226cc4c3c153723bae524ae831eeb8ad +U drh +Z 09fb8db5361f1b823e48611e4afd0f28 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 273a4023bd..f9b2c165b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4dcad2db743fdb9ef72871ca5a4d1384f76cb697161b0f5110e2670a83a18e8a \ No newline at end of file +27057ea76b5f72c73fb6f16094736685019643c665b49fd0bb8d60a812ce2338 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 858e33c8ac..923a4d77c7 100644 --- a/src/where.c +++ b/src/where.c @@ -5125,6 +5125,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){ rCost += -10; nOut += -30; + WHERETRACE(0x80,("VIEWSCAN cost reduction for %c\n",pWLoop->cId)); } /* Check to see if pWLoop should be added to the set of From 9246c85b61d37f071c4f6197423a0baf41de14f9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 18 Jul 2023 21:06:19 +0000 Subject: [PATCH 30/56] Do not use the [/info/609fbb94b8f01d67|viewscan optimization] on a query that has only a single loop, as the cost adjustments can cause problems for outer queries. Proposed fix for the performance regression reported by [forum:/forumpost/64d36440e473516c|forum post 64d36440e473516c]. FossilOrigin-Name: 76152ad2ffe56034f2fd93d9a1ce9358e1677a7e9cd3dcd9f3a34a5c956a463e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2333353d42..e1145c527f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\s".wheretrace"\sfor\sthe\sVIEWSCAN\soptimization. -D 2023-07-18T20:41:09.317 +C Do\snot\suse\sthe\s[/info/609fbb94b8f01d67|viewscan\soptimization]\son\sa\squery\nthat\shas\sonly\sa\ssingle\sloop,\sas\sthe\scost\sadjustments\scan\scause\sproblems\nfor\souter\squeries.\s\sProposed\sfix\sfor\sthe\sperformance\sregression\sreported\sby\n[forum:/forumpost/64d36440e473516c|forum\spost\s64d36440e473516c]. +D 2023-07-18T21:06:19.893 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -722,7 +722,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c dd843f619ac60d5dadab7109cf402432ba74dde0c301505fd1c202add07659e3 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c e551d13d4aff9176690368ae1b3acff10ede1f451652c75e9dfc9bb2d85d5970 +F src/where.c 477fcc5e561ef169e6002499602af6b805156c2aae6b2f5c2c93ef8c1cd64768 F src/whereInt.h c7d19902863beadec1d04e66aca39c0bcd60b74f05f0eaa7422c7005dfc5d51a F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c dc5096eca5ed503999be3bdee8a90c51361289a678d396a220912e9cb73b3c00 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4dcad2db743fdb9ef72871ca5a4d1384f76cb697161b0f5110e2670a83a18e8a -R 226cc4c3c153723bae524ae831eeb8ad +P 27057ea76b5f72c73fb6f16094736685019643c665b49fd0bb8d60a812ce2338 +R e083cfeae9842a7650b26a959bcc57e4 U drh -Z 09fb8db5361f1b823e48611e4afd0f28 +Z 4c6375f8dd8261298072fd9d19053ffb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f9b2c165b4..24332574c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27057ea76b5f72c73fb6f16094736685019643c665b49fd0bb8d60a812ce2338 \ No newline at end of file +76152ad2ffe56034f2fd93d9a1ce9358e1677a7e9cd3dcd9f3a34a5c956a463e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 923a4d77c7..35b9056ffe 100644 --- a/src/where.c +++ b/src/where.c @@ -5122,7 +5122,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* TUNING: A full-scan of a VIEW or subquery in the outer loop ** is not so bad. */ - if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 ){ + if( iLoop==0 && (pWLoop->wsFlags & WHERE_VIEWSCAN)!=0 && nLoop>1 ){ rCost += -10; nOut += -30; WHERETRACE(0x80,("VIEWSCAN cost reduction for %c\n",pWLoop->cId)); From 780bc4c557ac61294c32170b093e6463ce2fefa5 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 19 Jul 2023 08:18:25 +0000 Subject: [PATCH 31/56] Internal refactoring in opfs-sahpool. Move OPFS tests to the end of tester1.c-cpp.js. FossilOrigin-Name: 6bd5a7413dd830ca41b587a2826fb599a2196fb0186646a2333500f950b3cf4d --- ext/wasm/api/sqlite3-v-helper.js | 2 +- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 70 ++-- ext/wasm/tester1.c-pp.js | 387 ++++++++++++----------- manifest | 16 +- manifest.uuid | 2 +- 5 files changed, 249 insertions(+), 228 deletions(-) diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index cc9747aa5b..8ae1172cf6 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -100,7 +100,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ACHTUNG: because we cannot generically know how to transform JS exceptions into result codes, the installed functions do no - automatic catching of exceptions. It is critical, to avoid + automatic catching of exceptions. It is critical, to avoid undefined behavior in the C layer, that methods mapped via this function do not throw. The exception, as it were, to that rule is... diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 7547c01a8e..322ac977a9 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -89,7 +89,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const optionDefaults = Object.assign(Object.create(null),{ name: 'opfs-sahpool', - directory: undefined, + directory: undefined /* derived from .name */, initialCapacity: 6, clearOnInit: false, verbosity: 2 /*3+ == everything, 2 == warnings+errors, 1 == errors only*/ @@ -390,6 +390,21 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return rc; } + getFileForPtr(ptr){ + return this.mapIdToFile.get(ptr); + } + setFileForPtr(ptr,file){ + if(file) this.mapIdToFile.set(ptr, file); + else this.mapIdToFile.delete(ptr); + } + + hasFilename(name){ + return this.mapFilenameToSAH.has(name) + } + + getSAHForPath(path){ + return this.mapFilenameToSAH.get(path); + } }/*class OpfsSAHPool*/; @@ -576,6 +591,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ the default directory. If no directory is explicitly provided then a directory name is synthesized from the `name` option. + Peculiarities of this VFS: + + - Paths given to it _must_ be absolute. Relative paths will not + be properly recognized. This is arguably a bug but correcting it + requires some hoop-jumping and memory allocation in routines + which should not be allocating. + The API for the utility object passed on by this function's Promise, in alphabetical order... @@ -673,7 +695,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ throw new Error("Just testing rejection."); } if(initPromises[vfsName]){ - console.warn("Returning same OpfsSAHPool result",options,vfsName,initPromises[vfsName]); + //console.warn("Returning same OpfsSAHPool result",options,vfsName,initPromises[vfsName]); return initPromises[vfsName]; } if(!globalThis.FileSystemHandle || @@ -699,13 +721,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const opfsVfs = new capi.sqlite3_vfs() .addOnDispose(()=>opfsIoMethods.dispose()); - const promiseReject = (err)=>{ - error("rejecting promise:",err); - opfsVfs.dispose(); - initPromises[vfsName] = Promise.reject(err); - throw err; - }; - /* We fetch the default VFS so that we can inherit some methods from it. */ const pDVfs = capi.sqlite3_vfs_find(null); @@ -755,13 +770,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }, xClose: function(pFile){ thePool.storeErr(); - const file = thePool.mapIdToFile.get(pFile); + const file = thePool.getFileForPtr(pFile); if(file) { try{ log(`xClose ${file.path}`); if(file.sq3File) file.sq3File.dispose(); file.sah.flush(); - thePool.mapIdToFile.delete(pFile); + thePool.setFileForPtr(pFile,0); if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ thePool.deletePath(file.path); } @@ -780,7 +795,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }, xFileSize: function(pFile,pSz64){ log(`xFileSize`); - const file = thePool.mapIdToFile.get(pFile); + const file = thePool.getFileForPtr(pFile); const size = file.sah.getSize() - HEADER_OFFSET_DATA; //log(`xFileSize ${file.path} ${size}`); wasm.poke64(pSz64, BigInt(size)); @@ -789,14 +804,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xLock: function(pFile,lockType){ log(`xLock ${lockType}`); thePool.storeErr(); - const file = thePool.mapIdToFile.get(pFile); + const file = thePool.getFileForPtr(pFile); file.lockType = lockType; return 0; }, xRead: function(pFile,pDest,n,offset64){ log(`xRead ${n}@${offset64}`); thePool.storeErr(); - const file = thePool.mapIdToFile.get(pFile); + const file = thePool.getFileForPtr(pFile); log(`xRead ${file.path} ${n} ${offset64}`); try { const nRead = file.sah.read( @@ -819,7 +834,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xSync: function(pFile,flags){ log(`xSync ${flags}`); thePool.storeErr(); - const file = thePool.mapIdToFile.get(pFile); + const file = thePool.getFileForPtr(pFile); //log(`xSync ${file.path} ${flags}`); try{ file.sah.flush(); @@ -832,7 +847,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xTruncate: function(pFile,sz64){ log(`xTruncate ${sz64}`); thePool.storeErr(); - const file = thePool.mapIdToFile.get(pFile); + const file = thePool.getFileForPtr(pFile); //log(`xTruncate ${file.path} ${iSize}`); try{ file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); @@ -844,13 +859,13 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }, xUnlock: function(pFile,lockType){ log('xUnlock'); - const file = thePool.mapIdToFile.get(pFile); + const file = thePool.getFileForPtr(pFile); file.lockType = lockType; return 0; }, xWrite: function(pFile,pSrc,n,offset64){ thePool.storeErr(); - const file = thePool.mapIdToFile.get(pFile); + const file = thePool.getFileForPtr(pFile); log(`xWrite ${file.path} ${n} ${offset64}`); try{ const nBytes = file.sah.write( @@ -870,13 +885,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ const vfsMethods = { xAccess: function(pVfs,zName,flags,pOut){ - log(`xAccess ${wasm.cstrToJs(zName)}`); + //log(`xAccess ${wasm.cstrToJs(zName)}`); thePool.storeErr(); try{ - const name = this.getPath(zName); - wasm.poke32(pOut, thePool.mapFilenameToSAH.has(name) ? 1 : 0); + const name = thePool.getPath(zName); + wasm.poke32(pOut, thePool.hasFilename(name) ? 1 : 0); }catch(e){ - /*ignored*/; + /*ignored*/ + wasm.poke32(pOut, 0); } return 0; }, @@ -931,7 +947,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const path = (zName && wasm.peek8(zName)) ? thePool.getPath(zName) : getRandomName(); - let sah = thePool.mapFilenameToSAH.get(path); + let sah = thePool.getSAHForPath(path); if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { // File not found so try to create it. if(thePool.getFileCount() < thePool.getCapacity()) { @@ -949,7 +965,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ // Subsequent methods are only passed the file pointer, so // map the relevant info we need to that pointer. const file = {path, flags, sah}; - thePool.mapIdToFile.set(pFile, file); + thePool.setFileForPtr(pFile, file); wasm.poke32(pOutFlags, flags); file.sq3File = new capi.sqlite3_file(pFile); file.sq3File.$pMethods = opfsIoMethods.pointer; @@ -1020,6 +1036,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ log("VFS initialized."); return poolUtil; }); - }).catch(promiseReject); + }).catch((err)=>{ + error("rejecting promise:",err); + opfsVfs.dispose(); + return initPromises[vfsName] = Promise.reject(err); + }); }/*installOpfsSAHPoolVfs()*/; }/*sqlite3ApiBootstrap.initializers*/); diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 27a8118e8d..d1fbbb87df 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -2634,199 +2634,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }/*kvvfs sqlite3_js_vfs_create_file()*/) ;/* end kvvfs tests */ - //////////////////////////////////////////////////////////////////////// - T.g('OPFS: Origin-Private File System', - (sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs") - || 'requires "opfs" VFS')) - .t({ - name: 'OPFS db sanity checks', - test: async function(sqlite3){ - const filename = this.opfsDbFile = 'sqlite3-tester1.db'; - const pVfs = this.opfsVfs = capi.sqlite3_vfs_find('opfs'); - T.assert(pVfs); - const unlink = this.opfsUnlink = - (fn=filename)=>{wasm.sqlite3_wasm_vfs_unlink(pVfs,fn)}; - unlink(); - let db = new sqlite3.oo1.OpfsDb(filename); - try { - db.exec([ - 'create table p(a);', - 'insert into p(a) values(1),(2),(3)' - ]); - T.assert(3 === db.selectValue('select count(*) from p')); - db.close(); - db = new sqlite3.oo1.OpfsDb(filename); - db.exec('insert into p(a) values(4),(5),(6)'); - T.assert(6 === db.selectValue('select count(*) from p')); - this.opfsDbExport = capi.sqlite3_js_db_export(db); - T.assert(this.opfsDbExport instanceof Uint8Array) - .assert(this.opfsDbExport.byteLength>0 - && 0===this.opfsDbExport.byteLength % 512); - }finally{ - db.close(); - unlink(); - } - } - }/*OPFS db sanity checks*/) - .t({ - name: 'OPFS export/import', - test: async function(sqlite3){ - let db; - try { - const exp = this.opfsDbExport; - delete this.opfsDbExport; - capi.sqlite3_js_vfs_create_file("opfs", this.opfsDbFile, exp); - const db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); - T.assert(6 === db.selectValue('select count(*) from p')); - }finally{ - if(db) db.close(); - } - } - }/*OPFS export/import*/) - .t({ - name: 'OPFS utility APIs and sqlite3_js_vfs_create_file()', - test: async function(sqlite3){ - const filename = this.opfsDbFile; - const pVfs = this.opfsVfs; - const unlink = this.opfsUnlink; - T.assert(filename && pVfs && !!unlink); - delete this.opfsDbFile; - delete this.opfsVfs; - delete this.opfsUnlink; - unlink(); - // Sanity-test sqlite3_js_vfs_create_file()... - /************************************************************** - ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended - for client-side use. It is only for this project's own - internal use. Its APIs are subject to change or removal at - any time. - ***************************************************************/ - const opfs = sqlite3.opfs; - const fSize = 1379; - let sh; - try{ - T.assert(!(await opfs.entryExists(filename))); - capi.sqlite3_js_vfs_create_file( - pVfs, filename, null, fSize - ); - T.assert(await opfs.entryExists(filename)); - let fh = await opfs.rootDirectory.getFileHandle(filename); - sh = await fh.createSyncAccessHandle(); - T.assert(fSize === await sh.getSize()); - await sh.close(); - sh = undefined; - unlink(); - T.assert(!(await opfs.entryExists(filename))); - - const ba = new Uint8Array([1,2,3,4,5]); - capi.sqlite3_js_vfs_create_file( - "opfs", filename, ba - ); - T.assert(await opfs.entryExists(filename)); - fh = await opfs.rootDirectory.getFileHandle(filename); - sh = await fh.createSyncAccessHandle(); - T.assert(ba.byteLength === await sh.getSize()); - await sh.close(); - sh = undefined; - unlink(); - - T.mustThrowMatching(()=>{ - capi.sqlite3_js_vfs_create_file( - "no-such-vfs", filename, ba - ); - }, "SQLITE_NOTFOUND: Unknown sqlite3_vfs name: no-such-vfs"); - }finally{ - if(sh) await sh.close(); - unlink(); - } - - // Some sanity checks of the opfs utility functions... - const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); - const aDir = testDir+'/test/dir'; - T.assert(await opfs.mkdir(aDir), "mkdir failed") - .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") - .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") - .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") - .assert(!(await opfs.unlink(testDir+'/test/dir')), - "delete 2b should have failed (dir already deleted)") - .assert((await opfs.unlink(testDir, true)), "delete 3 failed") - .assert(!(await opfs.entryExists(testDir)), - "entryExists(",testDir,") should have failed"); - } - }/*OPFS util sanity checks*/) - ;/* end OPFS tests */ - - //////////////////////////////////////////////////////////////////////// - T.g('OPFS SyncAccessHandle Pool VFS', - (sqlite3)=>(hasOpfs() || "requires OPFS APIs")) - .t({ - name: 'SAH sanity checks', - test: async function(sqlite3){ - T.assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) - .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) < 0) - const inst = sqlite3.installOpfsSAHPoolVfs, - catcher = (e)=>{ - error("Cannot load SAH pool VFS.", - "This might not be a problem,", - "depending on the environment."); - return false; - }; - let u1, u2; - const P1 = inst(sahPoolConfig).then(u=>u1 = u).catch(catcher), - P2 = inst(sahPoolConfig).then(u=>u2 = u).catch(catcher); - await Promise.all([P1, P2]); - if(!P1) return; - T.assert(u1 === u2) - .assert(sahPoolConfig.name === u1.vfsName) - .assert(sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) - .assert(u1.getCapacity() >= sahPoolConfig.initialCapacity - /* If a test fails before we get to nuke the VFS, we - can have more than the initial capacity on the next - run. */) - .assert(u1.getCapacity() + 2 === (await u2.addCapacity(2))) - .assert(2 === (await u2.reduceCapacity(2))) - .assert(sqlite3.oo1.OpfsSAHPool.default instanceof Function) - .assert(sqlite3.oo1.OpfsSAHPool.default === - sqlite3.oo1.OpfsSAHPool[sahPoolConfig.name]) - .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) >= 0); - - T.assert(0 === u1.getFileCount()); - const DbCtor = sqlite3.oo1.OpfsSAHPool.default; - const dbName = '/foo.db'; - let db = new DbCtor(dbName); - T.assert(1 === u1.getFileCount()); - db.exec([ - 'create table t(a);', - 'insert into t(a) values(1),(2),(3)' - ]); - T.assert(1 === u1.getFileCount()); - T.assert(3 === db.selectValue('select count(*) from t')); - db.close(); - T.assert(1 === u1.getFileCount()); - db = new DbCtor(dbName); - T.assert(1 === u1.getFileCount()); - db.close(); - T.assert(1 === u1.getFileCount()) - .assert(true === u1.unlink(dbName)) - .assert(false === u1.unlink(dbName)) - .assert(0 === u1.getFileCount()); - - T.assert(true === await u2.removeVfs()) - .assert(false === await u1.removeVfs()) - .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)); - - let cErr, u3; - const conf2 = JSON.parse(JSON.stringify(sahPoolConfig)); - conf2.$testThrowInInit = new Error("Testing throwing during init."); - conf2.name = sahPoolConfig.name+'-err'; - const P3 = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e); - T.assert(P3 === conf2.$testThrowInInit) - .assert(cErr === P3) - .assert(undefined === u3) - .assert(!sqlite3.capi.sqlite3_vfs_find(conf2.name)); - } - }/*OPFS SAH Pool sanity checks*/) - //////////////////////////////////////////////////////////////////////// T.g('Hook APIs') .t({ @@ -3096,6 +2903,200 @@ globalThis.sqlite3InitModule = sqlite3InitModule; } })/*session API sanity tests*/ ;/*end of session API group*/; + + //////////////////////////////////////////////////////////////////////// + T.g('OPFS: Origin-Private File System', + (sqlite3)=>(sqlite3.capi.sqlite3_vfs_find("opfs") + || 'requires "opfs" VFS')) + .t({ + name: 'OPFS db sanity checks', + test: async function(sqlite3){ + const filename = this.opfsDbFile = 'sqlite3-tester1.db'; + const pVfs = this.opfsVfs = capi.sqlite3_vfs_find('opfs'); + T.assert(pVfs); + const unlink = this.opfsUnlink = + (fn=filename)=>{wasm.sqlite3_wasm_vfs_unlink(pVfs,fn)}; + unlink(); + let db = new sqlite3.oo1.OpfsDb(filename); + try { + db.exec([ + 'create table p(a);', + 'insert into p(a) values(1),(2),(3)' + ]); + T.assert(3 === db.selectValue('select count(*) from p')); + db.close(); + db = new sqlite3.oo1.OpfsDb(filename); + db.exec('insert into p(a) values(4),(5),(6)'); + T.assert(6 === db.selectValue('select count(*) from p')); + this.opfsDbExport = capi.sqlite3_js_db_export(db); + T.assert(this.opfsDbExport instanceof Uint8Array) + .assert(this.opfsDbExport.byteLength>0 + && 0===this.opfsDbExport.byteLength % 512); + }finally{ + db.close(); + unlink(); + } + } + }/*OPFS db sanity checks*/) + .t({ + name: 'OPFS export/import', + test: async function(sqlite3){ + let db; + try { + const exp = this.opfsDbExport; + delete this.opfsDbExport; + capi.sqlite3_js_vfs_create_file("opfs", this.opfsDbFile, exp); + const db = new sqlite3.oo1.OpfsDb(this.opfsDbFile); + T.assert(6 === db.selectValue('select count(*) from p')); + }finally{ + if(db) db.close(); + } + } + }/*OPFS export/import*/) + .t({ + name: 'OPFS utility APIs and sqlite3_js_vfs_create_file()', + test: async function(sqlite3){ + const filename = this.opfsDbFile; + const pVfs = this.opfsVfs; + const unlink = this.opfsUnlink; + T.assert(filename && pVfs && !!unlink); + delete this.opfsDbFile; + delete this.opfsVfs; + delete this.opfsUnlink; + unlink(); + // Sanity-test sqlite3_js_vfs_create_file()... + /************************************************************** + ATTENTION CLIENT-SIDE USERS: sqlite3.opfs is NOT intended + for client-side use. It is only for this project's own + internal use. Its APIs are subject to change or removal at + any time. + ***************************************************************/ + const opfs = sqlite3.opfs; + const fSize = 1379; + let sh; + try{ + T.assert(!(await opfs.entryExists(filename))); + capi.sqlite3_js_vfs_create_file( + pVfs, filename, null, fSize + ); + T.assert(await opfs.entryExists(filename)); + let fh = await opfs.rootDirectory.getFileHandle(filename); + sh = await fh.createSyncAccessHandle(); + T.assert(fSize === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + T.assert(!(await opfs.entryExists(filename))); + + const ba = new Uint8Array([1,2,3,4,5]); + capi.sqlite3_js_vfs_create_file( + "opfs", filename, ba + ); + T.assert(await opfs.entryExists(filename)); + fh = await opfs.rootDirectory.getFileHandle(filename); + sh = await fh.createSyncAccessHandle(); + T.assert(ba.byteLength === await sh.getSize()); + await sh.close(); + sh = undefined; + unlink(); + + T.mustThrowMatching(()=>{ + capi.sqlite3_js_vfs_create_file( + "no-such-vfs", filename, ba + ); + }, "SQLITE_NOTFOUND: Unknown sqlite3_vfs name: no-such-vfs"); + }finally{ + if(sh) await sh.close(); + unlink(); + } + + // Some sanity checks of the opfs utility functions... + const testDir = '/sqlite3-opfs-'+opfs.randomFilename(12); + const aDir = testDir+'/test/dir'; + T.assert(await opfs.mkdir(aDir), "mkdir failed") + .assert(await opfs.mkdir(aDir), "mkdir must pass if the dir exists") + .assert(!(await opfs.unlink(testDir+'/test')), "delete 1 should have failed (dir not empty)") + .assert((await opfs.unlink(testDir+'/test/dir')), "delete 2 failed") + .assert(!(await opfs.unlink(testDir+'/test/dir')), + "delete 2b should have failed (dir already deleted)") + .assert((await opfs.unlink(testDir, true)), "delete 3 failed") + .assert(!(await opfs.entryExists(testDir)), + "entryExists(",testDir,") should have failed"); + } + }/*OPFS util sanity checks*/) + ;/* end OPFS tests */ + + //////////////////////////////////////////////////////////////////////// + T.g('OPFS SyncAccessHandle Pool VFS', + (sqlite3)=>(hasOpfs() || "requires OPFS APIs")) + .t({ + name: 'SAH sanity checks', + test: async function(sqlite3){ + T.assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) + .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) < 0) + const inst = sqlite3.installOpfsSAHPoolVfs, + catcher = (e)=>{ + error("Cannot load SAH pool VFS.", + "This might not be a problem,", + "depending on the environment."); + return false; + }; + let u1, u2; + const P1 = inst(sahPoolConfig).then(u=>u1 = u).catch(catcher), + P2 = inst(sahPoolConfig).then(u=>u2 = u).catch(catcher); + await Promise.all([P1, P2]); + if(!P1) return; + T.assert(u1 === u2) + .assert(sahPoolConfig.name === u1.vfsName) + .assert(sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) + .assert(u1.getCapacity() >= sahPoolConfig.initialCapacity + /* If a test fails before we get to nuke the VFS, we + can have more than the initial capacity on the next + run. */) + .assert(u1.getCapacity() + 2 === (await u2.addCapacity(2))) + .assert(2 === (await u2.reduceCapacity(2))) + .assert(sqlite3.oo1.OpfsSAHPool.default instanceof Function) + .assert(sqlite3.oo1.OpfsSAHPool.default === + sqlite3.oo1.OpfsSAHPool[sahPoolConfig.name]) + .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) >= 0); + + T.assert(0 === u1.getFileCount()); + const DbCtor = sqlite3.oo1.OpfsSAHPool.default; + const dbName = '/foo.db'; + let db = new DbCtor(dbName); + T.assert(1 === u1.getFileCount()); + db.exec([ + 'create table t(a);', + 'insert into t(a) values(1),(2),(3)' + ]); + T.assert(1 === u1.getFileCount()); + T.assert(3 === db.selectValue('select count(*) from t')); + db.close(); + T.assert(1 === u1.getFileCount()); + db = new DbCtor(dbName); + T.assert(1 === u1.getFileCount()); + db.close(); + T.assert(1 === u1.getFileCount()) + .assert(true === u1.unlink(dbName)) + .assert(false === u1.unlink(dbName)) + .assert(0 === u1.getFileCount()); + + T.assert(true === await u2.removeVfs()) + .assert(false === await u1.removeVfs()) + .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)); + + let cErr, u3; + const conf2 = JSON.parse(JSON.stringify(sahPoolConfig)); + conf2.$testThrowInInit = new Error("Testing throwing during init."); + conf2.name = sahPoolConfig.name+'-err'; + const P3 = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e); + T.assert(P3 === conf2.$testThrowInInit) + .assert(cErr === P3) + .assert(undefined === u3) + .assert(!sqlite3.capi.sqlite3_vfs_find(conf2.name)); + } + }/*OPFS SAH Pool sanity checks*/) + //////////////////////////////////////////////////////////////////////// T.g('Bug Reports') .t({ diff --git a/manifest b/manifest index e7ddd646d8..a54d71323c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sopfs-sahpool\stests. -D 2023-07-18T19:47:19.982 +C Internal\srefactoring\sin\sopfs-sahpool.\sMove\sOPFS\stests\sto\sthe\send\sof\stester1.c-cpp.js. +D 2023-07-19T08:18:25.901 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,8 +501,8 @@ F ext/wasm/api/sqlite3-api-prologue.js d747cbb379e13881c9edf39dce019cbbbae860c45 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 -F ext/wasm/api/sqlite3-v-helper.js fc9ed95433d943a65905d16b7ed51515ddb6667d2a2c5a711c7ce33b29d3be31 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js a33b88beb55e0e19a8f9c3d37620e9aa58a0a32cacf7c398195d3d045d414815 +F ext/wasm/api/sqlite3-v-helper.js e4b7b27a8259e40407b3c16e42dd5df05b80726c609594cc23b1565dc2ad9ca2 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 9a8ba44b775b0e8faaffc8d877cfef0726713c2ca368e5776b59d9029f5ebf23 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4946af0d6fbd395aa39966562ca85900664605a5f0cc10fff50146dee527812c F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js f625ce01681956cfd895a834d8aeb68d59595200441628ac2e41adf48788d8c9 +F ext/wasm/tester1.c-pp.js a72fc43950ce26c1ad7cee47aa225dd18efdb92743cf616b2e114b4cd1cdf2dd F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9c8a73aff0f291e0c18072372e0d8961d3a05910489598d0d342227d99871954 -R 17462841136511130de69ca7a00ea9cc +P 60713fa9c4627ef17e0b8778eee37913d2b930c5a06414721a00af30e1395090 +R b81da7c28422da4e0ac61aef8c2566f8 U stephan -Z c3c355ff206bb421832084a0cbc9cd73 +Z 2d92190e2a732ef23d4b23ca479d2da6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a80d58ffe9..9aaea635b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60713fa9c4627ef17e0b8778eee37913d2b930c5a06414721a00af30e1395090 \ No newline at end of file +6bd5a7413dd830ca41b587a2826fb599a2196fb0186646a2333500f950b3cf4d \ No newline at end of file From 37fd50df1fdf67ba2d46dab956b921d14ad956be Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Jul 2023 09:52:10 +0000 Subject: [PATCH 32/56] Two minor improvements to sum(), one of which was inspired by [forum:/forumpost/af5be98dbc|forum post af5be98dbc]. FossilOrigin-Name: a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e1145c527f..575934e4e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sthe\s[/info/609fbb94b8f01d67|viewscan\soptimization]\son\sa\squery\nthat\shas\sonly\sa\ssingle\sloop,\sas\sthe\scost\sadjustments\scan\scause\sproblems\nfor\souter\squeries.\s\sProposed\sfix\sfor\sthe\sperformance\sregression\sreported\sby\n[forum:/forumpost/64d36440e473516c|forum\spost\s64d36440e473516c]. -D 2023-07-18T21:06:19.893 +C Two\sminor\simprovements\sto\ssum(),\sone\sof\swhich\swas\sinspired\sby\n[forum:/forumpost/af5be98dbc|forum\spost\saf5be98dbc]. +D 2023-07-19T09:52:10.467 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -590,7 +590,7 @@ F src/delete.c cd5f5cd06ed0b6a882ec1a8c2a0d73b3cecb28479ad19e9931c4706c5e2182be F src/expr.c 8d1656b65e26af3e34f78e947ac423f0d20c214ed25a67486e433bf16ca6b543 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a7fcbf7e66d14dbb73cf49f31489ebf66d0e6006c62b95246924a3bae9f37b36 -F src/func.c cffa6edb4aa4865d8e237022399ba9c2b22fd11e5581efba7c5b524b525952ca +F src/func.c 25f2e0204c011be56fc3c9a180534b68ca4866c61ec19806880136450434112d F src/global.c 29f56a330ed9d1b5cd9b79ac0ca36f97ac3afc730ff8bfa987b0db9e559d684d F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27057ea76b5f72c73fb6f16094736685019643c665b49fd0bb8d60a812ce2338 -R e083cfeae9842a7650b26a959bcc57e4 +P 76152ad2ffe56034f2fd93d9a1ce9358e1677a7e9cd3dcd9f3a34a5c956a463e +R 47d8d8ca9ec8151ad0210262a08ac711 U drh -Z 4c6375f8dd8261298072fd9d19053ffb +Z b98de7129167c58b9385610fbdec3119 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 24332574c1..4a5750f1d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76152ad2ffe56034f2fd93d9a1ce9358e1677a7e9cd3dcd9f3a34a5c956a463e \ No newline at end of file +a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 542d71a23d..8382e4a5c5 100644 --- a/src/func.c +++ b/src/func.c @@ -1764,11 +1764,10 @@ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ p->ovrfl = 1; kahanBabuskaNeumaierInit(p, p->iSum); p->approx = 1; - kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); } } }else{ - p->approx = 1; if( type==SQLITE_INTEGER ){ kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); }else{ From cd302a5e3207e775cc2455e77b2fdd9cedeb604f Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 19 Jul 2023 11:33:52 +0000 Subject: [PATCH 33/56] Appearance and legibility improvements for tester1.js. FossilOrigin-Name: 4fd3ed1b6405f690825c9993e56b199d4a7fb497cf9131815c803150b8b96528 --- ext/wasm/common/testing.css | 36 ++++++++++++++++++++++++++++++++++-- ext/wasm/tester1.c-pp.js | 27 +++++++++++++-------------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 55 insertions(+), 24 deletions(-) diff --git a/ext/wasm/common/testing.css b/ext/wasm/common/testing.css index fb44f1d612..71a19aa95e 100644 --- a/ext/wasm/common/testing.css +++ b/ext/wasm/common/testing.css @@ -40,8 +40,40 @@ span.labeled-input { .tests-pass { background-color: green; color: white } .tests-fail { background-color: red; color: yellow } .faded { opacity: 0.5; } -.group-start { color: blue; } -.group-end { color: blue; } +.group-start { + color: blue; + background-color: skyblue; + font-weight: bold; + border-top: 1px dotted blue; + padding: 0.5em; + margin-top: 0.5em; +} +.group-end { + padding: 0.5em; + margin-bottom: 0.25em; + /*border-bottom: 1px dotted blue;*/ +} +.group-end.green { + background: lightgreen; + border-bottom: 1px dotted green; +} +.one-test-line, .skipping-group { + margin-left: 3em; +} +.skipping-test, .skipping-group { + background-color: #ffff73; +} +.skipping-test { + margin-left: 6em; +} +.one-test-summary { + margin-left: 6em; +} +.full-test-summary { + padding-bottom: 0.5em; + padding-top: 0.5em; + border-top: 1px solid black; +} .input-wrapper { white-space: nowrap; display: flex; diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index d1fbbb87df..9d56340aae 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -167,8 +167,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; /** Running total of the number of tests run via this API. */ counter: 0, - /* Separator line for log messages. */ - separator: '------------------------------------------------------------', /** If expr is a function, it is called and its result is returned, coerced to a bool, else expr, coerced to @@ -256,13 +254,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule; return this; }, run: async function(sqlite3){ - log(TestUtil.separator); logClass('group-start',"Group #"+this.number+':',this.name); - const indent = ' '; if(this.predicate){ const p = this.predicate(sqlite3); if(!p || 'string'===typeof p){ - logClass('warning',indent, + logClass(['warning','skipping-group'], "SKIPPING group:", p ? p : "predicate says to" ); return; } @@ -274,11 +270,11 @@ globalThis.sqlite3InitModule = sqlite3InitModule; for(const t of this.tests){ ++i; const n = this.number+"."+i; - log(indent, n+":", t.name); + logClass('one-test-line', n+":", t.name); if(t.predicate){ const p = t.predicate(sqlite3); if(!p || 'string'===typeof p){ - logClass('warning',indent, + logClass(['warning','skipping-test'], "SKIPPING:", p ? p : "predicate says to" ); skipped.push( n+': '+t.name ); continue; @@ -295,12 +291,13 @@ globalThis.sqlite3InitModule = sqlite3InitModule; await rc; const then = performance.now(); runtime += then - now; - logClass('faded',indent, indent, + logClass(['faded','one-test-summary'], TestUtil.counter - tc, 'assertion(s) in', roundMs(then-now),'ms'); } - logClass('green', - "Group #"+this.number+":",(TestUtil.counter - assertCount), + logClass(['green','group-end'], + //"Group #"+this.number+":", + (TestUtil.counter - assertCount), "assertion(s) in",roundMs(runtime),"ms"); if(0 && skipped.length){ logClass('warning',"SKIPPED test(s) in group",this.number+":",skipped); @@ -336,8 +333,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; await g.run(sqlite3); runtime += performance.now() - now; } - log(TestUtil.separator); - logClass(['strong','green'], + logClass(['strong','green','full-test-summary'], "Done running tests.",TestUtil.counter,"assertions in", roundMs(runtime),'ms'); pok(); @@ -3042,10 +3038,13 @@ globalThis.sqlite3InitModule = sqlite3InitModule; return false; }; let u1, u2; + // Ensure that two immediately-consecutive installations + // resolve to the same Promise instead of triggering + // a locking error. const P1 = inst(sahPoolConfig).then(u=>u1 = u).catch(catcher), P2 = inst(sahPoolConfig).then(u=>u2 = u).catch(catcher); await Promise.all([P1, P2]); - if(!P1) return; + if(!(await P1)) return; T.assert(u1 === u2) .assert(sahPoolConfig.name === u1.vfsName) .assert(sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)) @@ -3201,7 +3200,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }else{ logClass('warning',"sqlite3_wasm_test_...() APIs unavailable."); } - log("registered vfs list =",capi.sqlite3_js_vfs_list()); + log("registered vfs list =",capi.sqlite3_js_vfs_list().join(', ')); TestUtil.runTests(sqlite3); }); })(self); diff --git a/manifest b/manifest index a54d71323c..d6fff1e2eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Internal\srefactoring\sin\sopfs-sahpool.\sMove\sOPFS\stests\sto\sthe\send\sof\stester1.c-cpp.js. -D 2023-07-19T08:18:25.901 +C Appearance\sand\slegibility\simprovements\sfor\stester1.js. +D 2023-07-19T11:33:52.985 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -512,7 +512,7 @@ F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c2 F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 -F ext/wasm/common/testing.css 0ff15602a3ab2bad8aef2c3bd120c7ee3fd1c2054ad2ace7e214187ae68d926f +F ext/wasm/common/testing.css cae1f8cbfc9f516e22af0524deab10569120860ea8f12bf73204838cee6b63a1 F ext/wasm/common/whwasmutil.js ae263dec9d7384f4c530f324b99d00516a4d6f26424372daee65031e00eb49b3 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js a72fc43950ce26c1ad7cee47aa225dd18efdb92743cf616b2e114b4cd1cdf2dd +F ext/wasm/tester1.c-pp.js d20a88f99fa78d50bed4de69270aed4902ae7901071797f3e3b095684736d830 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 60713fa9c4627ef17e0b8778eee37913d2b930c5a06414721a00af30e1395090 -R b81da7c28422da4e0ac61aef8c2566f8 +P 6bd5a7413dd830ca41b587a2826fb599a2196fb0186646a2333500f950b3cf4d +R b499063b7173978725e0c6adeaf3d1ef U stephan -Z 2d92190e2a732ef23d4b23ca479d2da6 +Z aeff227bd99478c07e8d0a15233f4c66 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9aaea635b7..8a229d33f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6bd5a7413dd830ca41b587a2826fb599a2196fb0186646a2333500f950b3cf4d \ No newline at end of file +4fd3ed1b6405f690825c9993e56b199d4a7fb497cf9131815c803150b8b96528 \ No newline at end of file From eafb4418a0a7efaaac3b2530a4b888659b08ba78 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 19 Jul 2023 13:31:29 +0000 Subject: [PATCH 34/56] Major restructuring of opfs-sahpool to improve maintainability and hopefully make it easier to reuse these pieces in the upcoming JSPI-based VFS experiment. FossilOrigin-Name: 534481cd0c2e6f62dd0a82f25d4b78fdcc671eb70d6966693c98212a6420891c --- ext/wasm/api/sqlite3-v-helper.js | 3 +- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 866 +++++++++++++---------- ext/wasm/common/testing.css | 1 + ext/wasm/tester1.c-pp.js | 2 +- manifest | 18 +- manifest.uuid | 2 +- 6 files changed, 489 insertions(+), 403 deletions(-) diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index 8ae1172cf6..e63da8afc3 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -295,7 +295,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - If `struct.$zName` is falsy and the entry has a string-type `name` property, `struct.$zName` is set to the C-string form of - that `name` value before registerVfs() is called. + that `name` value before registerVfs() is called. That string + gets added to the on-dispose state of the struct. On success returns this object. Throws on error. */ diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index 322ac977a9..b0d71c50ee 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -92,9 +92,327 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ directory: undefined /* derived from .name */, initialCapacity: 6, clearOnInit: false, - verbosity: 2 /*3+ == everything, 2 == warnings+errors, 1 == errors only*/ + /* Logging verbosity 3+ == everything, 2 == warnings+errors, 1 == + errors only. */ + verbosity: 2 }); + /** Logging routines, from most to least serious. */ + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const log = sqlite3.config.log; + const warn = sqlite3.config.warn; + const error = sqlite3.config.error; + + /* Maps (sqlite3_vfs*) to OpfsSAHPool instances */ + const __mapVfsToPool = new Map(); + const getPoolForVfs = (pVfs)=>__mapVfsToPool.get(pVfs); + const setPoolForVfs = (pVfs,pool)=>{ + if(pool) __mapVfsToPool.set(pVfs, pool); + else __mapVfsToPool.delete(pVfs); + }; + /* Maps (sqlite3_file*) to OpfsSAHPool instances */ + const __mapSqlite3File = new Map(); + const getPoolForPFile = (pFile)=>__mapSqlite3File.get(pFile); + const setPoolForPFile = (pFile,pool)=>{ + if(pool) __mapSqlite3File.set(pFile, pool); + else __mapSqlite3File.delete(pFile); + }; + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioMethods = { + xCheckReservedLock: function(pFile,pOut){ + const pool = getPoolForPFile(pFile); + pool.log('xCheckReservedLock'); + pool.storeErr(); + wasm.poke32(pOut, 1); + return 0; + }, + xClose: function(pFile){ + const pool = getPoolForPFile(pFile); + pool.storeErr(); + const file = pool.getFileForPtr(pFile); + if(file) { + try{ + pool.log(`xClose ${file.path}`); + pool.setFileForPtr(pFile, false); + file.sah.flush(); + if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ + pool.deletePath(file.path); + } + }catch(e){ + pool.storeErr(e); + return capi.SQLITE_IOERR; + } + } + return 0; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + const pool = getPoolForPFile(pFile); + pool.log(`xFileSize`); + const file = pool.getFileForPtr(pFile); + const size = file.sah.getSize() - HEADER_OFFSET_DATA; + //log(`xFileSize ${file.path} ${size}`); + wasm.poke64(pSz64, BigInt(size)); + return 0; + }, + xLock: function(pFile,lockType){ + const pool = getPoolForPFile(pFile); + pool.log(`xLock ${lockType}`); + pool.storeErr(); + const file = pool.getFileForPtr(pFile); + file.lockType = lockType; + return 0; + }, + xRead: function(pFile,pDest,n,offset64){ + const pool = getPoolForPFile(pFile); + pool.storeErr(); + const file = pool.getFileForPtr(pFile); + pool.log(`xRead ${file.path} ${n} @ ${offset64}`); + try { + const nRead = file.sah.read( + wasm.heap8u().subarray(pDest, pDest+n), + {at: HEADER_OFFSET_DATA + Number(offset64)} + ); + if(nRead < n){ + wasm.heap8u().fill(0, pDest + nRead, pDest + n); + return capi.SQLITE_IOERR_SHORT_READ; + } + return 0; + }catch(e){ + pool.storeErr(e); + return capi.SQLITE_IOERR; + } + }, + xSectorSize: function(pFile){ + return SECTOR_SIZE; + }, + xSync: function(pFile,flags){ + const pool = getPoolForPFile(pFile); + pool.log(`xSync ${flags}`); + pool.storeErr(); + const file = pool.getFileForPtr(pFile); + //log(`xSync ${file.path} ${flags}`); + try{ + file.sah.flush(); + return 0; + }catch(e){ + pool.storeErr(e); + return capi.SQLITE_IOERR; + } + }, + xTruncate: function(pFile,sz64){ + const pool = getPoolForPFile(pFile); + pool.log(`xTruncate ${sz64}`); + pool.storeErr(); + const file = pool.getFileForPtr(pFile); + //log(`xTruncate ${file.path} ${iSize}`); + try{ + file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); + return 0; + }catch(e){ + pool.storeErr(e); + return capi.SQLITE_IOERR; + } + }, + xUnlock: function(pFile,lockType){ + const pool = getPoolForPFile(pFile); + pool.log('xUnlock'); + const file = pool.getFileForPtr(pFile); + file.lockType = lockType; + return 0; + }, + xWrite: function(pFile,pSrc,n,offset64){ + const pool = getPoolForPFile(pFile); + pool.storeErr(); + const file = pool.getFileForPtr(pFile); + pool.log(`xWrite ${file.path} ${n} ${offset64}`); + try{ + const nBytes = file.sah.write( + wasm.heap8u().subarray(pSrc, pSrc+n), + { at: HEADER_OFFSET_DATA + Number(offset64) } + ); + return nBytes === n ? 0 : capi.SQLITE_IOERR; + }catch(e){ + pool.storeErr(e); + return capi.SQLITE_IOERR; + } + } + }/*ioMethods*/; + + const opfsIoMethods = new capi.sqlite3_io_methods(); + opfsIoMethods.$iVersion = 1; + sqlite3.vfs.installVfs({ + io: {struct: opfsIoMethods, methods: ioMethods} + }); + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsMethods = { + xAccess: function(pVfs,zName,flags,pOut){ + //log(`xAccess ${wasm.cstrToJs(zName)}`); + const pool = getPoolForVfs(pVfs); + pool.storeErr(); + try{ + const name = pool.getPath(zName); + wasm.poke32(pOut, pool.hasFilename(name) ? 1 : 0); + }catch(e){ + /*ignored*/ + wasm.poke32(pOut, 0); + } + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + const pool = getPoolForVfs(pVfs); + pool.log(`xDelete ${wasm.cstrToJs(zName)}`); + pool.storeErr(); + try{ + pool.deletePath(pool.getPath(zName)); + return 0; + }catch(e){ + pool.storeErr(e); + return capi.SQLITE_IOERR_DELETE; + } + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + //const pool = getPoolForVfs(pVfs); + //pool.log(`xFullPathname ${wasm.cstrToJs(zName)}`); + const i = wasm.cstrncpy(pOut, zName, nOut); + return i nOut) wasm.poke8(pOut + nOut - 1, 0); + }catch(e){ + return capi.SQLITE_NOMEM; + }finally{ + wasm.scopedAllocPop(scope); + } + } + return 0; + }, + //xSleep is optionally defined below + xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ + const pool = getPoolForVfs(pVfs); + try{ + pool.log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`); + // First try to open a path that already exists in the file system. + const path = (zName && wasm.peek8(zName)) + ? pool.getPath(zName) + : getRandomName(); + let sah = pool.getSAHForPath(path); + if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { + // File not found so try to create it. + if(pool.getFileCount() < pool.getCapacity()) { + // Choose an unassociated OPFS file from the pool. + sah = pool.nextAvailableSAH(); + pool.setAssociatedPath(sah, path, flags); + }else{ + // File pool is full. + toss('SAH pool is full. Cannot create file',path); + } + } + if(!sah){ + toss('file not found:',path); + } + // Subsequent I/O methods are only passed the sqlite3_file + // pointer, so map the relevant info we need to that pointer. + const file = {path, flags, sah}; + pool.setFileForPtr(pFile, file); + file.lockType = capi.SQLITE_LOCK_NONE; + const sq3File = new capi.sqlite3_file(pFile); + sq3File.$pMethods = opfsIoMethods.pointer; + sq3File.dispose(); + wasm.poke32(pOutFlags, flags); + return 0; + }catch(e){ + pool.storeErr(e); + return capi.SQLITE_CANTOPEN; + } + }/*xOpen()*/ + }/*vfsMethods*/; + + /** + Creates and initializes an sqlite3_vfs instance for an + OpfsSAHPool. The argument is the VFS's name (JS string). + + Maintenance reminder: the only detail about the returned object + which is specific to any given OpfsSAHPool instance is the $zName + member. All other state is identical. + */ + const createOpfsVfs = function(vfsName){ + const opfsVfs = new capi.sqlite3_vfs(); + /* We fetch the default VFS so that we can inherit some + methods from it. */ + const pDVfs = capi.sqlite3_vfs_find(null); + const dVfs = pDVfs + ? new capi.sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. */; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; + opfsVfs.addOnDispose( + opfsVfs.$zName = wasm.allocCString(vfsName), + ()=>setPoolForVfs(opfsVfs.pointer, 0) + ); + + if(dVfs){ + /* Inherit certain VFS members from the default VFS, + if available. */ + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; + dVfs.dispose(); + } + if(!opfsVfs.$xRandomness && !vfsMethods.xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + vfsMethods.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep && !vfsMethods.xSleep){ + vfsMethods.xSleep = (pVfs,ms)=>0; + } + sqlite3.vfs.installVfs({ + vfs: {struct: opfsVfs, methods: vfsMethods} + }); + return opfsVfs; + }; + /** Class for managing OPFS-related state for the OPFS SharedAccessHandle Pool sqlite3_vfs. @@ -103,31 +421,40 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* OPFS dir in which VFS metadata is stored. */ vfsDir; /* Directory handle to this.vfsDir. */ - dirHandle; - /* Directory handle to this.dirHandle's parent dir. Needed + #dhVfsRoot; + /* Directory handle to this.dhVfsRoot's parent dir. Needed for a VFS-wipe op. */ - parentDirHandle; + #dhVfsParent; /* Maps SAHs to their opaque file names. */ - mapSAHToName = new Map(); + #mapSAHToName = new Map(); /* Maps client-side file names to SAHs. */ - mapFilenameToSAH = new Map(); + #mapFilenameToSAH = new Map(); /* Set of currently-unused SAHs. */ - availableSAH = new Set(); + #availableSAH = new Set(); /* Maps (sqlite3_file*) to xOpen's file objects. */ - mapIdToFile = new Map(); + #mapSqlite3FileToFile = new Map(); /** Buffer used by [sg]etAssociatedPath(). */ - apBody = new Uint8Array(HEADER_CORPUS_SIZE); + #apBody = new Uint8Array(HEADER_CORPUS_SIZE); + #dvBody; - constructor(vfsObject, options = Object.create(null)){ + // associated sqlite3_vfs instance + #cVfs; + + // Logging verbosity. See optionDefaults.verbosity. + #verbosity; + + constructor(options = Object.create(null)){ this.vfsName = options.name || optionDefaults.name; if( sqlite3.capi.sqlite3_vfs_find(this.vfsName)){ toss3("VFS name is already registered:", this.vfsName); } - this.cVfs = vfsObject; + this.#verbosity = options.verbosity ?? optionDefaults.verbosity; + this.#cVfs = createOpfsVfs(this.vfsName); + setPoolForVfs(this.#cVfs.pointer, this); this.vfsDir = options.directory || ("."+this.vfsName); - this.dvBody = - new DataView(this.apBody.buffer, this.apBody.byteOffset); + this.#dvBody = + new DataView(this.#apBody.buffer, this.#apBody.byteOffset); this.isReady = this .reset(!!(options.clearOnInit ?? optionDefaults.clearOnInit)) .then(()=>{ @@ -139,11 +466,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }); } + #logImpl(level,...args){ + if(this.#verbosity>level) loggers[level](this.vfsName+":",...args); + }; + log(...args){this.#logImpl(2, ...args)}; + warn(...args){this.#logImpl(1, ...args)}; + error(...args){this.#logImpl(0, ...args)}; + + getVfs(){return this.#cVfs} + /* Current pool capacity. */ - getCapacity(){return this.mapSAHToName.size} + getCapacity(){return this.#mapSAHToName.size} /* Current number of in-use files from pool. */ - getFileCount(){return this.mapFilenameToSAH.size} + getFileCount(){return this.#mapFilenameToSAH.size} /** Adds n files to the pool's capacity. This change is @@ -153,24 +489,25 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ async addCapacity(n){ for(let i = 0; i < n; ++i){ const name = getRandomName(); - const h = await this.dirHandle.getFileHandle(name, {create:true}); + const h = await this.#dhVfsRoot.getFileHandle(name, {create:true}); const ah = await h.createSyncAccessHandle(); - this.mapSAHToName.set(ah,name); + this.#mapSAHToName.set(ah,name); this.setAssociatedPath(ah, '', 0); } return this.getCapacity(); } + async reduceCapacity(n){ let nRm = 0; - for(const ah of Array.from(this.availableSAH)){ + for(const ah of Array.from(this.#availableSAH)){ if(nRm === n || this.getFileCount() === this.getCapacity()){ break; } - const name = this.mapSAHToName.get(ah); + const name = this.#mapSAHToName.get(ah); ah.close(); - await this.dirHandle.removeEntry(name); - this.mapSAHToName.delete(ah); - this.availableSAH.delete(ah); + await this.#dhVfsRoot.removeEntry(name); + this.#mapSAHToName.delete(ah); + this.#availableSAH.delete(ah); ++nRm; } return nRm; @@ -180,14 +517,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Releases all currently-opened SAHs. */ releaseAccessHandles(){ - for(const ah of this.mapSAHToName.keys()) ah.close(); - this.mapSAHToName.clear(); - this.mapFilenameToSAH.clear(); - this.availableSAH.clear(); + for(const ah of this.#mapSAHToName.keys()) ah.close(); + this.#mapSAHToName.clear(); + this.#mapFilenameToSAH.clear(); + this.#availableSAH.clear(); } /** - Opens all files under this.vfsDir/this.dirHandle and acquires + Opens all files under this.vfsDir/this.#dhVfsRoot and acquires a SAH for each. returns a Promise which resolves to no value but completes once all SAHs are acquired. If acquiring an SAH throws, SAHPool.$error will contain the corresponding @@ -199,7 +536,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ async acquireAccessHandles(clearFiles){ const files = []; - for await (const [name,h] of this.dirHandle){ + for await (const [name,h] of this.#dhVfsRoot){ if('file'===h.kind){ files.push([name,h]); } @@ -207,16 +544,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return Promise.all(files.map(async([name,h])=>{ try{ const ah = await h.createSyncAccessHandle() - this.mapSAHToName.set(ah, name); + this.#mapSAHToName.set(ah, name); if(clearFiles){ ah.truncate(HEADER_OFFSET_DATA); this.setAssociatedPath(ah, '', 0); }else{ const path = this.getAssociatedPath(ah); if(path){ - this.mapFilenameToSAH.set(path, ah); + this.#mapFilenameToSAH.set(path, ah); }else{ - this.availableSAH.add(ah); + this.#availableSAH.add(ah); } } }catch(e){ @@ -235,32 +572,32 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ returns an empty string. */ getAssociatedPath(sah){ - sah.read(this.apBody, {at: 0}); + sah.read(this.#apBody, {at: 0}); // Delete any unexpected files left over by previous // untimely errors... - const flags = this.dvBody.getUint32(HEADER_OFFSET_FLAGS); - if(this.apBody[0] && + const flags = this.#dvBody.getUint32(HEADER_OFFSET_FLAGS); + if(this.#apBody[0] && ((flags & capi.SQLITE_OPEN_DELETEONCLOSE) || (flags & PERSISTENT_FILE_TYPES)===0)){ warn(`Removing file with unexpected flags ${flags.toString(16)}`, - this.apBody); + this.#apBody); this.setAssociatedPath(sah, '', 0); return ''; } const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); - const compDigest = this.computeDigest(this.apBody); + const compDigest = this.computeDigest(this.#apBody); if(fileDigest.every((v,i) => v===compDigest[i])){ // Valid digest - const pathBytes = this.apBody.findIndex((v)=>0===v); + const pathBytes = this.#apBody.findIndex((v)=>0===v); if(0===pathBytes){ // This file is unassociated, so truncate it to avoid // leaving stale db data laying around. sah.truncate(HEADER_OFFSET_DATA); } return pathBytes - ? textDecoder.decode(this.apBody.subarray(0,pathBytes)) + ? textDecoder.decode(this.#apBody.subarray(0,pathBytes)) : ''; }else{ // Invalid digest @@ -277,25 +614,25 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ in the metadata. */ setAssociatedPath(sah, path, flags){ - const enc = textEncoder.encodeInto(path, this.apBody); + const enc = textEncoder.encodeInto(path, this.#apBody); if(HEADER_MAX_PATH_SIZE <= enc.written){ toss("Path too long:",path); } - this.dvBody.setUint32(HEADER_OFFSET_FLAGS, flags); + this.#dvBody.setUint32(HEADER_OFFSET_FLAGS, flags); - const digest = this.computeDigest(this.apBody); - sah.write(this.apBody, {at: 0}); + const digest = this.computeDigest(this.#apBody); + sah.write(this.#apBody, {at: 0}); sah.write(digest, {at: HEADER_OFFSET_DIGEST}); sah.flush(); if(path){ - this.mapFilenameToSAH.set(path, sah); - this.availableSAH.delete(sah); + this.#mapFilenameToSAH.set(path, sah); + this.#availableSAH.delete(sah); }else{ // This is not a persistent file, so eliminate the contents. sah.truncate(HEADER_OFFSET_DATA); - this.availableSAH.add(sah); + this.#availableSAH.add(sah); } } @@ -314,8 +651,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } /** - Re-initializes the state of the SAH pool, - releasing and re-acquiring all handles. + Re-initializes the state of the SAH pool, releasing and + re-acquiring all handles. See acquireAccessHandles() for the specifics of the clearFiles argument. @@ -330,8 +667,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ h = await h.getDirectoryHandle(d,{create:true}); } } - this.dirHandle = h; - this.parentDirHandle = prev; + this.#dhVfsRoot = h; + this.#dhVfsParent = prev; this.releaseAccessHandles(); return this.acquireAccessHandles(clearFiles); } @@ -357,10 +694,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ is found, else false. */ deletePath(path) { - const sah = this.mapFilenameToSAH.get(path); + const sah = this.#mapFilenameToSAH.get(path); if(sah) { // Un-associate the name from the SAH. - this.mapFilenameToSAH.delete(path); + this.#mapFilenameToSAH.delete(path); this.setAssociatedPath(sah, '', 0); } return !!sah; @@ -370,7 +707,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Sets e as this object's current error. Pass a falsy (or no) value to clear it. */ - storeErr(e){return this.$error = e;} + storeErr(e){ + if(e) this.error(e); + return this.$error = e; + } /** Pops this object's Error object and returns it (a falsy value if no error is set). @@ -386,25 +726,79 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ it from the set. */ nextAvailableSAH(){ - const [rc] = this.availableSAH.keys(); + const [rc] = this.#availableSAH.keys(); return rc; } getFileForPtr(ptr){ - return this.mapIdToFile.get(ptr); + return this.#mapSqlite3FileToFile.get(ptr); } - setFileForPtr(ptr,file){ - if(file) this.mapIdToFile.set(ptr, file); - else this.mapIdToFile.delete(ptr); + /** + Maps or unmaps (if file is falsy) the given (sqlite3_file*) + to an xOpen file object and to this pool object. + */ + setFileForPtr(pFile,file){ + if(file){ + this.#mapSqlite3FileToFile.set(pFile, file); + setPoolForPFile(pFile, this); + }else{ + this.#mapSqlite3FileToFile.delete(pFile); + setPoolForPFile(pFile, false); + } } - hasFilename(name){ - return this.mapFilenameToSAH.has(name) + return this.#mapFilenameToSAH.has(name) } getSAHForPath(path){ - return this.mapFilenameToSAH.get(path); + return this.#mapFilenameToSAH.get(path); } + + async removeVfs(){ + if(!this.#cVfs.pointer) return false; + capi.sqlite3_vfs_unregister(this.#cVfs.pointer); + this.#cVfs.dispose(); + try{ + this.releaseAccessHandles(); + if(this.#dhVfsParent){ + await this.#dhVfsParent.removeEntry( + this.#dhVfsRoot.name, {recursive: true} + ); + this.#dhVfsRoot = this.#dhVfsParent = undefined; + } + }catch(e){ + sqlite3.config.error(this.vfsName,"removeVfs() failed:",e); + /*otherwise ignored - there is no recovery strategy*/ + } + return true; + } + + exportFile(name){ + const sah = this.#mapFilenameToSAH.get(name) || toss("File not found:",name); + const n = sah.getSize() - HEADER_OFFSET_DATA; + const b = new Uint8Array(n>=0 ? n : 0); + if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); + return b; + } + + importDb(name, bytes){ + const n = bytes.byteLength; + if(n<512 || n%512!=0){ + toss("Byte array size is invalid for an SQLite db."); + } + const header = "SQLite format 3"; + for(let i = 0; i < header.length; ++i){ + if( header.charCodeAt(i) !== bytes[i] ){ + toss("Input does not contain an SQLite database header."); + } + } + const sah = this.#mapFilenameToSAH.get(name) + || this.nextAvailableSAH() + || toss("No available handles to import to."); + sah.write(bytes, {at: HEADER_OFFSET_DATA}); + this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); + } + }/*class OpfsSAHPool*/; @@ -441,31 +835,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return (c < min) ? this.#p.addCapacity(min - c) : c; } - exportFile(name){ - const sah = this.#p.mapFilenameToSAH.get(name) || toss("File not found:",name); - const n = sah.getSize() - HEADER_OFFSET_DATA; - const b = new Uint8Array(n>=0 ? n : 0); - if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); - return b; - } + exportFile(name){ return this.#p.exportFile(name) } - importDb(name, bytes){ - const n = bytes.byteLength; - if(n<512 || n%512!=0){ - toss("Byte array size is invalid for an SQLite db."); - } - const header = "SQLite format 3"; - for(let i = 0; i < header.length; ++i){ - if( header.charCodeAt(i) !== bytes[i] ){ - toss("Input does not contain an SQLite database header."); - } - } - const sah = this.#p.mapFilenameToSAH.get(name) - || this.#p.nextAvailableSAH() - || toss("No available handles to import to."); - sah.write(bytes, {at: HEADER_OFFSET_DATA}); - this.#p.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); - } + importDb(name, bytes){ return this.#p.importDb(name,bytes) } async wipeFiles(){return this.#p.reset(true)} @@ -473,24 +845,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return this.#p.deletePath(filename); } - async removeVfs(){ - if(!this.#p.cVfs.pointer) return false; - capi.sqlite3_vfs_unregister(this.#p.cVfs.pointer); - this.#p.cVfs.dispose(); - try{ - this.#p.releaseAccessHandles(); - if(this.#p.parentDirHandle){ - await this.#p.parentDirHandle.removeEntry( - this.#p.dirHandle.name, {recursive: true} - ); - this.#p.dirHandle = this.#p.parentDirHandle = undefined; - } - }catch(e){ - sqlite3.config.error(this.#p.vfsName,"removeVfs() failed:",e); - /*otherwise ignored - there is no recovery strategy*/ - } - return true; - } + async removeVfs(){return this.#p.removeVfs()} }/* class SAHPoolUtil */; @@ -595,8 +950,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - Paths given to it _must_ be absolute. Relative paths will not be properly recognized. This is arguably a bug but correcting it - requires some hoop-jumping and memory allocation in routines - which should not be allocating. + requires some hoop-jumping in routines which have no business + doing tricks. + + - It is possible to install multiple instances under different + names, each sandboxed from one another inside their own private + directory. This feature exists primarily as a way for disparate + applications within a given HTTP origin to use this VFS without + introducing locking issues between them. The API for the utility object passed on by this function's @@ -616,9 +977,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Synchronously reads the contents of the given file into a Uint8Array and returns it. This will throw if the given name is not currently in active use or on I/O error. Note that the given name is _not_ - visible directly in OPFS (or, if it is, it's not from this VFS). The - reason for that is that this VFS manages name-to-file mappings in - a roundabout way in order to maintain its list of SAHs. + visible directly in OPFS (or, if it is, it's not from this VFS). - number getCapacity() @@ -705,37 +1064,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ !navigator?.storage?.getDirectory){ return (initPromises[vfsName] = Promise.reject(new Error("Missing required OPFS APIs."))); } - const verbosity = options.verbosity ?? optionDefaults.verbosity; - const loggers = [ - sqlite3.config.error, - sqlite3.config.warn, - sqlite3.config.log - ]; - const logImpl = (level,...args)=>{ - if(verbosity>level) loggers[level](vfsName+":",...args); - }; - const log = (...args)=>logImpl(2, ...args); - const warn = (...args)=>logImpl(1, ...args); - const error = (...args)=>logImpl(0, ...args); - const opfsIoMethods = new capi.sqlite3_io_methods(); - const opfsVfs = new capi.sqlite3_vfs() - .addOnDispose(()=>opfsIoMethods.dispose()); - - /* We fetch the default VFS so that we can inherit some - methods from it. */ - const pDVfs = capi.sqlite3_vfs_find(null); - const dVfs = pDVfs - ? new capi.sqlite3_vfs(pDVfs) - : null /* dVfs will be null when sqlite3 is built with - SQLITE_OS_OTHER. */; - opfsIoMethods.$iVersion = 1; - opfsVfs.$iVersion = 2/*yes, two*/; - opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; - opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; - opfsVfs.addOnDispose( - opfsVfs.$zName = wasm.allocCString(vfsName), - ()=>(dVfs ? dVfs.dispose() : null) - ); /** Maintenance reminder: the order of ASYNC ops in this function @@ -755,264 +1083,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(options.$testThrowInInit){ throw options.$testThrowInInit; } - const thePool = new OpfsSAHPool(opfsVfs, options); + const thePool = new OpfsSAHPool(options); return thePool.isReady.then(async()=>{ - /** - Impls for the sqlite3_io_methods methods. Maintenance reminder: - members are in alphabetical order to simplify finding them. - */ - const ioMethods = { - xCheckReservedLock: function(pFile,pOut){ - log('xCheckReservedLock'); - thePool.storeErr(); - wasm.poke32(pOut, 1); - return 0; - }, - xClose: function(pFile){ - thePool.storeErr(); - const file = thePool.getFileForPtr(pFile); - if(file) { - try{ - log(`xClose ${file.path}`); - if(file.sq3File) file.sq3File.dispose(); - file.sah.flush(); - thePool.setFileForPtr(pFile,0); - if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ - thePool.deletePath(file.path); - } - }catch(e){ - thePool.storeErr(e); - return capi.SQLITE_IOERR; - } - } - return 0; - }, - xDeviceCharacteristics: function(pFile){ - return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; - }, - xFileControl: function(pFile, opId, pArg){ - return capi.SQLITE_NOTFOUND; - }, - xFileSize: function(pFile,pSz64){ - log(`xFileSize`); - const file = thePool.getFileForPtr(pFile); - const size = file.sah.getSize() - HEADER_OFFSET_DATA; - //log(`xFileSize ${file.path} ${size}`); - wasm.poke64(pSz64, BigInt(size)); - return 0; - }, - xLock: function(pFile,lockType){ - log(`xLock ${lockType}`); - thePool.storeErr(); - const file = thePool.getFileForPtr(pFile); - file.lockType = lockType; - return 0; - }, - xRead: function(pFile,pDest,n,offset64){ - log(`xRead ${n}@${offset64}`); - thePool.storeErr(); - const file = thePool.getFileForPtr(pFile); - log(`xRead ${file.path} ${n} ${offset64}`); - try { - const nRead = file.sah.read( - wasm.heap8u().subarray(pDest, pDest+n), - {at: HEADER_OFFSET_DATA + Number(offset64)} - ); - if(nRead < n){ - wasm.heap8u().fill(0, pDest + nRead, pDest + n); - return capi.SQLITE_IOERR_SHORT_READ; - } - return 0; - }catch(e){ - thePool.storeErr(e); - return capi.SQLITE_IOERR; - } - }, - xSectorSize: function(pFile){ - return SECTOR_SIZE; - }, - xSync: function(pFile,flags){ - log(`xSync ${flags}`); - thePool.storeErr(); - const file = thePool.getFileForPtr(pFile); - //log(`xSync ${file.path} ${flags}`); - try{ - file.sah.flush(); - return 0; - }catch(e){ - thePool.storeErr(e); - return capi.SQLITE_IOERR; - } - }, - xTruncate: function(pFile,sz64){ - log(`xTruncate ${sz64}`); - thePool.storeErr(); - const file = thePool.getFileForPtr(pFile); - //log(`xTruncate ${file.path} ${iSize}`); - try{ - file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); - return 0; - }catch(e){ - thePool.storeErr(e); - return capi.SQLITE_IOERR; - } - }, - xUnlock: function(pFile,lockType){ - log('xUnlock'); - const file = thePool.getFileForPtr(pFile); - file.lockType = lockType; - return 0; - }, - xWrite: function(pFile,pSrc,n,offset64){ - thePool.storeErr(); - const file = thePool.getFileForPtr(pFile); - log(`xWrite ${file.path} ${n} ${offset64}`); - try{ - const nBytes = file.sah.write( - wasm.heap8u().subarray(pSrc, pSrc+n), - { at: HEADER_OFFSET_DATA + Number(offset64) } - ); - return nBytes === n ? 0 : capi.SQLITE_IOERR; - }catch(e){ - thePool.storeErr(e); - return capi.SQLITE_IOERR; - } - } - }/*ioMethods*/; - /** - Impls for the sqlite3_vfs methods. Maintenance reminder: members - are in alphabetical order to simplify finding them. - */ - const vfsMethods = { - xAccess: function(pVfs,zName,flags,pOut){ - //log(`xAccess ${wasm.cstrToJs(zName)}`); - thePool.storeErr(); - try{ - const name = thePool.getPath(zName); - wasm.poke32(pOut, thePool.hasFilename(name) ? 1 : 0); - }catch(e){ - /*ignored*/ - wasm.poke32(pOut, 0); - } - return 0; - }, - xCurrentTime: function(pVfs,pOut){ - wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), - 'double'); - return 0; - }, - xCurrentTimeInt64: function(pVfs,pOut){ - wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), - 'i64'); - return 0; - }, - xDelete: function(pVfs, zName, doSyncDir){ - log(`xDelete ${wasm.cstrToJs(zName)}`); - thePool.storeErr(); - try{ - thePool.deletePath(thePool.getPath(zName)); - return 0; - }catch(e){ - thePool.storeErr(e); - return capi.SQLITE_IOERR_DELETE; - } - }, - xFullPathname: function(pVfs,zName,nOut,pOut){ - log(`xFullPathname ${wasm.cstrToJs(zName)}`); - const i = wasm.cstrncpy(pOut, zName, nOut); - return i nOut) wasm.poke8(pOut + nOut - 1, 0); - }catch(e){ - return capi.SQLITE_NOMEM; - }finally{ - wasm.scopedAllocPop(scope); - } - } - return 0; - }, - //xSleep is optionally defined below - xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ - log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`); - try{ - // First try to open a path that already exists in the file system. - const path = (zName && wasm.peek8(zName)) - ? thePool.getPath(zName) - : getRandomName(); - let sah = thePool.getSAHForPath(path); - if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { - // File not found so try to create it. - if(thePool.getFileCount() < thePool.getCapacity()) { - // Choose an unassociated OPFS file from the pool. - sah = thePool.nextAvailableSAH(); - thePool.setAssociatedPath(sah, path, flags); - }else{ - // File pool is full. - toss('SAH pool is full. Cannot create file',path); - } - } - if(!sah){ - toss('file not found:',path); - } - // Subsequent methods are only passed the file pointer, so - // map the relevant info we need to that pointer. - const file = {path, flags, sah}; - thePool.setFileForPtr(pFile, file); - wasm.poke32(pOutFlags, flags); - file.sq3File = new capi.sqlite3_file(pFile); - file.sq3File.$pMethods = opfsIoMethods.pointer; - file.lockType = capi.SQLITE_LOCK_NONE; - return 0; - }catch(e){ - thePool.storeErr(e); - return capi.SQLITE_CANTOPEN; - } - }/*xOpen()*/ - }/*vfsMethods*/; - - if(dVfs){ - /* Inherit certain VFS members from the default VFS, - if available. */ - opfsVfs.$xRandomness = dVfs.$xRandomness; - opfsVfs.$xSleep = dVfs.$xSleep; - } - if(!opfsVfs.$xRandomness){ - /* If the default VFS has no xRandomness(), add a basic JS impl... */ - vfsMethods.xRandomness = function(pVfs, nOut, pOut){ - const heap = wasm.heap8u(); - let i = 0; - for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; - return i; - }; - } - if(!opfsVfs.$xSleep){ - vfsMethods.xSleep = (pVfs,ms)=>0; - } - /** The poolUtil object will be the result of the resolved Promise. */ const poolUtil = new SAHPoolUtil(thePool); - //log("vfs list:",capi.sqlite3_js_vfs_list()); - sqlite3.vfs.installVfs({ - io: {struct: opfsIoMethods, methods: ioMethods}, - vfs: {struct: opfsVfs, methods: vfsMethods} - }); - //log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods); - //log("vfs list:",capi.sqlite3_js_vfs_list()); if(sqlite3.oo1){ const oo1 = sqlite3.oo1; + const theVfs = thePool.getVfs(); const OpfsSAHPoolDb = function(...args){ const opt = oo1.DB.dbCtorHelper.normalizeArgs(...args); - opt.vfs = opfsVfs.$zName; + opt.vfs = theVfs.$zName; oo1.DB.dbCtorHelper.call(this, opt); }; OpfsSAHPoolDb.prototype = Object.create(oo1.DB.prototype); @@ -1023,7 +1105,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } oo1.OpfsSAHPool[vfsName] = OpfsSAHPoolDb; oo1.DB.dbCtorHelper.setVfsPostOpenSql( - opfsVfs.pointer, + theVfs.pointer, function(oo1Db, sqlite3){ sqlite3.capi.sqlite3_exec(oo1Db, [ /* See notes in sqlite3-vfs-opfs.js */ @@ -1033,12 +1115,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } ); }/*extend sqlite3.oo1*/ - log("VFS initialized."); + thePool.log("VFS initialized."); return poolUtil; + }).catch(async (e)=>{ + await thePool.removeVfs().catch(()=>{}); + return e; }); }).catch((err)=>{ - error("rejecting promise:",err); - opfsVfs.dispose(); + //error("rejecting promise:",err); return initPromises[vfsName] = Promise.reject(err); }); }/*installOpfsSAHPoolVfs()*/; diff --git a/ext/wasm/common/testing.css b/ext/wasm/common/testing.css index 71a19aa95e..a9be76764c 100644 --- a/ext/wasm/common/testing.css +++ b/ext/wasm/common/testing.css @@ -61,6 +61,7 @@ span.labeled-input { margin-left: 3em; } .skipping-test, .skipping-group { + padding: 0.25em 0.5em; background-color: #ffff73; } .skipping-test { diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 9d56340aae..56a4369cd5 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -296,7 +296,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; roundMs(then-now),'ms'); } logClass(['green','group-end'], - //"Group #"+this.number+":", + "#"+this.number+":", (TestUtil.counter - assertCount), "assertion(s) in",roundMs(runtime),"ms"); if(0 && skipped.length){ diff --git a/manifest b/manifest index d6fff1e2eb..c2f5821f0a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Appearance\sand\slegibility\simprovements\sfor\stester1.js. -D 2023-07-19T11:33:52.985 +C Major\srestructuring\sof\sopfs-sahpool\sto\simprove\smaintainability\sand\shopefully\smake\sit\seasier\sto\sreuse\sthese\spieces\sin\sthe\supcoming\sJSPI-based\sVFS\sexperiment. +D 2023-07-19T13:31:29.555 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -501,8 +501,8 @@ F ext/wasm/api/sqlite3-api-prologue.js d747cbb379e13881c9edf39dce019cbbbae860c45 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 -F ext/wasm/api/sqlite3-v-helper.js e4b7b27a8259e40407b3c16e42dd5df05b80726c609594cc23b1565dc2ad9ca2 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 9a8ba44b775b0e8faaffc8d877cfef0726713c2ca368e5776b59d9029f5ebf23 +F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js d3e41757230c8a41fccc4db077d029546f0ebccd13d4ba0111c52ca77779ab70 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4946af0d6fbd395aa39966562ca85900664605a5f0cc10fff50146dee527812c F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -512,7 +512,7 @@ F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c2 F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 -F ext/wasm/common/testing.css cae1f8cbfc9f516e22af0524deab10569120860ea8f12bf73204838cee6b63a1 +F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f F ext/wasm/common/whwasmutil.js ae263dec9d7384f4c530f324b99d00516a4d6f26424372daee65031e00eb49b3 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js d20a88f99fa78d50bed4de69270aed4902ae7901071797f3e3b095684736d830 +F ext/wasm/tester1.c-pp.js b99aa30d9b54c5c60f67381b249d290a542c529898852c32c5645f5a33be9498 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6bd5a7413dd830ca41b587a2826fb599a2196fb0186646a2333500f950b3cf4d -R b499063b7173978725e0c6adeaf3d1ef +P 4fd3ed1b6405f690825c9993e56b199d4a7fb497cf9131815c803150b8b96528 +R 304feb0212e249ee7b7b77df509eb682 U stephan -Z aeff227bd99478c07e8d0a15233f4c66 +Z 21bc5315f59e1cd6b79324f9b7546c07 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8a229d33f9..6e67b51d63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4fd3ed1b6405f690825c9993e56b199d4a7fb497cf9131815c803150b8b96528 \ No newline at end of file +534481cd0c2e6f62dd0a82f25d4b78fdcc671eb70d6966693c98212a6420891c \ No newline at end of file From 7a816e771a803d2be135626b7cb6d7506967acc6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Jul 2023 13:50:31 +0000 Subject: [PATCH 35/56] Performance optimization for parsing large JSONs that contain lots of text. FossilOrigin-Name: c9fbe0185cd5d64950724b00cd0bfb3a7939a985040465a0f35f445acb6e94a6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 21 ++++++++++----------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 575934e4e2..443ffbe17e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Two\sminor\simprovements\sto\ssum(),\sone\sof\swhich\swas\sinspired\sby\n[forum:/forumpost/af5be98dbc|forum\spost\saf5be98dbc]. -D 2023-07-19T09:52:10.467 +C Performance\soptimization\sfor\sparsing\slarge\sJSONs\sthat\scontain\slots\sof\stext. +D 2023-07-19T13:50:31.934 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 14c474fb1249a46eb44e878e2361f36abfe686b134039b0d1883d93d61505b4a +F src/json.c a1e70adf94131e29cdaabc19523a92b2df72ef6305b80d5493369e278267dfe6 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 76152ad2ffe56034f2fd93d9a1ce9358e1677a7e9cd3dcd9f3a34a5c956a463e -R 47d8d8ca9ec8151ad0210262a08ac711 +P a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5 +R 35635814f4d35d8abae453f3c0b6b67f U drh -Z b98de7129167c58b9385610fbdec3119 +Z a4b178ad01a69bfb9f8e963eafa3e384 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4a5750f1d2..cb58a338e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5 \ No newline at end of file +c9fbe0185cd5d64950724b00cd0bfb3a7939a985040465a0f35f445acb6e94a6 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6bad1c1e75..b67da9327a 100644 --- a/src/json.c +++ b/src/json.c @@ -1235,15 +1235,9 @@ json_parse_restart: jnFlags = 0; parse_string: cDelim = z[i]; - j = i+1; - for(;;){ - c = z[j]; - if( (c & ~0x1f)==0 ){ - /* Control characters are not allowed in strings */ - pParse->iErr = j; - return -1; - } - if( c=='\\' ){ + for(j=i+1; 1; j++){ + unsigned char uc = (unsigned char)z[j]; + if( uc=='\\' ){ c = z[++j]; if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' || c=='n' || c=='r' || c=='t' @@ -1263,10 +1257,15 @@ json_parse_restart: pParse->iErr = j; return -1; } - }else if( c==cDelim ){ + }else if( uc>'\'' ){ + continue; + }else if( uc==cDelim ){ break; + }else if( uc<=0x1f ){ + /* Control characters are not allowed in strings */ + pParse->iErr = j; + return -1; } - j++; } jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]); return j+1; From 8376ae7295a241fdac53ef67e8be261b069d39ba Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Jul 2023 15:06:29 +0000 Subject: [PATCH 36/56] Further performance enhancements to JSON parsing and rendering. Total performance gain for large JSONs so far on this branch is about 11%. FossilOrigin-Name: adb4d6b007cbe9d7c9670f5fc196443ebe0f3a89df1f3290ba6247fcf83fe5bd --- manifest | 12 +++---- manifest.uuid | 2 +- src/json.c | 94 +++++++++++++++++++++++++++++++++++---------------- 3 files changed, 72 insertions(+), 36 deletions(-) diff --git a/manifest b/manifest index 443ffbe17e..5c20d00512 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sfor\sparsing\slarge\sJSONs\sthat\scontain\slots\sof\stext. -D 2023-07-19T13:50:31.934 +C Further\sperformance\senhancements\sto\sJSON\sparsing\sand\srendering.\s\sTotal\nperformance\sgain\sfor\slarge\sJSONs\sso\sfar\son\sthis\sbranch\sis\sabout\s11%. +D 2023-07-19T15:06:29.535 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c a1e70adf94131e29cdaabc19523a92b2df72ef6305b80d5493369e278267dfe6 +F src/json.c 4a875e61ce6aa869bd6028836f198e7ab08d3efceb8ab8d0529bec8061b67e94 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5 -R 35635814f4d35d8abae453f3c0b6b67f +P c9fbe0185cd5d64950724b00cd0bfb3a7939a985040465a0f35f445acb6e94a6 +R dbdd171b9775d82e33a4a9295a6ddced U drh -Z a4b178ad01a69bfb9f8e963eafa3e384 +Z 04400ecbf8a4a25997341f5b2d7f0282 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cb58a338e7..7057be53c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c9fbe0185cd5d64950724b00cd0bfb3a7939a985040465a0f35f445acb6e94a6 \ No newline at end of file +adb4d6b007cbe9d7c9670f5fc196443ebe0f3a89df1f3290ba6247fcf83fe5bd \ No newline at end of file diff --git a/src/json.c b/src/json.c index b67da9327a..ed43a5ce96 100644 --- a/src/json.c +++ b/src/json.c @@ -218,12 +218,35 @@ static int jsonGrow(JsonString *p, u32 N){ /* Append N bytes from zIn onto the end of the JsonString string. */ -static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ - if( N==0 ) return; - if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; +static SQLITE_NOINLINE void jsonAppendExpand( + JsonString *p, + const char *zIn, + u32 N +){ + assert( N>0 ); + if( jsonGrow(p,N) ) return; memcpy(p->zBuf+p->nUsed, zIn, N); p->nUsed += N; } +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( N==0 ) return; + if( N+p->nUsed >= p->nAlloc ){ + jsonAppendExpand(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} +static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ + assert( N>0 ); + if( N+p->nUsed >= p->nAlloc ){ + jsonAppendExpand(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} + /* Append formatted text (not to exceed N bytes) to the JsonString. */ @@ -238,10 +261,17 @@ static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ /* Append a single character */ -static void jsonAppendChar(JsonString *p, char c){ - if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; +static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){ + if( jsonGrow(p,1) ) return; p->zBuf[p->nUsed++] = c; } +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc ){ + jsonAppendCharExpand(p,c); + }else{ + p->zBuf[p->nUsed++] = c; + } +} /* Append a comma separator to the output buffer, if the previous ** character is not '[' or '{'. @@ -250,7 +280,8 @@ static void jsonAppendSeparator(JsonString *p){ char c; if( p->nUsed==0 ) return; c = p->zBuf[p->nUsed-1]; - if( c!='[' && c!='{' ) jsonAppendChar(p, ','); + if( c=='[' || c=='{' ) return; + jsonAppendChar(p, ','); } /* Append the N-byte string in zIn to the end of the JsonString string @@ -310,7 +341,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ while( N>0 ){ for(i=0; i0 ){ - jsonAppendRaw(p, zIn, i); + jsonAppendRawNZ(p, zIn, i); zIn += i; N -= i; if( N==0 ) break; @@ -321,16 +352,16 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendChar(p, '\''); break; case 'v': - jsonAppendRaw(p, "\\u0009", 6); + jsonAppendRawNZ(p, "\\u0009", 6); break; case 'x': - jsonAppendRaw(p, "\\u00", 4); - jsonAppendRaw(p, &zIn[2], 2); + jsonAppendRawNZ(p, "\\u00", 4); + jsonAppendRawNZ(p, &zIn[2], 2); zIn += 2; N -= 2; break; case '0': - jsonAppendRaw(p, "\\u0000", 6); + jsonAppendRawNZ(p, "\\u0000", 6); break; case '\r': if( zIn[2]=='\n' ){ @@ -348,7 +379,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ N -= 2; break; default: - jsonAppendRaw(p, zIn, 2); + jsonAppendRawNZ(p, zIn, 2); break; } zIn += 2; @@ -378,11 +409,12 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ jsonPrintf(100,p,"%lld",i); }else{ assert( rc==2 ); - jsonAppendRaw(p, "9.0e999", 7); + jsonAppendRawNZ(p, "9.0e999", 7); } return; } - jsonAppendRaw(p, zIn, N); + assert( N>0 ); + jsonAppendRawNZ(p, zIn, N); } /* @@ -414,7 +446,7 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ } } if( N>0 ){ - jsonAppendRaw(p, zIn, N); + jsonAppendRawNZ(p, zIn, N); } } @@ -430,7 +462,7 @@ static void jsonAppendValue( ){ switch( sqlite3_value_type(pValue) ){ case SQLITE_NULL: { - jsonAppendRaw(p, "null", 4); + jsonAppendRawNZ(p, "null", 4); break; } case SQLITE_FLOAT: { @@ -538,15 +570,15 @@ static void jsonRenderNode( switch( pNode->eType ){ default: { assert( pNode->eType==JSON_NULL ); - jsonAppendRaw(pOut, "null", 4); + jsonAppendRawNZ(pOut, "null", 4); break; } case JSON_TRUE: { - jsonAppendRaw(pOut, "true", 4); + jsonAppendRawNZ(pOut, "true", 4); break; } case JSON_FALSE: { - jsonAppendRaw(pOut, "false", 5); + jsonAppendRawNZ(pOut, "false", 5); break; } case JSON_STRING: { @@ -562,7 +594,8 @@ static void jsonRenderNode( }else if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); }else{ - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + assert( pNode->n>0 ); + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); } break; } @@ -571,7 +604,8 @@ static void jsonRenderNode( if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); }else{ - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + assert( pNode->n>0 ); + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); } break; } @@ -580,7 +614,8 @@ static void jsonRenderNode( if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); }else{ - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + assert( pNode->n>0 ); + jsonAppendRawNZ(pOut, pNode->u.zJContent, pNode->n); } break; } @@ -882,7 +917,8 @@ static int jsonParseAddNode( const char *zContent /* Content */ ){ JsonNode *p; - if( pParse->aNode==0 || pParse->nNode>=pParse->nAlloc ){ + assert( pParse->aNode!=0 || pParse->nNode>=pParse->nAlloc ); + if( pParse->nNode>=pParse->nAlloc ){ return jsonParseAddNodeExpand(pParse, eType, n, zContent); } p = &pParse->aNode[pParse->nNode]; @@ -2002,12 +2038,12 @@ static void jsonParseFunc( assert( x.aNode[i].eU==0 || x.aNode[i].eU==1 ); if( x.aNode[i].u.zJContent!=0 ){ assert( x.aNode[i].eU==1 ); - jsonAppendRaw(&s, " ", 1); + jsonAppendChar(&s, ' '); jsonAppendRaw(&s, x.aNode[i].u.zJContent, x.aNode[i].n); }else{ assert( x.aNode[i].eU==0 ); } - jsonAppendRaw(&s, "\n", 1); + jsonAppendChar(&s, '\n'); } jsonParseReset(&x); jsonResult(&s); @@ -2174,11 +2210,11 @@ static void jsonExtractFunc( */ jsonInit(&jx, ctx); if( sqlite3Isdigit(zPath[0]) ){ - jsonAppendRaw(&jx, "$[", 2); + jsonAppendRawNZ(&jx, "$[", 2); jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); - jsonAppendRaw(&jx, "]", 2); + jsonAppendRawNZ(&jx, "]", 2); }else{ - jsonAppendRaw(&jx, "$.", 1 + (zPath[0]!='[')); + jsonAppendRawNZ(&jx, "$.", 1 + (zPath[0]!='[')); jsonAppendRaw(&jx, zPath, (int)strlen(zPath)); jsonAppendChar(&jx, 0); } @@ -2213,7 +2249,7 @@ static void jsonExtractFunc( if( pNode ){ jsonRenderNode(pNode, &jx, 0); }else{ - jsonAppendRaw(&jx, "null", 4); + jsonAppendRawNZ(&jx, "null", 4); } } if( i==argc ){ From 50834d8b74a88736dcf50e334197bcebe54b55ec Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Jul 2023 17:24:36 +0000 Subject: [PATCH 37/56] Further improvement to JSON parser performance. FossilOrigin-Name: 144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 32 +++++++++++++++++++++++++------- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 5c20d00512..140124f2f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sperformance\senhancements\sto\sJSON\sparsing\sand\srendering.\s\sTotal\nperformance\sgain\sfor\slarge\sJSONs\sso\sfar\son\sthis\sbranch\sis\sabout\s11%. -D 2023-07-19T15:06:29.535 +C Further\simprovement\sto\sJSON\sparser\sperformance. +D 2023-07-19T17:24:36.452 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 4a875e61ce6aa869bd6028836f198e7ab08d3efceb8ab8d0529bec8061b67e94 +F src/json.c e48136fce64e5004b1b8be76624cc9a4ac9ff6ae630a97df187c0ea4b9692d1b F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c9fbe0185cd5d64950724b00cd0bfb3a7939a985040465a0f35f445acb6e94a6 -R dbdd171b9775d82e33a4a9295a6ddced +P adb4d6b007cbe9d7c9670f5fc196443ebe0f3a89df1f3290ba6247fcf83fe5bd +R ab82eb710f468841d4023aaae186be2d U drh -Z 04400ecbf8a4a25997341f5b2d7f0282 +Z d9c18383c20eeea0feca70f3dd0775a9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7057be53c2..4a7e995a3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adb4d6b007cbe9d7c9670f5fc196443ebe0f3a89df1f3290ba6247fcf83fe5bd \ No newline at end of file +144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6 \ No newline at end of file diff --git a/src/json.c b/src/json.c index ed43a5ce96..176dcbfdb9 100644 --- a/src/json.c +++ b/src/json.c @@ -1272,8 +1272,30 @@ json_parse_restart: parse_string: cDelim = z[i]; for(j=i+1; 1; j++){ - unsigned char uc = (unsigned char)z[j]; - if( uc=='\\' ){ + static const char aOk[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + if( aOk[(unsigned char)z[j]] ) continue; + c = z[j]; + if( c==cDelim ){ + break; + }else if( c=='\\' ){ c = z[++j]; if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' || c=='n' || c=='r' || c=='t' @@ -1293,11 +1315,7 @@ json_parse_restart: pParse->iErr = j; return -1; } - }else if( uc>'\'' ){ - continue; - }else if( uc==cDelim ){ - break; - }else if( uc<=0x1f ){ + }else if( c<=0x1f ){ /* Control characters are not allowed in strings */ pParse->iErr = j; return -1; From 96cb7007a95387eeb63fc88835d47acbb867eca7 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 19 Jul 2023 17:46:28 +0000 Subject: [PATCH 38/56] Update the development-over-ssh docs for the wasm build. FossilOrigin-Name: 500109bd0a4c134b91c37f397ff1ee828e09c17f7ecd153f975ede748caee7bb --- ext/wasm/README.md | 17 +++++++++-------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/ext/wasm/README.md b/ext/wasm/README.md index e8d66865d8..769fe9edca 100644 --- a/ext/wasm/README.md +++ b/ext/wasm/README.md @@ -82,7 +82,7 @@ features in the apps which use them. # Testing on a remote machine that is accessed via SSH -*NB: The following are developer notes, last validated on 2022-08-18* +*NB: The following are developer notes, last validated on 2023-07-19* * Remote: Install git, emsdk, and althttpd * Use a [version of althttpd][althttpd] from @@ -90,16 +90,17 @@ features in the apps which use them. * Remote: Install the SQLite source tree. CD to ext/wasm * Remote: "`make`" to build WASM * Remote: `althttpd --enable-sab --port 8080 --popup` - * Local: `ssh -L 8180:localhost:8080 remote` + * Local: `ssh -L 8180:remote:8080 remote` * Local: Point your web-browser at http://localhost:8180/index.html -In order to enable [SharedArrayBuffers](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer), -the web-browser requires that the two extra Cross-Origin lines be present -in HTTP reply headers and that the request must come from "localhost". -Since the web-server is on a different machine from -the web-broser, the localhost requirement means that the connection must be tunneled -using SSH. +In order to enable [SharedArrayBuffer][], the web-browser requires +that the two extra Cross-Origin lines be present in HTTP reply headers +and that the request must come from "localhost" (_or_ over an SSL +connection). Since the web-server is on a different machine from the +web-broser, the localhost requirement means that the connection must +be tunneled using SSH. [emscripten]: https://emscripten.org [althttpd]: https://sqlite.org/althttpd +[SharedArrayBuffer]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer diff --git a/manifest b/manifest index c2f5821f0a..46c9678ff8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Major\srestructuring\sof\sopfs-sahpool\sto\simprove\smaintainability\sand\shopefully\smake\sit\seasier\sto\sreuse\sthese\spieces\sin\sthe\supcoming\sJSPI-based\sVFS\sexperiment. -D 2023-07-19T13:31:29.555 +C Update\sthe\sdevelopment-over-ssh\sdocs\sfor\sthe\swasm\sbuild. +D 2023-07-19T17:46:28.936 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865 F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c F ext/wasm/GNUmakefile b425091409491f0f9dca77f7a41143530f8b7d37abfb3ba59f1d50f4cc85d02f F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 -F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 +F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4fd3ed1b6405f690825c9993e56b199d4a7fb497cf9131815c803150b8b96528 -R 304feb0212e249ee7b7b77df509eb682 +P 534481cd0c2e6f62dd0a82f25d4b78fdcc671eb70d6966693c98212a6420891c +R 3098985ac29097adbc0d24d2b5daf2dc U stephan -Z 21bc5315f59e1cd6b79324f9b7546c07 +Z 78455e14df71dd2dbc1be007d9b53932 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6e67b51d63..7bb3f99cae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -534481cd0c2e6f62dd0a82f25d4b78fdcc671eb70d6966693c98212a6420891c \ No newline at end of file +500109bd0a4c134b91c37f397ff1ee828e09c17f7ecd153f975ede748caee7bb \ No newline at end of file From 55f318e53f0dc9343b2bc87956a3ea263e06eba3 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 19 Jul 2023 17:47:02 +0000 Subject: [PATCH 39/56] More internal refactoring and docs for opfs-sahpool. FossilOrigin-Name: 64ccf6177a019eab46fb3345ad1e8ba80eaf2c9da55767031f9f04ccd16afb4d --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 116 +++++++++++++++-------- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 81 insertions(+), 49 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index b0d71c50ee..f34c87004d 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -48,8 +48,9 @@ incompatible with that VFS. - This VFS requires the "semi-fully-sync" FileSystemSyncAccessHandle - (hereafter "SAH") APIs released with Chrome v108. If that API - is not detected, the VFS is not registered. + (hereafter "SAH") APIs released with Chrome v108 (and all other + major browsers released since March 2023). If that API is not + detected, the VFS is not registered. */ 'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ @@ -137,11 +138,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xClose: function(pFile){ const pool = getPoolForPFile(pFile); pool.storeErr(); - const file = pool.getFileForPtr(pFile); + const file = pool.getOFileForSFile(pFile); if(file) { try{ pool.log(`xClose ${file.path}`); - pool.setFileForPtr(pFile, false); + pool.mapSFileToOFile(pFile, false); file.sah.flush(); if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ pool.deletePath(file.path); @@ -162,7 +163,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xFileSize: function(pFile,pSz64){ const pool = getPoolForPFile(pFile); pool.log(`xFileSize`); - const file = pool.getFileForPtr(pFile); + const file = pool.getOFileForSFile(pFile); const size = file.sah.getSize() - HEADER_OFFSET_DATA; //log(`xFileSize ${file.path} ${size}`); wasm.poke64(pSz64, BigInt(size)); @@ -172,14 +173,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pool = getPoolForPFile(pFile); pool.log(`xLock ${lockType}`); pool.storeErr(); - const file = pool.getFileForPtr(pFile); + const file = pool.getOFileForSFile(pFile); file.lockType = lockType; return 0; }, xRead: function(pFile,pDest,n,offset64){ const pool = getPoolForPFile(pFile); pool.storeErr(); - const file = pool.getFileForPtr(pFile); + const file = pool.getOFileForSFile(pFile); pool.log(`xRead ${file.path} ${n} @ ${offset64}`); try { const nRead = file.sah.read( @@ -203,7 +204,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pool = getPoolForPFile(pFile); pool.log(`xSync ${flags}`); pool.storeErr(); - const file = pool.getFileForPtr(pFile); + const file = pool.getOFileForSFile(pFile); //log(`xSync ${file.path} ${flags}`); try{ file.sah.flush(); @@ -217,7 +218,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pool = getPoolForPFile(pFile); pool.log(`xTruncate ${sz64}`); pool.storeErr(); - const file = pool.getFileForPtr(pFile); + const file = pool.getOFileForSFile(pFile); //log(`xTruncate ${file.path} ${iSize}`); try{ file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); @@ -230,14 +231,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xUnlock: function(pFile,lockType){ const pool = getPoolForPFile(pFile); pool.log('xUnlock'); - const file = pool.getFileForPtr(pFile); + const file = pool.getOFileForSFile(pFile); file.lockType = lockType; return 0; }, xWrite: function(pFile,pSrc,n,offset64){ const pool = getPoolForPFile(pFile); pool.storeErr(); - const file = pool.getFileForPtr(pFile); + const file = pool.getOFileForSFile(pFile); pool.log(`xWrite ${file.path} ${n} ${offset64}`); try{ const nBytes = file.sah.write( @@ -349,7 +350,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ // Subsequent I/O methods are only passed the sqlite3_file // pointer, so map the relevant info we need to that pointer. const file = {path, flags, sah}; - pool.setFileForPtr(pFile, file); + pool.mapSFileToOFile(pFile, file); file.lockType = capi.SQLITE_LOCK_NONE; const sq3File = new capi.sqlite3_file(pFile); sq3File.$pMethods = opfsIoMethods.pointer; @@ -436,6 +437,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Buffer used by [sg]etAssociatedPath(). */ #apBody = new Uint8Array(HEADER_CORPUS_SIZE); + // DataView for this.#apBody #dvBody; // associated sqlite3_vfs instance @@ -497,6 +499,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return this.getCapacity(); } + /** + Reduce capacity by n, but can only reduce up to the limit + of currently-available SAHs. Returns a Promise which resolves + to the number of slots really removed. + */ async reduceCapacity(n){ let nRm = 0; for(const ah of Array.from(this.#availableSAH)){ @@ -514,7 +521,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } /** - Releases all currently-opened SAHs. + Releases all currently-opened SAHs. The only legal + operation after this is acquireAccessHandles(). */ releaseAccessHandles(){ for(const ah of this.#mapSAHToName.keys()) ah.close(); @@ -637,8 +645,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } /** - Computes a digest for the given byte array and - returns it as a two-element Uint32Array. + Computes a digest for the given byte array and returns it as a + two-element Uint32Array. This digest gets stored in the + metadata for each file as a validation check. Changing this + algorithm invalidates all existing databases for this VFS, so + don't do that. */ computeDigest(byteArray){ let h1 = 0xdeadbeef; @@ -730,14 +741,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return rc; } - getFileForPtr(ptr){ + /** + Given an (sqlite3_file*), returns the mapped + xOpen file object. + */ + getOFileForSFile(ptr){ return this.#mapSqlite3FileToFile.get(ptr); } /** Maps or unmaps (if file is falsy) the given (sqlite3_file*) to an xOpen file object and to this pool object. */ - setFileForPtr(pFile,file){ + mapSFileToOFile(pFile,file){ if(file){ this.#mapSqlite3FileToFile.set(pFile, file); setPoolForPFile(pFile, this); @@ -746,14 +761,34 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ setPoolForPFile(pFile, false); } } + + /** + Returns true if the given client-defined file name is in this + object's name-to-SAH map. + */ hasFilename(name){ return this.#mapFilenameToSAH.has(name) } + /** + Returns the SAH associated with the given + client-defined file name. + */ getSAHForPath(path){ return this.#mapFilenameToSAH.get(path); } + /** + Removes this object's sqlite3_vfs registration and shuts down + this object, releasing all handles, mappings, and whatnot, + including deleting its data directory. There is currently no + way to "revive" the object and reaquire its resources. + + This function is intended primarily for testing. + + Resolves to true if it did its job, false if the + VFS has already been shut down. + */ async removeVfs(){ if(!this.#cVfs.pointer) return false; capi.sqlite3_vfs_unregister(this.#cVfs.pointer); @@ -773,6 +808,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return true; } + //! Documented elsewhere in this file. exportFile(name){ const sah = this.#mapFilenameToSAH.get(name) || toss("File not found:",name); const n = sah.getSize() - HEADER_OFFSET_DATA; @@ -781,6 +817,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return b; } + //! Documented elsewhere in this file. importDb(name, bytes){ const n = bytes.byteLength; if(n<512 || n%512!=0){ @@ -803,13 +840,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** - A SAHPoolUtil instance is exposed to clients in order to manipulate an OpfsSAHPool object without directly exposing that + A OpfsSAHPoolUtil instance is exposed to clients in order to + manipulate an OpfsSAHPool object without directly exposing that object and allowing for some semantic changes compared to that class. - Class docs are in the client-level docs for installOpfsSAHPoolVfs(). + Class docs are in the client-level docs for + installOpfsSAHPoolVfs(). */ - class SAHPoolUtil { + class OpfsSAHPoolUtil { /* This object's associated OpfsSAHPool. */ #p; @@ -818,18 +857,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ this.vfsName = sahPool.vfsName; } - async addCapacity(n){ - return this.#p.addCapacity(n); - } - async reduceCapacity(n){ - return this.#p.reduceCapacity(n); - } - getCapacity(){ - return this.#p.getCapacity(this.#p); - } - getFileCount(){ - return this.#p.getFileCount(); - } + async addCapacity(n){ return this.#p.addCapacity(n) } + + async reduceCapacity(n){ return this.#p.reduceCapacity(n) } + + getCapacity(){ return this.#p.getCapacity(this.#p) } + + getFileCount(){ return this.#p.getFileCount() } + async reserveMinimumCapacity(min){ const c = this.#p.getCapacity(); return (c < min) ? this.#p.addCapacity(min - c) : c; @@ -839,20 +874,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ importDb(name, bytes){ return this.#p.importDb(name,bytes) } - async wipeFiles(){return this.#p.reset(true)} + async wipeFiles(){ return this.#p.reset(true) } - unlink(filename){ - return this.#p.deletePath(filename); - } + unlink(filename){ return this.#p.deletePath(filename) } - async removeVfs(){return this.#p.removeVfs()} + async removeVfs(){ return this.#p.removeVfs() } - }/* class SAHPoolUtil */; + }/* class OpfsSAHPoolUtil */; /** - Ensure that the client has a "fully-sync" SAH impl, - else reject the promise. Returns true on success, - throws on error. + Returns a resolved Promise if the current environment + has a "fully-sync" SAH impl, else a rejected Promise. */ const apiVersionCheck = async ()=>{ const dh = await navigator.storage.getDirectory(); @@ -1087,7 +1119,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return thePool.isReady.then(async()=>{ /** The poolUtil object will be the result of the resolved Promise. */ - const poolUtil = new SAHPoolUtil(thePool); + const poolUtil = new OpfsSAHPoolUtil(thePool); if(sqlite3.oo1){ const oo1 = sqlite3.oo1; diff --git a/manifest b/manifest index 46c9678ff8..9bf3bd22cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdevelopment-over-ssh\sdocs\sfor\sthe\swasm\sbuild. -D 2023-07-19T17:46:28.936 +C More\sinternal\srefactoring\sand\sdocs\sfor\sopfs-sahpool. +D 2023-07-19T17:47:02.768 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js d3e41757230c8a41fccc4db077d029546f0ebccd13d4ba0111c52ca77779ab70 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 05b5646b91faa947833d43a840e8b94abb441afa953ee5a11cc7f07f4e01361a F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4946af0d6fbd395aa39966562ca85900664605a5f0cc10fff50146dee527812c F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 534481cd0c2e6f62dd0a82f25d4b78fdcc671eb70d6966693c98212a6420891c -R 3098985ac29097adbc0d24d2b5daf2dc +P 500109bd0a4c134b91c37f397ff1ee828e09c17f7ecd153f975ede748caee7bb +R 44dc85544ec440f7c21f7b899d57ed02 U stephan -Z 78455e14df71dd2dbc1be007d9b53932 +Z e1c9bd04ae7a0c44d52816708800bbbb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7bb3f99cae..d4e846dedc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -500109bd0a4c134b91c37f397ff1ee828e09c17f7ecd153f975ede748caee7bb \ No newline at end of file +64ccf6177a019eab46fb3345ad1e8ba80eaf2c9da55767031f9f04ccd16afb4d \ No newline at end of file From 5e4d77fa888620e18ecfbd3c2c3701ab953ffbc9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Jul 2023 19:23:30 +0000 Subject: [PATCH 40/56] Describe JSON parser performance testing procedure in test/json/README.md FossilOrigin-Name: c47056e158073ee746c233dd2729cf224ea9e38821b15aa71ebf97607e602281 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/json/README.md | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 575934e4e2..c53b030eee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Two\sminor\simprovements\sto\ssum(),\sone\sof\swhich\swas\sinspired\sby\n[forum:/forumpost/af5be98dbc|forum\spost\saf5be98dbc]. -D 2023-07-19T09:52:10.467 +C Describe\sJSON\sparser\sperformance\stesting\sprocedure\sin\stest/json/README.md +D 2023-07-19T19:23:30.391 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1222,7 +1222,7 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json/README.md 506af1f54574b524106acb50d1a341ab5ddfa6d83fe25095007892b07e663e85 +F test/json/README.md e5a2fdbdf37612286fb07c6320cd88152a0d163c87c135b95ca4dce58a4fef7d F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 76152ad2ffe56034f2fd93d9a1ce9358e1677a7e9cd3dcd9f3a34a5c956a463e -R 47d8d8ca9ec8151ad0210262a08ac711 +P a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5 +R 9c85d46fcd15515aa3b030597dff0987 U drh -Z b98de7129167c58b9385610fbdec3119 +Z f1436c99b14877587c058fd4369d8d71 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4a5750f1d2..e37c386497 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5 \ No newline at end of file +c47056e158073ee746c233dd2729cf224ea9e38821b15aa71ebf97607e602281 \ No newline at end of file diff --git a/test/json/README.md b/test/json/README.md index 6a16114925..7e962418c3 100644 --- a/test/json/README.md +++ b/test/json/README.md @@ -25,3 +25,9 @@ of the SQLite JSON parser. 2. Run "`sh json-speed-check-1.sh x1`". The profile output will appear in jout-x1.txt. Substitute any label you want in place of "x1". + + 3. Run the query "`SELECT sum(json_valid(x)) FROM data1;`" on the + json100mb.db database file. Measure the + runtime. Divide 100 by the real elapse time of this test + to get an estimate for number of MB/s that the JSON parser is + able to process. From 93b3c1f3980629594bc237d29508d53fe8a37f37 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Jul 2023 20:23:34 +0000 Subject: [PATCH 41/56] Revise the new JSON parser performance test to make the test run 25 times longer, and thus provide a more repeatable number. FossilOrigin-Name: bee9e403ae47103938aabb9a7a7e120dfa741b464875965e58788a51fa56a8fe --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/json/README.md | 12 +++++++++--- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c53b030eee..20ecbfc77d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Describe\sJSON\sparser\sperformance\stesting\sprocedure\sin\stest/json/README.md -D 2023-07-19T19:23:30.391 +C Revise\sthe\snew\sJSON\sparser\sperformance\stest\sto\smake\sthe\stest\srun\s25\stimes\nlonger,\sand\sthus\sprovide\sa\smore\srepeatable\snumber. +D 2023-07-19T20:23:34.703 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1222,7 +1222,7 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json/README.md e5a2fdbdf37612286fb07c6320cd88152a0d163c87c135b95ca4dce58a4fef7d +F test/json/README.md 0992b8ccbecd8424b84c1173f9ac56bcc0ae96d49912fd2d86c04eb1f3cba7af F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a0d3e7571aded8d1e03908059d2d5aa5d62ec49bff099cb38f6f35df5e4b18b5 -R 9c85d46fcd15515aa3b030597dff0987 +P c47056e158073ee746c233dd2729cf224ea9e38821b15aa71ebf97607e602281 +R f95aec1d730e7f651d5bd3177562e93c U drh -Z f1436c99b14877587c058fd4369d8d71 +Z 755b9a0ce058340b12e7687869554354 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e37c386497..82949b4ca5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c47056e158073ee746c233dd2729cf224ea9e38821b15aa71ebf97607e602281 \ No newline at end of file +bee9e403ae47103938aabb9a7a7e120dfa741b464875965e58788a51fa56a8fe \ No newline at end of file diff --git a/test/json/README.md b/test/json/README.md index 7e962418c3..a100287b05 100644 --- a/test/json/README.md +++ b/test/json/README.md @@ -26,8 +26,14 @@ of the SQLite JSON parser. 2. Run "`sh json-speed-check-1.sh x1`". The profile output will appear in jout-x1.txt. Substitute any label you want in place of "x1". - 3. Run the query "`SELECT sum(json_valid(x)) FROM data1;`" on the - json100mb.db database file. Measure the - runtime. Divide 100 by the real elapse time of this test + 3. Run the script shown below in the CLI. + Divide 2500 by the real elapse time from this test to get an estimate for number of MB/s that the JSON parser is able to process. + +> ~~~~ +.open json100mb.db +.timer on +WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<25) +SELECT sum(json_valid(x)) FROM c, data1; +~~~~ From 0b518b8d656f8e76855fd010aa2ff72eeedc1f4c Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 20 Jul 2023 09:06:42 +0000 Subject: [PATCH 42/56] Filter the OPFS VFSes out of the sqlite3-node.mjs build. Add another level of subdirectory to the sahpool to later enable transparent support of client-provided files under the VFS's root dir. Rework the awkward sahpool-via-oo1 mapping. FossilOrigin-Name: 080a4d0aba30d8f3802b49be4a113205f069b3bdea8cebf525d654055642ff62 --- ext/wasm/GNUmakefile | 2 +- ext/wasm/api/README.md | 14 +++-- ...ol.js => sqlite3-vfs-opfs-sahpool.c-pp.js} | 58 ++++++++++++------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 4 ++ ext/wasm/tester1.c-pp.js | 18 +++--- manifest | 20 +++---- manifest.uuid | 2 +- 7 files changed, 72 insertions(+), 46 deletions(-) rename ext/wasm/api/{sqlite3-vfs-opfs-sahpool.js => sqlite3-vfs-opfs-sahpool.c-pp.js} (96%) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 486d1396d8..b6d81c00fc 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -376,7 +376,7 @@ sqlite3-api.jses += $(dir.api)/sqlite3-api-oo1.js sqlite3-api.jses += $(dir.api)/sqlite3-api-worker1.js sqlite3-api.jses += $(dir.api)/sqlite3-v-helper.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js -sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.js +sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs-sahpool.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js # SOAP.js is an external API file which is part of our distribution diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md index be53ac25aa..eb0f073cf7 100644 --- a/ext/wasm/api/README.md +++ b/ext/wasm/api/README.md @@ -91,7 +91,7 @@ browser client: directly to the (async) OPFS API and channels those results back to its synchronous counterpart. This file, because it must be started in its own Worker, is not part of the amalgamation. -- **`sqlite3-vfs-opfs-sahpool.js`**\ +- **`sqlite3-vfs-opfs-sahpool.c-pp.js`**\ is another sqlite3 VFS supporting the OPFS, but uses a completely different approach that the above-listed one. - **`sqlite3-api-cleanup.js`**\ @@ -111,13 +111,15 @@ browser client: with `c-pp`](#c-pp), noting that such preprocessing may be applied after all of the relevant files are concatenated. That extension is used primarily to keep the code maintainers cognisant of the fact that -those files contain constructs which will not run as-is in JavaScript. +those files contain constructs which may not run as-is in any given +JavaScript environment. The build process glues those files together, resulting in -`sqlite3-api.js`, which is everything except for the `post-js-*.js` -files, and `sqlite3.js`, which is the Emscripten-generated amalgamated -output and includes the `post-js-*.js` parts, as well as the -Emscripten-provided module loading pieces. +`sqlite3-api.js`, which is everything except for the +`pre/post-js-*.js` files, and `sqlite3.js`, which is the +Emscripten-generated amalgamated output and includes the +`pre/post-js-*.js` parts, as well as the Emscripten-provided module +loading pieces. The non-JS outlier file is `sqlite3-wasm.c`: it is a proxy for `sqlite3.c` which `#include`'s that file and adds a couple more diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js similarity index 96% rename from ext/wasm/api/sqlite3-vfs-opfs-sahpool.js rename to ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index f34c87004d..8a129d60f2 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -1,3 +1,4 @@ +//#ifnot target=node /* 2023-07-14 @@ -52,8 +53,8 @@ major browsers released since March 2023). If that API is not detected, the VFS is not registered. */ -'use strict'; globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + 'use strict'; const toss = sqlite3.util.toss; const toss3 = sqlite3.util.toss3; const initPromises = Object.create(null); @@ -79,6 +80,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.SQLITE_OPEN_WAL /* noting that WAL support is unavailable in the WASM build.*/; + /** Subdirectory of the VFS's space where "opaque" (randomly-named) + files are stored. Changing this effectively invalidates the data + stored under older names (orphaning it), so don't do that. */ + const OPAQUE_DIR_NAME = ".opaque"; + /** Returns short a string of random alphanumeric characters suitable for use as a random filename. @@ -423,6 +429,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ vfsDir; /* Directory handle to this.vfsDir. */ #dhVfsRoot; + /* Directory handle to the subdir of this.#dhVfsRoot which holds + the randomly-named "opaque" files. This subdir exists in the + hope that we can eventually support client-created files in + this.#dhVfsRoot. */ + #dhOpaque; /* Directory handle to this.dhVfsRoot's parent dir. Needed for a VFS-wipe op. */ #dhVfsParent; @@ -447,11 +458,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ #verbosity; constructor(options = Object.create(null)){ + this.#verbosity = options.verbosity ?? optionDefaults.verbosity; this.vfsName = options.name || optionDefaults.name; if( sqlite3.capi.sqlite3_vfs_find(this.vfsName)){ toss3("VFS name is already registered:", this.vfsName); } - this.#verbosity = options.verbosity ?? optionDefaults.verbosity; this.#cVfs = createOpfsVfs(this.vfsName); setPoolForVfs(this.#cVfs.pointer, this); this.vfsDir = options.directory || ("."+this.vfsName); @@ -491,7 +502,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ async addCapacity(n){ for(let i = 0; i < n; ++i){ const name = getRandomName(); - const h = await this.#dhVfsRoot.getFileHandle(name, {create:true}); + const h = await this.#dhOpaque.getFileHandle(name, {create:true}); const ah = await h.createSyncAccessHandle(); this.#mapSAHToName.set(ah,name); this.setAssociatedPath(ah, '', 0); @@ -512,7 +523,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } const name = this.#mapSAHToName.get(ah); ah.close(); - await this.#dhVfsRoot.removeEntry(name); + await this.#dhOpaque.removeEntry(name); this.#mapSAHToName.delete(ah); this.#availableSAH.delete(ah); ++nRm; @@ -532,7 +543,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } /** - Opens all files under this.vfsDir/this.#dhVfsRoot and acquires + Opens all files under this.vfsDir/this.#dhOpaque and acquires a SAH for each. returns a Promise which resolves to no value but completes once all SAHs are acquired. If acquiring an SAH throws, SAHPool.$error will contain the corresponding @@ -544,7 +555,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ async acquireAccessHandles(clearFiles){ const files = []; - for await (const [name,h] of this.#dhVfsRoot){ + for await (const [name,h] of this.#dhOpaque){ if('file'===h.kind){ files.push([name,h]); } @@ -680,6 +691,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } this.#dhVfsRoot = h; this.#dhVfsParent = prev; + this.#dhOpaque = await this.#dhVfsRoot.getDirectoryHandle( + OPAQUE_DIR_NAME,{create:true} + ); this.releaseAccessHandles(); return this.acquireAccessHandles(clearFiles); } @@ -691,6 +705,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ - a URL object - A JS string representing a file name - Wasm C-string representing a file name + + All "../" parts and duplicate slashes are resolve/removed from + the returned result. */ getPath(arg) { if(wasm.isPtr(arg)) arg = wasm.cstrToJs(arg); @@ -790,17 +807,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ VFS has already been shut down. */ async removeVfs(){ - if(!this.#cVfs.pointer) return false; + if(!this.#cVfs.pointer || !this.#dhOpaque) return false; capi.sqlite3_vfs_unregister(this.#cVfs.pointer); this.#cVfs.dispose(); try{ this.releaseAccessHandles(); - if(this.#dhVfsParent){ - await this.#dhVfsParent.removeEntry( - this.#dhVfsRoot.name, {recursive: true} - ); - this.#dhVfsRoot = this.#dhVfsParent = undefined; - } + await this.#dhVfsRoot.removeEntry(OPAQUE_DIR_NAME, {recursive: true}); + this.#dhOpaque = undefined; + await this.#dhVfsParent.removeEntry( + this.#dhVfsRoot.name, {recursive: true} + ); + this.#dhVfsRoot = this.#dhVfsParent = undefined; }catch(e){ sqlite3.config.error(this.vfsName,"removeVfs() failed:",e); /*otherwise ignored - there is no recovery strategy*/ @@ -1120,7 +1137,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** The poolUtil object will be the result of the resolved Promise. */ const poolUtil = new OpfsSAHPoolUtil(thePool); - if(sqlite3.oo1){ const oo1 = sqlite3.oo1; const theVfs = thePool.getVfs(); @@ -1130,12 +1146,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ oo1.DB.dbCtorHelper.call(this, opt); }; OpfsSAHPoolDb.prototype = Object.create(oo1.DB.prototype); - OpfsSAHPoolDb.PoolUtil = poolUtil; - if(!oo1.OpfsSAHPool){ - oo1.OpfsSAHPool = Object.create(null); - oo1.OpfsSAHPool.default = OpfsSAHPoolDb; - } - oo1.OpfsSAHPool[vfsName] = OpfsSAHPoolDb; + // yes or no? OpfsSAHPoolDb.PoolUtil = poolUtil; + poolUtil.OpfsSAHPoolDb = OpfsSAHPoolDb; oo1.DB.dbCtorHelper.setVfsPostOpenSql( theVfs.pointer, function(oo1Db, sqlite3){ @@ -1159,3 +1171,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }); }/*installOpfsSAHPoolVfs()*/; }/*sqlite3ApiBootstrap.initializers*/); +//#else +/* + The OPFS SAH Pool VFS parts are elided from builds targeting + node.js. +*/ +//#endif target=node diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 7d313a2635..61b7534de7 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -1,3 +1,4 @@ +//#ifnot target=node /* 2022-09-18 @@ -1370,3 +1371,6 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ } }); }/*sqlite3ApiBootstrap.initializers.push()*/); +//#else +/* The OPFS VFS parts are elided from builds targeting node.js. */ +//#endif target=node diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 56a4369cd5..905340b234 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3054,16 +3054,13 @@ globalThis.sqlite3InitModule = sqlite3InitModule; run. */) .assert(u1.getCapacity() + 2 === (await u2.addCapacity(2))) .assert(2 === (await u2.reduceCapacity(2))) - .assert(sqlite3.oo1.OpfsSAHPool.default instanceof Function) - .assert(sqlite3.oo1.OpfsSAHPool.default === - sqlite3.oo1.OpfsSAHPool[sahPoolConfig.name]) .assert(sqlite3.capi.sqlite3_js_vfs_list().indexOf(sahPoolConfig.name) >= 0); T.assert(0 === u1.getFileCount()); - const DbCtor = sqlite3.oo1.OpfsSAHPool.default; const dbName = '/foo.db'; - let db = new DbCtor(dbName); - T.assert(1 === u1.getFileCount()); + let db = new u1.OpfsSAHPoolDb(dbName); + T.assert(db instanceof sqlite3.oo1.DB) + .assert(1 === u1.getFileCount()); db.exec([ 'create table t(a);', 'insert into t(a) values(1),(2),(3)' @@ -3072,14 +3069,19 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(3 === db.selectValue('select count(*) from t')); db.close(); T.assert(1 === u1.getFileCount()); - db = new DbCtor(dbName); + db = new u2.OpfsSAHPoolDb(dbName); T.assert(1 === u1.getFileCount()); db.close(); T.assert(1 === u1.getFileCount()) .assert(true === u1.unlink(dbName)) .assert(false === u1.unlink(dbName)) .assert(0 === u1.getFileCount()); - + if(0){ + /* Enable this block to inspect vfs's contents via the dev + console or OPFS Explorer browser extension. The + following bits will remove them. */ + return; + } T.assert(true === await u2.removeVfs()) .assert(false === await u1.removeVfs()) .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)); diff --git a/manifest b/manifest index 9bf3bd22cd..9e1fc1f83f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sinternal\srefactoring\sand\sdocs\sfor\sopfs-sahpool. -D 2023-07-19T17:47:02.768 +C Filter\sthe\sOPFS\sVFSes\sout\sof\sthe\ssqlite3-node.mjs\sbuild.\sAdd\sanother\slevel\sof\ssubdirectory\sto\sthe\ssahpool\sto\slater\senable\stransparent\ssupport\sof\sclient-provided\sfiles\sunder\sthe\sVFS's\sroot\sdir.\sRework\sthe\sawkward\ssahpool-via-oo1\smapping. +D 2023-07-20T09:06:42.459 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,13 +482,13 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile b425091409491f0f9dca77f7a41143530f8b7d37abfb3ba59f1d50f4cc85d02f +F ext/wasm/GNUmakefile 437beb3e200cb8b2977dbda43caecd30edb87a7cf4fa2d86cb6179a1858fe466 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 -F ext/wasm/api/README.md f6cec6b0ce122cdff9440b30a3132dea3665b5b7baace910b43cbccdaaa376b9 +F ext/wasm/api/README.md 5eb44fa02e9c693a1884a3692428647894b0380b24bca120866b7a24c8786134 F ext/wasm/api/extern-post-js.c-pp.js 80f288131f9f4486a66e79dbf42d4402dc23e3cb4ef605377ae69f0545a6b8e6 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 @@ -502,8 +502,8 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 05b5646b91faa947833d43a840e8b94abb441afa953ee5a11cc7f07f4e01361a -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4946af0d6fbd395aa39966562ca85900664605a5f0cc10fff50146dee527812c +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js d9abf3cde87aea55ea901e80f70e55b36055e8e5120ed47321af35afb9facdaa w ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 7b6aa73e8af0379f96d17eb874bc02ed13e901c6924aa3804a075f7bc58c3146 F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js b99aa30d9b54c5c60f67381b249d290a542c529898852c32c5645f5a33be9498 +F ext/wasm/tester1.c-pp.js f835c9f703b562142f23a3607fa4a34cb6aece5fb5d674ea5bd7d37b0e47e104 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 500109bd0a4c134b91c37f397ff1ee828e09c17f7ecd153f975ede748caee7bb -R 44dc85544ec440f7c21f7b899d57ed02 +P 64ccf6177a019eab46fb3345ad1e8ba80eaf2c9da55767031f9f04ccd16afb4d +R 9b88832176f0fa72b79f47fe01f46afd U stephan -Z e1c9bd04ae7a0c44d52816708800bbbb +Z 925e0249217422a96f201bda7e782884 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d4e846dedc..450c8fa540 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64ccf6177a019eab46fb3345ad1e8ba80eaf2c9da55767031f9f04ccd16afb4d \ No newline at end of file +080a4d0aba30d8f3802b49be4a113205f069b3bdea8cebf525d654055642ff62 \ No newline at end of file From 35325a37ce6dcf4b68110d39765fd441e0829ddf Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 20 Jul 2023 23:25:32 +0000 Subject: [PATCH 43/56] Minor internal cleanups in JS code. FossilOrigin-Name: 21a2ca9fc46bf746874579897872e2a45cb07f278abb670dd22b122f7d6a9a6c --- ext/wasm/api/extern-post-js.c-pp.js | 2 +- ext/wasm/api/sqlite3-api-prologue.js | 21 ++++++++++++--------- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 2 +- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index bd5225d119..fac00370dd 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -59,7 +59,7 @@ const toExportForESM = initModuleState.sqlite3Dir = li.join('/') + '/'; } - globalThis.sqlite3InitModule = async function ff(...args){ + globalThis.sqlite3InitModule = function ff(...args){ //console.warn("Using replaced sqlite3InitModule()",globalThis.location); return originalInit(...args).then((EmscriptenModule)=>{ if('undefined'!==typeof WorkerGlobalScope && diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 22ec87ff31..f2d568e2ea 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1861,6 +1861,9 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( client: undefined, /** + This function is not part of the public interface, but a + piece of internal bootstrapping infrastructure. + Performs any optional asynchronous library-level initialization which might be required. This function returns a Promise which resolves to the sqlite3 namespace object. Any error in the @@ -1884,15 +1887,15 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( In Emscripten-based builds, this function is called automatically and deleted from this object. */ - asyncPostInit: function ff(){ - if(ff.ready instanceof Promise) return ff.ready; + asyncPostInit: async function ff(){ + if(ff.isReady instanceof Promise) return ff.isReady; let lip = sqlite3ApiBootstrap.initializersAsync; delete sqlite3ApiBootstrap.initializersAsync; if(!lip || !lip.length){ - return ff.ready = Promise.resolve(sqlite3); + return ff.isReady = Promise.resolve(sqlite3); } lip = lip.map((f)=>{ - return (f instanceof Promise) ? f : f(sqlite3); + return (f instanceof Function) ? async x=>f(sqlite3) : f; }); const catcher = (e)=>{ config.error("an async sqlite3 initializer failed:",e); @@ -1907,7 +1910,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( /* It's conceivable that we might want to expose StructBinder to client-side code, but it's only useful if clients build their own sqlite3.wasm which contains their - one C struct types. */ + own C struct types. */ delete sqlite3.StructBinder; } return sqlite3; @@ -1917,12 +1920,12 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( advantage is that it allows us to have post-init cleanup defined outside of this routine at the end of the list and have it run at a well-defined time. */ - let p = lip.shift(); + let p = Promise.resolve(sqlite3); while(lip.length) p = p.then(lip.shift()); - return ff.ready = p.then(postInit).catch(catcher); + return ff.isReady = p.then(postInit).catch(catcher); }else{ /* Run them in an arbitrary order. */ - return ff.ready = Promise.all(lip).then(postInit).catch(catcher); + return ff.isReady = Promise.all(lip).then(postInit).catch(catcher); } }, /** @@ -1983,7 +1986,7 @@ globalThis.sqlite3ApiBootstrap.initializers = []; specifically for initializers which are asynchronous. All entries in this list must be either async functions, non-async functions which return a Promise, or a Promise. Each function in the list is called - with the sqlite3 ojbect as its only argument. + with the sqlite3 object as its only argument. The resolved value of any Promise is ignored and rejection will kill the asyncPostInit() process (at an indeterminate point because all diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 61b7534de7..58c511082c 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -1367,7 +1367,7 @@ globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ }); }catch(e){ sqlite3.config.error("installOpfsVfs() exception:",e); - throw e; + return Promise.reject(e); } }); }/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/manifest b/manifest index 9e1fc1f83f..fbde0f5ec5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Filter\sthe\sOPFS\sVFSes\sout\sof\sthe\ssqlite3-node.mjs\sbuild.\sAdd\sanother\slevel\sof\ssubdirectory\sto\sthe\ssahpool\sto\slater\senable\stransparent\ssupport\sof\sclient-provided\sfiles\sunder\sthe\sVFS's\sroot\sdir.\sRework\sthe\sawkward\ssahpool-via-oo1\smapping. -D 2023-07-20T09:06:42.459 +C Minor\sinternal\scleanups\sin\sJS\scode. +D 2023-07-20T23:25:32.031 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -489,7 +489,7 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 5eb44fa02e9c693a1884a3692428647894b0380b24bca120866b7a24c8786134 -F ext/wasm/api/extern-post-js.c-pp.js 80f288131f9f4486a66e79dbf42d4402dc23e3cb4ef605377ae69f0545a6b8e6 +F ext/wasm/api/extern-post-js.c-pp.js 116749b7e55b7519129de06d3d353e19df68cfb24b12204aa4dc30c9a83023fe F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 @@ -497,13 +497,13 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 -F ext/wasm/api/sqlite3-api-prologue.js d747cbb379e13881c9edf39dce019cbbbae860c456ababe9d550b4b27666a1c9 +F ext/wasm/api/sqlite3-api-prologue.js fdd8b72323f09684a7addfb622d427477a58cd5a35523f150ce77236b56352a7 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js d9abf3cde87aea55ea901e80f70e55b36055e8e5120ed47321af35afb9facdaa w ext/wasm/api/sqlite3-vfs-opfs-sahpool.js -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 7b6aa73e8af0379f96d17eb874bc02ed13e901c6924aa3804a075f7bc58c3146 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js d9abf3cde87aea55ea901e80f70e55b36055e8e5120ed47321af35afb9facdaa +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e7a690e0e78ff4d563f2eca468f91db69f001ff4b79c6d2304cbb6f62dca437d F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 64ccf6177a019eab46fb3345ad1e8ba80eaf2c9da55767031f9f04ccd16afb4d -R 9b88832176f0fa72b79f47fe01f46afd +P 080a4d0aba30d8f3802b49be4a113205f069b3bdea8cebf525d654055642ff62 +R 729d5d7f424013b7282496b5ada001e8 U stephan -Z 925e0249217422a96f201bda7e782884 +Z fa107c664214632c0e2de6d34ddd182b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 450c8fa540..d513af32a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -080a4d0aba30d8f3802b49be4a113205f069b3bdea8cebf525d654055642ff62 \ No newline at end of file +21a2ca9fc46bf746874579897872e2a45cb07f278abb670dd22b122f7d6a9a6c \ No newline at end of file From 3068d6622e6f4def5dca4b9db8bde765fcc617bb Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 21 Jul 2023 09:10:42 +0000 Subject: [PATCH 44/56] Internal cleanups in the async part of the JS library bootstrap phase. FossilOrigin-Name: b6d57ab63793241a500ea527c5b3216c54b3ff1972d3adbbf42a9a53bfec0aa1 --- ext/wasm/api/sqlite3-api-prologue.js | 38 +++++++++++----------------- manifest | 12 ++++----- manifest.uuid | 2 +- 3 files changed, 22 insertions(+), 30 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index f2d568e2ea..0179091c02 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1889,19 +1889,9 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( */ asyncPostInit: async function ff(){ if(ff.isReady instanceof Promise) return ff.isReady; - let lip = sqlite3ApiBootstrap.initializersAsync; + let lia = sqlite3ApiBootstrap.initializersAsync; delete sqlite3ApiBootstrap.initializersAsync; - if(!lip || !lip.length){ - return ff.isReady = Promise.resolve(sqlite3); - } - lip = lip.map((f)=>{ - return (f instanceof Function) ? async x=>f(sqlite3) : f; - }); - const catcher = (e)=>{ - config.error("an async sqlite3 initializer failed:",e); - throw e; - }; - const postInit = ()=>{ + const postInit = async ()=>{ if(!sqlite3.__isUnderTest){ /* Delete references to internal-only APIs which are used by some initializers. Retain them when running in test mode @@ -1915,18 +1905,20 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( } return sqlite3; }; - if(1){ - /* Run all initializers in the sequence they were added. The - advantage is that it allows us to have post-init cleanup - defined outside of this routine at the end of the list and - have it run at a well-defined time. */ - let p = Promise.resolve(sqlite3); - while(lip.length) p = p.then(lip.shift()); - return ff.isReady = p.then(postInit).catch(catcher); - }else{ - /* Run them in an arbitrary order. */ - return ff.isReady = Promise.all(lip).then(postInit).catch(catcher); + const catcher = (e)=>{ + config.error("an async sqlite3 initializer failed:",e); + throw e; + }; + if(!lia || !lia.length){ + return ff.isReady = postInit().catch(catcher); } + lia = lia.map((f)=>{ + return (f instanceof Function) ? async x=>f(sqlite3) : f; + }); + lia.push(postInit); + let p = Promise.resolve(sqlite3); + while(lia.length) p = p.then(lia.shift()); + return ff.isReady = p.catch(catcher); }, /** scriptInfo ideally gets injected into this object by the diff --git a/manifest b/manifest index fbde0f5ec5..2ae10ae239 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\scleanups\sin\sJS\scode. -D 2023-07-20T23:25:32.031 +C Internal\scleanups\sin\sthe\sasync\spart\sof\sthe\sJS\slibrary\sbootstrap\sphase. +D 2023-07-21T09:10:42.614 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -497,7 +497,7 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 -F ext/wasm/api/sqlite3-api-prologue.js fdd8b72323f09684a7addfb622d427477a58cd5a35523f150ce77236b56352a7 +F ext/wasm/api/sqlite3-api-prologue.js cbd7d6ba185f3a844a8b0020e954b49bbc2ca78b305d117bec2ceca21431795a F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 080a4d0aba30d8f3802b49be4a113205f069b3bdea8cebf525d654055642ff62 -R 729d5d7f424013b7282496b5ada001e8 +P 21a2ca9fc46bf746874579897872e2a45cb07f278abb670dd22b122f7d6a9a6c +R 29a8f4f8ea37e0c15f874df4805828de U stephan -Z fa107c664214632c0e2de6d34ddd182b +Z 8b913efdb9ce5e9239da5b5f12183785 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d513af32a7..d10306e8a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21a2ca9fc46bf746874579897872e2a45cb07f278abb670dd22b122f7d6a9a6c \ No newline at end of file +b6d57ab63793241a500ea527c5b3216c54b3ff1972d3adbbf42a9a53bfec0aa1 \ No newline at end of file From bfe6dd0100d54e43601b96da97b27a5cda82578a Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 21 Jul 2023 10:51:35 +0000 Subject: [PATCH 45/56] Minor internal cleanups in the opfs-sahpool VFS. FossilOrigin-Name: 74ad31e2908af8225b7aa527dbcd1877423d58163e365317a78453b31e322ea3 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 58 +++++++++++++------ manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 46 insertions(+), 26 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 8a129d60f2..28524b613c 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -144,11 +144,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xClose: function(pFile){ const pool = getPoolForPFile(pFile); pool.storeErr(); - const file = pool.getOFileForSFile(pFile); + const file = pool.getOFileForS3File(pFile); if(file) { try{ pool.log(`xClose ${file.path}`); - pool.mapSFileToOFile(pFile, false); + pool.mapS3FileToOFile(pFile, false); file.sah.flush(); if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){ pool.deletePath(file.path); @@ -169,7 +169,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xFileSize: function(pFile,pSz64){ const pool = getPoolForPFile(pFile); pool.log(`xFileSize`); - const file = pool.getOFileForSFile(pFile); + const file = pool.getOFileForS3File(pFile); const size = file.sah.getSize() - HEADER_OFFSET_DATA; //log(`xFileSize ${file.path} ${size}`); wasm.poke64(pSz64, BigInt(size)); @@ -179,14 +179,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pool = getPoolForPFile(pFile); pool.log(`xLock ${lockType}`); pool.storeErr(); - const file = pool.getOFileForSFile(pFile); + const file = pool.getOFileForS3File(pFile); file.lockType = lockType; return 0; }, xRead: function(pFile,pDest,n,offset64){ const pool = getPoolForPFile(pFile); pool.storeErr(); - const file = pool.getOFileForSFile(pFile); + const file = pool.getOFileForS3File(pFile); pool.log(`xRead ${file.path} ${n} @ ${offset64}`); try { const nRead = file.sah.read( @@ -210,7 +210,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pool = getPoolForPFile(pFile); pool.log(`xSync ${flags}`); pool.storeErr(); - const file = pool.getOFileForSFile(pFile); + const file = pool.getOFileForS3File(pFile); //log(`xSync ${file.path} ${flags}`); try{ file.sah.flush(); @@ -224,7 +224,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const pool = getPoolForPFile(pFile); pool.log(`xTruncate ${sz64}`); pool.storeErr(); - const file = pool.getOFileForSFile(pFile); + const file = pool.getOFileForS3File(pFile); //log(`xTruncate ${file.path} ${iSize}`); try{ file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64)); @@ -237,14 +237,14 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xUnlock: function(pFile,lockType){ const pool = getPoolForPFile(pFile); pool.log('xUnlock'); - const file = pool.getOFileForSFile(pFile); + const file = pool.getOFileForS3File(pFile); file.lockType = lockType; return 0; }, xWrite: function(pFile,pSrc,n,offset64){ const pool = getPoolForPFile(pFile); pool.storeErr(); - const file = pool.getOFileForSFile(pFile); + const file = pool.getOFileForS3File(pFile); pool.log(`xWrite ${file.path} ${n} ${offset64}`); try{ const nBytes = file.sah.write( @@ -356,7 +356,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ // Subsequent I/O methods are only passed the sqlite3_file // pointer, so map the relevant info we need to that pointer. const file = {path, flags, sah}; - pool.mapSFileToOFile(pFile, file); + pool.mapS3FileToOFile(pFile, file); file.lockType = capi.SQLITE_LOCK_NONE; const sq3File = new capi.sqlite3_file(pFile); sq3File.$pMethods = opfsIoMethods.pointer; @@ -374,11 +374,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Creates and initializes an sqlite3_vfs instance for an OpfsSAHPool. The argument is the VFS's name (JS string). + Throws if the VFS name is already registered or if something + goes terribly wrong via sqlite3.vfs.installVfs(). + Maintenance reminder: the only detail about the returned object which is specific to any given OpfsSAHPool instance is the $zName member. All other state is identical. */ const createOpfsVfs = function(vfsName){ + if( sqlite3.capi.sqlite3_vfs_find(vfsName)){ + toss3("VFS name is already registered:", vfsName); + } const opfsVfs = new capi.sqlite3_vfs(); /* We fetch the default VFS so that we can inherit some methods from it. */ @@ -444,7 +450,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Set of currently-unused SAHs. */ #availableSAH = new Set(); /* Maps (sqlite3_file*) to xOpen's file objects. */ - #mapSqlite3FileToFile = new Map(); + #mapS3FileToOFile_ = new Map(); + + /* Maps SAH to an abstract File Object which contains + various metadata about that handle. */ + //#mapSAHToMeta = new Map(); /** Buffer used by [sg]etAssociatedPath(). */ #apBody = new Uint8Array(HEADER_CORPUS_SIZE); @@ -460,9 +470,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ constructor(options = Object.create(null)){ this.#verbosity = options.verbosity ?? optionDefaults.verbosity; this.vfsName = options.name || optionDefaults.name; - if( sqlite3.capi.sqlite3_vfs_find(this.vfsName)){ - toss3("VFS name is already registered:", this.vfsName); - } this.#cVfs = createOpfsVfs(this.vfsName); setPoolForVfs(this.#cVfs.pointer, this); this.vfsDir = options.directory || ("."+this.vfsName); @@ -494,6 +501,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Current number of in-use files from pool. */ getFileCount(){return this.#mapFilenameToSAH.size} +// #createFileObject(sah,clientName,opaqueName){ +// const f = Object.assign(Object.create(null),{ +// clientName, opaqueName +// }); +// this.#mapSAHToMeta.set(sah, f); +// return f; +// } +// #unmapFileObject(sah){ +// this.#mapSAHToMeta.delete(sah); +// } + /** Adds n files to the pool's capacity. This change is persistent across settings. Returns a Promise which resolves @@ -506,6 +524,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const ah = await h.createSyncAccessHandle(); this.#mapSAHToName.set(ah,name); this.setAssociatedPath(ah, '', 0); + //this.#createFileObject(ah,undefined,name); } return this.getCapacity(); } @@ -522,6 +541,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ break; } const name = this.#mapSAHToName.get(ah); + //this.#unmapFileObject(ah); ah.close(); await this.#dhOpaque.removeEntry(name); this.#mapSAHToName.delete(ah); @@ -762,19 +782,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Given an (sqlite3_file*), returns the mapped xOpen file object. */ - getOFileForSFile(ptr){ - return this.#mapSqlite3FileToFile.get(ptr); + getOFileForS3File(pFile){ + return this.#mapS3FileToOFile_.get(pFile); } /** Maps or unmaps (if file is falsy) the given (sqlite3_file*) to an xOpen file object and to this pool object. */ - mapSFileToOFile(pFile,file){ + mapS3FileToOFile(pFile,file){ if(file){ - this.#mapSqlite3FileToFile.set(pFile, file); + this.#mapS3FileToOFile_.set(pFile, file); setPoolForPFile(pFile, this); }else{ - this.#mapSqlite3FileToFile.delete(pFile); + this.#mapS3FileToOFile_.delete(pFile); setPoolForPFile(pFile, false); } } diff --git a/manifest b/manifest index 2ae10ae239..a0994da299 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Internal\scleanups\sin\sthe\sasync\spart\sof\sthe\sJS\slibrary\sbootstrap\sphase. -D 2023-07-21T09:10:42.614 +C Minor\sinternal\scleanups\sin\sthe\sopfs-sahpool\sVFS. +D 2023-07-21T10:51:35.142 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js d9abf3cde87aea55ea901e80f70e55b36055e8e5120ed47321af35afb9facdaa +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js bb99a931388966a032f635a0cc9cd72685e067f21b95b2a58a660c055020b739 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e7a690e0e78ff4d563f2eca468f91db69f001ff4b79c6d2304cbb6f62dca437d F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21a2ca9fc46bf746874579897872e2a45cb07f278abb670dd22b122f7d6a9a6c -R 29a8f4f8ea37e0c15f874df4805828de +P b6d57ab63793241a500ea527c5b3216c54b3ff1972d3adbbf42a9a53bfec0aa1 +R 04390a13b454e3d10f0ad00241913292 U stephan -Z 8b913efdb9ce5e9239da5b5f12183785 +Z 911ec9b641009d60f6f5bfe849c03052 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d10306e8a0..7fd82b9688 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b6d57ab63793241a500ea527c5b3216c54b3ff1972d3adbbf42a9a53bfec0aa1 \ No newline at end of file +74ad31e2908af8225b7aa527dbcd1877423d58163e365317a78453b31e322ea3 \ No newline at end of file From 7a2280fe6579c69235a78c5df83f34a555558049 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 15:01:53 +0000 Subject: [PATCH 46/56] Multiple optimizations that try to preserve or infer the zero-terminated property of TEXT values. Avoid unnecessary copying of text values destined to become function parameters. All changes help improve performance of doing UPDATEs on large JSON values that are indexed multiple ways. FossilOrigin-Name: d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 3 +-- src/json.c | 3 ++- src/vdbe.c | 6 ++++++ src/vdbeapi.c | 9 +++++++++ 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 1eaf96f72f..2bf0e377ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancements\sto\sthe\sJSON\sparser\sand\scode\sgenerator. -D 2023-07-19T23:02:45.962 +C Multiple\soptimizations\sthat\stry\sto\spreserve\sor\sinfer\sthe\szero-terminated\nproperty\sof\sTEXT\svalues.\s\sAvoid\sunnecessary\scopying\sof\stext\svalues\sdestined\nto\sbecome\sfunction\sparameters.\s\sAll\schanges\shelp\simprove\sperformance\sof\ndoing\sUPDATEs\son\slarge\sJSON\svalues\sthat\sare\sindexed\smultiple\sways. +D 2023-07-21T15:01:53.046 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -587,7 +587,7 @@ F src/date.c f73f203b3877cef866c60ab402aec2bf89597219b60635cf50cbe3c5e4533e94 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c cd5f5cd06ed0b6a882ec1a8c2a0d73b3cecb28479ad19e9931c4706c5e2182be -F src/expr.c 8d1656b65e26af3e34f78e947ac423f0d20c214ed25a67486e433bf16ca6b543 +F src/expr.c ef4a81822da6f767696bd7f4b9983328af061158958138540142285a5b1181b7 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a7fcbf7e66d14dbb73cf49f31489ebf66d0e6006c62b95246924a3bae9f37b36 F src/func.c 25f2e0204c011be56fc3c9a180534b68ca4866c61ec19806880136450434112d @@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e48136fce64e5004b1b8be76624cc9a4ac9ff6ae630a97df187c0ea4b9692d1b +F src/json.c 2bf3223892b4b9be39d701a5d349efcce5f5eebf71baaf91ba5866e10127e839 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -707,10 +707,10 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c fea6fffdee3cdae917a66b70deec59d4f238057cfd6de623d15cf990c196940d F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c 74282a947234513872a83b0bab1b8c644ece64b3e27b053ef17677c8ff9c81e0 +F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae -F src/vdbeapi.c de9703f8705afc393cc2864669ce28cf9516983c8331d59aa2b978de01634365 +F src/vdbeapi.c 1b17d1a53c3b88eb8bf2534f4f3557957f59f9431e5283734452b191cd45898d F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce F src/vdbemem.c cf4a1556dd5b18c071cf7c243373c29ce752eb516022e3ad49ba72f08b785033 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bee9e403ae47103938aabb9a7a7e120dfa741b464875965e58788a51fa56a8fe 144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6 -R af54725f832c177294e9e511cdacd53d +P 0a745897d66e9f36ea6e787fad54f0c413d52c226dc96fad36ccd33a0aa1930e +R ad6898efbc6819851ab380be79271f0e U drh -Z 1c679f829efc53f4f55993e30939aa14 +Z ecdcb9c56dec3594bcd0055088d236ad # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 10f9cdd123..000a57e974 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a745897d66e9f36ea6e787fad54f0c413d52c226dc96fad36ccd33a0aa1930e \ No newline at end of file +d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index a8e552f2c7..0c41f6684a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4690,8 +4690,7 @@ expr_code_doover: } } - sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, - SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_FACTOR); }else{ r1 = 0; } diff --git a/src/json.c b/src/json.c index 176dcbfdb9..a9425aecda 100644 --- a/src/json.c +++ b/src/json.c @@ -501,7 +501,8 @@ static void jsonAppendValue( */ static void jsonResult(JsonString *p){ if( p->bErr==0 ){ - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + if( p->nAlloc>=p->nUsed+1 ) p->zBuf[p->nUsed] = 0; + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, SQLITE_UTF8); jsonZero(p); diff --git a/src/vdbe.c b/src/vdbe.c index b248a664d3..075a632117 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -556,6 +556,9 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr){ sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.'); } sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]); + if( f & MEM_Term ){ + sqlite3_str_appendf(pStr, "(0-term)"); + } } } #endif @@ -3085,6 +3088,9 @@ op_column_restart: rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); if( rc!=SQLITE_OK ) goto abort_due_to_error; sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ + pDest->flags |= MEM_Term; + } pDest->flags &= ~MEM_Ephem; } } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 920780a896..7f44d22bc7 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -514,6 +514,15 @@ void sqlite3_result_text64( (void)invokeValueDestructor(z, xDel, pCtx); }else{ setResultStrOrError(pCtx, z, (int)n, enc, xDel); + if( xDel==sqlite3_free && enc==SQLITE_UTF8 ){ + Mem *pOut = pCtx->pOut; + if( pOut->z==z + && sqlite3_msize(pOut->z)>=pOut->n+1 + && pOut->z[n]==0 + ){ + pOut->flags |= MEM_Term; + } + } } } #ifndef SQLITE_OMIT_UTF16 From eee8687a9ff50465a720298bd8d50cbd85bb0fe7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 15:07:56 +0000 Subject: [PATCH 47/56] Fix harmless compiler warning about signed/unsigned comparison. FossilOrigin-Name: 75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2bf0e377ab..b4f361aba9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Multiple\soptimizations\sthat\stry\sto\spreserve\sor\sinfer\sthe\szero-terminated\nproperty\sof\sTEXT\svalues.\s\sAvoid\sunnecessary\scopying\sof\stext\svalues\sdestined\nto\sbecome\sfunction\sparameters.\s\sAll\schanges\shelp\simprove\sperformance\sof\ndoing\sUPDATEs\son\slarge\sJSON\svalues\sthat\sare\sindexed\smultiple\sways. -D 2023-07-21T15:01:53.046 +C Fix\sharmless\scompiler\swarning\sabout\ssigned/unsigned\scomparison. +D 2023-07-21T15:07:56.630 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -710,7 +710,7 @@ F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae -F src/vdbeapi.c 1b17d1a53c3b88eb8bf2534f4f3557957f59f9431e5283734452b191cd45898d +F src/vdbeapi.c b8866af13401c9779759640457fa37dafe7bc2466c8428ade10ef625d012e52b F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce F src/vdbemem.c cf4a1556dd5b18c071cf7c243373c29ce752eb516022e3ad49ba72f08b785033 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a745897d66e9f36ea6e787fad54f0c413d52c226dc96fad36ccd33a0aa1930e -R ad6898efbc6819851ab380be79271f0e +P d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e +R e667842b6755e6f486934eef004d9f18 U drh -Z ecdcb9c56dec3594bcd0055088d236ad +Z 993e102a87ed849f759c01f42525f684 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 000a57e974..247f7c86b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e \ No newline at end of file +75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 7f44d22bc7..079fb3976a 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -517,7 +517,7 @@ void sqlite3_result_text64( if( xDel==sqlite3_free && enc==SQLITE_UTF8 ){ Mem *pOut = pCtx->pOut; if( pOut->z==z - && sqlite3_msize(pOut->z)>=pOut->n+1 + && sqlite3_msize(pOut->z) >= (sqlite3_uint64)(pOut->n+1) && pOut->z[n]==0 ){ pOut->flags |= MEM_Term; From 569700a72e58a1cf9efc7fb743768a64cfd77d3c Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 18:09:07 +0000 Subject: [PATCH 48/56] Further improvements to large string handling in relation to JSON. FossilOrigin-Name: 1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/json.c | 4 ++-- src/vdbeInt.h | 1 + src/vdbeapi.c | 10 +--------- src/vdbemem.c | 31 ++++++++++++++++++++++++++++++- 6 files changed, 44 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index b4f361aba9..0ff7e75f17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sabout\ssigned/unsigned\scomparison. -D 2023-07-21T15:07:56.630 +C Further\simprovements\sto\slarge\sstring\shandling\sin\srelation\sto\sJSON. +D 2023-07-21T18:09:07.505 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 2bf3223892b4b9be39d701a5d349efcce5f5eebf71baaf91ba5866e10127e839 +F src/json.c 46ea5566e1363f4f353db79b0378c2bf8ffdf9d4667daee3df67b14669767bed F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -709,11 +709,11 @@ F src/util.c fea6fffdee3cdae917a66b70deec59d4f238057cfd6de623d15cf990c196940d F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 -F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae -F src/vdbeapi.c b8866af13401c9779759640457fa37dafe7bc2466c8428ade10ef625d012e52b +F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 +F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c cf4a1556dd5b18c071cf7c243373c29ce752eb516022e3ad49ba72f08b785033 +F src/vdbemem.c d6588e873aec78077c95c95df03920a1bcff1ee59fe58579261c957bf6118988 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e -R e667842b6755e6f486934eef004d9f18 +P 75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 +R b0bbcf196bde05ae8f599a4f16022c78 U drh -Z 993e102a87ed849f759c01f42525f684 +Z 8dfc9dc2f2e2752cbadaab8d45cf08cb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 247f7c86b6..9ac6a758a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 \ No newline at end of file +1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c \ No newline at end of file diff --git a/src/json.c b/src/json.c index a9425aecda..05046b5b54 100644 --- a/src/json.c +++ b/src/json.c @@ -501,8 +501,8 @@ static void jsonAppendValue( */ static void jsonResult(JsonString *p){ if( p->bErr==0 ){ - if( p->nAlloc>=p->nUsed+1 ) p->zBuf[p->nUsed] = 0; - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + jsonAppendChar(p, 0); + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed-1, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, SQLITE_UTF8); jsonZero(p); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 4c3394716b..3a5b961a25 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -611,6 +611,7 @@ int sqlite3VdbeMemSetZeroBlob(Mem*,int); int sqlite3VdbeMemIsRowSet(const Mem*); #endif int sqlite3VdbeMemSetRowSet(Mem*); +void sqlite3VdbeMemZeroTerminateIfAble(Mem*); int sqlite3VdbeMemMakeWriteable(Mem*); int sqlite3VdbeMemStringify(Mem*, u8, u8); int sqlite3IntFloatCompare(i64,double); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 079fb3976a..92bf38160f 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -514,15 +514,7 @@ void sqlite3_result_text64( (void)invokeValueDestructor(z, xDel, pCtx); }else{ setResultStrOrError(pCtx, z, (int)n, enc, xDel); - if( xDel==sqlite3_free && enc==SQLITE_UTF8 ){ - Mem *pOut = pCtx->pOut; - if( pOut->z==z - && sqlite3_msize(pOut->z) >= (sqlite3_uint64)(pOut->n+1) - && pOut->z[n]==0 - ){ - pOut->flags |= MEM_Term; - } - } + sqlite3VdbeMemZeroTerminateIfAble(pCtx->pOut); } } #ifndef SQLITE_OMIT_UTF16 diff --git a/src/vdbemem.c b/src/vdbemem.c index 1250eea77d..2baa35e24f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -314,6 +314,32 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ return SQLITE_OK; } +/* +** If pMem is already a string, detect if it is a zero-terminated +** string, or make it into one if possible, and mark it as such. +** +** This is an optimization. Correct operation continues even if +** this routine is a no-op. +*/ +void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ + if( (pMem->flags & (MEM_Str|MEM_Term))!=MEM_Str ) return; + if( pMem->enc!=SQLITE_UTF8 ) return; + if( pMem->z==0 ) return; + if( pMem->flags & MEM_Dyn ){ + if( pMem->xDel==sqlite3_free + && sqlite3_msize(pMem->z) >= pMem->n+1 + && pMem->z[pMem->n]==0 + ){ + pMem->flags |= MEM_Term; + return; + } + }else if( pMem->szMalloc>0 && pMem->szMalloc >= pMem->n+1 ){ + pMem->z[pMem->n] = 0; + pMem->flags |= MEM_Term; + return; + } +} + /* ** It is already known that pMem contains an unterminated string. ** Add the zero terminator. @@ -803,6 +829,7 @@ int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ break; } default: { + int rc; assert( aff==SQLITE_AFF_TEXT ); assert( MEM_Str==(MEM_Blob>>3) ); pMem->flags |= (pMem->flags&MEM_Blob)>>3; @@ -810,7 +837,9 @@ int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1; - return sqlite3VdbeChangeEncoding(pMem, encoding); + rc = sqlite3VdbeChangeEncoding(pMem, encoding); + if( rc ) return rc; + sqlite3VdbeMemZeroTerminateIfAble(pMem); } } return SQLITE_OK; From 028acd974f3c03c489dfaa02da618af99613cde7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 18:38:59 +0000 Subject: [PATCH 49/56] Avoid a potentially large strlen() in sqlite3DecOrHexToI64(). FossilOrigin-Name: 5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/util.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0ff7e75f17..1fd56ed788 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\simprovements\sto\slarge\sstring\shandling\sin\srelation\sto\sJSON. -D 2023-07-21T18:09:07.505 +C Avoid\sa\spotentially\slarge\sstrlen()\sin\ssqlite3DecOrHexToI64(). +D 2023-07-21T18:38:59.114 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -705,7 +705,7 @@ F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0 F src/update.c 0aa36561167a7c40d01163238c297297962f31a15a8d742216b3c37cdf25f731 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c fea6fffdee3cdae917a66b70deec59d4f238057cfd6de623d15cf990c196940d +F src/util.c c2aa170f2eb429235b1dddce8952770787ffa5124dc89d405bfbe8ebad8e7ebd F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 -R b0bbcf196bde05ae8f599a4f16022c78 +P 1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c +R 985a846b4f2b586d366fd130e337d17a U drh -Z 8dfc9dc2f2e2752cbadaab8d45cf08cb +Z f082f5b7a15300eec42c5fbe873f6275 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9ac6a758a7..ff10b67548 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c \ No newline at end of file +5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa \ No newline at end of file diff --git a/src/util.c b/src/util.c index 409482d395..1200aef0e3 100644 --- a/src/util.c +++ b/src/util.c @@ -846,7 +846,9 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ }else #endif /* SQLITE_OMIT_HEX_INTEGER */ { - return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); + int n = (int)(0x3fffffff&strspn(z,"+- \n\t0123456789")); + if( z[n] ) n++; + return sqlite3Atoi64(z, pOut, n, SQLITE_UTF8); } } From 89f07b12769548185c122bb3cb071cb4748c4f04 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 19:17:26 +0000 Subject: [PATCH 50/56] Fix harmless compiler warning introduced by the previous check-in. FossilOrigin-Name: 06f829e9e57c5aa495f519ad7bb379be611e8e21ef60d36e63d055e45f7a6117 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1fd56ed788..083967c4a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\spotentially\slarge\sstrlen()\sin\ssqlite3DecOrHexToI64(). -D 2023-07-21T18:38:59.114 +C Fix\sharmless\scompiler\swarning\sintroduced\sby\sthe\sprevious\scheck-in. +D 2023-07-21T19:17:26.591 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -713,7 +713,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c d6588e873aec78077c95c95df03920a1bcff1ee59fe58579261c957bf6118988 +F src/vdbemem.c 490342e7ab9ac561e8e210e1efe107bc290a97bc39046637fffceb84c02cc8e4 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c -R 985a846b4f2b586d366fd130e337d17a +P 5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa +R 99401c28791119c3b9a7ff587d00483e U drh -Z f082f5b7a15300eec42c5fbe873f6275 +Z 82c739c7a09ae5acca56a5e43e87910d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ff10b67548..c901fe6612 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa \ No newline at end of file +06f829e9e57c5aa495f519ad7bb379be611e8e21ef60d36e63d055e45f7a6117 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 2baa35e24f..b007a89cda 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -327,7 +327,7 @@ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ if( pMem->z==0 ) return; if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free - && sqlite3_msize(pMem->z) >= pMem->n+1 + && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) && pMem->z[pMem->n]==0 ){ pMem->flags |= MEM_Term; From 21f22f9088f5e9ce2ae4c1e666ab73cc96bfa99e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 22:22:13 +0000 Subject: [PATCH 51/56] Mark an unreachable branch as NEVER(). FossilOrigin-Name: 414a4d660c3b505640227066b468db1268fcb1bd3fad931644a769617c6c00cd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 083967c4a7..6095497a50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sintroduced\sby\sthe\sprevious\scheck-in. -D 2023-07-21T19:17:26.591 +C Mark\san\sunreachable\sbranch\sas\sNEVER(). +D 2023-07-21T22:22:13.828 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -713,7 +713,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c 490342e7ab9ac561e8e210e1efe107bc290a97bc39046637fffceb84c02cc8e4 +F src/vdbemem.c b3013dd11696db423bb410a50126d46d700fe3133b936a32195700e9731d2960 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa -R 99401c28791119c3b9a7ff587d00483e +P 06f829e9e57c5aa495f519ad7bb379be611e8e21ef60d36e63d055e45f7a6117 +R f650c894c3b7e9fe01bf709dbdf65d35 U drh -Z 82c739c7a09ae5acca56a5e43e87910d +Z ff67cbf0340dd6d2b6c094e434a4b95a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c901fe6612..58efe19050 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06f829e9e57c5aa495f519ad7bb379be611e8e21ef60d36e63d055e45f7a6117 \ No newline at end of file +414a4d660c3b505640227066b468db1268fcb1bd3fad931644a769617c6c00cd \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index b007a89cda..3f845452c4 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -324,7 +324,7 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ if( (pMem->flags & (MEM_Str|MEM_Term))!=MEM_Str ) return; if( pMem->enc!=SQLITE_UTF8 ) return; - if( pMem->z==0 ) return; + if( NEVER(pMem->z==0) ) return; if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) From 61a5b6e3bc0482ed169c86760e83c6d95a5b94c7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Jul 2023 15:21:41 +0000 Subject: [PATCH 52/56] Fix a harmless use-of-initialized-value warning from OSSFuzz that results from recent large string optmizations. FossilOrigin-Name: 1a6b3dd1c40277a0d0f0bb562ddc4868aadd632fc2d29be1b17bb33fc22c46c8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ad864a87b7..84820321d7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancements\sfor\squeries\susing\svery\slarge\sstring\svalues. -D 2023-07-21T22:28:34.848 +C Fix\sa\sharmless\suse-of-initialized-value\swarning\sfrom\sOSSFuzz\sthat\sresults\nfrom\srecent\slarge\sstring\soptmizations. +D 2023-07-22T15:21:41.842 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -590,7 +590,7 @@ F src/delete.c cd5f5cd06ed0b6a882ec1a8c2a0d73b3cecb28479ad19e9931c4706c5e2182be F src/expr.c ef4a81822da6f767696bd7f4b9983328af061158958138540142285a5b1181b7 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a7fcbf7e66d14dbb73cf49f31489ebf66d0e6006c62b95246924a3bae9f37b36 -F src/func.c 25f2e0204c011be56fc3c9a180534b68ca4866c61ec19806880136450434112d +F src/func.c cc1da67fd643a43cfe691784158ec656d8ec6d13bb17e67018b01b38b3e4f5ab F src/global.c 29f56a330ed9d1b5cd9b79ac0ca36f97ac3afc730ff8bfa987b0db9e559d684d F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a745897d66e9f36ea6e787fad54f0c413d52c226dc96fad36ccd33a0aa1930e 414a4d660c3b505640227066b468db1268fcb1bd3fad931644a769617c6c00cd -R f650c894c3b7e9fe01bf709dbdf65d35 +P 3661b9cd98565ea23b0cafd030668f09885dcbd54eb12b7b3cd85c289aef4c3c +R 7861df33493b30fd58a73dc8ed59a062 U drh -Z 591aef3c08eb8a7097f1d29488b469be +Z 08afe3692a916f7ae05aca68accdd5b2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 654e87bd4c..e58ba7645d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3661b9cd98565ea23b0cafd030668f09885dcbd54eb12b7b3cd85c289aef4c3c \ No newline at end of file +1a6b3dd1c40277a0d0f0bb562ddc4868aadd632fc2d29be1b17bb33fc22c46c8 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 8382e4a5c5..ef06a79fb9 100644 --- a/src/func.c +++ b/src/func.c @@ -1227,6 +1227,7 @@ static void charFunc( *zOut++ = 0x80 + (u8)(c & 0x3F); } \ } + *zOut = 0; sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); } From 5d03b1610f268916a7609d5fb290441a07787357 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Jul 2023 16:37:28 +0000 Subject: [PATCH 53/56] Do not read past the end of a text buffer looking for a zero terminator, as that space might not be initialized. If the buffer is owned, just set the null terminator. This is a better fix for the OSSFuzz-detected use-of-initialized-value problem. FossilOrigin-Name: 931bccb0cc290b8bf3027641e7a7fac30e3244d7dc84aa9e38b24b7e9544ca06 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 84820321d7..b24ee0ddd5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\suse-of-initialized-value\swarning\sfrom\sOSSFuzz\sthat\sresults\nfrom\srecent\slarge\sstring\soptmizations. -D 2023-07-22T15:21:41.842 +C Do\snot\sread\spast\sthe\send\sof\sa\stext\sbuffer\slooking\sfor\sa\szero\sterminator,\sas\nthat\sspace\smight\snot\sbe\sinitialized.\s\sIf\sthe\sbuffer\sis\sowned,\sjust\sset\sthe\nnull\sterminator.\s\sThis\sis\sa\sbetter\sfix\sfor\sthe\sOSSFuzz-detected\nuse-of-initialized-value\sproblem. +D 2023-07-22T16:37:28.699 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -713,7 +713,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c b3013dd11696db423bb410a50126d46d700fe3133b936a32195700e9731d2960 +F src/vdbemem.c 40afb83ed848e235848ffdd3ba25adca4ba602111b8ed3b05ae3b1b12e0eacee F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3661b9cd98565ea23b0cafd030668f09885dcbd54eb12b7b3cd85c289aef4c3c -R 7861df33493b30fd58a73dc8ed59a062 +P 1a6b3dd1c40277a0d0f0bb562ddc4868aadd632fc2d29be1b17bb33fc22c46c8 +R 2635d5866cacdf7a016f4afea5d617d0 U drh -Z 08afe3692a916f7ae05aca68accdd5b2 +Z 94bd1f809dedbfd836a0b0e28480b2ba # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e58ba7645d..42e6969aa7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a6b3dd1c40277a0d0f0bb562ddc4868aadd632fc2d29be1b17bb33fc22c46c8 \ No newline at end of file +931bccb0cc290b8bf3027641e7a7fac30e3244d7dc84aa9e38b24b7e9544ca06 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 3f845452c4..b5a794ae8f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -328,8 +328,8 @@ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) - && pMem->z[pMem->n]==0 ){ + pMem->z[pMem->n] = 0; pMem->flags |= MEM_Term; return; } From 479cfd5af3d46e70468eda8e31c4d033955bdd29 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 23 Jul 2023 22:14:41 +0000 Subject: [PATCH 54/56] When writing the filename header in a opfs-sahpool file, ensure that all remaining bytes in that part of the header are zeroed out to avoid the downstream problems reported in [forum:d50ec48a293988a5|forum post d50ec48a293988a5]. FossilOrigin-Name: c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 4 ++-- manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 28524b613c..9bf1421a9f 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -654,10 +654,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ */ setAssociatedPath(sah, path, flags){ const enc = textEncoder.encodeInto(path, this.#apBody); - if(HEADER_MAX_PATH_SIZE <= enc.written){ + if(HEADER_MAX_PATH_SIZE <= enc.written + 1/*NUL byte*/){ toss("Path too long:",path); } - + this.#apBody.fill(0, enc.written, HEADER_MAX_PATH_SIZE); this.#dvBody.setUint32(HEADER_OFFSET_FLAGS, flags); const digest = this.computeDigest(this.#apBody); diff --git a/manifest b/manifest index af4dd71733..ef9ea0e9d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sopfs-sahpool\ssqlite3_vfs\simplementation\sto\sJS,\soffering\san\salternative\sto\sthe\sother\sOPFS\sVFS\s(with\stradeoffs). -D 2023-07-22T19:57:42.982 +C When\swriting\sthe\sfilename\sheader\sin\sa\sopfs-sahpool\sfile,\sensure\sthat\sall\sremaining\sbytes\sin\sthat\spart\sof\sthe\sheader\sare\szeroed\sout\sto\savoid\sthe\sdownstream\sproblems\sreported\sin\s[forum:d50ec48a293988a5|forum\spost\sd50ec48a293988a5]. +D 2023-07-23T22:14:41.081 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js bb99a931388966a032f635a0cc9cd72685e067f21b95b2a58a660c055020b739 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 7931b50b63246a3d6b46a4c703c28820aa10c5b1ae7c0718e1f58dae2cf6db85 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e7a690e0e78ff4d563f2eca468f91db69f001ff4b79c6d2304cbb6f62dca437d F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2044,9 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 931bccb0cc290b8bf3027641e7a7fac30e3244d7dc84aa9e38b24b7e9544ca06 74ad31e2908af8225b7aa527dbcd1877423d58163e365317a78453b31e322ea3 -R 00ca22e33a29604f58c1aae1d32cd34b -T +closed 74ad31e2908af8225b7aa527dbcd1877423d58163e365317a78453b31e322ea3 Closed\sby\sintegrate-merge. +P d2e602cda44bf35e76167143262b4f91826d25780d0e095e680a31d5dedb2018 +R 47af828f435c00e0fb124001796bea21 U stephan -Z bed86151ea151954dfefb0b67895169f +Z 3f8f8cf2fbd99e80728947340dd883ad # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 49bce2c199..9921927a14 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2e602cda44bf35e76167143262b4f91826d25780d0e095e680a31d5dedb2018 \ No newline at end of file +c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3 \ No newline at end of file From c76a8862a73fdf90ec5ab1aa088b8b600bba2365 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 24 Jul 2023 15:41:58 +0000 Subject: [PATCH 55/56] Resolve an ES6 module export construct which is incompatible with node.js, as reported in [forum:b9680fa9ad|forum post b9680fa9ad]. FossilOrigin-Name: 80927c3913561dddf75cf73be871d93ae06b16f83e8cc36fc360765014209615 --- ext/wasm/api/extern-post-js.c-pp.js | 3 ++- ext/wasm/demo-123.js | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index fac00370dd..00cb3fab90 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -119,5 +119,6 @@ const toExportForESM = return globalThis.sqlite3InitModule /* required for ESM */; })(); //#if target=es6-module -export { toExportForESM as default, toExportForESM as sqlite3InitModule } +sqlite3InitModule = toExportForESM; +export default sqlite3InitModule; //#endif diff --git a/ext/wasm/demo-123.js b/ext/wasm/demo-123.js index 311afcc827..8e03ee80fa 100644 --- a/ext/wasm/demo-123.js +++ b/ext/wasm/demo-123.js @@ -235,10 +235,10 @@ - get change count (total or statement-local, 32- or 64-bit) - get a DB's file name - + Misc. Stmt features: - - Various forms of bind() + - Various forms of bind() - clearBindings() - reset() - Various forms of step() diff --git a/manifest b/manifest index ef9ea0e9d6..8d61da8af1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\swriting\sthe\sfilename\sheader\sin\sa\sopfs-sahpool\sfile,\sensure\sthat\sall\sremaining\sbytes\sin\sthat\spart\sof\sthe\sheader\sare\szeroed\sout\sto\savoid\sthe\sdownstream\sproblems\sreported\sin\s[forum:d50ec48a293988a5|forum\spost\sd50ec48a293988a5]. -D 2023-07-23T22:14:41.081 +C Resolve\san\sES6\smodule\sexport\sconstruct\swhich\sis\sincompatible\swith\snode.js,\sas\sreported\sin\s[forum:b9680fa9ad|forum\spost\sb9680fa9ad]. +D 2023-07-24T15:41:58.448 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -489,7 +489,7 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 5eb44fa02e9c693a1884a3692428647894b0380b24bca120866b7a24c8786134 -F ext/wasm/api/extern-post-js.c-pp.js 116749b7e55b7519129de06d3d353e19df68cfb24b12204aa4dc30c9a83023fe +F ext/wasm/api/extern-post-js.c-pp.js e7257ea56d4077d0773d7537fcb8eade59aad0c118dab92497c27939edca41cb F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 @@ -516,7 +516,7 @@ F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a7 F ext/wasm/common/whwasmutil.js ae263dec9d7384f4c530f324b99d00516a4d6f26424372daee65031e00eb49b3 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 -F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 +F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5 F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d2e602cda44bf35e76167143262b4f91826d25780d0e095e680a31d5dedb2018 -R 47af828f435c00e0fb124001796bea21 +P c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3 +R 1ab3847cf4557b99f62130a0f45e24bd U stephan -Z 3f8f8cf2fbd99e80728947340dd883ad +Z 4a3a3c6a8d937dc54de34852d2276ed0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9921927a14..649aa580be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3 \ No newline at end of file +80927c3913561dddf75cf73be871d93ae06b16f83e8cc36fc360765014209615 \ No newline at end of file From 322a3257033d2b7e1a1a136909ea1e73dc7d9291 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 25 Jul 2023 12:26:05 +0000 Subject: [PATCH 56/56] Reformulate a (sed -i) construct in ext/wasm/GNUmakefile to account for Mac's sed -i being different than GNU's. FossilOrigin-Name: 907dfc4a7aa129cdcedeb3ba2d75e1b68a8f22c2545ee1c8cf7d705041644e5c --- ext/wasm/GNUmakefile | 7 ++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 8fedaee7b6..2346773337 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -677,10 +677,15 @@ sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles) # # Upstream RFE: # https://github.com/emscripten-core/emscripten/issues/18237 +# +# Maintenance reminder: Mac sed -i works differently than GNU sed, so +# don't use that flag here. define SQLITE3.xJS.ESM-EXPORT-DEFAULT if [ x1 = x$(1) ]; then \ echo "Fragile workaround for emscripten/issues/18237. See SQLITE3.xJS.RECIPE."; \ - sed -i -e '0,/^export default/{/^export default/d;}' $@ || exit $$?; \ + {\ + sed -e '0,/^export default/{/^export default/d;}' $@ > $@.tmp && mv $@.tmp $@; \ + } || exit $$?; \ if [ x != x$(2) ]; then \ if ! grep -q '^export default' $@; then \ echo "Cannot find export default." 1>&2; \ diff --git a/manifest b/manifest index 8d61da8af1..55ef9f2d35 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Resolve\san\sES6\smodule\sexport\sconstruct\swhich\sis\sincompatible\swith\snode.js,\sas\sreported\sin\s[forum:b9680fa9ad|forum\spost\sb9680fa9ad]. -D 2023-07-24T15:41:58.448 +C Reformulate\sa\s(sed\s-i)\sconstruct\sin\sext/wasm/GNUmakefile\sto\saccount\sfor\sMac's\ssed\s-i\sbeing\sdifferent\sthan\sGNU's. +D 2023-07-25T12:26:05.338 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 4e8260d05c52d9924b853efbdfe052bd483cfe42f055567c1bbf29d274794b22 +F ext/wasm/GNUmakefile a3e316bf51d8915a49d4728e0d41f19ec4c3ead3409d1171989a4fe163ec2214 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c1b080e39397c983c13a5e79303223827de7b4946c18a79396851ec1814782f3 -R 1ab3847cf4557b99f62130a0f45e24bd +P 80927c3913561dddf75cf73be871d93ae06b16f83e8cc36fc360765014209615 +R 6a5bd7ff24557139b19e78c4a48cb406 U stephan -Z 4a3a3c6a8d937dc54de34852d2276ed0 +Z 524f840828c91fa63ac22024dbea11bc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 649aa580be..d8bafbc0f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -80927c3913561dddf75cf73be871d93ae06b16f83e8cc36fc360765014209615 \ No newline at end of file +907dfc4a7aa129cdcedeb3ba2d75e1b68a8f22c2545ee1c8cf7d705041644e5c \ No newline at end of file