diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index b19568d637..b951873ea5 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -142,14 +142,28 @@ const getDirForFilename = async function f(absFilename, createDirs = false){ /** Returns the sync access handle associated with the given file handle object (which must be a valid handle object), lazily opening - it if needed. + it if needed. Timestamps the handle for use in relinquishing it + during idle time. */ -const getSyncHandle = async (f)=>( - f.accessHandle || ( - f.accessHandle = - await f.fileHandle.createSyncAccessHandle() - ) -); +const getSyncHandle = async (fh)=>{ + if(!fh.syncHandle){ + //const t = performance.now(); + //warn("Creating sync handle for",fh.filenameAbs); + fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); + //warn("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + } + fh.syncHandleTime = performance.now(); + return fh.syncHandle; +}; + +const closeSyncHandle = async (fh)=>{ + if(fh.syncHandle){ + //warn("Closing sync handle for",fh.filenameAbs); + const h = fh.syncHandle; + delete fh.syncHandle; + return h.close(); + } +}; /** Stores the given value at state.sabOPView[state.opIds.rc] and then @@ -255,7 +269,7 @@ const vfsAsyncImpls = { wTimeStart('xClose'); if(fh){ delete __openFiles[fid]; - if(fh.accessHandle) await fh.accessHandle.close(); + await closeSyncHandle(fh); if(fh.deleteOnClose){ try{ await fh.dirHandle.removeEntry(fh.filenamePart) } catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } @@ -342,12 +356,12 @@ const vfsAsyncImpls = { const hFile = await hDir.getFileHandle(filenamePart, {create}); /** wa-sqlite, at this point, grabs a SyncAccessHandle and - assigns it to the accessHandle prop of the file state + assigns it to the syncHandle prop of the file state object, but only for certain cases and it's unclear why it places that limitation on it. */ wTimeEnd(); - const fobj = Object.assign(Object.create(null),{ + __openFiles[fid] = Object.assign(Object.create(null),{ filenameAbs: filename, filenamePart: filenamePart, dirHandle: hDir, @@ -357,7 +371,6 @@ const vfsAsyncImpls = { ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), deleteOnClose: deleteOnClose }); - __openFiles[fid] = fobj; storeAndNotify(opName, 0); }catch(e){ wTimeEnd(); @@ -370,8 +383,8 @@ const vfsAsyncImpls = { xRead: async function(fid,n,offset){ mTimeStart('xRead'); let rc = 0; + const fh = __openFiles[fid]; try{ - const fh = __openFiles[fid]; wTimeStart('xRead'); const nRead = (await getSyncHandle(fh)).read( fh.sabView.subarray(0, n), @@ -394,10 +407,10 @@ const vfsAsyncImpls = { mTimeStart('xSync'); const fh = __openFiles[fid]; let rc = 0; - if(!fh.readOnly && fh.accessHandle){ + if(!fh.readOnly && fh.syncHandle){ try { wTimeStart('xSync'); - await fh.accessHandle.flush(); + await fh.syncHandle.flush(); }catch(e){ state.s11n.storeException(2,e); }finally{ @@ -563,23 +576,58 @@ const waitLoop = async function f(){ o.key = k; o.f = vi || toss("No vfsAsyncImpls[",k,"]"); } + /** + waitTime is how long (ms) to wait for each Atomics.wait(). + We need to wait up periodically to give the thread a chance + to do other things. + */ + const waitTime = 250; + /** + relinquishTime defines the_approximate_ number of ms + after which a db sync access handle will be relinquished + so that we do not hold a persistent lock on it. When the following loop + times out while waiting, every (approximate) increment of this + value it will relinquish any db handles which have been idle for + at least this much time. + + Reaquisition of a sync handle seems to take an average of + 0.6-0.9ms on this dev machine but takes anywhere from 1-3ms + every once in a while (maybe 1 time in 5 or 10). + */ + const relinquishTime = 1000; + let lastOpTime = performance.now(); + let now; while(true){ try { - if('timed-out'===Atomics.wait(state.sabOPView, state.opIds.whichOp, 0, 500)){ + if('timed-out'===Atomics.wait( + state.sabOPView, state.opIds.whichOp, 0, waitTime + )){ + if(relinquishTime && + (lastOpTime + relinquishTime <= (now = performance.now()))){ + for(const fh of Object.values(__openFiles)){ + if(fh.syncHandle && ( + now - relinquishTime >= fh.syncHandleTime + )){ + //warn("Relinquishing for timeout:",fh.filenameAbs); + closeSyncHandle(fh)/*async!*/; + } + } + } continue; } + lastOpTime = performance.now(); const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); Atomics.store(state.sabOPView, state.opIds.whichOp, 0); const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); const args = state.s11n.deserialize() || []; - state.s11n.serialize()/* clear s11n to keep the caller from - confusing this with an exception string - written by the upcoming operation */; + state.s11n.serialize(/* clear s11n to keep the caller from + confusing this with an exception string + written by the upcoming operation */); //warn("waitLoop() whichOp =",opId, hnd, args); if(hnd.f) await hnd.f(...args); else error("Missing callback for opId",opId); }catch(e){ - error('in waitLoop():',e.message); + error('in waitLoop():',e); } }; }; diff --git a/ext/wasm/test-opfs-vfs.js b/ext/wasm/test-opfs-vfs.js index 78f86e380f..755b73a97c 100644 --- a/ext/wasm/test-opfs-vfs.js +++ b/ext/wasm/test-opfs-vfs.js @@ -11,18 +11,9 @@ *********************************************************************** A testing ground for the OPFS VFS. - - Summary of how this works: - - This file uses the sqlite3.StructBinder-created struct wrappers for - sqlite3_vfs, sqlite3_io_methods, ans sqlite3_file to set up a - conventional sqlite3_vfs (except that it's implemented in JS). The - methods which require OPFS APIs use a separate worker (hereafter called the - OPFS worker) to access that functionality. This worker and that one - use SharedArrayBuffer. */ 'use strict'; -const tryOpfsVfs = function(sqlite3){ +const tryOpfsVfs = async function(sqlite3){ const toss = function(...args){throw new Error(args.join(' '))}; const logPrefix = "OPFS tester:"; const log = (...args)=>console.log(logPrefix,...args); @@ -40,17 +31,27 @@ const tryOpfsVfs = function(sqlite3){ const oVfs = capi.sqlite3_vfs.instanceForPointer(pVfs) || toss("Unexpected instanceForPointer() result.");; log("OPFS VFS:",pVfs, oVfs); + const wait = async (ms)=>{ + return new Promise((resolve)=>setTimeout(resolve, ms)); + }; + const waitForRelinquish = async ()=>{ + log("Waiting briefly to test sync handle relinquishing..."); + return wait(1500); + }; + const urlArgs = new URL(self.location.href).searchParams; const dbFile = "my-persistent.db"; if(urlArgs.has('delete')) sqlite3.opfs.deleteEntry(dbFile); const db = new opfs.OpfsDb(dbFile); log("db file:",db.filename); + await waitForRelinquish(); try{ if(opfs.entryExists(dbFile)){ let n = db.selectValue("select count(*) from sqlite_schema"); log("Persistent data found. sqlite_schema entry count =",n); } + await waitForRelinquish(); db.transaction((db)=>{ db.exec({ sql:[ @@ -62,6 +63,7 @@ const tryOpfsVfs = function(sqlite3){ (performance.now() |0) / 4] }); }); + await waitForRelinquish(); log("count(*) from t =",db.selectValue("select count(*) from t")); // Some sanity checks of the opfs utility functions... diff --git a/manifest b/manifest index 55695edc1f..1e61d92d7e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sjournal=MODE\sto\sthe\slist\sof\ssupported\sURL\sflags\sfor\sspeedtest1-worker.html. -D 2022-10-03T11:23:47.443 +C Experimentally\srelinquish\sthe\sOPFS\sVFS\ssync\saccess\shandle\swhen\sthe\sdb\sis\sidle\sand\sreacquire\sit\son\sdemand,\sthe\sgoal\sbeing\sto\shelp\salleviate\scross-tab\slocking\sissues. +D 2022-10-03T11:33:35.536 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -522,11 +522,11 @@ F ext/wasm/speedtest1.html e4cb5d722b494104fc1249e7c008ca018f820a784833c51004c95 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js 4894fc7767d3ac3cd4b9e377585ad0f07f0fc7da7cf47b8fd302454082908715 +F ext/wasm/sqlite3-opfs-async-proxy.js d571a40230f54b99863f469e050cc4d4317e90239e64026d8d574689956ff76c F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8 F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 -F ext/wasm/test-opfs-vfs.js a59ff9210b17d46b0c6fbf6a0ba60143c033327865f2e556e14f06280cef62ac +F ext/wasm/test-opfs-vfs.js 623de926913fa8cb7bac170f105a0c397f1951aba52d9f54792aadce7892a3c2 F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893 F ext/wasm/testing-worker1-promiser.js bd788e33c1807e0a6dda9c9a9d784bd3350ca49c9dd8ae2cc8719b506b6e013e F ext/wasm/testing1.html 50575755e43232dbe4c2f97c9086b3118eb91ec2ee1fae931e6d7669fb17fcae @@ -2029,8 +2029,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a984e1ba435731413a541f86c50232bc7d6e33aff6ba4cca90f89188e7b82a2c -R 21cc3050e6217e8423f4278520aeac9f +P 5c43e8d2ec23f28fdce63874d9d4c8ccbb4a8f81030b19d65ff5711a7c32697e +R 1a27bfc63f0ef541783eed07221dd339 U stephan -Z c1aed4bb1f5e7222d859541f8a82e5f6 +Z 39513e5d3a3cf9877958a5d88a7ecd20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9162ec001c..4f33119d84 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c43e8d2ec23f28fdce63874d9d4c8ccbb4a8f81030b19d65ff5711a7c32697e \ No newline at end of file +2703ac9842335962e488e597168d70b1389b95a6ad39edf70a211b95979b4708 \ No newline at end of file