Experimentally relinquish the OPFS VFS sync access handle when the db is idle and reacquire it on demand, the goal being to help alleviate cross-tab locking issues.

FossilOrigin-Name: 2703ac9842335962e488e597168d70b1389b95a6ad39edf70a211b95979b4708

ext/wasm/sqlite3-opfs-async-proxy.js

@@ -142,14 +142,28 @@ const getDirForFilename = async function f(absFilename, createDirs = false){
 /**
    Returns the sync access handle associated with the given file
    handle object (which must be a valid handle object), lazily opening
-   it if needed.
+   it if needed. Timestamps the handle for use in relinquishing it
+   during idle time.
 */
-const getSyncHandle = async (f)=>(
-  f.accessHandle || (
-    f.accessHandle =
-      await f.fileHandle.createSyncAccessHandle()
-  )
-);
+const getSyncHandle = async (fh)=>{
+  if(!fh.syncHandle){
+    //const t = performance.now();
+    //warn("Creating sync handle for",fh.filenameAbs);
+    fh.syncHandle = await fh.fileHandle.createSyncAccessHandle();
+    //warn("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
+  }
+  fh.syncHandleTime = performance.now();
+  return fh.syncHandle;
+};
+
+const closeSyncHandle = async (fh)=>{
+  if(fh.syncHandle){
+    //warn("Closing sync handle for",fh.filenameAbs);
+    const h = fh.syncHandle;
+    delete fh.syncHandle;
+    return h.close();
+  }
+};
 
 /**
    Stores the given value at state.sabOPView[state.opIds.rc] and then
@@ -255,7 +269,7 @@ const vfsAsyncImpls = {
     wTimeStart('xClose');
     if(fh){
       delete __openFiles[fid];
-      if(fh.accessHandle) await fh.accessHandle.close();
+      await closeSyncHandle(fh);
       if(fh.deleteOnClose){
         try{ await fh.dirHandle.removeEntry(fh.filenamePart) }
         catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) }
@@ -342,12 +356,12 @@ const vfsAsyncImpls = {
       const hFile = await hDir.getFileHandle(filenamePart, {create});
       /**
          wa-sqlite, at this point, grabs a SyncAccessHandle and
-         assigns it to the accessHandle prop of the file state
+         assigns it to the syncHandle prop of the file state
          object, but only for certain cases and it's unclear why it
          places that limitation on it.
       */
       wTimeEnd();
-      const fobj = Object.assign(Object.create(null),{
+      __openFiles[fid] = Object.assign(Object.create(null),{
         filenameAbs: filename,
         filenamePart: filenamePart,
         dirHandle: hDir,
@@ -357,7 +371,6 @@ const vfsAsyncImpls = {
           ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags),
         deleteOnClose: deleteOnClose
       });
-      __openFiles[fid] = fobj;
       storeAndNotify(opName, 0);
     }catch(e){
       wTimeEnd();
@@ -370,8 +383,8 @@ const vfsAsyncImpls = {
   xRead: async function(fid,n,offset){
     mTimeStart('xRead');
     let rc = 0;
-    try{
     const fh = __openFiles[fid];
+    try{
       wTimeStart('xRead');
       const nRead = (await getSyncHandle(fh)).read(
         fh.sabView.subarray(0, n),
@@ -394,10 +407,10 @@ const vfsAsyncImpls = {
     mTimeStart('xSync');
     const fh = __openFiles[fid];
     let rc = 0;
-    if(!fh.readOnly && fh.accessHandle){
+    if(!fh.readOnly && fh.syncHandle){
       try {
         wTimeStart('xSync');
-        await fh.accessHandle.flush();
+        await fh.syncHandle.flush();
       }catch(e){
         state.s11n.storeException(2,e);
       }finally{
@@ -563,23 +576,58 @@ const waitLoop = async function f(){
     o.key = k;
     o.f = vi || toss("No vfsAsyncImpls[",k,"]");
   }
+  /**
+     waitTime is how long (ms) to wait for each Atomics.wait().
+     We need to wait up periodically to give the thread a chance
+     to do other things.
+  */
+  const waitTime = 250;
+  /**
+     relinquishTime defines the_approximate_ number of ms
+     after which a db sync access handle will be relinquished
+     so that we do not hold a persistent lock on it. When the following loop
+     times out while waiting, every (approximate) increment of this
+     value it will relinquish any db handles which have been idle for
+     at least this much time.
+
+     Reaquisition of a sync handle seems to take an average of
+     0.6-0.9ms on this dev machine but takes anywhere from 1-3ms
+     every once in a while (maybe 1 time in 5 or 10).
+  */
+  const relinquishTime = 1000;
+  let lastOpTime = performance.now();
+  let now;
   while(true){
     try {
-      if('timed-out'===Atomics.wait(state.sabOPView, state.opIds.whichOp, 0, 500)){
+      if('timed-out'===Atomics.wait(
+        state.sabOPView, state.opIds.whichOp, 0, waitTime
+      )){
+        if(relinquishTime &&
+           (lastOpTime + relinquishTime <= (now = performance.now()))){
+          for(const fh of Object.values(__openFiles)){
+            if(fh.syncHandle && (
+              now - relinquishTime >= fh.syncHandleTime
+            )){
+              //warn("Relinquishing for timeout:",fh.filenameAbs);
+              closeSyncHandle(fh)/*async!*/;
+            }
+          }
+        }
         continue;
       }
+      lastOpTime = performance.now();
       const opId = Atomics.load(state.sabOPView, state.opIds.whichOp);
       Atomics.store(state.sabOPView, state.opIds.whichOp, 0);
       const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId);
       const args = state.s11n.deserialize() || [];
-      state.s11n.serialize()/* clear s11n to keep the caller from
+      state.s11n.serialize(/* clear s11n to keep the caller from
                               confusing this with an exception string
-                               written by the upcoming operation */;
+                              written by the upcoming operation */);
       //warn("waitLoop() whichOp =",opId, hnd, args);
       if(hnd.f) await hnd.f(...args);
       else error("Missing callback for opId",opId);
     }catch(e){
-      error('in waitLoop():',e.message);
+      error('in waitLoop():',e);
     }
   };
 };

ext/wasm/test-opfs-vfs.js

@@ -11,18 +11,9 @@
   ***********************************************************************
 
   A testing ground for the OPFS VFS.
-
-  Summary of how this works:
-
-  This file uses the sqlite3.StructBinder-created struct wrappers for
-  sqlite3_vfs, sqlite3_io_methods, ans sqlite3_file to set up a
-  conventional sqlite3_vfs (except that it's implemented in JS). The
-  methods which require OPFS APIs use a separate worker (hereafter called the
-  OPFS worker) to access that functionality. This worker and that one
-  use SharedArrayBuffer.
 */
 'use strict';
-const tryOpfsVfs = function(sqlite3){
+const tryOpfsVfs = async function(sqlite3){
   const toss = function(...args){throw new Error(args.join(' '))};
   const logPrefix = "OPFS tester:";
   const log = (...args)=>console.log(logPrefix,...args);
@@ -40,17 +31,27 @@ const tryOpfsVfs = function(sqlite3){
   const oVfs = capi.sqlite3_vfs.instanceForPointer(pVfs) || toss("Unexpected instanceForPointer() result.");;
   log("OPFS VFS:",pVfs, oVfs);
 
+  const wait = async (ms)=>{
+    return new Promise((resolve)=>setTimeout(resolve, ms));
+  };
+  const waitForRelinquish = async ()=>{
+    log("Waiting briefly to test sync handle relinquishing...");
+    return wait(1500);
+  };
+
   const urlArgs = new URL(self.location.href).searchParams;
   const dbFile = "my-persistent.db";
   if(urlArgs.has('delete')) sqlite3.opfs.deleteEntry(dbFile);
 
   const db = new opfs.OpfsDb(dbFile);
   log("db file:",db.filename);
+  await waitForRelinquish();
   try{
     if(opfs.entryExists(dbFile)){
       let n = db.selectValue("select count(*) from sqlite_schema");
       log("Persistent data found. sqlite_schema entry count =",n);
     }
+    await waitForRelinquish();
     db.transaction((db)=>{
       db.exec({
         sql:[
@@ -62,6 +63,7 @@ const tryOpfsVfs = function(sqlite3){
                (performance.now() |0) / 4]
       });
     });
+    await waitForRelinquish();
     log("count(*) from t =",db.selectValue("select count(*) from t"));
 
     // Some sanity checks of the opfs utility functions...

manifest

@@ -1,5 +1,5 @@
-C Add\sjournal=MODE\sto\sthe\slist\sof\ssupported\sURL\sflags\sfor\sspeedtest1-worker.html.
-D 2022-10-03T11:23:47.443
+C Experimentally\srelinquish\sthe\sOPFS\sVFS\ssync\saccess\shandle\swhen\sthe\sdb\sis\sidle\sand\sreacquire\sit\son\sdemand,\sthe\sgoal\sbeing\sto\shelp\salleviate\scross-tab\slocking\sissues.
+D 2022-10-03T11:33:35.536
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -522,11 +522,11 @@ F ext/wasm/speedtest1.html e4cb5d722b494104fc1249e7c008ca018f820a784833c51004c95
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
-F ext/wasm/sqlite3-opfs-async-proxy.js 4894fc7767d3ac3cd4b9e377585ad0f07f0fc7da7cf47b8fd302454082908715
+F ext/wasm/sqlite3-opfs-async-proxy.js d571a40230f54b99863f469e050cc4d4317e90239e64026d8d574689956ff76c
 F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8
 F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf
 F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
-F ext/wasm/test-opfs-vfs.js a59ff9210b17d46b0c6fbf6a0ba60143c033327865f2e556e14f06280cef62ac
+F ext/wasm/test-opfs-vfs.js 623de926913fa8cb7bac170f105a0c397f1951aba52d9f54792aadce7892a3c2
 F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893
 F ext/wasm/testing-worker1-promiser.js bd788e33c1807e0a6dda9c9a9d784bd3350ca49c9dd8ae2cc8719b506b6e013e
 F ext/wasm/testing1.html 50575755e43232dbe4c2f97c9086b3118eb91ec2ee1fae931e6d7669fb17fcae
@@ -2029,8 +2029,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a984e1ba435731413a541f86c50232bc7d6e33aff6ba4cca90f89188e7b82a2c
-R 21cc3050e6217e8423f4278520aeac9f
+P 5c43e8d2ec23f28fdce63874d9d4c8ccbb4a8f81030b19d65ff5711a7c32697e
+R 1a27bfc63f0ef541783eed07221dd339
 U stephan
-Z c1aed4bb1f5e7222d859541f8a82e5f6
+Z 39513e5d3a3cf9877958a5d88a7ecd20
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-5c43e8d2ec23f28fdce63874d9d4c8ccbb4a8f81030b19d65ff5711a7c32697e
+2703ac9842335962e488e597168d70b1389b95a6ad39edf70a211b95979b4708