Replace time-based auto-unlock of opfs sync handles with lock acquisition/release via sqlite3_io_methods::xLock/xUnlock().

FossilOrigin-Name: 2625b7cfe1640c1d7e779ec1f37db970541598c0dc3e22e5eecf3c772d95ad40

ext/wasm/api/sqlite3-api-opfs.js

@@ -169,6 +169,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
     W.onerror = function(err){
       // The error object doesn't contain any useful info when the
       // failure is, e.g., that the remote script is 404.
+      error("Error initializing OPFS asyncer:",err);
       promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons."));
     };
     const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/;
@@ -202,7 +203,6 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
        environment or the other when sqlite3_os_end() is called (_if_ it
        gets called at all in a wasm build, which is undefined).
     */
-
     /**
        State which we send to the async-api Worker or share with it.
        This object must initially contain only cloneable or sharable
@@ -224,7 +224,12 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
     */
     const state = Object.create(null);
     state.verbose = options.verbose;
-    state.littleEndian = true;
+    state.littleEndian = (()=>{
+      const buffer = new ArrayBuffer(2);
+      new DataView(buffer).setInt16(0, 256, true /* littleEndian */);
+      // Int16Array uses the platform's endianness.
+      return new Int16Array(buffer)[0] === 256;
+    })();
     /** Whether the async counterpart should log exceptions to
         the serialization channel. That produces a great deal of
         noise for seemingly innocuous things like xAccess() checks
@@ -265,7 +270,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
          between both workers. This worker writes to it and the other
          listens for changes. */
       state.opIds.whichOp = i++;
-      /* Slot for storing return values. This work listens to that
+      /* Slot for storing return values. This worker listens to that
          slot and the other worker writes to it. */
       state.opIds.rc = i++;
       /* Each function gets an ID which this worker writes to
@@ -278,11 +283,13 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
       state.opIds.xDeleteNoWait = i++;
       state.opIds.xFileControl = i++;
       state.opIds.xFileSize = i++;
+      state.opIds.xLock = i++;
       state.opIds.xOpen = i++;
       state.opIds.xRead = i++;
       state.opIds.xSleep = i++;
       state.opIds.xSync = i++;
       state.opIds.xTruncate = i++;
+      state.opIds.xUnlock = i++;
       state.opIds.xWrite = i++;
       state.opIds.mkdir = i++;
       state.opIds['opfs-async-metrics'] = i++;
@@ -290,7 +297,6 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
       state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/);
       opfsUtil.metrics.reset();
     }
-
     /**
        SQLITE_xxx constants to export to the async worker
        counterpart...
@@ -304,10 +310,17 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
       'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
       'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
       'SQLITE_IOERR_DELETE',
+      'SQLITE_LOCK_NONE',
+      'SQLITE_LOCK_SHARED',
+      'SQLITE_LOCK_RESERVED',
+      'SQLITE_LOCK_PENDING',
+      'SQLITE_LOCK_EXCLUSIVE',
       'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE',
       'SQLITE_OPEN_READONLY'
-    ].forEach(function(k){
+    ].forEach((k)=>{
-      state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
+      if(undefined === (state.sq3Codes[k] = capi[k])){
+        toss("Maintenance required: not found:",k);
+      }
     });
 
     /**
@@ -605,7 +618,8 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
       xCheckReservedLock: function(pFile,pOut){
         // Exclusive lock is automatically acquired when opened
         //warn("xCheckReservedLock(",arguments,") is a no-op");
-        wasm.setMemValue(pOut,1,'i32');
+        const f = __openFiles[pFile];
+        wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32');
         return 0;
       },
       xClose: function(pFile){
@@ -643,9 +657,17 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
         return rc;
       },
       xLock: function(pFile,lockType){
-        //2022-09: OPFS handles lock when opened
+        mTimeStart('xLock');
-        //warn("xLock(",arguments,") is a no-op");
+        const f = __openFiles[pFile];
-        return 0;
+        let rc = 0;
+        if( capi.SQLITE_LOCK_NONE === f.lockType ) {
+          rc = opRun('xLock', pFile, lockType);
+          if( 0===rc ) f.lockType = lockType;
+        }else{
+          f.lockType = lockType;
+        }
+        mTimeEnd();
+        return rc;
       },
       xRead: function(pFile,pDest,n,offset64){
         /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */
@@ -676,9 +698,16 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
         return rc;
       },
       xUnlock: function(pFile,lockType){
-        //2022-09: OPFS handles lock when opened
+        mTimeStart('xUnlock');
-        //warn("xUnlock(",arguments,") is a no-op");
+        const f = __openFiles[pFile];
-        return 0;
+        let rc = 0;
+        if( capi.SQLITE_LOCK_NONE === lockType
+          && f.lockType ){
+          rc = opRun('xUnlock', pFile, lockType);
+        }
+        if( 0===rc ) f.lockType = lockType;
+        mTimeEnd();
+        return rc;
       },
       xWrite: function(pFile,pSrc,n,offset64){
         /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */
@@ -790,6 +819,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
           fh.sabView = state.sabFileBufView;
           fh.sq3File = new sqlite3_file(pFile);
           fh.sq3File.$pMethods = opfsIoMethods.pointer;
+          fh.lockType = capi.SQLITE_LOCK_NONE;
         }
         mTimeEnd();
         return rc;
@@ -1061,5 +1091,12 @@ installOpfsVfs.defaultProxyUri =
     //self.location.pathname.replace(/[^/]*$/, "sqlite3-opfs-async-proxy.js");
   "sqlite3-opfs-async-proxy.js";
 //console.warn("sqlite3.installOpfsVfs.defaultProxyUri =",sqlite3.installOpfsVfs.defaultProxyUri);
-self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>installOpfsVfs());
+self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{
+  try{
+    return installOpfsVfs();
+  }catch(e){
+    console.error("installOpfsVfs() exception:",e);
+    throw e;
+  }
+});
 }/*sqlite3ApiBootstrap.initializers.push()*/);

ext/wasm/api/sqlite3-api-prologue.js

@@ -1312,14 +1312,25 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
       // Is it okay to resolve these in parallel or do we need them
       // to resolve in order? We currently only have 1, so it
       // makes no difference.
-      lip = lip.map((f)=>f(sqlite3).catch(()=>{}));
+      lip = lip.map((f)=>f(sqlite3).catch((e)=>{
+        console.error("An async sqlite3 initializer failed:",e);
+      }));
       //let p = lip.shift();
       //while(lip.length) p = p.then(lip.shift());
       //return p.then(()=>sqlite3);
       return Promise.all(lip).then(()=>sqlite3);
     }
   };
-  sqlite3ApiBootstrap.initializers.forEach((f)=>f(sqlite3));
+  try{
+    sqlite3ApiBootstrap.initializers.forEach((f)=>{
+      f(sqlite3);
+    });
+  }catch(e){
+    /* If we don't report this here, it can get completely swallowed
+       up and disappear into the abyss of Promises and Workers. */
+    console.error("sqlite3 bootstrap initializer threw:",e);
+    throw e;
+  }
   delete sqlite3ApiBootstrap.initializers;
   sqlite3ApiBootstrap.sqlite3 = sqlite3;
   return sqlite3;

ext/wasm/sqlite3-opfs-async-proxy.js

@@ -41,6 +41,9 @@ if(self.window === self){
 }else if(!navigator.storage.getDirectory){
   toss("This API requires navigator.storage.getDirectory.");
 }
+
+//warn("This file is very much experimental and under construction.",self.location.pathname);
+
 /**
    Will hold state copied to this object from the syncronous side of
    this API.
@@ -97,8 +100,6 @@ metrics.dump = ()=>{
   console.log("Serialization metrics:",metrics.s11n);
 };
 
-//warn("This file is very much experimental and under construction.",self.location.pathname);
-
 /**
    Map of sqlite3_file pointers (integers) to metadata related to a
    given OPFS file handles. The pointers are, in this side of the
@@ -142,8 +143,7 @@ const getDirForFilename = async function f(absFilename, createDirs = false){
 /**
    Returns the sync access handle associated with the given file
    handle object (which must be a valid handle object), lazily opening
-   it if needed. Timestamps the handle for use in relinquishing it
+   it if needed.
-   during idle time.
 
    In order to help alleviate cross-tab contention for a dabase,
    if an exception is thrown while acquiring the handle, this routine
@@ -177,13 +177,12 @@ const getSyncHandle = async (fh)=>{
     }
     log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
   }
-  fh.syncHandleTime = performance.now();
   return fh.syncHandle;
 };
 
 const closeSyncHandle = async (fh)=>{
   if(fh.syncHandle){
-    //warn("Closing sync handle for",fh.filenameAbs);
+    log("Closing sync handle for",fh.filenameAbs);
     const h = fh.syncHandle;
     delete fh.syncHandle;
     return h.close();
@@ -239,6 +238,7 @@ const wTimeEnd = ()=>(
 */
 let flagAsyncShutdown = false;
 
+
 /**
    Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods
    methods. Maintenance reminder: members are in alphabetical order
@@ -373,6 +373,20 @@ const vfsAsyncImpls = {
     storeAndNotify('xFileSize', sz);
     mTimeEnd();
   },
+  xLock: async function(fid,lockType){
+    mTimeStart('xLock');
+    const fh = __openFiles[fid];
+    let rc = 0;
+    if( !fh.syncHandle ){
+      try { await getSyncHandle(fh) }
+      catch(e){
+        state.s11n.storeException(1,e);
+        rc = state.sq3Codes.SQLITE_IOERR;
+      }
+    }
+    storeAndNotify('xLock',rc);
+    mTimeEnd();
+  },
   xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags){
     const opName = 'xOpen';
     mTimeStart(opName);
@@ -473,6 +487,23 @@ const vfsAsyncImpls = {
     storeAndNotify('xTruncate',rc);
     mTimeEnd();
   },
+  xUnlock: async function(fid,lockType){
+    mTimeStart('xUnlock');
+    let rc = 0;
+    const fh = __openFiles[fid];
+    if( state.sq3Codes.SQLITE_LOCK_NONE===lockType
+        && fh.syncHandle ){
+      try { await closeSyncHandle(fh) }
+      catch(e){
+        state.s11n.storeException(1,e);
+        rc = state.sq3Codes.SQLITE_IOERR;
+        /* Maybe we want to not report this? "Destructors do not
+           throw." */
+      }
+    }
+    storeAndNotify('xUnlock',rc);
+    mTimeEnd();
+  },
   xWrite: async function(fid,n,offset){
     mTimeStart('xWrite');
     let rc;
@@ -495,7 +526,7 @@ const vfsAsyncImpls = {
     storeAndNotify('xWrite',rc);
     mTimeEnd();
   }
-};
+}/*vfsAsyncImpls*/;
 
 const initS11n = ()=>{
   /**
@@ -617,21 +648,7 @@ const waitLoop = async function f(){
      We need to wake up periodically to give the thread a chance
      to do other things.
   */
-  const waitTime = 500;
+  const waitTime = 1000;
-  /**
-     relinquishTime defines the_approximate_ number of ms after which
-     a db sync access handle will be relinquished so that we do not
-     hold a persistent lock on it. When the following loop times out
-     while waiting, every (approximate) increment of this value it
-     will relinquish any db handles which have been idle for at least
-     this much time.
-
-     Reaquisition of a sync handle seems to take an average of
-     0.6-0.9ms on this dev machine but takes anywhere from 1-3ms every
-     once in a while (maybe 1 time in 5 or 10). Outliers as long as
-     7ms have been witnessed, but they're rare.
-  */
-  const relinquishTime = 500;
   let lastOpTime = performance.now();
   let now;
   while(!flagAsyncShutdown){
@@ -639,21 +656,6 @@ const waitLoop = async function f(){
       if('timed-out'===Atomics.wait(
         state.sabOPView, state.opIds.whichOp, 0, waitTime
       )){
-        if(relinquishTime &&
-           (lastOpTime + relinquishTime <= (now = performance.now()))){
-          for(const fh of Object.values(__openFiles)){
-            if(fh.syncHandle && (
-              now - relinquishTime >= fh.syncHandleTime
-            )){
-              log("Relinquishing for timeout:",fh.filenameAbs);
-              await closeSyncHandle(fh)
-              /* Testing shows that we have to wait on this async
-                 op to finish, else we might try to re-open it
-                 before the close has run. The FS layer does not
-                 retain the order those operations, apparently. */;
-            }
-          }
-        }
         continue;
       }
       lastOpTime = performance.now();
@@ -719,4 +721,4 @@ navigator.storage.getDirectory().then(function(d){
     }
   };
   wMsg('opfs-async-loaded');
-}).catch((e)=>error(e));
+}).catch((e)=>error("error initializing OPFS asyncer:",e));

ext/wasm/test-opfs-vfs.js

@@ -34,10 +34,6 @@ const tryOpfsVfs = async function(sqlite3){
   const wait = async (ms)=>{
     return new Promise((resolve)=>setTimeout(resolve, ms));
   };
-  const waitForRelinquish = async ()=>{
-    log("Waiting briefly to test sync handle relinquishing...");
-    return wait(1500);
-  };
 
   const urlArgs = new URL(self.location.href).searchParams;
   const dbFile = "my-persistent.db";
@@ -45,13 +41,11 @@ const tryOpfsVfs = async function(sqlite3){
 
   const db = new opfs.OpfsDb(dbFile,'ct');
   log("db file:",db.filename);
-  await waitForRelinquish();
   try{
     if(opfs.entryExists(dbFile)){
       let n = db.selectValue("select count(*) from sqlite_schema");
       log("Persistent data found. sqlite_schema entry count =",n);
     }
-    await waitForRelinquish();
     db.transaction((db)=>{
       db.exec({
         sql:[
@@ -63,7 +57,6 @@ const tryOpfsVfs = async function(sqlite3){
                (performance.now() |0) / 4]
       });
     });
-    await waitForRelinquish();
     log("count(*) from t =",db.selectValue("select count(*) from t"));
 
     // Some sanity checks of the opfs utility functions...

manifest

@@ -1,5 +1,5 @@
-C Tweaks\sto\sthe\sopfs\sasync\swait/relinquish\stimes.
+C Replace\stime-based\sauto-unlock\sof\sopfs\ssync\shandles\swith\slock\sacquisition/release\svia\ssqlite3_io_methods::xLock/xUnlock().
-D 2022-10-04T11:14:23.298
+D 2022-10-04T17:06:51.812
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -487,8 +487,8 @@ F ext/wasm/api/pre-js.js 2db711eb637991b383fc6b5c0f3df65ec48a7201e5730e304beba8d
 F ext/wasm/api/sqlite3-api-cleanup.js 5d22d1d3818ecacb23bfa223d5970cd0617d8cdbb48c8bc4bbd463f05b021a99
 F ext/wasm/api/sqlite3-api-glue.js a5a1bd620e2e2c26b6e843cf439548b35c5bb5ed21b24b89a412e0b0a8592c42
 F ext/wasm/api/sqlite3-api-oo1.js ac1e08d36bdfb5aa0a2d75b7d4c892fd51819d34c932370c3282810672bcc086
-F ext/wasm/api/sqlite3-api-opfs.js 0db417af18fdcd39171366d9ac9e7bf57c5236aea4d2de26c759e65a22b7c446
+F ext/wasm/api/sqlite3-api-opfs.js 78fd272d6ec4fe6da93910b978f693fff63beed8e0e5e7dab42a25dbdfefca4b
-F ext/wasm/api/sqlite3-api-prologue.js baf88cf253a35153674b068b5125238993ea2a7804152d238b5e40140b506ef5
+F ext/wasm/api/sqlite3-api-prologue.js 587e6aa373bd85e83bcc3fb82b3636b01aaf2caefa39679e02574e766560d711
 F ext/wasm/api/sqlite3-api-worker1.js 7f4f46cb6b512a48572d7567233896e6a9c46570c44bdc3d13419730c7c221c8
 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
 F ext/wasm/api/sqlite3-wasm.c 0c8cb242b80b8de27b74f05ed781f427958e04c9339c4720f30c3a7e78ee6242
@@ -522,11 +522,11 @@ F ext/wasm/speedtest1.html e4cb5d722b494104fc1249e7c008ca018f820a784833c51004c95
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
-F ext/wasm/sqlite3-opfs-async-proxy.js 0861d0eff1ffb441b4ad98781a37825252ae2f08482cc5b29c27f34bcf952049
+F ext/wasm/sqlite3-opfs-async-proxy.js 40fc1f779e18e08b31404ef832381ffd36b9d11447007aaf96223198324e4467
 F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8
 F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf
 F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
-F ext/wasm/test-opfs-vfs.js 0115d56f3f2a6475040dffe75a6851a353b1690708795944cf0fe1890bc7ff54
+F ext/wasm/test-opfs-vfs.js 56c3d725044c668fa7910451e96c1195d25ad95825f9ac79f747a7759d1973d0
 F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893
 F ext/wasm/testing-worker1-promiser.js bd788e33c1807e0a6dda9c9a9d784bd3350ca49c9dd8ae2cc8719b506b6e013e
 F ext/wasm/testing1.html 50575755e43232dbe4c2f97c9086b3118eb91ec2ee1fae931e6d7669fb17fcae
@@ -2029,8 +2029,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9d488081fc82a9abe3e81c7897fe6b29b6b337f62d0c62cb9cec7517bd54b53f
+P 35f33c23e5849de1c43c4499ee0a7fa11d26ae34949c1e820c3fa8e8873f9c2b
-R 616ce8132e7f98fd04bd3d25f88d4168
+R 447ab40d8f9d9bacdb3bbf1fc5e8416f
 U stephan
-Z da44b1ca0ef51fa028d53641db451066
+Z 1f6ebc0e7b8d0de3ff6621f210427c47
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-35f33c23e5849de1c43c4499ee0a7fa11d26ae34949c1e820c3fa8e8873f9c2b
+2625b7cfe1640c1d7e779ec1f37db970541598c0dc3e22e5eecf3c772d95ad40