More tweaking of OPFS concurrency measures and the related test app.

FossilOrigin-Name: a8d4da1501d411085ec2fd48c4a056c8b1d97ef3c3203c5b403a854ac2864870

ext/wasm/api/sqlite3-opfs-async-proxy.js

@@ -220,22 +220,19 @@ class GetSyncHandleError extends Error {
   }
 };
 GetSyncHandleError.convertRc = (e,rc)=>{
-  if(1){
-    /* This approach returns SQLITE_LOCKED to the C API
-       when getSyncHandle() fails but makes the very
-       wild assumption that such a failure _is_ a locking
-       error. In practice that appears to be the most
-       common error, by far, but we cannot unambiguously
+  if(0){
+    /* This approach makes the very wild assumption that such a
+       failure _is_ a locking error. In practice that appears to be
+       the most common error, by far, but we cannot unambiguously
        distinguish that from other errors.
 
-       This approach demonstrably reduces concurrency-related
-       errors but is highly questionable.
+       This approach is highly questionable.
     */
     return (e instanceof GetSyncHandleError)
-      ? state.sq3Codes.SQLITE_LOCKED
+      ? state.sq3Codes.SQLITE_IOERR_LOCK
       : rc;
   }else{
-    return ec;
+    return rc;
   }
 }
 /**
@@ -253,7 +250,7 @@ const getSyncHandle = async (fh)=>{
   if(!fh.syncHandle){
     const t = performance.now();
     log("Acquiring sync handle for",fh.filenameAbs);
-    const maxTries = 4, msBase = 300;
+    const maxTries = 6, msBase = 300;
     let i = 1, ms = msBase;
     for(; true; ms = msBase * ++i){
       try {
@@ -271,7 +268,7 @@ const getSyncHandle = async (fh)=>{
         }
         warn("Error getting sync handle. Waiting",ms,
              "ms and trying again.",fh.filenameAbs,e);
-        //await closeAutoLocks();
+        await closeAutoLocks();
         Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms);
       }
     }

ext/wasm/tests/opfs/concurrency/index.html

@@ -18,7 +18,21 @@
     <h1></h1>
     <p>
       OPFS concurrency tester using multiple independent Workers.
-      This app is incomplete.
+      Disclaimer: concurrency in OPFS is currently a pain point
+      and timing/concurrency mitigation in this environment is
+      highly unpredictable!
+    </p>
+    <p>
+      URL flags: pass a number of workers using
+      the <code>workers=N</code> URL flag and the worker work interval
+      as <code>interval=N</code> (milliseconds).  Enable OPFS VFS
+      verbosity with <code>verbose=1-3</code> (output goes to the
+      dev console).
+    </p>
+    <p>Achtung: if it does not start to do anything within a couple of
+      seconds, check the dev console: Chrome often fails with "cannot allocate
+      WasmMemory" at startup. Closing and re-opening the tab usually resolves
+      it.
     </p>
     <div class='input-wrapper'>
       <input type='checkbox' id='cb-log-reverse'>

ext/wasm/tests/opfs/concurrency/test.js

@@ -57,6 +57,12 @@
   options.workerCount = (
     urlArgsHtml.has('workers') ? +urlArgsHtml.get('workers') : 3
   ) || 3;
+  options.opfsVerbose = (
+    urlArgsHtml.has('verbose') ? +urlArgsHtml.get('verbose') : 1
+  ) || 1;
+  options.interval = (
+    urlArgsHtml.has('interval') ? +urlArgsHtml.get('interval') : 750
+  ) || 750;
   const workers = [];
   workers.post = (type,...args)=>{
     for(const w of workers) w.postMessage({type, payload:args});
@@ -91,7 +97,8 @@
   workers.uri = (
     'worker.js?'
       + 'sqlite3.dir='+options.sqlite3Dir
-      + '&opfs-verbose=2'
+      + '&interval='+options.interval
+      + '&opfs-verbose='+options.opfsVerbose
   );
   for(let i = 0; i < options.workerCount; ++i){
     stdout("Launching worker...");

ext/wasm/tests/opfs/concurrency/worker.js

@@ -8,7 +8,6 @@ self.sqlite3InitModule().then(async function(sqlite3){
   };
   const stdout = (...args)=>wPost('stdout',...args);
   const stderr = (...args)=>wPost('stderr',...args);
-  const postErr = (...args)=>wPost('error',...args);
   if(!sqlite3.opfs){
     stderr("OPFS support not detected. Aborting.");
     return;
@@ -19,15 +18,33 @@ self.sqlite3InitModule().then(async function(sqlite3){
   };
 
   const dbName = 'concurrency-tester.db';
-  if((new URL(self.location.href).searchParams).has('unlink-db')){
+  const urlArgs = new URL(self.location.href).searchParams;
+  if(urlArgs.has('unlink-db')){
     await sqlite3.opfs.unlink(dbName);
     stdout("Unlinked",dbName);
   }
   wPost('loaded');
-
+  let db;
+  const interval = Object.assign(Object.create(null),{
+    delay: urlArgs.has('interval') ? (+urlArgs.get('interval') || 750) : 750,
+    handle: undefined,
+    count: 0
+  });
+  const finish = ()=>{
+    if(db){
+      if(!db.pointer) return;
+      db.close();
+    }
+    if(interval.error){
+      wPost('failed',"Ending work after interval #"+interval.count,
+            "due to error:",interval.error);
+    }else{
+      wPost('finished',"Ending work after",interval.count,"intervals.");
+    }
+  };
   const run = async function(){
-    const db = new sqlite3.opfs.OpfsDb(dbName,'c');
-    //sqlite3.capi.sqlite3_busy_timeout(db.pointer, 2000);
+    db = new sqlite3.opfs.OpfsDb(dbName,'c');
+    sqlite3.capi.sqlite3_busy_timeout(db.pointer, 5000);
     db.transaction((db)=>{
       db.exec([
         "create table if not exists t1(w TEXT UNIQUE ON CONFLICT REPLACE,v);",
@@ -36,11 +53,6 @@ self.sqlite3InitModule().then(async function(sqlite3){
     });
 
     const maxIterations = 10;
-    const interval = Object.assign(Object.create(null),{
-      delay: 500,
-      handle: undefined,
-      count: 0
-    });
     stdout("Starting interval-based db updates with delay of",interval.delay,"ms.");
     const doWork = async ()=>{
       const tm = new Date().getTime();
@@ -57,15 +69,6 @@ self.sqlite3InitModule().then(async function(sqlite3){
         interval.error = e;
       }
     };
-    const finish = ()=>{
-      db.close();
-      if(interval.error){
-        wPost('failed',"Ending work after interval #"+interval.count,
-              "due to error:",interval.error);
-      }else{
-        wPost('finished',"Ending work after",interval.count,"intervals.");
-      }
-    };
     if(1){/*use setInterval()*/
       interval.handle = setInterval(async ()=>{
         await doWork();
@@ -89,7 +92,10 @@ self.sqlite3InitModule().then(async function(sqlite3){
 
   self.onmessage = function({data}){
     switch(data.type){
-        case 'run': run().catch((e)=>postErr(e.message));
+        case 'run': run().catch((e)=>{
+          if(!interval.error) interval.error = e;
+          finish();
+        });
           break;
         default:
           stderr("Unhandled message type '"+data.type+"'.");

manifest

@@ -1,5 +1,5 @@
-C Resolve\smissing\sSQLITE_LOCKED\sresult\scode\swhich\striggered\sa\snew\s(since\slast\scheckin)\sexception\sin\sthe\sOPFS\sVFS.\sImprove\soutput\sof\sthe\sOPFS\scontention\stester\sapp.
-D 2022-11-21T04:12:38.735
+C More\stweaking\sof\sOPFS\sconcurrency\smeasures\sand\sthe\srelated\stest\sapp.
+D 2022-11-21T05:18:24.071
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -506,7 +506,7 @@ F ext/wasm/api/sqlite3-api-opfs.js 38d368e33f470f9ba196f1a2b0c9ce076c930c70df233
 F ext/wasm/api/sqlite3-api-prologue.js 08e96d26d329e8c1e08813fe0b84ee93e0e78b087efdd6eb2809ae2672902437
 F ext/wasm/api/sqlite3-api-worker1.js e94ba98e44afccfa482874cd9acb325883ade50ed1f9f9526beb9de1711f182f
 F ext/wasm/api/sqlite3-license-version-header.js a661182fc93fc2cf212dfd0b987f8e138a3ac98f850b1112e29b5fbdaecc87c3
-F ext/wasm/api/sqlite3-opfs-async-proxy.js 021af8b3d1754e308c09eebee5f8d235fb245bea1f9b1c1414141cc2ebd5649c
+F ext/wasm/api/sqlite3-opfs-async-proxy.js 1ec10873f1d59d305f6f3b435c50a1b75d693d5fb739b226f3da46fcbb11261a
 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
 F ext/wasm/api/sqlite3-wasm.c 8fc8f47680df0e9a6c0f2f03cb004148645ecc983aa216daba09cb21f7e092a2
 F ext/wasm/api/sqlite3-worker1-promiser.js 0c7a9826dbf82a5ed4e4f7bf7816e825a52aff253afbf3350431f5773faf0e4b
@@ -552,9 +552,9 @@ F ext/wasm/test-opfs-vfs.js 44363db07b2a20e73b0eb1808de4400ca71b703af718d0fa6d96
 F ext/wasm/tester1-worker.html 5ef353348c37cf2e4fd0b23da562d3275523e036260b510734e9a3239ba8c987
 F ext/wasm/tester1.c-pp.html 74aa9b31c75f12490653f814b53c3dd39f40cd3f70d6a53a716f4e8587107399
 F ext/wasm/tester1.c-pp.js 0c129495d057c77788b59715152d51f9bf9002ebbcce759ef8b028272ce3519d
-F ext/wasm/tests/opfs/concurrency/index.html c7cf329e5b206dd8226d94ab9fec02f5f350d8ed69a57c96d84e876afd3d3d1b
-F ext/wasm/tests/opfs/concurrency/test.js 6f7d49d97c27906f8f5a39d6da176122350f375cd811e51182128bc2db74e464
-F ext/wasm/tests/opfs/concurrency/worker.js 571bcc525520fedfc71e25486265247633a4bf8aee554923aa7219399ed749fd
+F ext/wasm/tests/opfs/concurrency/index.html bb9b0f6da86df34c67fa506db9c45b7c4cf0045a211611cc6b8d2b53fa983481
+F ext/wasm/tests/opfs/concurrency/test.js cea04456ffae7bbfd726ef0e859d2afc2a51ead66940fc4a2499884ac44d16dd
+F ext/wasm/tests/opfs/concurrency/worker.js a92fa4da1431d3c6db3e1dd0b98794e09d1021e23a8c490335410cce283b11c1
 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5
 F ext/wasm/wasmfs.make 8fea9b4f3cde06141de1fc4c586ab405bd32c3f401554f4ebb18c797401a678d
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
@@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 96f76e7616f8157a342b9e1c42f7b1feab200d182268871a2b25f67d4ee2564c
-R 37b708b698da9468667dd15ac1e5f125
+P 2debbbca33bd4170a1dc4dbb5eb3e68523e51d289b06c551e5560ac4e32e433b
+R cdfdba12cf28ae07442f6b59e52570f8
 U stephan
-Z 415bce67c5fc2cb5709db14ccdfe0252
+Z bce379ce157581c096a0eabd8577e81e
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-2debbbca33bd4170a1dc4dbb5eb3e68523e51d289b06c551e5560ac4e32e433b
+a8d4da1501d411085ec2fd48c4a056c8b1d97ef3c3203c5b403a854ac2864870