WASM API renaming. Reworked JS API bootstrap's async post-init into a generic mechanism, no longer OPFS-specific.

FossilOrigin-Name: c42a8cb090cad1108dfd6be574202d744c59e053b505bc4c17252dc6b65d26bf

ext/wasm/api/README.md

@@ -66,24 +66,33 @@ browser client:
   thread via the Worker message-passing interface. Like OO API #1,
   this is an optional component, offering one of any number of
   potential implementations for such an API.
-    - `sqlite3-worker1.js`\  
+    - `../sqlite3-worker1.js`\  
       Is not part of the amalgamated sources and is intended to be
       loaded by a client Worker thread. It loads the sqlite3 module
       and runs the Worker #1 API which is implemented in
       `sqlite3-api-worker1.js`.
+    - `../sqlite3-worker1-promiser.js`\  
+      Is likewise not part of the amalgamated sources and provides
+      a Promise-based interface into the Worker #1 API. This is
+      a far user-friendlier way to interface with databases running
+      in a Worker thread.
 - `sqlite3-api-opfs.js`\  
-  is an in-development/experimental sqlite3 VFS wrapper, the goal of
-  which being to use Google Chrome's Origin-Private FileSystem (OPFS)
-  storage layer to provide persistent storage for database files in a
-  browser. It is far from complete.
+  is an sqlite3 VFS implementation which supports Google Chrome's
+  Origin-Private FileSystem (OPFS) as a storage layer to provide
+  persistent storage for database files in a browser. It requires...
+    - `../sqlite3-opfs-async-proxy.js`\  
+      is the asynchronous backend part of the OPFS proxy. It speaks
+      directly to the (async) OPFS API and channels those results back
+      to its synchronous counterpart. This file, because it must be
+      started in its own Worker, is not part of the amalgamation.
 - `sqlite3-api-cleanup.js`\  
   the previous files temporarily create global objects in order to
   communicate their state to the files which follow them, and _this_
   file connects any final components together and cleans up those
   globals. As of this writing, this code ensures that the previous
-  files leave no global symbols installed, and it moves the sqlite3
-  namespace object into the in-scope Emscripten module. Abstracting
-  this for other WASM toolchains is TODO.
+  files leave no more than a single global symbol installed. When
+  adapting the API for non-Emscripten toolchains, this "should"
+  be the only file where changes are needed.
 - `post-js-footer.js`\  
   Emscripten-specific footer for the `--post-js` input. This closes
   off the lexical scope opened by `post-js-header.js`.

ext/wasm/api/sqlite3-api-opfs.js

@@ -20,7 +20,7 @@
 'use strict';
 self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
 /**
-   sqlite3.installOpfsVfs() returns a Promise which, on success, installs
+   installOpfsVfs() returns a Promise which, on success, installs
    an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs
    which accept a VFS. It uses the Origin-Private FileSystem API for
    all file storage. On error it is rejected with an exception
@@ -32,7 +32,6 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
    - The environment does not support OPFS. That includes when
      this function is called from the main window thread.
 
-
   Significant notes and limitations:
 
   - As of this writing, OPFS is still very much in flux and only
@@ -73,8 +72,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
    object and that object gets a new object installed in its
    `opfs` property, containing several OPFS-specific utilities.
 */
-sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
-  delete sqlite3.installOpfsVfs;
+const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
   if(!self.SharedArrayBuffer ||
      !self.FileSystemHandle ||
      !self.FileSystemDirectoryHandle ||
@@ -1027,5 +1025,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
   })/*thePromise*/;
   return thePromise;
 }/*installOpfsVfs()*/;
-sqlite3.installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js";
+installOpfsVfs.defaultProxyUri =
+    //self.location.pathname.replace(/[^/]*$/, "sqlite3-opfs-async-proxy.js");
+  "sqlite3-opfs-async-proxy.js";
+//console.warn("sqlite3.installOpfsVfs.defaultProxyUri =",sqlite3.installOpfsVfs.defaultProxyUri);
+self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>installOpfsVfs());
 }/*sqlite3ApiBootstrap.initializers.push()*/);

ext/wasm/api/sqlite3-api-prologue.js

@@ -698,8 +698,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
     ["sqlite3_wasm_vfs_unlink", "int", "string"]
   ];
 
-  /** State for sqlite3_web_persistent_dir(). */
-  let __persistentDir;
+  /** State for sqlite3_wasmfs_opfs_dir(). */
+  let __persistentDir = undefined;
   /**
      An experiment. Do not use in client code.
 
@@ -713,8 +713,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
 
      This function currently only recognizes the WASMFS/OPFS storage
      combination. "Plain" OPFS is provided via a separate VFS which
-     can optionally be installed (if OPFS is available on the system)
-     using sqlite3.installOpfsVfs().
+     is optionally be installed via sqlite3.asyncPostInit().
 
      TODOs and caveats:
 
@@ -724,7 +723,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
        happen when using the JS-native "opfs" VFS, as opposed to the
        WASMFS/OPFS combination.
   */
-  capi.sqlite3_web_persistent_dir = function(){
+  capi.sqlite3_wasmfs_opfs_dir = function(){
     if(undefined !== __persistentDir) return __persistentDir;
     // If we have no OPFS, there is no persistent dir
     const pdir = config.wasmfsOpfsDir;
@@ -738,17 +737,6 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
       if(pdir && 0===capi.wasm.xCallWrapped(
         'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir
       )){
-        /** OPFS does not support locking and will trigger errors if
-            we try to lock. We don't _really_ want to
-            _unconditionally_ install a non-locking sqlite3 VFS as the
-            default, but we do so here for simplicy's sake for the
-            time being. That said: locking is a no-op on all of the
-            current WASM storage, so this isn't (currently) as bad as
-            it may initially seem. */
-        const pVfs = sqlite3.capi.sqlite3_vfs_find("unix-none");
-        if(pVfs){
-          capi.sqlite3_vfs_register(pVfs,1);
-        }
         return __persistentDir = pdir;
       }else{
         return __persistentDir = "";
@@ -762,7 +750,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
   /**
      Experimental and subject to change or removal.
 
-     Returns true if sqlite3.capi.sqlite3_web_persistent_dir() is a
+     Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a
      non-empty string and the given name starts with (that string +
      '/'), else returns false.
 
@@ -771,7 +759,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
      kvvfs is available.
   */
   capi.sqlite3_web_filename_is_persistent = function(name){
-    const p = capi.sqlite3_web_persistent_dir();
+    const p = capi.sqlite3_wasmfs_opfs_dir();
     return (p && name) ? name.startsWith(p+'/') : false;
   };
 
@@ -922,11 +910,42 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
 
   }/* main-window-only bits */
 
+
   /* The remainder of the API will be set up in later steps. */
   const sqlite3 = {
     WasmAllocError: WasmAllocError,
     capi,
-    config
+    config,
+    /**
+       Performs any optional asynchronous library-level initialization
+       which might be required. This function returns a Promise which
+       resolves to the sqlite3 namespace object. It _ignores any
+       errors_ in the asynchronous init process, as such components
+       are all optional. If called more than once, the second and
+       subsequent calls are no-ops which return a pre-resolved
+       Promise.
+
+       If called at all, this function must be called by client-level
+       code, which must not use the library until the returned promise
+       resolves.
+
+       Bug: if called while a prior call is still resolving, the 2nd
+       call will resolve prematurely, before the 1st call has finished
+       resolving.
+    */
+    asyncPostInit: async function(){
+      let lip = sqlite3ApiBootstrap.initializersAsync;
+      delete sqlite3ApiBootstrap.initializersAsync;
+      if(!lip || !lip.length) return Promise.resolve(sqlite3);
+      // Is it okay to resolve these in parallel or do we need them
+      // to resolve in order? We currently only have 1, so it
+      // makes no difference.
+      lip = lip.map((f)=>f(sqlite3).catch(()=>{}));
+      //let p = lip.shift();
+      //while(lip.length) p = p.then(lip.shift());
+      //return p.then(()=>sqlite3);
+      return Promise.all(lip).then(()=>sqlite3);
+    }
   };
   sqlite3ApiBootstrap.initializers.forEach((f)=>f(sqlite3));
   delete sqlite3ApiBootstrap.initializers;
@@ -946,8 +965,29 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
   their features (noting that most will also require that certain
   features alread have been installed).  At the end of that process,
   this array is deleted.
+
+  Note that the order of insertion into this array is significant for
+  some pieces. e.g. sqlite3.capi.wasm cannot be fully utilized until
+  the whwasmutil.js part is plugged in.
 */
 self.sqlite3ApiBootstrap.initializers = [];
+/**
+  self.sqlite3ApiBootstrap.initializersAsync is an internal detail
+  used by the sqlite3 API's amalgamation process. It must not be
+  modified by client code except when plugging such code into the
+  amalgamation process.
+
+  Counterpart of self.sqlite3ApiBootstrap.initializers, specifically
+  for initializers which are asynchronous. All functions in this list
+  take the sqlite3 object as their argument and MUST return a
+  Promise. Both the resolved value and rejection cases are ignored.
+
+  This list is not processed until the client calls
+  sqlite3.asyncPostInit(). This means, for example, that intializers
+  added to self.sqlite3ApiBootstrap.initializers may push entries to
+   this list.
+*/
+self.sqlite3ApiBootstrap.initializersAsync = [];
 /**
    Client code may assign sqlite3ApiBootstrap.defaultConfig an
    object-type value before calling sqlite3ApiBootstrap() (without

ext/wasm/api/sqlite3-api-worker1.js

@@ -154,12 +154,12 @@
     messageId: ...as above...,
     result: {
 
-      persistentDirName: path prefix, if any, of persistent storage.
+      wasmfsOpfsDir: path prefix, if any, of persistent storage.
       An empty string denotes that no persistent storage is available.
 
       bigIntEnabled: bool. True if BigInt support is enabled.
 
-      persistenceEnabled: true if persistent storage is enabled in the
+      wasmfsOpfsEnabled: true if persistent storage is enabled in the
       current environment. Only files stored under persistentDirName
       will persist, however.
 
@@ -183,7 +183,7 @@
       See the sqlite3.oo1.DB constructor for peculiarities and transformations,
 
       persistent [=false]: if true and filename is not one of ("",
-      ":memory:"), prepend sqlite3.capi.sqlite3_web_persistent_dir()
+      ":memory:"), prepend sqlite3.capi.sqlite3_wasmfs_opfs_dir()
       to the given filename so that it is stored in persistent storage
       _if_ the environment supports it.  If persistent storage is not
       supported, the filename is used as-is.
@@ -438,7 +438,7 @@ sqlite3.initWorker1API = function(){
         toss("Throwing because of simulateError flag.");
       }
       const rc = Object.create(null);
-      const pDir = sqlite3.capi.sqlite3_web_persistent_dir();
+      const pDir = sqlite3.capi.sqlite3_wasmfs_opfs_dir();
       if(!args.filename || ':memory:'===args.filename){
         oargs.filename = args.filename || '';
       }else if(pDir){
@@ -521,11 +521,11 @@ sqlite3.initWorker1API = function(){
     'config-get': function(){
       const rc = Object.create(null), src = sqlite3.config;
       [
-        'persistentDirName', 'bigIntEnabled'
+        'wasmfsOpfsDir', 'bigIntEnabled'
       ].forEach(function(k){
         if(Object.getOwnPropertyDescriptor(src, k)) rc[k] = src[k];
       });
-      rc.persistenceEnabled = !!sqlite3.capi.sqlite3_web_persistent_dir();
+      rc.wasmfsOpfsEnabled = !!sqlite3.capi.sqlite3_wasmfs_opfs_dir();
       return rc;
     },
 

ext/wasm/api/sqlite3-wasm.c

@@ -543,9 +543,10 @@ int sqlite3_wasm_vfs_unlink(const char * zName){
   return rc;
 }
 
-#if defined(__EMSCRIPTEN__) && defined(SQLITE_WASM_WASMFS)
-#include <emscripten/wasmfs.h>
+#if defined(__EMSCRIPTEN__)
 #include <emscripten/console.h>
+#if defined(SQLITE_WASM_WASMFS)
+#include <emscripten/wasmfs.h>
 
 /*
 ** This function is NOT part of the sqlite3 public API. It is strictly
@@ -596,10 +597,11 @@ int sqlite3_wasm_init_wasmfs(const char *zMountPoint){
 #else
 WASM_KEEP
 int sqlite3_wasm_init_wasmfs(const char *zUnused){
+  emscripten_console_warn("WASMFS OPFS is not compiled in.");
   if(zUnused){/*unused*/}
   return SQLITE_NOTFOUND;
 }
 #endif /* __EMSCRIPTEN__ && SQLITE_WASM_WASMFS */
-
+#endif
 
 #undef WASM_KEEP

ext/wasm/batch-runner.js

@@ -361,7 +361,7 @@
         dbFile = 1 ? 'local' : 'session';
         this.logHtml("Using KVVFS storage:",dbFile);
       }else{
-        pDir = capi.sqlite3_web_persistent_dir();
+        pDir = capi.sqlite3_wasmfs_opfs_dir();
         if(pDir){
           dbFile = pDir+"/speedtest.db";
           this.logHtml("Using persistent storage:",dbFile);

ext/wasm/scratchpad-wasmfs-main.js

@@ -42,7 +42,7 @@
           oo = sqlite3.oo1,
           wasm = capi.wasm;
     stdout("Loaded sqlite3:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
-    const persistentDir = capi.sqlite3_web_persistent_dir();
+    const persistentDir = capi.sqlite3_wasmfs_opfs_dir();
     if(persistentDir){
       stdout("Persistent storage dir:",persistentDir);
     }else{

ext/wasm/speedtest1-worker.js

@@ -81,11 +81,10 @@
     setStatus: (text)=>mPost('load-status',text)
   };
   self.sqlite3Speedtest1InitModule(EmscriptenModule).then(function(EModule){
-    const S = EModule.sqlite3;
     log("Module inited.");
-    return S.installOpfsVfs()
-      .catch((e)=>console.warn(e.message))
-      .then(()=>{
+    return EModule.sqlite3.asyncPostInit()
+      .then((sqlite3)=>{
+        const S = sqlite3;
         const vfsUnlink = S.capi.wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["string"]);
         App.unlink = function(fname){
           vfsUnlink(fname);

ext/wasm/sqlite3-worker1-promiser.js

@@ -248,7 +248,10 @@ self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){
   };
 }/*sqlite3Worker1Promiser()*/;
 self.sqlite3Worker1Promiser.defaultConfig = {
-  worker: ()=>new Worker('sqlite3-worker1.js'),
+  worker: ()=>{
+    //const p = self.location.pathname.replace(/[^/]*$/, "sqlite3-worker1.js");
+    return new Worker("sqlite3-worker1.js");
+  },
   onerror: (...args)=>console.error('worker1 error',...args),
   dbId: undefined
 };

ext/wasm/sqlite3-worker1.js

@@ -28,4 +28,9 @@
 */
 "use strict";
 importScripts('sqlite3.js');
-sqlite3InitModule().then((EmscriptenModule)=>EmscriptenModule.sqlite3.initWorker1API());
+sqlite3InitModule().then((EmscriptenModule)=>{
+  EmscriptenModule.sqlite3.asyncPostInit().then((sqlite3)=>{
+    sqlite3.capi.sqlite3_wasmfs_opfs_dir();
+    sqlite3.initWorker1API();
+  });
+});

ext/wasm/test-opfs-vfs.js

@@ -30,7 +30,7 @@ const tryOpfsVfs = function(sqlite3){
   const error =  (...args)=>console.error(logPrefix,...args);
   log("tryOpfsVfs()");
   const capi = sqlite3.capi;
-  const pVfs = capi.sqlite3_vfs_find("opfs") || toss("Unexpectedly missing 'opfs' VFS.");
+  const pVfs = capi.sqlite3_vfs_find("opfs") || toss("Missing 'opfs' VFS.");
   const oVfs = capi.sqlite3_vfs.instanceForPointer(pVfs) || toss("Unexpected instanceForPointer() result.");;
   log("OPFS VFS:",pVfs, oVfs);
 
@@ -78,7 +78,7 @@ const tryOpfsVfs = function(sqlite3){
 
 importScripts('sqlite3.js');
 self.sqlite3InitModule()
-  .then((EmscriptenModule)=>EmscriptenModule.sqlite3.installOpfsVfs())
+  .then((EmscriptenModule)=>EmscriptenModule.sqlite3.asyncPostInit())
   .then((sqlite3)=>tryOpfsVfs(sqlite3))
   .catch((e)=>{
     console.error("Error initializing module:",e);

ext/wasm/testing-worker1-promiser.js

@@ -77,8 +77,8 @@
       const r = ev.result;
       log('sqlite3.config subset:', r);
       T.assert('boolean' === typeof r.bigIntEnabled)
-        .assert('string'===typeof r.persistentDirName)
-        .assert('boolean' === typeof r.persistenceEnabled);
+        .assert('string'===typeof r.wasmfsOpfsDir)
+        .assert('boolean' === typeof r.wasmfsOpfsEnabled);
       sqConfig = r;
     });
     logHtml('',
@@ -86,12 +86,12 @@
     
     await wtest('open', {
       filename: dbFilename,
-      persistent: sqConfig.persistenceEnabled,
+      persistent: sqConfig.wasmfsOpfsEnabled,
       simulateError: 0 /* if true, fail the 'open' */,
     }, function(ev){
       const r = ev.result;
       log("then open result",r);
-      T.assert(r.persistent === sqConfig.persistenceEnabled)
+      T.assert(r.persistent === sqConfig.wasmfsOpfsEnabled)
         .assert(r.persistent
                ? (dbFilename!==r.filename)
                : (dbFilename==r.filename))

ext/wasm/testing1.js

@@ -1019,7 +1019,7 @@
           wasm = capi.wasm;
     log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
     log("Build options:",wasm.compileOptionUsed());
-    capi.sqlite3_web_persistent_dir()/*will install OPFS if available, plus a and non-locking VFS*/;
+    capi.sqlite3_wasmfs_opfs_dir()/*will install OPFS if available, plus a and non-locking VFS*/;
     if(1){
       /* Let's grab those last few lines of test coverage for
          sqlite3-api.js... */

ext/wasm/testing2.js

@@ -11,6 +11,10 @@
   ***********************************************************************
 
   A basic test script for sqlite3-worker1.js.
+
+  Note that the wrapper interface demonstrated in
+  testing-worker1-promiser.js is much easier to use from client code, as it
+  lacks the message-passing acrobatics demonstrated in this file.
 */
 'use strict';
 (function(){