Initial sketches for an alternate OPFS VFS which uses a pool of pre-opened SyncAccessHandles to bypass the need for a dedicated I/O worker and the COOP/COEP HTTP response headers. Currently completely non-functional.

FossilOrigin-Name: a93de9f2a553a3a4edd1b361dd6f465a1b0b5b51f7bb8ede432067aedcfefda4

ext/wasm/api/sqlite3-vfs-opfs-sahpool.js

@@ -0,0 +1,310 @@
+/*
+  2023-07-14
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  INCOMPLETE! WORK IN PROGRESS!
+
+  This file holds an experimental sqlite3_vfs backed by OPFS storage
+  which uses a different implementation strategy than the "opfs"
+  VFS. This one is a port of Roy Hashimoto's OPFS SyncAccessHandle
+  pool:
+
+  https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/AccessHandlePoolVFS.js
+
+  As described at:
+
+  https://github.com/rhashimoto/wa-sqlite/discussions/67
+
+  with Roy's explicit permission to permit us to port his to our
+  infrastructure rather than having to clean-room reverse-engineer it:
+
+  https://sqlite.org/forum/forumpost/e140d84e71
+
+  Primary differences from the original "opfs" VFS include:
+
+  - This one avoids the need for a sub-worker to synchronize
+  communication between the synchronous C API and the only-partly
+  synchronous OPFS API.
+
+  - It does so by opening a fixed number of OPFS files at
+  library-level initialization time, obtaining SyncAccessHandles to
+  each, and manipulating those handles via the synchronous sqlite3_vfs
+  interface.
+
+  - Because of that, this one lacks all library-level concurrency
+  support.
+
+  - Also because of that, it does not require the SharedArrayBuffer,
+  so can function without the COOP/COEP HTTP response headers.
+
+  - It can hypothetically support Safari 16.4+, whereas the "opfs"
+  VFS requires v17 due to a bug in 16.x which makes it incompatible
+  with that VFS.
+
+  - This VFS requires the "semi-fully-sync" FileSystemSyncAccessHandle
+  (hereafter "SAH") APIs released with Chrome v108. There is
+  unfortunately no known programmatic way to determine whether a given
+  API is from that release or newer without actually calling it and
+  checking whether one of the "fully-sync" functions returns a Promise
+  (in which case it's the older version). (Reminder to self: when
+  opening up the initial pool of files, we can close() the first one
+  we open and see if close() returns a Promise. If it does, it's the
+  older version so fail VFS initialization. If it doesn't, re-open it.)
+
+*/
+'use strict';
+globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
+const installOpfsVfs = async function(sqlite3){
+  const pToss = (...args)=>Promise.reject(new Error(args.join(' ')));
+  if(!globalThis.FileSystemHandle ||
+     !globalThis.FileSystemDirectoryHandle ||
+     !globalThis.FileSystemFileHandle ||
+     !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle ||
+     !navigator?.storage?.getDirectory){
+    return pToss("Missing required OPFS APIs.");
+  }
+  const thePromise = new Promise(function(promiseResolve, promiseReject_){
+    const verbosity = 3;
+    const loggers = [
+      sqlite3.config.error,
+      sqlite3.config.warn,
+      sqlite3.config.log
+    ];
+    const logImpl = (level,...args)=>{
+      if(verbosity>level) loggers[level]("OPFS syncer:",...args);
+    };
+    const log =    (...args)=>logImpl(2, ...args);
+    const warn =   (...args)=>logImpl(1, ...args);
+    const error =  (...args)=>logImpl(0, ...args);
+    const toss = sqlite3.util.toss;
+    const capi = sqlite3.capi;
+    const wasm = sqlite3.wasm;
+    const opfsIoMethods = new capi.sqlite3_io_methods();
+    const opfsVfs = new capi.sqlite3_vfs()
+          .addOnDispose(()=>opfsIoMethods.dispose());
+    const promiseReject = (err)=>{
+      opfsVfs.dispose();
+      return promiseReject_(err);
+    };
+
+    // Config opts for the VFS...
+    const SECTOR_SIZE = 4096;
+    const HEADER_MAX_PATH_SIZE = 512;
+    const HEADER_FLAGS_SIZE = 4;
+    const HEADER_DIGEST_SIZE = 8;
+    const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE;
+    const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE;
+    const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE;
+    const HEADER_OFFSET_DATA = SECTOR_SIZE;
+    const DEFAULT_CAPACITY = 6;
+    /* Bitmask of file types which may persist across sessions.
+       SQLITE_OPEN_xyz types not listed here may be inadvertently
+       left in OPFS but are treated as transient by this VFS and
+       they will be cleaned up during VFS init. */
+    const PERSISTENT_FILE_TYPES =
+          capi.SQLITE_OPEN_MAIN_DB |
+          capi.SQLITE_OPEN_MAIN_JOURNAL |
+          capi.SQLITE_OPEN_SUPER_JOURNAL |
+          capi.SQLITE_OPEN_WAL /* noting that WAL support is
+                                  unavailable in the WASM build.*/;
+    const pDVfs = capi.sqlite3_vfs_find(null)/*default VFS*/;
+    const dVfs = pDVfs
+          ? new sqlite3_vfs(pDVfs)
+          : null /* dVfs will be null when sqlite3 is built with
+                    SQLITE_OS_OTHER. */;
+    opfsVfs.$iVersion = 2/*yes, two*/;
+    opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof;
+    opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE;
+    opfsVfs.$zName = wasm.allocCString("opfs-sahpool");
+    opfsVfs.addOnDispose(
+      '$zName', opfsVfs.$zName,
+      'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null)
+    );
+
+    const VState = Object.assign(Object.create(null),{
+      /* OPFS dir in which VFS metadata is stored. */
+      vfsDir: ".sqlite3-sahpool",
+      dirHandle: undefined,
+      /* Maps OPFS access handles to their opaque file names. */
+      mapAH2Name: new Map(),
+      mapPath2AH: new Map(),
+      availableAH: new Set(),
+      mapId2File: new Map(),
+      getCapacity: function(){return this.mapAH2Name.size},
+      getFileCount: function(){return this.mapPath2AH.size},
+      addCapacity: async function(n){
+        for(let i = 0; i < n; ++i){
+          const name = Math.random().toString(36).replace('0.','');
+          const h = await this.dirHandle.getFileName(name, {create:true});
+          const ah = await h.createSyncAccessHandle();
+          this.mapAH2Name(ah,name);
+          this.setAssociatedPath(ah, '', 0);
+        }
+      },
+      setAssociatedPath: function(accessHandle, path, flags){
+        // TODO
+      },
+      releaseAccessHandles: function(){
+        for(const ah of this.mapAH2Name.keys()) ah.close();
+        this.mapAH2Name.clear();
+        this.mapPath2AH.clear();
+        this.availableAH.clear();
+      },
+      acquireAccessHandles: async function(){
+        // TODO
+      },
+      reset: async function(){
+        await this.isReady;
+        let h = await navigator.storage.getDirectory();
+        for(const d of this.vfsDir.split('/')){
+          if(d){
+            h = await h.getDirectoryHandle(d,{create:true});
+          }
+        }
+        this.dirHandle = h;
+        this.releaseAccessHandles();
+        await this.acquireAccessHandles();
+      }
+      // much more TODO
+    })/*VState*/;
+
+    // Much, much more TODO...
+    /**
+       Impls for the sqlite3_io_methods methods. Maintenance reminder:
+       members are in alphabetical order to simplify finding them.
+    */
+    const ioSyncWrappers = {
+      xCheckReservedLock: function(pFile,pOut){
+        return 0;
+      },
+      xClose: function(pFile){
+        let rc = 0;
+        return rc;
+      },
+      xDeviceCharacteristics: function(pFile){
+        return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
+      },
+      xFileControl: function(pFile, opId, pArg){
+        return capi.SQLITE_NOTFOUND;
+      },
+      xFileSize: function(pFile,pSz64){
+        let rc = 0;
+        return rc;
+      },
+      xLock: function(pFile,lockType){
+        let rc = capi.SQLITE_IOERR_LOCK;
+        return rc;
+      },
+      xRead: function(pFile,pDest,n,offset64){
+        let rc = capi.SQLITE_IOERR_READ;
+        return rc;
+      },
+      xSync: function(pFile,flags){
+        let rc = capi.SQLITE_IOERR_FSYNC;
+        return rc;
+      },
+      xTruncate: function(pFile,sz64){
+        let rc = capi.SQLITE_IOERR_TRUNCATE;
+        return rc;
+      },
+      xUnlock: function(pFile,lockType){
+        let rc = capi.SQLITE_IOERR_UNLOCK;
+        return rc;
+      },
+      xWrite: function(pFile,pSrc,n,offset64){
+        let rc = capi.SQLITE_IOERR_WRITE;
+        return rc;
+      }
+    }/*ioSyncWrappers*/;
+
+    /**
+       Impls for the sqlite3_vfs methods. Maintenance reminder: members
+       are in alphabetical order to simplify finding them.
+    */
+    const vfsSyncWrappers = {
+      xAccess: function(pVfs,zName,flags,pOut){
+        const rc = capi.SQLITE_ERROR;
+        return rc;
+      },
+      xCurrentTime: function(pVfs,pOut){
+        wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000),
+                  'double');
+        return 0;
+      },
+      xCurrentTimeInt64: function(pVfs,pOut){
+        wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(),
+                  'i64');
+        return 0;
+      },
+      xDelete: function(pVfs, zName, doSyncDir){
+        const rc = capi.SQLITE_ERROR;
+        return rc;
+      },
+      xFullPathname: function(pVfs,zName,nOut,pOut){
+        const i = wasm.cstrncpy(pOut, zName, nOut);
+        return i<nOut ? 0 : capi.SQLITE_CANTOPEN;
+      },
+      xGetLastError: function(pVfs,nOut,pOut){
+        /* TODO: store exception state somewhere and serve
+           it from here. */
+        warn("OPFS xGetLastError() has nothing sensible to return.");
+        return 0;
+      },
+      //xSleep is optionally defined below
+      xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
+        let rc = capi.SQLITE_ERROR;
+        return rc;
+      }/*xOpen()*/
+    }/*vfsSyncWrappers*/;
+
+    if(dVfs){
+      /* Inherit certain VFS members from the default VFS,
+         if available. */
+      opfsVfs.$xRandomness = dVfs.$xRandomness;
+      opfsVfs.$xSleep = dVfs.$xSleep;
+    }
+    if(!opfsVfs.$xRandomness){
+      /* If the default VFS has no xRandomness(), add a basic JS impl... */
+      vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){
+        const heap = wasm.heap8u();
+        let i = 0;
+        for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF;
+        return i;
+      };
+    }
+
+    try{
+      sqlite3.vfs.installVfs({
+        io: {struct: opfsIoMethods, methods: ioSyncWrappers},
+        vfs: {struct: opfsVfs, methods: vfsSyncWrappers}
+      });
+    }catch(e){
+      promiseReject(e);
+      return;
+    }
+
+    VState.isReady = VState.reset().then(async ()=>{
+      if(0===VState.getCapacity())[
+        await VState.addCapacity(DEFAULT_CAPACITY);
+      }
+      promiseResolve(sqlite3);
+    }).catch(promiseReject);
+  })/*thePromise*/;
+  return thePromise;
+}/*installOpfsVfs()*/;
+
+globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{
+  return installOpfsVfs(sqlite3).catch((e)=>{
+    sqlite3.config.warn("Ignoring inability to install opfs-sahpool sqlite3_vfs:",
+                        e.message);
+  });
+}/*sqlite3ApiBootstrap.initializersAsync*/);
+}/*sqlite3ApiBootstrap.initializers*/);

manifest

@@ -1,5 +1,5 @@
-C Minor\sinternal\scleanups\sin\sthe\sOPFS\sVFS.
+C Initial\ssketches\sfor\san\salternate\sOPFS\sVFS\swhich\suses\sa\spool\sof\spre-opened\sSyncAccessHandles\sto\sbypass\sthe\sneed\sfor\sa\sdedicated\sI/O\sworker\sand\sthe\sCOOP/COEP\sHTTP\sresponse\sheaders.\sCurrently\scompletely\snon-functional.
-D 2023-07-14T21:06:00.870
+D 2023-07-14T21:17:29.056
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -502,6 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386
 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2
 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487
+F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js d0bc04c29983e967e37a91ca4e849beae6db6c883a6612da9717ca22a24119d1
 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87
 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda
 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
@@ -2043,8 +2044,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 816b503f093c4e6d92d0eb2f9fbd841acd01cc9bc89ee58d961b56c64f71406a
+P 984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e
-R 6597235c929ad6ed0b90d6716177ff1b
+R 81377ab74a003e9bec4077c3a186be8e
+T *branch * opfs-sahpool
+T *sym-opfs-sahpool *
+T -sym-trunk * Cancelled\sby\sbranch.
 U stephan
-Z b7b5463364c5290e11704a96855173c7
+Z bc905b63d521aded8c5f64b8ee091932
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e
+a93de9f2a553a3a4edd1b361dd6f465a1b0b5b51f7bb8ede432067aedcfefda4