Redefine what the opfs-sahpool installation promise resolves to. Fix addCapacity(). Add utility methods to import/export files.

FossilOrigin-Name: 809c6f4de3653ad7a7751af45a7a0d6cb20c3ee3be80c69833c729242227d970

ext/wasm/api/sqlite3-vfs-opfs-sahpool.js

@@ -56,10 +56,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
 const toss = sqlite3.util.toss;
 let vfsRegisterResult = undefined;
 /**
-   installOpfsSAHPoolVfs() asynchronously initializes the
-   OPFS SyncAccessHandle Pool VFS. It returns a Promise
-   which either resolves to the sqlite3 object or rejects
-   with an Error value.
+   installOpfsSAHPoolVfs() asynchronously initializes the OPFS
+   SyncAccessHandle Pool VFS. It returns a Promise which either
+   resolves to a utility object described below or rejects with an
+   Error value.
 
    Initialization of this VFS is not automatic because its
    registration requires that it lock all resources it
@@ -75,6 +75,12 @@ let vfsRegisterResult = undefined;
    resolved or rejected Promise. If called while the first call is
    still pending resolution, a rejected promise with a descriptive
    error is returned.
+
+   On success, the resulting Promise resolves to a utility object
+   which can be used to query and manipulate the pool. Its API is...
+
+   TODO
+
 */
 sqlite3.installOpfsSAHPoolVfs = async function(){
   if(sqlite3===vfsRegisterResult) return Promise.resolve(sqlite3);
@@ -112,8 +118,11 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
     vfsRegisterResult = err;
     return Promise.reject(err);
   };
+  /** The PoolUtil object will be the result of the
+      resolved Promise. */
+  const PoolUtil = Object.create(null);
   const promiseResolve =
-        ()=>Promise.resolve(vfsRegisterResult = sqlite3);
+        ()=>Promise.resolve(vfsRegisterResult = PoolUtil);
   // Config opts for the VFS...
   const SECTOR_SIZE = 4096;
   const HEADER_MAX_PATH_SIZE = 512;
@@ -184,15 +193,14 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
        to the new capacity.
     */
     addCapacity: async function(n){
-      const cap = this.getCapacity();
-      for(let i = cap; i < cap+n; ++i){
+      for(let i = 0; i < n; ++i){
         const name = getRandomName();
         const h = await this.dirHandle.getFileHandle(name, {create:true});
         const ah = await h.createSyncAccessHandle();
         this.mapSAHToName.set(ah,name);
         this.setAssociatedPath(ah, '', 0);
       }
-      return i;
+      return this.getCapacity();
     },
     /**
        Removes n entries from the pool's current capacity
@@ -399,14 +407,19 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
       const rc = this.$error;
       this.$error = undefined;
       return rc;
+    },
+    nextAvailableSAH: function(){
+      const [rc] = this.availableSAH.keys();
+      return rc;
     }
+
   })/*SAHPool*/;
-  sqlite3.SAHPool = SAHPool/*only for testing*/;
+  //sqlite3.SAHPool = SAHPool/*only for testing*/;
   /**
      Impls for the sqlite3_io_methods methods. Maintenance reminder:
      members are in alphabetical order to simplify finding them.
   */
-  const ioSyncWrappers = {
+  const ioMethods = {
     xCheckReservedLock: function(pFile,pOut){
       log('xCheckReservedLock');
       SAHPool.storeErr();
@@ -523,13 +536,13 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
         return capi.SQLITE_IOERR;
       }
     }
-  }/*ioSyncWrappers*/;
+  }/*ioMethods*/;
 
   /**
      Impls for the sqlite3_vfs methods. Maintenance reminder: members
      are in alphabetical order to simplify finding them.
   */
-  const vfsSyncWrappers = {
+  const vfsMethods = {
     xAccess: function(pVfs,zName,flags,pOut){
       log(`xAccess ${wasm.cstrToJs(zName)}`);
       SAHPool.storeErr();
@@ -597,7 +610,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
           // File not found so try to create it.
           if(SAHPool.getFileCount() < SAHPool.getCapacity()) {
             // Choose an unassociated OPFS file from the pool.
-            [sah] = SAHPool.availableSAH.keys();
+            sah = SAHPool.nextAvailableSAH();
             SAHPool.setAssociatedPath(sah, path, flags);
           }else{
             // File pool is full.
@@ -621,7 +634,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
         return capi.SQLITE_CANTOPEN;
       }
     }/*xOpen()*/
-  }/*vfsSyncWrappers*/;
+  }/*vfsMethods*/;
 
   if(dVfs){
     /* Inherit certain VFS members from the default VFS,
@@ -631,7 +644,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
   }
   if(!opfsVfs.$xRandomness){
     /* If the default VFS has no xRandomness(), add a basic JS impl... */
-    vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){
+    vfsMethods.xRandomness = function(pVfs, nOut, pOut){
       const heap = wasm.heap8u();
       let i = 0;
       for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF;
@@ -639,7 +652,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
     };
   }
   if(!opfsVfs.$xSleep){
-    vfsSyncWrappers.xSleep = (pVfs,ms)=>0;
+    vfsMethods.xSleep = (pVfs,ms)=>0;
   }
 
   /**
@@ -669,6 +682,58 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
   if(true!==apiVersionCheck){
     return promiseReject(apiVersionCheck);
   }
+
+  PoolUtil.$SAHPool = SAHPool/* ONLY for testing and debugging */;
+  PoolUtil.addCapacity = async (n)=>SAHPool.addCapacity(n);
+  PoolUtil.reduceCapacity = async (n)=>SAHPool.reduceCapacity(n);
+  PoolUtil.getCapacity = SAHPool.getCapacity.bind(SAHPool);
+  PoolUtil.getActiveFileCount = SAHPool.getFileCount.bind(SAHPool);
+  /**
+     Synchronously reads the contents of the given file into a
+     Uint8Array and returns it. This will throw if the given name is
+     not currently in active use or on I/O error.
+  */
+  PoolUtil.exportFile = function(name){
+    const sah = SAHPool.mapPathToSAH.get(name) || toss("File not found:",name);
+    const n = sah.getSize() - HEADER_OFFSET_DATA;
+    const b = new Uint8Array(n>=0 ? n : 0);
+    if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA});
+    return b;
+  };
+
+  /**
+     The counterpart of exportFile(), this imports the contents of an
+     SQLite database, provided as a byte array, under the given name,
+     overwriting any existing content. Throws if the pool has no
+     available file slots, on I/O error, or if the input does not
+     appear to be a database. In the latter case, only a cursory
+     examination is made.
+
+     Note that this routine is _only_ for importing database files,
+     not arbitrary files, the reason being that this VFS will
+     automatically clean up any non-database files so importing them
+     is pointless.
+
+     Returns undefined.
+  */
+  PoolUtil.importDb = function(name, bytes){
+    const n = bytes.byteLength;
+    if(n<512 || n%512!=0){
+      toss("Byte array size is invalid for an SQLite db.");
+    }
+    const header = "SQLite format 3";
+    for(let i = 0; i < header.length; ++i){
+      if( header.charCodeAt(i) !== bytes[i] ){
+        toss("Input does not contain an SQLite database header.");
+      }
+    }
+    const sah = SAHPool.mapPathToSAH.get(name)
+          || SAHPool.nextAvailableSAH()
+          || toss("No available handles to import to.");
+    sah.write(bytes, {at: HEADER_OFFSET_DATA});
+    SAHPool.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
+  };
+
   return SAHPool.isReady = SAHPool.reset().then(async ()=>{
     if(SAHPool.$error){
       throw SAHPool.$error;
@@ -678,12 +743,11 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
     }
     //log("vfs list:",capi.sqlite3_js_vfs_list());
     sqlite3.vfs.installVfs({
-      io: {struct: opfsIoMethods, methods: ioSyncWrappers},
-      vfs: {struct: opfsVfs, methods: vfsSyncWrappers},
-      applyArgcCheck: true
+      io: {struct: opfsIoMethods, methods: ioMethods},
+      vfs: {struct: opfsVfs, methods: vfsMethods}
     });
-    log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods);
-    log("vfs list:",capi.sqlite3_js_vfs_list());
+    //log("opfsVfs",opfsVfs,"opfsIoMethods",opfsIoMethods);
+    //log("vfs list:",capi.sqlite3_js_vfs_list());
     if(sqlite3.oo1){
       const OpfsSAHPoolDb = function(...args){
         const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args);
@@ -691,39 +755,14 @@ sqlite3.installOpfsSAHPoolVfs = async function(){
         sqlite3.oo1.DB.dbCtorHelper.call(this, opt);
       };
       OpfsSAHPoolDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
-      OpfsSAHPoolDb.addPoolCapacity = async (n)=>SAHPool.addCapacity(n);
-      OpfsSAHPoolDb.reducePoolCapacity = async (n)=>SAHPool.reduceCapacity(n);
-      OpfsSAHPoolDb.getPoolCapacity = ()=>SAHPool.getCapacity();
-      OpfsSAHPoolDb.getPoolUsage = ()=>SAHPool.getFileCount();
+      OpfsSAHPoolDb.PoolUtil;
       sqlite3.oo1.OpfsSAHPoolDb = OpfsSAHPoolDb;
       sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql(
         opfsVfs.pointer,
         function(oo1Db, sqlite3){
           sqlite3.capi.sqlite3_exec(oo1Db, [
-            /* As of July 2023, the PERSIST journal mode on OPFS is
-               somewhat slower than DELETE or TRUNCATE (it was faster
-               before Chrome version 108 or 109). TRUNCATE and DELETE
-               have very similar performance on OPFS.
-
-               Roy Hashimoto notes that TRUNCATE and PERSIST modes may
-               decrease OPFS concurrency because multiple connections
-               can open the journal file in those modes:
-
-               https://github.com/rhashimoto/wa-sqlite/issues/68
-
-               Given that, and the fact that testing has not revealed
-               any appreciable difference between performance of
-               TRUNCATE and DELETE modes on OPFS, we currently (as of
-               2023-07-13) default to DELETE mode.
-            */
+            /* See notes in sqlite3-vfs-opfs.js */
             "pragma journal_mode=DELETE;",
-            /*
-              OPFS benefits hugely from cache on moderate/large
-              speedtest1 --size 50 and --size 100 workloads. We
-              currently rely on setting a non-default cache size when
-              building sqlite3.wasm. If that policy changes, the cache
-              can be set here.
-            */
             "pragma cache_size=-16384;"
           ], 0, 0, 0);
         }

ext/wasm/speedtest1-worker.js

@@ -107,7 +107,10 @@
     const S = globalThis.S = sqlite3;
     log("Loaded speedtest1 module. Setting up...");
     if(S.installOpfsSAHPoolVfs){
-      await S.installOpfsSAHPoolVfs().catch(e=>{
+      await S.installOpfsSAHPoolVfs().then(P=>{
+        S.SAHPoolUtil = P;
+        //return P.addCapacity(5).then(log("pool capacity:",P.getCapacity()));;
+      }).catch(e=>{
         logErr("Error setting up opfs-sahpool:",e.message);
       });
     }

manifest

@@ -1,5 +1,5 @@
-C Minor\scleanups\sin\sthe\sopfs-sahpool\sVFS.
-D 2023-07-15T21:08:48.986
+C Redefine\swhat\sthe\sopfs-sahpool\sinstallation\spromise\sresolves\sto.\sFix\saddCapacity().\sAdd\sutility\smethods\sto\simport/export\sfiles.
+D 2023-07-16T10:02:41.870
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386
 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2
 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487
-F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 0032097c168c8fe7e753abc5b35e65323116d04b0dbaaa97176604660b7bb98c
+F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 0e982cc4f1b9ed4786086d1115e740b7efd628de5cbaf16caf3a71913f91241b
 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js a5c3195203e6085d7aa89fae4b84cf3f3eec4ff4f928c6d0e5d3ef8b14cbc1c0
 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda
 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
@@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150
 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d
 F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe
 F ext/wasm/speedtest1-worker.html bbcf1e7fd79541040c1a7ca2ebf1cb7793ddaf9900d6bde1784148f11b807c34
-F ext/wasm/speedtest1-worker.js 554b0985f791758e40ff2b1a04b771e315ab84b4e26b4b8a1c7a5ba968086c45
+F ext/wasm/speedtest1-worker.js faa4a06ec21921aaa0e0b672a94b56037da837e16732bdd6545b99f1cadbb32e
 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
@@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 41bf1fe31f2f3d0daa2bac25dc57262a4b90f22fed6fa97e4e92467c32ae02dc
-R a6cacf00d5cb9e3eccf543ab6eefecd8
+P 279e09070918dab7b60c39179ebb7eb931ca6bd4e589b414f436740499a2f910
+R 3700fe28ac7e053b813d4612e5083eb3
 U stephan
-Z d51af2a044894ea09e404a3df277218e
+Z 2285d7e58406d05c66733e55ed60721b
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-279e09070918dab7b60c39179ebb7eb931ca6bd4e589b414f436740499a2f910
+809c6f4de3653ad7a7751af45a7a0d6cb20c3ee3be80c69833c729242227d970