Cut the speedtest1 runtime of the OPFS VFS proxy by approximately 3/4ths via xRead/xWrite buffer-copying optimizations. Still slower than the WASMFS impl by approx. 1/5th.

FossilOrigin-Name: fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51

ext/wasm/api/sqlite3-api-opfs.js

@@ -134,6 +134,34 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
        by the Promise setup and, on success, installed as sqlite3.opfs.
     */
     const opfsUtil = Object.create(null);
+    /**
+       Not part of the public API. Solely for internal/development
+       use.
+    */
+    opfsUtil.metrics = {
+      dump: function(){
+        let k, n = 0, t = 0;
+        for(k in metrics){
+          const m = metrics[k];
+          n += m.count;
+          t += m.time;
+          m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0;
+          m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0;
+        }
+        console.log("metrics for",self.location.href,":",metrics,
+                    "\nTotal of",n,"op(s) for",t,"ms");
+      },
+      reset: function(){
+        let k;
+        const r = (m)=>(m.count = m.time = m.wait = 0);
+        for(k in state.opIds){
+          r(metrics[k] = Object.create(null));
+        }
+        [ // timed routines which are not in state.opIds
+          'xFileControl'
+        ].forEach((k)=>r(metrics[k] = Object.create(null)));
+      }
+    }/*metrics*/;      
 
     /**
        State which we send to the async-api Worker or share with it.
@@ -165,6 +193,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
     state.fbInt64Offset =
       state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */;
     state.opIds = Object.create(null);
+    const metrics = Object.create(null);
     {
       let i = 0;
       state.opIds.xAccess = i++;
@@ -180,6 +209,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       state.opIds.xWrite = i++;
       state.opIds.mkdir = i++;
       state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/);
+      opfsUtil.metrics.reset();
     }
 
     state.sq3Codes = Object.create(null);
@@ -207,9 +237,11 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
        given operation's signature in the async API counterpart.
     */
     const opRun = (op,args)=>{
+      const t = performance.now();
       Atomics.store(state.opSABView, state.opIds[op], -1);
       wMsg(op, args);
       Atomics.wait(state.opSABView, state.opIds[op], -1);
+      metrics[op].wait += performance.now() - t;
       return Atomics.load(state.opSABView, state.opIds[op]);
     };
 
@@ -338,7 +370,20 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       tgt.ondispose.__removeFuncList.push(memKey, pFunc);
       return (n,f)=>callee(tgt, n, f);
     }/*installMethod*/;
-    
+
+    const opTimer = Object.create(null);
+    opTimer.op = undefined;
+    opTimer.start = undefined;
+    const mTimeStart = (op)=>{
+      opTimer.start = performance.now();
+      opTimer.op = op;
+      //metrics[op] || toss("Maintenance required: missing metrics for",op);
+      ++metrics[op].count;
+    };
+    const mTimeEnd = ()=>(
+      metrics[opTimer.op].time += performance.now() - opTimer.start
+    );
+
     /**
        Impls for the sqlite3_io_methods methods. Maintenance reminder:
        members are in alphabetical order to simplify finding them.
@@ -351,6 +396,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
         return 0;
       },
       xClose: function(pFile){
+        mTimeStart('xClose');
         let rc = 0;
         const f = __openFiles[pFile];
         if(f){
@@ -358,22 +404,29 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
           rc = opRun('xClose', pFile);
           if(f.sq3File) f.sq3File.dispose();
         }
+        mTimeEnd();
         return rc;
       },
       xDeviceCharacteristics: function(pFile){
         //debug("xDeviceCharacteristics(",pFile,")");
         return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
       },
-      xFileControl: function(pFile,op,pArg){
+      xFileControl: function(pFile, opId, pArg){
-        //debug("xFileControl(",arguments,") is a no-op");
+        mTimeStart('xFileControl');
+        if(capi.SQLITE_FCNTL_SYNC===opId){
+          return opRun('xSync', {fid:pFile, flags:0});
+        }
+        mTimeEnd();
         return capi.SQLITE_NOTFOUND;
       },
       xFileSize: function(pFile,pSz64){
+        mTimeStart('xFileSize');
         const rc = opRun('xFileSize', pFile);
         if(!isWorkerErrCode(rc)){
           const f = __openFiles[pFile];
-          wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0) ,'i64');
+          wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0,true) ,'i64');
         }
+        mTimeEnd();
         return rc;
       },
       xLock: function(pFile,lockType){
@@ -383,26 +436,31 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       },
       xRead: function(pFile,pDest,n,offset){
         /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */
+        mTimeStart('xRead');
         const f = __openFiles[pFile];
         let rc;
         try {
           // FIXME(?): block until we finish copying the xRead result buffer. How?
           rc = opRun('xRead',{fid:pFile, n, offset});
           if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){
-            let i = 0;
+            // set() seems to be the fastest way to copy this...
-            for(; i < n; ++i) wasm.setMemValue(pDest + i, f.sabView[i]);
+            wasm.heap8u().set(f.sabView.subarray(0, n), pDest);
           }
         }catch(e){
           error("xRead(",arguments,") failed:",e,f);
           rc = capi.SQLITE_IOERR_READ;
         }
+        mTimeEnd();
         return rc;
       },
       xSync: function(pFile,flags){
-        return opRun('xSync', {fid:pFile, flags});
+        return 0; // impl'd in xFileControl(). opRun('xSync', {fid:pFile, flags});
       },
       xTruncate: function(pFile,sz64){
-        return opRun('xTruncate', {fid:pFile, size: sz64});
+        mTimeStart('xTruncate');
+        const rc = opRun('xTruncate', {fid:pFile, size: sz64});
+        mTimeEnd();
+        return rc;
       },
       xUnlock: function(pFile,lockType){
         //2022-09: OPFS handles lock when opened
@@ -411,16 +469,19 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       },
       xWrite: function(pFile,pSrc,n,offset){
         /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */
+        mTimeStart('xWrite');
         const f = __openFiles[pFile];
+        let rc;
         try {
-          let i = 0;
           // FIXME(?): block from here until we finish the xWrite. How?
-          for(; i < n; ++i) f.sabView[i] = wasm.getMemValue(pSrc+i);
+          f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n));
-          return opRun('xWrite',{fid:pFile, n, offset});
+          rc = opRun('xWrite',{fid:pFile, n, offset});
         }catch(e){
           error("xWrite(",arguments,") failed:",e,f);
-          return capi.SQLITE_IOERR_WRITE;
+          rc = capi.SQLITE_IOERR_WRITE;
         }
+        mTimeEnd();
+        return rc;
       }
     }/*ioSyncWrappers*/;
     
@@ -430,8 +491,10 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
     */
     const vfsSyncWrappers = {
       xAccess: function(pVfs,zName,flags,pOut){
+        mTimeStart('xAccess');
         const rc = opRun('xAccess', wasm.cstringToJs(zName));
         wasm.setMemValue(pOut, rc ? 0 : 1, 'i32');
+        mTimeEnd();
         return 0;
       },
       xCurrentTime: function(pVfs,pOut){
@@ -448,9 +511,11 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
         return 0;
       },
       xDelete: function(pVfs, zName, doSyncDir){
+        mTimeStart('xDelete');
         opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir});
         /* We're ignoring errors because we cannot yet differentiate
            between harmless and non-harmless failures. */
+        mTimeEnd();
         return 0;
       },
       xFullPathname: function(pVfs,zName,nOut,pOut){
@@ -469,6 +534,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       },
       //xSleep is optionally defined below
       xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
+        mTimeStart('xOpen');
         if(!f._){
           f._ = {
             fileTypes: {
@@ -518,6 +584,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
           args.sq3File.$pMethods = opfsIoMethods.pointer;
           args.ba = new Uint8Array(args.sab);
         }
+        mTimeEnd();
         return rc;
       }/*xOpen()*/
     }/*vfsSyncWrappers*/;
@@ -600,7 +667,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       };
       opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
     }
-
+    
     /**
        Potential TODOs:
 
@@ -670,6 +737,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       }
     }/*sanityCheck()*/;
 
+    
     W.onmessage = function({data}){
       //log("Worker.onmessage:",data);
       switch(data.type){

ext/wasm/common/whwasmutil.js

@@ -603,6 +603,10 @@ self.WhWasmUtilInstaller = function(target){
      out) the pointer's value, else it will contain an essentially
      random value.
 
+     ACHTUNG: calling this often, e.g. in a loop, can have a noticably
+     painful impact on performance. Rather than doing so, use
+     heapForSize() to fetch the heap object and read directly from it.
+
      See: setMemValue()
   */
   target.getMemValue = function(ptr, type='i8'){
@@ -633,6 +637,10 @@ self.WhWasmUtilInstaller = function(target){
      this function behaves as if the 3rd argument were `i32`.
 
      This function returns itself.
+
+     ACHTUNG: calling this often, e.g. in a loop, can have a noticably
+     painful impact on performance. Rather than doing so, use
+     heapForSize() to fetch the heap object and assign directly to it.
   */
   target.setMemValue = function f(ptr, value, type='i8'){
     if (type.endsWith('*')) type = ptrIR;

ext/wasm/speedtest1-worker.html

@@ -260,9 +260,12 @@
         flags["without"] = "rowid     Use WITHOUT ROWID where appropriate";
         const preselectedFlags = [
           'big-transactions',
-          'memdb',
           'singlethread'
         ];
+        if('opfs'!==urlParams.get('vfs')){
+          preselectedFlags.push('memdb');
+        }
+
         Object.keys(flags).sort().forEach(function(f){
           const opt = document.createElement('option');
           eFlags.appendChild(opt);

ext/wasm/speedtest1-worker.js

@@ -45,7 +45,7 @@
 
   const runSpeedtest = function(cliFlagsArray){
     const scope = App.wasm.scopedAllocPush();
-    const dbFile = App.pDir+"/speedtest1.db";
+    const dbFile = App.pDir+"/speedtest1.sqlite3";
     App.unlink(dbFile);
     try{
       const argv = [

ext/wasm/sqlite3-opfs-async-proxy.js

@@ -60,6 +60,7 @@ const logImpl = (level,...args)=>{
 const log =    (...args)=>logImpl(2, ...args);
 const warn =   (...args)=>logImpl(1, ...args);
 const error =  (...args)=>logImpl(0, ...args);
+const metrics = Object.create(null);
 
 warn("This file is very much experimental and under construction.",
      self.location.pathname);
@@ -220,7 +221,7 @@ const vfsAsyncImpls = {
     let sz;
     try{
       sz = await fh.accessHandle.getSize();
-      fh.sabViewFileSize.setBigInt64(0, BigInt(sz));
+      fh.sabViewFileSize.setBigInt64(0, BigInt(sz), true);
       sz = 0;
     }catch(e){
       error("xFileSize():",e, fh);

manifest

@@ -1,5 +1,5 @@
-C Remove\san\sobsolete\sfile.\sUpdate\swasmfs\sbuild\sfor\stoday's\schanges.\sFix\sthe\scase\sthat\sOPFS.xTruncate()\sis\spassed\sa\sBigInt\sinstead\sof\sa\sNumber.
+C Cut\sthe\sspeedtest1\sruntime\sof\sthe\sOPFS\sVFS\sproxy\sby\sapproximately\s3/4ths\svia\sxRead/xWrite\sbuffer-copying\soptimizations.\sStill\sslower\sthan\sthe\sWASMFS\simpl\sby\sapprox.\s1/5th.
-D 2022-09-19T14:56:13.838
+D 2022-09-19T17:09:09.850
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a
 F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
 F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30
 F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48
-F ext/wasm/api/sqlite3-api-opfs.js e6b3a168f18b94f01fd21028ffdbaf4e576c53c2f2762032c9ee7d24b7a3bd99
+F ext/wasm/api/sqlite3-api-opfs.js 627b2d8fcd37b6479d7f0e8d5ba891d00c1eaf9b8cf99981081c2bf2e7ed8560
 F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655
 F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0
 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
@@ -495,7 +495,7 @@ F ext/wasm/batch-runner.js 6f5b86e0b5519a9a941d9f17ee9c5ecdc63f452f157602fe7fdf8
 F ext/wasm/common/SqliteTestUtil.js 529161a624265ba84271a52db58da022649832fa1c71309fb1e02cc037327a2b
 F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f
 F ext/wasm/common/testing.css 3a5143699c2b73a85b962271e1a9b3241b30d90e30d895e4f55665e648572962
-F ext/wasm/common/whwasmutil.js f7282ef36c9625330d4e6e82d1beec6678cd101e95e7108cd85db587a788c145
+F ext/wasm/common/whwasmutil.js f64f94e2a3e1d8fd2139b4d80f00a3ab504985e38103c45b1ff15ec8b44fcce3
 F ext/wasm/demo-123-worker.html e419b66495d209b5211ec64903b4cfb3ca7df20d652b41fcd28bf018a773234f
 F ext/wasm/demo-123.html aa281d33b7eefa755f3122b7b5a18f39a42dc5fb69c8879171bf14b4c37c4ec4
 F ext/wasm/demo-123.js 234655683e35a4543a23de7b10800d76b0369947b33e089e5613171fa7795afb
@@ -513,13 +513,13 @@ F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e5
 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06
 F ext/wasm/scratchpad-wasmfs-main.js f0836e3576df7a89390d777bb53e142e559e8a79becfb2a5a976490b05a1c4fa
 F ext/wasm/speedtest1-wasmfs.html 9d8cd19eab8854d17f7129aa11607cae6f6d9857c505a4aef13000588583d93e
-F ext/wasm/speedtest1-worker.html f7adcc68af76d347741ecd884e0c773f707fa2cdec87cfd9b2e147f5fc2c56c7
+F ext/wasm/speedtest1-worker.html 7bd3ede33e08067f59e8d6e3dc25016fc930222b059d66b2554203bf262aba02
-F ext/wasm/speedtest1-worker.js d7e02ff74c28c8f2ef19eb6585642d44b7c8e7b83ac19b7848b209a880945706
+F ext/wasm/speedtest1-worker.js 11e7f68cedd2a83b0e638f94c1d2f58406ba672a7e88b66bff5d4f4284e8ba16
 F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0aa0d828cf1d
 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
 F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
-F ext/wasm/sqlite3-opfs-async-proxy.js 7ce90feb4ed582a7709f6a99a46fd43a240a1c1ad2f9f47efff1bd32136ab09c
+F ext/wasm/sqlite3-opfs-async-proxy.js 7139530dd8a408f2897115d3ea04dfe03f175df66870b1e8a49733a77a06f970
 F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365
 F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
 F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
@@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2ec7e09139a510b9fd29e4c97283b20740a00f369193c6fecbb734f187e81b48
+P 976053925013bf1975f5f9222e28ba648af28e305bb6bdae600eb24d0e136bec
-R 1715d33dc730c3ab31438d0a66f7aa0a
+R 79e6bae24f2a8f4f545c24b5d2562e72
 U stephan
-Z 0f0e9b03805cac5450f028e863da1f77
+Z 251c239274312fdfb8787b5b1f3a9b52
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-976053925013bf1975f5f9222e28ba648af28e305bb6bdae600eb24d0e136bec
+fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51