Cut the speedtest1 runtime of the OPFS VFS proxy by approximately 3/4ths via xRead/xWrite buffer-copying optimizations. Still slower than the WASMFS impl by approx. 1/5th.

FossilOrigin-Name: fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51

ext/wasm/api/sqlite3-api-opfs.js

@@ -134,6 +134,34 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
        by the Promise setup and, on success, installed as sqlite3.opfs.
     */
     const opfsUtil = Object.create(null);
+    /**
+       Not part of the public API. Solely for internal/development
+       use.
+    */
+    opfsUtil.metrics = {
+      dump: function(){
+        let k, n = 0, t = 0;
+        for(k in metrics){
+          const m = metrics[k];
+          n += m.count;
+          t += m.time;
+          m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0;
+          m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0;
+        }
+        console.log("metrics for",self.location.href,":",metrics,
+                    "\nTotal of",n,"op(s) for",t,"ms");
+      },
+      reset: function(){
+        let k;
+        const r = (m)=>(m.count = m.time = m.wait = 0);
+        for(k in state.opIds){
+          r(metrics[k] = Object.create(null));
+        }
+        [ // timed routines which are not in state.opIds
+          'xFileControl'
+        ].forEach((k)=>r(metrics[k] = Object.create(null)));
+      }
+    }/*metrics*/;      
 
     /**
        State which we send to the async-api Worker or share with it.
@@ -165,6 +193,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
     state.fbInt64Offset =
       state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */;
     state.opIds = Object.create(null);
+    const metrics = Object.create(null);
     {
       let i = 0;
       state.opIds.xAccess = i++;
@@ -180,6 +209,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       state.opIds.xWrite = i++;
       state.opIds.mkdir = i++;
       state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/);
+      opfsUtil.metrics.reset();
     }
 
     state.sq3Codes = Object.create(null);
@@ -207,9 +237,11 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
        given operation's signature in the async API counterpart.
     */
     const opRun = (op,args)=>{
+      const t = performance.now();
       Atomics.store(state.opSABView, state.opIds[op], -1);
       wMsg(op, args);
       Atomics.wait(state.opSABView, state.opIds[op], -1);
+      metrics[op].wait += performance.now() - t;
       return Atomics.load(state.opSABView, state.opIds[op]);
     };
 
@@ -338,7 +370,20 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       tgt.ondispose.__removeFuncList.push(memKey, pFunc);
       return (n,f)=>callee(tgt, n, f);
     }/*installMethod*/;
-    
+
+    const opTimer = Object.create(null);
+    opTimer.op = undefined;
+    opTimer.start = undefined;
+    const mTimeStart = (op)=>{
+      opTimer.start = performance.now();
+      opTimer.op = op;
+      //metrics[op] || toss("Maintenance required: missing metrics for",op);
+      ++metrics[op].count;
+    };
+    const mTimeEnd = ()=>(
+      metrics[opTimer.op].time += performance.now() - opTimer.start
+    );
+
     /**
        Impls for the sqlite3_io_methods methods. Maintenance reminder:
        members are in alphabetical order to simplify finding them.
@@ -351,6 +396,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
         return 0;
       },
       xClose: function(pFile){
+        mTimeStart('xClose');
         let rc = 0;
         const f = __openFiles[pFile];
         if(f){
@@ -358,22 +404,29 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
           rc = opRun('xClose', pFile);
           if(f.sq3File) f.sq3File.dispose();
         }
+        mTimeEnd();
         return rc;
       },
       xDeviceCharacteristics: function(pFile){
         //debug("xDeviceCharacteristics(",pFile,")");
         return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
       },
-      xFileControl: function(pFile,op,pArg){
-        //debug("xFileControl(",arguments,") is a no-op");
+      xFileControl: function(pFile, opId, pArg){
+        mTimeStart('xFileControl');
+        if(capi.SQLITE_FCNTL_SYNC===opId){
+          return opRun('xSync', {fid:pFile, flags:0});
+        }
+        mTimeEnd();
         return capi.SQLITE_NOTFOUND;
       },
       xFileSize: function(pFile,pSz64){
+        mTimeStart('xFileSize');
         const rc = opRun('xFileSize', pFile);
         if(!isWorkerErrCode(rc)){
           const f = __openFiles[pFile];
-          wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0) ,'i64');
+          wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0,true) ,'i64');
         }
+        mTimeEnd();
         return rc;
       },
       xLock: function(pFile,lockType){
@@ -383,26 +436,31 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       },
       xRead: function(pFile,pDest,n,offset){
         /* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */
+        mTimeStart('xRead');
         const f = __openFiles[pFile];
         let rc;
         try {
           // FIXME(?): block until we finish copying the xRead result buffer. How?
           rc = opRun('xRead',{fid:pFile, n, offset});
           if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){
-            let i = 0;
-            for(; i < n; ++i) wasm.setMemValue(pDest + i, f.sabView[i]);
+            // set() seems to be the fastest way to copy this...
+            wasm.heap8u().set(f.sabView.subarray(0, n), pDest);
           }
         }catch(e){
           error("xRead(",arguments,") failed:",e,f);
           rc = capi.SQLITE_IOERR_READ;
         }
+        mTimeEnd();
         return rc;
       },
       xSync: function(pFile,flags){
-        return opRun('xSync', {fid:pFile, flags});
+        return 0; // impl'd in xFileControl(). opRun('xSync', {fid:pFile, flags});
       },
       xTruncate: function(pFile,sz64){
-        return opRun('xTruncate', {fid:pFile, size: sz64});
+        mTimeStart('xTruncate');
+        const rc = opRun('xTruncate', {fid:pFile, size: sz64});
+        mTimeEnd();
+        return rc;
       },
       xUnlock: function(pFile,lockType){
         //2022-09: OPFS handles lock when opened
@@ -411,16 +469,19 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       },
       xWrite: function(pFile,pSrc,n,offset){
         /* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */
+        mTimeStart('xWrite');
         const f = __openFiles[pFile];
+        let rc;
         try {
-          let i = 0;
           // FIXME(?): block from here until we finish the xWrite. How?
-          for(; i < n; ++i) f.sabView[i] = wasm.getMemValue(pSrc+i);
-          return opRun('xWrite',{fid:pFile, n, offset});
+          f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n));
+          rc = opRun('xWrite',{fid:pFile, n, offset});
         }catch(e){
           error("xWrite(",arguments,") failed:",e,f);
-          return capi.SQLITE_IOERR_WRITE;
+          rc = capi.SQLITE_IOERR_WRITE;
         }
+        mTimeEnd();
+        return rc;
       }
     }/*ioSyncWrappers*/;
     
@@ -430,8 +491,10 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
     */
     const vfsSyncWrappers = {
       xAccess: function(pVfs,zName,flags,pOut){
+        mTimeStart('xAccess');
         const rc = opRun('xAccess', wasm.cstringToJs(zName));
         wasm.setMemValue(pOut, rc ? 0 : 1, 'i32');
+        mTimeEnd();
         return 0;
       },
       xCurrentTime: function(pVfs,pOut){
@@ -448,9 +511,11 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
         return 0;
       },
       xDelete: function(pVfs, zName, doSyncDir){
+        mTimeStart('xDelete');
         opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir});
         /* We're ignoring errors because we cannot yet differentiate
            between harmless and non-harmless failures. */
+        mTimeEnd();
         return 0;
       },
       xFullPathname: function(pVfs,zName,nOut,pOut){
@@ -469,6 +534,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       },
       //xSleep is optionally defined below
       xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
+        mTimeStart('xOpen');
         if(!f._){
           f._ = {
             fileTypes: {
@@ -518,6 +584,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
           args.sq3File.$pMethods = opfsIoMethods.pointer;
           args.ba = new Uint8Array(args.sab);
         }
+        mTimeEnd();
         return rc;
       }/*xOpen()*/
     }/*vfsSyncWrappers*/;
@@ -600,7 +667,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       };
       opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
     }
-
+    
     /**
        Potential TODOs:
 
@@ -670,6 +737,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
       }
     }/*sanityCheck()*/;
 
+    
     W.onmessage = function({data}){
       //log("Worker.onmessage:",data);
       switch(data.type){

ext/wasm/common/whwasmutil.js

@@ -603,6 +603,10 @@ self.WhWasmUtilInstaller = function(target){
      out) the pointer's value, else it will contain an essentially
      random value.
 
+     ACHTUNG: calling this often, e.g. in a loop, can have a noticably
+     painful impact on performance. Rather than doing so, use
+     heapForSize() to fetch the heap object and read directly from it.
+
      See: setMemValue()
   */
   target.getMemValue = function(ptr, type='i8'){
@@ -633,6 +637,10 @@ self.WhWasmUtilInstaller = function(target){
      this function behaves as if the 3rd argument were `i32`.
 
      This function returns itself.
+
+     ACHTUNG: calling this often, e.g. in a loop, can have a noticably
+     painful impact on performance. Rather than doing so, use
+     heapForSize() to fetch the heap object and assign directly to it.
   */
   target.setMemValue = function f(ptr, value, type='i8'){
     if (type.endsWith('*')) type = ptrIR;

ext/wasm/speedtest1-worker.html

@@ -260,9 +260,12 @@
         flags["without"] = "rowid     Use WITHOUT ROWID where appropriate";
         const preselectedFlags = [
           'big-transactions',
-          'memdb',
           'singlethread'
         ];
+        if('opfs'!==urlParams.get('vfs')){
+          preselectedFlags.push('memdb');
+        }
+
         Object.keys(flags).sort().forEach(function(f){
           const opt = document.createElement('option');
           eFlags.appendChild(opt);

ext/wasm/speedtest1-worker.js

@@ -45,7 +45,7 @@
 
   const runSpeedtest = function(cliFlagsArray){
     const scope = App.wasm.scopedAllocPush();
-    const dbFile = App.pDir+"/speedtest1.db";
+    const dbFile = App.pDir+"/speedtest1.sqlite3";
     App.unlink(dbFile);
     try{
       const argv = [

ext/wasm/sqlite3-opfs-async-proxy.js

@@ -60,6 +60,7 @@ const logImpl = (level,...args)=>{
 const log =    (...args)=>logImpl(2, ...args);
 const warn =   (...args)=>logImpl(1, ...args);
 const error =  (...args)=>logImpl(0, ...args);
+const metrics = Object.create(null);
 
 warn("This file is very much experimental and under construction.",
      self.location.pathname);
@@ -220,7 +221,7 @@ const vfsAsyncImpls = {
     let sz;
     try{
       sz = await fh.accessHandle.getSize();
-      fh.sabViewFileSize.setBigInt64(0, BigInt(sz));
+      fh.sabViewFileSize.setBigInt64(0, BigInt(sz), true);
       sz = 0;
     }catch(e){
       error("xFileSize():",e, fh);