Fix a long-standing filename digest computation bug in the OPFS SAHPool VFS which caused all VFS-stored filenames to have a digest value of 0. See [/forumpost/042d53c928382021] and for full details.

FossilOrigin-Name: 493cbe74504e8eb1ca8f2edf49fdab6bebc7fe36ffab06932a4b8c5a4eea86cd

ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js

@@ -79,6 +79,48 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
         capi.SQLITE_OPEN_MAIN_JOURNAL |
         capi.SQLITE_OPEN_SUPER_JOURNAL |
         capi.SQLITE_OPEN_WAL;
+  const FLAG_COMPUTE_DIGEST_V2 = capi.SQLITE_OPEN_MEMORY
+  /* Part of the fix for
+     https://github.com/sqlite/sqlite-wasm/issues/97
+
+     Summary: prior to version 3.50.0 computeDigest() always computes
+     a value of [0,0] due to overflows, so it does not do anything
+     useful.  Fixing it invalidates old persistent files, so we
+     instead only fix it for files created or updated since the bug
+     was discovered and fixed.
+
+     This flag determines whether we use the broken legacy
+     computeDigest() or the v2 variant. We only use this flag for
+     newly-created/overwritten files. Pre-existing files have the
+     broken digest stored in them so need to continue to use that.
+
+     What this means, in terms of db file compatibility between
+     versions:
+
+     - DBs created with versions older than this fix (<3.50.0)
+     can be read by post-fix versions. Such DBs which are written
+     to in-place (not replaced) by newer versions can still be read
+     by older versions, as the affected digest is only modified
+     when the SAH slot is assigned to a given filename.
+
+     - DBs created with post-fix versions will, when read by a pre-fix
+     version, be seen as having a "bad digest" and will be
+     unceremoniously replaced by that pre-fix version. When swapping
+     back to a post-fix version, that version will see that the file
+     entry is missing the FLAG_COMPUTE_DIGEST_V2 bit so will treat it
+     as a legacy file.
+
+     This flag is stored in the same memory as the various
+     SQLITE_OPEN_... flags and we must be careful here to not use a
+     flag bit which is otherwise relevant for the VFS.
+     SQLITE_OPEN_MEMORY is handled by sqlite3_open_v2() and friends,
+     not the VFS, so we'll repurpose that one.  If we take a
+     currently-unused bit and it ends up, at some later point, being
+     used, we would have to invalidate existing VFS files in order to
+     move to another bit.  Similarly, if the SQLITE_OPEN_MEMORY bit
+     were ever reassigned (which it won't be!), we'd invalidate all
+     VFS-side files.
+  */;
 
   /** Subdirectory of the VFS's space where "opaque" (randomly-named)
       files are stored. Changing this effectively invalidates the data
@@ -329,6 +371,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
     xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
       const pool = getPoolForVfs(pVfs);
       try{
+        flags &= ~FLAG_COMPUTE_DIGEST_V2;
         pool.log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`);
         // First try to open a path that already exists in the file system.
         const path = (zName && wasm.peek8(zName))
@@ -624,7 +667,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
 
       const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4);
       sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST});
-      const compDigest = this.computeDigest(this.#apBody);
+      const compDigest = this.computeDigest(this.#apBody, flags);
+      //warn("getAssociatedPath() flags",'0x'+flags.toString(16), "compDigest", compDigest);
       if(fileDigest.every((v,i) => v===compDigest[i])){
         // Valid digest
         const pathBytes = this.#apBody.findIndex((v)=>0===v);
@@ -633,6 +677,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
           // leaving stale db data laying around.
           sah.truncate(HEADER_OFFSET_DATA);
         }
+        //warn("getAssociatedPath() flags",'0x'+flags.toString(16), "compDigest", compDigest,"pathBytes",pathBytes);
         return pathBytes
           ? textDecoder.decode(this.#apBody.subarray(0,pathBytes))
           : '';
@@ -655,10 +700,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
       if(HEADER_MAX_PATH_SIZE <= enc.written + 1/*NUL byte*/){
         toss("Path too long:",path);
       }
+      if(path && flags){
+        /* When creating or re-writing files, update their digest, if
+           needed, to v2. We continue to use v1 for the (!path) case
+           (empty files) because there's little reason not to use a
+           digest of 0 for empty entries. */
+        flags |= FLAG_COMPUTE_DIGEST_V2;
+      }
       this.#apBody.fill(0, enc.written, HEADER_MAX_PATH_SIZE);
       this.#dvBody.setUint32(HEADER_OFFSET_FLAGS, flags);
-
+      const digest = this.computeDigest(this.#apBody, flags);
-      const digest = this.computeDigest(this.#apBody);
+      //console.warn("setAssociatedPath(",path,") digest",digest);
       sah.write(this.#apBody, {at: 0});
       sah.write(digest, {at: HEADER_OFFSET_DIGEST});
       sah.flush();
@@ -679,15 +731,22 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
        metadata for each file as a validation check. Changing this
        algorithm invalidates all existing databases for this VFS, so
        don't do that.
+
+       See the docs for FLAG_COMPUTE_DIGEST_V2 for more details.
     */
-    computeDigest(byteArray){
+    computeDigest(byteArray, fileFlags){
-      let h1 = 0xdeadbeef;
+      if( fileFlags & FLAG_COMPUTE_DIGEST_V2 ){
-      let h2 = 0x41c6ce57;
+        let h1 = 0xdeadbeef;
-      for(const v of byteArray){
+        let h2 = 0x41c6ce57;
-        h1 = 31 * h1 + (v * 307);
+        for(const v of byteArray){
-        h2 = 31 * h2 + (v * 307);
+          h1 = Math.imul(h1 ^ v, 2654435761);
+          h2 = Math.imul(h2 ^ v, 104729);
+        }
+        return new Uint32Array([h1>>>0, h2>>>0]);
+      }else{
+        /* this is what the buggy legacy computation worked out to */
+        return new Uint32Array([0,0]);
       }
-      return new Uint32Array([h1>>>0, h2>>>0]);
     }
 
     /**

ext/wasm/tester1.c-pp.js

@@ -3506,7 +3506,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
           });
           db.exec([
             "create table t(a);",
-            "insert into t(a) values(1),(2),(3);",
+            "insert into t(a) values(1),(2),(1);",
             "select auxtest(1,a), auxtest(1,a) from t order by a"
           ]);
         }finally{

ext/wasm/tests/opfs/sahpool/digest-worker.js

@@ -0,0 +1,94 @@
+/*
+  2025-01-31
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  This file is part of testing the OPFS SAHPool VFS's computeDigest()
+  fix.  See ./digest.html for the details.
+*/
+const clog = console.log.bind(console);
+const wPost = (type,...args)=>postMessage({type, payload:args});
+const log = (...args)=>{
+  clog("Worker:",...args);
+  wPost('log',...args);
+}
+
+const hasOpfs = ()=>{
+  return globalThis.FileSystemHandle
+    && globalThis.FileSystemDirectoryHandle
+    && globalThis.FileSystemFileHandle
+    && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle
+    && navigator?.storage?.getDirectory;
+};
+if( !hasOpfs() ){
+  wPost('error',"OPFS not detected");
+  throw new Error("OPFS not detected");
+}
+
+clog("Importing sqlite3...");
+const searchParams = new URL(self.location.href).searchParams;
+importScripts(searchParams.get('sqlite3.dir') + '/sqlite3.js');
+
+const runTests = function(sqlite3, poolUtil){
+  const fname = '/my.db';
+  let db = new poolUtil.OpfsSAHPoolDb(fname);
+  let n = (new Date()).valueOf();
+  try {
+    db.exec([
+      "create table if not exists t(a);"
+    ]);
+    db.exec({
+      sql: "insert into t(a) values(?)",
+      bind: n++
+    });
+    log(fname,"record count: ",db.selectValue("select count(*) from t"));
+  }finally{
+    db.close();
+  }
+
+  db = new poolUtil.OpfsSAHPoolDb(fname);
+  try {
+    db.exec({
+      sql: "insert into t(a) values(?)",
+      bind: n++
+    });
+    log(fname,"record count: ",db.selectValue("select count(*) from t"));
+  }finally{
+    db.close();
+  }
+
+  const fname2 = '/my2.db';
+  db = new poolUtil.OpfsSAHPoolDb(fname2);
+  try {
+    db.exec([
+      "create table if not exists t(a);"
+    ]);
+    db.exec({
+      sql: "insert into t(a) values(?)",
+      bind: n++
+    });
+    log(fname2,"record count: ",db.selectValue("select count(*) from t"));
+  }finally{
+    db.close();
+  }
+};
+
+globalThis.sqlite3InitModule().then(async function(sqlite3){
+  log("sqlite3 version:",sqlite3.version);
+  const sahPoolConfig = {
+    name: 'opfs-sahpool-digest',
+    clearOnInit: false,
+    initialCapacity: 6
+  };
+  return sqlite3.installOpfsSAHPoolVfs(sahPoolConfig).then(poolUtil=>{
+    log('vfs acquired');
+    runTests(sqlite3, poolUtil);
+  });
+});

ext/wasm/tests/opfs/sahpool/digest.html

@@ -0,0 +1,151 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <link rel="stylesheet" href="../../../common/emscripten.css"/>
+    <link rel="stylesheet" href="../../../common/testing.css"/>
+    <title>sqlite3 tester: OpfsSAHPool Digest</title>
+    <style></style>
+  </head>
+  <body><h1 id='color-target'></h1>
+
+    <p>
+      This is a test app for the digest calculation of the OPFS
+      SAHPool VFS. It requires running it with a new database created using
+      v3.49.0 or older, then running it again with a newer version, then
+      again with 3.49.0 or older.
+    </p>
+    <div class='input-wrapper'>
+      <input type='checkbox' id='cb-log-reverse'>
+      <label for='cb-log-reverse'>Reverse log order?</label>
+    </div>
+    <div id='test-output'></div>
+    <script>
+      /*
+        2025-02-03
+
+        The author disclaims copyright to this source code.  In place of a
+        legal notice, here is a blessing:
+
+        *   May you do good and not evil.
+        *   May you find forgiveness for yourself and forgive others.
+        *   May you share freely, never taking more than you give.
+
+        ***********************************************************************
+
+        This is a bugfix test for the OPFS SAHPool VFS. It requires
+        setting up a database created using v3.49.0 or older, then
+        running it again with a newer version. In that case, the newer
+        version should be able to read the older version's db files
+        just fine. Revering back to the old version should also still
+        work - it should be able to read databases modified by the
+        newer version. However, a database _created_ by a version with
+        this fix will _not_ be legible by a version which predates
+        this fix, in which case the older version will see that VFS
+        file slot as corrupt and will clear it for re-use.
+
+        This is unfortunately rather cumbersome to test properly,
+        and essentially impossible to automate.
+      */
+      (function(){
+        'use strict';
+        document.querySelector('h1').innerHTML =
+        document.querySelector('title').innerHTML;
+        const mapToString = (v)=>{
+          switch(typeof v){
+            case 'number': case 'string': case 'boolean':
+            case 'undefined': case 'bigint':
+              return ''+v;
+            default: break;
+          }
+          if(null===v) return 'null';
+          if(v instanceof Error){
+            v = {
+              message: v.message,
+              stack: v.stack,
+              errorClass: v.name
+            };
+          }
+          return JSON.stringify(v,undefined,2);
+        };
+        const normalizeArgs = (args)=>args.map(mapToString);
+        const logTarget = document.querySelector('#test-output');
+        const logClass = function(cssClass,...args){
+          const ln = document.createElement('div');
+          if(cssClass){
+            for(const c of (Array.isArray(cssClass) ? cssClass : [cssClass])){
+              ln.classList.add(c);
+            }
+          }
+          ln.append(document.createTextNode(normalizeArgs(args).join(' ')));
+          logTarget.append(ln);
+        };
+        const cbReverse = document.querySelector('#cb-log-reverse');
+        //cbReverse.setAttribute('checked','checked');
+        const cbReverseKey = 'tester1:cb-log-reverse';
+        const cbReverseIt = ()=>{
+          logTarget.classList[cbReverse.checked ? 'add' : 'remove']('reverse');
+          //localStorage.setItem(cbReverseKey, cbReverse.checked ? 1 : 0);
+        };
+        cbReverse.addEventListener('change', cbReverseIt, true);
+        /*if(localStorage.getItem(cbReverseKey)){
+          cbReverse.checked = !!(+localStorage.getItem(cbReverseKey));
+          }*/
+        cbReverseIt();
+
+        const log = (...args)=>{
+          //console.log(...args);
+          logClass('',...args);
+        }
+        const warn = (...args)=>{
+          console.warn(...args);
+          logClass('warning',...args);
+        }
+        const error = (...args)=>{
+          console.error(...args);
+          logClass('error',...args);
+        };
+
+        const toss = (...args)=>{
+          error(...args);
+          throw new Error(args.join(' '));
+        };
+
+        const endOfWork = (passed=true)=>{
+          const eH = document.querySelector('#color-target');
+          const eT = document.querySelector('title');
+          if(passed){
+            log("End of work chain. If you made it this far, you win.");
+            eH.innerText = 'PASS: '+eH.innerText;
+            eH.classList.add('tests-pass');
+            eT.innerText = 'PASS: '+eT.innerText;
+          }else{
+            eH.innerText = 'FAIL: '+eH.innerText;
+            eH.classList.add('tests-fail');
+            eT.innerText = 'FAIL: '+eT.innerText;
+          }
+        };
+
+        log("Running opfs-sahpool digest tests...");
+        const W1 = new Worker('digest-worker.js?sqlite3.dir=../../../jswasm');
+        W1.onmessage = function({data}){
+          //log("onmessage:",data);
+          switch(data.type){
+            case 'log':
+              log('worker says:', ...data.payload);
+              break;
+            case 'error':
+              error('worker says:', ...data.payload);
+              endOfWork(false);
+              break;
+            case 'initialized':
+              log(data.workerId, ': Worker initialized',...data.payload);
+              break;
+          }
+        };
+      })();
+    </script>
+  </body>
+</html>

manifest

@@ -1,5 +1,5 @@
-C Add\s--asan-fsanitize=...\sconfigure\sflag\sto\sthe\scanonical\sbuild\sto\soptionally\sset\s-fsantize\sflags\sfor\sthe\sfuzzcheck-asan\stool.\sTeach\sproj-check-fsanitiz\sto\sfail\sfor\sflags\swhich\sthe\scompiler\semits\sany\swarning\sfor,\sfor\sreasons\sdescribed\sin\sits\scomments.
+C Fix\sa\slong-standing\sfilename\sdigest\scomputation\sbug\sin\sthe\sOPFS\sSAHPool\sVFS\swhich\scaused\sall\sVFS-stored\sfilenames\sto\shave\sa\sdigest\svalue\sof\s0.\sSee\s[/forumpost/042d53c928382021]\sand\sfor\sfull\sdetails.
-D 2025-03-16T13:09:21.699
+D 2025-03-16T14:05:42.209
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
@@ -645,7 +645,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js f646a65257973b8c4481f8a6a216370b85644
 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966
 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
-F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 16d80af915bfd2529824b999b304425503094eedf34fb113d0791e002b13e5cf
+F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 0f68a64e508598910e7c01214ae27d603dfc8baec6a184506fafac603a901931
 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 4ab0704ee198de7d1059eccedc7703c931510b588d10af0ee36ea5b3ebbac284
 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616
 F ext/wasm/api/sqlite3-wasm.c 6a4cd40267eaf08400895c5b9de39c56976c3b97b3c1bbe53fc2e80fa074e9c7
@@ -696,10 +696,12 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c
 F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
 F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
-F ext/wasm/tester1.c-pp.js f3a3cbf0207287c4caa9f84b4e2934804e4b5720c71eaf143f33c8122bd3c9f2
+F ext/wasm/tester1.c-pp.js 419717b16e12703487a7ccf3ea4e63d693bdfbf7657e55a7e6c559bbccf027d3
 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e
 F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88
 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
+F ext/wasm/tests/opfs/sahpool/digest-worker.js b0ab6218588f1f0a6d15a363b493ceaf29bfb87804d9e0165915a9996377cf79
+F ext/wasm/tests/opfs/sahpool/digest.html 206d08a34dc8bd570b2581d3d9ab3ecad3201b516a598dd096dcf3cf8cd81df8
 F ext/wasm/tests/opfs/sahpool/index.html be736567fd92d3ecb9754c145755037cbbd2bca01385e2732294b53f4c842328
 F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36e0f6991460fff0cb7c15079454679a4e2
 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61
@@ -2213,8 +2215,9 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P be3a2e631100b711996b9524a54fc604966513a62d83fc916270a6226da7adab
+P 013730e9b92af39cb7fd2871df9b4bc81b8990f918892bd79370704421672da0 500f2e6ec74b4c0e4ac0365ba4e0d81ed6df8dd09dc0f8af65d294c3453f8865
-R 699bb2f6a1869a511409b94aecd9b45c
+R 9266ed4d88e1ccecb16602d4a66164a0
+T +closed 500f2e6ec74b4c0e4ac0365ba4e0d81ed6df8dd09dc0f8af65d294c3453f8865 Closed\sby\sintegrate-merge.
 U stephan
-Z b465670c31108cfd99bca4e96066f044
+Z 8dfe9f7d9c954162ba84353f5fc2343c
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-013730e9b92af39cb7fd2871df9b4bc81b8990f918892bd79370704421672da0
+493cbe74504e8eb1ca8f2edf49fdab6bebc7fe36ffab06932a4b8c5a4eea86cd