kvvfs docs and 64-bit fixes.

FossilOrigin-Name: cf58e17fa2dc2e183a6ea1d41795c701efb303c9b378aa9b90953c9b568c621a
2026-10-28 10:57:26 +03:00 · 2025-11-30 18:43:59 +00:00
parent 4694b91988
commit 8ff58b4f41
5 changed files with 151 additions and 83 deletions
--- a/ext/wasm/GNUmakefile
+++ b/ext/wasm/GNUmakefile
@@ -1267,6 +1267,8 @@ speedtest1: b-speedtest1
 #
 # Generate 64-bit variants of speedtest1*.{js,html}
 #
+# TODO: preprocess these like we do the rest.
+#
 define gen-st64
 $(2): $(1)
 	@$$(call b.echo,speedtest164,$$(emo.disk)$(emo.lock) Creating from $$<)
--- a/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js
+++ b/ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js
@@ -115,21 +115,66 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
  );
  util.assert( 32<=kvvfsMethods.$nKeySize, "unexpected kvvfsMethods.$nKeySize: "+kvvfsMethods.$nKeySize);

+  /**
+     Most of the VFS-internal state.
+   */
  const cache = Object.assign(Object.create(null),{
-    fixedPageSize: 8192/*used in some validation*/,
+    /**
+       Bug: kvvfs is currently fixed at a page size of 8kb because
+       changing it is leaving corrupted dbs for reasons as yet
+       unknown. This value is used in certain validation to ensure
+       that if that limitation is lifted, it will break potentially
+       affected code.
+    */
+    fixedPageSize: 8192,
+    /** Regex matching journal file names. */
    rxJournalSuffix: /-journal$/,
+    /** Frequently-used C-string. */
    zKeyJrnl: wasm.allocCString("jrnl"),
+    /** Frequently-used C-string. */
    zKeySz: wasm.allocCString("sz"),
+    /**
+       The maximum size of a kvvfs record key. It is historically only
+       32, a limitation currently retained only because it's convenient to
+       do so (the underlying code has outgrown the need for the artifically
+       low limit).
+
+       We cache this value here because the end of this init code will
+       dispose of kvvfsMethods, invalidating it.
+    */
    keySize: kvvfsMethods.$nKeySize,
+    /**
+       WASM heap memory buffers to optimize out some frequent
+       allocations.
+    */
    buffer: Object.assign(Object.create(null),{
+      /**
+         The size of each buffer in this.pool.
+
+         kvvfsMethods.$nBufferSize is slightly larger than the output
+         space needed for a kvvfs-encoded 64kb db page in a worse-cast
+         encoding (128kb). It is not suitable for arbitrary buffer
+         use, only page de/encoding.  As the VFS system has no hook
+         into library finalization, these buffers are effectively
+         leaked except in the few places which use memBufferFree().
+      */
      n: kvvfsMethods.$nBufferSize,
+      /**
+         Map of buffer ids to wasm.alloc()'d pointers of size
+         this.n. (Re)used by various internals.
+
+         Buffer ids 0 and 1 are used in the API internals.  Other
+         names are used in higher-level APIs.
+
+         See memBuffer() and memBufferFree().
+      */
      pool: Object.create(null)
    })
  });

  /**
-     A wasm.alloc()'d buffer large enough for all kvvfs
-     encoding/decoding needs (cache.buffer.n).
+     Returns a (cached) wasm.alloc()'d buffer of cache.buffer.n size,
+     throwing on OOM.

     We leak this one-time alloc because we've no better option.
     sqlite3_vfs does not have a finalizer, so we've no place to hook
@@ -139,7 +184,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
  */
  cache.memBuffer = (id=0)=>cache.buffer.pool[id] ??= wasm.alloc(cache.buffer.n);

-  /** Freeds the buffer with the given id. */
+  /** Frees the buffer with the given id. */
  cache.memBufferFree = (id)=>{
    const b = cache.buffer.pool[id];
    if( b ){
@@ -148,19 +193,21 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
    }
  };

+  const noop = ()=>{};
  const debug = sqlite3.__isUnderTest
-        ? function(){sqlite3.config.debug("kvvfs:", ...arguments)}
-        : function(){};
-  const warn = function(){sqlite3.config.warn("kvvfs:", ...arguments)};
-  const error = function(){sqlite3.config.error("kvvfs:", ...arguments)};
+        ? (...args)=>sqlite3.config.debug("kvvfs:", ...args)
+        : noop;
+  const warn = (...args)=>sqlite3.config.warn("kvvfs:", ...args);
+  const error = (...args)=>sqlite3.config.error("kvvfs:", ...args);

  /**
     Implementation of JS's Storage interface for use as backing store
     of the kvvfs. Storage is a native class and its constructor
     cannot be legally called from JS, making it impossible to
     directly subclass Storage. This class implements (only) the
-     Storage interface, however, to make it a drop-in replacement for
-     localStorage/sessionStorage.
+     Storage interface, to make it a drop-in replacement for
+     localStorage/sessionStorage. (Any behavioral discrepancies are to
+     be considered bugs.)

     This impl simply proxies a plain, prototype-less Object, suitable
     for JSON-ing.
@@ -266,26 +313,31 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){

  /**
     Create a new instance of the objects which go into
-     cache.storagePool.
+     cache.storagePool, with a refcount of 1. If passed a Storage-like
+     object as its second argument, it is used for the storage,
+     otherwise it creates a new KVVfsStorage object.
  */
-  const newStorageObj = (name,storage)=>Object.assign(Object.create(null),{
+  const newStorageObj = (name,storage=undefined)=>Object.assign(Object.create(null),{
    /**
       JS string value of this KVVfsFile::$zClass. i.e. the storage's
       name.
    */
    jzClass: name,
    /**
-       Refcount to keep dbs and journals pointing to the same storage
-       for the life of both. Managed by xOpen() and xClose().
+       Refcount. This keeps dbs and journals pointing to the same
+       storage for the life of both and enables kvvfs to behave more
+       like a conventional filesystem (a stepping stone towards
+       downstream API goals). Managed by xOpen() and xClose().
    */
    refc: 1,
    /**
-       deleteAtRefc0 objects will be removed by xClose() when refc
-       reaches 0. The others will persist, to give the illusion of
-       real back-end storage. Managed by xOpen(). By default this is
-       false but the delete-on-close=1 flag can be used to set this to
-       true.
-     */
+       If true, this storage will be removed by xClose() or
+       sqlite3_js_kvvfs_unlink() when refc reaches 0. The others will
+       persist when refc==0, to give the illusion of real back-end
+       storage. Managed by xOpen() and sqlite3_js_kvvfs_reserve(). By
+       default this is false but the delete-on-close=1 flag can be
+       used to set this to true.
+    */
    deleteAtRefc0: false,
    /**
       The backing store. Must implement the Storage interface.
@@ -294,10 +346,10 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
    /**
       The storage prefix used for kvvfs keys.  It is
       "kvvfs-STORAGENAME-" for local/session storage and an empty
-       string for transient storage. local/session storage must use
-       the long form (A) for backwards compatibility and (B) so that
-       kvvfs can coexist with non-db client data in those backends.
-       Neither (A) nor (B) are concerns for KVVfsStorage objects.
+       string for other storage. local/session storage must use the
+       long form (A) for backwards compatibility and (B) so that kvvfs
+       can coexist with non-db client data in those backends.  Neither
+       (A) nor (B) are concerns for KVVfsStorage objects.

       This prefix mirrors the one generated by os_kv.c's
       kvrecordMakeKey() and must stay in sync with that one.
@@ -310,14 +362,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
    files: [],
    /**
       If set, it's an array of objects with various event
-       callbacks. See sqlite3_js_kvvfs_listen().
+       callbacks. See sqlite3_js_kvvfs_listen(). When there are no
+       listeners, this member is set to undefined (instead of an empty
+       array) to allow us to more easily optimize out calls to
+       notifyListeners() for the common case of no listeners.
    */
    listeners: undefined
  });

  /**
-     Deletes the cache.storagePool entries for store and its
-     db/journal counterpart.
+     Deletes the cache.storagePool entries for store (a
+     cache.storagePool entry) and its db/journal counterpart.
  */
  const deleteStorage = function(store){
    const other = cache.rxJournalSuffix.test(store.jzClass)
@@ -402,6 +457,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
    return e;
  };

+  /** Exception handler for notifyListeners(). */
  const catchForNotify = (e)=>{
    warn("kvvfs.listener handler threw:",e);
  };
@@ -425,58 +481,63 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
     xFileControl().
  */
  const notifyListeners = async function(eventName,store,...args){
-    if( store.listeners ){
-      //cache.rxPageNoSuffix ??= /(\d+)$/;
-      if( store.keyPrefix && args[0] ){
-        args[0] = args[0].replace(store.keyPrefix,'');
-      }
-      let u8enc, z0, z1, wcache;
-      for(const ear of store.listeners){
-        const ev = Object.create(null);
-        ev.storageName = store.jzClass;
-        ev.type = eventName;
-        const decodePages = ear.decodePages;
-        const f = ear.events[eventName];
-        if( f ){
-          if( !ear.includeJournal && args[0]==='jrnl' ){
-            continue;
-          }
-          if( 'write'===eventName && ear.decodePages && +args[0]>0 ){
-            /* Decode pages to Uint8Array. */
-            ev.data = [args[0]];
-            if( wcache?.[args[0]] ){
-              ev.data[1] = wcache[args[0]];
+    try{
+      if( store.listeners ){
+        //cache.rxPageNoSuffix ??= /(\d+)$/;
+        if( store.keyPrefix && args[0] ){
+          args[0] = args[0].replace(store.keyPrefix,'');
+        }
+        let u8enc, z0, z1, wcache;
+        for(const ear of store.listeners){
+          const ev = Object.create(null);
+          ev.storageName = store.jzClass;
+          ev.type = eventName;
+          const decodePages = ear.decodePages;
+          const f = ear.events[eventName];
+          if( f ){
+            if( !ear.includeJournal && args[0]==='jrnl' ){
              continue;
            }
-            u8enc ??= new TextEncoder('utf-8');
-            z0 ??= cache.memBuffer(10);
-            z1 ??= cache.memBuffer(11);
-            const u = u8enc.encode(args[1]);
-            const heap = wasm.heap8u();
-            heap.set(u, z0);
-            heap[wasm.ptr.add(z0, u.length)] = 0;
-            const rc = kvvfsDecode(z0, z1, cache.buffer.n);
-            if( rc>0 ){
-              wcache ??= Object.create(null);
-              wcache[args[0]]
-                = ev.data[1]
-                = heap.slice(z1, wasm.ptr.add(z1,rc));
+            if( 'write'===eventName && ear.decodePages && +args[0]>0 ){
+              /* Decode pages to Uint8Array, caching the result in
+                 wcache in case we have more listeners. */
+              ev.data = [args[0]];
+              if( wcache?.[args[0]] ){
+                ev.data[1] = wcache[args[0]];
+                continue;
+              }
+              u8enc ??= new TextEncoder('utf-8');
+              z0 ??= cache.memBuffer(10);
+              z1 ??= cache.memBuffer(11);
+              const u = u8enc.encode(args[1]);
+              const heap = wasm.heap8u();
+              heap.set(u, Number(z0));
+              heap[wasm.ptr.addn(z0, u.length)] = 0;
+              const rc = kvvfsDecode(z0, z1, cache.buffer.n);
+              if( rc>0 ){
+                wcache ??= Object.create(null);
+                wcache[args[0]]
+                  = ev.data[1]
+                  = heap.slice(Number(z1), wasm.ptr.addn(z1,rc));
+              }else{
+                continue;
+              }
            }else{
-              continue;
+              ev.data = args.length
+                ? ((args.length===1) ? args[0] : args)
+                : undefined;
+            }
+            try{f(ev)?.catch?.(catchForNotify)}
+            catch(e){
+              warn("notifyListeners [",store.jzClass,"]",eventName,e);
            }
-          }else{
-            ev.data = args.length
-              ? ((args.length===1) ? args[0] : args)
-              : undefined;
-          }
-          try{f(ev)?.catch?.(catchForNotify)}
-          catch(e){
-            warn("notifyListeners [",store.jzClass,"]",eventName,e);
          }
        }
      }
+    }catch(e){
+      catchForNotify(e);
    }
-  };
+  }/*notifyListeners()*/;

  /**
     Returns the storage object mapped to the given string zClass
@@ -1223,7 +1284,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
                util.toss3(capi.SQLITE_ERROR,"Unexpected decoded page size:",nDec);
              }
              //debug("Decoded",nDec,"page bytes");
-              pages[kk] = heap.slice(zDec, wasm.ptr.add(zDec, nDec));
+              pages[kk] = heap.slice(Number(zDec), wasm.ptr.addn(zDec, nDec));
            }else{
              pages[kk] = s.getItem(k);
            }
@@ -1325,8 +1386,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
          /* Copy u to the heap and encode the heap copy via C. This
             is _presumably_ faster than porting the encoding algo to
             JS. */
-          heap.set(u, zBin);
-          heap[wasm.ptr.add(zBin,n)] = 0;
+          heap.set(u, Number(zBin));
+          heap[wasm.ptr.addn(zBin,n)] = 0;
          const rc = kvvfsEncode(zBin, n, zEnc);
          util.assert( rc < cache.buffer.n,
                       "Impossibly long output - possibly smashed the heap" );
--- a/ext/wasm/index.html
+++ b/ext/wasm/index.html
@@ -105,6 +105,11 @@
              (<a href='speedtest1-worker.html?size=15'>32-bit</a>,
              <a href='speedtest1-worker-64bit.html?size=15'>64-bit</a>):
              an interactive Worker-thread variant of speedtest1.</li>
+            <li>speedtest1-worker?vfs=kvvfs
+              (<a href='speedtest1-worker.html?vfs=kvvfs&size=10'>32-bit</a>,
+              <a href='speedtest1-worker-64bit.html?vfs=kvvfs&size=10'>64-bit</a>):
+              speedtest1-worker with the
+              kvvfs VFS preselected and configured for a moderate workload.</li>
            <li>speedtest1-worker?vfs=opfs
              (<a href='speedtest1-worker.html?vfs=opfs&size=10'>32-bit</a>,
              <a href='speedtest1-worker-64bit.html?vfs=opfs&size=10'>64-bit</a>):
--- a/16
+++ b/16
@@ -1,5 +1,5 @@
-C Optimize\sout\sa\skvvfs\sevent\snotification\scall\sin\sthe\scommon\scase\swhere\sthere\sare\sno\slisteners.
-D 2025-11-30T16:37:47.709
+C kvvfs\sdocs\sand\s64-bit\sfixes.
+D 2025-11-30T18:43:59.989
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -578,7 +578,7 @@ F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009e
 F ext/session/sqlite3session.c b3de195ce668cace9b324599bf6255a70290cbfb5451e826e946f3aee6e64c54
 F ext/session/sqlite3session.h 7404723606074fcb2afdc6b72c206072cdb2b7d8ba097ca1559174a80bc26f7a
 F ext/session/test_session.c 8766b5973a6323934cb51248f621c3dc87ad2a98f023c3cc280d79e7d78d36fb
-F ext/wasm/GNUmakefile 2fe52720144ab481755c6df45f46dcf6c540f8622a5387ed280f1d0b26a6d28f
+F ext/wasm/GNUmakefile 0b40ca7fc6310a1375b813dde2b26b549e6a650c10045418fe8440619a3e826c
 F ext/wasm/README-dist.txt f01081a850ce38a56706af6b481e3a7878e24e42b314cfcd4b129f0f8427066a
 F ext/wasm/README.md 2e87804e12c98f1d194b7a06162a88441d33bb443efcfe00dc6565a780d2f259
 F ext/wasm/SQLTester/GNUmakefile e0794f676d55819951bbfae45cc5e8d7818dc460492dc317ce7f0d2eca15caff
@@ -600,7 +600,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 1041dd645e8e821c082b628cd8d9acf70c667
 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
 F ext/wasm/api/sqlite3-opfs-async-proxy.js 9654b565b346dc609b75d15337f20acfa7af7d9d558da1afeb9b6d8eaa404966
 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d
-F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js 278c7882968b01e9fc8269ae94f725e85acce1dc0a38e91b71397a17d8c1876e
+F ext/wasm/api/sqlite3-vfs-kvvfs.c-pp.js bc83c21b2211f1bcaa2b200a08411d86962426284987a39477543bc45bd64260
 F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js a2eea6442556867b589e04107796c6e1d04a472219529eeb45b7cd221d7d048b
 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 88ce2078267a2d1af57525a32d896295f4a8db7664de0e17e82dc9ff006ed8d3
 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js 366596d8ff73d4cefb938bbe95bc839d503c3fab6c8335ce4bf52f0d8a7dee81
@@ -627,7 +627,7 @@ F ext/wasm/fiddle/fiddle-worker.js 7798af02e672e088ff192716f80626c8895e19301a65b
 F ext/wasm/fiddle/fiddle.js 84fd75967e0af8b69d3dd849818342227d0f81d13db92e0dcbc63649b31a4893
 F ext/wasm/fiddle/index.c-pp.html 72c7e5517217960b3809648429ea396a7cbad0ffb2c92f6a2f5703abecb27317
 F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73607dc54d290
-F ext/wasm/index.html 54e27db740695ab2cb296e02d42c4c66b3f11b65797340d19fa6590f5b287da1
+F ext/wasm/index.html 475bc283338749db4e3fbf24cf3f5aa020cc85a1fffb780d400a915fcb5f1756
 F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b
 F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41
 F ext/wasm/mkdist.sh 64d53f469c823ed311f6696f69cec9093f745e467334b34f5ceabdf9de3c5b28 x
@@ -2180,8 +2180,8 @@ F tool/version-info.c 33d0390ef484b3b1cb685d59362be891ea162123cea181cb8e6d2cf6dd
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 7f0eca9d0aeb49c86a785ae930d235902bbd0f15877cc8f6083daac8efb2d1c1
-R bd382764c774d080ec17e2301611eab2
+P 8405c19d32f1e8b7273953a038f8bdfd4ea1a6548300bac5421cdf6fc6840285
+R 2f3b7cfa7869c544f8a6af729f308175
 U stephan
-Z 249c02695d60433ade4ac626997fb17e
+Z d109a9becc744b2aad4397bfe3d32fc6
 # Remove this line to create a well-formed Fossil manifest.
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-8405c19d32f1e8b7273953a038f8bdfd4ea1a6548300bac5421cdf6fc6840285
+cf58e17fa2dc2e183a6ea1d41795c701efb303c9b378aa9b90953c9b568c621a