Initial build of kvvfs in wasm. It loads but cannot find the VFS for as-yet-unknown reasons (sqlite3 shell works fine), and most APIs throw "null function or function signature mismatch" from deep within wasm, presumably as a side effect of the "missing" VFS.

FossilOrigin-Name: 1a2f24a0bdfc6eaae478916b8f4f9c6b63ead9964534fc2951fb4e995ffe61f1

ext/wasm/GNUmakefile

@@ -262,6 +262,7 @@ all: $(sqlite3.js)
 # End main Emscripten-based module build
 ########################################################################
 
+include kvvfs.make
 
 ########################################################################
 # fiddle_remote is the remote destination for the fiddle app. It

ext/wasm/kvvfs.make

@@ -0,0 +1,82 @@
+#!/usr/bin/make
+#^^^^ help emacs select makefile mode
+#
+# This is a sub-make for building a standalone kvvfs-based
+# sqlite3.wasm.  It is intended to be "include"d from the main
+# GNUMakefile.
+#
+# Notable potential TODOs:
+#
+# - Trim down a custom sqlite3-api.js for this build. We can elimate
+#   the jaccwabyt dependency, for example, because this build won't
+#   make use of the VFS bits. Similarly, we can eliminate or replace
+#   parts of the OO1 API, or provide a related API which manages
+#   singletons of the localStorage/sessionStorage instances.
+#
+########################################################################
+MAKEFILE.kvvfs := $(lastword $(MAKEFILE_LIST))
+
+kvvfs.js     := sqlite3-kvvfs.js
+kvvfs.wasm   := sqlite3-kvvfs.wasm
+kvvfs.wasm.c := $(dir.api)/sqlite3-wasm.c
+
+CLEAN_FILES += $(kvvfs.js) $(kvvfs.wasm)
+
+########################################################################
+# emcc flags for .c/.o/.wasm.
+kvvfs.flags =
+#kvvfs.flags += -v # _very_ loud but also informative about what it's doing
+
+########################################################################
+# emcc flags for .c/.o.
+kvvfs.cflags :=
+kvvfs.cflags += -std=c99 -fPIC
+kvvfs.cflags += -I. -I$(dir.top)
+kvvfs.cflags += -DSQLITE_OS_KV=1 $(SQLITE_OPT)
+
+########################################################################
+# emcc flags specific to building the final .js/.wasm file...
+kvvfs.jsflags := -fPIC
+kvvfs.jsflags += --no-entry
+kvvfs.jsflags += -sENVIRONMENT=web
+kvvfs.jsflags += -sMODULARIZE
+kvvfs.jsflags += -sSTRICT_JS
+kvvfs.jsflags += -sDYNAMIC_EXECUTION=0
+kvvfs.jsflags += -sNO_POLYFILL
+kvvfs.jsflags += -sEXPORTED_FUNCTIONS=@$(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api
+kvvfs.jsflags += -sEXPORTED_RUNTIME_METHODS=FS,wasmMemory,allocateUTF8OnStack
+                                            # wasmMemory ==> for -sIMPORTED_MEMORY
+                                            # allocateUTF8OnStack ==> kvvfs internals
+kvvfs.jsflags += -sUSE_CLOSURE_COMPILER=0
+kvvfs.jsflags += -sIMPORTED_MEMORY
+#kvvfs.jsflags += -sINITIAL_MEMORY=13107200
+#kvvfs.jsflags += -sTOTAL_STACK=4194304
+kvvfs.jsflags += -sEXPORT_NAME=sqlite3InitModule
+kvvfs.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr.
+kvvfs.jsflags += --post-js=$(post-js.js)
+#kvvfs.jsflags += -sFILESYSTEM=0 # only for experimentation. sqlite3 needs the FS API
+#                                Perhaps the kvvfs build doesn't?
+#kvvfs.jsflags += -sABORTING_MALLOC
+kvvfs.jsflags += -sALLOW_MEMORY_GROWTH
+kvvfs.jsflags += -sALLOW_TABLE_GROWTH
+kvvfs.jsflags += -Wno-limited-postlink-optimizations
+# ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag.
+kvvfs.jsflags += -sERROR_ON_UNDEFINED_SYMBOLS=0
+kvvfs.jsflags += -sLLD_REPORT_UNDEFINED
+#kvvfs.jsflags += --import-undefined
+kvvfs.jsflags += -sMEMORY64=0
+ifneq (0,$(enable_bigint))
+kvvfs.jsflags += -sWASM_BIGINT
+endif
+
+$(kvvfs.js): $(MAKEFILE) $(MAKEFILE.kvvfs) $(kvvfs.wasm.c) \
+    EXPORTED_FUNCTIONS.api \
+    $(post-js.js)
+	$(emcc.bin) -o $@ $(emcc_opt) $(emcc.flags) $(kvvfs.cflags) $(kvvfs.jsflags) $(kvvfs.wasm.c)
+	chmod -x $(kvvfs.wasm)
+ifneq (,$(wasm-strip))
+	$(wasm-strip) $(kvvfs.wasm)
+endif
+	@ls -la $@ $(kvvfs.wasm)
+
+kvvfs: $(kvvfs.js)

ext/wasm/kvvfs1.html

@@ -0,0 +1,34 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
+    <link rel="stylesheet" href="common/emscripten.css"/>
+    <link rel="stylesheet" href="common/testing.css"/>
+    <title>sqlite3-kvvfs.js tests</title>
+  </head>
+  <body>
+    <header id='titlebar'><span>sqlite3-kvvfs.js tests</span></header>
+    <!-- emscripten bits -->
+    <figure id="module-spinner">
+      <div class="spinner"></div>
+      <div class='center'><strong>Initializing app...</strong></div>
+      <div class='center'>
+        On a slow internet connection this may take a moment.  If this
+        message displays for "a long time", intialization may have
+        failed and the JavaScript console may contain clues as to why.
+      </div>
+    </figure>
+    <div class="emscripten" id="module-status">Downloading...</div>
+    <div class="emscripten">
+      <progress value="0" max="100" id="module-progress" hidden='1'></progress>  
+    </div><!-- /emscripten bits -->
+    <div>Everything on this page happens in the dev console.</div>
+    <hr>
+    <div id='test-output'></div>
+    <script src="sqlite3-kvvfs.js"></script>
+    <script src="common/SqliteTestUtil.js"></script>
+    <script src="kvvfs1.js"></script>
+  </body>
+</html>

ext/wasm/kvvfs1.js

@@ -0,0 +1,52 @@
+/*
+  2022-09-12
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  A basic test script for sqlite3-kvvfs.wasm. This file must be run in
+  main JS thread and sqlite3-kvvfs.js must have been loaded before it.
+*/
+'use strict';
+(function(){
+  const T = self.SqliteTestUtil;
+  const toss = function(...args){throw new Error(args.join(' '))};
+  const debug = console.debug.bind(console);
+  const eOutput = document.querySelector('#test-output');
+  const log = console.log.bind(console)
+  const logHtml = function(...args){
+    log.apply(this, args);
+    const ln = document.createElement('div');
+    ln.append(document.createTextNode(args.join(' ')));
+    eOutput.append(ln);
+  };
+
+  const runTests = function(Module){
+    //log("Module",Module);
+    const sqlite3 = Module.sqlite3,
+          capi = sqlite3.capi,
+          oo = sqlite3.oo1,
+          wasm = capi.wasm;
+    log("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
+    log("Build options:",wasm.compileOptionUsed());
+    self.S = sqlite3;
+
+    log("vfs(null) =",capi.sqlite3_vfs_find(null))
+    log("vfs('kvvfs') =",capi.sqlite3_vfs_find('kvvfs'))
+    //const db = new oo.DB("session");
+    
+    log("Init done. Proceed from the dev console.");
+  };
+
+  sqlite3InitModule(self.sqlite3TestModule).then(function(theModule){
+    console.warn("Installing Emscripten module as global EM for dev console access.");
+    self.EM = theModule;
+    runTests(theModule);
+  });
+})();

manifest

@@ -1,5 +1,5 @@
-C Improved\scomments\sin\sos_kv.c.\s\sBetter\snames\sfor\sthe\skey/value\sname\sspaces.
+C Initial\sbuild\sof\skvvfs\sin\swasm.\sIt\sloads\sbut\scannot\sfind\sthe\sVFS\sfor\sas-yet-unknown\sreasons\s(sqlite3\sshell\sworks\sfine),\sand\smost\sAPIs\sthrow\s"null\sfunction\sor\sfunction\ssignature\smismatch"\sfrom\sdeep\swithin\swasm,\spresumably\sas\sa\sside\seffect\sof\sthe\s"missing"\sVFS.
-D 2022-09-10T18:38:21.976
+D 2022-09-12T12:39:28.819
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -474,7 +474,7 @@ F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
 F ext/wasm/EXPORTED_FUNCTIONS.fiddle 7fb73f7150ab79d83bb45a67d257553c905c78cd3d693101699243f36c5ae6c3
 F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02
-F ext/wasm/GNUmakefile 5359a37fc13b68fad2259228590450339a0c59687744edd0db7bb93d3b1ae2b1
+F ext/wasm/GNUmakefile 12a672ab9125dc860457c2853f7651b98517e424d7a0e9714c89b28c5ff73800
 F ext/wasm/README.md 4b00ae7c7d93c4591251245f0996a319e2651361013c98d2efb0b026771b7331
 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api c5eaceabb9e759aaae7d3101a4a3e542f96ab2c99d89a80ce20ec18c23115f33
 F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287
@@ -502,6 +502,9 @@ F ext/wasm/jaccwabyt/jaccwabyt.js 99b424b4d467d4544e82615b58e2fe07532a898540bf9d
 F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106
 F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f
 F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19
+F ext/wasm/kvvfs.make 7cc9cf10e744c3ba523c3eaf5c4af47028f3a5bb76db304ea8044a9b2a9d496f
+F ext/wasm/kvvfs1.html 2acb241a6110a4ec581adbf07a23d5fc2ef9c7142aa9d60856732a102abc5016
+F ext/wasm/kvvfs1.js 461d571234244bd682725f1dd73195b1d9a7eabdd57c0181ff41dc7c55252313
 F ext/wasm/testing1.html 0bf3ff224628c1f1e3ed22a2dc1837c6c73722ad8c0ad9c8e6fb9e6047667231
 F ext/wasm/testing1.js cba7134901a965743fa9289d82447ab71de4690b1ee5d06f6cb83e8b569d7943
 F ext/wasm/testing2.html 73e5048e666fd6fb28b6e635677a9810e1e139c599ddcf28d687c982134b92b8
@@ -571,7 +574,7 @@ F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6
 F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9
 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
-F src/os_kv.c c658ddfb433b982afb3d12d133117e8db9eee11af33db41c30a6f30375f9b99a
+F src/os_kv.c a188e92dac693b1c1b512d93b0c4dc85c1baad11e322b01121f87057996e4d11
 F src/os_setup.h 0711dbc4678f3ac52d7fe736951b6384a0615387c4ba5135a4764e4e31f4b6a6
 F src/os_unix.c 102f7e5c5b59c18ea3dbc929dc3be8acb3afc0e0b6ad572e032335c9c27f44f1
 F src/os_win.c e9454cb141908e8eef2102180bad353a36480612d5b736e4c2bd5777d9b25a34
@@ -2001,8 +2004,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f6632e69c2ec1a7ddc4e51f3567e3bc082ee94a6dd51fdafdc0c3bf386a32d4c
+P 8e1652a3856765c9146cc2a7e3b598a2dc7dc84c556b35eab9a35184514384b9
-R 8cd4482ba17a8f6c143ea61b8462f46c
+R 87fbf09a4dec22281b7328e04378fb76
-U drh
+U stephan
-Z 81846bec91bdcab04bc60c9c032d039e
+Z bc0b4e3041260a10b10ef1c5f11f1ba3
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-8e1652a3856765c9146cc2a7e3b598a2dc7dc84c556b35eab9a35184514384b9
+1a2f24a0bdfc6eaae478916b8f4f9c6b63ead9964534fc2951fb4e995ffe61f1

src/os_kv.c

@@ -166,10 +166,6 @@ static sqlite3_io_methods kvvfs_jrnl_io_methods = {
 /* Forward declarations for the low-level storage engine
 */
 #define KVSTORAGE_KEY_SZ  32
-static int kvstorageWrite(const char*, const char *zKey, const char *zData);
-static int kvstorageDelete(const char*, const char *zKey);
-static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf);
-
 
 /* Expand the key name with an appropriate prefix and put the result
 ** zKeyOut[].  The zKeyOut[] buffer is assumed to hold at least
@@ -183,6 +179,207 @@ static void kvstorageMakeKey(
   sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn);
 }
 
+#ifdef __EMSCRIPTEN__
+/* Provide Emscripten-based impls of kvstorageWrite/Read/Delete()... */
+#include <emscripten.h>
+#include <emscripten/console.h>
+
+/*
+** WASM_KEEP is identical to EMSCRIPTEN_KEEPALIVE but is not
+** Emscripten-specific. It explicitly includes marked functions for
+** export into the target wasm file without requiring explicit listing
+** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list
+** (or equivalent in other build platforms). Any function with neither
+** this attribute nor which is listed as an explicit export will not
+** be exported from the wasm file (but may still be used internally
+** within the wasm file).
+**
+** The functions in this file (sqlite3-wasm.c) which require exporting
+** are marked with this flag. They may also be added to any explicit
+** build-time export list but need not be. All of these APIs are
+** intended for use only within the project's own JS/WASM code, and
+** not by client code, so an argument can be made for reducing their
+** visibility by not including them in any build-time export lists.
+**
+** 2022-09-11: it's not yet _proven_ that this approach works in
+** non-Emscripten builds. If not, such builds will need to export
+** those using the --export=... wasm-ld flag (or equivalent). As of
+** this writing we are tied to Emscripten for various reasons
+** and cannot test the library with other build environments.
+*/
+#define WASM_KEEP __attribute__((used,visibility("default")))
+/*
+** An internal level of indirection for accessing the static
+** kvstorageMakeKey() from EM_JS()-generated functions. This must be
+** made available for export via Emscripten but is not intended to be
+** used from client code. If called with a NULL zKeyOut it is a no-op.
+** It returns KVSTORAGE_KEY_SZ, so JS code (which cannot see that
+** constant) may call it with NULL arguments to get the size of the
+** allocation they'll need for a kvvfs key.
+**
+** Maintenance reminder: Emscripten will install this in the Module
+** init scope and will prefix its name with "_".
+*/
+WASM_KEEP
+int sqlite3_wasm__kvvfsMakeKey(const char *zClass, const char *zKeyIn,
+                               char *zKeyOut){
+  if( 0!=zKeyOut ) kvstorageMakeKey(zClass, zKeyIn, zKeyOut);
+  return KVSTORAGE_KEY_SZ;
+}
+/*
+** Internal helper for kvstorageWrite/Read/Delete() which creates a
+** storage key for the given zClass/zKeyIn combination. Returns a
+** pointer to the key: a C string allocated on the WASM stack, or 0 if
+** allocation fails. It is up to the caller to save/restore the stack
+** before/after this operation.
+*/
+EM_JS(const char *, kvstorageMakeKeyOnJSStack,
+      (const char *zClass, const char *zKeyIn),{
+  if( 0==zClass || 0==zKeyIn) return 0;
+  const zXKey = stackAlloc(_sqlite3_wasm__kvvfsMakeKey(0,0,0));
+  if(zXKey) _sqlite3_wasm__kvvfsMakeKey(zClass, zKeyIn, zXKey);
+  return zXKey;
+});
+
+/*
+** JS impl of kvstorageWrite(). Main docs are in the C impl. This impl
+** writes zData to the global sessionStorage (if zClass starts with
+** 's') or localStorage, using a storage key derived from zClass and
+** zKey.
+*/
+EM_JS(int, kvstorageWrite,
+      (const char *zClass, const char *zKey, const char *zData),{
+  const stack = stackSave();
+  try {
+    const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey);
+    if(!zXKey) return 1/*OOM*/;
+    const jKey = UTF8ToString(zXKey);
+    /**
+       We could simplify this function and eliminate the
+       kvstorageMakeKey() symbol acrobatics if we'd simply hard-code
+       the key algo into the 3 functions which need it:
+
+       const jKey = "kvvfs-"+UTF8ToString(zClass)+"-"+UTF8ToString(zKey);
+    */
+    ((115/*=='s'*/===getValue(zClass))
+     ? sessionStorage : localStorage).setItem(jKey, UTF8ToString(zData));
+  }catch(e){
+    console.error("kvstorageWrite()",e);
+    return 1; // Can't access SQLITE_xxx from here
+  }finally{
+    stackRestore(stack);
+  }
+  return 0;
+});
+
+/*
+** JS impl of kvstorageDelete(). Main docs are in the C impl. This
+** impl generates a key derived from zClass and zKey, and removes the
+** matching entry (if any) from global sessionStorage (if zClass
+** starts with 's') or localStorage.
+*/
+EM_JS(int, kvstorageDelete,
+      (const char *zClass, const char *zKey),{
+  const stack = stackSave();
+  try {
+    const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey);
+    if(!zXKey) return 1/*OOM*/;
+    _sqlite3_wasm__kvvfsMakeKey(zClass, zKey, zXKey);
+    const jKey = UTF8ToString(zXKey);
+    ((115/*=='s'*/===getValue(zClass))
+     ? sessionStorage : localStorage).removeItem(jKey);
+  }catch(e){
+    console.error("kvstorageDelete()",e);
+    return 1;
+  }finally{
+    stackRestore(stack);
+  }
+  return 0;
+});
+
+/*
+** JS impl of kvstorageRead(). Main docs are in the C impl. This impl
+** reads its data from the global sessionStorage (if zClass starts
+** with 's') or localStorage, using a storage key derived from zClass
+** and zKey.
+*/
+EM_JS(int, kvstorageRead,
+      (const char *zClass, const char *zKey, char *zBuf, int nBuf),{
+  const stack = stackSave();
+  try {
+    const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey);
+    if(!zXKey) return -3/*OOM*/;
+    const jKey = UTF8ToString(zXKey);
+    const jV = ((115/*=='s'*/===getValue(zClass))
+                ? sessionStorage : localStorage).getItem(jKey);
+    if(!jV) return -1;
+    const nV = jV.length /* Note that we are relying 100% on v being
+                            ASCII so that jV.length is equal to the
+                            C-string's byte length. */;
+    if(nBuf<=0) return nV;
+    else if(1===nBuf){
+      setValue(zBuf, 0);
+      return nV;
+    }
+    const zV = allocateUTF8OnStack(jV);
+    if(nBuf > nV + 1) nBuf = nV + 1;
+    HEAPU8.copyWithin(zBuf, zV, zV + nBuf - 1);
+    setValue( zBuf + nBuf - 1, 0 );
+    return nBuf - 1;
+  }catch(e){
+    console.error("kvstorageRead()",e);
+    return -2;
+  }finally{
+    stackRestore(stack);
+  }
+});
+
+/*
+** This function exists for (1) WASM testing purposes and (2) as a
+** hook to get Emscripten to export several EM_JS()-generated
+** functions (if we don't reference them from exported C functions
+** then they get stripped away at build time). It is not part of the
+** public API and its signature and semantics may change at any time.
+** It's not even part of the private API, for that matter - it's part
+** of the Emscripten C/JS/WASM glue.
+*/
+WASM_KEEP
+int sqlite3__wasm_emjs_kvvfs(int whichOp){
+  int rc = 0;
+  const char * zClass =
+    "sezzion" /*don't collide with "session" records!*/;
+  const char * zKey = "hello";
+  switch( whichOp ){
+    case 0: break;
+    case 1:
+      kvstorageWrite(zClass, zKey, "world");
+      break;
+    case 2: {
+      char buffer[128] = {0};
+      char * zBuf = &buffer[0];
+      rc = kvstorageRead(zClass, zKey, zBuf, (int)sizeof(buffer));
+      emscripten_console_logf("kvstorageRead()=%d %s\n", rc, zBuf);
+      break;
+    }
+    case 3:
+      kvstorageDelete(zClass, zKey);
+      break;
+    case 4:
+      kvstorageMakeKeyOnJSStack(0,0);
+      break;
+    default: break;
+  }
+  return rc;
+}
+
+#undef WASM_KEEP
+#else /* end ifdef __EMSCRIPTEN__ */
+/* Forward declarations for the low-level storage engine
+*/
+static int kvstorageWrite(const char*, const char *zKey, const char *zData);
+static int kvstorageDelete(const char*, const char *zKey);
+static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf);
+
 /* Write content into a key.  zClass is the particular namespace of the
 ** underlying key/value store to use - either "local" or "session".
 **
@@ -277,6 +474,7 @@ static int kvstorageRead(
     return (int)n;
   }
 }
+#endif /* ifdef __EMSCRIPTEN__ */
 
 /****** Utility subroutines ************************************************/