Refactor os_kv.c so that the kvvfs read/write/delete methods can be swapped out at runtime by JS implementations. This eliminates the kvvfs dependency on Emscripten. Checkin part 1 of 2, to account for cherrypicking.

FossilOrigin-Name: ea370b9b05f7ed7eaa154ba58019f6642217eabc18517e721567adf948d93980

manifest

@@ -1,5 +1,5 @@
-C Add\s-DSQLITE_OS_KV_OPTIONAL\sto\sspeedtest1\sflags\sfor\skvvfs\stesting\spurposes.
+C Refactor\sos_kv.c\sso\sthat\sthe\skvvfs\sread/write/delete\smethods\scan\sbe\sswapped\sout\sat\sruntime\sby\sJS\simplementations.\sThis\seliminates\sthe\skvvfs\sdependency\son\sEmscripten.\sCheckin\spart\s1\sof\s2,\sto\saccount\sfor\scherrypicking.
-D 2022-10-09T11:42:12.888
+D 2022-10-09T13:19:27.791
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -600,7 +600,7 @@ F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6
 F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9
 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
-F src/os_kv.c 88872a68ba37e4ed4306eb6fa78d559dcc675761ce1f6458ba710e1c10a2d667
+F src/os_kv.c c3d991bfb6207560d3416bbf241cade7113e4ce852781c53b7e40a3d6ac25fcb
 F src/os_setup.h 0711dbc4678f3ac52d7fe736951b6384a0615387c4ba5135a4764e4e31f4b6a6
 F src/os_unix.c 287aa5f5691a2b356780c63e83abaa33549add84227b8313395f04088486d79c
 F src/os_win.c 8d129ae3e59e0fa900e20d0ad789e96f2e08177f0b00b53cdda65c40331e0902
@@ -2030,8 +2030,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2625b7cfe1640c1d7e779ec1f37db970541598c0dc3e22e5eecf3c772d95ad40
+P 92b500da70a3dc64e910c232a2cac7620b6609162c2a5058b269d3b83d763c02
-R 7aaffd9c020ac6e72a26849a8062d65e
+R 4791c9c6d2e81be0b277d92f5c368eb4
 U stephan
-Z 7690e0fae23fd2c65e8199412cec0d90
+Z 01e42f1c0aeb881a7fdcbb6522393054
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-92b500da70a3dc64e910c232a2cac7620b6609162c2a5058b269d3b83d763c02
+ea370b9b05f7ed7eaa154ba58019f6642217eabc18517e721567adf948d93980

src/os_kv.c

@@ -23,14 +23,14 @@
 
 /* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */
 #if 0
-#define SQLITE_KV_TRACE(X)  printf X;
+#define SQLITE_KV_TRACE(X)  printf X
 #else
 #define SQLITE_KV_TRACE(X)
 #endif
 
 /* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */
 #if 0
-#define SQLITE_KV_LOG(X)  printf X;
+#define SQLITE_KV_LOG(X)  printf X
 #else
 #define SQLITE_KV_LOG(X)
 #endif
@@ -166,6 +166,9 @@ static sqlite3_io_methods kvvfs_jrnl_io_methods = {
 
 /* Forward declarations for the low-level storage engine
 */
+static int kvstorageWrite(const char*, const char *zKey, const char *zData);
+static int kvstorageDelete(const char*, const char *zKey);
+static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf);
 #define KVSTORAGE_KEY_SZ  32
 
 /* Expand the key name with an appropriate prefix and put the result
@@ -180,206 +183,6 @@ static void kvstorageMakeKey(
   sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn);
 }
 
-#ifdef __EMSCRIPTEN__
-/* Provide Emscripten-based impls of kvstorageWrite/Read/Delete()... */
-#include <emscripten.h>
-#include <emscripten/console.h>
-
-/*
-** WASM_KEEP is identical to EMSCRIPTEN_KEEPALIVE but is not
-** Emscripten-specific. It explicitly includes marked functions for
-** export into the target wasm file without requiring explicit listing
-** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list
-** (or equivalent in other build platforms). Any function with neither
-** this attribute nor which is listed as an explicit export will not
-** be exported from the wasm file (but may still be used internally
-** within the wasm file).
-**
-** The functions in this filewhich require exporting are marked with
-** this flag. They may also be added to any explicit build-time export
-** list but need not be. All of these APIs are intended for use only
-** within the project's own JS/WASM code, and not by client code, so
-** an argument can be made for reducing their visibility by not
-** including them in any build-time export lists.
-**
-** 2022-09-11: it's not yet _proven_ that this approach works in
-** non-Emscripten builds. If not, such builds will need to export
-** those using the --export=... wasm-ld flag (or equivalent). As of
-** this writing we are tied to Emscripten for various reasons
-** and cannot test the library with other build environments.
-*/
-#define WASM_KEEP __attribute__((used,visibility("default")))
-/*
-** An internal level of indirection for accessing the static
-** kvstorageMakeKey() from EM_JS()-generated functions. This must be
-** made available for export via Emscripten but is not intended to be
-** used from client code. If called with a NULL zKeyOut it is a no-op.
-** It returns KVSTORAGE_KEY_SZ, so JS code (which cannot see that
-** constant) may call it with NULL arguments to get the size of the
-** allocation they'll need for a kvvfs key.
-**
-** Maintenance reminder: Emscripten will install this in the Module
-** init scope and will prefix its name with "_".
-*/
-WASM_KEEP
-int sqlite3_wasm__kvvfsMakeKey(const char *zClass, const char *zKeyIn,
-                               char *zKeyOut){
-  if( 0!=zKeyOut ) kvstorageMakeKey(zClass, zKeyIn, zKeyOut);
-  return KVSTORAGE_KEY_SZ;
-}
-/*
-** Internal helper for kvstorageWrite/Read/Delete() which creates a
-** storage key for the given zClass/zKeyIn combination. Returns a
-** pointer to the key: a C string allocated on the WASM stack, or 0 if
-** allocation fails. It is up to the caller to save/restore the stack
-** before/after this operation.
-*/
-EM_JS(const char *, kvstorageMakeKeyOnJSStack,
-      (const char *zClass, const char *zKeyIn),{
-  if( 0==zClass || 0==zKeyIn) return 0;
-  const zXKey = stackAlloc(_sqlite3_wasm__kvvfsMakeKey(0,0,0));
-  if(zXKey) _sqlite3_wasm__kvvfsMakeKey(zClass, zKeyIn, zXKey);
-  return zXKey;
-});
-
-/*
-** JS impl of kvstorageWrite(). Main docs are in the C impl. This impl
-** writes zData to the global sessionStorage (if zClass starts with
-** 's') or localStorage, using a storage key derived from zClass and
-** zKey.
-*/
-EM_JS(int, kvstorageWrite,
-      (const char *zClass, const char *zKey, const char *zData),{
-  const stack = stackSave();
-  try {
-    const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey);
-    if(!zXKey) return 1/*OOM*/;
-    const jKey = UTF8ToString(zXKey);
-    /**
-       We could simplify this function and eliminate the
-       kvstorageMakeKey() symbol acrobatics if we'd simply hard-code
-       the key algo into the 3 functions which need it:
-
-       const jKey = "kvvfs-"+UTF8ToString(zClass)+"-"+UTF8ToString(zKey);
-    */
-    ((115/*=='s'*/===getValue(zClass))
-     ? sessionStorage : localStorage).setItem(jKey, UTF8ToString(zData));
-  }catch(e){
-    console.error("kvstorageWrite()",e);
-    return 1; // Can't access SQLITE_xxx from here
-  }finally{
-    stackRestore(stack);
-  }
-  return 0;
-});
-
-/*
-** JS impl of kvstorageDelete(). Main docs are in the C impl. This
-** impl generates a key derived from zClass and zKey, and removes the
-** matching entry (if any) from global sessionStorage (if zClass
-** starts with 's') or localStorage.
-*/
-EM_JS(int, kvstorageDelete,
-      (const char *zClass, const char *zKey),{
-  const stack = stackSave();
-  try {
-    const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey);
-    if(!zXKey) return 1/*OOM*/;
-    const jKey = UTF8ToString(zXKey);
-    ((115/*=='s'*/===getValue(zClass))
-     ? sessionStorage : localStorage).removeItem(jKey);
-  }catch(e){
-    console.error("kvstorageDelete()",e);
-    return 1;
-  }finally{
-    stackRestore(stack);
-  }
-  return 0;
-});
-
-/*
-** JS impl of kvstorageRead(). Main docs are in the C impl. This impl
-** reads its data from the global sessionStorage (if zClass starts
-** with 's') or localStorage, using a storage key derived from zClass
-** and zKey.
-*/
-EM_JS(int, kvstorageRead,
-      (const char *zClass, const char *zKey, char *zBuf, int nBuf),{
-  const stack = stackSave();
-  try {
-    const zXKey = kvstorageMakeKeyOnJSStack(zClass,zKey);
-    if(!zXKey) return -3/*OOM*/;
-    const jKey = UTF8ToString(zXKey);
-    const jV = ((115/*=='s'*/===getValue(zClass))
-                ? sessionStorage : localStorage).getItem(jKey);
-    if(!jV) return -1;
-    const nV = jV.length /* Note that we are relying 100% on v being
-                            ASCII so that jV.length is equal to the
-                            C-string's byte length. */;
-    if(nBuf<=0) return nV;
-    else if(1===nBuf){
-      setValue(zBuf, 0);
-      return nV;
-    }
-    const zV = allocateUTF8OnStack(jV);
-    if(nBuf > nV + 1) nBuf = nV + 1;
-    HEAPU8.copyWithin(zBuf, zV, zV + nBuf - 1);
-    setValue( zBuf + nBuf - 1, 0 );
-    return nBuf - 1;
-  }catch(e){
-    console.error("kvstorageRead()",e);
-    return -2;
-  }finally{
-    stackRestore(stack);
-  }
-});
-
-/*
-** This function exists for (1) WASM testing purposes and (2) as a
-** hook to get Emscripten to export several EM_JS()-generated
-** functions (if we don't reference them from exported C functions
-** then they get stripped away at build time). It is not part of the
-** public API and its signature and semantics may change at any time.
-** It's not even part of the private API, for that matter - it's part
-** of the Emscripten C/JS/WASM glue.
-*/
-WASM_KEEP
-int sqlite3__wasm_emjs_kvvfs(int whichOp){
-  int rc = 0;
-  const char * zClass =
-    "sezzion" /*don't collide with "session" records!*/;
-  const char * zKey = "hello";
-  switch( whichOp ){
-    case 0: break;
-    case 1:
-      rc = kvstorageWrite(zClass, zKey, "world");
-      break;
-    case 2: {
-      char buffer[128] = {0};
-      char * zBuf = &buffer[0];
-      rc = kvstorageRead(zClass, zKey, zBuf, (int)sizeof(buffer));
-      emscripten_console_logf("kvstorageRead()=%d %s\n", rc, zBuf);
-      break;
-    }
-    case 3:
-      kvstorageDelete(zClass, zKey);
-      break;
-    case 4:
-      kvstorageMakeKeyOnJSStack(0,0);
-      break;
-    default: break;
-  }
-  return rc;
-}
-
-#undef WASM_KEEP
-#else /* end ifdef __EMSCRIPTEN__ */
-/* Forward declarations for the low-level storage engine
-*/
-static int kvstorageWrite(const char*, const char *zKey, const char *zData);
-static int kvstorageDelete(const char*, const char *zKey);
-static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf);
-
 /* Write content into a key.  zClass is the particular namespace of the
 ** underlying key/value store to use - either "local" or "session".
 **
@@ -474,7 +277,45 @@ static int kvstorageRead(
     return (int)n;
   }
 }
-#endif /* ifdef __EMSCRIPTEN__ */
+
+/*
+** An internal level of indirection which enables us to replace the
+** kvvfs i/o methods with JavaScript implementations in WASM builds.
+** Maintenance reminder: if this struct changes in any way, the JSON
+** rendering of its structure must be updated in
+** sqlite3_wasm_enum_json(). There are no binary compatibility
+** concerns, so it does not need an iVersion member. This file is
+** necessarily always compiled together with sqlite3_wasm_enum_json(),
+** and JS code dynamically creates the mapping of members based on
+** that JSON description.
+*/
+typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods;
+struct sqlite3_kvvfs_methods {
+  int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf);
+  int (*xWrite)(const char *zClass, const char *zKey, const char *zData);
+  int (*xDelete)(const char *zClass, const char *zKey);
+  const int nKeySize;
+};
+
+/*
+** This object holds the kvvfs I/O methods which may be swapped out
+** for JavaScript-side implementations in WASM builds. In such builds
+** it cannot be const, but in native builds it should be so that
+** the compiler can hopefully optimize this level of indirection out.
+** That said, kvvfs is intended primarily for use in WASM builds.
+**
+** Note that this is not explicitly flagged as static because the
+** amalgamation build will tag it with SQLITE_PRIVATE.
+*/
+#ifndef SQLITE_WASM
+const
+#endif
+sqlite3_kvvfs_methods sqlite3KvvfsMethods = {
+kvstorageRead,
+kvstorageWrite,
+kvstorageDelete,
+KVSTORAGE_KEY_SZ
+};
 
 /****** Utility subroutines ************************************************/
 
@@ -632,13 +473,13 @@ static void kvvfsDecodeJournal(
 static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){
   char zData[50];
   zData[0] = 0;
-  kvstorageRead(pFile->zClass, "sz", zData, sizeof(zData)-1);
+  sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1);
   return strtoll(zData, 0, 0);
 }
 static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){
   char zData[50];
   sqlite3_snprintf(sizeof(zData), zData, "%lld", sz);
-  return kvstorageWrite(pFile->zClass, "sz", zData);
+  return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData);
 }
 
 /****** sqlite3_io_methods methods ******************************************/
@@ -717,7 +558,7 @@ static int kvvfsReadDb(
     pgno = 1;
   }
   sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno);
-  got = kvstorageRead(pFile->zClass, zKey, aData, sizeof(aData)-1);
+  got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, aData, sizeof(aData)-1);
   if( got<0 ){
     n = 0;
   }else{
@@ -791,7 +632,7 @@ static int kvvfsWriteDb(
   pgno = 1 + iOfst/iAmt;
   sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno);
   kvvfsEncode(zBuf, iAmt, aData);
-  if( kvstorageWrite(pFile->zClass, zKey, aData) ){
+  if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){
     return SQLITE_IOERR;
   }
   if( iOfst+iAmt > pFile->szDb ){
@@ -807,7 +648,7 @@ static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){
   KVVfsFile *pFile = (KVVfsFile *)pProtoFile;
   SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size));
   assert( size==0 );
-  kvstorageDelete(pFile->zClass, "jrnl");
+  sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl");
   sqlite3_free(pFile->aJrnl);
   pFile->aJrnl = 0;
   pFile->nJrnl = 0;
@@ -826,7 +667,7 @@ static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){
     pgnoMax = 2 + pFile->szDb/pFile->szPage;
     while( pgno<=pgnoMax ){
       sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno);
-      kvstorageDelete(pFile->zClass, zKey);
+      sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey);
       pgno++;
     }
     pFile->szDb = size;
@@ -858,7 +699,7 @@ static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){
   }while( n>0 );
   zOut[i++] = ' ';
   kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]);
-  i = kvstorageWrite(pFile->zClass, "jrnl", zOut);
+  i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut);
   sqlite3_free(zOut);
   return i ? SQLITE_IOERR : SQLITE_OK;
 }
@@ -1004,10 +845,10 @@ static int kvvfsOpen(
 */
 static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
   if( strcmp(zPath, "local-journal")==0 ){
-    kvstorageDelete("local", "jrnl");
+    sqlite3KvvfsMethods.xDelete("local", "jrnl");
   }else
   if( strcmp(zPath, "session-journal")==0 ){
-    kvstorageDelete("session", "jrnl");
+    sqlite3KvvfsMethods.xDelete("session", "jrnl");
   }
   return SQLITE_OK;
 }
@@ -1024,16 +865,16 @@ static int kvvfsAccess(
 ){
   SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath));
   if( strcmp(zPath, "local-journal")==0 ){
-    *pResOut = kvstorageRead("local", "jrnl", 0, 0)>0;
+    *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0;
   }else
   if( strcmp(zPath, "session-journal")==0 ){
-    *pResOut = kvstorageRead("session", "jrnl", 0, 0)>0;
+    *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0;
   }else
   if( strcmp(zPath, "local")==0 ){
-    *pResOut = kvstorageRead("local", "sz", 0, 0)>0;
+    *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0;
   }else
   if( strcmp(zPath, "session")==0 ){
-    *pResOut = kvstorageRead("session", "sz", 0, 0)>0;
+    *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0;
   }else
   {
     *pResOut = 0;