Fiddle: fix makefile dependency issue and duplicate inclusion of post-js.js. Reimplement db export using sqlite3_serialize().

FossilOrigin-Name: 29db7de79232c21d19b91bb0fc253114e02e21dd9bf90619453dbe72a4c8bf7f

ext/wasm/api/sqlite3-api-glue.js

@@ -291,7 +291,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
        For the given integer, returns the SQLITE_xxx result code as a
        string, or undefined if no such mapping is found.
     */
-    capi.sqlite3_wasm_rc_str = (rc)=>__rcMap[rc];
+    capi.sqlite3_web_rc_str = (rc)=>__rcMap[rc];
     /* Bind all registered C-side structs... */
     for(const s of wasm.ctype.structs){
       capi[s.name] = sqlite3.StructBinder(s);

ext/wasm/api/sqlite3-api-prologue.js

@@ -870,6 +870,51 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
     return rc;
   };
 
+  /**
+     Serializes the given `sqlite3*` pointer to a Uint8Array, as per
+     sqlite3_serialize(). On success it returns a Uint8Array. On
+     error it throws with a description of the problem.
+  */
+  capi.sqlite3_web_db_export = function(pDb){
+    if(!pDb) toss('Invalid sqlite3* argument.');
+    const wasm = capi.wasm;
+    if(!wasm.bigIntEnabled) toss('BigInt64 support is not enabled.');
+    const scope = wasm.scopedAllocPush();
+    let pOut;
+    try{
+      const pSize = wasm.scopedAlloc(8/*i64*/ + wasm.ptrSizeof);
+      const ppOut = pSize + 8;
+      /**
+         Maintenance reminder, since this cost a full hour of grief
+         and confusion: if the order of pSize/ppOut are reversed in
+         that memory block, fetching the value of pSize after the
+         export reads a garbage size because it's not on an 8-byte
+         memory boundary!
+      */
+      wasm.setPtrValue(ppOut, 0);
+      wasm.setMemValue(pSize, 0, 'i64');
+      let rc = wasm.exports.sqlite3_wasm_db_serialize(
+        pDb, ppOut, pSize, 0
+      );
+      if(rc){
+        toss("Database serialization failed with code",
+             sqlite3.capi.sqlite3_web_rc_str(rc));
+      }
+      const pOut = wasm.getPtrValue(ppOut);
+      const nOut = wasm.getMemValue(pSize, 'i64');
+      rc = nOut
+        ? wasm.heap8u().slice(pOut, pOut + Number(nOut))
+        : new Uint8Array();
+      return rc;
+    }catch(e){
+      console.error('internal error?',e);
+      throw w;
+    }finally{
+      if(pOut) wasm.exports.sqlite3_free(pOut);
+      wasm.scopedAllocPop(scope);
+    }
+  };
+  
   if( capi.util.isMainWindow() ){
     /* Features specific to the main window thread... */
 
@@ -945,7 +990,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
           }
         }
       });
-      return sz * 2 /* because JS uses UC16 encoding */;
+      return sz * 2 /* because JS uses 2-byte char encoding */;
     };
 
   }/* main-window-only bits */

ext/wasm/api/sqlite3-wasm.c

@@ -406,6 +406,7 @@ const char * sqlite3_wasm_enum_json(void){
   } _DefGroup;
 
   DefGroup(serialize){
+    DefInt(SQLITE_SERIALIZE_NOCOPY);
     DefInt(SQLITE_DESERIALIZE_FREEONCLOSE);
     DefInt(SQLITE_DESERIALIZE_READONLY);
     DefInt(SQLITE_DESERIALIZE_RESIZEABLE);
@@ -567,6 +568,83 @@ int sqlite3_wasm_vfs_unlink(const char * zName){
   return rc;
 }
 
+/*
+** Uses the current database's VFS xRead to stream the db file's
+** contents out to the given callback. The callback gets a single
+** chunk of size n (its 2nd argument) on each call and must return 0
+** on success, non-0 on error. This function returns 0 on success,
+** SQLITE_NOTFOUND if no db is open, or propagates any other non-0
+** code from the callback. Note that this is not thread-friendly: it
+** expects that it will be the only thread reading the db file and
+** takes no measures to ensure that is the case.
+**
+** This implementation appears to work fine, but
+** sqlite3_wasm_db_serialize() is arguably the better way to achieve
+** this.
+*/
+WASM_KEEP
+int sqlite3_wasm_db_export_chunked( sqlite3* pDb,
+                                    int (*xCallback)(unsigned const char *zOut, int n) ){
+  sqlite3_int64 nSize = 0;
+  sqlite3_int64 nPos = 0;
+  sqlite3_file * pFile = 0;
+  unsigned char buf[1024 * 8];
+  int nBuf = (int)sizeof(buf);
+  int rc = pDb
+    ? sqlite3_file_control(pDb, "main",
+                           SQLITE_FCNTL_FILE_POINTER, &pFile)
+    : SQLITE_NOTFOUND;
+  if( rc ) return rc;
+  rc = pFile->pMethods->xFileSize(pFile, &nSize);
+  if( rc ) return rc;
+  if(nSize % nBuf){
+    /* DB size is not an even multiple of the buffer size. Reduce
+    ** buffer size so that we do not unduly inflate the db size
+    ** with zero-padding when exporting. */
+    if(0 == nSize % 4096) nBuf = 4096;
+    else if(0 == nSize % 2048) nBuf = 2048;
+    else if(0 == nSize % 1024) nBuf = 1024;
+    else nBuf = 512;
+  }
+  for( ; 0==rc && nPos<nSize; nPos += nBuf ){
+    rc = pFile->pMethods->xRead(pFile, buf, nBuf, nPos);
+    if(SQLITE_IOERR_SHORT_READ == rc){
+      rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/;
+    }
+    if( 0==rc ) rc = xCallback(buf, nBuf);
+  }
+  return rc;
+}
+
+/*
+** A proxy for sqlite3_serialize() which serializes the "main" schema
+** of pDb, placing the serialized output in pOut and nOut. nOut may be
+** NULL. If pDb or pOut are NULL then SQLITE_MISUSE is returned. If
+** allocation of the serialized copy fails, SQLITE_NOMEM is returned.
+** On success, 0 is returned and `*pOut` will contain a pointer to the
+** memory unless mFlags includes SQLITE_SERIALIZE_NOCOPY and the
+** database has no contiguous memory representation, in which case
+** `*pOut` will be NULL but 0 will be returned.
+**
+** If `*pOut` is not NULL, the caller is responsible for passing it to
+** sqlite3_free() to free it.
+*/
+WASM_KEEP
+int sqlite3_wasm_db_serialize( sqlite3* pDb, unsigned char **pOut, sqlite3_int64 * nOut,
+                               unsigned int mFlags ){
+  unsigned char * z;
+  if( !pDb || !pOut ) return SQLITE_MISUSE;
+  if(nOut) *nOut = 0;
+  z = sqlite3_serialize(pDb, "main", nOut, mFlags);
+  if( z || (SQLITE_SERIALIZE_NOCOPY & mFlags) ){
+    *pOut = z;
+    return 0;
+  }else{
+    return SQLITE_NOMEM;
+  }
+}
+
+
 #if defined(__EMSCRIPTEN__)
 #include <emscripten/console.h>
 #if defined(SQLITE_WASM_WASMFS)

ext/wasm/fiddle.make

@@ -37,7 +37,6 @@ fiddle.emcc-flags = \
   $(sqlite3.js.flags.--post-js) \
   -sEXPORTED_RUNTIME_METHODS=@$(dir.wasm)/EXPORTED_RUNTIME_METHODS.fiddle \
   -sEXPORTED_FUNCTIONS=@$(dir.wasm)/EXPORTED_FUNCTIONS.fiddle \
-  --post-js=$(post-js.js) \
   $(SQLITE_OPT) $(SHELL_OPT) \
   -DSQLITE_SHELL_FIDDLE
 # -D_POSIX_C_SOURCE is needed for strdup() with emcc
@@ -60,9 +59,9 @@ $(dir.fiddle)/$(SOAP.js): $(SOAP.js)
 $(eval $(call call-make-pre-js,fiddle-module))
 $(fiddle-module.js): $(MAKEFILE) $(MAKEFILE.fiddle) \
     EXPORTED_FUNCTIONS.fiddle EXPORTED_RUNTIME_METHODS.fiddle \
-    $(fiddle.cs) $(pre-post-fiddle.deps) $(dir.fiddle)/$(SOAP.js)
+    $(fiddle.cs) $(pre-post-fiddle-module.deps) $(dir.fiddle)/$(SOAP.js)
 	$(emcc.bin) -o $@ $(fiddle.emcc-flags) \
-    $(pre-post-common.flags) $(pre-post-fiddle.flags) \
+    $(pre-post-common.flags) $(pre-post-fiddle-module.flags) \
     $(fiddle.cs)
 	$(maybe-wasm-strip) $(fiddle-module.wasm)
 	gzip < $@ > $@.gz

ext/wasm/fiddle/fiddle-worker.js

@@ -227,94 +227,6 @@
       f._();
     }
   };
-
-  /**
-     Exports the shell's current db file in such a way that it can
-     export DBs hosted in the Emscripten-supplied FS or in native OPFS
-     (and, hypothetically, kvvfs). Throws on error. On success returns
-     a Blob containing the whole db contents.
-
-     Bug/to investigate: xFileSize() is returning garbage for the
-     default VFS but works in OPFS. Thus for exporting that impl we'll
-     use the fiddleModule.FS API for the time being. The equivalent
-     native impl, fiddle_export_db(), works okay with both VFSes, so
-     the bug is apparently in (or via) this code.
-  */
-  const brokenExportDbFileToBlob = function(){
-    const capi = sqlite3.capi, wasm = capi.wasm;
-    const pDb = Sqlite3Shell.dbHandle();
-    if(!pDb) toss("No db is opened.");
-    const scope = wasm.scopedAllocPush();
-    try{
-      const ppFile = wasm.scopedAlloc(12/*sizeof(i32 + i64)*/);
-      const pFileSize = ppFile + 4;
-      wasm.setMemValue(ppFile, 0, '*');
-      let rc = capi.sqlite3_file_control(
-        pDb, "main", capi.SQLITE_FCNTL_FILE_POINTER, ppFile
-      );
-      if(rc) toss("Cannot get sqlite3_file handle.");
-      const jFile = new capi.sqlite3_file(wasm.getPtrValue(ppFile));
-      const jIom = new capi.sqlite3_io_methods(jFile.$pMethods);
-      const xFileSize = wasm.functionEntry(jIom.$xFileSize);
-      const xRead = wasm.functionEntry(jIom.$xRead);
-      wasm.setMemValue(pFileSize, 0, 'i64');
-      //stderr("nFileSize =",wasm.getMemValue(pFileSize,'i64'),"pFileSize =",pFileSize);
-      rc = xFileSize( jFile.pointer, pFileSize );
-      if(rc) toss("Cannot get db file size.");
-      //stderr("nFileSize =",wasm.getMemValue(pFileSize,'i64'),"pFileSize =",pFileSize);
-      const nFileSize = Number( wasm.getMemValue(pFileSize,'i64') );
-      if(nFileSize <= 0n || nFileSize>=Number.MAX_SAFE_INTEGER){
-        toss("Unexpected DB size:",nFileSize);
-      }
-      //stderr("nFileSize =",nFileSize,"pFileSize =",pFileSize);
-      const nIobuf = 1024 * 4;
-      const iobuf = wasm.scopedAlloc(nIobuf);
-      let nPos = 0;
-      const blobList = [];
-      const heap = wasm.heap8u();
-      for( ; nPos < nFileSize; nPos += nIobuf ){
-        rc = xRead(jFile.pointer, iobuf, nIobuf, BigInt(nPos));
-        if(rc){
-          if(capi.SQLITE_IOERR_SHORT_READ === rc){
-            //stderr('rc =',rc,'nPos =',nPos,'nIobuf =',nIobuf,'nFileSize =',nFileSize);
-            rc = ((nPos + nIobuf) < nFileSize) ? rc : 0/*assume EOF*/;
-          }
-          if(rc) toss("xRead() SQLITE_xxx error #"+rc,capi.sqlite3_wasm_rc_str(rc));
-        }
-        blobList.push(heap.slice(iobuf, iobuf+nIobuf));
-      }
-      return new Blob(blobList);
-    }catch(e){
-      console.error('exportDbFileToBlob()',e);
-      stderr("exportDbFileToBlob():",e.message);
-    }finally{
-      wasm.scopedAllocPop(scope);
-    }
-  }/*brokenExportDbFileToBlob()*/;
-
-  const exportDbFileToBlob = function f(){
-    if(!f._){
-      f._ = sqlite3.capi.wasm.xWrap('fiddle_export_db', 'int', '*');
-    }
-    const capi = sqlite3.capi;
-    const wasm = capi.wasm;
-    const blobList = [];
-    const heap = wasm.heap8u();
-    const callback = wasm.installFunction('ipi', function(buf, n){
-      blobList.push(heap.slice(buf, buf+n));
-      return 0;
-    });
-    try {
-      const rc = wasm.exports.fiddle_export_db( callback );
-      if(rc) toss("DB export failed with code", capi.sqlite3_wasm_rc_str(rc));
-      return new Blob(blobList);
-    }catch(e){
-      console.error("exportDbFileToBlob():",e.message);
-      throw(e);
-    }finally{
-      wasm.uninstallFunction(callback);
-    }
-  }/*exportDbFileToBlob()*/;
   
   self.onmessage = function f(ev){
     ev = ev.data;
@@ -344,11 +256,13 @@
           stdout("Exporting",fn+".");
           const fn2 = fn ? fn.split(/[/\\]/).pop() : null;
           try{
-            if(!fn2) throw new Error("DB appears to be closed.");
+            if(!fn2) toss("DB appears to be closed.");
-            exportDbFileToBlob().arrayBuffer().then((buffer)=>{
+            const buffer = sqlite3.capi.sqlite3_web_db_export(
-              wMsg('db-export',{filename: fn2, buffer}, [buffer]);
+              Sqlite3Shell.dbHandle()
-            });
+            );
+            wMsg('db-export',{filename: fn2, buffer: buffer.buffer}, [buffer.buffer]);
           }catch(e){
+            console.error("Export failed:",e);
             /* Post a failure message so that UI elements disabled
                during the export can be re-enabled. */
             wMsg('db-export',{

manifest

@@ -1,5 +1,5 @@
-C Merge\strunk\sinto\sfiddle-opfs\sbranch.
+C Fiddle:\sfix\smakefile\sdependency\sissue\sand\sduplicate\sinclusion\sof\spost-js.js.\sReimplement\sdb\sexport\susing\ssqlite3_serialize().
-D 2022-10-01T13:45:14.035
+D 2022-10-01T16:01:41.242
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -485,13 +485,13 @@ F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba814
 F ext/wasm/api/post-js-header.js 2e5c886398013ba2af88028ecbced1e4b22dc96a86467f1ecc5ba9e64ef90a8b
 F ext/wasm/api/pre-js.js 2db711eb637991b383fc6b5c0f3df65ec48a7201e5730e304beba8de2d3f9b0b
 F ext/wasm/api/sqlite3-api-cleanup.js 5d22d1d3818ecacb23bfa223d5970cd0617d8cdbb48c8bc4bbd463f05b021a99
-F ext/wasm/api/sqlite3-api-glue.js ead29e6008ba148e7c67ad2bd928819dc72313ad2dcd34510cc61e089ee01c4e
+F ext/wasm/api/sqlite3-api-glue.js b15a51b88aaa472d36bf82d5123dbfdafe8ddf6ca75fba934510e4a20bbe4adb
 F ext/wasm/api/sqlite3-api-oo1.js 9caed0757a5e039ed92467e827fd3ca347fa08f19fe086fcbdd14a4ebe9c2f01
 F ext/wasm/api/sqlite3-api-opfs.js 1b097808b7b081b0f0700cf97d49ef19760e401706168edff9cd45cf9169f541
-F ext/wasm/api/sqlite3-api-prologue.js 04e0c929deeb28c9a2509d8004dfe3214992b988d277c4f50afa84953689c23c
+F ext/wasm/api/sqlite3-api-prologue.js d7904da82691f68b9ffb081c072cf4725716154284a63447f2d1dce40ab4b85f
 F ext/wasm/api/sqlite3-api-worker1.js 7f4f46cb6b512a48572d7567233896e6a9c46570c44bdc3d13419730c7c221c8
 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
-F ext/wasm/api/sqlite3-wasm.c 336389b23c9b83763177499e49a0967949c392b2f7d84fbbb52ad6678e159f18
+F ext/wasm/api/sqlite3-wasm.c 61c6bf8404a07f3d5f2861fbc1d3596474cecfc38801fb3a38c560fe847f79a5
 F ext/wasm/batch-runner.html c363032aba7a525920f61f8be112a29459f73f07e46f0ba3b7730081a617826e
 F ext/wasm/batch-runner.js ce92650a6681586c89bef26ceae96674a55ca5a9727815202ca62e1a00ff5015
 F ext/wasm/common/SqliteTestUtil.js 647bf014bd30bdd870a7e9001e251d12fc1c9ec9ce176a1004b838a4b33c5c05
@@ -503,9 +503,9 @@ F ext/wasm/demo-123.html aa281d33b7eefa755f3122b7b5a18f39a42dc5fb69c8879171bf14b
 F ext/wasm/demo-123.js 536579fd587974c2511c5bf82034b253d4fdeceabb726927ad7599ef6b7578e8
 F ext/wasm/demo-kvvfs1.html 7d4f28873de67f51ac18c584b7d920825139866a96049a49c424d6f5a0ea5e7f
 F ext/wasm/demo-kvvfs1.js 105596bd2ccd0b1deb5fde8e99b536e8242d4bb5932fac0c8403ff3a6bc547e8
-F ext/wasm/fiddle.make d343d44c58bca06ac0ec0296207f6441560bff89f1e587bbaf843b73c2ca5d76
+F ext/wasm/fiddle.make 3f4efd62bc2a9c883bfcea52ae2755114a62d444d6d042df287f4aef301d6c6c
 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f
-F ext/wasm/fiddle/fiddle-worker.js 2a7107b06e5be3b9b063c340ec952f687e37ba6e0aa736b58c280dfb5e16625a
+F ext/wasm/fiddle/fiddle-worker.js ebf6e95fe031738cd9f79d1ccdf6d80640a2884a43f7006a5e3459995f93069b
 F ext/wasm/fiddle/fiddle.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2
 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715
 F ext/wasm/index.html 63b370619e4f849ac76f1baed435c05edc29dbb6795bc7c1c935561ff667dd27
@@ -599,7 +599,7 @@ F src/notify.c 89a97dc854c3aa62ad5f384ef50c5a4a11d70fcc69f86de3e991573421130ed6
 F src/os.c 0eb831ba3575af5277e47f4edd14fdfc90025c67eb25ce5cda634518d308d4e9
 F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85
-F src/os_kv.c cf02a39ab3271d237890c7de03a49e5ecc19c930b8f4ad6afd43651dc05c9400
+F src/os_kv.c 88872a68ba37e4ed4306eb6fa78d559dcc675761ce1f6458ba710e1c10a2d667
 F src/os_setup.h 0711dbc4678f3ac52d7fe736951b6384a0615387c4ba5135a4764e4e31f4b6a6
 F src/os_unix.c 287aa5f5691a2b356780c63e83abaa33549add84227b8313395f04088486d79c
 F src/os_win.c 8d129ae3e59e0fa900e20d0ad789e96f2e08177f0b00b53cdda65c40331e0902
@@ -2029,8 +2029,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 000ef7059bfb54dc4f829b81a8d8c927c0382980218d8a3d60cd2c4d89151c90 b7da0bcdf70e53ab1ec00a0694e17c7429e23bc6eb3f39b622d06a930aa2f6a3
+P 64ebcbe41615a6d7776597564105ea7638e4a9095a764ea558c2620640429cf8
-R 5df8cd3132a0539ad9a559117068898c
+R 624d71ac921985ea3965d7f2eda6af5a
 U stephan
-Z 7ab2cbdb07e5b8371b60f5611bb9584c
+Z 9261b853936f36a4eb842018d682a85c
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-64ebcbe41615a6d7776597564105ea7638e4a9095a764ea558c2620640429cf8
+29db7de79232c21d19b91bb0fc253114e02e21dd9bf90619453dbe72a4c8bf7f

src/os_kv.c

@@ -195,12 +195,12 @@ static void kvstorageMakeKey(
 ** be exported from the wasm file (but may still be used internally
 ** within the wasm file).
 **
-** The functions in this file (sqlite3-wasm.c) which require exporting
+** The functions in this filewhich require exporting are marked with
-** are marked with this flag. They may also be added to any explicit
+** this flag. They may also be added to any explicit build-time export
-** build-time export list but need not be. All of these APIs are
+** list but need not be. All of these APIs are intended for use only
-** intended for use only within the project's own JS/WASM code, and
+** within the project's own JS/WASM code, and not by client code, so
-** not by client code, so an argument can be made for reducing their
+** an argument can be made for reducing their visibility by not
-** visibility by not including them in any build-time export lists.
+** including them in any build-time export lists.
 **
 ** 2022-09-11: it's not yet _proven_ that this approach works in
 ** non-Emscripten builds. If not, such builds will need to export