Build refactoring for the fiddle/wasm bits. Set up wasm binding of a chunk of the core C API and added some infastructure for creating test pages for it.

FossilOrigin-Name: dea098b64eb95c395b346ebcae687afe42b7d21df48833527808c02226300a66

ext/fiddle/EXPORTED_FUNCTIONS.sqlite3

@@ -0,0 +1,39 @@
+_sqlite3_bind_blob
+_sqlite3_bind_double
+_sqlite3_bind_int
+_sqlite3_bind_parameter_index
+_sqlite3_bind_text
+_sqlite3_changes
+_sqlite3_clear_bindings
+_sqlite3_close_v2
+_sqlite3_column_blob
+_sqlite3_column_bytes
+_sqlite3_column_count
+_sqlite3_column_count
+_sqlite3_column_double
+_sqlite3_column_name
+_sqlite3_column_text
+_sqlite3_column_type
+_sqlite3_create_function_v2
+_sqlite3_data_count
+_sqlite3_errmsg
+_sqlite3_exec
+_sqlite3_finalize
+_sqlite3_libversion
+_sqlite3_open
+_sqlite3_prepare_v2
+_sqlite3_prepare_v2
+_sqlite3_reset
+_sqlite3_result_blob
+_sqlite3_result_double
+_sqlite3_result_error
+_sqlite3_result_int
+_sqlite3_result_null
+_sqlite3_result_text
+_sqlite3_sourceid
+_sqlite3_step
+_sqlite3_value_blob
+_sqlite3_value_bytes
+_sqlite3_value_double
+_sqlite3_value_text
+_sqlite3_value_type

ext/fiddle/Makefile

@@ -1,7 +1,10 @@
 # This makefile exists primarily to simplify/speed up development from
 # emacs. It is not part of the canonical build process.
 default:
-	make -C ../.. fiddle -e emcc_opt=-O0
+	make -C ../.. wasm -e emcc_opt=-O0
 
 clean:
-	make -C ../../ clean-fiddle
+	make -C ../../ clean-wasm
+
+push-demo:
+	rsync -va fiddle*.js fiddle*.wasm fiddle.html *.css wh2:www/wh/sqlite3/.

ext/fiddle/emscripten.css

@@ -0,0 +1,24 @@
+/* emcscript-related styling, used during the module load/intialization processes... */
+.emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; }
+div.emscripten { text-align: center; }
+div.emscripten_border { border: 1px solid black; }
+#module-spinner { overflow: visible; }
+#module-spinner > * {
+    margin-top: 1em;
+}
+.spinner {
+    height: 50px;
+    width: 50px;
+    margin: 0px auto;
+    animation: rotation 0.8s linear infinite;
+    border-left: 10px solid rgb(0,150,240);
+    border-right: 10px solid rgb(0,150,240);
+    border-bottom: 10px solid rgb(0,150,240);
+    border-top: 10px solid rgb(100,0,200);
+    border-radius: 100%;
+    background-color: rgb(200,100,250);
+}
+@keyframes rotation {
+    from {transform: rotate(0deg);}
+    to {transform: rotate(360deg);}
+}

ext/fiddle/fiddle-worker.js

@@ -14,6 +14,10 @@
   sqlite3 wasm module and offers access to the db via the Worker
   message-passing interface.
 
+  Forewarning: this API is still very much Under Construction and
+  subject to any number of changes as experience reveals what those
+  need to be.
+
   Because we can have only a single message handler, as opposed to an
   arbitrary number of discrete event listeners like with DOM elements,
   we have to define a lower-level message API. Messages abstractly
@@ -70,23 +74,23 @@
   Noting that it happens in Firefox as well as Chrome. Harmless but
   annoying.
 */
-
-const thisWorker = self;
+"use strict";
 
 const wMsg = (type,data)=>postMessage({type, data});
 
 self.onerror = function(/*message, source, lineno, colno, error*/) {
     const err = arguments[4];
     if(err && 'ExitStatus'==err.name){
+        /* This is relevant for the sqlite3 shell binding but not the
+           lower-level binding. */
         Module._isDead = true;
         Module.printErr("FATAL ERROR:", err.message);
         Module.printErr("Restarting the app requires reloading the page.");
-        //const taOutput = document.querySelector('#output');
-        //if(taOutput) taOutput.classList.add('error');
+        wMsg('error', err);
     }
     Module.setStatus('Exception thrown, see JavaScript console');
     Module.setStatus = function(text) {
-        if(text) console.error('[post-exception status] ' + text);
+        console.error('[post-exception status]', text);
     };
 };
 
@@ -139,47 +143,10 @@ self.onmessage = function(ev){
 self.Module.setStatus('Downloading...');
 importScripts('fiddle-module.js')
 /* loads the wasm module and notifies, via Module.setStatus() and
-   Module.onRuntimeInitialized(), when it's done loading. */;
+   Module.onRuntimeInitialized(), when it's done loading.  The latter
+   is called _before_ the final call to Module.setStatus(). */;
 
 Module["onRuntimeInitialized"] = function onRuntimeInitialized() {
-    /* For reference: sql.js does essentially everything we want and
-       it solves much of the wasm-related voodoo, but we'll need a
-       different structure because we want the db connection to run in
-       a worker thread and feed data back into the main
-       thread. Regardless of those differences, it makes a great point
-       of reference:
-
-       https://github.com/sql-js/sql.js
-
-       Some of the specific design goals here:
-
-       - Bind a low-level sqlite3 API which is close to the native one in
-         terms of usage.
-
-       - Create a higher-level one, more akin to sql.js and
-         node.js-style implementations. This one would speak directly
-         to the low-level API. This API could be used by clients who
-         import the low-level API directly into their main thread
-         (which we don't want to recommend but also don't want to
-         outright forbid).
-
-       - Create a second higher-level one which speaks to the
-         low-level API via worker messages. This one would be intended
-         for use in the main thread, talking to the low-level UI via
-         worker messages. Because workers have only a single message
-         channel, some acrobatics will be needed here to feed async
-         work results back into client-side callbacks (as those
-         callbacks cannot simply be passed to the worker). Exactly
-         what those acrobatics should look like is not yet entirely
-         clear and much experimentation is pending.
-
-    */
-    console.log('onRuntimeInitialized');
-
-    /*
-      TODO: create the main sqlite API here. We'll have another for
-      use in the main thread which will talk to this one via worker
-      messages.
-    */
-    
-}
+    //console.log('onRuntimeInitialized');
+    //wMsg('module','done');
+};

ext/fiddle/fiddle.html

@@ -6,34 +6,9 @@
     <title>sqlite3 fiddle</title>
     <!--script src="jqterm/jqterm-bundle.min.js"></script>
     <link rel="stylesheet" href="jqterm/jquery.terminal.min.css"/-->
+    <link rel="stylesheet" href="emscripten.css"/>
     <style>
-      /* emcscript-related styling, used during the intialization phase... */
-      .emscripten { padding-right: 0; margin-left: auto; margin-right: auto; display: block; }
-      div.emscripten { text-align: center; }
-      div.emscripten_border { border: 1px solid black; }
-      #module-spinner { overflow: visible; }
-      #module-spinner > * {
-          margin-top: 1em;
-      }
-      .spinner {
-        height: 50px;
-        width: 50px;
-        margin: 0px auto;
-        animation: rotation 0.8s linear infinite;
-        border-left: 10px solid rgb(0,150,240);
-        border-right: 10px solid rgb(0,150,240);
-        border-bottom: 10px solid rgb(0,150,240);
-        border-top: 10px solid rgb(100,0,200);
-        border-radius: 100%;
-        background-color: rgb(200,100,250);
-      }
-      @keyframes rotation {
-        from {transform: rotate(0deg);}
-        to {transform: rotate(360deg);}
-      }
-
       /* The following styles are for app-level use. */
-      
       textarea {
           font-family: monospace;
           flex: 1 1 auto;
@@ -70,9 +45,6 @@
           filter: invert(100%);
           flex: 10 1 auto;
       }
-      /*#main-wrapper:not(.side-by-side) .ta-wrapper.input {
-          flex: 5 1 auto;
-      }*/
       .button-bar {
           display: flex;
           justify-content: center;
@@ -142,6 +114,7 @@
   </head>
   <body>
     <header id='titlebar'><span>sqlite3 fiddle</span></header>
+    <!-- emscripten bits -->
     <figure id="module-spinner">
       <div class="spinner"></div>
       <div class='center'><strong>Initializing app...</strong></div>
@@ -154,7 +127,7 @@
     <div class="emscripten" id="module-status">Downloading...</div>
     <div class="emscripten">
       <progress value="0" max="100" id="module-progress" hidden='1'></progress>  
-    </div>
+    </div><!-- /emscripten bits -->
 
     <div id='view-terminal' class='app-view hidden initially-hidden'>
       This is a placeholder for a terminal-like view.

ext/fiddle/fiddle.js

@@ -159,8 +159,9 @@
             progressElement.remove();
             if(!text) spinnerElement.remove();
         }
-        if(text) statusElement.innerText = text;
-        else {
+        if(text){
+            statusElement.innerText = text;
+        }else{
             console.log("Finalizing status.");
             statusElement.remove();
             SF.clearMsgHandlers('module');

ext/fiddle/sqlite3-api.js

@@ -0,0 +1,165 @@
+/*
+  2022-05-22
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  This file is intended to be loaded after loading
+  sqlite3-module.wasm. It sets one of any number of potential
+  bindings using that API, this one as closely matching the C-native
+  API as is feasible.
+
+  Note that this file is not named sqlite3.js because that file gets
+  generated by emscripten as the JS-glue counterpart of sqlite3.wasm.
+
+  The API gets installed as self.sqlite3, where self is expected to be
+  either the global window or Worker object.
+
+  Because using this API properly requires some degree of WASM-related
+  magic, it is not recommended that this API be used as-is in
+  client-level code, but instead is intended to be used as a basis for
+  APIs more appropriate for high-level client code.
+
+  This file installs namespace.sqlite3, where namespace is `self`,
+  meaning either the global window or worker, depending on where this
+  is loaded from.
+*/
+(function(namespace){
+    /* For reference: sql.js does essentially everything we want and
+       it solves much of the wasm-related voodoo, but we'll need a
+       different structure because we want the db connection to run in
+       a worker thread and feed data back into the main
+       thread. Regardless of those differences, it makes a great point
+       of reference:
+
+       https://github.com/sql-js/sql.js
+
+       Some of the specific design goals here:
+
+       - Bind a low-level sqlite3 API which is close to the native one
+         in terms of usage.
+
+       - Create a higher-level one, more akin to sql.js and
+         node.js-style implementations. This one would speak directly
+         to the low-level API. This API could be used by clients who
+         import the low-level API directly into their main thread
+         (which we don't want to recommend but also don't want to
+         outright forbid).
+
+       - Create a second higher-level one which speaks to the
+         low-level API via worker messages. This one would be intended
+         for use in the main thread, talking to the low-level UI via
+         worker messages. Because workers have only a single message
+         channel, some acrobatics will be needed here to feed async
+         work results back into client-side callbacks (as those
+         callbacks cannot simply be passed to the worker). Exactly
+         what those acrobatics should look like is not yet entirely
+         clear and much experimentation is pending.
+    */
+
+    /** 
+      Set up the main sqlite3 binding API here, mimicking the C API as
+      closely as we can.
+
+      Attribution: though not a direct copy/paste, much of what
+      follows is strongly influenced by the sql.js implementation.
+    */
+    const api = {
+        /* It is important that the following integer values match
+           those from the C code. Ideally we could fetch them from the
+           C API, e.g., in the form of a JSON object, but getting that
+           JSON string constructed within our current confised is
+           currently not worth the effort. */
+        /* Minimum subset of sqlite result codes we'll need. */
+        SQLITE_OK: 0,
+        SQLITE_ROW: 100,
+        SQLITE_DONE: 101,
+        /* sqlite data types */
+        SQLITE_INTEGER: 1,
+        SQLITE_FLOAT: 2,
+        SQLITE_TEXT: 3,
+        SQLITE_BLOB: 4,
+        /* sqlite encodings, used for creating UDFs, noting that we
+           will only support UTF8. */
+        SQLITE_UTF8: 1
+    };
+    const cwrap = Module.cwrap;
+    [/* C-side functions to bind. Each entry is an array with 3 or 4
+        elements:
+        
+        ["c-side name",
+         "result type" (cwrap() syntax),
+         [arg types in cwrap() syntax]
+        ]
+
+        If it has 4 elements, the first one is an alternate name to
+        use for the JS-side binding. That's required when overloading
+        a binding for two different uses.
+     */
+        ["sqlite3_open", "number", ["string", "number"]],
+        ["sqlite3_close_v2", "number", ["number"]],
+        ["sqlite3_exec", "number",
+         ["number", "string", "number", "number", "number"]],
+        ["sqlite3_changes", "number", ["number"]],
+        ["sqlite3_prepare_v2", "number", ["number", "string", "number", "number", "number"]],
+        ["sqlite3_prepare_v2_sqlptr",
+         /* Impl which requires that the 2nd argument be a pointer to
+            the SQL, instead of a string. This is used for cases where
+            we require a non-NULL value for the final argument. We may
+            or may not need this, depending on how our higher-level
+            API shapes up, but this code's spiritual guide (sql.js)
+            uses it we we'll include it. */
+         "sqlite3_prepare_v2",
+         "number", ["number", "number", "number", "number", "number"]],
+        ["sqlite3_bind_text","number",["number", "number", "number", "number", "number"]],
+        ["sqlite3_bind_blob","number",["number", "number", "number", "number", "number"]],
+        ["sqlite3_bind_double","number",["number", "number", "number"]],
+        ["sqlite3_bind_int","number",["number", "number", "number"]],
+        ["sqlite3_bind_parameter_index","number",["number", "string"]],
+        ["sqlite3_step", "number", ["number"]],
+        ["sqlite3_errmsg", "string", ["number"]],
+        ["sqlite3_column_count","number",["number"]],
+        ["sqlite3_data_count", "number", ["number"]],
+        ["sqlite3_column_count", "number", ["number"]],
+        ["sqlite3_column_double","number",["number", "number"]],
+        ["sqlite3_column_text","string",["number", "number"]],
+        ["sqlite3_column_blob","number", ["number", "number"]],
+        ["sqlite3_column_bytes","number",["number", "number"]],
+        ["sqlite3_column_type","number",["number", "number"]],
+        ["sqlite3_column_name","string",["number", "number"]],
+        ["sqlite3_reset", "number", ["number"]],
+        ["sqlite3_clear_bindings","number",["number"]],
+        ["sqlite3_finalize", "number", ["number"]],
+        ["sqlite3_create_function_v2", "number",
+         ["number", "string", "number", "number",
+          "number", "number", "number", "number",
+          "number"]],
+        ["sqlite3_value_type", "number", ["number"]],
+        ["sqlite3_value_bytes","number",["number"]],
+        ["sqlite3_value_text", "string", ["number"]],
+        ["sqlite3_value_blob", "number", ["number"]],
+        ["sqlite3_value_double","number",["number"]],
+        ["sqlite3_result_double",null,["number", "number"]],
+        ["sqlite3_result_null",null,["number"]],
+        ["sqlite3_result_text",null,["number", "string", "number", "number"]],
+        ["sqlite3_result_blob",null,["number", "number", "number", "number"]],
+        ["sqlite3_result_int",null,["number", "number"]],
+        ["sqlite3_result_error",null,["number", "string", "number"]],
+        ["sqlite3_libversion", "string", []],
+        ["sqlite3_sourceid", "string", []]
+        //["sqlite3_sql", "string", ["number"]],
+        //["sqlite3_normalized_sql", "string", ["number"]]
+    ].forEach(function(e){
+        const a = Array.prototype.slice.call(e);
+        const k = (4==a.length) ? a.shift() : a[0];
+        api[k] = cwrap.apply(this, a);
+    });
+    //console.debug("libversion =",api.sqlite3_libversion());
+    namespace.sqlite3 = api;
+})(self/*worker or window*/);

ext/fiddle/testing-common.js

@@ -0,0 +1,96 @@
+/*
+  2022-05-22
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  This file contains bootstrapping code used by various test scripts
+  which live in this file's directory.
+*/
+(function(){
+    /* querySelectorAll() proxy */
+    const EAll = function(/*[element=document,] cssSelector*/){
+        return (arguments.length>1 ? arguments[0] : document)
+            .querySelectorAll(arguments[arguments.length-1]);
+    };
+    /* querySelector() proxy */
+    const E = function(/*[element=document,] cssSelector*/){
+        return (arguments.length>1 ? arguments[0] : document)
+            .querySelector(arguments[arguments.length-1]);
+    };
+    
+    const statusElement = E('#module-status');
+    const progressElement = E('#module-progress');
+    const spinnerElement = E('#module-spinner');
+
+    self.Module = {
+        /* ^^^ cannot declare that const because fiddle-module.js
+           (auto-generated) includes a decl for it and runs in this scope. */
+        preRun: [],
+        postRun: [],
+        //onRuntimeInitialized: function(){},
+        print: function(){
+            console.log(Array.prototype.slice.call(arguments));
+        },
+        printErr: function(){
+            console.error(Array.prototype.slice.call(arguments));
+        },
+        setStatus: function f(text){
+            if(!f.last) f.last = { time: Date.now(), text: '' };
+            if(text === f.last.text) return;
+            const m = text.match(/([^(]+)\((\d+(\.\d+)?)\/(\d+)\)/);
+            const now = Date.now();
+            if(m && now - f.last.time < 30) return; // if this is a progress update, skip it if too soon
+            f.last.time = now;
+            f.last.text = text;
+            if(m) {
+                text = m[1];
+                progressElement.value = parseInt(m[2])*100;
+                progressElement.max = parseInt(m[4])*100;
+                progressElement.hidden = false;
+                spinnerElement.hidden = false;
+            } else {
+                progressElement.remove();
+                if(!text) spinnerElement.remove();
+            }
+            if(text) statusElement.innerText = text;
+            else statusElement.remove();
+        },
+        totalDependencies: 0,
+        monitorRunDependencies: function(left) {
+            this.totalDependencies = Math.max(this.totalDependencies, left);
+            this.setStatus(left
+                           ? ('Preparing... (' + (this.totalDependencies-left)
+                              + '/' + this.totalDependencies + ')')
+                           : 'All downloads complete.');
+        },
+        /* Loads sqlite3-api.js and calls the given callback (if
+           provided), passing it the sqlite3 module. Whether this is
+           synchronous or async depends on whether it's run in the
+           main thread or a worker.*/
+        loadSqliteAPI: function(callback){
+            const theScript = 'sqlite3-api.js';
+            if(self.importScripts){/*worker*/
+                importScripts(theScript);
+                if(callback) callback(self.sqlite3);
+            }else{/*main thread*/
+                new Promise((resolve, reject) => {
+                    const script = document.createElement('script');
+                    document.body.appendChild(script);
+                    script.onload = resolve;
+                    script.onerror = reject;
+                    script.async = true;
+                    script.src = theScript;
+                }).then(() => {
+                    if(callback) callback(self.sqlite3);
+                });
+            }
+        }
+    };
+})(self/*window or worker*/);

ext/fiddle/testing1.html

@@ -0,0 +1,31 @@
+<!doctype html>
+<html lang="en-us">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    <link rel="stylesheet" href="emscripten.css"/>
+    <title>sqlite3-api.js tests</title>
+    <style></style>
+  </head>
+  <body>
+    <header id='titlebar'><span>sqlite3-api.js tests</span></header>
+    <!-- emscripten bits -->
+    <figure id="module-spinner">
+      <div class="spinner"></div>
+      <div class='center'><strong>Initializing app...</strong></div>
+      <div class='center'>
+        On a slow internet connection this may take a moment.  If this
+        message displays for "a long time", intialization may have
+        failed and the JavaScript console may contain clues as to why.
+      </div>
+    </figure>
+    <div class="emscripten" id="module-status">Downloading...</div>
+    <div class="emscripten">
+      <progress value="0" max="100" id="module-progress" hidden='1'></progress>  
+    </div><!-- /emscripten bits -->
+    <div>Everything on this page happens in the dev console.</div>
+    <script src="testing-common.js"></script>
+    <script src="testing1.js"></script>
+    <script src="sqlite3.js"></script>
+  </body>
+</html>

ext/fiddle/testing1.js

@@ -0,0 +1,23 @@
+/*
+  2022-05-22
+
+  The author disclaims copyright to this source code.  In place of a
+  legal notice, here is a blessing:
+
+  *   May you do good and not evil.
+  *   May you find forgiveness for yourself and forgive others.
+  *   May you share freely, never taking more than you give.
+
+  ***********************************************************************
+
+  A basic test script for sqlite3-api.js.
+*/
+(function(){
+    self.Module.onRuntimeInitialized = function(){
+        console.log("Loading sqlite3-api.js...");
+        self.Module.loadSqliteAPI(function(S){
+            console.log("Loaded module:",S.sqlite3_libversion(),
+                        S.sqlite3_sourceid());
+        });
+    };
+})(self/*window or worker*/);