Add (#base64 -f filename) support to the preprocessor and slightly simplify the wasm embedding with it. Fix a bug in the preprocessor's @token@ call expansion when the length of the configured @ is greater than 1 byte.

FossilOrigin-Name: 1e90bedec99054f4856bfb89efc4d676486504db67bd1304c45d14c638ddfc7e
2026-10-23 21:49:02 +03:00 · 2026-07-22 14:06:41 +00:00
parent 9c20fed370
commit 7d1796e488
4 changed files with 105 additions and 32 deletions
--- a/ext/wasm/api/pre-js.c-pp.js
+++ b/ext/wasm/api/pre-js.c-pp.js
@@ -136,11 +136,9 @@
 */}
 //%define ? base64.wasm 0
 //%if base64.wasm and defined sqlite3.wasm.base64
-    let b64Wasm = `
-//%base64
-//%include -raw [arg sqlite3.wasm.base64]
-//%/base64
-`;
+//%@ push policy error delimiter '<<<' '>>>'
+    let b64Wasm = `<<<[base64 -file sqlite3.wasm.base64]>>>`;
+//%@ pop policy delimiter
    const bytes = Uint8Array.fromBase64(b64Wasm);
    b64Wasm = null;

--- a/ext/wasm/libcmpp.c
+++ b/ext/wasm/libcmpp.c
@@ -15,13 +15,13 @@

  ./c-pp -I. -I./src -Dsrcdir=./src -Dsed=/usr/bin/sed -o libcmpp.h ./tool/libcmpp.c-pp.h -o libcmpp.c ./tool/libcmpp.c-pp.c

-  with libcmpp 2.0.x ed6b3b14709bba639fa506517cfe9f7415d4a807912cb28df06cfcf19a526176 @ 2026-07-21 11:54:45.237 UTC
+  with libcmpp 2.0.x 7f7a9127b675829bf0f834bc13db17b69f7c53061d54efbfd32cfd5ae6e735f5 @ 2026-07-22 14:02:02.120 UTC
 */
 #define CMPP_PACKAGE_NAME "libcmpp"
 #define CMPP_LIB_VERSION "2.0.x"
-#define CMPP_LIB_VERSION_HASH "ed6b3b14709bba639fa506517cfe9f7415d4a807912cb28df06cfcf19a526176"
-#define CMPP_LIB_VERSION_TIMESTAMP "2026-07-21 11:54:45.237 UTC"
-#define CMPP_LIB_CONFIG_TIMESTAMP "2026-07-21 14:56 GMT"
+#define CMPP_LIB_VERSION_HASH "7f7a9127b675829bf0f834bc13db17b69f7c53061d54efbfd32cfd5ae6e735f5"
+#define CMPP_LIB_VERSION_TIMESTAMP "2026-07-22 14:02:02.120 UTC"
+#define CMPP_LIB_CONFIG_TIMESTAMP "2026-07-22 14:02 GMT"
 #define CMPP_VERSION CMPP_LIB_VERSION " " CMPP_LIB_VERSION_HASH " @ " CMPP_LIB_VERSION_TIMESTAMP
 #define CMPP_PLATFORM_EXT_DLL ".so"
 #define CMPP_MODULE_PATH ".:/usr/local/lib/cmpp"
@@ -1008,7 +1008,7 @@ CMPP_EXPORT int cmpp_flush_f_FILE(void * pFile);

   ```
   cmpp_b os = cmpp_b_empty;
-   FILE * f = cmpp_fopen(...);
+   cmpp_FILE * f = cmpp_fopen(...);
   rc = cmpp_stream(cmpp_input_f_FILE, f, cmpp_output_f_b, &os);
   // On error os might be partially populated.
   // Eventually clean up the buffer:
@@ -6716,8 +6716,8 @@ static int cmpp__out_expand(cmpp * pp, cmpp_outputer * pOut,
            if( nl && dxp && dxp->flags.countLines ){
              dxp->pos.lineNo +=nl;
            }
-            cmpp_call_str(pp, z+delim->open.n,
-                          (zb - z - delim->open.n),
+            //g_warn("Found: <<%.*s>>", (int)(zb - z -1), z+1);
+            cmpp_call_str(pp, z+1, (zb - z - 1),
                          cmpp_b_reuse(bCall), 0);
            if( 0==ppCode ){
              cmpp__out2(pp, pOut, bCall->z, bCall->n);
@@ -13404,37 +13404,104 @@ end:

 /** Impl for #base64. */
 static void cmpp_dx_f_base64(cmpp_dx *dx){
-  cmpp_api_init(dx->pp);
  cmpp_b ob = cmpp_b_empty;
  cmpp_flag32_t consumeFlags = cmpp_dx_consume_F_PROCESS_OTHER_D;
  bool comma = false;
  bool quote = false;
  cmpp_arg const *arg = 0;
-  for( arg = dx->args.arg0; arg; arg = arg->next ){
+  char * zResolved = 0;
+  cmpp_args args = cmpp_args_empty;
+  cmpp_b * const bFile = cmpp_b_borrow(dx->pp) /* -f|-file NAME */;
+  const bool isCall = cmpp_dx_is_call(dx);
+  //bool interpretFile = false;
+  if( !bFile || cmpp_dx_args_clone(dx, &args) ){
+    goto end;
+  }
+  for( arg = args.arg0; arg; arg = arg->next ){

 #define M(STR) cmpp_arg_equals(arg, STR)
    if( M("-comma") ) comma = true;
    else if( M("-quote") ) quote = true;
-    else{
+    else if( M("-f") || M("-file")
+             //|| M("-F") || M("-FILE")
+    ){
+      if( !arg->next ){
+        cmpp_dx_err_set(dx, CMPP_RC_MISUSE,
+                        "Expecting a filename argument for %s", arg->z);
+        goto end;
+      }
+      arg = arg->next;
+      assert( !bFile->z );
+      if( cmpp_arg_to_b(dx, arg, bFile,
+                        cmpp_arg_to_b_F_BRACE_CALL) ){
+        goto end;
+      }
+      if( !bFile->n || !bFile->z[0] ){
+        cmpp_dx_err_set(dx, CMPP_RC_MISUSE, "Empty filename is not permitted.");
+        goto end;
+      }
+#if 1
+      zResolved = sqlite3_mprintf("%.*s", (int)bFile->n, bFile->z)
+        /* memory must come from our allocator, not strndup() */;
+      if( cmpp_check_oom(dx->pp, zResolved) ){
+        goto end;
+      }
+#else
+      int nResolved = 0;
+      zResolved = cmpp__include_search(dx->pp, bFile->z, &nResolved);
+      if( !zResolved ){
+        if( !dxppCode ){
+          cmpp_dx_err_set(dx, CMPP_RC_NOT_FOUND, "file not found: %s",
+                          bFile->z);
+        }
+        goto end;
+      }
+#endif
+      //interpretFile = (char)'F'==(char)arg->z[1];
+    }else{
      cmpp_dx_err_set(dx, CMPP_RC_MISUSE,
                      "Unhandled argument: %s", arg->z);
-      return;
+      goto end;
    }
 #undef M
  }
-  if( cmpp_dx_consume_b(dx, &ob, &dx->d->closer, 1, consumeFlags) ){
+  if( zResolved ){
+    cmpp_outputer out = cmpp_outputer_b;
+    out.state = cmpp_b_reuse(bFile);
+    cmpp_FILE * const fp = cmpp_fopen(zResolved, "r");
+    if( fp ){
+      /* TODO (2026-07-22): -F|-FILE NAME tells it to preprocess the
+         input file before encoding, for which we can use
+         cmpp_process_stream(dx->pp,zResolved,cmpp_input_f_FILE, fp).
+         We need to temporarily redirect dx->pp's output to ob before
+         doing so. */
+      int const rc = cmpp_stream(cmpp_input_f_FILE, fp, out.out, out.state);
+      if( rc ){
+        cmpp_dx_err_set(dx, rc, "Unknown error streaming file %s.",
+                        zResolved);
+      }
+      cmpp_fclose(fp);
+    }else{
+      cmpp_dx_err_set(dx, cmpp_errno_rc(errno, CMPP_RC_IO),
+                      "Unknown error opening file %s.", zResolved);
+    }
+    cmpp_b_swap(bFile, &ob);
+  }else if( isCall ){
+    cmpp_dx_err_set(dx, CMPP_RC_MISUSE,
+                    "#base64.../#base64 is not legal for [call]. "
+                    "Pass the -f FILENAME flag instead.");
    goto end;
-  }else{
+  }else if( cmpp_dx_consume_b(dx, &ob, &dx->d->closer, 1, consumeFlags) ){
+    goto end;
+  }
+  if( ob.n ){
    cmpp_b * const b64 = cmpp_b_borrow(dx->pp);
    if( !b64 ) goto end;
    int cmpp__b_base64_encode(cmpp *, cmpp_b const *, cmpp_b *)/* in b.c */;
    cmpp__b_base64_encode(dx->pp, &ob, b64);
-    if( cmpp_dx_err_check(dx) ){
-      cmpp_b_return(dx->pp, b64);
-      goto end;
-    }
    cmpp_b_swap(&ob, b64);
    cmpp_b_return(dx->pp, b64);
+    if( cmpp_dx_err_check(dx) ) goto end;
  }
  unsigned char const * zEnd = ob.z + ob.n;
 #define out(P,N) if( cmpp_dx_out_raw(dx, P, N) ) goto end
@@ -13470,6 +13537,14 @@ static void cmpp_dx_f_base64(cmpp_dx *dx){
  }
 #undef out
 end:
+  if( bFile ){
+    /* ob has its whole contents, i.e. probably a large buffer, so
+       let's keep that one. */
+    cmpp_b_swap(&ob, bFile);
+    cmpp_b_return(dx->pp, bFile);
+  }
+  cmpp_mfree(zResolved);
+  cmpp_args_cleanup(&args);
  cmpp_b_clear(&ob);
  return;
 }
@@ -13670,6 +13745,7 @@ int cmpp__d_delayed_load(cmpp *pp, char const *zName){
    dNoRows->closer = dQ->closer;
    assert( !dQ->impl.state );
    dQ->impl.state = dNoRows;
+    goto end;
  }
 #endif /*CMPP_OMIT_D_DB*/

--- a/17
+++ b/17
@@ -1,5 +1,5 @@
-C Do\snot\scache\sthe\sresults\sof\sgetenv()\sin\sthe\stemporary\sfilename\sgeneration\nlogic\sas\sthat\smight\sresult\sin\sa\sUAF\sif\san\senvironment\svariable\svalue\sis\nmodified\susing\ssetenv()\sor\ssimilar.
-D 2026-07-22T12:55:23.697
+C Add\s(#base64\s-f\sfilename)\ssupport\sto\sthe\spreprocessor\sand\sslightly\ssimplify\sthe\swasm\sembedding\swith\sit.\sFix\sa\sbug\sin\sthe\spreprocessor's\s@token@\scall\sexpansion\swhen\sthe\slength\sof\sthe\sconfigured\s@\sis\sgreater\sthan\s1\sbyte.
+D 2026-07-22T14:06:41.366
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -598,7 +598,7 @@ F ext/wasm/api/opfs-common-inline.c-pp.js 496ca858af09b7fef2efaece467960611d35f5
 F ext/wasm/api/opfs-common-shared.c-pp.js 9a9019bcc070d7c90829af5f5eb81d25135bf9543944a691631c0184f0c62416
 F ext/wasm/api/post-js-footer.js a50c1a2c4d008aede7b2aa1f18891a7ee71437c2f415b8aeb3db237ddce2935b
 F ext/wasm/api/post-js-header.js f35d2dcf1ab7f22a93d565f8e0b622a2934fc4e743edf3b708e4dd8140eeff55
-F ext/wasm/api/pre-js.c-pp.js da89ab1d97b02b18cf2c33618b330ff747f6182667d8d164e7fe9acc39ea9a0a
+F ext/wasm/api/pre-js.c-pp.js f573ffab962c14d43119ef8879e42c8fb0804260888492c8da9151a8e1ca215e
 F ext/wasm/api/sqlite3-api-glue.c-pp.js 31a721ada7225838a61310a9f3f797fa5275353f8e9b0ae769d85b437be061f5
 F ext/wasm/api/sqlite3-api-oo1.c-pp.js 35e4727010f15fd72ead0dd1eb4e3c2c9bb1cc60e51544cbdff1f7c14f209de2
 F ext/wasm/api/sqlite3-api-prologue.js 0084e15d66fbcd75cacbaa58e3b473d5e57082c30f3122be7fdadff5589cf6b6
@@ -636,7 +636,7 @@ F ext/wasm/index-dist.html db23748044e286773f2768eec287669501703b5d5f72755e8db73
 F ext/wasm/index.html 5bf6cf1b0a3c8b9f5f54d77f2219d7ae87a15162055ce308109c49b1dcab4239
 F ext/wasm/jaccwabyt/jaccwabyt.js 4e2b797dc170851c9c530c3567679f4aa509eec0fab73b466d945b00b356574b
 F ext/wasm/jaccwabyt/jaccwabyt.md 6aa90fa1a973d0ad10d077088bea163b241d8470c75eafdef87620a1de1dea41
-F ext/wasm/libcmpp.c 58ca45c18bc68710fa8f37bd11aa49a0ea937b9d95580c40b5ad89174aa07976
+F ext/wasm/libcmpp.c 62b8aa62c8920fdced9abb00c46c0ceb98814ef6724800c433a52fc288632bc2
 F ext/wasm/mkdist.sh f8883b077a2ca47cf92e6f0ce305fbf72ca648c3501810125056c4b09c2d5554 x
 F ext/wasm/mkwasmbuilds.c d4648ef369df6ea3b4e046494b3077f0baa5732282301a7418fdda39ec505a0f
 F ext/wasm/module-symbols.html e54f42112e0aac2a31f850ab33e7f2630a2ea4f63496f484a12469a2501e07e2
@@ -2217,9 +2217,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee
 F tool/warnings.sh a554d13f6e5cf3760f041b87939e3d616ec6961859c3245e8ef701d1eafc2ca2
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
 F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c
-P 5fc1ede8b2e559db8a5d783d1065b1f1b2eef73054486552b700706208cbdfd1 82e522ff1429e7eed2ffb0e960badc5757fd015023b7c0679ba32108e44e4abd
-R 46939929c46affb76858b635fea36ba9
-T +closed 82e522ff1429e7eed2ffb0e960badc5757fd015023b7c0679ba32108e44e4abd
-U drh
-Z bdf1a4651337684379b93c8dd7823792
+P cefd5807d8be41f8ff0ce1ce487064dcbcf2f05583b86dbe68a3ae1960b13d9f
+R 8ea013803e4cd9648be5185c211d4bc6
+U stephan
+Z 65de0d9d521302a56e9a802fadaa1c25
 # Remove this line to create a well-formed Fossil manifest.
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-cefd5807d8be41f8ff0ce1ce487064dcbcf2f05583b86dbe68a3ae1960b13d9f
+1e90bedec99054f4856bfb89efc4d676486504db67bd1304c45d14c638ddfc7e