/* ** 2024-09-23 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ************************************************************************* ** ** This app's single purpose is to emit parts of the Makefile code for ** sqlite3's canonical WASM build. The main motivation is to generate ** code which "could" be created via GNU Make's eval command but is ** highly illegible when constructed that way. Attempts to write this ** app in Bash and TCL have suffered from the problem that those ** languages require escaping $ symbols, making the resulting script ** code as illegible as the eval spaghetti we want to get away ** from. Maintaining it in C is, somewhat surprisingly, _slightly_ ** less illegible than writing it in bash, tcl, or native Make code. ** ** The emitted makefile code is not standalone - it depends on ** variables and $(call)able functions from the main makefile. */ #undef NDEBUG #define DEBUG 1 #include #include #include #define pf printf #define ps puts /* ** Valid build names. Each build is a combination of one of these and ** one of JS_BUILD_MODES, but only certain combinations are legal. ** This macro and JS_BUILD_MODES exist solely for documentation ** purposes: they are not expanded into code anywhere. */ #define JS_BUILD_NAMES sqlite3 sqlite3-wasmfs /* ** Valid build modes. For the "sqlite3-wasmfs" build, only "esm" (ES6 ** Module) is legal. */ #define JS_BUILD_MODES vanilla esm bundler-friendly node /* Separator to help eyeballs find the different output sections */ static const char * zBanner = "\n########################################################################\n"; /* ** Flags for use with BuildDef::flags and the 3rd argument to ** mk_pre_post(). ** ** Maintenance reminder: do not combine flags within this enum, ** e.g. LIBMODE_BUNDLER_FRIENDLY=0x02|LIBMODE_ESM, as that will lead ** to breakage in some of the flag checks. */ enum LibModeFlags { /* Indicates an ESM module build. */ LIBMODE_ESM = 0x01, /* Indicates a "bundler-friendly" build mode. */ LIBMODE_BUNDLER_FRIENDLY = 0x02, /* Indicates that this build is unsupported. Such builds are not ** added to the 'all' target. The unsupported builds exist primarily ** for experimentation's sake. */ LIBMODE_UNSUPPORTED = 0x04, /* Indicates a node.js-for-node.js build (untested and ** unsupported). */ LIBMODE_NODEJS = 0x08, /* Indicates a wasmfs build (untested and unsupported). */ LIBMODE_WASMFS = 0x10 }; /* ** Info needed for building one combination of JS_BUILD_NAMES and ** JS_BUILD_MODE, noting that only a subset of those combinations are ** legal/sensical. */ struct BuildDef { const char *zName; /* Name from JS_BUILD_NAMES */ const char *zMode; /* Name from JS_BUILD_MODES */ int flags; /* Flags from LibModeFlags */ const char *zJsOut; /* Name of generated sqlite3.js/.mjs */ /* TODO: dynamically determine zJsOut based on zName, zMode, and flags. */ const char *zCmppD; /* Extra -D... flags for c-pp */ const char *zEmcc; /* Extra flags for emcc */ }; typedef struct BuildDef BuildDef; /* ** The set of WASM builds for the library (as opposed to the apps ** (fiddle, speedtest1)). This array must end with an empty sentinel ** entry. Their order is mostly insignificant, but some makefile vars ** used by some builds are set up by prior builds. Because of that, ** the (sqlite3, vanilla), (sqlite3, esm), and (sqlite3, ** bundler-friendly) builds should be defined first (in that order). */ const BuildDef aBuildDefs[] = { {/* Core build */ "sqlite3", "vanilla", 0, "$(sqlite3.js)", 0, 0}, {/* Core ESM */ "sqlite3", "esm", LIBMODE_ESM, "$(sqlite3.mjs)", "-Dtarget=es6-module", 0}, {/* Core bundler-friendly build. Untested and "not really" ** supported, but required by the downstream npm subproject. ** Testing these would require special-purpose node-based tools and ** custom test apps. Or we can pass them off as-is to the npm ** subproject and they spot failures pretty quickly ;). */ "sqlite3", "bundler-friendly", LIBMODE_BUNDLER_FRIENDLY | LIBMODE_ESM, "$(dir.dout)/sqlite3-bundler-friendly.mjs", "$(c-pp.D.sqlite3-esm) -Dtarget=es6-bundler-friendly", 0}, {/* node.js mode. Untested and unsupported. */ "sqlite3", "node", LIBMODE_UNSUPPORTED | LIBMODE_NODEJS, "$(dir.dout)/sqlite3-node.mjs", "$(c-pp.D.sqlite3-bundler-friendly) -Dtarget=node", 0}, {/* Wasmfs build. Fully unsupported and largely untested. */ "sqlite3-wasmfs", "esm" , LIBMODE_UNSUPPORTED | LIBMODE_WASMFS | LIBMODE_ESM, "$(dir.wasmfs)/sqlite3-wasmfs.mjs", "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs", "-sEXPORT_ES6 -sUSE_ES6_IMPORT_META"}, {/*End-of-list sentinel*/0,0,0,0,0,0} }; /* ** Emits common vars needed by the rest of the emitted code (but not ** needed by makefile code outside of these generated pieces). */ static void mk_prologue(void){ /* A 0-terminated list of makefile vars which we expect to have been ** set up by this point in the build process. */ char const * aRequiredVars[] = { "dir.top", "dir.api", "dir.dout", "dir.tmp", "sqlite3-license-version.js", "MAKEFILE", "MAKEFILE_LIST", /* Fiddle... */ "dir.fiddle", "dir.fiddle-debug", "MAKEFILE.fiddle", "EXPORTED_FUNCTIONS.fiddle", /*"just-testing",*/ 0 }; char const * zVar; int i; pf("%s# Build setup sanity checks...\n", zBanner); for( i = 0; (zVar = aRequiredVars[i]); ++i ){ pf("ifeq (,$(%s))\n", zVar); pf(" $(error build process error: expecting make var $$(%s) to " "have been set up by now)\n", zVar); ps("endif"); } pf("%s", zBanner); ps("# extern-post-js* and extern-pre-js* are files for use with"); ps("# Emscripten's --extern-pre-js and --extern-post-js flags."); ps("extern-pre-js.js = $(dir.api)/extern-pre-js.js"); ps("extern-post-js.js.in = $(dir.api)/extern-post-js.c-pp.js"); ps("# Emscripten flags for --[extern-][pre|post]-js=... for the"); ps("# various builds."); ps("pre-post-common.flags = --extern-pre-js=$(sqlite3-license-version.js)"); ps("# pre-post-jses.deps.* = a list of dependencies for the\n" "# --[extern-][pre/post]-js files."); ps("pre-post-jses.deps.common = $(extern-pre-js.js) $(sqlite3-license-version.js)"); { /* SQLITE.CALL.WASM-OPT = shell code to run $(1) (source wasm file ** name) through $(bin.wasm-opt) */ const char * zOptFlags = /* ** Flags for wasm-opt. It has many, many, MANY "passes" options ** and the ones which appear here were selected solely on the ** basis of trial and error. ** ** All wasm file size savings/costs mentioned below are based on ** the vanilla build of sqlite3.wasm with -Oz (our shipping ** configuration). Comments like "saves nothing" may not be ** technically correct: "nothing" means "some neglible amount." ** ** Note that performance gains/losses are _not_ taken into ** account here: only wasm file size. */ "--enable-bulk-memory-opt " /* required */ "--all-features " /* required */ "--post-emscripten " /* Saves roughly 12kb */ "--strip-debug " /* We already wasm-strip, but in ** case this environment has no ** wasm-strip... */ /* ** The rest are trial-and-error. See wasm-opt --help and search ** for "Optimization passes" to find the full list. ** ** With many flags this gets unusuably slow. */ /*"--converge " saves nothing for the options we're using */ /*"--dce " saves nothing */ /*"--directize " saves nothing */ /*"--gsi " no: requires --closed-world flag, which does not ** sound like something we want. */ /*"--gufa --gufa-cast-all --gufa-optimizing " costs roughly 2kb */ /*"--heap-store-optimization " saves nothing */ /*"--heap2local " saves nothing */ //"--inlining --inlining-optimizing " costs roughly 3kb */ "--local-cse " /* saves roughly 1kb */ /*"--once-reduction " saves nothing */ /*"--remove-memory-init " presumably a performance tweak */ /*"--remove-unused-names " saves nothing */ /*"--safe-heap "*/ /*"--vacuum " saves nothing */ ; ps("ifeq (,$(bin.wasm-opt))"); ps("define SQLITE.CALL.WASM-OPT"); ps("echo 'wasm-opt not available for $(1)'"); ps("endef"); ps("else"); ps("define SQLITE.CALL.WASM-OPT"); pf("echo -n 'Before wasm-opt:'; ls -l $(1);\\\n" "\trm -f wasm-opt-tmp.wasm;\\\n" /* It's very likely that the set of wasm-opt flags varies from ** version to version, so we'll ignore any errors here. */ "\tif $(bin.wasm-opt) $(1) -o wasm-opt-tmp.wasm \\\n" "\t\t%s; then \\\n" "\t\tmv wasm-opt-tmp.wasm $(1); \\\n" "\t\techo -n 'After wasm-opt: '; \\\n" "\t\tls -l $(1); \\\n" "\telse \\\n" "\t\techo 'WARNING: ignoring wasm-opt failure for $(1)'; \\\n" "\tfi\n", zOptFlags ); ps("endef"); ps("endif"); } } /* ** Emits makefile code for setting up values for the --pre-js=FILE, ** --post-js=FILE, and --extern-post-js=FILE emcc flags, as well as ** populating those files. */ static void mk_pre_post(const char *zName /* build name */, const char *zMode /* build mode */, const char *zCmppD /* optional -D flags for c-pp for the ** --pre/--post-js files. */){ /* Very common printf() args combo. */ #define zNM zName, zMode pf("%s# Begin --pre/--post flags for %s-%s\n", zBanner, zNM); pf("c-pp.D.%s-%s = %s\n", zNM, zCmppD ? zCmppD : ""); pf("pre-post-%s-%s.flags ?=\n", zNM); /* --pre-js=... */ pf("pre-js.js.%s-%s = $(dir.tmp)/pre-js.%s-%s.js\n", zNM, zNM); pf("$(pre-js.js.%s-%s): $(MAKEFILE_LIST) $(sqlite3-license-version.js)\n", zNM); #if 1 pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.%s-%s)," "$(c-pp.D.%s-%s)))\n", zNM, zNM); #else /* This part is needed if/when we re-enable the custom ** Module.instantiateModule() impl in api/pre-js.c-pp.js. */ pf("pre-js.js.%s-%s.intermediary = $(dir.tmp)/pre-js.%s-%s.intermediary.js\n", zNM, zNM); pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(pre-js.js.in),$(pre-js.js.%s-%s.intermediary)," "$(c-pp.D.%s-%s) -Dcustom-Module.instantiateModule))\n", zNM, zNM); pf("$(pre-js.js.%s-%s): $(pre-js.js.%s-%s.intermediary)\n", zNM, zNM); pf("\tcp $(pre-js.js.%s-%s.intermediary) $@\n", zNM); /* Amend $(pre-js.js.zName-zMode) for all targets except the plain ** "sqlite3" and the "sqlite3-wasmfs" builds... */ if( 0!=strcmp("sqlite3-wasmfs", zName) && 0!=strcmp("sqlite3", zName) ){ #error "This part ^^^ is needs adapting for use with the LIBMODE_... flags" pf("\t@echo 'Module[xNameOfInstantiateWasm].uri = " "\"%s.wasm\";' >> $@\n", zName); } #endif /* --post-js=... */ pf("post-js.js.%s-%s = $(dir.tmp)/post-js.%s-%s.js\n", zNM, zNM); pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(post-js.js.in)," "$(post-js.js.%s-%s),$(c-pp.D.%s-%s)))\n", zNM, zNM); /* --extern-post-js=... */ pf("extern-post-js.js.%s-%s = $(dir.tmp)/extern-post-js.%s-%s.js\n", zNM, zNM); pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.%s-%s)," "$(c-pp.D.%s-%s)))\n", zNM, zNM); /* Combined flags for use with emcc... */ pf("pre-post-common.flags.%s-%s = " "$(pre-post-common.flags) " "--post-js=$(post-js.js.%s-%s) " "--extern-post-js=$(extern-post-js.js.%s-%s)\n", zNM, zNM, zNM); pf("pre-post-%s-%s.flags += $(pre-post-common.flags.%s-%s) " "--pre-js=$(pre-js.js.%s-%s)\n", zNM, zNM, zNM); /* Set up deps... */ pf("pre-post-jses.%s-%s.deps = $(pre-post-jses.deps.common) " "$(post-js.js.%s-%s) $(extern-post-js.js.%s-%s)\n", zNM, zNM, zNM); pf("pre-post-%s-%s.deps = $(pre-post-jses.%s-%s.deps) $(dir.tmp)/pre-js.%s-%s.js\n", zNM, zNM, zNM); pf("# End --pre/--post flags for %s-%s%s", zNM, zBanner); #undef zNM } /* ** Emits rules for the fiddle builds. */ static void mk_fiddle(void){ int i = 0; mk_pre_post("fiddle-module","vanilla", 0); for( ; i < 2; ++i ){ /* 0==normal, 1==debug */ const char *zTail = i ? ".debug" : ""; const char *zDir = i ? "$(dir.fiddle-debug)" : "$(dir.fiddle)"; pf("%s# Begin fiddle%s\n", zBanner, zTail); pf("fiddle-module.js%s = %s/fiddle-module.js\n", zTail, zDir); pf("$(fiddle-module.js%s):%s $(MAKEFILE_LIST) $(MAKEFILE.fiddle) " "$(EXPORTED_FUNCTIONS.fiddle) " "$(fiddle.cses) $(pre-post-fiddle-module-vanilla.deps) " "$(SOAP.js)\n", zTail, (i ? " $(fiddle-module.js)" : "")); if( 1==i ){/*fiddle.debug*/ pf("\t@test -d \"$(dir $@)\" || mkdir -p \"$(dir $@)\"\n"); } pf("\t$(bin.emcc) -o $@ $(fiddle.emcc-flags%s) " "$(pre-post-fiddle-module-vanilla.flags) $(fiddle.cses)\n", zTail); ps("\t@chmod -x $(basename $@).wasm"); ps("\t@$(maybe-wasm-strip) $(basename $@).wasm"); ps("\t@$(SQLITE.strip-createExportWrapper)"); pf("\t@cp -p $(SOAP.js) $(dir $@)\n"); if( 1==i ){/*fiddle.debug*/ pf("\tcp -p $(dir.fiddle)/index.html " "$(dir.fiddle)/fiddle.js " "$(dir.fiddle)/fiddle-worker.js " "$(dir $@)\n"); } pf("\t@for i in %s/*.*js %s/*.html %s/*.wasm; do \\\n" "\t\ttest -f $${i} || continue; \\\n" "\t\tgzip < $${i} > $${i}.gz; \\\n" "\tdone\n", zDir, zDir, zDir); if( 0==i ){ ps("fiddle: $(fiddle-module.js)"); }else{ ps("fiddle-debug: $(fiddle-module.js.debug)"); } pf("# End fiddle%s%s", zTail, zBanner); } } /* ** Emits makefile code for one build of the library, primarily defined ** by the combination of zName and zMode, each of which must be values ** from JS_BUILD_NAMES resp. JS_BUILD_MODES. */ static void mk_lib_mode(const BuildDef * pB){ const char * zWasmOut = "$(basename $@).wasm" /* The various targets named X.js or X.mjs (zJsOut) also generate ** X.wasm, and we need that part of the name to perform some ** post-processing after Emscripten generates X.wasm. */; assert( pB->zName ); assert( pB->zMode ); assert( pB->zJsOut ); /* Very common printf() args combo. */ #define zNM pB->zName, pB->zMode pf("%s# Begin build [%s-%s]. flags=0x%02x\n", zBanner, zNM, pB->flags); pf("# zJsOut=%s\n# zCmppD=%s\n", pB->zJsOut, pB->zCmppD ? pB->zCmppD : ""); pf("$(info Setting up build [%s-%s]: %s)\n", zNM, pB->zJsOut); mk_pre_post(zNM, pB->zCmppD); pf("\nemcc.flags.%s.%s ?=\n", zNM); if( pB->zEmcc && pB->zEmcc[0] ){ pf("emcc.flags.%s.%s += %s\n", zNM, pB->zEmcc); } /* target pB->zJsOut */ pf("%s: $(MAKEFILE_LIST) $(sqlite3-wasm.cfiles) $(EXPORTED_FUNCTIONS.api) " "$(pre-post-%s-%s.deps) " "$(sqlite3-api.ext.jses)" /* ^^^ maintenance reminder: we set these as deps so that they ** get copied into place early. That allows the developer to ** reload the base-most test pages while the later-stage builds ** are still compiling, which is especially helpful when running ** builds with long build times (like -Oz). */ "\n", pB->zJsOut, zNM); pf("\t@echo \"Building $@ ...\"\n"); if( LIBMODE_UNSUPPORTED & pB->flags ){ ps("\t@echo 'ACHTUNG: $@ is an unsupported build. " "Use at your own risk.'"); } pf("\t$(bin.emcc) -o $@ $(emcc_opt_full) $(emcc.flags) \\\n"); pf("\t\t$(emcc.jsflags) -sENVIRONMENT=$(emcc.environment.%s) \\\n", pB->zMode); pf("\t\t$(pre-post-%s-%s.flags) \\\n", zNM); pf("\t\t$(emcc.flags.%s) $(emcc.flags.%s.%s) \\\n", pB->zName, zNM); pf("\t\t$(cflags.common) $(SQLITE_OPT) \\\n" "\t\t$(cflags.%s) $(cflags.%s.%s) \\\n" "\t\t$(cflags.wasm_extra_init) $(sqlite3-wasm.cfiles)\n", pB->zName, zNM); if( (LIBMODE_ESM & pB->flags) || (LIBMODE_NODEJS & pB->flags) ){ /* TODO? Replace this $(call) with the corresponding makefile ** code. OTOH, we also use this $(call) in the speedtest1-wasmfs ** build, which is not part of the rules emitted by this ** program. */ pf("\t@$(call SQLITE.CALL.xJS.ESM-EXPORT-DEFAULT,1,%d)\n", (LIBMODE_WASMFS & pB->flags) ? 1 : 0); } pf("\t@chmod -x %s\n", zWasmOut); pf("\t@$(maybe-wasm-strip) %s\n", zWasmOut); pf("\t@$(call SQLITE.CALL.WASM-OPT,%s)\n", zWasmOut); ps("\t@$(SQLITE.strip-createExportWrapper)"); /* ** The above $(bin.emcc) call will write pB->zJsOut, a.k.a. $@, and ** will create a like-named .wasm file (zWasmOut). That .wasm file ** name gets hard-coded into $@ so we need to, for some cases, patch ** zJsOut to use the name sqlite3.wasm instead. Note that the ** resulting .wasm file is identical for all builds for which zEmcc ** is empty. */ if( (LIBMODE_BUNDLER_FRIENDLY & pB->flags) ){ pf("\t@echo 'Patching $@ for %s.wasm...'; \\\n", pB->zName); pf("\t\trm -f %s; \\\n", zWasmOut); pf("\t\tsed -i -e 's/%s-%s.wasm/%s.wasm/g' $@ || exit;\n", /* ^^^^^^ reminder: Mac/BSD sed has no -i flag but this ** build process explicitly requires a Linux system. */ zNM, pB->zName); pf("\t@ls -la $@\n"); if( LIBMODE_BUNDLER_FRIENDLY & pB->flags ){ /* Avoid a 3rd occurrence of the bug fixed by 65798c09a00662a3, ** which was (in two cases) caused by makefile refactoring and ** not recognized until after a release was made with the broken ** sqlite3-bundler-friendly.mjs (which is used by the npm ** subproject but is otherwise untested/unsupported): */ pf("\t@if grep -e '^ *importScripts(' $@; " "then echo 'ERROR: bug fixed in 65798c09a00662a3 has re-appeared'; " "exit 1; fi;\n"); } }else{ pf("\t@ls -la %s $@\n", zWasmOut); } if( 0==(LIBMODE_UNSUPPORTED & pB->flags) ){ pf("all: %s\n", pB->zJsOut); } pf("# End build [%s-%s]%s", zNM, zBanner); #undef zNM } int main(void){ int rc = 0; const BuildDef *pB = &aBuildDefs[0]; pf("# What follows was GENERATED by %s. Edit at your own risk.\n", __FILE__); mk_prologue(); for( ; pB->zName; ++pB ){ mk_lib_mode( pB ); } mk_fiddle(); mk_pre_post("speedtest1","vanilla", 0); mk_pre_post("speedtest1-wasmfs","esm", "$(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs"); return rc; }