From 248c1abdf4c61bf0761994db40c214272a092c7c Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 8 Mar 2023 10:05:42 +0000 Subject: [PATCH] Extend wasm build to support a custom sqlite3.c to support building against sqlite3-see.c. The JS code now binds the SEE-specific functions if it detects an SEE build. FossilOrigin-Name: dd8612c8adbaf9d06bf0d7319b9afc9bd8ca3d0fcfa1cb591a7a2fcb86480048 --- ext/wasm/GNUmakefile | 46 ++++++++++++++++----- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see | 5 +++ ext/wasm/api/sqlite3-api-glue.js | 9 ++++ ext/wasm/api/sqlite3-wasm.c | 29 ++++++++++--- manifest | 19 +++++---- manifest.uuid | 2 +- 6 files changed, 85 insertions(+), 25 deletions(-) create mode 100644 ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index ae8a7d98e8..250b8be0c7 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -145,8 +145,27 @@ ifeq (,$(wildcard $(dir.tmp))) dir._tmp := $(shell mkdir -p $(dir.tmp)) endif -sqlite3.c := $(dir.top)/sqlite3.c +######################################################################## +# Set up sqlite3.c and sqlite3.h... +# +# To build with SEE (https://sqlite.org/see), either put sqlite3-see.c +# in the top of this build tree or pass +# sqlite3.c=PATH_TO_sqlite3-see.c to the build. Note that only +# encryption modules with no 3rd-party dependencies will currently +# work here: AES256-OFB, AES128-OFB, and AES128-CCM. Not +# coincidentally, those 3 modules are included in the sqlite3-see.c +# bundle. +# +# A custom sqlite3.c must not have any spaces in its name. +sqlite3.canonical.c := $(dir.top)/sqlite3.c +sqlite3.c ?= $(firstword $(wildcard $(dir.top)/sqlite3-see.c) $(sqlite3.canonical.c)) sqlite3.h := $(dir.top)/sqlite3.h +ifeq (,$(shell grep sqlite3_activate_see $(sqlite3.c) 2>/dev/null)) + SQLITE_C_IS_SEE := 0 +else + SQLITE_C_IS_SEE := 1 + $(info This is an SEE build.) +endif # Most SQLITE_OPT flags are set in sqlite3-wasm.c but we need them # made explicit here for building speedtest1.c. SQLITE_OPT = \ @@ -169,7 +188,8 @@ SQLITE_OPT = \ -DSQLITE_OS_KV_OPTIONAL=1 \ '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ -DSQLITE_USE_URI=1 \ - -DSQLITE_WASM_ENABLE_C_TESTS + -DSQLITE_WASM_ENABLE_C_TESTS \ + -DSQLITE_C=$(sqlite3.c) .NOTPARALLEL: $(sqlite3.h) $(sqlite3.h): @@ -226,7 +246,7 @@ endif # cleanup process for the dist build to work properly. bin.version-info := $(dir.wasm)/version-info $(bin.version-info): $(dir.wasm)/version-info.c $(sqlite3.h) $(MAKEFILE) - $(CC) -O0 -I$(dir.top) -o $@ $< + $(CC) -O0 -I$(dir $(sqlite3.c)) -o $@ $< DISTCLEAN_FILES += $(bin.version-info) # bin.stripcomments is used for stripping C/C++-style comments from JS @@ -284,7 +304,7 @@ endef ######################################################################## # cflags.common = C compiler flags for all builds -cflags.common := -I. -I.. -I$(dir.top) +cflags.common := -I. -I$(dir $(sqlite3.c)) # emcc.WASM_BIGINT = 1 for BigInt (C int64) support, else 0. The API # disables certain features if BigInt is not enabled and such builds # _are not tested_ on any regular basis. @@ -327,10 +347,14 @@ emcc_opt_full := $(emcc_opt) -g3 # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. -EXPORTED_FUNCTIONS.api.in := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) +EXPORTED_FUNCTIONS.api.main := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) +EXPORTED_FUNCTIONS.api.in := $(EXPORTED_FUNCTIONS.api.main) +ifeq (1,$(SQLITE_C_IS_SEE)) + EXPORTED_FUNCTIONS.api.in += $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see) +endif EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api -$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) - cp $(EXPORTED_FUNCTIONS.api.in) $@ +$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) + cat $(EXPORTED_FUNCTIONS.api.in) > $@ # sqlite3-license-version.js = generated JS file with the license # header and version info. @@ -834,9 +858,9 @@ speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 # -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app # which runs speedtest1 multiple times. -$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) +$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api.main) @echo "Making $@ ..." - @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api); } > $@ + @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.main); } > $@ speedtest1.js := $(dir.dout)/speedtest1.js speedtest1.wasm := $(dir.dout)/speedtest1.wasm cflags.speedtest1 := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM @@ -847,9 +871,11 @@ $(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." $(emcc.bin) \ - $(emcc.speedtest1) $(emcc.speedtest1.common) \ + $(emcc.speedtest1) -I$(dir $(sqlite3.canonical.c)) \ + $(emcc.speedtest1.common) \ $(cflags.speedtest1) $(pre-post-speedtest1.flags.vanilla) \ $(SQLITE_OPT) \ + -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c) \ $(speedtest1.exit-runtime0) \ -o $@ $(speedtest1.cses) -lm $(maybe-wasm-strip) $(speedtest1.wasm) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see new file mode 100644 index 0000000000..83f3a97dbc --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see @@ -0,0 +1,5 @@ +_sqlite3_key +_sqlite3_key_v2 +_sqlite3_rekey +_sqlite3_rekey_v2 +_sqlite3_activate_see diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 1cb00b9419..f444ec975c 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -327,6 +327,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); } + if(wasm.exports.sqlite3_activate_see instanceof Function){ + wasm.bindingSignatures.push( + ["sqlite3_key", "int", "sqlite3*", "string", "int"], + ["sqlite3_key_v2","int","sqlite3*","string","*","int"], + ["sqlite3_rekey", "int", "sqlite3*", "string", "int"], + ["sqlite3_rekey_v2", "int", "sqlite3*", "string", "*", "int"], + ["sqlite3_activate_see", undefined, "string"] + ); + } /** Functions which require BigInt (int64) support are separated from the others because we need to conditionally bind them or apply diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index bcabe922bf..fd074541ad 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -168,11 +168,6 @@ #endif #include -#include "sqlite3.c" /* yes, .c instead of .h. */ - -#if defined(__EMSCRIPTEN__) -# include -#endif /* ** SQLITE_WASM_EXPORT is functionally identical to EMSCRIPTEN_KEEPALIVE @@ -201,6 +196,30 @@ // See also: //__attribute__((export_name("theExportedName"), used, visibility("default"))) +/* +** Which sqlite3.c we're using needs to be configurable to enable +** building against a custom copy, e.g. the SEE variant. Note that we +** #include the .c file, rather than the header, so that the WASM +** extensions have access to private API internals. +** +** The caveat here is that custom variants need to account for +** exporting any necessary symbols (e.g. sqlite3_activate_see()). We +** cannot export them from here using SQLITE_WASM_EXPORT because that +** attribute (apparently) has to be part of the function definition. +*/ +#ifndef SQLITE_C +# define SQLITE_C sqlite3.c /* yes, .c instead of .h. */ +#endif +#define INC__STRINGIFY_(f) #f +#define INC__STRINGIFY(f) INC__STRINGIFY_(f) +#include INC__STRINGIFY(SQLITE_C) +#undef INC__STRINGIFY_ +#undef INC__STRINGIFY +#undef SQLITE_C + +#if defined(__EMSCRIPTEN__) +# include +#endif #if 0 /* diff --git a/manifest b/manifest index 7bfa0a5377..3f8746bfab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\scount-of-view\soptimization\sthat\scan\slead\sto\sincorrect\nbytecode.\s\sdbsqlfuzz\s23d782160b71c3f8f535ccb2da313dfc8eb8c631. -D 2023-03-08T00:47:53.498 +C Extend\swasm\sbuild\sto\ssupport\sa\scustom\ssqlite3.c\sto\ssupport\sbuilding\sagainst\ssqlite3-see.c.\sThe\sJS\scode\snow\sbinds\sthe\sSEE-specific\sfunctions\sif\sit\sdetects\san\sSEE\sbuild. +D 2023-03-08T10:05:42.118 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -468,10 +468,11 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 6c0c0e9d3b6ef2090f230bc47f8da5d9614fbdefacb72e8bafcc9a41d035605b +F ext/wasm/GNUmakefile 97f01bb84b0b745e2ba642e12ab24a682e369538e8d68b6e67bdbe502450fc6c F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 77a2f1f2fc60a35def7455dffc8d3f2c56385d6ac5c6cecc60fa938252ea2c54 F ext/wasm/api/extern-post-js.c-pp.js 5c4997d3442756e4e4819303fa4a7045de4a2a7b79c3ad6c26cdcf1d9141fac6 @@ -480,7 +481,7 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js cc21e3486da748463e02bbe51e2464c6ac136587cdfd5aa00cd0b5385f6ca808 -F ext/wasm/api/sqlite3-api-glue.js 32091c2730ecef8f5795c3527d3db6b7bbf054d12e27311ce2da59db559b2e0e +F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 2691a34a741015127b210954a1b9586764d3ff0c8a20f00fd15c00f339ecc79f F ext/wasm/api/sqlite3-api-prologue.js df8646e4f92b8b09cef255da3530e11dc264a2e8d53b0e78daa2ee04f99c584d F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d660194ddfc51eab1c5b9bf @@ -489,7 +490,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 26f6240170d415726d9cfe2fa7a0163e153775e1a74fa91c9ba5446502c71097 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 223d30c41d811cae8b9f1175fa68f2f1fb3cc056d16ad0def3b0ea5c65757a6c +F ext/wasm/api/sqlite3-wasm.c 4b09550e1a73f6318b47bdacdbd39045acc23f562fedbdc1d8af1b7473763e7b F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2048,8 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1096b5a7cc8104db01f8820ace47020baad2f12e6711e3a7b4514ed1becc7b66 -R 132c7e609c38a7be8ca466782d538c7f -U drh -Z 36cb7a4789e4e3ff0efced0434fbf8d1 +P f45009533a79a59b302598ee2545ef787c51d0128f4e1dca60dd83589f660619 +R cd7604a48b0beb018a7a99fee097a4b7 +U stephan +Z 52942618230bb2090251a07b83122b0f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c0bc28f12c..38299f9abf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f45009533a79a59b302598ee2545ef787c51d0128f4e1dca60dd83589f660619 \ No newline at end of file +dd8612c8adbaf9d06bf0d7319b9afc9bd8ca3d0fcfa1cb591a7a2fcb86480048 \ No newline at end of file