From 8a1104940d76c9c948cfdda5a61cf2e0aa8c3d04 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 11 Jul 2023 19:54:17 +0000 Subject: [PATCH 01/72] Reduce divergence between branches: trunk and wal-shm-exceptions. FossilOrigin-Name: d38ed6b1fc6b572997d520c36f256c11404fb1bd5f28acab07466bfc96554a8f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 18 ++++-------------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 58dc9db6f2..e4e210686c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\suse\sthe\s"LL"\ssuffix\son\s64-bit\sinteger\sliterals. -D 2023-07-08T17:42:24.748 +C Reduce\sdivergence\sbetween\sbranches:\strunk\sand\swal-shm-exceptions. +D 2023-07-11T19:54:17.575 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -719,7 +719,7 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac F src/vtab.c 1ecf8c3745d29275688d583e12822fa984d421e0286b5ef50c137bc3bf6d7a64 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c cbfeeb7415baa545efa244dd34bb5af4ae953a206fed720c6fa7f1ef763ec122 +F src/wal.c dd843f619ac60d5dadab7109cf402432ba74dde0c301505fd1c202add07659e3 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 F src/where.c 2dc708cf8b6a691fb79f16bbc46567497ee6f991043318d421e294b2da114d93 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P beab3c98639be531744e60440223bb9ee76bc15234aff05e5efb273c8241dfd8 -R 0aeac1606c7bbc75dfae0a73d28601e5 +P 07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7 +R 9ba462c35151490c11339704cc10d776 U drh -Z 7fe2cdb685f595b3ff5ea313f1ff0511 +Z 72367f42719b51ed960b0f45070bb83e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1161b8bd7f..90dbe2fcde 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7 \ No newline at end of file +d38ed6b1fc6b572997d520c36f256c11404fb1bd5f28acab07466bfc96554a8f \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index f4d0328b2b..747cf099e3 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1818,23 +1818,15 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3_malloc64(nByte); + p = (WalIterator *)sqlite3_malloc64(nByte + + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) + ); if( !p ){ return SQLITE_NOMEM_BKPT; } memset(p, 0, nByte); p->nSegment = nSegment; - - /* Allocate temporary space used by the merge-sort routine. This block - ** of memory will be freed before this function returns. - */ - aTmp = (ht_slot *)sqlite3_malloc64( - sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) - ); - if( !aTmp ){ - rc = SQLITE_NOMEM_BKPT; - } - + aTmp = (ht_slot*)&(((u8*)p)[nByte]); for(i=walFramePage(nBackfill+1); rc==SQLITE_OK && iaSegment[i].aPgno = (u32 *)sLoc.aPgno; } } - sqlite3_free(aTmp); - if( rc!=SQLITE_OK ){ walIteratorFree(p); p = 0; From a17b04f84035ffc5f979cfa2b776c2290b982e87 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 12 Jul 2023 09:16:18 +0000 Subject: [PATCH 02/72] Correct handling of the speedtest1 --without-rowid flag in the HTML front-end to speedtest1.wasm. Reduce default speedtest1-with-OPFS workload size in ext/wasm/index.html to speed up manual testing. FossilOrigin-Name: 2daba2cdd3890c87899ff3eb0c1706c814f0ae62e72822eef3b0d4a885679a6b --- ext/wasm/index.html | 4 ++-- ext/wasm/speedtest1-worker.html | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/wasm/index.html b/ext/wasm/index.html index a9872a9eaf..42a5fe6638 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -93,8 +93,8 @@ diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index 0d88049b95..3d0fda86ac 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -259,7 +259,7 @@ flags["--utf16be"] = "Set text encoding to UTF-16BE"; flags["--utf16le"] = "Set text encoding to UTF-16LE"; flags["--verify"] = "Run additional verification steps."; - flags["--without"] = "rowid Use WITHOUT ROWID where appropriate"; + flags["--without-rowid"] = "Use WITHOUT ROWID where appropriate"; const preselectedFlags = [ '--big-transactions', '--singlethread' diff --git a/manifest b/manifest index e4e210686c..78af027310 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sdivergence\sbetween\sbranches:\strunk\sand\swal-shm-exceptions. -D 2023-07-11T19:54:17.575 +C Correct\shandling\sof\sthe\sspeedtest1\s--without-rowid\sflag\sin\sthe\sHTML\sfront-end\sto\sspeedtest1.wasm.\sReduce\sdefault\sspeedtest1-with-OPFS\sworkload\ssize\sin\sext/wasm/index.html\sto\sspeed\sup\smanual\stesting. +D 2023-07-12T09:16:18.602 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -531,14 +531,14 @@ F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa0 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e -F ext/wasm/index.html dd900891844caebd9cadbddd704f66bd841d7c12fd69ce5af490e2c10fb49f45 +F ext/wasm/index.html bba92ca0c173b579356533226a85a28797d1789e893c9b483fc0623842842a0f F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 F ext/wasm/speedtest1-wasmfs.html 7a301f4f5b6ad4f5d37fd6e7ca03a2f5d5547fd289da60a39075a93d7646d354 -F ext/wasm/speedtest1-worker.html 82869822e641c1bef3ec0cd2d7d2b6a42d0b4f68a7b160fb2e1dd0b523940a9b +F ext/wasm/speedtest1-worker.html 97c2bf5f8534091ce718de05801090d5a80c3f13575996f095ba23638e1bdca0 F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07d95ed60f0a17ea13b4bc19c2ab2ec9052fedd27c9e1e57a1ec6e3a6470e5b7 -R 9ba462c35151490c11339704cc10d776 -U drh -Z 72367f42719b51ed960b0f45070bb83e +P d38ed6b1fc6b572997d520c36f256c11404fb1bd5f28acab07466bfc96554a8f +R eae9af4d4222dea3bda054d259796f22 +U stephan +Z 73b3cb1c198f637825060346ee535239 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 90dbe2fcde..145db7a72a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d38ed6b1fc6b572997d520c36f256c11404fb1bd5f28acab07466bfc96554a8f \ No newline at end of file +2daba2cdd3890c87899ff3eb0c1706c814f0ae62e72822eef3b0d4a885679a6b \ No newline at end of file From b3b64179dfca82f9bb5d7ad6e9fe48eb6b0acd36 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 12 Jul 2023 09:30:13 +0000 Subject: [PATCH 03/72] Internal JS doc fix. FossilOrigin-Name: 7c5ea02205a360a11fa9043be12c678eb7fe6f6a0b63b09a3dc976beae881e3f --- ext/wasm/api/sqlite3-api-cleanup.js | 2 +- ext/wasm/api/sqlite3-api-prologue.js | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index d38b401bf7..3661464e1b 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -23,7 +23,7 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build const SABC = Object.assign( Object.create(null), { exports: Module['asm'], - memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */ + memory: Module.wasmMemory /* gets set if built with -sIMPORTED_MEMORY */ }, globalThis.sqlite3ApiConfig || {} ); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index c882d5b247..77739db57d 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -53,7 +53,7 @@ - `memory`[^1]: optional WebAssembly.Memory object, defaulting to `exports.memory`. In Emscripten environments this should be set - to `Module.wasmMemory` if the build uses `-sIMPORT_MEMORY`, or be + to `Module.wasmMemory` if the build uses `-sIMPORTED_MEMORY`, or be left undefined/falsy to default to `exports.memory` when using WASM-exported memory. @@ -809,7 +809,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( || toss3("Missing API config.exports (WASM module exports)."), /** - When Emscripten compiles with `-sIMPORT_MEMORY`, it + When Emscripten compiles with `-sIMPORTED_MEMORY`, it initalizes the heap and imports it into wasm, as opposed to the other way around. In this case, the memory is not available via this.exports.memory. diff --git a/manifest b/manifest index 78af027310..2610aac269 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\shandling\sof\sthe\sspeedtest1\s--without-rowid\sflag\sin\sthe\sHTML\sfront-end\sto\sspeedtest1.wasm.\sReduce\sdefault\sspeedtest1-with-OPFS\sworkload\ssize\sin\sext/wasm/index.html\sto\sspeed\sup\smanual\stesting. -D 2023-07-12T09:16:18.602 +C Internal\sJS\sdoc\sfix. +D 2023-07-12T09:30:13.598 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,10 +494,10 @@ F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 -F ext/wasm/api/sqlite3-api-cleanup.js cc21e3486da748463e02bbe51e2464c6ac136587cdfd5aa00cd0b5385f6ca808 +F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 -F ext/wasm/api/sqlite3-api-prologue.js 17f4ec398ba34c5c666fea8e8c4eb82064a35b302f2f2eb355283cd8d3f68ed5 +F ext/wasm/api/sqlite3-api-prologue.js 9608d7844f1ffa50add1d8a3138219d9cba87a6515067120dafe37e0d009deec F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d38ed6b1fc6b572997d520c36f256c11404fb1bd5f28acab07466bfc96554a8f -R eae9af4d4222dea3bda054d259796f22 +P 2daba2cdd3890c87899ff3eb0c1706c814f0ae62e72822eef3b0d4a885679a6b +R b36af1931765289ef77591fcd1f62a3a U stephan -Z 73b3cb1c198f637825060346ee535239 +Z 81dbb73ce7c058b780d72b7803ddc1b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 145db7a72a..044eb05c28 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2daba2cdd3890c87899ff3eb0c1706c814f0ae62e72822eef3b0d4a885679a6b \ No newline at end of file +7c5ea02205a360a11fa9043be12c678eb7fe6f6a0b63b09a3dc976beae881e3f \ No newline at end of file From a5512fb17910ca7369fb93302cf92bafdcead360 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 13 Jul 2023 01:48:53 +0000 Subject: [PATCH 04/72] Remove obsolete ext/wasm/api/sqlite3-wasi.h. FossilOrigin-Name: f661a6e0c1ac081ee402e64590eabd427440db1b717be9166bdfc025d0b0d899 --- ext/wasm/api/sqlite3-wasi.h | 69 ------------------------------------- manifest | 11 +++--- manifest.uuid | 2 +- 3 files changed, 6 insertions(+), 76 deletions(-) delete mode 100644 ext/wasm/api/sqlite3-wasi.h diff --git a/ext/wasm/api/sqlite3-wasi.h b/ext/wasm/api/sqlite3-wasi.h deleted file mode 100644 index 096f45dfec..0000000000 --- a/ext/wasm/api/sqlite3-wasi.h +++ /dev/null @@ -1,69 +0,0 @@ -/** - Dummy function stubs to get sqlite3.c compiling with - wasi-sdk. This requires, in addition: - - -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_GETPID - - -lwasi-emulated-getpid -*/ -typedef unsigned mode_t; -int fchmod(int fd, mode_t mode); -int fchmod(int fd, mode_t mode){ - return (fd && mode) ? 0 : 0; -} -typedef unsigned uid_t; -typedef uid_t gid_t; -int fchown(int fd, uid_t owner, gid_t group); -int fchown(int fd, uid_t owner, gid_t group){ - return (fd && owner && group) ? 0 : 0; -} -uid_t geteuid(void); -uid_t geteuid(void){return 0;} -#if !defined(F_WRLCK) -enum { -F_WRLCK, -F_RDLCK, -F_GETLK, -F_SETLK, -F_UNLCK -}; -#endif - -#undef HAVE_PREAD - -#include -#define WASM__KEEP __attribute__((used)) - -#if 0 -/** - wasi-sdk cannot build sqlite3's default VFS without at least the following - functions. They are apparently syscalls which clients have to implement or - otherwise obtain. - - https://github.com/WebAssembly/WASI/blob/main/phases/snapshot/docs.md -*/ -environ_get -environ_sizes_get -clock_time_get -fd_close -fd_fdstat_get -fd_fdstat_set_flags -fd_filestat_get -fd_filestat_set_size -fd_pread -fd_prestat_get -fd_prestat_dir_name -fd_read -fd_seek -fd_sync -fd_write -path_create_directory -path_filestat_get -path_filestat_set_times -path_open -path_readlink -path_remove_directory -path_unlink_file -poll_oneoff -proc_exit -#endif diff --git a/manifest b/manifest index 2610aac269..501a8cd76c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Internal\sJS\sdoc\sfix. -D 2023-07-12T09:30:13.598 +C Remove\sobsolete\sext/wasm/api/sqlite3-wasi.h. +D 2023-07-13T01:48:53.998 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -503,7 +503,6 @@ F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d299 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 05f2563ddebfdc7a0f0ac0eb7cb381bb72043299aae1600ba9367c12f52b3fcc -F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 @@ -2043,8 +2042,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2daba2cdd3890c87899ff3eb0c1706c814f0ae62e72822eef3b0d4a885679a6b -R b36af1931765289ef77591fcd1f62a3a +P 7c5ea02205a360a11fa9043be12c678eb7fe6f6a0b63b09a3dc976beae881e3f +R 119e33c4ca65919c57a8b13cbaf75c29 U stephan -Z 81dbb73ce7c058b780d72b7803ddc1b3 +Z ed73669b2c57061d23ff2e04c6cf3602 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 044eb05c28..bca95f7257 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c5ea02205a360a11fa9043be12c678eb7fe6f6a0b63b09a3dc976beae881e3f \ No newline at end of file +f661a6e0c1ac081ee402e64590eabd427440db1b717be9166bdfc025d0b0d899 \ No newline at end of file From 5da49ff5629c3e598a71081ab45d1bef79926f4b Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 13 Jul 2023 01:51:16 +0000 Subject: [PATCH 05/72] JS: change the default OPFS VFS journal mode from PERSIST to DELETE, as DELETE is faster in current browsers. FossilOrigin-Name: a061a43e36b8f3cb7c465fd2e54cfcd9dceb07e232ee78602bbdd1818acd05ea --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 25 +++++++++++++++---------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 07dfa0f960..31099c7cec 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -298,6 +298,7 @@ const installOpfsVfs = function callee(options){ lock contention to free up. */ state.asyncIdleWaitTime = 150; + /** Whether the async counterpart should log exceptions to the serialization channel. That produces a great deal of @@ -1089,7 +1090,7 @@ const installOpfsVfs = function callee(options){ propagate any exception on error, rather than returning false. */ opfsUtil.unlink = async function(fsEntryName, recursive = false, - throwOnError = false){ + throwOnError = false){ try { const [hDir, filenamePart] = await opfsUtil.getDirForFilename(fsEntryName, false); @@ -1186,19 +1187,23 @@ const installOpfsVfs = function callee(options){ contention. */ sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); sqlite3.capi.sqlite3_exec(oo1Db, [ - /* Truncate journal mode is faster than delete for - this vfs, per speedtest1. That gap seems to have closed with - Chrome version 108 or 109, but "persist" is very roughly 5-6% - faster than truncate in initial tests. + /* As of July 2023, the PERSIST journal mode on OPFS is + somewhat slower than DELETE or TRUNCATE (it was faster + before Chrome version 108 or 109). TRUNCATE and DELETE + have very similar performance on OPFS. - For later analysis: Roy Hashimoto notes that TRUNCATE - and PERSIST modes may decrease OPFS concurrency because - multiple connections can open the journal file in those - modes: + Roy Hashimoto notes that TRUNCATE and PERSIST modes may + decrease OPFS concurrency because multiple connections + can open the journal file in those modes: https://github.com/rhashimoto/wa-sqlite/issues/68 + + Given that, and the fact that testing has not revealed + any appreciable difference between performance of + TRUNCATE and DELETE modes on OPFS, we currently (as of + 2023-07-13) default to DELETE mode. */ - "pragma journal_mode=persist;", + "pragma journal_mode=DELETE;", /* This vfs benefits hugely from cache on moderate/large speedtest1 --size 50 and --size 100 workloads. We diff --git a/manifest b/manifest index 501a8cd76c..5912114cba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sobsolete\sext/wasm/api/sqlite3-wasi.h. -D 2023-07-13T01:48:53.998 +C JS:\schange\sthe\sdefault\sOPFS\sVFS\sjournal\smode\sfrom\sPERSIST\sto\sDELETE,\sas\sDELETE\sis\sfaster\sin\scurrent\sbrowsers. +D 2023-07-13T01:51:16.511 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 05f2563ddebfdc7a0f0ac0eb7cb381bb72043299aae1600ba9367c12f52b3fcc +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js f42feb4f492f770330e95400d0ff9b03b90978f6bcc7f3b34ec4cb70b57dc40e F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 @@ -2042,8 +2042,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7c5ea02205a360a11fa9043be12c678eb7fe6f6a0b63b09a3dc976beae881e3f -R 119e33c4ca65919c57a8b13cbaf75c29 +P f661a6e0c1ac081ee402e64590eabd427440db1b717be9166bdfc025d0b0d899 +R 68f402355eff532485c97ad92873be22 U stephan -Z ed73669b2c57061d23ff2e04c6cf3602 +Z e4c28ef3556be6a3a591892b05be6957 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bca95f7257..74410aa9fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f661a6e0c1ac081ee402e64590eabd427440db1b717be9166bdfc025d0b0d899 \ No newline at end of file +a061a43e36b8f3cb7c465fd2e54cfcd9dceb07e232ee78602bbdd1818acd05ea \ No newline at end of file From 81602595a0eb65272e83f98a2a4e581bed757996 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 13 Jul 2023 03:46:43 +0000 Subject: [PATCH 06/72] Replace 'self' with 'globalThis' in some JS test code. FossilOrigin-Name: 53eeffc9b1ea8c3ee7d7cdcc220997ed893403e45b35d8ea10805ea3b0435587 --- ext/wasm/tester1.c-pp.js | 18 +++++++++--------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 1fb3303df1..ea4a2b479a 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -45,7 +45,7 @@ */ //#if target=es6-module import {default as sqlite3InitModule} from './jswasm/sqlite3.mjs'; -self.sqlite3InitModule = sqlite3InitModule; +globalThis.sqlite3InitModule = sqlite3InitModule; //#else 'use strict'; //#endif @@ -57,7 +57,7 @@ self.sqlite3InitModule = sqlite3InitModule; */ let logClass; /* Predicate for tests/groups. */ - const isUIThread = ()=>(self.window===self && self.document); + const isUIThread = ()=>(globalThis.window===self && globalThis.document); /* Predicate for tests/groups. */ const isWorker = ()=>!isUIThread(); /* Predicate for tests/groups. */ @@ -3050,14 +3050,14 @@ self.sqlite3InitModule = sqlite3InitModule; //////////////////////////////////////////////////////////////////////// log("Loading and initializing sqlite3 WASM module..."); if(0){ - self.sqlite3ApiConfig = { + globalThis.sqlite3ApiConfig = { debug: ()=>{}, log: ()=>{}, warn: ()=>{}, error: ()=>{} } } - if(!self.sqlite3InitModule && !isUIThread()){ + if(!globalThis.sqlite3InitModule && !isUIThread()){ /* Vanilla worker, as opposed to an ES6 module worker */ /* If sqlite3.js is in a directory other than this script, in order @@ -3070,27 +3070,27 @@ self.sqlite3InitModule = sqlite3InitModule; that's not needed. URL arguments passed as part of the filename via importScripts() - are simply lost, and such scripts see the self.location of + are simply lost, and such scripts see the globalThis.location of _this_ script. */ let sqlite3Js = 'sqlite3.js'; - const urlParams = new URL(self.location.href).searchParams; + const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('sqlite3.dir')){ sqlite3Js = urlParams.get('sqlite3.dir') + '/' + sqlite3Js; } importScripts(sqlite3Js); } - self.sqlite3InitModule.__isUnderTest = + globalThis.sqlite3InitModule.__isUnderTest = true /* disables certain API-internal cleanup so that we can test internal APIs from here */; - self.sqlite3InitModule({ + globalThis.sqlite3InitModule({ print: log, printErr: error }).then(function(sqlite3){ //console.log('sqlite3 =',sqlite3); log("Done initializing WASM/JS bits. Running tests..."); sqlite3.config.warn("Installing sqlite3 bits as global S for local dev/test purposes."); - self.S = sqlite3; + globalThis.S = sqlite3; capi = sqlite3.capi; wasm = sqlite3.wasm; log("sqlite3 version:",capi.sqlite3_libversion(), diff --git a/manifest b/manifest index 5912114cba..39db92f161 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C JS:\schange\sthe\sdefault\sOPFS\sVFS\sjournal\smode\sfrom\sPERSIST\sto\sDELETE,\sas\sDELETE\sis\sfaster\sin\scurrent\sbrowsers. -D 2023-07-13T01:51:16.511 +C Replace\s'self'\swith\s'globalThis'\sin\ssome\sJS\stest\scode. +D 2023-07-13T03:46:43.944 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -547,7 +547,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 1a05497ae2b2fcca008d43b37072f9b841e1c970c06c01eb0faf675db567bfc8 +F ext/wasm/tester1.c-pp.js 439a1264bc3224b90bc722833568cc36fc77ed0ba6d2e4379b40ea7c417c97ab F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2042,8 +2042,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f661a6e0c1ac081ee402e64590eabd427440db1b717be9166bdfc025d0b0d899 -R 68f402355eff532485c97ad92873be22 +P a061a43e36b8f3cb7c465fd2e54cfcd9dceb07e232ee78602bbdd1818acd05ea +R e5f9c2b543ee2e3a15f3537a9d58e252 U stephan -Z e4c28ef3556be6a3a591892b05be6957 +Z c5917bb24dee9d73dec2b26ad6758a4f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 74410aa9fd..2830c84e4a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a061a43e36b8f3cb7c465fd2e54cfcd9dceb07e232ee78602bbdd1818acd05ea \ No newline at end of file +53eeffc9b1ea8c3ee7d7cdcc220997ed893403e45b35d8ea10805ea3b0435587 \ No newline at end of file From 01bdff7a1c64b258261e1555957d95a203aba590 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 13 Jul 2023 04:26:13 +0000 Subject: [PATCH 07/72] Get wasmfs build and its bare-bones test app working again, albeit currently in ES6 mode only. FossilOrigin-Name: 647761ed422f196f94facc88bbddd7219a2c1a6301a5f847b0a32d3e405233a7 --- ext/wasm/GNUmakefile | 13 ++++++--- ext/wasm/api/extern-post-js.c-pp.js | 7 ++--- ext/wasm/api/sqlite3-api-prologue.js | 11 ++----- ext/wasm/scratchpad-wasmfs-main.html | 12 ++++++-- ext/wasm/scratchpad-wasmfs-main.js | 16 ++++++++++- ext/wasm/speedtest1-wasmfs.html | 2 +- ext/wasm/wasmfs.make | 43 +++++++++++++++++----------- manifest | 27 +++++++++-------- manifest.uuid | 2 +- 9 files changed, 81 insertions(+), 52 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index a99513bfa8..85a5ef5054 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -655,7 +655,8 @@ sqlite3-wasm.cses := $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) # SQLITE3.xJS.EXPORT-DEFAULT is part of SQLITE3-WASMFS.xJS.RECIPE and # SETUP_LIB_BUILD_MODE, factored into a separate piece to avoid code # duplication. $1 is 1 if the build mode needs this workaround (esm, -# bundler-friendly) and 0 if not (vanilla). +# bundler-friendly) and 0 if not (vanilla). $2 must be empty for all +# builds except sqlite3-wasmfs.mjs, in which case it must be 1. # # Reminder for ESM builds: even if we use -sEXPORT_ES6=0, emcc _still_ # adds: @@ -673,9 +674,11 @@ define SQLITE3.xJS.ESM-EXPORT-DEFAULT if [ x1 = x$(1) ]; then \ echo "Fragile workaround for an Emscripten annoyance. See SQLITE3.xJS.RECIPE."; \ sed -i -e '0,/^export default/{/^export default/d;}' $@ || exit $$?; \ - if ! grep -q '^export default' $@; then \ - echo "Cannot find export default." 1>&2; \ - exit 1; \ + if [ x != x$(2) ]; then \ + if ! grep -q '^export default' $@; then \ + echo "Cannot find export default." 1>&2; \ + exit 1; \ + fi; \ fi; \ fi endef @@ -981,6 +984,8 @@ include fiddle.make ifneq (,$(filter wasmfs,$(MAKECMDGOALS))) wasmfs.enable ?= 1 else +# Unconditionally enable wasmfs for [dist]clean so that the wasmfs +# sub-make can clean up. wasmfs.enable ?= $(if $(filter %clean,$(MAKECMDGOALS)),1,0) endif ifeq (1,$(wasmfs.enable)) diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index 927bf64f9e..6b544fe1cd 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -23,10 +23,7 @@ const toExportForESM = impls which Emscripten installs at some point in the file above this. */ - const originalInit = - /* Maintenance reminder: DO NOT use `self.` here. It's correct - for non-ES6 Module cases but wrong for ES6 modules because those - resolve this symbol differently. */ sqlite3InitModule; + const originalInit = sqlite3InitModule; if(!originalInit){ throw new Error("Expecting globalThis.sqlite3InitModule to be defined by the Emscripten build."); } @@ -124,5 +121,5 @@ const toExportForESM = return globalThis.sqlite3InitModule /* required for ESM */; })(); //#if target=es6-module -export default toExportForESM; +export { toExportForESM as default, toExportForESM as sqlite3InitModule } //#endif diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 77739db57d..98e97355c7 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -92,8 +92,8 @@ currently work due to incompatible Emscripten-side changes made in the WASMFS+OPFS combination. This option is currently ignored. - [^1] = This property may optionally be a function, in which case this - function re-assigns calls that function to fetch the value, + [^1] = This property may optionally be a function, in which case + this function calls that function to fetch the value, enabling delayed evaluation. The returned object is the top-level sqlite3 namespace object. @@ -149,11 +149,6 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( config[k] = config[k](); } }); - config.wasmOpfsDir = - /* 2022-12-17: WASMFS+OPFS can no longer be activated from the - main thread (aborts via a failed assert() if it's attempted), - which eliminates any(?) benefit to supporting it. */ false; - /** The main sqlite3 binding API gets installed into this object, mimicking the C API as closely as we can. The numerous members @@ -1200,8 +1195,6 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( if(undefined !== __wasmfsOpfsDir) return __wasmfsOpfsDir; // If we have no OPFS, there is no persistent dir const pdir = config.wasmfsOpfsDir; - console.error("sqlite3_wasmfs_opfs_dir() can no longer work due "+ - "to incompatible WASMFS changes. It will be removed."); if(!pdir || !globalThis.FileSystemHandle || !globalThis.FileSystemDirectoryHandle diff --git a/ext/wasm/scratchpad-wasmfs-main.html b/ext/wasm/scratchpad-wasmfs-main.html index 91f61526cd..f78b412885 100644 --- a/ext/wasm/scratchpad-wasmfs-main.html +++ b/ext/wasm/scratchpad-wasmfs-main.html @@ -33,8 +33,16 @@

All stuff on this page happens in the dev console.


- + - + + + diff --git a/ext/wasm/scratchpad-wasmfs-main.js b/ext/wasm/scratchpad-wasmfs-main.js index 56f9325de5..3fb8d6af06 100644 --- a/ext/wasm/scratchpad-wasmfs-main.js +++ b/ext/wasm/scratchpad-wasmfs-main.js @@ -14,6 +14,9 @@ main JS thread and sqlite3.js must have been loaded before it. */ 'use strict'; +//importScripts('jswasm/sqlite3-wasmfs.js'); +import sqlite3InitModule from './jswasm/sqlite3-wasmfs.mjs'; +//console.log('sqlite3InitModule =',sqlite3InitModule); (function(){ const toss = function(...args){throw new Error(args.join(' '))}; const log = console.log.bind(console), @@ -38,6 +41,7 @@ const capi = sqlite3.capi, oo = sqlite3.oo1, wasm = sqlite3.wasm; + stdout("Loaded module:",sqlite3); stdout("Loaded sqlite3:",capi.sqlite3_libversion(), capi.sqlite3_sourceid()); const persistentDir = capi.sqlite3_wasmfs_opfs_dir(); if(persistentDir){ @@ -66,5 +70,15 @@ stdout("Total test time:",(performance.now() - startTime),"ms"); }; - sqlite3InitModule(self.sqlite3TestModule).then(runTests); + sqlite3InitModule(globalThis.sqlite3TestModule).then((X)=>{ + /* + 2023-07-13: we're passed the Emscripten Module object here + instead of the sqlite3 object. This differs from the canonical + build and is a side effect of WASMFS's requirement + that the module init function (i.e. sqlite3InitModule()) + return its initial arrgument (the Emscripten Module). + */ + stdout("then() got",X.sqlite3,X); + runTests(X.sqlite3 || X); + }); })(); diff --git a/ext/wasm/speedtest1-wasmfs.html b/ext/wasm/speedtest1-wasmfs.html index 4c4db32bca..3c69285a18 100644 --- a/ext/wasm/speedtest1-wasmfs.html +++ b/ext/wasm/speedtest1-wasmfs.html @@ -91,7 +91,7 @@ }; const runTests = function(sqlite3){ - console.log("Module inited."); + console.log("Module inited.",sqlite3); const wasm = sqlite3.wasm; const __unlink = wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["*","string"]); const unlink = (fn)=>__unlink(0,fn); diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make index 020014f3e8..791f7ba839 100644 --- a/ext/wasm/wasmfs.make +++ b/ext/wasm/wasmfs.make @@ -6,14 +6,10 @@ # GNUMakefile. ######################################################################## MAKEFILE.wasmfs := $(lastword $(MAKEFILE_LIST)) +$(warning The WASMFS build is currently incomplete.) -# Maintenance reminder: these particular files cannot be built into a -# subdirectory because loading of the auxiliary -# sqlite3-wasmfs.worker.js file it creates fails if sqlite3-wasmfs.js -# is loaded from any directory other than the one in which the -# containing HTML lives. Similarly, they cannot be loaded from a -# Worker to an Emscripten quirk regarding loading nested Workers. -dir.wasmfs := $(dir.wasm) +#dir.wasmfs := $(dir.wasm) +dir.wasmfs := $(dir.dout) sqlite3-wasmfs.js := $(dir.wasmfs)/sqlite3-wasmfs.js sqlite3-wasmfs.mjs := $(dir.wasmfs)/sqlite3-wasmfs.mjs sqlite3-wasmfs.wasm := $(dir.wasmfs)/sqlite3-wasmfs.wasm @@ -38,7 +34,6 @@ emcc.flags.sqlite3-wasmfs += --no-entry emcc.flags.sqlite3-wasmfs += --minify 0 emcc.flags.sqlite3-wasmfs += -sMODULARIZE emcc.flags.sqlite3-wasmfs += -sEXPORT_NAME=$(sqlite3.js.init-func) -emcc.flags.sqlite3-wasmfs += -sSTRICT_JS emcc.flags.sqlite3-wasmfs += -sDYNAMIC_EXECUTION=0 emcc.flags.sqlite3-wasmfs += -sNO_POLYFILL emcc.flags.sqlite3-wasmfs += -sWASM_BIGINT=$(emcc.WASM_BIGINT) @@ -68,8 +63,8 @@ emcc.flags.sqlite3-wasmfs += $(sqlite3-wasmfs.fsflags) # And, indeed, it runs slowly if memory is permitted to grow. emcc.flags.sqlite3-wasmfs.vanilla := emcc.flags.sqlite3-wasmfs.esm := -sEXPORT_ES6 -sUSE_ES6_IMPORT_META -$(eval $(call call-make-pre-js,sqlite3-wasmfs,vanilla)) -$(eval $(call call-make-pre-js,sqlite3-wasmfs,esm)) +$(eval $(call call-make-pre-post,sqlite3-wasmfs,vanilla)) +$(eval $(call call-make-pre-post,sqlite3-wasmfs,esm)) Xemcc.flags.sqlite3-wasmfs.vanilla += \ $(pre-post-common.flags.vanilla) \ $(pre-post-sqlite3-wasmfs.flags.vanilla) @@ -89,17 +84,32 @@ define SQLITE3-WASMFS.xJS.RECIPE $(emcc.flags.sqlite3-wasmfs) $(emcc.flags.sqlite3-wasmfs.$(1)) \ $(pre-post-sqlite3-wasmfs.flags.$(1)) \ $(sqlite3-wasm.c) - @$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(1)) + @$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(if $(filter %.mjs,$@),1,)) chmod -x $(sqlite3-wasmfs.wasm) $(maybe-wasm-strip) $(sqlite3-wasmfs.wasm) - @ls -la $(sqlite3-wasmfs.wasm) sqlite3-wasmfs*js + @ls -la $(sqlite3-wasmfs.wasm) $(dir.wasmfs)/sqlite3-wasmfs*js endef +######################################################################## +# Build quirk: we cannot build BOTH .js and .mjs with our current +# build infrastructure because the supplemental *.worker.js files get +# generated with the name of the main module file +# ($(sqlite3-wasmfs.{js,mjs})) hard-coded in them. Thus the last one +# to get built gets the *.worker.js files mapped to it. In order to +# build both modes they would need to have distinct base names or +# output directories. +# +wasmfs.build.ext := mjs +ifeq (js,$(wasmfs.build.mode)) $(sqlite3-wasmfs.js): $(call SQLITE3-WASMFS.xJS.RECIPE,vanilla) -$(sqlite3-wasmfs.mjs): $(sqlite3-wasmfs.js) - $(call SQLITE3-WASMFS.xJS.RECIPE,esm) $(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.js) -wasmfs: $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs) +wasmfs: $(sqlite3-wasmfs.js) +else +$(sqlite3-wasmfs.mjs): # $(sqlite3-wasmfs.js) + $(call SQLITE3-WASMFS.xJS.RECIPE,esm) +$(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.mjs) +wasmfs: $(sqlite3-wasmfs.mjs) +endif #all: wasmfs ######################################################################## @@ -124,8 +134,7 @@ $(speedtest1-wasmfs.js): $(speedtest1.cses) $(sqlite3-wasmfs.js) \ $(maybe-wasm-strip) $(speedtest1-wasmfs.wasm) ls -la $@ $(speedtest1-wasmfs.wasm) -#speedtest1: $(speedtest1-wasmfs.js) -wasmfs: $(speedtest1-wasmfs.js) +#wasmfs: $(speedtest1-wasmfs.js) CLEAN_FILES += $(speedtest1-wasmfs.js) $(speedtest1-wasmfs.wasm) \ $(subst .js,.worker.js,$(speedtest1-wasmfs.js)) # end speedtest1.js diff --git a/manifest b/manifest index 39db92f161..c99cedfe6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\s'self'\swith\s'globalThis'\sin\ssome\sJS\stest\scode. -D 2023-07-13T03:46:43.944 +C Get\swasmfs\sbuild\sand\sits\sbare-bones\stest\sapp\sworking\sagain,\salbeit\scurrently\sin\sES6\smode\sonly. +D 2023-07-13T04:26:13.675 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,14 +482,14 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 38700d5074af690f004e4e5f3533164ab49693b9d0832929c4ecf97a0bc09494 +F ext/wasm/GNUmakefile 319505a36ceb8debffd5f0d3243e6396ee5f623a68529d205fb0c7155df832cf F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 77a2f1f2fc60a35def7455dffc8d3f2c56385d6ac5c6cecc60fa938252ea2c54 -F ext/wasm/api/extern-post-js.c-pp.js 393ab78b807da94096eae1a68bfddb999a2299936a185d910162fe87a57a9a3a +F ext/wasm/api/extern-post-js.c-pp.js a91c73765b8909d0fddad72328f9396fee1f5fb58435e73cff20ef713e67d014 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 @@ -497,7 +497,7 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 -F ext/wasm/api/sqlite3-api-prologue.js 9608d7844f1ffa50add1d8a3138219d9cba87a6515067120dafe37e0d009deec +F ext/wasm/api/sqlite3-api-prologue.js b20856fb72a07a99de2bea24cf66dd8fc7dbf6514280884e0ff91e3d0859ed3b F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 @@ -534,9 +534,9 @@ F ext/wasm/index.html bba92ca0c173b579356533226a85a28797d1789e893c9b483fc0623842 F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10 -F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 -F ext/wasm/scratchpad-wasmfs-main.js 4c140457f4d6da9d646a49addd91edb6e9ad1643c6c48e3258b5bce24725dc18 -F ext/wasm/speedtest1-wasmfs.html 7a301f4f5b6ad4f5d37fd6e7ca03a2f5d5547fd289da60a39075a93d7646d354 +F ext/wasm/scratchpad-wasmfs-main.html a70452766ed8dd88f7c4b3603dbf8de34c76b734614fd67729f0961a82af19c3 +F ext/wasm/scratchpad-wasmfs-main.js 59cbc259921f3ff521cdfafdbe278df885b5f92e0e279aacacf6fb12ff3be5eb +F ext/wasm/speedtest1-wasmfs.html 3493c46b24bad00e9f6ebe14804143ae67a880f6c59c16ee13bf5f06e7dc4313 F ext/wasm/speedtest1-worker.html 97c2bf5f8534091ce718de05801090d5a80c3f13575996f095ba23638e1bdca0 F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da @@ -552,7 +552,7 @@ F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a9 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 -F ext/wasm/wasmfs.make cf9a68162d92ca2bcb0b9528b244cb36d5cc2d84ccc9c2d398461927d6e75aea +F ext/wasm/wasmfs.make 4d34310493108e297b46084959ae06f634ac21446898fc464223ca33cd05a508 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 @@ -2042,8 +2042,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a061a43e36b8f3cb7c465fd2e54cfcd9dceb07e232ee78602bbdd1818acd05ea -R e5f9c2b543ee2e3a15f3537a9d58e252 +P 53eeffc9b1ea8c3ee7d7cdcc220997ed893403e45b35d8ea10805ea3b0435587 +R 6632e735d7b6678fb19e864586c8977f +T *branch * wasmfs-2023 +T *sym-wasmfs-2023 * +T -sym-trunk * Cancelled\sby\sbranch. U stephan -Z c5917bb24dee9d73dec2b26ad6758a4f +Z 18788228280efa16edd900f1ed072054 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2830c84e4a..269dc636d7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53eeffc9b1ea8c3ee7d7cdcc220997ed893403e45b35d8ea10805ea3b0435587 \ No newline at end of file +647761ed422f196f94facc88bbddd7219a2c1a6301a5f847b0a32d3e405233a7 \ No newline at end of file From 0f0e5700ba0c88de229eacb797d87551d987cf33 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 13 Jul 2023 04:54:18 +0000 Subject: [PATCH 08/72] Minor cleanups in the wasmfs build. FossilOrigin-Name: f64477f21040c265061ad9b7f601e74159fd6141e7e4b122c80bf244ab6ddb76 --- ext/wasm/wasmfs.make | 25 ++++++++++++++----------- manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make index 791f7ba839..36290ffbf6 100644 --- a/ext/wasm/wasmfs.make +++ b/ext/wasm/wasmfs.make @@ -85,9 +85,10 @@ define SQLITE3-WASMFS.xJS.RECIPE $(pre-post-sqlite3-wasmfs.flags.$(1)) \ $(sqlite3-wasm.c) @$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(if $(filter %.mjs,$@),1,)) - chmod -x $(sqlite3-wasmfs.wasm) - $(maybe-wasm-strip) $(sqlite3-wasmfs.wasm) - @ls -la $(sqlite3-wasmfs.wasm) $(dir.wasmfs)/sqlite3-wasmfs*js + @dotwasm=$(basename $@).wasm; \ + chmod -x $$dotwasm; \ + $(maybe-wasm-strip) $$dotwasm; \ + ls -la $$dotwasm $@ endef ######################################################################## # Build quirk: we cannot build BOTH .js and .mjs with our current @@ -96,19 +97,21 @@ endef # ($(sqlite3-wasmfs.{js,mjs})) hard-coded in them. Thus the last one # to get built gets the *.worker.js files mapped to it. In order to # build both modes they would need to have distinct base names or -# output directories. +# output directories. "The problem" with giving them distinct base +# names is that it means that the corresponding .wasm file is also +# built/saved multiple times. # wasmfs.build.ext := mjs -ifeq (js,$(wasmfs.build.mode)) $(sqlite3-wasmfs.js): $(call SQLITE3-WASMFS.xJS.RECIPE,vanilla) -$(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.js) -wasmfs: $(sqlite3-wasmfs.js) -else -$(sqlite3-wasmfs.mjs): # $(sqlite3-wasmfs.js) +$(sqlite3-wasmfs.mjs): $(call SQLITE3-WASMFS.xJS.RECIPE,esm) -$(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.mjs) -wasmfs: $(sqlite3-wasmfs.mjs) +ifeq (js,$(wasmfs.build.ext)) + $(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.js) + wasmfs: $(sqlite3-wasmfs.js) +else + $(sqlite3-wasmfs.wasm): $(sqlite3-wasmfs.mjs) + wasmfs: $(sqlite3-wasmfs.mjs) endif #all: wasmfs diff --git a/manifest b/manifest index c99cedfe6a..38342083a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\swasmfs\sbuild\sand\sits\sbare-bones\stest\sapp\sworking\sagain,\salbeit\scurrently\sin\sES6\smode\sonly. -D 2023-07-13T04:26:13.675 +C Minor\scleanups\sin\sthe\swasmfs\sbuild. +D 2023-07-13T04:54:18.496 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -552,7 +552,7 @@ F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a9 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 -F ext/wasm/wasmfs.make 4d34310493108e297b46084959ae06f634ac21446898fc464223ca33cd05a508 +F ext/wasm/wasmfs.make e7e1af321302e390b854a9940c16514c755070a2225a830d51e84eb9fea47222 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 @@ -2042,11 +2042,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 53eeffc9b1ea8c3ee7d7cdcc220997ed893403e45b35d8ea10805ea3b0435587 -R 6632e735d7b6678fb19e864586c8977f -T *branch * wasmfs-2023 -T *sym-wasmfs-2023 * -T -sym-trunk * Cancelled\sby\sbranch. +P 647761ed422f196f94facc88bbddd7219a2c1a6301a5f847b0a32d3e405233a7 +R e7ab42412381a677950ba361ea3eaf82 U stephan -Z 18788228280efa16edd900f1ed072054 +Z 333900e3d11fac56a338fcc42eb12fb0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 269dc636d7..271ae2c55c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -647761ed422f196f94facc88bbddd7219a2c1a6301a5f847b0a32d3e405233a7 \ No newline at end of file +f64477f21040c265061ad9b7f601e74159fd6141e7e4b122c80bf244ab6ddb76 \ No newline at end of file From d41d454fc1e95a6b8d92dd143667392f8effd5d6 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 13 Jul 2023 10:41:41 +0000 Subject: [PATCH 09/72] More work on the wasmfs build. Resolve the inconsistent argument type passed to sqlite3InitModule() in such builds. FossilOrigin-Name: 4850a9e2d2b41b91e91b9ef99a6d26ddf11a161b4c970fc00d2d095606729a04 --- ext/wasm/GNUmakefile | 16 ++++++---------- ext/wasm/api/extern-post-js.c-pp.js | 14 ++++++-------- ext/wasm/scratchpad-wasmfs-main.html | 18 ++---------------- ext/wasm/scratchpad-wasmfs-main.js | 13 ++----------- ext/wasm/wasmfs.make | 6 +++--- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- 7 files changed, 30 insertions(+), 59 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 85a5ef5054..60b33a3338 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -381,18 +381,14 @@ sqlite3-api.jses += $(dir.api)/sqlite3-v-helper.js sqlite3-api.jses += $(dir.api)/sqlite3-vfs-opfs.c-pp.js sqlite3-api.jses += $(dir.api)/sqlite3-api-cleanup.js -# "External" API files which are part of our distribution +# SOAP.js is an external API file which is part of our distribution # but not part of the sqlite3-api.js amalgamation. SOAP.js := $(dir.api)/sqlite3-opfs-async-proxy.js -# COPY_XAPI = a $(call)able function to copy $1 to $(dir.dout), where -# $1 must be one of the "external" JS API files. -define COPY_XAPI -sqlite3-api.ext.jses += $$(dir.dout)/$$(notdir $(1)) -$$(dir.dout)/$$(notdir $(1)): $(1) $$(MAKEFILE) - cp $$< $$@ -endef -$(foreach X,$(SOAP.js),\ - $(eval $(call COPY_XAPI,$(X)))) +SOAP.js.bld := $(dir.dout)/$(notdir $(SOAP.js)) +sqlite3-api.ext.jses += $(SOAP.js.bld) +$(SOAP.js.bld): $(SOAP.js) + cp $< $@ + all quick: $(sqlite3-api.ext.jses) q: quick diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index 6b544fe1cd..fac00370dd 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -63,18 +63,16 @@ const toExportForESM = //console.warn("Using replaced sqlite3InitModule()",globalThis.location); return originalInit(...args).then((EmscriptenModule)=>{ if('undefined'!==typeof WorkerGlobalScope && - (EmscriptenModule['ENVIRONMENT_IS_PTHREAD'] - || EmscriptenModule['_pthread_self'] - || 'function'===typeof threadAlert - || globalThis?.location?.pathname?.endsWith?.('.worker.js') - )){ + EmscriptenModule['ENVIRONMENT_IS_PTHREAD']){ /** Workaround for wasmfs-generated worker, which calls this routine from each individual thread and requires that its - argument be returned. All of the criteria above are fragile, - based solely on inspection of the offending code, not public - Emscripten details. */ + argument be returned. The conditional criteria above are + fragile, based solely on inspection of the offending code, + not public Emscripten details. */ + //console.warn("sqlite3InitModule() returning E-module.",EmscriptenModule); return EmscriptenModule; } + //console.warn("sqlite3InitModule() returning sqlite3 object."); const s = EmscriptenModule.sqlite3; s.scriptInfo = initModuleState; //console.warn("sqlite3.scriptInfo =",s.scriptInfo); diff --git a/ext/wasm/scratchpad-wasmfs-main.html b/ext/wasm/scratchpad-wasmfs-main.html index f78b412885..79635706c8 100644 --- a/ext/wasm/scratchpad-wasmfs-main.html +++ b/ext/wasm/scratchpad-wasmfs-main.html @@ -10,20 +10,6 @@
sqlite3 WASMFS/OPFS Main-thread Scratchpad
- -
-
-
Initializing app...
-
- On a slow internet connection this may take a moment. If this - message displays for "a long time", intialization may have - failed and the JavaScript console may contain clues as to why. -
-
-
Downloading...
-
- -

Scratchpad/test app for the WASMF/OPFS integration in the main window thread. This page requires that the sqlite3 API have been built with WASMFS support. If OPFS support is available then @@ -34,8 +20,8 @@


- - + + - - + diff --git a/ext/wasm/speedtest1-wasmfs.mjs b/ext/wasm/speedtest1-wasmfs.mjs new file mode 100644 index 0000000000..266ab64e7f --- /dev/null +++ b/ext/wasm/speedtest1-wasmfs.mjs @@ -0,0 +1,91 @@ +import sqlite3InitModule from './jswasm/speedtest1-wasmfs.mjs'; +const wMsg = (type,...args)=>{ + console.log("wMsg(",type,...args,")"); + postMessage({type, args}); +}; +wMsg('log',"speedtest1-wasmfs starting..."); +/** + If this environment contains OPFS, this function initializes it and + returns the name of the dir on which OPFS is mounted, else it returns + an empty string. +*/ +const wasmfsDir = function f(wasmUtil,dirName="/opfs"){ + if(undefined !== f._) return f._; + if( !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return f._ = ""; + } + try{ + if(0===wasmUtil.xCallWrapped( + 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], dirName + )){ + return f._ = dirName; + }else{ + return f._ = ""; + } + }catch(e){ + // sqlite3_wasm_init_wasmfs() is not available + return f._ = ""; + } +}; +wasmfsDir._ = undefined; + +const log = (...args)=>wMsg('log',...args); +const logErr = (...args)=>wMsg('logErr',...args); + +const runTests = function(sqlite3){ + console.log("Module inited.",sqlite3); + const wasm = sqlite3.wasm; + const __unlink = wasm.xWrap("sqlite3_wasm_vfs_unlink", "int", ["*","string"]); + const unlink = (fn)=>__unlink(0,fn); + const pDir = wasmfsDir(wasm); + if(pDir) log("Persistent storage:",pDir); + else{ + logErr("Expecting persistent storage in this build."); + return; + } + const scope = wasm.scopedAllocPush(); + const dbFile = pDir+"/speedtest1.db"; + const urlParams = new URL(self.location.href).searchParams; + const argv = ["speedtest1"]; + if(urlParams.has('flags')){ + argv.push(...(urlParams.get('flags').split(','))); + let i = argv.indexOf('--vfs'); + if(i>=0) argv.splice(i,2); + }else{ + argv.push( + "--singlethread", + "--nomutex", + //"--nosync", + "--nomemstat", + "--size", "10" + ); + } + + if(argv.indexOf('--memdb')>=0){ + logErr("WARNING: --memdb flag trumps db filename."); + } + argv.push("--big-transactions"/*important for tests 410 and 510!*/, + dbFile); + //log("argv =",argv); + // These log messages are not emitted to the UI until after main() returns. Fixing that + // requires moving the main() call and related cleanup into a timeout handler. + if(pDir) unlink(dbFile); + log("Starting native app:\n ",argv.join(' ')); + log("This will take a while and the browser might warn about the runaway JS.", + "Give it time..."); + setTimeout(function(){ + if(pDir) unlink(dbFile); + wasm.xCall('wasm_main', argv.length, + wasm.scopedAllocMainArgv(argv)); + wasm.scopedAllocPop(scope); + //if(pDir) unlink(dbFile); + log("Done running native main()"); + }, 25); +}/*runTests()*/; + +sqlite3InitModule({ + print: log, + printErr: logErr +}).then(runTests); diff --git a/ext/wasm/wasmfs.make b/ext/wasm/wasmfs.make index a2415b79ef..b97f812750 100644 --- a/ext/wasm/wasmfs.make +++ b/ext/wasm/wasmfs.make @@ -29,9 +29,10 @@ cflags.sqlite3-wasmfs += -DSQLITE_ENABLE_WASMFS ######################################################################## # emcc flags specific to building the final .js/.wasm file... emcc.flags.sqlite3-wasmfs := -emcc.flags.sqlite3-wasmfs += -sEXPORTED_RUNTIME_METHODS=wasmMemory,allocateUTF8OnStack +emcc.flags.sqlite3-wasmfs += \ + -sEXPORTED_RUNTIME_METHODS=wasmMemory,allocateUTF8OnStack,stringToUTF8OnStack # wasmMemory ==> for -sIMPORTED_MEMORY - # allocateUTF8OnStack ==> wasmfs internals + # *OnStack ==> wasmfs internals (leaky abstraction) emcc.flags.sqlite3-wasmfs += -sUSE_CLOSURE_COMPILER=0 emcc.flags.sqlite3-wasmfs += -Wno-limited-postlink-optimizations # ^^^^^ it likes to warn when we have "limited optimizations" via the -g3 flag. @@ -56,10 +57,7 @@ $(eval $(call SETUP_LIB_BUILD_MODE,sqlite3-wasmfs,esm,1,\ $(c-pp.D.sqlite3-bundler-friendly) -Dwasmfs,\ -sEXPORT_ES6 -sUSE_ES6_IMPORT_META\ )) -#$(eval $(call call-make-pre-post,sqlite3-wasmfs,vanilla)) $(sqlite3-wasmfs.js) $(sqlite3-wasmfs.mjs): $(MAKEFILE.wasmfs) -#$(sqlite3-wasmfs.js): $(pre-post-sqlite3-wasmfs.deps.vanilla) -#$(sqlite3-wasmfs.mjs): $(pre-post-sqlite3-wasmfs.deps.esm) ######################################################################## # Build quirk: we cannot build BOTH .js and .mjs with our current # build infrastructure because the supplemental *.worker.js files get @@ -84,28 +82,33 @@ endif ######################################################################## # speedtest1 for wasmfs. -speedtest1-wasmfs.js := $(dir.wasmfs)/speedtest1-wasmfs.js -speedtest1-wasmfs.wasm := $(subst .js,.wasm,$(speedtest1-wasmfs.js)) +speedtest1-wasmfs.mjs := $(dir.wasmfs)/speedtest1-wasmfs.mjs +speedtest1-wasmfs.wasm := $(subst .mjs,.wasm,$(speedtest1-wasmfs.mjs)) emcc.flags.speedtest1-wasmfs := $(sqlite3-wasmfs.fsflags) -emcc.flags.speedtest1-wasmfs += $(SQLITE_OPT) -DSQLITE_ENABLE_WASMFS +emcc.flags.speedtest1-wasmfs += $(SQLITE_OPT) emcc.flags.speedtest1-wasmfs += -sALLOW_MEMORY_GROWTH=0 emcc.flags.speedtest1-wasmfs += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.128) -#$(eval $(call call-make-pre-js,speedtest1-wasmfs,vanilla)) -$(speedtest1-wasmfs.js): $(speedtest1.cses) $(sqlite3-wasmfs.js) \ +#$(eval $(call call-make-pre-js,speedtest1-wasmfs,ems)) +$(speedtest1-wasmfs.mjs): $(speedtest1.cfiles) $(sqlite3-wasmfs.js) \ $(MAKEFILE) $(MAKEFILE.wasmfs) \ - $(pre-post-sqlite3-wasmfs.deps) \ + $(pre-post-sqlite3-wasmfs-esm.deps) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." $(emcc.bin) \ - $(emcc.speedtest1.common) $(emcc.flags.speedtest1-wasmfs) \ - $(pre-post-sqlite3-wasmfs.flags.vanilla) \ + $(pre-post-sqlite3-wasmfs-esm.flags) \ + $(cflags.common) \ $(cflags.sqlite3-wasmfs) \ - -o $@ $(speedtest1.cses) -lm + $(emcc.speedtest1.common) \ + $(emcc.flags.speedtest1-vanilla) \ + $(emcc.flags.sqlite3-wasmfs) \ + $(emcc.flags.speedtest1-wasmfs) \ + -o $@ $(speedtest1.cfiles) -lm + @$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,1) $(maybe-wasm-strip) $(speedtest1-wasmfs.wasm) ls -la $@ $(speedtest1-wasmfs.wasm) -#wasmfs: $(speedtest1-wasmfs.js) -CLEAN_FILES += $(speedtest1-wasmfs.js) $(speedtest1-wasmfs.wasm) \ - $(subst .js,.worker.js,$(speedtest1-wasmfs.js)) +wasmfs: $(speedtest1-wasmfs.mjs) +CLEAN_FILES += $(speedtest1-wasmfs.mjs) $(speedtest1-wasmfs.wasm) \ + $(subst .js,.worker.js,$(speedtest1-wasmfs.mjs)) # end speedtest1.js ######################################################################## diff --git a/manifest b/manifest index 83cc47a93d..a84635b9a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Significant\ssurgery\son\sthe\swasm\sbuild\son\sthe\sway\sto\sincorporating\swasmfs\sas\sa\sfirst-class\sbuild\soption. -D 2023-07-13T14:08:30.624 +C Get\sspeedtest1\swasmfs\svariant\srunning\sagain. +D 2023-07-13T16:33:45.070 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -535,8 +535,9 @@ F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789c F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10 F ext/wasm/scratchpad-wasmfs-main.html bf23812d059da37399b34e27d6b4c5980e2e1fb2153e5d3318d39b41aa854c32 -F ext/wasm/scratchpad-wasmfs-main.js 97f8eff738871030b98b75166b75555ce140605c791143dace93058643275137 -F ext/wasm/speedtest1-wasmfs.html 3493c46b24bad00e9f6ebe14804143ae67a880f6c59c16ee13bf5f06e7dc4313 +F ext/wasm/scratchpad-wasmfs-main.js 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 +F ext/wasm/speedtest1-wasmfs.html ce92a78f745e4fb837742624300a733046150507b1cac5c1aca81f396a8f46db +F ext/wasm/speedtest1-wasmfs.mjs 934417fea36e7c8b41bad24fe5550aa2903140e2c2195d876215d6c902961cd2 F ext/wasm/speedtest1-worker.html 97c2bf5f8534091ce718de05801090d5a80c3f13575996f095ba23638e1bdca0 F ext/wasm/speedtest1-worker.js 13b57c4a41729678a1194014afec2bd5b94435dcfc8d1039dfa9a533ac819ee1 F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da @@ -552,7 +553,7 @@ F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a9 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd72273503ae7d5 -F ext/wasm/wasmfs.make 6cfe6cb04859ca7aa611f71f6e770811d023a519a7fb4cc4de1419189594c2a8 +F ext/wasm/wasmfs.make 889be53a3d0f6bb0d270e25c4fb29f679b8d764c7fe864217180e61537cb9ac0 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 @@ -2042,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4850a9e2d2b41b91e91b9ef99a6d26ddf11a161b4c970fc00d2d095606729a04 -R 68d56fe3eae005ed95f6c7df371b3baf +P 215c37fce38cf647e073480689b79d952af7eb8165ae08c7f5caed27003fecfc +R 65bf4d4d1cc787c57405349531ba9382 U stephan -Z a358a57a699b604bd4cd582e774f0d4e +Z 63407a73d630f95e1718b20fb63b6e0d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4800ae6aca..7f1ab781d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -215c37fce38cf647e073480689b79d952af7eb8165ae08c7f5caed27003fecfc \ No newline at end of file +0f77e031daf0121f7603409cedf57f138edbb43083cbbcf00d2a423e16de25a7 \ No newline at end of file From bb9549e0be7abd5516eea9736c4fd2891d14b5d0 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 13 Jul 2023 17:27:05 +0000 Subject: [PATCH 13/72] More work on the wasmfs build and its test apps. FossilOrigin-Name: 953b8557194e9451dcf9f3bb433eafb67961487325e6519e675e73fecf850bfb --- ext/wasm/index.html | 19 +++++++++--------- ...asmfs-main.html => scratchpad-wasmfs.html} | 6 +----- ...d-wasmfs-main.js => scratchpad-wasmfs.mjs} | 0 ext/wasm/speedtest1-wasmfs.html | 3 ++- ext/wasm/speedtest1-wasmfs.mjs | 3 +-- manifest | 20 +++++++++---------- manifest.uuid | 2 +- 7 files changed, 25 insertions(+), 28 deletions(-) rename ext/wasm/{scratchpad-wasmfs-main.html => scratchpad-wasmfs.html} (77%) rename ext/wasm/{scratchpad-wasmfs-main.js => scratchpad-wasmfs.mjs} (100%) diff --git a/ext/wasm/index.html b/ext/wasm/index.html index 42a5fe6638..2c4bc4eb87 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -117,18 +117,19 @@ - + diff --git a/ext/wasm/scratchpad-wasmfs-main.html b/ext/wasm/scratchpad-wasmfs.html similarity index 77% rename from ext/wasm/scratchpad-wasmfs-main.html rename to ext/wasm/scratchpad-wasmfs.html index 79635706c8..c37febff18 100644 --- a/ext/wasm/scratchpad-wasmfs-main.html +++ b/ext/wasm/scratchpad-wasmfs.html @@ -19,13 +19,9 @@

All stuff on this page happens in the dev console.


- - - -