From 71e2bdb2b45c03d4ed5b8cfe3998a18881afbd76 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 22 Jul 2024 20:58:51 +0000 Subject: [PATCH] OPFS VFSes: remove the on-open() pragma calls, as those (A) already reflected the build-time default settings and (B) they made it illegal to run locking_mode=exclusive, which is a requirement for WAL mode without shared memory. Modify part of the test suite to demonstrate that the SAHPool VFS can run in WAL mode so long as locking_mode=exclusive is used. FossilOrigin-Name: 19cd8e2b056d7842ee39afb7160c901c9dc55a5bac8049cb0b5246210f6b920d --- ext/wasm/api/sqlite3-api-oo1.c-pp.js | 52 ++++++++++++------- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 13 +---- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 29 +---------- ext/wasm/tester1.c-pp.js | 19 ++++++- manifest | 18 +++---- manifest.uuid | 2 +- 6 files changed, 61 insertions(+), 72 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-oo1.c-pp.js b/ext/wasm/api/sqlite3-api-oo1.c-pp.js index c579b5ddf7..55d7356e25 100644 --- a/ext/wasm/api/sqlite3-api-oo1.c-pp.js +++ b/ext/wasm/api/sqlite3-api-oo1.c-pp.js @@ -79,14 +79,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }.bind({counter: 0})); - /** - A map of sqlite3_vfs pointers to SQL code or a callback function - to run when the DB constructor opens a database with the given - VFS. In the latter case, the call signature is (theDbObject,sqlite3Namespace) - and the callback is expected to throw on error. - */ - const __vfsPostOpenSql = Object.create(null); - //#if enable-see /** Converts ArrayBuffer or Uint8Array ba into a string of hex @@ -279,7 +271,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ // Check for per-VFS post-open SQL/callback... const pVfs = capi.sqlite3_js_db_vfs(pDb) || toss3("Internal error: cannot get VFS for new db handle."); - const postInitSql = __vfsPostOpenSql[pVfs]; + const postInitSql = __vfsPostOpenCallback[pVfs]; if(postInitSql){ /** Reminder: if this db is encrypted and the client did _not_ pass @@ -302,19 +294,39 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; + /** - Sets SQL which should be exec()'d on a DB instance after it is - opened with the given VFS pointer. The SQL may be any type - supported by the "string:flexible" function argument conversion. - Alternately, the 2nd argument may be a function, in which case it - is called with (theOo1DbObject,sqlite3Namespace) at the end of - the DB() constructor. The function must throw on error, in which - case the db is closed and the exception is propagated. This - function is intended only for use by DB subclasses or sqlite3_vfs - implementations. + A map of sqlite3_vfs pointers to SQL code or a callback function + to run when the DB constructor opens a database with the given + VFS. In the latter case, the call signature is + (theDbObject,sqlite3Namespace) and the callback is expected to + throw on error. */ - dbCtorHelper.setVfsPostOpenSql = function(pVfs, sql){ - __vfsPostOpenSql[pVfs] = sql; + const __vfsPostOpenCallback = Object.create(null); + + /** + Sets a callback which should be called after a db is opened with + the given sqlite3_vfs pointer. The 2nd argument must be a + function, which gets called with + (theOo1DbObject,sqlite3Namespace) at the end of the DB() + constructor. The function must throw on error, in which case the + db is closed and the exception is propagated. This function is + intended only for use by DB subclasses or sqlite3_vfs + implementations. + + Prior to 2024-07-22, it was legal to pass SQL code as the second + argument, but that can interfere with a client's ability to run + pragmas which must be run before anything else, namely (pragma + locking_mode=exclusive) for use with WAL mode. That capability + had only ever been used as an internal detail of the two OPFS + VFSes, and they no longer use it that way. + */ + dbCtorHelper.setVfsPostOpenCallback = function(pVfs, callback){ + if( !(callback instanceof Function)){ + toss3("dbCtorHelper.setVfsPostOpenCallback() should not be used with "+ + "a non-function argument.",arguments); + } + __vfsPostOpenCallback[pVfs] = callback; }; /** diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 86ba3add8e..d423bb0bbc 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -78,8 +78,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.SQLITE_OPEN_MAIN_DB | capi.SQLITE_OPEN_MAIN_JOURNAL | capi.SQLITE_OPEN_SUPER_JOURNAL | - capi.SQLITE_OPEN_WAL /* noting that WAL support is - unavailable in the WASM build.*/; + capi.SQLITE_OPEN_WAL; /** Subdirectory of the VFS's space where "opaque" (randomly-named) files are stored. Changing this effectively invalidates the data @@ -1280,16 +1279,6 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ }; OpfsSAHPoolDb.prototype = Object.create(oo1.DB.prototype); poolUtil.OpfsSAHPoolDb = OpfsSAHPoolDb; - oo1.DB.dbCtorHelper.setVfsPostOpenSql( - theVfs.pointer, - function(oo1Db, sqlite3){ - sqlite3.capi.sqlite3_exec(oo1Db, [ - /* See notes in sqlite3-vfs-opfs.js */ - "pragma journal_mode=DELETE;", - "pragma cache_size=-16384;" - ], 0, 0, 0); - } - ); }/*extend sqlite3.oo1*/ thePool.log("VFS initialized."); return poolUtil; diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index ef36b60626..2d11b35831 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -1288,40 +1288,13 @@ const installOpfsVfs = function callee(options){ OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); sqlite3.oo1.OpfsDb = OpfsDb; OpfsDb.importDb = opfsUtil.importDb; - sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenCallback( opfsVfs.pointer, function(oo1Db, sqlite3){ /* Set a relatively high default busy-timeout handler to help OPFS dbs deal with multi-tab/multi-worker contention. */ sqlite3.capi.sqlite3_busy_timeout(oo1Db, 10000); - sqlite3.capi.sqlite3_exec(oo1Db, [ - /* As of July 2023, the PERSIST journal mode on OPFS is - somewhat slower than DELETE or TRUNCATE (it was faster - before Chrome version 108 or 109). TRUNCATE and DELETE - have very similar performance on OPFS. - - Roy Hashimoto notes that TRUNCATE and PERSIST modes may - decrease OPFS concurrency because multiple connections - can open the journal file in those modes: - - https://github.com/rhashimoto/wa-sqlite/issues/68 - - Given that, and the fact that testing has not revealed - any appreciable difference between performance of - TRUNCATE and DELETE modes on OPFS, we currently (as of - 2023-07-13) default to DELETE mode. - */ - "pragma journal_mode=DELETE;", - /* - This vfs benefits hugely from cache on moderate/large - speedtest1 --size 50 and --size 100 workloads. We - currently rely on setting a non-default cache size when - building sqlite3.wasm. If that policy changes, the cache - can be set here. - */ - "pragma cache_size=-16384;" - ], 0, 0, 0); } ); }/*extend sqlite3.oo1*/ diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index d99d56fa96..b5125b0d8e 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3113,11 +3113,25 @@ globalThis.sqlite3InitModule = sqlite3InitModule; T.assert(db instanceof sqlite3.oo1.DB) .assert(1 === u1.getFileCount()); db.exec([ + 'pragma locking_mode=exclusive;', + 'pragma journal_mode=wal;' + /* WAL mode only works in this VFS if locking_mode=exclusive + is invoked prior to the first db access, as this build + does not have the shared-memory APIs needed for WAL without + exclusive-mode locking. See: + + https://sqlite.org/wal.html#use_of_wal_without_shared_memory + + Note that WAL mode here DOES NOT add any concurrency capabilities + to this VFS, but it MAY provide slightly improved performance + over the other journaling modes. + */, 'create table t(a);', 'insert into t(a) values(1),(2),(3)' ]); - T.assert(1 === u1.getFileCount()); - T.assert(3 === db.selectValue('select count(*) from t')); + T.assert(2 === u1.getFileCount() /* one is the journal file */) + .assert(3 === db.selectValue('select count(*) from t')) + .assert('wal'===db.selectValue('pragma journal_mode')); db.close(); T.assert(1 === u1.getFileCount()); db = new u2.OpfsSAHPoolDb(dbName); @@ -3137,6 +3151,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert( dbytes.byteLength == nWrote ); let db2 = new u1.OpfsSAHPoolDb(dbName2); T.assert(db2 instanceof sqlite3.oo1.DB) + //.assert('wal' == db2.selectValue("pragma journal_mode=WAL")) .assert(3 === db2.selectValue('select count(*) from t')); db2.close(); T.assert(true === u1.unlink(dbName2)) diff --git a/manifest b/manifest index 1c9ec496eb..16a34220e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C wasm\sbuild:\sresolve\sa\scircular\sdep\sand\sdo\ssome\sminor\stidying\sup. -D 2024-07-22T19:52:02.340 +C OPFS\sVFSes:\sremove\sthe\son-open()\spragma\scalls,\sas\sthose\s(A)\salready\sreflected\sthe\sbuild-time\sdefault\ssettings\sand\s(B)\sthey\smade\sit\sillegal\sto\srun\slocking_mode=exclusive,\swhich\sis\sa\srequirement\sfor\sWAL\smode\swithout\sshared\smemory.\sModify\spart\sof\sthe\stest\ssuite\sto\sdemonstrate\sthat\sthe\sSAHPool\sVFS\scan\srun\sin\sWAL\smode\sso\slong\sas\slocking_mode=exclusive\sis\sused. +D 2024-07-22T20:58:51.797 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -612,14 +612,14 @@ F ext/wasm/api/post-js-header.js 04dc12c3edd666b64a1b4ef3b6690c88dcc653f26451fd4 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e F ext/wasm/api/sqlite3-api-glue.c-pp.js 21a0f8c1e4b4675b6563759c74bef954ac36aa99acce79c56802b661429f43d0 -F ext/wasm/api/sqlite3-api-oo1.c-pp.js aba93e986b141454af2be42f37dfcfaaa981434a3801af3e48f621b620e43061 +F ext/wasm/api/sqlite3-api-oo1.c-pp.js 2e6ac2fc9cf77f7a77980a71930ce0b3e0469b0c87da7a161abd3bc365e4e3ec F ext/wasm/api/sqlite3-api-prologue.js b347a0c5350247f90174a0ad9b9e72a99a5f837f31f78f60fcdb829b2ca30b63 F ext/wasm/api/sqlite3-api-worker1.c-pp.js 5cc22a3c0d52828cb32aad8691488719f47d27567e63e8bc8b832d74371c352d F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js e8f1df56e97a29004a95a2eddd26778f52c33b3e797d32d4b1b668a38e6493dd F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 8c59ff35224adbe926b85d0c6debedc63c3c949d4cee761b3a74867b56155341 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 5868293eda205e74f2f5d1ada0077a6afca6981c7eba9f147736c31cde165e8b +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js e529a99b7d5a088284821e2902b20d3404b561126969876997d5a73a656c9199 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e99e3d99f736937914527070f00ab13e9391d3f1cef884ab99a64cbcbee8d675 F ext/wasm/api/sqlite3-vtab-helper.c-pp.js a2fcbc3fecdd0eea229283584ebc122f29d98194083675dbe5cb2cf3a17fe309 F ext/wasm/api/sqlite3-wasm.c 9267174b9b0591b4f71193542ab57adf95bb9415f7d3453acf4a8ca8052f5e6c F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 46f303ba8ddd1b2f0a391798837beddfa72e8c897038c8047eda49ce7d5ed46b @@ -667,7 +667,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 619964ecb359f5385ed0724f43e0eaee218daceaebc27997a230687dd7333499 +F ext/wasm/tester1.c-pp.js 4f68682b64d5cd3e956803c0ee90457a3c47af4eecda4775e7fa4e66fde4a183 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2195,8 +2195,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b6aed8bcb06edc7f0221fd707d5adc227856fe62dbcaae5ffe1fb4faa5c542e1 -R 202e5ba85a746eaa1bad2f1008edf0f0 +P 9df3f1f24c6346dc94695bf533501c54379bb6e3cf492b67dda8a64a6a1eb495 +R 1a7770728d958b8cba816bd7bdd2e0a2 U stephan -Z 9b5070ff548645dbd81801cffcf25f1a +Z ad279efb20ab2e4ae7b776e5a3c20df2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8c46cfc11a..e7add708c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9df3f1f24c6346dc94695bf533501c54379bb6e3cf492b67dda8a64a6a1eb495 +19cd8e2b056d7842ee39afb7160c901c9dc55a5bac8049cb0b5246210f6b920d