From 40ce00b54613bce948251264c2dae694db881aa3 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 3 Feb 2025 16:26:30 +0000 Subject: [PATCH 01/38] Initial work on a fix for the SAHPool VFS's effectively-no-op digest calculation, as reported in [https://github.com/sqlite/sqlite-wasm/issues/97|ticket #97 of the downstream npm subproject]. This requires more testing alongside databases created before this version to ensure that it's backwards-compatible. FossilOrigin-Name: 9234c33f92d92bfddc6211c9c587f1072e70837c0ffe1416ef7d84d59bacd364 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 56 ++++++++++++++++--- ext/wasm/tester1.c-pp.js | 2 +- manifest | 17 +++--- manifest.uuid | 2 +- 4 files changed, 61 insertions(+), 16 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 6551b5c89c..38a1b91dde 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -79,6 +79,32 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.SQLITE_OPEN_MAIN_JOURNAL | capi.SQLITE_OPEN_SUPER_JOURNAL | capi.SQLITE_OPEN_WAL; + const FLAG_COMPUTE_DIGEST_V2 = 1 ? capi.SQLITE_OPEN_MEMORY : 0 + /* Part of the fix for + https://github.com/sqlite/sqlite-wasm/issues/97 + + Summary: prior to versions 3.49.1 and 3.50 computeDigest() always + computes a value of [0,0], so it does not actually do anything + useful. Fixing it invalidates old persistent files, so we + instead only fix it for files created or updated since the bug + was discovered and fixed. + + This flag determines whether we use the broken legacy + computeDigest() or the v2 variant. We only use this flag for + newly-created/overwritten files. Pre-existing files have the + broken digest stored in them so need to continue to use that. + + This flag is stored in the same space as the + SQLITE_OPEN_... flags and we must be careful here to not use an + flag bit which is otherwise relevant for the VFS. + SQLITE_OPEN_MEMORY is handled by sqlite3_open_v2() and friends, + not the VFS, so we'll repurpose that one. If we take a + currently-unused bit and it ends up, at some later point, being + used, we would have to invalidate existing VFS files in order to + move to another bit. Similarly, if the SQLITE_OPEN_MEMORY bit + were ever reassigned (which it won't be!), we'd invalidate all + VFS-side files. + */; /** Subdirectory of the VFS's space where "opaque" (randomly-named) files are stored. Changing this effectively invalidates the data @@ -329,6 +355,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ const pool = getPoolForVfs(pVfs); try{ + flags &= ~FLAG_COMPUTE_DIGEST_V2; pool.log(`xOpen ${wasm.cstrToJs(zName)} ${flags}`); // First try to open a path that already exists in the file system. const path = (zName && wasm.peek8(zName)) @@ -631,7 +658,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); - const compDigest = this.computeDigest(this.#apBody); + const compDigest = this.computeDigest(this.#apBody, flags); + //console.warn("getAssociatedPath() compDigest",compDigest); if(fileDigest.every((v,i) => v===compDigest[i])){ // Valid digest const pathBytes = this.#apBody.findIndex((v)=>0===v); @@ -662,10 +690,17 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ if(HEADER_MAX_PATH_SIZE <= enc.written + 1/*NUL byte*/){ toss("Path too long:",path); } + if(path && flags){ + /* When re-writing files, update their digest, if needed, + to v2. We continue to use v1 for the (!path) case + (empty files) because there's little reason not to + use a digest of 0 for empty entries. */ + flags |= FLAG_COMPUTE_DIGEST_V2; + } this.#apBody.fill(0, enc.written, HEADER_MAX_PATH_SIZE); this.#dvBody.setUint32(HEADER_OFFSET_FLAGS, flags); - - const digest = this.computeDigest(this.#apBody); + const digest = this.computeDigest(this.#apBody, flags); + //console.warn("setAssociatedPath(",path,") digest",digest); sah.write(this.#apBody, {at: 0}); sah.write(digest, {at: HEADER_OFFSET_DIGEST}); sah.flush(); @@ -686,13 +721,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ metadata for each file as a validation check. Changing this algorithm invalidates all existing databases for this VFS, so don't do that. + + See the docs for FLAG_COMPUTE_DIGEST_V2 for more details. */ - computeDigest(byteArray){ + computeDigest(byteArray, fileFlags){ let h1 = 0xdeadbeef; let h2 = 0x41c6ce57; - for(const v of byteArray){ - h1 = 31 * h1 + (v * 307); - h2 = 31 * h2 + (v * 307); + if( fileFlags & FLAG_COMPUTE_DIGEST_V2 ){ + for(const v of byteArray){ + h1 = Math.imul(h1 ^ v, 2654435761); + h2 = Math.imul(h2 ^ v, 104729); + } + }else{ + /* this is what the buggy legacy computation worked out to */ + h1 = h2 = 0; } return new Uint32Array([h1>>>0, h2>>>0]); } diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 880edcec1d..6d603752be 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3492,7 +3492,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule; }); db.exec([ "create table t(a);", - "insert into t(a) values(1),(2),(3);", + "insert into t(a) values(1),(2),(1);", "select auxtest(a,a), auxtest(a,a) from t order by a" ]); }finally{ diff --git a/manifest b/manifest index 54fc8aa8ae..05212cec00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smore\scomplete\stest\sfor\s[76c8435a]\sand\sadd\ssome\scommentary\sabout\s(A)\sthe\sinability\sto\sautomatically\sclean\sup\sautomatically-generated\sWASM\sproxy\sfunctions\sfor\ssqlite3_set_auxdata()\sdestructors\sand\s(B)\show\sto\sdeal\swith\s(A)\sto\savoid\sleaking\sWASM\sproxy\sfunctions. -D 2025-02-03T14:55:56.185 +C Initial\swork\son\sa\sfix\sfor\sthe\sSAHPool\sVFS's\seffectively-no-op\sdigest\scalculation,\sas\sreported\sin\s[https://github.com/sqlite/sqlite-wasm/issues/97|ticket\s#97\sof\sthe\sdownstream\snpm\ssubproject].\sThis\srequires\smore\stesting\salongside\sdatabases\screated\sbefore\sthis\sversion\sto\sensure\sthat\sit's\sbackwards-compatible. +D 2025-02-03T16:26:30.488 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -645,7 +645,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 5cc22a3c0d52828cb32aad8691488719f47d2 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 3774befd97cd1a5e2895c8225a894aad946848c6d9b4028acc988b5d123475af F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js bb5e96cd0fd6e1e54538256433f1c60a4e3095063c4d1a79a8a022fc59be9571 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 2635b4c7f63a8ab496bcff91cb33a890d060e979c34d34e5c782d970fdbce36d F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 9b86ca2d8276cf919fbc9ba2a10e9786033b64f92c2db844d951804dee6c4b4e F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616 F ext/wasm/api/sqlite3-wasm.c 6f9d8529072d072359cd22dc5dfb0572c524684686569cfbd0f9640d7619fc10 @@ -696,7 +696,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js 1618670e466f424aa289859fe0ec8ded223e42e9e69b5c851f809baaaca1a00c F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js fb8d0761daaa69bd40c8253cc2d6c8c37ada97e1751b7f07af7369842ba2aeae +F ext/wasm/tester1.c-pp.js 005a94be87b888ca33aff130aeaef58045781ebfb92e559063d14e0fee68c9bf F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2209,8 +2209,11 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 91ef45fc2902e46813366ec6b8317209f39f10e4a23c3808e33aceedab9da6c7 -R f2c90877762eddda2efda339cfb2ee34 +P d693c2dddbd10a2e0b77893b04b11502e30b768f1b06814105f7f35172845fb9 +R f5d8b5467421514674c485e5d9d6f1f0 +T *branch * sahpool-digest +T *sym-sahpool-digest * +T -sym-trunk * Cancelled\sby\sbranch. U stephan -Z 6cfffa02c18a4e52a298c977368cc8d7 +Z fa5f726034dfcd33aa64a420eace023f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c7225ebda1..0ea45d1c37 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d693c2dddbd10a2e0b77893b04b11502e30b768f1b06814105f7f35172845fb9 +9234c33f92d92bfddc6211c9c587f1072e70837c0ffe1416ef7d84d59bacd364 From c97abeac0b33a06a274bc4363b5162c5eed6a5d0 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 3 Feb 2025 17:21:54 +0000 Subject: [PATCH 02/38] Add a test app to assist in validating the SAHPool digest calculation fix. FossilOrigin-Name: a1e304b8020025cc73a658bd8c7697d59b4f3ad96cac0a3e36553a3207d13dc6 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 4 +- ext/wasm/tests/opfs/sahpool/digest-worker.js | 95 ++++++++++++ ext/wasm/tests/opfs/sahpool/digest.html | 141 ++++++++++++++++++ manifest | 17 +-- manifest.uuid | 2 +- 5 files changed, 247 insertions(+), 12 deletions(-) create mode 100644 ext/wasm/tests/opfs/sahpool/digest-worker.js create mode 100644 ext/wasm/tests/opfs/sahpool/digest.html diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 38a1b91dde..08b77c9c1e 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -659,7 +659,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); const compDigest = this.computeDigest(this.#apBody, flags); - //console.warn("getAssociatedPath() compDigest",compDigest); + console.warn("getAssociatedPath() flags",flags.toString(16), "compDigest", compDigest); if(fileDigest.every((v,i) => v===compDigest[i])){ // Valid digest const pathBytes = this.#apBody.findIndex((v)=>0===v); @@ -700,7 +700,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ this.#apBody.fill(0, enc.written, HEADER_MAX_PATH_SIZE); this.#dvBody.setUint32(HEADER_OFFSET_FLAGS, flags); const digest = this.computeDigest(this.#apBody, flags); - //console.warn("setAssociatedPath(",path,") digest",digest); + console.warn("setAssociatedPath(",path,") digest",digest); sah.write(this.#apBody, {at: 0}); sah.write(digest, {at: HEADER_OFFSET_DIGEST}); sah.flush(); diff --git a/ext/wasm/tests/opfs/sahpool/digest-worker.js b/ext/wasm/tests/opfs/sahpool/digest-worker.js new file mode 100644 index 0000000000..5396252a83 --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/digest-worker.js @@ -0,0 +1,95 @@ +/* + 2025-01-31 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is part of sahpool-pausing.js's demonstration of the + pause/unpause feature of the opfs-sahpool VFS. +*/ +const clog = console.log.bind(console); +const wPost = (type,...args)=>postMessage({type, payload:args}); +const log = (...args)=>{ + clog("Worker:",...args); + wPost('log',...args); +} + +const hasOpfs = ()=>{ + return globalThis.FileSystemHandle + && globalThis.FileSystemDirectoryHandle + && globalThis.FileSystemFileHandle + && globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle + && navigator?.storage?.getDirectory; +}; +if( !hasOpfs() ){ + wPost('error',"OPFS not detected"); + throw new Error("OPFS not detected"); +} + +clog("Importing sqlite3..."); +const searchParams = new URL(self.location.href).searchParams; +importScripts(searchParams.get('sqlite3.dir') + '/sqlite3.js'); + +const runTests = function(sqlite3, poolUtil){ + const fname = '/my.db'; + let db = new poolUtil.OpfsSAHPoolDb(fname); + let n = (new Date()).valueOf(); + try { + db.exec([ + "create table if not exists t(a);" + ]); + db.exec({ + sql: "insert into t(a) values(?)", + bind: n++ + }); + log("Record count: ",db.selectValue("select count(*) from t")); + }finally{ + db.close(); + } + + db = new poolUtil.OpfsSAHPoolDb(fname); + try { + db.exec({ + sql: "insert into t(a) values(?)", + bind: n++ + }); + log("Record count: ",db.selectValue("select count(*) from t")); + }finally{ + db.close(); + } + + const fname2 = '/my2.db'; + db = new poolUtil.OpfsSAHPoolDb(fname2); + try { + db.exec([ + "create table if not exists t(a);" + ]); + db.exec({ + sql: "insert into t(a) values(?)", + bind: n++ + }); + log("Record count: ",db.selectValue("select count(*) from t")); + }finally{ + db.close(); + } +}; + +globalThis.sqlite3InitModule().then(async function(sqlite3){ + log("sqlite3 version:",sqlite3.capi.sqlite3_libversion(), + sqlite3.capi.sqlite3_sourceid()); + const sahPoolConfig = { + name: 'opfs-sahpool-digest', + clearOnInit: false, + initialCapacity: 6 + }; + return sqlite3.installOpfsSAHPoolVfs(sahPoolConfig).then(poolUtil=>{ + log('vfs acquired'); + runTests(sqlite3, poolUtil); + }); +}); diff --git a/ext/wasm/tests/opfs/sahpool/digest.html b/ext/wasm/tests/opfs/sahpool/digest.html new file mode 100644 index 0000000000..c4ba6cc1aa --- /dev/null +++ b/ext/wasm/tests/opfs/sahpool/digest.html @@ -0,0 +1,141 @@ + + + + + + + + + sqlite3 tester: OpfsSAHPool Digest + + +

+ +

+ This is a test app for the digest calculation of the OPFS + SAHPool VFS. It requires running it with a new database created using + v3.49.0 or older, then running it again with a newer version, then + again with 3.49.0 or older. +

+
+ + +
+
+ + + diff --git a/manifest b/manifest index 05212cec00..638b314db7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\swork\son\sa\sfix\sfor\sthe\sSAHPool\sVFS's\seffectively-no-op\sdigest\scalculation,\sas\sreported\sin\s[https://github.com/sqlite/sqlite-wasm/issues/97|ticket\s#97\sof\sthe\sdownstream\snpm\ssubproject].\sThis\srequires\smore\stesting\salongside\sdatabases\screated\sbefore\sthis\sversion\sto\sensure\sthat\sit's\sbackwards-compatible. -D 2025-02-03T16:26:30.488 +C Add\sa\stest\sapp\sto\sassist\sin\svalidating\sthe\sSAHPool\sdigest\scalculation\sfix. +D 2025-02-03T17:21:54.248 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -645,7 +645,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 5cc22a3c0d52828cb32aad8691488719f47d2 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 3774befd97cd1a5e2895c8225a894aad946848c6d9b4028acc988b5d123475af F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 2635b4c7f63a8ab496bcff91cb33a890d060e979c34d34e5c782d970fdbce36d +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js d1d17004bfee7a9159b56888b83345c6219262448297a81966faae4fd28f749c F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 9b86ca2d8276cf919fbc9ba2a10e9786033b64f92c2db844d951804dee6c4b4e F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616 F ext/wasm/api/sqlite3-wasm.c 6f9d8529072d072359cd22dc5dfb0572c524684686569cfbd0f9640d7619fc10 @@ -700,6 +700,8 @@ F ext/wasm/tester1.c-pp.js 005a94be87b888ca33aff130aeaef58045781ebfb92e559063d14 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 +F ext/wasm/tests/opfs/sahpool/digest-worker.js cdf4bb6dad5e274985e4b0022d0d85f9e782df03352044d8c7c22532845d7304 +F ext/wasm/tests/opfs/sahpool/digest.html 83d0c2f1dd6fe83e9088c603091514ac8549ee04a4632d8cc51d27fd094f05b3 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 F main.mk 8cfe182232ac7bbc87530792db6f31c09f2a2f35e9887d0412978746efe42ea9 @@ -2209,11 +2211,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d693c2dddbd10a2e0b77893b04b11502e30b768f1b06814105f7f35172845fb9 -R f5d8b5467421514674c485e5d9d6f1f0 -T *branch * sahpool-digest -T *sym-sahpool-digest * -T -sym-trunk * Cancelled\sby\sbranch. +P 9234c33f92d92bfddc6211c9c587f1072e70837c0ffe1416ef7d84d59bacd364 +R 176b17e8248aa3df480fb81ba267bf4c U stephan -Z fa5f726034dfcd33aa64a420eace023f +Z 9279d9607c28db362f757a9d03a072e6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0ea45d1c37..fef85d18dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9234c33f92d92bfddc6211c9c587f1072e70837c0ffe1416ef7d84d59bacd364 +a1e304b8020025cc73a658bd8c7697d59b4f3ad96cac0a3e36553a3207d13dc6 From d2f7dfa6190f529f7e543380b6663074e9078207 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 26 Feb 2025 03:03:08 +0000 Subject: [PATCH 03/38] More work on the sahpool digest fix. New/fixed versions can read legacy (no digest) files but the reverse is only possible in limited circumstances (when files originated from a legacy version). The burning question is whether the real fix would be to remove this digest check altogether, as it only applies in a very limited context, and the fact that it was broken for some 18 months unnoticed suggests that its value might not be worth the CPU cycles. FossilOrigin-Name: 0df62b776c68bebb0e187b353b6f29b0a41a29f0a1c8d6728fa6b9f7ce0d13f7 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 52 ++++++++++++------- ext/wasm/tests/opfs/sahpool/digest-worker.js | 9 ++-- ext/wasm/tests/opfs/sahpool/digest.html | 2 +- manifest | 16 +++--- manifest.uuid | 2 +- 5 files changed, 48 insertions(+), 33 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 08b77c9c1e..94c890850d 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -79,23 +79,39 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ capi.SQLITE_OPEN_MAIN_JOURNAL | capi.SQLITE_OPEN_SUPER_JOURNAL | capi.SQLITE_OPEN_WAL; - const FLAG_COMPUTE_DIGEST_V2 = 1 ? capi.SQLITE_OPEN_MEMORY : 0 + const FLAG_COMPUTE_DIGEST_V2 = capi.SQLITE_OPEN_MEMORY /* Part of the fix for https://github.com/sqlite/sqlite-wasm/issues/97 - Summary: prior to versions 3.49.1 and 3.50 computeDigest() always - computes a value of [0,0], so it does not actually do anything - useful. Fixing it invalidates old persistent files, so we - instead only fix it for files created or updated since the bug - was discovered and fixed. + Summary: prior to versions 3.49.2 and 3.50.0 computeDigest() + always computes a value of [0,0] due to overflows, so it does not + do anything useful. Fixing it invalidates old persistent files, + so we instead only fix it for files created or updated since the + bug was discovered and fixed. This flag determines whether we use the broken legacy computeDigest() or the v2 variant. We only use this flag for newly-created/overwritten files. Pre-existing files have the broken digest stored in them so need to continue to use that. - This flag is stored in the same space as the - SQLITE_OPEN_... flags and we must be careful here to not use an + What this means, in terms of db file compatibility between + versions: + + - DBs created with versions older than this fix (<=3.49.1) + can be read by post-fix versions. Such DBs which are written + to in-place (not replaced) by newer versions can still be read + by older versions, as the affected digest is only modified + when the SAH slot is assigned to a given filename. + + - DBs created with post-fix versions will, when read by a pre-fix + version, be seen as having a "bad digest" and will be + unceremoniously replaced by that pre-fix version. When swapping + back to a post-fix version, that version will see that the file + entry is missing the FLAG_COMPUTE_DIGEST_V2 bit so will treat it + as a legacy file. + + This flag is stored in the same memory as the variour + SQLITE_OPEN_... flags and we must be careful here to not use a flag bit which is otherwise relevant for the VFS. SQLITE_OPEN_MEMORY is handled by sqlite3_open_v2() and friends, not the VFS, so we'll repurpose that one. If we take a @@ -659,7 +675,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4); sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST}); const compDigest = this.computeDigest(this.#apBody, flags); - console.warn("getAssociatedPath() flags",flags.toString(16), "compDigest", compDigest); + //warn("getAssociatedPath() flags",'0x'+flags.toString(16), "compDigest", compDigest); if(fileDigest.every((v,i) => v===compDigest[i])){ // Valid digest const pathBytes = this.#apBody.findIndex((v)=>0===v); @@ -691,16 +707,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ toss("Path too long:",path); } if(path && flags){ - /* When re-writing files, update their digest, if needed, - to v2. We continue to use v1 for the (!path) case - (empty files) because there's little reason not to - use a digest of 0 for empty entries. */ + /* When creating or re-writing files, update their digest, if + needed, to v2. We continue to use v1 for the (!path) case + (empty files) because there's little reason not to use a + digest of 0 for empty entries. */ flags |= FLAG_COMPUTE_DIGEST_V2; } this.#apBody.fill(0, enc.written, HEADER_MAX_PATH_SIZE); this.#dvBody.setUint32(HEADER_OFFSET_FLAGS, flags); const digest = this.computeDigest(this.#apBody, flags); - console.warn("setAssociatedPath(",path,") digest",digest); + //console.warn("setAssociatedPath(",path,") digest",digest); sah.write(this.#apBody, {at: 0}); sah.write(digest, {at: HEADER_OFFSET_DIGEST}); sah.flush(); @@ -725,18 +741,18 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ See the docs for FLAG_COMPUTE_DIGEST_V2 for more details. */ computeDigest(byteArray, fileFlags){ - let h1 = 0xdeadbeef; - let h2 = 0x41c6ce57; if( fileFlags & FLAG_COMPUTE_DIGEST_V2 ){ + let h1 = 0xdeadbeef; + let h2 = 0x41c6ce57; for(const v of byteArray){ h1 = Math.imul(h1 ^ v, 2654435761); h2 = Math.imul(h2 ^ v, 104729); } + return new Uint32Array([h1>>>0, h2>>>0]); }else{ /* this is what the buggy legacy computation worked out to */ - h1 = h2 = 0; + return new Uint32Array([0,0]); } - return new Uint32Array([h1>>>0, h2>>>0]); } /** diff --git a/ext/wasm/tests/opfs/sahpool/digest-worker.js b/ext/wasm/tests/opfs/sahpool/digest-worker.js index 5396252a83..4308566675 100644 --- a/ext/wasm/tests/opfs/sahpool/digest-worker.js +++ b/ext/wasm/tests/opfs/sahpool/digest-worker.js @@ -48,7 +48,7 @@ const runTests = function(sqlite3, poolUtil){ sql: "insert into t(a) values(?)", bind: n++ }); - log("Record count: ",db.selectValue("select count(*) from t")); + log(fname,"record count: ",db.selectValue("select count(*) from t")); }finally{ db.close(); } @@ -59,7 +59,7 @@ const runTests = function(sqlite3, poolUtil){ sql: "insert into t(a) values(?)", bind: n++ }); - log("Record count: ",db.selectValue("select count(*) from t")); + log(fname,"record count: ",db.selectValue("select count(*) from t")); }finally{ db.close(); } @@ -74,15 +74,14 @@ const runTests = function(sqlite3, poolUtil){ sql: "insert into t(a) values(?)", bind: n++ }); - log("Record count: ",db.selectValue("select count(*) from t")); + log(fname2,"record count: ",db.selectValue("select count(*) from t")); }finally{ db.close(); } }; globalThis.sqlite3InitModule().then(async function(sqlite3){ - log("sqlite3 version:",sqlite3.capi.sqlite3_libversion(), - sqlite3.capi.sqlite3_sourceid()); + log("sqlite3 version:",sqlite3.version); const sahPoolConfig = { name: 'opfs-sahpool-digest', clearOnInit: false, diff --git a/ext/wasm/tests/opfs/sahpool/digest.html b/ext/wasm/tests/opfs/sahpool/digest.html index c4ba6cc1aa..fdcd98ec13 100644 --- a/ext/wasm/tests/opfs/sahpool/digest.html +++ b/ext/wasm/tests/opfs/sahpool/digest.html @@ -36,7 +36,7 @@ *********************************************************************** This is a bugfix test for the OPFS SAHPool VFS. It requires setting up - a database created using v3.49.0 or older, then runnig it again with + a database created using v3.49.0 or older, then running it again with a newer version. */ (function(){ diff --git a/manifest b/manifest index 638b314db7..21e7317364 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\sapp\sto\sassist\sin\svalidating\sthe\sSAHPool\sdigest\scalculation\sfix. -D 2025-02-03T17:21:54.248 +C More\swork\son\sthe\ssahpool\sdigest\sfix.\sNew/fixed\sversions\scan\sread\slegacy\s(no\sdigest)\sfiles\sbut\sthe\sreverse\sis\sonly\spossible\sin\slimited\scircumstances\s(when\sfiles\soriginated\sfrom\sa\slegacy\sversion).\sThe\sburning\squestion\sis\swhether\sthe\sreal\sfix\swould\sbe\sto\sremove\sthis\sdigest\scheck\saltogether,\sas\sit\sonly\sapplies\sin\sa\svery\slimited\scontext,\sand\sthe\sfact\sthat\sit\swas\sbroken\sfor\ssome\s18\smonths\sunnoticed\ssuggests\sthat\sits\svalue\smight\snot\sbe\sworth\sthe\sCPU\scycles. +D 2025-02-26T03:03:08.171 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -645,7 +645,7 @@ F ext/wasm/api/sqlite3-api-worker1.c-pp.js 5cc22a3c0d52828cb32aad8691488719f47d2 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 3774befd97cd1a5e2895c8225a894aad946848c6d9b4028acc988b5d123475af F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js d1d17004bfee7a9159b56888b83345c6219262448297a81966faae4fd28f749c +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js c18540f86e2d10a48d67f933edb05e29a0e2c904dbdc996f3c765a8b63c49aea F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 9b86ca2d8276cf919fbc9ba2a10e9786033b64f92c2db844d951804dee6c4b4e F ext/wasm/api/sqlite3-vtab-helper.c-pp.js e809739d71e8b35dfe1b55d24d91f02d04239e6aef7ca1ea92a15a29e704f616 F ext/wasm/api/sqlite3-wasm.c 6f9d8529072d072359cd22dc5dfb0572c524684686569cfbd0f9640d7619fc10 @@ -700,8 +700,8 @@ F ext/wasm/tester1.c-pp.js 005a94be87b888ca33aff130aeaef58045781ebfb92e559063d14 F ext/wasm/tests/opfs/concurrency/index.html 657578a6e9ce1e9b8be951549ed93a6a471f4520a99e5b545928668f4285fb5e F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65ad09f510589c779b7cc6a803a88 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 -F ext/wasm/tests/opfs/sahpool/digest-worker.js cdf4bb6dad5e274985e4b0022d0d85f9e782df03352044d8c7c22532845d7304 -F ext/wasm/tests/opfs/sahpool/digest.html 83d0c2f1dd6fe83e9088c603091514ac8549ee04a4632d8cc51d27fd094f05b3 +F ext/wasm/tests/opfs/sahpool/digest-worker.js f8320caaf6368ee8c59f259de5bebe68232f678390dc773ee61c466e171881dc +F ext/wasm/tests/opfs/sahpool/digest.html 08122dabf8ef56ac7d40f43660e178811166842bb47b37cbb11003030bb9ce09 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 F main.mk 8cfe182232ac7bbc87530792db6f31c09f2a2f35e9887d0412978746efe42ea9 @@ -2211,8 +2211,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9234c33f92d92bfddc6211c9c587f1072e70837c0ffe1416ef7d84d59bacd364 -R 176b17e8248aa3df480fb81ba267bf4c +P a1e304b8020025cc73a658bd8c7697d59b4f3ad96cac0a3e36553a3207d13dc6 +R 428e6412102dca0ea6c390f44246e72c U stephan -Z 9279d9607c28db362f757a9d03a072e6 +Z eddb8c14b5cfddbfe410d5840cc6ef5c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fef85d18dd..2130fc3e97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1e304b8020025cc73a658bd8c7697d59b4f3ad96cac0a3e36553a3207d13dc6 +0df62b776c68bebb0e187b353b6f29b0a41a29f0a1c8d6728fa6b9f7ce0d13f7 From 642479d1cd44171e258aa5e7a36f9d2472356008 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 10 Mar 2025 22:31:55 +0000 Subject: [PATCH 04/38] Ensure that the TEMP database has been initialized at the beginning of a call to sqlite3_open_blob() for the TEMP database. Fix for the issue reported by [forum:/forumpost/0a556d619b|forum post 0a556d619b]. FossilOrigin-Name: 2cfccdbe08b7b14a6b255f7157ac20d0807327adefcb33fcffeeed14c7603fe1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeblob.c | 9 ++++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b3d524e4c0..9e74479a52 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexplicit\sdb\sclose\sto\stest/walsetlk.test\sto\swork\saround\san\sunjustified\stest\sfailure\son\sWindows\swhen\sthe\swalsetlk\stests\sare\srun\sin\sthe\ssame\sinvocation\sof\stestfixture.exe\sin\sWindows. -D 2025-03-10T17:28:43.120 +C Ensure\sthat\sthe\sTEMP\sdatabase\shas\sbeen\sinitialized\sat\sthe\sbeginning\sof\na\scall\sto\ssqlite3_open_blob()\sfor\sthe\sTEMP\sdatabase.\s\sFix\sfor\sthe\sissue\nreported\sby\s[forum:/forumpost/0a556d619b|forum\spost\s0a556d619b]. +D 2025-03-10T22:31:55.627 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -855,7 +855,7 @@ F src/vdbe.h 3d26d5c7660c5c7bd33ffb0d8784615072d8b23c81f8110870efe2631136bc89 F src/vdbeInt.h 078b1c15b26587b54c1c1879d0d2f4dec812b9de4c337fed9faf73fbcc3bf091 F src/vdbeapi.c cb8eb9e41a16f5fa3ce5b8f3910edfbba336d10156cfb7a79f92cf7bf443977b F src/vdbeaux.c d7ef1a0a7233589d789eda1ba9ffa4b0ea61fca9651e4f47fb4250d03d62bcaf -F src/vdbeblob.c 9166b6eb7054e5da82e35255892fb1ed551355a4716452539e8e3ac14f25fbe3 +F src/vdbeblob.c b1b4032cac46b41e44b957c4d00aee9851f862dfd85ecb68116ba49884b03dfd F src/vdbemem.c 571ae3116dbf840a62c4aaa6bc09d577dfef8ad4d3978cf37275bb5f9653217b F src/vdbesort.c f7ce6eb4c0e8b0273329d2f43b8b6e5ebe8f2d853fc323d5787dada702ea0b66 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d7013b63932b2f5750572ae6bdd259a2b6e6548c20fb9a5559edd22d2f2fc6cb -R cb2e10fc56d1e1fbf885d5500a62f25a -U stephan -Z c867479e9a5d0c3a65bf2562516a55c9 +P f418de109335cd7cb29d2b587540c163bbaaa7129c662c2908ef67492139b2d7 +R c32606fb5e8adbe46b53d16576b6e8b5 +U drh +Z 031b66c2465b667491ccd6797fe6a4c2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 843ccdc623..1609d2fc22 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f418de109335cd7cb29d2b587540c163bbaaa7129c662c2908ef67492139b2d7 +2cfccdbe08b7b14a6b255f7157ac20d0807327adefcb33fcffeeed14c7603fe1 diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 79698d0af4..42edcf7de8 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -133,6 +133,7 @@ int sqlite3_blob_open( char *zErr = 0; Table *pTab; Incrblob *pBlob = 0; + int iDb; Parse sParse; #ifdef SQLITE_ENABLE_API_ARMOR @@ -178,7 +179,10 @@ int sqlite3_blob_open( sqlite3ErrorMsg(&sParse, "cannot open view: %s", zTable); } #endif - if( !pTab ){ + if( pTab==0 + || ((iDb = sqlite3SchemaToIndex(db, pTab->pSchema))==1 && + sqlite3OpenTempDatabase(&sParse)) + ){ if( sParse.zErrMsg ){ sqlite3DbFree(db, zErr); zErr = sParse.zErrMsg; @@ -189,7 +193,7 @@ int sqlite3_blob_open( goto blob_open_out; } pBlob->pTab = pTab; - pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; + pBlob->zDb = db->aDb[iDb].zDbSName; /* Now search pTab for the exact column. */ iCol = sqlite3ColumnIndex(pTab, zColumn); @@ -273,7 +277,6 @@ int sqlite3_blob_open( {OP_Halt, 0, 0, 0}, /* 5 */ }; Vdbe *v = (Vdbe *)pBlob->pStmt; - int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); VdbeOp *aOp; sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, From a7829ecbdd9f0d20dd1d4e1386038ab1e7655f5a Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 11 Mar 2025 12:19:27 +0000 Subject: [PATCH 05/38] The substitute "puts" command used by the Windows implementation of sqlite3_analyzer must invoke fflush() after each line of output. Otherwise the output can be truncated when redirected into a file. FossilOrigin-Name: ba058ce90a2ba9ebc4d8fb289108c04f80fa85da01c0b8bd58855681836ba83d --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/sqlite3_analyzer.c.in | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9e74479a52..1b5df60ebc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\sTEMP\sdatabase\shas\sbeen\sinitialized\sat\sthe\sbeginning\sof\na\scall\sto\ssqlite3_open_blob()\sfor\sthe\sTEMP\sdatabase.\s\sFix\sfor\sthe\sissue\nreported\sby\s[forum:/forumpost/0a556d619b|forum\spost\s0a556d619b]. -D 2025-03-10T22:31:55.627 +C The\ssubstitute\s"puts"\scommand\sused\sby\sthe\sWindows\simplementation\sof\nsqlite3_analyzer\smust\sinvoke\sfflush()\safter\seach\sline\sof\soutput.\s\sOtherwise\nthe\soutput\scan\sbe\struncated\swhen\sredirected\sinto\sa\sfile. +D 2025-03-11T12:19:27.605 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -2193,7 +2193,7 @@ F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd F tool/spellsift.tcl 52b4b04dc4333c7ab024f09d9d66ed6b6f7c6eb00b38497a09f338fa55d40618 x F tool/split-sqlite3c.tcl 07e18a1d8cc3f6b3a4a1f3528e63c9b29a5c8a7bca0b8d394b231da464ce1247 F tool/sqldiff.c 134be7866be19f8beb32043d5aea5657f01aaeae2df8d33d758ff722c78666b9 -F tool/sqlite3_analyzer.c.in fc7735c499d226a49d843d8209b2543e4e5229eeb71a674c331323a2217b65b4 +F tool/sqlite3_analyzer.c.in 5a2984ac457a49c7ad4f0924c738df55aafdc97661e912fd99c5cfcdfd4725fd F tool/sqlite3_rsync.c 9a1cca2ab1271c59b37a6493c15dc1bcd0ab9149197a9125926bc08dd26b83fb F tool/sqltclsh.c.in 1bcc2e9da58fadf17b0bf6a50e68c1159e602ce057210b655d50bad5aaaef898 F tool/sqltclsh.tcl 862f4cf1418df5e1315b5db3b5ebe88969e2a784525af5fbf9596592f14ed848 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f418de109335cd7cb29d2b587540c163bbaaa7129c662c2908ef67492139b2d7 -R c32606fb5e8adbe46b53d16576b6e8b5 +P 2cfccdbe08b7b14a6b255f7157ac20d0807327adefcb33fcffeeed14c7603fe1 +R 7158e964e2d9b27882ac231601582e3d U drh -Z 031b66c2465b667491ccd6797fe6a4c2 +Z 67031cafd0f114410834393ff697fc3b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1609d2fc22..537d580928 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2cfccdbe08b7b14a6b255f7157ac20d0807327adefcb33fcffeeed14c7603fe1 +ba058ce90a2ba9ebc4d8fb289108c04f80fa85da01c0b8bd58855681836ba83d diff --git a/tool/sqlite3_analyzer.c.in b/tool/sqlite3_analyzer.c.in index 9c11752b81..945c3c5b90 100644 --- a/tool/sqlite3_analyzer.c.in +++ b/tool/sqlite3_analyzer.c.in @@ -61,6 +61,7 @@ static int subst_puts( } sqlite3_fputs(zOut, pOut); if( addNewLine ) sqlite3_fputs("\n", pOut); + fflush(pOut); return TCL_OK; } #endif /* defined(_WIN32) */ From 62d9d70eddda991bd3dedb55c1beb5a23fb6cae8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 12 Mar 2025 15:17:13 +0000 Subject: [PATCH 06/38] The --echo flag on the CLI also echos dot-commands provided on the command-line. FossilOrigin-Name: 6ec0c03b954cf705da076d035a1cc2e784233ae28857385379e44a59af6c5ec4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1b5df60ebc..c474735113 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\ssubstitute\s"puts"\scommand\sused\sby\sthe\sWindows\simplementation\sof\nsqlite3_analyzer\smust\sinvoke\sfflush()\safter\seach\sline\sof\soutput.\s\sOtherwise\nthe\soutput\scan\sbe\struncated\swhen\sredirected\sinto\sa\sfile. -D 2025-03-11T12:19:27.605 +C The\s--echo\sflag\son\sthe\sCLI\salso\sechos\sdot-commands\sprovided\son\sthe\scommand-line. +D 2025-03-12T15:17:13.208 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -782,7 +782,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 626c24b258b111f75c22107aa5614ad89810df3026f5ca071116d3fe75925c75 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c df63f64ef91c132dd12d37c876653f8b5493d2d5cf330a27158912ee5a065451 -F src/shell.c.in ad3cb02ead5551be11ecf1433899d7585ad3bed669de0de9a70dabfd6a8a7256 +F src/shell.c.in 248050551cad788f8bb4b4728e00d8e36a10130d2d101e55cd51cfee03df91ff F src/sqlite.h.in 3db05f6603c78d9e6fe035e9e12bed4ca8140135d05ff092becc2cf7d7d1fefb F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2cfccdbe08b7b14a6b255f7157ac20d0807327adefcb33fcffeeed14c7603fe1 -R 7158e964e2d9b27882ac231601582e3d +P ba058ce90a2ba9ebc4d8fb289108c04f80fa85da01c0b8bd58855681836ba83d +R 8a620b0dc40f645c425a37f6c9520afe U drh -Z 67031cafd0f114410834393ff697fc3b +Z 59b1c4ee8192208101e011858b1f1e61 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 537d580928..30117bd21c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba058ce90a2ba9ebc4d8fb289108c04f80fa85da01c0b8bd58855681836ba83d +6ec0c03b954cf705da076d035a1cc2e784233ae28857385379e44a59af6c5ec4 diff --git a/src/shell.c.in b/src/shell.c.in index 655982e732..93d73e6ac7 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -13475,6 +13475,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** the database filename. */ for(i=0; i Date: Thu, 13 Mar 2025 18:51:18 +0000 Subject: [PATCH 07/38] Fix the generate_series extension for the case where the termination value is not an even multiple of the step from the start value and there is also a value=NNN constraint in the WHERE clause. [forum:/info/bf2dc8e909983511|Forum post bf2dc8e9] FossilOrigin-Name: 75e72e3b0d0d689d39e00a01dc361dd6ce2649e68d200bf501ddcf04063041b2 --- ext/misc/series.c | 6 ++---- manifest | 14 +++++++------- manifest.uuid | 2 +- test/tabfunc01.test | 13 +++++++++++++ 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/ext/misc/series.c b/ext/misc/series.c index d500842d56..04644dd603 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -519,8 +519,7 @@ static int seriesFilter( pCur->ss.iBase += ((d+szStep-1)/szStep)*szStep; } if( pCur->ss.iTerm>iMax ){ - sqlite3_uint64 d = pCur->ss.iTerm - iMax; - pCur->ss.iTerm -= ((d+szStep-1)/szStep)*szStep; + pCur->ss.iTerm = iMax; } }else{ sqlite3_int64 szStep = -pCur->ss.iStep; @@ -530,8 +529,7 @@ static int seriesFilter( pCur->ss.iBase -= ((d+szStep-1)/szStep)*szStep; } if( pCur->ss.iTermss.iTerm; - pCur->ss.iTerm += ((d+szStep-1)/szStep)*szStep; + pCur->ss.iTerm = iMin; } } } diff --git a/manifest b/manifest index c474735113..3096adc66e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\s--echo\sflag\son\sthe\sCLI\salso\sechos\sdot-commands\sprovided\son\sthe\scommand-line. -D 2025-03-12T15:17:13.208 +C Fix\sthe\sgenerate_series\sextension\sfor\sthe\scase\swhere\sthe\stermination\svalue\nis\snot\san\seven\smultiple\sof\sthe\sstep\sfrom\sthe\sstart\svalue\sand\sthere\sis\salso\na\svalue=NNN\sconstraint\sin\sthe\sWHERE\sclause.\n[forum:/info/bf2dc8e909983511|Forum\spost\sbf2dc8e9] +D 2025-03-13T18:51:18.706 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -436,7 +436,7 @@ F ext/misc/regexp.c 388e7f237307c7dfbfb8dde44e097946f6c437801d63f0d7ad63f3320d4e F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c 13c13768db923313c561851c7feb0adf5c456a40d01bf7c57051895f3e4b81c7 +F ext/misc/series.c 69e0d2b5d193c67bdfee5baf79c10ec24a61a40212c7ca9c219edf7afa24305f F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 F ext/misc/shathree.c fd22d70620f86a0467acfdd3acd8435d5cb54eb1e2d9ff36ae44e389826993df F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 @@ -1717,7 +1717,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 7be82bd50c7ede7f01b2dd17cd1b84f352c516078222d0b067d858f081e3f9a7 +F test/tabfunc01.test 66d1ea27289c19317bf111744a4d5fda901759945d22ddb5bc964641fc38c185 F test/table.test 7862a00b58b5541511a26757ea9c5c7c3f8298766e98aa099deec703d9c0a8e0 F test/tableapi.test e37c33e6be2276e3a96bb54b00eea7f321277115d10e5b30fdb52a112b432750 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ba058ce90a2ba9ebc4d8fb289108c04f80fa85da01c0b8bd58855681836ba83d -R 8a620b0dc40f645c425a37f6c9520afe +P 6ec0c03b954cf705da076d035a1cc2e784233ae28857385379e44a59af6c5ec4 +R ed704471ff935c4c658477722480723c U drh -Z 59b1c4ee8192208101e011858b1f1e61 +Z 7365cbca0dfe1a68adc01dee6650e845 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 30117bd21c..f66e3d0bb3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ec0c03b954cf705da076d035a1cc2e784233ae28857385379e44a59af6c5ec4 +75e72e3b0d0d689d39e00a01dc361dd6ce2649e68d200bf501ddcf04063041b2 diff --git a/test/tabfunc01.test b/test/tabfunc01.test index b6797171e0..cbf9865eda 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -181,6 +181,19 @@ do_execsql_test tabfunc01-4.4 { SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10; } {1 3 5} +# 2025-03-13 forum post bf2dc8e909983511 +# +do_execsql_test tabfunc01-5.1 { + SELECT value + FROM generate_series(60,73,6) + WHERE value=66; +} 66 +do_execsql_test tabfunc01-5.2 { + SELECT value + FROM generate_series(73,60,-6) + WHERE value=67; +} 67 + # The next series of tests is verifying that virtual table are able # to optimize the IN operator, even on terms that are not marked "omit". # When the generate_series virtual table is compiled for the testfixture, From 1560045c328e5dc3b9a09e9c975b8626f984ef4f Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 14 Mar 2025 09:34:09 +0000 Subject: [PATCH 08/38] Cherrypick the [2b582c0097e33] doc addition, which was initially committed to the wrong branch. FossilOrigin-Name: f786de8d1873cd27b1bf83273a1e100e9d481144674888ccf65974e003a3caad --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 2 ++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3096adc66e..d18c8051a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sgenerate_series\sextension\sfor\sthe\scase\swhere\sthe\stermination\svalue\nis\snot\san\seven\smultiple\sof\sthe\sstep\sfrom\sthe\sstart\svalue\sand\sthere\sis\salso\na\svalue=NNN\sconstraint\sin\sthe\sWHERE\sclause.\n[forum:/info/bf2dc8e909983511|Forum\spost\sbf2dc8e9] -D 2025-03-13T18:51:18.706 +C Cherrypick\sthe\s[2b582c0097e33]\sdoc\saddition,\swhich\swas\sinitially\scommitted\sto\sthe\swrong\sbranch. +D 2025-03-14T09:34:09.128 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -783,7 +783,7 @@ F src/resolve.c 626c24b258b111f75c22107aa5614ad89810df3026f5ca071116d3fe75925c75 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c df63f64ef91c132dd12d37c876653f8b5493d2d5cf330a27158912ee5a065451 F src/shell.c.in 248050551cad788f8bb4b4728e00d8e36a10130d2d101e55cd51cfee03df91ff -F src/sqlite.h.in 3db05f6603c78d9e6fe035e9e12bed4ca8140135d05ff092becc2cf7d7d1fefb +F src/sqlite.h.in fd70afd92948cf7cc93f687ac960bad1b0b6fbc436752419eff2fd65a1809380 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 F src/sqliteInt.h 130217107c0425ab43d098c6eadf8aa2e1a037e26d79384127e2d950b27eec77 @@ -2213,8 +2213,9 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6ec0c03b954cf705da076d035a1cc2e784233ae28857385379e44a59af6c5ec4 -R ed704471ff935c4c658477722480723c -U drh -Z 7365cbca0dfe1a68adc01dee6650e845 +P 75e72e3b0d0d689d39e00a01dc361dd6ce2649e68d200bf501ddcf04063041b2 +Q +2b582c0097e3374beb280dfa6b03e0dacb9911da1bceb0dce0468e6b7291e74f +R 56a19e59ade4736dd825679ace2bb72a +U stephan +Z 2c7e1671f57e7e6f44a52a14c045869a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f66e3d0bb3..e509045678 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75e72e3b0d0d689d39e00a01dc361dd6ce2649e68d200bf501ddcf04063041b2 +f786de8d1873cd27b1bf83273a1e100e9d481144674888ccf65974e003a3caad diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c53218aa00..71338031c8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7071,6 +7071,8 @@ int sqlite3_autovacuum_pages( ** ** ^The second argument is a pointer to the function to invoke when a ** row is updated, inserted or deleted in a rowid table. +** ^The update hook is disabled by invoking sqlite3_update_hook() +** with a NULL pointer as the second parameter. ** ^The first argument to the callback is a copy of the third argument ** to sqlite3_update_hook(). ** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], From dae87df198a40df2c04507f86b49c291c1e917e3 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 14 Mar 2025 12:37:36 +0000 Subject: [PATCH 09/38] Fix an internal doc typo reported in [forum:e25e581f917|forum post e25e581f917]. FossilOrigin-Name: fa6f6ccdffc50024624306900efd2538c7415d8bdd0f02835b2e9c05adab3cf1 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/vdbe.h | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d18c8051a4..eb3d849e1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cherrypick\sthe\s[2b582c0097e33]\sdoc\saddition,\swhich\swas\sinitially\scommitted\sto\sthe\swrong\sbranch. -D 2025-03-14T09:34:09.128 +C Fix\san\sinternal\sdoc\stypo\sreported\sin\s[forum:e25e581f917|forum\spost\se25e581f917]. +D 2025-03-14T12:37:36.487 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -851,7 +851,7 @@ F src/utf.c d4d55ca95106a2029ec1cdbd2497a34e69ea1d338f1a9d80ef15ebf4ff01690d F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 F src/vacuum.c fbfc3e074c865d2b5b10b8a65a3783275b80c152653590690747a102bb6cc770 F src/vdbe.c 014769c8f7e528d59f5a8e25d0035258396cc2c755673dee29885b6049725ee6 -F src/vdbe.h 3d26d5c7660c5c7bd33ffb0d8784615072d8b23c81f8110870efe2631136bc89 +F src/vdbe.h 31eddcffc1d14c76c2a20fe4e137e1ee43d44f370896fae14a067052801a3625 F src/vdbeInt.h 078b1c15b26587b54c1c1879d0d2f4dec812b9de4c337fed9faf73fbcc3bf091 F src/vdbeapi.c cb8eb9e41a16f5fa3ce5b8f3910edfbba336d10156cfb7a79f92cf7bf443977b F src/vdbeaux.c d7ef1a0a7233589d789eda1ba9ffa4b0ea61fca9651e4f47fb4250d03d62bcaf @@ -2213,9 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 75e72e3b0d0d689d39e00a01dc361dd6ce2649e68d200bf501ddcf04063041b2 -Q +2b582c0097e3374beb280dfa6b03e0dacb9911da1bceb0dce0468e6b7291e74f -R 56a19e59ade4736dd825679ace2bb72a +P f786de8d1873cd27b1bf83273a1e100e9d481144674888ccf65974e003a3caad +R d7baa5b59cd39b651d61c163fdc55c15 U stephan -Z 2c7e1671f57e7e6f44a52a14c045869a +Z bd2d36d9f3639b6b3b36c30d817df77d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e509045678..38ff83c8db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f786de8d1873cd27b1bf83273a1e100e9d481144674888ccf65974e003a3caad +fa6f6ccdffc50024624306900efd2538c7415d8bdd0f02835b2e9c05adab3cf1 diff --git a/src/vdbe.h b/src/vdbe.h index 476f1b4ea2..dc98e270e3 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -319,8 +319,8 @@ int sqlite3NotPureFunc(sqlite3_context*); int sqlite3VdbeBytecodeVtabInit(sqlite3*); #endif -/* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on -** each VDBE opcode. +/* Use SQLITE_ENABLE_EXPLAIN_COMMENTS to enable generation of extra +** comments on each VDBE opcode. ** ** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op ** comments in VDBE programs that show key decision points in the code From cebf06c7980109ab459b5d90dd563ae621a78f94 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 14 Mar 2025 18:10:02 +0000 Subject: [PATCH 10/38] Make use of the flexible-array feature of C99, when available, to try to pacify -fsanitize=strict-bounds. This check-in fixes the core. There is more yet to do in FTS3, RTREE, and in FTS5. FossilOrigin-Name: 6fd6b32d06bd6a705e5140cd613af823b8183a6f6a9ceeeedfcf5e8b50821d68 --- manifest | 45 +++++++++++++++++++------------------ manifest.uuid | 2 +- src/build.c | 17 ++++++-------- src/expr.c | 13 +++++------ src/main.c | 2 +- src/resolve.c | 16 ++++++++------ src/select.c | 12 +++++----- src/sqliteInt.h | 59 +++++++++++++++++++++++++++++++++++++++++-------- src/trigger.c | 16 ++++++++------ src/vdbe.c | 12 +++++----- src/vdbeInt.h | 23 ++++++++++++++----- src/vdbeaux.c | 4 +--- src/vdbesort.c | 9 +++++--- src/wal.c | 9 +++++--- src/where.c | 20 +++++++++-------- src/whereInt.h | 7 +++++- src/wherecode.c | 17 +++++++------- 17 files changed, 175 insertions(+), 108 deletions(-) diff --git a/manifest b/manifest index eb3d849e1e..549a0c86df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sinternal\sdoc\stypo\sreported\sin\s[forum:e25e581f917|forum\spost\se25e581f917]. -D 2025-03-14T12:37:36.487 +C Make\suse\sof\sthe\sflexible-array\sfeature\sof\sC99,\swhen\savailable,\sto\stry\sto\npacify\s-fsanitize=strict-bounds.\s\sThis\scheck-in\sfixes\sthe\score.\sThere\sis\nmore\syet\sto\sdo\sin\sFTS3,\sRTREE,\sand\sin\sFTS5. +D 2025-03-14T18:10:02.673 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -725,14 +725,14 @@ F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea F src/btree.c 00fcee37947641f48d4b529d96143e74d056b7afa8f26d61292c90ee59c056b2 F src/btree.h 18e5e7b2124c23426a283523e5f31a4bff029131b795bb82391f9d2f3136fc50 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886 -F src/build.c 20793695ef64d2d0c147501e37f344f828f09f16d346a987b516316186030996 +F src/build.c 3fe9b9d0f411cc2139a2d5ffa1c9b555417f89332f4dbf7f8e311c2e69e40c81 F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9 F src/dbpage.c 2e677acb658a29965e55398bbc61161cb7819da538057c8032adac7ab8e4a8c0 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42 -F src/expr.c f27e2692e65f0600cf4c989ec2af36bdfab52ada6365c0cc0f434630157b877f +F src/expr.c 61c3baab38f1b50eb4696e1f37c8f7ae1d1ecbfc1a35d446cfd1886624784131 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f F src/func.c 7686ea382b20e8bfe2ab9de76150c99ee7b6e83523561f3c7787e0f68cb435c2 @@ -745,7 +745,7 @@ F src/insert.c a5f0366266be993ebf533808f22cb7a788624805b55bc45424ceed3f48c54a16 F src/json.c 81e2012796a0e139b18c50ee3444c8ef86a020ab360511882216f5b610657e0c F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 -F src/main.c 5102588cbe7668d5ee19fd9945546e4f1fd99e41815432decf5dd29f01f55597 +F src/main.c 07f78d917ffcdf327982840cfd8e855fd000527a2ea5ace372ce4febcbd0bf97 F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2 @@ -779,14 +779,14 @@ F src/pragma.c 30b535d0a66348df844ee36f890617b4cf45e9a22dcbc47ec3ca92909c50aaf1 F src/prepare.c 1832be043fce7d489959aae6f994c452d023914714c4d5457beaed51c0f3d126 F src/printf.c 33fc0d7643c848a098afdcb6e1db6de12379d47084b1cd0912cfce1d09345e44 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c 626c24b258b111f75c22107aa5614ad89810df3026f5ca071116d3fe75925c75 +F src/resolve.c 20e1fbe8f840ffc0cd835e33f68a802a22e34faa918d7a269f3de242fda02f99 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c df63f64ef91c132dd12d37c876653f8b5493d2d5cf330a27158912ee5a065451 +F src/select.c 8c273248e38a8a7286abe3f41c1931cc65eff602fef128acc0cc0484e1b7abb3 F src/shell.c.in 248050551cad788f8bb4b4728e00d8e36a10130d2d101e55cd51cfee03df91ff F src/sqlite.h.in fd70afd92948cf7cc93f687ac960bad1b0b6fbc436752419eff2fd65a1809380 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 130217107c0425ab43d098c6eadf8aa2e1a037e26d79384127e2d950b27eec77 +F src/sqliteInt.h 79d0f2705e62a1acaa11c10ac723749726ebbd96fb9be9f93c31fb44bdd4da1f F src/sqliteLimit.h 6d817c28a8f19af95e6f4921933b7fbbca48a962bce0eb0ec81e8bb3ef38e68b F src/status.c 0e72e4f6be6ccfde2488eb63210297e75f569f3ce9920f6c3d77590ec6ce5ffd F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -844,30 +844,30 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 3e37ac2b6cbb9b0abe33827b0153c27595269afd7152b48019808974481aca2c F src/treeview.c d85ce76e6d1498d781957c07cb234da6d77ce0ed2d196480d516f54dabc62279 -F src/trigger.c da3c25786870d8bf97cd46b493374c2375d1abaf20a9b0f5f8629a3f2f2ce383 +F src/trigger.c 3ffb8ed6b64dbcc0ccae6e82435d01be3bf547e13b814e2d46f7df9bef84748e F src/update.c 3e5e7ff66fa19ebe4d1b113d480639a24cc1175adbefabbd1a948a07f28e37cf F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1 F src/utf.c d4d55ca95106a2029ec1cdbd2497a34e69ea1d338f1a9d80ef15ebf4ff01690d F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 F src/vacuum.c fbfc3e074c865d2b5b10b8a65a3783275b80c152653590690747a102bb6cc770 -F src/vdbe.c 014769c8f7e528d59f5a8e25d0035258396cc2c755673dee29885b6049725ee6 +F src/vdbe.c b5deed01000b3970cfca089dc531cf9342afd96d00cc8b4ad26d303f088116ee F src/vdbe.h 31eddcffc1d14c76c2a20fe4e137e1ee43d44f370896fae14a067052801a3625 -F src/vdbeInt.h 078b1c15b26587b54c1c1879d0d2f4dec812b9de4c337fed9faf73fbcc3bf091 +F src/vdbeInt.h 11041d559992165651413e5868a907f698a2ec7af169fb2d20eb058a9ccd2313 F src/vdbeapi.c cb8eb9e41a16f5fa3ce5b8f3910edfbba336d10156cfb7a79f92cf7bf443977b -F src/vdbeaux.c d7ef1a0a7233589d789eda1ba9ffa4b0ea61fca9651e4f47fb4250d03d62bcaf +F src/vdbeaux.c 932105750c15378176980288daea80008ffdac360716ae4e7c7f85bf462d272d F src/vdbeblob.c b1b4032cac46b41e44b957c4d00aee9851f862dfd85ecb68116ba49884b03dfd F src/vdbemem.c 571ae3116dbf840a62c4aaa6bc09d577dfef8ad4d3978cf37275bb5f9653217b -F src/vdbesort.c f7ce6eb4c0e8b0273329d2f43b8b6e5ebe8f2d853fc323d5787dada702ea0b66 +F src/vdbesort.c 49e366d0216c782eba287bf602384e4330d2526a22f1275492d2785ce103c79b F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 F src/vtab.c 828221bdbeaaa6d62126ee6d07fd4ec0d09dcaea846f87ad01944d8b7e548859 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 2c69a5f92270429db72d853691b0640c76c671d5b2465396dadb9d9873e1efce +F src/wal.c 554a6b1afaaecb98cb47bb598bccf1374c9d3b624e5c4c3c4eb2ad364cc579f8 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 -F src/where.c 12cca5dfbe96e2589f951c43c0720fc58e52611787c37d85a0d9c10376202e8b -F src/whereInt.h d20cddddb1d61b18d5cb1fcfa9b77fbeebbc4afe44d996e603452a23b3009ee1 -F src/wherecode.c 5baa06f0daae7d38aca1d4814030b82ad4f127fe6bad18f0644776a474f6088b +F src/where.c e80177e452b4e436abc6ece0cb0249631000434f2a7425cc1df709015fce74ad +F src/whereInt.h 8373791e8dfbef3ca0d4ad7eab350be860b2740c44d94d0c54d2a8d36209c41a +F src/wherecode.c 0d3de258d7922aede028841c6e0060633c50be26737c92cc62ce8be280535430 F src/whereexpr.c 2415c8eee5ff89a8b709d7d83d71c1ff986cd720d0520057e1d8a5371339012a F src/window.c d01227141f622f24fbe36ca105fbe6ef023f9fd98f1ccd65da95f88886565db5 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -2213,8 +2213,11 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f786de8d1873cd27b1bf83273a1e100e9d481144674888ccf65974e003a3caad -R d7baa5b59cd39b651d61c163fdc55c15 -U stephan -Z bd2d36d9f3639b6b3b36c30d817df77d +P fa6f6ccdffc50024624306900efd2538c7415d8bdd0f02835b2e9c05adab3cf1 +R a7f539cde629f0570849a89a860ffc9d +T *branch * flex-array +T *sym-flex-array * +T -sym-trunk * +U drh +Z ea27c767023e4b4e578a073476796f47 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 38ff83c8db..45864ab067 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa6f6ccdffc50024624306900efd2538c7415d8bdd0f02835b2e9c05adab3cf1 +6fd6b32d06bd6a705e5140cd613af823b8183a6f6a9ceeeedfcf5e8b50821d68 diff --git a/src/build.c b/src/build.c index 907494b193..2679626668 100644 --- a/src/build.c +++ b/src/build.c @@ -3633,7 +3633,7 @@ void sqlite3CreateForeignKey( }else{ nCol = pFromCol->nExpr; } - nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; + nByte = SZ_FKEY(nCol) + pTo->n + 1; if( pToCol ){ for(i=0; inExpr; i++){ nByte += sqlite3Strlen30(pToCol->a[i].zEName) + 1; @@ -4692,12 +4692,11 @@ IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){ sqlite3 *db = pParse->db; int i; if( pList==0 ){ - pList = sqlite3DbMallocZero(db, sizeof(IdList) ); + pList = sqlite3DbMallocZero(db, SZ_IDLIST(1)); if( pList==0 ) return 0; }else{ IdList *pNew; - pNew = sqlite3DbRealloc(db, pList, - sizeof(IdList) + pList->nId*sizeof(pList->a)); + pNew = sqlite3DbRealloc(db, pList, SZ_IDLIST(pList->nId+1)); if( pNew==0 ){ sqlite3IdListDelete(db, pList); return 0; @@ -4796,8 +4795,7 @@ SrcList *sqlite3SrcListEnlarge( return 0; } if( nAlloc>SQLITE_MAX_SRCLIST ) nAlloc = SQLITE_MAX_SRCLIST; - pNew = sqlite3DbRealloc(db, pSrc, - sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); + pNew = sqlite3DbRealloc(db, pSrc, SZ_SRCLIST(nAlloc)); if( pNew==0 ){ assert( db->mallocFailed ); return 0; @@ -4872,7 +4870,7 @@ SrcList *sqlite3SrcListAppend( assert( pParse->db!=0 ); db = pParse->db; if( pList==0 ){ - pList = sqlite3DbMallocRawNN(pParse->db, sizeof(SrcList) ); + pList = sqlite3DbMallocRawNN(pParse->db, SZ_SRCLIST(1)); if( pList==0 ) return 0; pList->nAlloc = 1; pList->nSrc = 1; @@ -5758,10 +5756,9 @@ With *sqlite3WithAdd( } if( pWith ){ - sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); - pNew = sqlite3DbRealloc(db, pWith, nByte); + pNew = sqlite3DbRealloc(db, pWith, SZ_WITH(pWith->nCte+1)); }else{ - pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + pNew = sqlite3DbMallocZero(db, SZ_WITH(1)); } assert( (pNew!=0 && zName!=0) || db->mallocFailed ); diff --git a/src/expr.c b/src/expr.c index a3c41a1f57..dd5ea20383 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1738,7 +1738,7 @@ static Expr *exprDup( With *sqlite3WithDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ - sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + sqlite3_int64 nByte = SZ_WITH(p->nCte); pRet = sqlite3DbMallocZero(db, nByte); if( pRet ){ int i; @@ -1865,11 +1865,9 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){ SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ SrcList *pNew; int i; - int nByte; assert( db!=0 ); if( p==0 ) return 0; - nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); - pNew = sqlite3DbMallocRawNN(db, nByte ); + pNew = sqlite3DbMallocRawNN(db, SZ_SRCLIST(p->nSrc) ); if( pNew==0 ) return 0; pNew->nSrc = pNew->nAlloc = p->nSrc; for(i=0; inSrc; i++){ @@ -1931,7 +1929,7 @@ IdList *sqlite3IdListDup(sqlite3 *db, const IdList *p){ int i; assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew)+(p->nId-1)*sizeof(p->a[0]) ); + pNew = sqlite3DbMallocRawNN(db, SZ_IDLIST(p->nId)); if( pNew==0 ) return 0; pNew->nId = p->nId; for(i=0; inId; i++){ @@ -2015,7 +2013,7 @@ SQLITE_NOINLINE ExprList *sqlite3ExprListAppendNew( struct ExprList_item *pItem; ExprList *pList; - pList = sqlite3DbMallocRawNN(db, sizeof(ExprList)+sizeof(pList->a[0])*4 ); + pList = sqlite3DbMallocRawNN(db, SZ_EXPRLIST(4)); if( pList==0 ){ sqlite3ExprDelete(db, pExpr); return 0; @@ -2035,8 +2033,7 @@ SQLITE_NOINLINE ExprList *sqlite3ExprListAppendGrow( struct ExprList_item *pItem; ExprList *pNew; pList->nAlloc *= 2; - pNew = sqlite3DbRealloc(db, pList, - sizeof(*pList)+(pList->nAlloc-1)*sizeof(pList->a[0])); + pNew = sqlite3DbRealloc(db, pList, SZ_EXPRLIST(pList->nAlloc)); if( pNew==0 ){ sqlite3ExprListDelete(db, pList); sqlite3ExprDelete(db, pExpr); diff --git a/src/main.c b/src/main.c index ba2faf0b7e..9b67de8153 100644 --- a/src/main.c +++ b/src/main.c @@ -3845,7 +3845,7 @@ int sqlite3_set_clientdata( return SQLITE_OK; }else{ size_t n = strlen(zName); - p = sqlite3_malloc64( sizeof(DbClientData)+n+1 ); + p = sqlite3_malloc64( SZ_DBCLIENTDATA(n+1) ); if( p==0 ){ if( xDestructor ) xDestructor(pData); sqlite3_mutex_leave(db->mutex); diff --git a/src/resolve.c b/src/resolve.c index 54ce4fb1ea..c3ab2d557c 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -2277,20 +2277,22 @@ int sqlite3ResolveSelfReference( Expr *pExpr, /* Expression to resolve. May be NULL. */ ExprList *pList /* Expression list to resolve. May be NULL. */ ){ - SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ + SrcList *pSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ int rc; + u8 srcSpace[SZ_SRCLIST_1]; /* Memory space for the fake SrcList */ assert( type==0 || pTab!=0 ); assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr || type==NC_GenCol || pTab==0 ); memset(&sNC, 0, sizeof(sNC)); - memset(&sSrc, 0, sizeof(sSrc)); + pSrc = (SrcList*)srcSpace; + memset(pSrc, 0, SZ_SRCLIST_1); if( pTab ){ - sSrc.nSrc = 1; - sSrc.a[0].zName = pTab->zName; - sSrc.a[0].pSTab = pTab; - sSrc.a[0].iCursor = -1; + pSrc->nSrc = 1; + pSrc->a[0].zName = pTab->zName; + pSrc->a[0].pSTab = pTab; + pSrc->a[0].iCursor = -1; if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP ** schema elements */ @@ -2298,7 +2300,7 @@ int sqlite3ResolveSelfReference( } } sNC.pParse = pParse; - sNC.pSrcList = &sSrc; + sNC.pSrcList = pSrc; sNC.ncFlags = type | NC_IsDDL; if( (rc = sqlite3ResolveExprNames(&sNC, pExpr))!=SQLITE_OK ) return rc; if( pList ) rc = sqlite3ResolveExprListNames(&sNC, pList); diff --git a/src/select.c b/src/select.c index 904290223e..eedbd82818 100644 --- a/src/select.c +++ b/src/select.c @@ -154,7 +154,7 @@ Select *sqlite3SelectNew( pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; - if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc)); + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; @@ -1537,8 +1537,8 @@ static void selectInnerLoop( ** X extra columns. */ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ - int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); - KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); + int nExtra = (N+X)*(sizeof(CollSeq*)+1); + KeyInfo *p = sqlite3DbMallocRawNN(db, SZ_KEYINFO(0) + nExtra); if( p ){ p->aSortFlags = (u8*)&p->aColl[N+X]; p->nKeyField = (u16)N; @@ -1546,7 +1546,7 @@ KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ p->enc = ENC(db); p->db = db; p->nRef = 1; - memset(&p[1], 0, nExtra); + memset(p->aColl, 0, nExtra); }else{ return (KeyInfo*)sqlite3OomFault(db); } @@ -6062,7 +6062,7 @@ static int selectExpander(Walker *pWalker, Select *p){ pEList = p->pEList; if( pParse->pWith && (p->selFlags & SF_View) ){ if( p->pWith==0 ){ - p->pWith = (With*)sqlite3DbMallocZero(db, sizeof(With)); + p->pWith = (With*)sqlite3DbMallocZero(db, SZ_WITH(1) ); if( p->pWith==0 ){ return WRC_Abort; } @@ -7250,7 +7250,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ pExpr = 0; pSub = sqlite3SubqueryDetach(db, pFrom); sqlite3SrcListDelete(db, p->pSrc); - p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); + p->pSrc = sqlite3DbMallocZero(pParse->db, SZ_SRCLIST_1); while( pSub ){ Expr *pTerm; pPrior = pSub->pPrior; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 034f986901..7b71ace561 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -768,6 +768,16 @@ #define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) #endif +/* +** Work around C99 "flex-array" syntax for pre-C99 compilers, so as +** to avoid complaints from -fsanitize=strict-bounds. +*/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + /* ** Macros to compute minimum and maximum of two numbers. */ @@ -2577,9 +2587,13 @@ struct FKey { struct sColMap { /* Mapping of columns in pFrom to columns in zTo */ int iFrom; /* Index of column in pFrom */ char *zCol; /* Name of column in zTo. If NULL use PRIMARY KEY */ - } aCol[1]; /* One entry for each of nCol columns */ + } aCol[FLEXARRAY]; /* One entry for each of nCol columns */ }; +/* The size (in bytes) of an FKey object holding N columns. The answer +** does NOT include space to hold the zTo name. */ +#define SZ_FKEY(N) (offsetof(FKey,aCol)+(N)*sizeof(struct sColMap)) + /* ** SQLite supports many different ways to resolve a constraint ** error. ROLLBACK processing means that a constraint violation @@ -2641,9 +2655,12 @@ struct KeyInfo { u16 nAllField; /* Total columns, including key plus others */ sqlite3 *db; /* The database connection */ u8 *aSortFlags; /* Sort order for each column. */ - CollSeq *aColl[1]; /* Collating sequence for each term of the key */ + CollSeq *aColl[FLEXARRAY]; /* Collating sequence for each term of the key */ }; +/* The size (in bytes) of a KeyInfo object with up to N fields */ +#define SZ_KEYINFO(N) (offsetof(KeyInfo,aColl) + (N)*sizeof(CollSeq*)) + /* ** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. */ @@ -3216,9 +3233,14 @@ struct ExprList { int iConstExprReg; /* Register in which Expr value is cached. Used only ** by Parse.pConstExpr */ } u; - } a[1]; /* One slot for each expression in the list */ + } a[FLEXARRAY]; /* One slot for each expression in the list */ }; +/* The size (in bytes) of an ExprList object that is big enough to hold +** as many as N expressions. */ +#define SZ_EXPRLIST(N) \ + (offsetof(ExprList,a) + (N)*sizeof(struct ExprList_item)) + /* ** Allowed values for Expr.a.eEName */ @@ -3246,9 +3268,12 @@ struct IdList { int nId; /* Number of identifiers on the list */ struct IdList_item { char *zName; /* Name of the identifier */ - } a[1]; + } a[FLEXARRAY]; }; +/* The size (in bytes) of an IdList object that can hold up to N IDs. */ +#define SZ_IDLIST(N) (offsetof(IdList,a)+(N)*sizeof(struct IdList_item)) + /* ** Allowed values for IdList.eType, which determines which value of the a.u4 ** is valid. @@ -3368,11 +3393,19 @@ struct OnOrUsing { ** */ struct SrcList { - int nSrc; /* Number of tables or subqueries in the FROM clause */ - u32 nAlloc; /* Number of entries allocated in a[] below */ - SrcItem a[1]; /* One entry for each identifier on the list */ + int nSrc; /* Number of tables or subqueries in the FROM clause */ + u32 nAlloc; /* Number of entries allocated in a[] below */ + SrcItem a[FLEXARRAY]; /* One entry for each identifier on the list */ }; +/* Size (in bytes) of a SrcList object that can hold as many as N +** SrcItem objects. */ +#define SZ_SRCLIST(N) (offsetof(SrcList,a)+(N)*sizeof(SrcItem)) + +/* Size (in bytes( of a SrcList object that holds 1 SrcItem. This is a +** special case of SZ_SRCITEM(1) that comes up often. */ +#define SZ_SRCLIST_1 (offsetof(SrcList,a)+sizeof(SrcItem)) + /* ** Permitted values of the SrcList.a.jointype field */ @@ -4436,9 +4469,13 @@ struct With { int nCte; /* Number of CTEs in the WITH clause */ int bView; /* Belongs to the outermost Select of a view */ With *pOuter; /* Containing WITH clause, or NULL */ - Cte a[1]; /* For each CTE in the WITH clause.... */ + Cte a[FLEXARRAY]; /* For each CTE in the WITH clause.... */ }; +/* The size (in bytes) of a With object that can hold as many +** as N different CTEs. */ +#define SZ_WITH(N) (offsetof(With,a) + (N)*sizeof(Cte)) + /* ** The Cte object is not guaranteed to persist for the entire duration ** of code generation. (The query flattener or other parser tree @@ -4467,9 +4504,13 @@ struct DbClientData { DbClientData *pNext; /* Next in a linked list */ void *pData; /* The data */ void (*xDestructor)(void*); /* Destructor. Might be NULL */ - char zName[1]; /* Name of this client data. MUST BE LAST */ + char zName[FLEXARRAY]; /* Name of this client data. MUST BE LAST */ }; +/* The size (in bytes) of a DbClientData object that can has a name +** that is N bytes long, including the zero-terminator. */ +#define SZ_DBCLIENTDATA(N) (offsetof(DbClientData,zName)+(N)) + #ifdef SQLITE_DEBUG /* ** An instance of the TreeView object is used for printing the content of diff --git a/src/trigger.c b/src/trigger.c index 604c3ab42f..779da5e5fb 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -1039,7 +1039,8 @@ static void codeReturningTrigger( ExprList *pNew; Returning *pReturning; Select sSelect; - SrcList sFrom; + SrcList *pFrom; + u8 fromSpace[SZ_SRCLIST_1]; assert( v!=0 ); if( !pParse->bReturning ){ @@ -1055,13 +1056,14 @@ static void codeReturningTrigger( return; } memset(&sSelect, 0, sizeof(sSelect)); - memset(&sFrom, 0, sizeof(sFrom)); + pFrom = (SrcList*)fromSpace; + memset(pFrom, 0, SZ_SRCLIST_1); sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); - sSelect.pSrc = &sFrom; - sFrom.nSrc = 1; - sFrom.a[0].pSTab = pTab; - sFrom.a[0].zName = pTab->zName; /* tag-20240424-1 */ - sFrom.a[0].iCursor = -1; + sSelect.pSrc = pFrom; + pFrom->nSrc = 1; + pFrom->a[0].pSTab = pTab; + pFrom->a[0].zName = pTab->zName; /* tag-20240424-1 */ + pFrom->a[0].iCursor = -1; sqlite3SelectPrep(pParse, &sSelect, 0); if( pParse->nErr==0 ){ assert( db->mallocFailed==0 ); diff --git a/src/vdbe.c b/src/vdbe.c index ed5687f251..6ded15c799 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -278,9 +278,9 @@ static VdbeCursor *allocateCursor( i64 nByte; VdbeCursor *pCx = 0; - nByte = - ROUND8P(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + - (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); + nByte = SZ_VDBECURSOR(nField); + assert( ROUND8(nByte)==nByte ); + if( eCurType==CURTYPE_BTREE ) nByte += sqlite3BtreeCursorSize(); assert( iCur>=0 && iCurnCursor ); if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ @@ -313,8 +313,8 @@ static VdbeCursor *allocateCursor( pCx->nField = nField; pCx->aOffset = &pCx->aType[nField]; if( eCurType==CURTYPE_BTREE ){ - pCx->uc.pCursor = (BtCursor*) - &pMem->z[ROUND8P(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; + assert( ROUND8(SZ_VDBECURSOR(nField))==SZ_VDBECURSOR(nField) ); + pCx->uc.pCursor = (BtCursor*)&pMem->z[SZ_VDBECURSOR(nField)]; sqlite3BtreeCursorZero(pCx->uc.pCursor); } return pCx; @@ -7705,7 +7705,7 @@ case OP_AggStep: { ** ** Note: We could avoid this by using a regular memory cell from aMem[] for ** the accumulator, instead of allocating one here. */ - nAlloc = ROUND8P( sizeof(pCtx[0]) + (n-1)*sizeof(sqlite3_value*) ); + nAlloc = ROUND8P( SZ_CONTEXT(n) ); pCtx = sqlite3DbMallocRawNN(db, nAlloc + sizeof(Mem)); if( pCtx==0 ) goto no_mem; pCtx->pOut = (Mem*)((u8*)pCtx + nAlloc); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 12af827268..a86453f569 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -133,12 +133,18 @@ struct VdbeCursor { #endif VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */ - /* 2*nField extra array elements allocated for aType[], beyond the one - ** static element declared in the structure. nField total array slots for - ** aType[] and nField+1 array slots for aOffset[] */ - u32 aType[1]; /* Type values record decode. MUST BE LAST */ + /* Space is allocated for aType to hold at least 2*nField+1 entries: + ** nField slots for aType[] and nField+1 array slots for aOffset[] */ + u32 aType[FLEXARRAY]; /* Type values record decode. MUST BE LAST */ }; +/* +** The size (in bytes) of a VdbeCursor object that has an nField value of N +** or less. The value of SZ_VDBECURSOR(n) is guaranteed to be a multiple +** of 8. +*/ +#define SZ_VDBECURSOR(N) (offsetof(VdbeCursor,aType) + ((N)+1)*sizeof(u64)) + /* Return true if P is a null-only cursor */ #define IsNullCursor(P) \ @@ -395,9 +401,16 @@ struct sqlite3_context { u8 enc; /* Encoding to use for results */ u8 skipFlag; /* Skip accumulator loading if true */ u16 argc; /* Number of arguments */ - sqlite3_value *argv[1]; /* Argument set */ + sqlite3_value *argv[FLEXARRAY]; /* Argument set */ }; +/* +** The size (in bytes) of an sqlite3_context object that holds N +** argv[] arguments. +*/ +#define SZ_CONTEXT(N) \ + (offsetof(sqlite3_context,argv)+(N)*sizeof(sqlite3_value*)) + /* The ScanStatus object holds a single value for the ** sqlite3_stmt_scanstatus() interface. diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 6a8db6f394..ccb0f8867f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -445,12 +445,10 @@ int sqlite3VdbeAddFunctionCall( int eCallCtx /* Calling context */ ){ Vdbe *v = pParse->pVdbe; - int nByte; int addr; sqlite3_context *pCtx; assert( v ); - nByte = sizeof(*pCtx) + (nArg-1)*sizeof(sqlite3_value*); - pCtx = sqlite3DbMallocRawNN(pParse->db, nByte); + pCtx = sqlite3DbMallocRawNN(pParse->db, SZ_CONTEXT(nArg)); if( pCtx==0 ){ assert( pParse->db->mallocFailed ); freeEphemeralFunction(pParse->db, (FuncDef*)pFunc); diff --git a/src/vdbesort.c b/src/vdbesort.c index c9da88f6e1..9a7e0760c6 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -332,9 +332,12 @@ struct VdbeSorter { u8 iPrev; /* Previous thread used to flush PMA */ u8 nTask; /* Size of aTask[] array */ u8 typeMask; - SortSubtask aTask[1]; /* One or more subtasks */ + SortSubtask aTask[FLEXARRAY]; /* One or more subtasks */ }; +/* Size (in bytes) of a VdbeSorter object that works with N or fewer subtasks */ +#define SZ_VDBESORTER(N) (offsetof(VdbeSorter,aTask)+(N)*sizeof(SortSubtask)) + #define SORTER_TYPE_INTEGER 0x01 #define SORTER_TYPE_TEXT 0x02 @@ -966,8 +969,8 @@ int sqlite3VdbeSorterInit( assert( pCsr->eCurType==CURTYPE_SORTER ); assert( sizeof(KeyInfo) + UMXV(pCsr->pKeyInfo->nKeyField)*sizeof(CollSeq*) < 0x7fffffff ); - szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nKeyField-1)*sizeof(CollSeq*); - sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); + szKeyInfo = SZ_KEYINFO(pCsr->pKeyInfo->nKeyField+1); + sz = SZ_VDBESORTER(nWorker+1); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->uc.pSorter = pSorter; diff --git a/src/wal.c b/src/wal.c index d374d6eb4d..7f091a48cf 100644 --- a/src/wal.c +++ b/src/wal.c @@ -597,9 +597,13 @@ struct WalIterator { u32 *aPgno; /* Array of page numbers. */ int nEntry; /* Nr. of entries in aPgno[] and aIndex[] */ int iZero; /* Frame number associated with aPgno[0] */ - } aSegment[1]; /* One for every 32KB page in the wal-index */ + } aSegment[FLEXARRAY]; /* One for every 32KB page in the wal-index */ }; +/* Size (in bytes) of a WalIterator object suitable for N or fewer segments */ +#define SZ_WALITERATOR(N) \ + (offsetof(WalIterator,aSegment)*(N)*sizeof(struct WalSegment)) + /* ** Define the parameters of the hash tables in the wal-index file. There ** is a hash-table following every HASHTABLE_NPAGE page numbers in the @@ -1960,8 +1964,7 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ /* Allocate space for the WalIterator object. */ nSegment = walFramePage(iLast) + 1; - nByte = sizeof(WalIterator) - + (nSegment-1)*sizeof(struct WalSegment) + nByte = SZ_WALITERATOR(nSegment) + iLast*sizeof(ht_slot); p = (WalIterator *)sqlite3_malloc64(nByte + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) diff --git a/src/where.c b/src/where.c index 1275250067..ed63203495 100644 --- a/src/where.c +++ b/src/where.c @@ -35,11 +35,16 @@ struct HiddenIndexInfo { int eDistinct; /* Value to return from sqlite3_vtab_distinct() */ u32 mIn; /* Mask of terms that are IN (...) */ u32 mHandleIn; /* Terms that vtab will handle as IN (...) */ - sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST - ** because extra space is allocated to hold up - ** to nTerm such values */ + sqlite3_value *aRhs[FLEXARRAY]; /* RHS values for constraints. MUST BE LAST + ** Extra space is allocated to hold up + ** to nTerm such values */ }; +/* Size (in bytes) of a HiddenIndeInfo object sufficient to hold as +** many as N constraints */ +#define SZ_HIDDENINDEXINFO(N) \ + (offsetof(HiddenIndexInfo,aRhs) + (N)*sizeof(sqlite3_value*)) + /* Forward declaration of methods */ static int whereLoopResize(sqlite3*, WhereLoop*, int); @@ -1517,8 +1522,8 @@ static sqlite3_index_info *allocateIndexInfo( */ pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm - + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) - + sizeof(sqlite3_value*)*nTerm ); + + sizeof(*pIdxOrderBy)*nOrderBy + + SZ_HIDDENINDEXINFO(nTerm) ); if( pIdxInfo==0 ){ sqlite3ErrorMsg(pParse, "out of memory"); return 0; @@ -6712,10 +6717,7 @@ WhereInfo *sqlite3WhereBegin( ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - nByteWInfo = ROUND8P(sizeof(WhereInfo)); - if( nTabList>1 ){ - nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel)); - } + nByteWInfo = SZ_WHEREINFO(nTabList); pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); diff --git a/src/whereInt.h b/src/whereInt.h index 8ba8a7072d..1b0a02d8bc 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -501,9 +501,14 @@ struct WhereInfo { Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ WhereClause sWC; /* Decomposition of the WHERE clause */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ - WhereLevel a[1]; /* Information about each nest loop in WHERE */ + WhereLevel a[FLEXARRAY]; /* Information about each nest loop in WHERE */ }; +/* +** The size (in bytes) of a WhereInfo object that holds N WhereLevels. +*/ +#define SZ_WHEREINFO(N) (offsetof(WhereInfo,a)+(N)*sizeof(WhereLevel)) + /* ** Private interfaces - callable only by other where.c routines. ** diff --git a/src/wherecode.c b/src/wherecode.c index 1a0cdc6d71..5fe2308137 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -2313,8 +2313,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int nNotReady; /* The number of notReady tables */ SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3DbMallocRawNN(db, - sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + pOrTab = sqlite3DbMallocRawNN(db, SZ_SRCLIST(nNotReady+1)); if( pOrTab==0 ) return notReady; pOrTab->nAlloc = (u8)(nNotReady + 1); pOrTab->nSrc = pOrTab->nAlloc; @@ -2857,7 +2856,8 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( WhereInfo *pSubWInfo; WhereLoop *pLoop = pLevel->pWLoop; SrcItem *pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; - SrcList sFrom; + SrcList *pFrom; + u8 fromSpace[SZ_SRCLIST_1]; Bitmask mAll = 0; int k; @@ -2901,13 +2901,14 @@ SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); } } - sFrom.nSrc = 1; - sFrom.nAlloc = 1; - memcpy(&sFrom.a[0], pTabItem, sizeof(SrcItem)); - sFrom.a[0].fg.jointype = 0; + pFrom = (SrcList*)fromSpace; + pFrom->nSrc = 1; + pFrom->nAlloc = 1; + memcpy(&pFrom->a[0], pTabItem, sizeof(SrcItem)); + pFrom->a[0].fg.jointype = 0; assert( pParse->withinRJSubrtn < 100 ); pParse->withinRJSubrtn++; - pSubWInfo = sqlite3WhereBegin(pParse, &sFrom, pSubWhere, 0, 0, 0, + pSubWInfo = sqlite3WhereBegin(pParse, pFrom, pSubWhere, 0, 0, 0, WHERE_RIGHT_JOIN, 0); if( pSubWInfo ){ int iCur = pLevel->iTabCur; From b6e8f65ffe7dccf0c65176ed72241a61bd0d655a Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 14 Mar 2025 19:07:11 +0000 Subject: [PATCH 11/38] KeyInfo is now an indeterminate size, so we cannot declare a variable of that type, only a pointer to an instance of that type. FossilOrigin-Name: 37b687dc2d3b9dc82ed09a9c5b2c00e576b1eebe358a20d18a3da190347b644e --- manifest | 19 ++++++++----------- manifest.uuid | 2 +- src/vdbeInt.h | 3 ++- src/vdbeapi.c | 6 +++--- src/vdbeaux.c | 13 +++++++------ 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 549a0c86df..d294d91a35 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\suse\sof\sthe\sflexible-array\sfeature\sof\sC99,\swhen\savailable,\sto\stry\sto\npacify\s-fsanitize=strict-bounds.\s\sThis\scheck-in\sfixes\sthe\score.\sThere\sis\nmore\syet\sto\sdo\sin\sFTS3,\sRTREE,\sand\sin\sFTS5. -D 2025-03-14T18:10:02.673 +C KeyInfo\sis\snow\san\sindeterminate\ssize,\sso\swe\scannot\sdeclare\sa\svariable\sof\sthat\ntype,\sonly\sa\spointer\sto\san\sinstance\sof\sthat\stype. +D 2025-03-14T19:07:11.842 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -852,9 +852,9 @@ F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 F src/vacuum.c fbfc3e074c865d2b5b10b8a65a3783275b80c152653590690747a102bb6cc770 F src/vdbe.c b5deed01000b3970cfca089dc531cf9342afd96d00cc8b4ad26d303f088116ee F src/vdbe.h 31eddcffc1d14c76c2a20fe4e137e1ee43d44f370896fae14a067052801a3625 -F src/vdbeInt.h 11041d559992165651413e5868a907f698a2ec7af169fb2d20eb058a9ccd2313 -F src/vdbeapi.c cb8eb9e41a16f5fa3ce5b8f3910edfbba336d10156cfb7a79f92cf7bf443977b -F src/vdbeaux.c 932105750c15378176980288daea80008ffdac360716ae4e7c7f85bf462d272d +F src/vdbeInt.h 366843cb88740595011e091e180bb7258487ec9fd91c0b5a47cf9ff25620a482 +F src/vdbeapi.c a9ad72afed9aaec2acdde4daa5caa2f342b298f8c8859704143f6e3b78cb9966 +F src/vdbeaux.c 72a1c99d9300a5e0adff2c708074ac1355a619664301db474a48e147418fba05 F src/vdbeblob.c b1b4032cac46b41e44b957c4d00aee9851f862dfd85ecb68116ba49884b03dfd F src/vdbemem.c 571ae3116dbf840a62c4aaa6bc09d577dfef8ad4d3978cf37275bb5f9653217b F src/vdbesort.c 49e366d0216c782eba287bf602384e4330d2526a22f1275492d2785ce103c79b @@ -2213,11 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fa6f6ccdffc50024624306900efd2538c7415d8bdd0f02835b2e9c05adab3cf1 -R a7f539cde629f0570849a89a860ffc9d -T *branch * flex-array -T *sym-flex-array * -T -sym-trunk * +P 6fd6b32d06bd6a705e5140cd613af823b8183a6f6a9ceeeedfcf5e8b50821d68 +R fe593c8c6938e177844eba6146548bc4 U drh -Z ea27c767023e4b4e578a073476796f47 +Z 1a2d68d765d400a0cfddfeaade189b5f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 45864ab067..36487e89dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6fd6b32d06bd6a705e5140cd613af823b8183a6f6a9ceeeedfcf5e8b50821d68 +37b687dc2d3b9dc82ed09a9c5b2c00e576b1eebe358a20d18a3da190347b644e diff --git a/src/vdbeInt.h b/src/vdbeInt.h index a86453f569..600d9b8bc5 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -544,7 +544,7 @@ struct PreUpdate { VdbeCursor *pCsr; /* Cursor to read old values from */ int op; /* One of SQLITE_INSERT, UPDATE, DELETE */ u8 *aRecord; /* old.* database record */ - KeyInfo keyinfo; + KeyInfo *pKeyinfo; /* Key information */ UnpackedRecord *pUnpacked; /* Unpacked version of aRecord[] */ UnpackedRecord *pNewUnpacked; /* Unpacked version of new.* record */ int iNewReg; /* Register for new.* values */ @@ -556,6 +556,7 @@ struct PreUpdate { Table *pTab; /* Schema object being updated */ Index *pPk; /* PK index if pTab is WITHOUT ROWID */ sqlite3_value **apDflt; /* Array of default values, if required */ + u8 keyinfoSpace[SZ_KEYINFO(0)]; /* Space to hold pKeyinfo[0] content */ }; /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 31880d85b5..523f80097d 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2216,7 +2216,7 @@ int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ if( !aRec ) goto preupdate_old_out; rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec); if( rc==SQLITE_OK ){ - p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec); + p->pUnpacked = vdbeUnpackRecord(p->pKeyinfo, nRec, aRec); if( !p->pUnpacked ) rc = SQLITE_NOMEM; } if( rc!=SQLITE_OK ){ @@ -2281,7 +2281,7 @@ int sqlite3_preupdate_count(sqlite3 *db){ #else p = db->pPreUpdate; #endif - return (p ? p->keyinfo.nKeyField : 0); + return (p ? p->pKeyinfo->nKeyField : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -2364,7 +2364,7 @@ int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ Mem *pData = &p->v->aMem[p->iNewReg]; rc = ExpandBlob(pData); if( rc!=SQLITE_OK ) goto preupdate_new_out; - pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z); + pUnpack = vdbeUnpackRecord(p->pKeyinfo, pData->n, pData->z); if( !pUnpack ){ rc = SQLITE_NOMEM; goto preupdate_new_out; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ccb0f8867f..6d36f7280a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -5524,10 +5524,11 @@ void sqlite3VdbePreUpdateHook( preupdate.pCsr = pCsr; preupdate.op = op; preupdate.iNewReg = iReg; - preupdate.keyinfo.db = db; - preupdate.keyinfo.enc = ENC(db); - preupdate.keyinfo.nKeyField = pTab->nCol; - preupdate.keyinfo.aSortFlags = (u8*)&fakeSortOrder; + preupdate.pKeyinfo = (KeyInfo*)&preupdate.keyinfoSpace; + preupdate.pKeyinfo->db = db; + preupdate.pKeyinfo->enc = ENC(db); + preupdate.pKeyinfo->nKeyField = pTab->nCol; + preupdate.pKeyinfo->aSortFlags = (u8*)&fakeSortOrder; preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; preupdate.pTab = pTab; @@ -5537,8 +5538,8 @@ void sqlite3VdbePreUpdateHook( db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); db->pPreUpdate = 0; sqlite3DbFree(db, preupdate.aRecord); - vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pUnpacked); - vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pNewUnpacked); + vdbeFreeUnpacked(db, preupdate.pKeyinfo->nKeyField+1,preupdate.pUnpacked); + vdbeFreeUnpacked(db, preupdate.pKeyinfo->nKeyField+1,preupdate.pNewUnpacked); sqlite3VdbeMemRelease(&preupdate.oldipk); if( preupdate.aNew ){ int i; From 01ef1dfc1f147490bcd9dd47f2f41fb18dfcf477 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 14 Mar 2025 20:19:49 +0000 Subject: [PATCH 12/38] Use flexible arrays for RTREE. FossilOrigin-Name: 2b41776179c726586e3ff836edcf235938cf02f7c5e33c1d6954b84d4061b8d5 --- ext/rtree/rtree.c | 17 ++++++++++++++--- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 21c87bdc76..5e9fa6996e 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -94,6 +94,14 @@ typedef unsigned int u32; # define ALWAYS(X) (X) # define NEVER(X) (X) #endif +#ifndef offsetof +#define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif #endif /* !defined(SQLITE_AMALGAMATION) */ /* Macro to check for 4-byte alignment. Only used inside of assert() */ @@ -414,9 +422,13 @@ struct RtreeMatchArg { RtreeGeomCallback cb; /* Info about the callback functions */ int nParam; /* Number of parameters to the SQL function */ sqlite3_value **apSqlParam; /* Original SQL parameter values */ - RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ + RtreeDValue aParam[FLEXARRAY]; /* Values for parameters to the SQL function */ }; +/* Size of an RtreeMatchArg object with N parameters */ +#define SZ_RTREEMATCHARG(N) \ + (offsetof(RtreeMatchArg,aParam)+(N)*sizeof(RtreeDValue)) + #ifndef MAX # define MAX(x,y) ((x) < (y) ? (y) : (x)) #endif @@ -4369,8 +4381,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ sqlite3_int64 nBlob; int memErr = 0; - nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue) - + nArg*sizeof(sqlite3_value*); + nBlob = SZ_RTREEMATCHARG(nArg) + nArg*sizeof(sqlite3_value*); pBlob = (RtreeMatchArg *)sqlite3_malloc64(nBlob); if( !pBlob ){ sqlite3_result_error_nomem(ctx); diff --git a/manifest b/manifest index d294d91a35..2e3a0d5c67 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C KeyInfo\sis\snow\san\sindeterminate\ssize,\sso\swe\scannot\sdeclare\sa\svariable\sof\sthat\ntype,\sonly\sa\spointer\sto\san\sinstance\sof\sthat\stype. -D 2025-03-14T19:07:11.842 +C Use\sflexible\sarrays\sfor\sRTREE. +D 2025-03-14T20:19:49.223 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -537,7 +537,7 @@ F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c3350 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 734aa36238bcd2dee91db5dba107d5fcbdb02396612811377a8ad50f1272b1c1 F ext/rtree/geopoly.c f0573d5109fdc658a180db0db6eec86ab2a1cf5ce58ec66cbf3356167ea757eb -F ext/rtree/rtree.c 980f84e9fecfa99f80ae066e13b198505790449542f802d6b6466ed839149af4 +F ext/rtree/rtree.c 99dade93b5ca1c1fa4a5ba1381140d88b27a52573a92897827d9eb2a8059a460 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test e0608db762b2aadca0ecb6f97396cf66244490adc3ba88f2a292b27be3e1da3e F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d @@ -786,7 +786,7 @@ F src/shell.c.in 248050551cad788f8bb4b4728e00d8e36a10130d2d101e55cd51cfee03df91f F src/sqlite.h.in fd70afd92948cf7cc93f687ac960bad1b0b6fbc436752419eff2fd65a1809380 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 79d0f2705e62a1acaa11c10ac723749726ebbd96fb9be9f93c31fb44bdd4da1f +F src/sqliteInt.h 96133c5b4371629b30644a88108a0ca99e6a95a55509cdfc8de9961fba4bbd26 F src/sqliteLimit.h 6d817c28a8f19af95e6f4921933b7fbbca48a962bce0eb0ec81e8bb3ef38e68b F src/status.c 0e72e4f6be6ccfde2488eb63210297e75f569f3ce9920f6c3d77590ec6ce5ffd F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6fd6b32d06bd6a705e5140cd613af823b8183a6f6a9ceeeedfcf5e8b50821d68 -R fe593c8c6938e177844eba6146548bc4 +P 37b687dc2d3b9dc82ed09a9c5b2c00e576b1eebe358a20d18a3da190347b644e +R c3a6881fe54f628cb6eebb7a30e7280b U drh -Z 1a2d68d765d400a0cfddfeaade189b5f +Z 9b83357076de4b4a7589bb4979f7b8b7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 36487e89dc..dae93f569a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -37b687dc2d3b9dc82ed09a9c5b2c00e576b1eebe358a20d18a3da190347b644e +2b41776179c726586e3ff836edcf235938cf02f7c5e33c1d6954b84d4061b8d5 diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7b71ace561..35e5b94d71 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -765,7 +765,7 @@ ** ourselves. */ #ifndef offsetof -#define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) +#define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) #endif /* From dbd455a0fd4c05129d61d64cd5a1dc141ca7a283 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 14 Mar 2025 21:15:11 +0000 Subject: [PATCH 13/38] Fix one of two flexible arrays in FTS3. FossilOrigin-Name: ddfa87c17906ecf7fd5639a87bbfa9a87d17ab688159acd2fd80cc5b6f25f09b --- ext/fts3/fts3Int.h | 19 ++++++++++++++++++- ext/fts3/fts3_expr.c | 6 +++--- ext/fts3/fts3_snippet.c | 4 ++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 06f3c6efec..a53ecdf17c 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -202,6 +202,19 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ #define deliberate_fall_through +/* +** Macros needed to provide flexible arrays in a portable way +*/ +#ifndef offsetof +# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + + #endif /* SQLITE_AMALGAMATION */ #ifdef SQLITE_DEBUG @@ -431,9 +444,13 @@ struct Fts3Phrase { */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ - Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ + Fts3PhraseToken aToken[FLEXARRAY]; /* One for each token in the phrase */ }; +/* Size (in bytes) of an Fts3Phrase object large enough to hold N tokens */ +#define SZ_FTS3PHRASE(N) \ + (offsetof(Fts3Phrase,aToken)+(N)*sizeof(Fts3PhraseToken)) + /* ** A tree of these objects forms the RHS of a MATCH operator. ** diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index ca857835e2..55943c271f 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -202,7 +202,7 @@ static int getNextToken( rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); if( rc==SQLITE_OK ){ - nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + nByte = sizeof(Fts3Expr) + SZ_FTS3PHRASE(1) + nToken; pRet = (Fts3Expr *)sqlite3Fts3MallocZero(nByte); if( !pRet ){ rc = SQLITE_NOMEM; @@ -212,7 +212,7 @@ static int getNextToken( pRet->pPhrase->nToken = 1; pRet->pPhrase->iColumn = iCol; pRet->pPhrase->aToken[0].n = nToken; - pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + pRet->pPhrase->aToken[0].z = (char*)&pRet->pPhrase->aToken[1]; memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); if( iEnd Date: Fri, 14 Mar 2025 23:20:12 +0000 Subject: [PATCH 14/38] In FTS3, rename the MatchinfoBuffer.aMatchinfo field to aMI, to avoid confusing it with MatchInfo.aMatchinfo. Make aMI a flexiable array. FossilOrigin-Name: bb00b973980d259ca85af84c054501cae78b3a9d33ccffa54d7034235dd8d50d --- ext/fts3/fts3_snippet.c | 24 ++++++++++++------------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 5272d46bf0..36ee94a48b 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -108,12 +108,12 @@ struct MatchinfoBuffer { int nElem; int bGlobal; /* Set if global data is loaded */ char *zMatchinfo; - u32 aMatchinfo[1]; + u32 aMI[FLEXARRAY]; }; /* Size (in bytes) of a MatchinfoBuffer sufficient for N elements */ #define SZ_MATCHINFOBUFFER(N) \ - (offsetof(MatchinfoBuffer,aMatchinfo)+((((N)+1)/2)*sizeof(u64)) + (offsetof(MatchinfoBuffer,aMI)+(((N)+1)/2)*sizeof(u64)) /* @@ -139,13 +139,13 @@ struct StrBuffer { static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ MatchinfoBuffer *pRet; sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) - + sizeof(MatchinfoBuffer); + + SZ_MATCHINFOBUFFER(1); sqlite3_int64 nStr = strlen(zMatchinfo); pRet = sqlite3Fts3MallocZero(nByte + nStr+1); if( pRet ){ - pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; - pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + pRet->aMI[0] = (u8*)(&pRet->aMI[1]) - (u8*)pRet; + pRet->aMI[1+nElem] = pRet->aMI[0] + sizeof(u32)*((int)nElem+1); pRet->nElem = (int)nElem; pRet->zMatchinfo = ((char*)pRet) + nByte; @@ -159,10 +159,10 @@ static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ static void fts3MIBufferFree(void *p){ MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); - assert( (u32*)p==&pBuf->aMatchinfo[1] - || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + assert( (u32*)p==&pBuf->aMI[1] + || (u32*)p==&pBuf->aMI[pBuf->nElem+2] ); - if( (u32*)p==&pBuf->aMatchinfo[1] ){ + if( (u32*)p==&pBuf->aMI[1] ){ pBuf->aRef[1] = 0; }else{ pBuf->aRef[2] = 0; @@ -179,18 +179,18 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ if( p->aRef[1]==0 ){ p->aRef[1] = 1; - aOut = &p->aMatchinfo[1]; + aOut = &p->aMI[1]; xRet = fts3MIBufferFree; } else if( p->aRef[2]==0 ){ p->aRef[2] = 1; - aOut = &p->aMatchinfo[p->nElem+2]; + aOut = &p->aMI[p->nElem+2]; xRet = fts3MIBufferFree; }else{ aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32)); if( aOut ){ xRet = sqlite3_free; - if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + if( p->bGlobal ) memcpy(aOut, &p->aMI[1], p->nElem*sizeof(u32)); } } @@ -200,7 +200,7 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ p->bGlobal = 1; - memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); + memcpy(&p->aMI[2+p->nElem], &p->aMI[1], p->nElem*sizeof(u32)); } /* diff --git a/manifest b/manifest index 999dcec3d4..8fa55dd409 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sone\sof\stwo\sflexible\sarrays\sin\sFTS3. -D 2025-03-14T21:15:11.288 +C In\sFTS3,\srename\sthe\sMatchinfoBuffer.aMatchinfo\sfield\sto\saMI,\sto\savoid\sconfusing\nit\swith\sMatchInfo.aMatchinfo.\s\sMake\saMI\sa\sflexiable\sarray. +D 2025-03-14T23:20:12.428 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -87,7 +87,7 @@ F ext/fts3/fts3_hash.c d9dba473741445789330c7513d4f65737c92df23c3212784312931641 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116 F ext/fts3/fts3_porter.c 024417020c57dd1ab39816f5fe6cf45222a857b78a1f6412f040ada1ceabd4ff -F ext/fts3/fts3_snippet.c 53cb456dbf7f1cfed57f031bf94efc5df7bcd20158fb0442f891879f3b9337f8 +F ext/fts3/fts3_snippet.c 55506af9c656d06ad6acef0735b67749d199617421f2e66c5b7101745b9cf1ba F ext/fts3/fts3_term.c 6a96027ad364001432545fe43322b6af04ed28bb5619ec51af1f59d0710d6d69 F ext/fts3/fts3_test.c 7a9cb3d61774134211bf4bfdf1adcb581a1a0377f2d050a121ae7ab44baef0e3 F ext/fts3/fts3_tokenize_vtab.c 7fd9ef364f257b97218b9c331f2378e307375c592f70fd541f714e747d944962 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2b41776179c726586e3ff836edcf235938cf02f7c5e33c1d6954b84d4061b8d5 -R b928acd4f3ec564994a518332f83b638 +P ddfa87c17906ecf7fd5639a87bbfa9a87d17ab688159acd2fd80cc5b6f25f09b +R 5924af19939adfea920b9a7d74b7564d U drh -Z f48ebbbdb7fe4570c0fe613e575ace55 +Z e36dec86ff4d157017d3250585c57577 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a4f78c7437..f55aab8e29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ddfa87c17906ecf7fd5639a87bbfa9a87d17ab688159acd2fd80cc5b6f25f09b +bb00b973980d259ca85af84c054501cae78b3a9d33ccffa54d7034235dd8d50d From 502b7a236e4a4ae16f88d46417256216ebedc49a Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 14 Mar 2025 23:57:53 +0000 Subject: [PATCH 15/38] Turn Fts5Colset.aiCol into a flexible array. FossilOrigin-Name: 0c4d9c74741794468adc444908f6024f016738aa2852d3a646f2c28d079d9446 --- ext/fts5/fts5Int.h | 17 +++++++++++++++-- ext/fts5/fts5_expr.c | 10 +++++----- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 832f9ad477..2da347862e 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -75,6 +75,18 @@ typedef sqlite3_uint64 u64; # define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif +/* +** Macros needed to provide flexible arrays in a portable way +*/ +#ifndef offsetof +# define offsetof(STRUCTURE,FIELD) ((size_t)((char*)&((STRUCTURE*)0)->FIELD)) +#endif +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + #endif /* Truncate very long tokens to this many bytes. Hard limit is @@ -147,10 +159,11 @@ typedef struct Fts5Colset Fts5Colset; */ struct Fts5Colset { int nCol; - int aiCol[1]; + int aiCol[FLEXARRAY]; }; - +/* Size (int bytes) of a complete Fts5Colset object with N columns. */ +#define SZ_FTS5COLSET(N) (sizeof(i64)*((N+2)/2)) /************************************************************************** ** Interface to code in fts5_config.c. fts5_config.c contains contains code diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index dc2f7fb39c..85dbbec4de 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -292,7 +292,7 @@ int sqlite3Fts5ExprNew( /* If the LHS of the MATCH expression was a user column, apply the ** implicit column-filter. */ if( sParse.rc==SQLITE_OK && iColnCol ){ - int n = sizeof(Fts5Colset); + int n = SZ_FTS5COLSET(1); Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); if( pColset ){ pColset->nCol = 1; @@ -1912,7 +1912,7 @@ int sqlite3Fts5ExprClonePhrase( if( pColsetOrig ){ sqlite3_int64 nByte; Fts5Colset *pColset; - nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); + nByte = SZ_FTS5COLSET(pColsetOrig->nCol); pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); if( pColset ){ memcpy(pColset, pColsetOrig, (size_t)nByte); @@ -2034,7 +2034,7 @@ static Fts5Colset *fts5ParseColset( assert( pParse->rc==SQLITE_OK ); assert( iCol>=0 && iColpConfig->nCol ); - pNew = sqlite3_realloc64(p, sizeof(Fts5Colset) + sizeof(int)*nCol); + pNew = sqlite3_realloc64(p, SZ_FTS5COLSET(nCol+1)); if( pNew==0 ){ pParse->rc = SQLITE_NOMEM; }else{ @@ -2069,7 +2069,7 @@ Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ int nCol = pParse->pConfig->nCol; pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, - sizeof(Fts5Colset) + sizeof(int)*nCol + SZ_FTS5COLSET(nCol+1) ); if( pRet ){ int i; @@ -2130,7 +2130,7 @@ Fts5Colset *sqlite3Fts5ParseColset( static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ Fts5Colset *pRet; if( pOrig ){ - sqlite3_int64 nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); + sqlite3_int64 nByte = SZ_FTS5COLSET(pOrig->nCol); pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); if( pRet ){ memcpy(pRet, pOrig, (size_t)nByte); diff --git a/manifest b/manifest index 8fa55dd409..408a20c931 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sFTS3,\srename\sthe\sMatchinfoBuffer.aMatchinfo\sfield\sto\saMI,\sto\savoid\sconfusing\nit\swith\sMatchInfo.aMatchinfo.\s\sMake\saMI\sa\sflexiable\sarray. -D 2025-03-14T23:20:12.428 +C Turn\sFts5Colset.aiCol\sinto\sa\sflexible\sarray. +D 2025-03-14T23:57:53.780 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -106,11 +106,11 @@ F ext/fts3/unicode/mkunicode.tcl cbf5f7b5c8ce8014bad731f246f2e520eece908465de477 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h ff5d3cc88b29e41612bfb29eb723e29e38973de62ca75ba3e8f94ccb67f5b5f2 -F ext/fts5/fts5Int.h 6abff7dd770dc5969c994c281e6e77fc277ce414d56cc4a62c145cc7036b0b67 +F ext/fts5/fts5Int.h bffbd0acdcdf509899681f4e1cfeef1c955030acd9fe15ff9082410f80c3bead F ext/fts5/fts5_aux.c da4a7a9a11ec15c6df0699d908915a209bcde48f0b04101461316b59f71abffb F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d7c53066c7 F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 -F ext/fts5/fts5_expr.c 4a35c6edc4a36862597532ace43db20f5dfd4a2f789d4fa1dd7786b677b73036 +F ext/fts5/fts5_expr.c 390502c9954e20a9b6fcc31e4dc699df78a4ad52fd64e5b8aef0bfb82aa4a40d F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 F ext/fts5/fts5_index.c 2f35dd8408946f0e0bfea8f3bfbe8dfaafe90a5345885b43d678546c19266673 F ext/fts5/fts5_main.c b0e95a793f3c649d313c536269403e1a413ee665223adb5f8196edd2bc1146f7 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ddfa87c17906ecf7fd5639a87bbfa9a87d17ab688159acd2fd80cc5b6f25f09b -R 5924af19939adfea920b9a7d74b7564d +P bb00b973980d259ca85af84c054501cae78b3a9d33ccffa54d7034235dd8d50d +R 05b17a99b8821529239b9da49f396cc0 U drh -Z e36dec86ff4d157017d3250585c57577 +Z 04bcbcb01c8f02443f6768740e1f8171 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f55aab8e29..ec9f099abf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb00b973980d259ca85af84c054501cae78b3a9d33ccffa54d7034235dd8d50d +0c4d9c74741794468adc444908f6024f016738aa2852d3a646f2c28d079d9446 From 17abe9e2516dffefbd2ba5a545a3ce83aed1fb42 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 15 Mar 2025 00:11:22 +0000 Subject: [PATCH 16/38] Convert the Fts5Sorter.aIdx field to a flexible array. FossilOrigin-Name: 28ac776a23da2753265a7fe2ee2ebb09964815fc9058e69c08275fc217842edc --- ext/fts5/fts5_main.c | 6 ++++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 4d5e3cd4f3..e888abf215 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -170,9 +170,11 @@ struct Fts5Sorter { i64 iRowid; /* Current rowid */ const u8 *aPoslist; /* Position lists for current row */ int nIdx; /* Number of entries in aIdx[] */ - int aIdx[1]; /* Offsets into aPoslist for current row */ + int aIdx[FLEXARRAY]; /* Offsets into aPoslist for current row */ }; +/* Size (int bytes) of an Fts5Sorter object with N indexes */ +#define SZ_FTS5SORTER(N) (offsetof(Fts5Sorter,nIdx)+((N+2)/2)*sizeof(i64)) /* ** Virtual-table cursor object. @@ -1050,7 +1052,7 @@ static int fts5CursorFirstSorted( const char *zRankArgs = pCsr->zRankArgs; nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); - nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); + nByte = SZ_FTS5SORTER(nPhrase); pSorter = (Fts5Sorter*)sqlite3_malloc64(nByte); if( pSorter==0 ) return SQLITE_NOMEM; memset(pSorter, 0, (size_t)nByte); diff --git a/manifest b/manifest index 408a20c931..2d0615da6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Turn\sFts5Colset.aiCol\sinto\sa\sflexible\sarray. -D 2025-03-14T23:57:53.780 +C Convert\sthe\sFts5Sorter.aIdx\sfield\sto\sa\sflexible\sarray. +D 2025-03-15T00:11:22.914 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -113,7 +113,7 @@ F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773 F ext/fts5/fts5_expr.c 390502c9954e20a9b6fcc31e4dc699df78a4ad52fd64e5b8aef0bfb82aa4a40d F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 F ext/fts5/fts5_index.c 2f35dd8408946f0e0bfea8f3bfbe8dfaafe90a5345885b43d678546c19266673 -F ext/fts5/fts5_main.c b0e95a793f3c649d313c536269403e1a413ee665223adb5f8196edd2bc1146f7 +F ext/fts5/fts5_main.c 57933c18efe1058d8871199875c7a59744dabc3904f3aefbf9ff4a4e11fc79e2 F ext/fts5/fts5_storage.c 1ad05dab4830a4e2eaf2900bb143477f93bc17437093582f36f4b818809e88d8 F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 F ext/fts5/fts5_test_mi.c d35fdd50db99a775a040fb57127a45adc968e97da94ae784eec664256ac86db2 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bb00b973980d259ca85af84c054501cae78b3a9d33ccffa54d7034235dd8d50d -R 05b17a99b8821529239b9da49f396cc0 +P 0c4d9c74741794468adc444908f6024f016738aa2852d3a646f2c28d079d9446 +R 36e1481e3adfdafd4b3b226a43a9ad79 U drh -Z 04bcbcb01c8f02443f6768740e1f8171 +Z 97caa33dc844f08a198788bd53346fc8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ec9f099abf..103c672440 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c4d9c74741794468adc444908f6024f016738aa2852d3a646f2c28d079d9446 +28ac776a23da2753265a7fe2ee2ebb09964815fc9058e69c08275fc217842edc From bd0e3ed522a10f32d689bda1991068e35add65fb Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 15 Mar 2025 12:22:39 +0000 Subject: [PATCH 17/38] Use flexible arrays whereever appropriate in FTS5. FossilOrigin-Name: 16dfc415b6e98a2acae79a24bb0afd401e60efc27cbdd1603a426fd33e17d427 --- ext/fts5/fts5_expr.c | 42 +++++++++++++++---------- ext/fts5/fts5_index.c | 73 ++++++++++++++++++++++++++----------------- manifest | 14 ++++----- manifest.uuid | 2 +- 4 files changed, 79 insertions(+), 52 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 85dbbec4de..d7574aec52 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -86,9 +86,13 @@ struct Fts5ExprNode { /* Child nodes. For a NOT node, this array always contains 2 entries. For ** AND or OR nodes, it contains 2 or more entries. */ int nChild; /* Number of child nodes */ - Fts5ExprNode *apChild[1]; /* Array of child nodes */ + Fts5ExprNode *apChild[FLEXARRAY]; /* Array of child nodes */ }; +/* Size (in bytes) of an Fts5ExprNode object that holds up to N children */ +#define SZ_FTS5EXPRNODE(N) \ + (offsetof(Fts5ExprNode,apChild) + (N)*sizeof(Fts5ExprNode*)) + #define Fts5NodeIsString(p) ((p)->eType==FTS5_TERM || (p)->eType==FTS5_STRING) /* @@ -119,9 +123,13 @@ struct Fts5ExprPhrase { Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */ Fts5Buffer poslist; /* Current position list */ int nTerm; /* Number of entries in aTerm[] */ - Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ + Fts5ExprTerm aTerm[FLEXARRAY]; /* Terms that make up this phrase */ }; +/* Size (in bytes) of an Fts5ExprPhrase object that holds up to N terms */ +#define SZ_FTS5EXPRPHRASE(N) \ + (offsetof(Fts5ExprPhrase,aTerm) + (N)*sizeof(Fts5ExprTerm)) + /* ** One or more phrases that must appear within a certain token distance of ** each other within each matching document. @@ -130,9 +138,12 @@ struct Fts5ExprNearset { int nNear; /* NEAR parameter */ Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */ int nPhrase; /* Number of entries in aPhrase[] array */ - Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ + Fts5ExprPhrase *apPhrase[FLEXARRAY]; /* Array of phrase pointers */ }; +/* Size (in bytes) of an Fts5ExprNearset object covering up to N phrases */ +#define SZ_FTS5EXPRNEARSET(N) \ + (offsetof(Fts5ExprNearset,apPhrase)+(N)*sizeof(Fts5ExprPhrase*)) /* ** Parse context. @@ -1650,7 +1661,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( if( pParse->rc==SQLITE_OK ){ if( pNear==0 ){ sqlite3_int64 nByte; - nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); + nByte = SZ_FTS5EXPRNEARSET(SZALLOC+1); pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; @@ -1661,7 +1672,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset( int nNew = pNear->nPhrase + SZALLOC; sqlite3_int64 nByte; - nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); + nByte = SZ_FTS5EXPRNEARSET(nNew+1); pRet = (Fts5ExprNearset*)sqlite3_realloc64(pNear, nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; @@ -1752,12 +1763,12 @@ static int fts5ParseTokenize( int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); pNew = (Fts5ExprPhrase*)sqlite3_realloc64(pPhrase, - sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew + SZ_FTS5EXPRPHRASE(nNew+1) ); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase)); + if( pPhrase==0 ) memset(pNew, 0, SZ_FTS5EXPRPHRASE(1)); pCtx->pPhrase = pPhrase = pNew; pNew->nTerm = nNew - SZALLOC; } @@ -1865,7 +1876,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm( if( sCtx.pPhrase==0 ){ /* This happens when parsing a token or quoted phrase that contains ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); + sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, SZ_FTS5EXPRPHRASE(1)); }else if( sCtx.pPhrase->nTerm ){ sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix; } @@ -1900,12 +1911,11 @@ int sqlite3Fts5ExprClonePhrase( sizeof(Fts5ExprPhrase*)); } if( rc==SQLITE_OK ){ - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNode)); + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRNODE(1)); } if( rc==SQLITE_OK ){ - pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, - sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + SZ_FTS5EXPRNEARSET(2)); } if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){ Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; @@ -1940,7 +1950,7 @@ int sqlite3Fts5ExprClonePhrase( }else{ /* This happens when parsing a token or quoted phrase that contains ** no token characters at all. (e.g ... MATCH '""'). */ - sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRPHRASE(1)); } } @@ -2298,7 +2308,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( assert( pNear->nPhrase==1 ); assert( pParse->bPhraseToAnd ); - nByte = sizeof(Fts5ExprNode) + nTerm*sizeof(Fts5ExprNode*); + nByte = SZ_FTS5EXPRNODE(nTerm+1); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ pRet->eType = FTS5_AND; @@ -2308,7 +2318,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( pParse->nPhrase--; for(ii=0; iirc, sizeof(Fts5ExprPhrase) + &pParse->rc, SZ_FTS5EXPRPHRASE(1) ); if( pPhrase ){ if( parseGrowPhraseArray(pParse) ){ @@ -2377,7 +2387,7 @@ Fts5ExprNode *sqlite3Fts5ParseNode( if( pRight->eType==eType ) nChild += pRight->nChild-1; } - nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); + nByte = SZ_FTS5EXPRNODE(nChild); pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); if( pRet ){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 70a4752e26..63840de1fb 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -422,9 +422,13 @@ struct Fts5Structure { u64 nOriginCntr; /* Origin value for next top-level segment */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ - Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ + Fts5StructureLevel aLevel[FLEXARRAY]; /* Array of nLevel level objects */ }; +/* Size (in bytes) of an Fts5Structure object holding up to N levels */ +#define SZ_FTS5STRUCTURE(N) \ + (offsetof(Fts5Structure,aLevel) + (N)*sizeof(Fts5StructureLevel)) + /* ** An object of type Fts5SegWriter is used to write to segments. */ @@ -554,11 +558,15 @@ struct Fts5SegIter { ** Array of tombstone pages. Reference counted. */ struct Fts5TombstoneArray { - int nRef; /* Number of pointers to this object */ + int nRef; /* Number of pointers to this object */ int nTombstone; - Fts5Data *apTombstone[1]; /* Array of tombstone pages */ + Fts5Data *apTombstone[FLEXARRAY]; /* Array of tombstone pages */ }; +/* Size (in bytes) of an Fts5TombstoneArray holding up to N tombstones */ +#define SZ_FTS5TOMBSTONEARRAY(N) \ + (offsetof(Fts5TombstoneArray,apTombstone)+(N)*sizeof(Fts5Data*)) + /* ** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. @@ -627,9 +635,12 @@ struct Fts5Iter { i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ - Fts5SegIter aSeg[1]; /* Array of segment iterators */ + Fts5SegIter aSeg[FLEXARRAY]; /* Array of segment iterators */ }; +/* Size (in bytes) of an Fts5Iter object holding up to N segment iterators */ +#define SZ_FTS5ITER(N) (offsetof(Fts5Iter,aSeg)+(N)*sizeof(Fts5SegIter)) + /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. @@ -656,9 +667,13 @@ struct Fts5DlidxLvl { struct Fts5DlidxIter { int nLvl; int iSegid; - Fts5DlidxLvl aLvl[1]; + Fts5DlidxLvl aLvl[FLEXARRAY]; }; +/* Size (in bytes) of an Fts5DlidxIter object with up to N levels */ +#define SZ_FTS5DLIDXITER(N) \ + (offsetof(Fts5DlidxIter,aLvl)+(N)*sizeof(Fts5DlidxLvl)) + static void fts5PutU16(u8 *aOut, u16 iVal){ aOut[0] = (iVal>>8); aOut[1] = (iVal&0xFF); @@ -1026,7 +1041,7 @@ int sqlite3Fts5StructureTest(Fts5Index *p, void *pStruct){ static void fts5StructureMakeWritable(int *pRc, Fts5Structure **pp){ Fts5Structure *p = *pp; if( *pRc==SQLITE_OK && p->nRef>1 ){ - i64 nByte = sizeof(Fts5Structure)+(p->nLevel-1)*sizeof(Fts5StructureLevel); + i64 nByte = SZ_FTS5STRUCTURE(p->nLevel); Fts5Structure *pNew; pNew = (Fts5Structure*)sqlite3Fts5MallocZero(pRc, nByte); if( pNew ){ @@ -1100,10 +1115,7 @@ static int fts5StructureDecode( ){ return FTS5_CORRUPT; } - nByte = ( - sizeof(Fts5Structure) + /* Main structure */ - sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ - ); + nByte = SZ_FTS5STRUCTURE(nLevel); pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); if( pRet ){ @@ -1183,10 +1195,7 @@ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ if( *pRc==SQLITE_OK ){ Fts5Structure *pStruct = *ppStruct; int nLevel = pStruct->nLevel; - sqlite3_int64 nByte = ( - sizeof(Fts5Structure) + /* Main structure */ - sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ - ); + sqlite3_int64 nByte = SZ_FTS5STRUCTURE(nLevel+2); pStruct = sqlite3_realloc64(pStruct, nByte); if( pStruct ){ @@ -1725,7 +1734,7 @@ static Fts5DlidxIter *fts5DlidxIterInit( int bDone = 0; for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ - sqlite3_int64 nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); + sqlite3_int64 nByte = SZ_FTS5DLIDXITER(i+1); Fts5DlidxIter *pNew; pNew = (Fts5DlidxIter*)sqlite3_realloc64(pIter, nByte); @@ -1943,7 +1952,7 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ const int nTomb = pIter->pSeg->nPgTombstone; if( nTomb>0 ){ - int nByte = nTomb * sizeof(Fts5Data*) + sizeof(Fts5TombstoneArray); + int nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1); Fts5TombstoneArray *pNew; pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -3404,8 +3413,7 @@ static Fts5Iter *fts5MultiIterAlloc( for(nSlot=2; nSlotaSeg[] */ + SZ_FTS5ITER(nSlot) + /* pNew + pNew->aSeg[] */ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ ); if( pNew ){ @@ -5771,7 +5779,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( Fts5Structure *pStruct ){ Fts5Structure *pNew = 0; - sqlite3_int64 nByte = sizeof(Fts5Structure); + sqlite3_int64 nByte = SZ_FTS5STRUCTURE(1); int nSeg = pStruct->nSegment; int i; @@ -5801,6 +5809,7 @@ static Fts5Structure *fts5IndexOptimizeStruct( } nByte += (((i64)pStruct->nLevel)+1) * sizeof(Fts5StructureLevel); + assert( nByte==SZ_FTS5STRUCTURE(pStruct->nLevel+2) ); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ @@ -6377,9 +6386,13 @@ struct Fts5TokenDataIter { int nIterAlloc; Fts5PoslistReader *aPoslistReader; int *aPoslistToIter; - Fts5Iter *apIter[1]; + Fts5Iter *apIter[FLEXARRAY]; }; +/* Size in bytes of an Fts5TokenDataIter object holding up to N iterators */ +#define SZ_FTS5TOKENDATAITER(N) \ + (offsetof(Fts5TokenDataIter,apIter) + (N)*sizeof(Fts5Iter)) + /* ** The two input arrays - a1[] and a2[] - are in sorted order. This function ** merges the two arrays together and writes the result to output array @@ -6642,7 +6655,7 @@ static void fts5SetupPrefixIter( && p->pConfig->bPrefixInsttoken ){ s.pTokendata = &s2; - s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, sizeof(*s2.pT)); + s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, SZ_FTS5TOKENDATAITER(1)); } if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ @@ -6770,15 +6783,17 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){ ** and the initial version of the "averages" record (a zero-byte blob). */ int sqlite3Fts5IndexReinit(Fts5Index *p){ - Fts5Structure s; + Fts5Structure *pTmp; + u8 tmpSpace[SZ_FTS5STRUCTURE(1)]; fts5StructureInvalidate(p); fts5IndexDiscardData(p); - memset(&s, 0, sizeof(Fts5Structure)); + pTmp = (Fts5Structure*)tmpSpace; + memset(pTmp, 0, SZ_FTS5STRUCTURE(1)); if( p->pConfig->bContentlessDelete ){ - s.nOriginCntr = 1; + pTmp->nOriginCntr = 1; } fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); - fts5StructureWrite(p, &s); + fts5StructureWrite(p, pTmp); return fts5IndexReturn(p); } @@ -6986,7 +7001,7 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( if( p->rc==SQLITE_OK ){ if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; - int nByte = nAlloc * sizeof(Fts5Iter*) + sizeof(Fts5TokenDataIter); + int nByte = SZ_FTS5TOKENDATAITER(nAlloc+1); Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); if( pNew==0 ){ @@ -7502,7 +7517,8 @@ static int fts5SetupPrefixIterTokendata( fts5BufferGrow(&p->rc, &token, nToken+1); assert( token.p!=0 || p->rc!=SQLITE_OK ); - ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*ctx.pT)); + ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, + SZ_FTS5TOKENDATAITER(1)); if( p->rc==SQLITE_OK ){ @@ -7633,7 +7649,8 @@ int sqlite3Fts5IndexIterWriteTokendata( if( pIter->nSeg>0 ){ /* This is a prefix term iterator. */ if( pT==0 ){ - pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*pT)); + pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, + SZ_FTS5TOKENDATAITER(1)); pIter->pTokenDataIter = pT; } if( pT ){ diff --git a/manifest b/manifest index 2d0615da6d..46dafc1357 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sthe\sFts5Sorter.aIdx\sfield\sto\sa\sflexible\sarray. -D 2025-03-15T00:11:22.914 +C Use\sflexible\sarrays\swhereever\sappropriate\sin\sFTS5. +D 2025-03-15T12:22:39.392 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -110,9 +110,9 @@ F ext/fts5/fts5Int.h bffbd0acdcdf509899681f4e1cfeef1c955030acd9fe15ff9082410f80c F ext/fts5/fts5_aux.c da4a7a9a11ec15c6df0699d908915a209bcde48f0b04101461316b59f71abffb F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d7c53066c7 F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 -F ext/fts5/fts5_expr.c 390502c9954e20a9b6fcc31e4dc699df78a4ad52fd64e5b8aef0bfb82aa4a40d +F ext/fts5/fts5_expr.c 887a611b34094c828ff5fb19bbc50a6b1bbfd28791db01b0c8bf722e3c9f437a F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 -F ext/fts5/fts5_index.c 2f35dd8408946f0e0bfea8f3bfbe8dfaafe90a5345885b43d678546c19266673 +F ext/fts5/fts5_index.c d171f2a507abccb3d524bf461b01f0d3971a9bf221be622ac7c671a991cb62ee F ext/fts5/fts5_main.c 57933c18efe1058d8871199875c7a59744dabc3904f3aefbf9ff4a4e11fc79e2 F ext/fts5/fts5_storage.c 1ad05dab4830a4e2eaf2900bb143477f93bc17437093582f36f4b818809e88d8 F ext/fts5/fts5_tcl.c 7fb5a3d3404099075aaa2457307cb459bbc257c0de3dbd52b1e80a5b503e0329 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0c4d9c74741794468adc444908f6024f016738aa2852d3a646f2c28d079d9446 -R 36e1481e3adfdafd4b3b226a43a9ad79 +P 28ac776a23da2753265a7fe2ee2ebb09964815fc9058e69c08275fc217842edc +R c3399a4ca4e26a3efb3d2d24d674f2fd U drh -Z 97caa33dc844f08a198788bd53346fc8 +Z 06abea0099d4abff8fb7955f85c850ef # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 103c672440..855bbdd989 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28ac776a23da2753265a7fe2ee2ebb09964815fc9058e69c08275fc217842edc +16dfc415b6e98a2acae79a24bb0afd401e60efc27cbdd1603a426fd33e17d427 From 0a4f10e6e269c268207459fa68790c9e542c44f3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 15 Mar 2025 13:04:16 +0000 Subject: [PATCH 18/38] Use flexible arrays in the recovery extension and in the fuzzcheck test program. Adjust the unix makefile to use -fsanitize=bounds-strict when building fuzzcheck-asan. FossilOrigin-Name: 6ea6a6b211fed1a14d7bec1ab1790dec09e2a00423860498a60b760c4a4561fa --- ext/recover/sqlite3recover.c | 7 +++++-- main.mk | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fuzzcheck.c | 17 +++++++++++------ 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index b13719083d..7eedae6699 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -140,9 +140,12 @@ struct RecoverColumn { typedef struct RecoverBitmap RecoverBitmap; struct RecoverBitmap { i64 nPg; /* Size of bitmap */ - u32 aElem[1]; /* Array of 32-bit bitmasks */ + u32 aElem[]; /* Array of 32-bit bitmasks */ }; +/* Size in bytes of a RecoverBitmap object sufficient to cover 32 pages */ +#define SZ_RECOVERBITMAP_32 (16) + /* ** State variables (part of the sqlite3_recover structure) used while ** recovering data for tables identified in the recovered schema (state @@ -382,7 +385,7 @@ static int recoverError( */ static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ int nElem = (nPg+1+31) / 32; - int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32); + int nByte = SZ_RECOVERBITMAP_32 + nElem*sizeof(u32); RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte); if( pRet ){ diff --git a/main.mk b/main.mk index 2803e623a2..e3682a1a9e 100644 --- a/main.mk +++ b/main.mk @@ -2169,7 +2169,7 @@ fuzzy: fuzzcheck$(T.exe) xbin: fuzzcheck$(T.exe) fuzzcheck-asan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) - $(T.link) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ + $(T.link) -o $@ -fsanitize=address,bounds-strict $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ sqlite3.c $(LDFLAGS.libsqlite3) fuzzy: fuzzcheck-asan$(T.exe) xbin: fuzzcheck-asan$(T.exe) diff --git a/manifest b/manifest index 46dafc1357..ffc30b9622 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sflexible\sarrays\swhereever\sappropriate\sin\sFTS5. -D 2025-03-15T12:22:39.392 +C Use\sflexible\sarrays\sin\sthe\srecovery\sextension\sand\sin\sthe\sfuzzcheck\stest\sprogram.\nAdjust\sthe\sunix\smakefile\sto\suse\s-fsanitize=bounds-strict\swhen\sbuilding\nfuzzcheck-asan. +D 2025-03-15T13:04:16.138 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -523,7 +523,7 @@ F ext/recover/recoverpgsz.test 88766fcb810e52ee05335c456d4e5fb06d02b73d3ccb48c52 F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f F ext/recover/recoverslowidx.test c90d59c46bb8924a973ac6fbc38f3163cee38cc240256addcab1cf1a322c37dc F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 -F ext/recover/sqlite3recover.c 0ecdcb4df8967c84aa4dfe786816998bf2ef5cce55f4ac85ad4079e76f271027 +F ext/recover/sqlite3recover.c 098b622d34499625a3c87bc31abc237f8e9b992fa93ac5071f280bb0d90e7fd8 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 F ext/recover/test_recover.c 072260d7452a3b81aba995b2b3269e7ec2aa7f06725544ba4c25b1b0a1dbc61a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -705,7 +705,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk f2f6af216cf14ec010d317e2f75ed5dc2134a2f9d6be7df3a96ee11149598ca1 +F main.mk c7716a7f5559e9055519796fb27f257e4588bfae0f5c59e38b40f4e6ea51f5f0 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -1278,7 +1278,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 97eab1b916d576a0f734b921598bdac05ff04d1f15c494dbe40ca71a772c56bb +F test/fuzzcheck.c 128789e46d14f2a3ab452fb341165897ff471c4bbef09f8e113c80839acf8941 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 28ac776a23da2753265a7fe2ee2ebb09964815fc9058e69c08275fc217842edc -R c3399a4ca4e26a3efb3d2d24d674f2fd +P 16dfc415b6e98a2acae79a24bb0afd401e60efc27cbdd1603a426fd33e17d427 +R 16bdd7a30aeb17479eb9d6ab2f69c679 U drh -Z 06abea0099d4abff8fb7955f85c850ef +Z f00ea2f5fbf083c38b332f57e1d5d154 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 855bbdd989..d61ea27233 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16dfc415b6e98a2acae79a24bb0afd401e60efc27cbdd1603a426fd33e17d427 +6ea6a6b211fed1a14d7bec1ab1790dec09e2a00423860498a60b760c4a4561fa diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 140ad6944d..f43057a16e 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -129,9 +129,12 @@ struct Blob { int id; /* Id of this Blob */ int seq; /* Sequence number */ int sz; /* Size of this Blob in bytes */ - unsigned char a[1]; /* Blob content. Extra space allocated as needed. */ + unsigned char a[]; /* Blob content. Extra space allocated as needed. */ }; +/* Size in bytes of a Blob object sufficient to store N byte of content */ +#define SZ_BLOB(N) (offsetof(Blob,a) + (((N)+7)&~7)) + /* ** Maximum number of files in the in-memory virtual filesystem. */ @@ -512,13 +515,15 @@ static void blobListLoadFromDb( int *pN, /* OUT: Write number of blobs loaded here */ Blob **ppList /* OUT: Write the head of the blob list here */ ){ - Blob head; + Blob *head; Blob *p; sqlite3_stmt *pStmt; int n = 0; int rc; char *z2; + unsigned char tmp[SZ_BLOB(8)]; + head = (Blob*)tmp; if( firstId>0 ){ z2 = sqlite3_mprintf("%s WHERE rowid BETWEEN %d AND %d", zSql, firstId, lastId); @@ -528,11 +533,11 @@ static void blobListLoadFromDb( rc = sqlite3_prepare_v2(db, z2, -1, &pStmt, 0); sqlite3_free(z2); if( rc ) fatalError("%s", sqlite3_errmsg(db)); - head.pNext = 0; - p = &head; + head->pNext = 0; + p = head; while( SQLITE_ROW==sqlite3_step(pStmt) ){ int sz = sqlite3_column_bytes(pStmt, 1); - Blob *pNew = safe_realloc(0, sizeof(*pNew)+sz ); + Blob *pNew = safe_realloc(0, SZ_BLOB(sz+1)); pNew->id = sqlite3_column_int(pStmt, 0); pNew->sz = sz; pNew->seq = n++; @@ -544,7 +549,7 @@ static void blobListLoadFromDb( } sqlite3_finalize(pStmt); *pN = n; - *ppList = head.pNext; + *ppList = head->pNext; } /* From d389fd369901aa1384afcd64fa93321611d1a5fc Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 15 Mar 2025 13:11:24 +0000 Subject: [PATCH 19/38] Omit the -fsanitize=bounds-strict for now, as that is still not widely implemented. In particular, it does not work on Macs. FossilOrigin-Name: 3e1c2ac7817e73ea736a39bb0c0ec8212ceedbc89b265b4caf1b53871d27d7c0 --- main.mk | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/main.mk b/main.mk index e3682a1a9e..2803e623a2 100644 --- a/main.mk +++ b/main.mk @@ -2169,7 +2169,7 @@ fuzzy: fuzzcheck$(T.exe) xbin: fuzzcheck$(T.exe) fuzzcheck-asan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) - $(T.link) -o $@ -fsanitize=address,bounds-strict $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ + $(T.link) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ sqlite3.c $(LDFLAGS.libsqlite3) fuzzy: fuzzcheck-asan$(T.exe) xbin: fuzzcheck-asan$(T.exe) diff --git a/manifest b/manifest index ffc30b9622..edc538e5b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sflexible\sarrays\sin\sthe\srecovery\sextension\sand\sin\sthe\sfuzzcheck\stest\sprogram.\nAdjust\sthe\sunix\smakefile\sto\suse\s-fsanitize=bounds-strict\swhen\sbuilding\nfuzzcheck-asan. -D 2025-03-15T13:04:16.138 +C Omit\sthe\s-fsanitize=bounds-strict\sfor\snow,\sas\sthat\sis\sstill\snot\swidely\nimplemented.\s\sIn\sparticular,\sit\sdoes\snot\swork\son\sMacs. +D 2025-03-15T13:11:24.810 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -705,7 +705,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk c7716a7f5559e9055519796fb27f257e4588bfae0f5c59e38b40f4e6ea51f5f0 +F main.mk f2f6af216cf14ec010d317e2f75ed5dc2134a2f9d6be7df3a96ee11149598ca1 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 16dfc415b6e98a2acae79a24bb0afd401e60efc27cbdd1603a426fd33e17d427 -R 16bdd7a30aeb17479eb9d6ab2f69c679 +P 6ea6a6b211fed1a14d7bec1ab1790dec09e2a00423860498a60b760c4a4561fa +R 230925c95f57711ebe48c1e2c164b78b U drh -Z f00ea2f5fbf083c38b332f57e1d5d154 +Z 2c119b268b3dca0dc5ff030ca58b304e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d61ea27233..6a992e7ba7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ea6a6b211fed1a14d7bec1ab1790dec09e2a00423860498a60b760c4a4561fa +3e1c2ac7817e73ea736a39bb0c0ec8212ceedbc89b265b4caf1b53871d27d7c0 From f792cda1a1f042673a749c756eaab175244333fa Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 15 Mar 2025 13:36:01 +0000 Subject: [PATCH 20/38] For fuzzcheck-asan, dynamically determine the list of -fsanitize flags to use based on configure-time feature tests. FossilOrigin-Name: b70f9cc81516e57e73960bed4b4d2abdcf3dab0ad4a400ca1aed49365c25231e --- Makefile.in | 3 +++ auto.def | 2 ++ autosetup/proj.tcl | 23 +++++++++++++++++++++++ main.mk | 5 ++++- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/Makefile.in b/Makefile.in index 2abf714e5f..45e2cb086d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -135,6 +135,9 @@ ENABLE_LIB_STATIC = @ENABLE_LIB_STATIC@ HAVE_WASI_SDK = @HAVE_WASI_SDK@ libsqlite3.DLL.install-rules = @SQLITE_DLL_INSTALL_RULES@ +# -fsanitize flags for the fuzzcheck-asap app +LDFLAGS.fuzzcheck.fsanitize = @LDFLAGS_FUZZCHECK_FSANITIZE@ + T.cc.sqlite = $(T.cc) @TARGET_DEBUG@ # diff --git a/auto.def b/auto.def index c4fb2c5ab5..905237d65c 100644 --- a/auto.def +++ b/auto.def @@ -46,6 +46,8 @@ sqlite-configure canonical { define LINK_TOOLS_DYNAMICALLY [proj-opt-was-provided dynlink-tools] + define LDFLAGS_FUZZCHECK_FSANITIZE [proj-check-fsanitize {address bounds-strict}] + sqlite-handle-tcl sqlite-handle-emsdk diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index adf31a1ad6..dc5f9a092b 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1031,6 +1031,29 @@ proc proj-check-soname {{libname "libfoo.so.0"}} { } } +######################################################################## +# @proj-check-fsanitize ?list-of-opts? +# +# Checks whether CC supports -fsanitize=X, where X is each entry of +# the given list of flags. If any of those flags are supported, it +# returns the string "-fsanitize=X..." where X... is a comma-separated +# list of all flags which passed. If none of the given options are +# supported then it returns an empty string. +proc proj-check-fsanitize {{opts {address bounds-strict}}} { + set sup {} + foreach opt $opts { + cc-with {-link 1} { + if {[cc-check-flags "-fsanitize=$opt"]} { + lappend sup $opt + } + } + } + if {[llength $sup] > 0} { + return "-fsanitize=[join $sup ,]" + } + return "" +} + ######################################################################## # Internal helper for proj-dump-defs-json. Expects to be passed a # [define] name and the variadic $args which are passed to diff --git a/main.mk b/main.mk index 2803e623a2..bf74817d9a 100644 --- a/main.mk +++ b/main.mk @@ -2168,8 +2168,11 @@ fuzzcheck$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) fuzzy: fuzzcheck$(T.exe) xbin: fuzzcheck$(T.exe) +# -fsanitize=... flags for fuzzcheck-asan. +LDFLAGS.fuzzcheck.fsanitize ?= -fsanitize=address + fuzzcheck-asan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) - $(T.link) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ + $(T.link) -o $@ $(LDFLAGS.fuzzcheck.fsanitize) $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ sqlite3.c $(LDFLAGS.libsqlite3) fuzzy: fuzzcheck-asan$(T.exe) xbin: fuzzcheck-asan$(T.exe) diff --git a/manifest b/manifest index edc538e5b3..5162836083 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Omit\sthe\s-fsanitize=bounds-strict\sfor\snow,\sas\sthat\sis\sstill\snot\swidely\nimplemented.\s\sIn\sparticular,\sit\sdoes\snot\swork\son\sMacs. -D 2025-03-15T13:11:24.810 +C For\sfuzzcheck-asan,\sdynamically\sdetermine\sthe\slist\sof\s-fsanitize\sflags\sto\suse\sbased\son\sconfigure-time\sfeature\stests. +D 2025-03-15T13:36:01.334 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d -F Makefile.in 88f74a1b9fcd903fe3414fe9f8484f8491dc403615dbf1c28c6f415f5220b8b2 +F Makefile.in 39c3f70ffacbe3e0e0d7b13e73d2c7bd5e402d326413d91f75c8ddf466c6ae1e F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 F Makefile.msc ef04498c7e227a0f459b105bb4952f26cc985d1d6340a367e62d5a79c4689dfb F README.md a953c0cffd6e4f2501a306c00ee2b6e1e6630c25031e094629307fe99dd003d1 @@ -14,7 +14,7 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 -F auto.def 0612f87776956cff7ba1585ad3ca7ab7d2e88735da0e9b4321dbacb05479cb94 +F auto.def c26f6784f6320862e858528636746e4995276f81ba3c1f915f7f682ec2404de9 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac F autoconf/Makefile.in 6c98c82f52aa27a5c586080cf7c61c811174c2b6d8b8de33fd657d78d541dd7d F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2038dcd8b @@ -49,7 +49,7 @@ F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1d F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl e69b91f814ea510057ce7663845de703c3746d71cff9a0db6b2563ee3e7fd25e +F autosetup/proj.tcl 408ba69a0b13df8bbedaa565897c3304aa3a614fdd1dc11cc1151da1ea0fde8c F autosetup/sqlite-config.tcl 831985320d98002fcd5ea064cae8a49f8afcd9685d83178ef1ebb79189b5045c F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x @@ -705,7 +705,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk f2f6af216cf14ec010d317e2f75ed5dc2134a2f9d6be7df3a96ee11149598ca1 +F main.mk e79f9c0ce4d2dd85e4d07ece408525ecabf8cb8ea0cc080fe0ad21416b727b04 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6ea6a6b211fed1a14d7bec1ab1790dec09e2a00423860498a60b760c4a4561fa -R 230925c95f57711ebe48c1e2c164b78b -U drh -Z 2c119b268b3dca0dc5ff030ca58b304e +P 3e1c2ac7817e73ea736a39bb0c0ec8212ceedbc89b265b4caf1b53871d27d7c0 +R ea1d897400ea4fcd1d782c3f3f3eb197 +U stephan +Z 3c46d5c6b2fecf10bb4cea80942c2207 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6a992e7ba7..5d565accb6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e1c2ac7817e73ea736a39bb0c0ec8212ceedbc89b265b4caf1b53871d27d7c0 +b70f9cc81516e57e73960bed4b4d2abdcf3dab0ad4a400ca1aed49365c25231e From 8c0e9227219a6276de19b8944a3d41e37994aa0d Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 15 Mar 2025 13:50:07 +0000 Subject: [PATCH 21/38] -fsanitize is a CFLAG, not LDFLAG, so rename some vars accordingly and simplify the feature check to not run the linker. FossilOrigin-Name: 44f2c64ec16f4720dc538be30410863c4138ea4ce41c94521bd7980535261735 --- Makefile.in | 2 +- auto.def | 2 +- autosetup/proj.tcl | 4 ++-- main.mk | 4 ++-- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Makefile.in b/Makefile.in index 45e2cb086d..91135edba3 100644 --- a/Makefile.in +++ b/Makefile.in @@ -136,7 +136,7 @@ HAVE_WASI_SDK = @HAVE_WASI_SDK@ libsqlite3.DLL.install-rules = @SQLITE_DLL_INSTALL_RULES@ # -fsanitize flags for the fuzzcheck-asap app -LDFLAGS.fuzzcheck.fsanitize = @LDFLAGS_FUZZCHECK_FSANITIZE@ +CFLAGS.fuzzcheck.fsanitize = @CFLAGS_FUZZCHECK_FSANITIZE@ T.cc.sqlite = $(T.cc) @TARGET_DEBUG@ diff --git a/auto.def b/auto.def index 905237d65c..38850dfa4d 100644 --- a/auto.def +++ b/auto.def @@ -46,7 +46,7 @@ sqlite-configure canonical { define LINK_TOOLS_DYNAMICALLY [proj-opt-was-provided dynlink-tools] - define LDFLAGS_FUZZCHECK_FSANITIZE [proj-check-fsanitize {address bounds-strict}] + define CFLAGS_FUZZCHECK_FSANITIZE [proj-check-fsanitize {address bounds-strict}] sqlite-handle-tcl sqlite-handle-emsdk diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index dc5f9a092b..4d4719d615 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1037,12 +1037,12 @@ proc proj-check-soname {{libname "libfoo.so.0"}} { # Checks whether CC supports -fsanitize=X, where X is each entry of # the given list of flags. If any of those flags are supported, it # returns the string "-fsanitize=X..." where X... is a comma-separated -# list of all flags which passed. If none of the given options are +# list of all supported flags. If none of the given options are # supported then it returns an empty string. proc proj-check-fsanitize {{opts {address bounds-strict}}} { set sup {} foreach opt $opts { - cc-with {-link 1} { + cc-with {} { if {[cc-check-flags "-fsanitize=$opt"]} { lappend sup $opt } diff --git a/main.mk b/main.mk index bf74817d9a..4ca92d4477 100644 --- a/main.mk +++ b/main.mk @@ -2169,10 +2169,10 @@ fuzzy: fuzzcheck$(T.exe) xbin: fuzzcheck$(T.exe) # -fsanitize=... flags for fuzzcheck-asan. -LDFLAGS.fuzzcheck.fsanitize ?= -fsanitize=address +CFLAGS.fuzzcheck.fsanitize ?= -fsanitize=address fuzzcheck-asan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) - $(T.link) -o $@ $(LDFLAGS.fuzzcheck.fsanitize) $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ + $(T.link) -o $@ $(CFLAGS.fuzzcheck.fsanitize) $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ sqlite3.c $(LDFLAGS.libsqlite3) fuzzy: fuzzcheck-asan$(T.exe) xbin: fuzzcheck-asan$(T.exe) diff --git a/manifest b/manifest index 5162836083..9c009aa5ef 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C For\sfuzzcheck-asan,\sdynamically\sdetermine\sthe\slist\sof\s-fsanitize\sflags\sto\suse\sbased\son\sconfigure-time\sfeature\stests. -D 2025-03-15T13:36:01.334 +C -fsanitize\sis\sa\sCFLAG,\snot\sLDFLAG,\sso\srename\ssome\svars\saccordingly\sand\ssimplify\sthe\sfeature\scheck\sto\snot\srun\sthe\slinker. +D 2025-03-15T13:50:07.673 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d -F Makefile.in 39c3f70ffacbe3e0e0d7b13e73d2c7bd5e402d326413d91f75c8ddf466c6ae1e +F Makefile.in 2788f5a3c36e26817707003170871936d44f46c5b45f8cbefe07c9f1a51a8988 F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 F Makefile.msc ef04498c7e227a0f459b105bb4952f26cc985d1d6340a367e62d5a79c4689dfb F README.md a953c0cffd6e4f2501a306c00ee2b6e1e6630c25031e094629307fe99dd003d1 @@ -14,7 +14,7 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 -F auto.def c26f6784f6320862e858528636746e4995276f81ba3c1f915f7f682ec2404de9 +F auto.def 3eae53b31c798490a4af13ab8f14948f176c742f4778cc58d8b1f6d6d2e01a62 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac F autoconf/Makefile.in 6c98c82f52aa27a5c586080cf7c61c811174c2b6d8b8de33fd657d78d541dd7d F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2038dcd8b @@ -49,7 +49,7 @@ F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1d F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 408ba69a0b13df8bbedaa565897c3304aa3a614fdd1dc11cc1151da1ea0fde8c +F autosetup/proj.tcl 409dd44d07b2bc9ee98778619f6c0d0123e79f16a0dc726463e037d10f45745e F autosetup/sqlite-config.tcl 831985320d98002fcd5ea064cae8a49f8afcd9685d83178ef1ebb79189b5045c F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x @@ -705,7 +705,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk e79f9c0ce4d2dd85e4d07ece408525ecabf8cb8ea0cc080fe0ad21416b727b04 +F main.mk 09d801f71ac6e84dd6f6afda009b76c2a712cb02edac50c4bcfdae1027dfb4da F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3e1c2ac7817e73ea736a39bb0c0ec8212ceedbc89b265b4caf1b53871d27d7c0 -R ea1d897400ea4fcd1d782c3f3f3eb197 +P b70f9cc81516e57e73960bed4b4d2abdcf3dab0ad4a400ca1aed49365c25231e +R 5d61542419e2a8828e0b0a6d6bc8149b U stephan -Z 3c46d5c6b2fecf10bb4cea80942c2207 +Z b02345883d043cd5c1744f64864d3542 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5d565accb6..3f6c9a6fd4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b70f9cc81516e57e73960bed4b4d2abdcf3dab0ad4a400ca1aed49365c25231e +44f2c64ec16f4720dc538be30410863c4138ea4ce41c94521bd7980535261735 From 9f8a238fb5ab0e292931492e949665c1a104c838 Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 15 Mar 2025 15:19:42 +0000 Subject: [PATCH 22/38] Configure-internal build cleanups (no functional changes). Add EXTRA_SRC to the deps of sqlite3.c. FossilOrigin-Name: 8afb8bbce8654d6f76207fb136e79dc52b6724a71eae82a4c098690a68eb75a1 --- auto.def | 2 +- autosetup/proj.tcl | 2 ++ autosetup/sqlite-config.tcl | 67 ++++++++++++++++++------------------- main.mk | 2 +- manifest | 18 +++++----- manifest.uuid | 2 +- 6 files changed, 47 insertions(+), 46 deletions(-) diff --git a/auto.def b/auto.def index c4fb2c5ab5..d7f46392eb 100644 --- a/auto.def +++ b/auto.def @@ -25,7 +25,7 @@ sqlite-configure canonical { # -------------^^^^^^^ intentionally using [get-env] instead of # [proj-get-env] here because [sqlite-setup-default-cflags] uses # [proj-get-env] and we want this to supercede that. - sqlite-munge-cflags + sqlite-munge-cflags; # straighten out -DSQLITE_ENABLE/OMIT flags } sqlite-check-common-bins ;# must come before [sqlite-handle-wasi-sdk] sqlite-handle-wasi-sdk ;# must run relatively early, as it changes the environment diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index adf31a1ad6..aef8aed1f0 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -183,6 +183,8 @@ proc proj-lshift_ {listVar {count 1}} { } ######################################################################## +# @proj-strip-hash-comments value +# # Expects to receive string input, which it splits on newlines, strips # out any lines which begin with any number of whitespace followed by # a '#', and returns a value containing the [append]ed results of each diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 56567ddb96..3644b4872a 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -61,13 +61,21 @@ array set sqliteConfig [proj-strip-hash-comments { set sqliteConfig(is-cross-compiling) [proj-is-cross-compiling] ######################################################################## -# Processes all configure --flags for this build $buildMode must be -# either "canonical" or "autoconf", and others may be added in the +# Processes all configure --flags for this build, run build-specific +# config checks, then finalize the configure process. $buildMode must +# be either "canonical" or "autoconf", and others may be added in the # future. After bootstrapping, $configScript is eval'd in the caller's # scope, then post-configuration finalization is run. $configScript is # intended to hold configure code which is specific to the given # $buildMode, with the caveat that _some_ build-specific code is # encapsulated in the configuration finalization step. +# +# The intent is that all build-mode-specific configuration goes inside +# the $configScript argument to this function, and that an auto.def file +# contains only two commands: +# +# use sqlite-config +# sqlite-configure BUILD_NAME { build-specific configure script } proc sqlite-configure {buildMode configScript} { set allBuildModes {canonical autoconf} if {$buildMode ni $allBuildModes} { @@ -131,7 +139,7 @@ proc sqlite-configure {buildMode configScript} { ######################################################################## set allFlags { # Structure: a list of M {Z} pairs, where M is a descriptive - # option group name and Z is a list of X Y pairs. X is a list of + # option group name and Z is a list of X Y pairs. X is a list of # $buildMode name(s) to which the Y flags apply, or {*} to apply # to all builds. Y is a {block} in the form expected by # autosetup's [options] command. Each block which is applicable @@ -168,8 +176,8 @@ proc sqlite-configure {buildMode configScript} { largefile=1 => {This legacy flag has no effect on the library but may influence the contents of the generated sqlite_cfg.h} - # ^^^ It's not clear that this actually does anything, as - # HAVE_LFS is not checked anywhere in the .c/.h/.in files. + # ^^^ It's not clear that LFS support actually does anything, + # as HAVE_LFS is not checked anywhere in the .c/.h/.in files. load-extension=1 => {Disable loading of external extensions} math=1 => {Disable math functions} json=1 => {Disable JSON functions} @@ -365,12 +373,13 @@ proc sqlite-configure {buildMode configScript} { }; # sqlite-configure ######################################################################## -# Performs late-stage config steps common to both the canonical and -# autoconf bundle builds. +# Performs late-stage config steps common to all supported +# $::sqliteConfig(build-mode) values. proc sqlite-configure-finalize {} { set buildMode $::sqliteConfig(build-mode) set isCanonical [expr {$buildMode eq "canonical"}] set isAutoconf [expr {$buildMode eq "autoconf"}] + proj-assert {$isCanonical || $isAutoconf} "Unknown build mode: $buildMode" define HAVE_LFS 0 if {[opt-bool largefile]} { @@ -417,6 +426,13 @@ proc sqlite-configure-finalize {} { sqlite-handle-math sqlite-handle-icu sqlite-handle-env-quirks + sqlite-handle-common-feature-flags + sqlite-finalize-feature-flags + ######################################################################## + # When cross-compiling, we have to avoid using the -s flag to + # /usr/bin/install: + # https://sqlite.org/forum/forumpost/9a67df63eda9925c + define IS_CROSS_COMPILING $::sqliteConfig(is-cross-compiling) sqlite-process-dot-in-files sqlite-post-config-validation sqlite-dump-defines @@ -433,12 +449,10 @@ proc sqlite-post-options-init {} { define PACKAGE_URL {https://sqlite.org} define PACKAGE_BUGREPORT [get-define PACKAGE_URL]/forum define PACKAGE_STRING "[get-define PACKAGE_NAME] [get-define PACKAGE_VERSION]" - # - # Carry values from hidden --flag aliases over to their canonical - # flag forms. This list must include only options which are common - # to both the top-level auto.def and autoconf/auto.def. - # proj-xfer-options-aliases { + # Carry values from hidden --flag aliases over to their canonical + # flag forms. This list must include only options which are common + # to all build modes supported by [sqlite-configure]. with-readline-inc => with-readline-cflags with-readline-lib => with-readline-ldflags with-debug => debug @@ -468,11 +482,9 @@ proc msg-debug {msg} { ######################################################################## # Sets up the SQLITE_AUTORECONFIG define. proc sqlite-autoreconfig {} { - # # SQLITE_AUTORECONFIG contains make target rules for re-running the # configure script with the same arguments it was initially invoked - # with. This can be used to automatically reconfigure - # + # with. This can be used to automatically reconfigure. set squote {{arg} { # Wrap $arg in single-quotes if it looks like it might need that # to avoid mis-handling as a shell argument. We assume that $arg @@ -507,6 +519,8 @@ proc sqlite-add-feature-flag {args} { define-append OPT_FEATURE_FLAGS {*}$args } } + +######################################################################## # Appends $args, if not empty, to OPT_SHELL. proc sqlite-add-shell-opt {args} { if {"" ne $args} { @@ -545,13 +559,11 @@ proc sqlite-check-common-bins {} { # Run checks for system-level includes and libs which are common to # both the canonical build and the "autoconf" bundle. proc sqlite-check-common-system-deps {} { - # # Check for needed/wanted data types cc-with {-includes stdint.h} \ {cc-check-types int8_t int16_t int32_t int64_t intptr_t \ uint8_t uint16_t uint32_t uint64_t uintptr_t} - # # Check for needed/wanted functions cc-check-functions gmtime_r isnan localtime_r localtime_s \ malloc_usable_size strchrnul usleep utime pread pread64 pwrite pwrite64 @@ -566,7 +578,6 @@ proc sqlite-check-common-system-deps {} { } define LDFLAGS_RT [join [lsort -unique $ldrt] ""] - # # Check for needed/wanted headers cc-check-includes \ sys/types.h sys/stat.h dlfcn.h unistd.h \ @@ -588,7 +599,6 @@ proc sqlite-check-common-system-deps {} { ######################################################################## # Move -DSQLITE_OMIT... and -DSQLITE_ENABLE... flags from CFLAGS and # CPPFLAGS to OPT_FEATURE_FLAGS and remove them from BUILD_CFLAGS. -# This is derived from the legacy build but is still practical. proc sqlite-munge-cflags {} { # Move CFLAGS and CPPFLAGS entries matching -DSQLITE_OMIT* and # -DSQLITE_ENABLE* to OPT_FEATURE_FLAGS. This behavior is derived @@ -735,6 +745,10 @@ proc sqlite-finalize-feature-flags {} { define OPT_SHELL [lsort -unique $oFF] msg-result "Shell options: [get-define OPT_SHELL]" } + if {"" ne [set extraSrc [get-define AMALGAMATION_EXTRA_SRC ""]]} { + proj-assert {"canonical" eq $::sqliteConfig(build-mode)} + msg-result "Appending source files to amalgamation: $extraSrc" + } } ######################################################################## @@ -1557,21 +1571,6 @@ proc sqlite-handle-env-quirks {} { # Perform some late-stage work and generate the configure-process # output file(s). proc sqlite-process-dot-in-files {} { - ######################################################################## - # When cross-compiling, we have to avoid using the -s flag to - # /usr/bin/install: - # https://sqlite.org/forum/forumpost/9a67df63eda9925c - define IS_CROSS_COMPILING $::sqliteConfig(is-cross-compiling) - - # Finish up handling of the various feature flags here because it's - # convenient for both the canonical build and autoconf bundles that - # it be done here. - sqlite-handle-common-feature-flags - sqlite-finalize-feature-flags - if {"" ne [set extraSrc [get-define AMALGAMATION_EXTRA_SRC ""]]} { - msg-result "Appending source files to amalgamation: $extraSrc" - } - ######################################################################## # "Re-export" the autoconf-conventional --XYZdir flags into something # which is more easily overridable from a make invocation. See the docs diff --git a/main.mk b/main.mk index 2803e623a2..439ec7f48f 100644 --- a/main.mk +++ b/main.mk @@ -1107,7 +1107,7 @@ sqlite3.h: $(MAKE_SANITY_CHECK) $(TOP)/src/sqlite.h.in \ $(B.tclsh) $(TOP)/tool/mksqlite3h.tcl $(TOP) -o sqlite3.h sqlite3.c: .target_source sqlite3.h $(TOP)/tool/mksqlite3c.tcl src-verify$(B.exe) \ - $(B.tclsh) + $(B.tclsh) $(EXTRA_SRC) $(B.tclsh) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_GEN_FLAGS) $(EXTRA_SRC) cp tsrc/sqlite3ext.h . cp $(TOP)/ext/session/sqlite3session.h . diff --git a/manifest b/manifest index eb3d849e1e..3b99e06a1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sinternal\sdoc\stypo\sreported\sin\s[forum:e25e581f917|forum\spost\se25e581f917]. -D 2025-03-14T12:37:36.487 +C Configure-internal\sbuild\scleanups\s(no\sfunctional\schanges).\sAdd\sEXTRA_SRC\sto\sthe\sdeps\sof\ssqlite3.c. +D 2025-03-15T15:19:42.945 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -14,7 +14,7 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 -F auto.def 0612f87776956cff7ba1585ad3ca7ab7d2e88735da0e9b4321dbacb05479cb94 +F auto.def 16e64dacd2c556b67df0c33fa08d564224bdb3137756679f39c45ed389b5a07b F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac F autoconf/Makefile.in 6c98c82f52aa27a5c586080cf7c61c811174c2b6d8b8de33fd657d78d541dd7d F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2038dcd8b @@ -49,8 +49,8 @@ F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1d F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl e69b91f814ea510057ce7663845de703c3746d71cff9a0db6b2563ee3e7fd25e -F autosetup/sqlite-config.tcl 831985320d98002fcd5ea064cae8a49f8afcd9685d83178ef1ebb79189b5045c +F autosetup/proj.tcl f8b5402faf3ee3fe9b10b946a085fc826184d26df1c17b5380e90edf89ea5fb6 +F autosetup/sqlite-config.tcl a7f4d093d63bc1da9ec3d44f392f377ce4c86aa7e48532ae51619e55558f5fbe F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 @@ -705,7 +705,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk f2f6af216cf14ec010d317e2f75ed5dc2134a2f9d6be7df3a96ee11149598ca1 +F main.mk 88af9562d8f4d921e37ffa4d18f59d6d3749f357c7b876393d37298ec78f8110 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f786de8d1873cd27b1bf83273a1e100e9d481144674888ccf65974e003a3caad -R d7baa5b59cd39b651d61c163fdc55c15 +P fa6f6ccdffc50024624306900efd2538c7415d8bdd0f02835b2e9c05adab3cf1 +R fa3f42bd795b87d6c693892aa92ef559 U stephan -Z bd2d36d9f3639b6b3b36c30d817df77d +Z 60eb85208863d91adaf5e3aa8394c9ef # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 38ff83c8db..a3c2630055 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa6f6ccdffc50024624306900efd2538c7415d8bdd0f02835b2e9c05adab3cf1 +8afb8bbce8654d6f76207fb136e79dc52b6724a71eae82a4c098690a68eb75a1 From f212fb3362447fe38dd0dbe23c0da000d014c8f7 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 15 Mar 2025 16:58:39 +0000 Subject: [PATCH 23/38] Speed up parsing of very long fts3 query expressions. FossilOrigin-Name: 2dd5b6895a3b23c2b9cbf0c1c1e802faf8f2b41ef60819eea25d609755266e64 --- ext/fts3/fts3_expr.c | 47 +++++++++++++++++++++++++++++++++----------- manifest | 14 ++++++------- manifest.uuid | 2 +- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index ca857835e2..1372cd933b 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -161,6 +161,23 @@ int sqlite3Fts3OpenTokenizer( */ static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); +/* +** Search buffer z[], size n, for a '"' character. Or, if enable_parenthesis +** is defined, search for '(' and ')' as well. Return the index of the first +** such character in the buffer. If there is no such character, return -1. +*/ +static int findBarredChar(const char *z, int n){ + int ii; + for(ii=0; iiiLangid, z, i, &pCursor); + *pnConsumed = n; + rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor); if( rc==SQLITE_OK ){ const char *zToken; int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; @@ -202,6 +212,18 @@ static int getNextToken( rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); if( rc==SQLITE_OK ){ + + /* Check that this tokenization did not gobble up any " characters. Or, + ** if enable_parenthesis is true, that it did not gobble up any + ** open or close parenthesis characters either. If it did, call + ** getNextToken() again, but pass only that part of the input buffer + ** up to the first such character. */ + int iBarred = findBarredChar(z, iEnd); + if( iBarred>=0 ){ + pModule->xClose(pCursor); + return getNextToken(pParse, iCol, z, iBarred, ppExpr, pnConsumed); + } + nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; pRet = (Fts3Expr *)sqlite3Fts3MallocZero(nByte); if( !pRet ){ @@ -236,7 +258,11 @@ static int getNextToken( } *pnConsumed = iEnd; - }else if( i && rc==SQLITE_DONE ){ + }else if( n && rc==SQLITE_DONE ){ + int iBarred = findBarredChar(z, n); + if( iBarred>=0 ){ + *pnConsumed = iBarred; + } rc = SQLITE_OK; } @@ -1239,7 +1265,6 @@ static void fts3ExprTestCommon( } if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ - sqlite3Fts3ExprFree(pExpr); sqlite3_result_error(context, "Error parsing expression", -1); }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); diff --git a/manifest b/manifest index 3b99e06a1d..07cc260a22 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Configure-internal\sbuild\scleanups\s(no\sfunctional\schanges).\sAdd\sEXTRA_SRC\sto\sthe\sdeps\sof\ssqlite3.c. -D 2025-03-15T15:19:42.945 +C Speed\sup\sparsing\sof\svery\slong\sfts3\squery\sexpressions. +D 2025-03-15T16:58:39.639 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -82,7 +82,7 @@ F ext/fts3/fts3.c fd2a8642fa4701ef5dd6bce7947ecb3c7ae472e1d44022772454a8b74a1315 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 75b9cf37c93d3c56d8e569d64527c927cb54a5279afb3823740ca1e29e481c15 F ext/fts3/fts3_aux.c 7eab82a9cf0830f6551ba3abfdbe73ed39e322a4d3940ee82fbf723674ecd9f3 -F ext/fts3/fts3_expr.c 673bf600655f5080239ff0e1e80eaae0176389f7e7d3af54c6d51491280ca360 +F ext/fts3/fts3_expr.c b8ff0d3775f33eddae559b444df2000d48768d7d9bdb642e3f6434c9f2543ffc F ext/fts3/fts3_hash.c d9dba473741445789330c7513d4f65737c92df23c3212784312931641814672a F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c 305ce7fb6036484085b5556a9c8e62acdc7763f0f4cdf5fd538212a9f3720116 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fa6f6ccdffc50024624306900efd2538c7415d8bdd0f02835b2e9c05adab3cf1 -R fa3f42bd795b87d6c693892aa92ef559 -U stephan -Z 60eb85208863d91adaf5e3aa8394c9ef +P 8afb8bbce8654d6f76207fb136e79dc52b6724a71eae82a4c098690a68eb75a1 +R 1bd8f3cfa90570231383a500e0910fe3 +U dan +Z 1c83f58e6b1181c1bb790bbf7879f1cb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a3c2630055..a2320ead9e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8afb8bbce8654d6f76207fb136e79dc52b6724a71eae82a4c098690a68eb75a1 +2dd5b6895a3b23c2b9cbf0c1c1e802faf8f2b41ef60819eea25d609755266e64 From 7bd72d4abf8e4659966bf6e61b32f07ee800c34a Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 15 Mar 2025 18:26:27 +0000 Subject: [PATCH 24/38] Fix alignment problems on Linux with -m32 and on Mac PPC. FossilOrigin-Name: 8a91aeca60548d5cd19add128cf65b9c3815c9103b1ef8ff6bc02711b6d709de --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbeInt.h | 3 ++- src/whereInt.h | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9c009aa5ef..5f7bfb31fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C -fsanitize\sis\sa\sCFLAG,\snot\sLDFLAG,\sso\srename\ssome\svars\saccordingly\sand\ssimplify\sthe\sfeature\scheck\sto\snot\srun\sthe\slinker. -D 2025-03-15T13:50:07.673 +C Fix\salignment\sproblems\son\sLinux\swith\s-m32\sand\son\sMac\sPPC. +D 2025-03-15T18:26:27.012 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -852,7 +852,7 @@ F src/util.c 36fb1150062957280777655976f3f9a75db236cb8207a0770ceae8d5ec17fcd3 F src/vacuum.c fbfc3e074c865d2b5b10b8a65a3783275b80c152653590690747a102bb6cc770 F src/vdbe.c b5deed01000b3970cfca089dc531cf9342afd96d00cc8b4ad26d303f088116ee F src/vdbe.h 31eddcffc1d14c76c2a20fe4e137e1ee43d44f370896fae14a067052801a3625 -F src/vdbeInt.h 366843cb88740595011e091e180bb7258487ec9fd91c0b5a47cf9ff25620a482 +F src/vdbeInt.h 5446f60e89b2aa7cdf3ab0ec4e7b01b8732cd9d52d9092a0b8b1bf700768f784 F src/vdbeapi.c a9ad72afed9aaec2acdde4daa5caa2f342b298f8c8859704143f6e3b78cb9966 F src/vdbeaux.c 72a1c99d9300a5e0adff2c708074ac1355a619664301db474a48e147418fba05 F src/vdbeblob.c b1b4032cac46b41e44b957c4d00aee9851f862dfd85ecb68116ba49884b03dfd @@ -866,7 +866,7 @@ F src/wal.c 554a6b1afaaecb98cb47bb598bccf1374c9d3b624e5c4c3c4eb2ad364cc579f8 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014 F src/where.c e80177e452b4e436abc6ece0cb0249631000434f2a7425cc1df709015fce74ad -F src/whereInt.h 8373791e8dfbef3ca0d4ad7eab350be860b2740c44d94d0c54d2a8d36209c41a +F src/whereInt.h ecdbfb5551cf394f04ec7f0bc7ad963146d80eee3071405ac29aa84950128b8e F src/wherecode.c 0d3de258d7922aede028841c6e0060633c50be26737c92cc62ce8be280535430 F src/whereexpr.c 2415c8eee5ff89a8b709d7d83d71c1ff986cd720d0520057e1d8a5371339012a F src/window.c d01227141f622f24fbe36ca105fbe6ef023f9fd98f1ccd65da95f88886565db5 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b70f9cc81516e57e73960bed4b4d2abdcf3dab0ad4a400ca1aed49365c25231e -R 5d61542419e2a8828e0b0a6d6bc8149b -U stephan -Z b02345883d043cd5c1744f64864d3542 +P 44f2c64ec16f4720dc538be30410863c4138ea4ce41c94521bd7980535261735 +R 0aa0704287ff919c67b51206bc5ac75c +U drh +Z 83d96f08f2610b8988f70e4446915dd1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3f6c9a6fd4..d121f7294a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -44f2c64ec16f4720dc538be30410863c4138ea4ce41c94521bd7980535261735 +8a91aeca60548d5cd19add128cf65b9c3815c9103b1ef8ff6bc02711b6d709de diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 600d9b8bc5..13262cd4e2 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -143,7 +143,8 @@ struct VdbeCursor { ** or less. The value of SZ_VDBECURSOR(n) is guaranteed to be a multiple ** of 8. */ -#define SZ_VDBECURSOR(N) (offsetof(VdbeCursor,aType) + ((N)+1)*sizeof(u64)) +#define SZ_VDBECURSOR(N) \ + (ROUND8(offsetof(VdbeCursor,aType)) + ((N)+1)*sizeof(u64)) /* Return true if P is a null-only cursor */ diff --git a/src/whereInt.h b/src/whereInt.h index 1b0a02d8bc..40a720ab9e 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -507,7 +507,7 @@ struct WhereInfo { /* ** The size (in bytes) of a WhereInfo object that holds N WhereLevels. */ -#define SZ_WHEREINFO(N) (offsetof(WhereInfo,a)+(N)*sizeof(WhereLevel)) +#define SZ_WHEREINFO(N) ROUND8(offsetof(WhereInfo,a)+(N)*sizeof(WhereLevel)) /* ** Private interfaces - callable only by other where.c routines. From b7b060401ed28cdfd67c11b6b2e578bea6e22dba Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 15 Mar 2025 19:00:46 +0000 Subject: [PATCH 25/38] Work around compilers that do not understand flexible arrays, in the recovery extension and in the fuzzcheck test module. FossilOrigin-Name: f101c46cf83e532fd33034abccba496bf395ef10c161af003211614d6581d5eb --- ext/recover/sqlite3recover.c | 12 +++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fuzzcheck.c | 7 ++++++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 7eedae6699..d50aa25ea0 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -33,6 +33,16 @@ typedef unsigned int u32; typedef unsigned char u8; typedef sqlite3_int64 i64; +/* +** Work around C99 "flex-array" syntax for pre-C99 compilers, so as +** to avoid complaints from -fsanitize=strict-bounds. +*/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif + typedef struct RecoverTable RecoverTable; typedef struct RecoverColumn RecoverColumn; @@ -140,7 +150,7 @@ struct RecoverColumn { typedef struct RecoverBitmap RecoverBitmap; struct RecoverBitmap { i64 nPg; /* Size of bitmap */ - u32 aElem[]; /* Array of 32-bit bitmasks */ + u32 aElem[FLEXARRAY]; /* Array of 32-bit bitmasks */ }; /* Size in bytes of a RecoverBitmap object sufficient to cover 32 pages */ diff --git a/manifest b/manifest index 5f7bfb31fe..aa813d962f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\salignment\sproblems\son\sLinux\swith\s-m32\sand\son\sMac\sPPC. -D 2025-03-15T18:26:27.012 +C Work\saround\scompilers\sthat\sdo\snot\sunderstand\sflexible\sarrays,\sin\sthe\nrecovery\sextension\sand\sin\sthe\sfuzzcheck\stest\smodule. +D 2025-03-15T19:00:46.484 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -523,7 +523,7 @@ F ext/recover/recoverpgsz.test 88766fcb810e52ee05335c456d4e5fb06d02b73d3ccb48c52 F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f F ext/recover/recoverslowidx.test c90d59c46bb8924a973ac6fbc38f3163cee38cc240256addcab1cf1a322c37dc F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 -F ext/recover/sqlite3recover.c 098b622d34499625a3c87bc31abc237f8e9b992fa93ac5071f280bb0d90e7fd8 +F ext/recover/sqlite3recover.c 56c216332ea91233d6d820d429f3384adbec9ecedda67aa98186b691d427cc57 F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 F ext/recover/test_recover.c 072260d7452a3b81aba995b2b3269e7ec2aa7f06725544ba4c25b1b0a1dbc61a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -1278,7 +1278,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 128789e46d14f2a3ab452fb341165897ff471c4bbef09f8e113c80839acf8941 +F test/fuzzcheck.c e94511f5f5b8c134c83535b4799004b85ead69ed8375d74e9af90e41549a82ad F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 44f2c64ec16f4720dc538be30410863c4138ea4ce41c94521bd7980535261735 -R 0aa0704287ff919c67b51206bc5ac75c +P 8a91aeca60548d5cd19add128cf65b9c3815c9103b1ef8ff6bc02711b6d709de +R 1db0ab70ab59a8097f7bb0d50c768431 U drh -Z 83d96f08f2610b8988f70e4446915dd1 +Z 9641d2a6129bd2783700fa8c8a1376d8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d121f7294a..e5551412ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a91aeca60548d5cd19add128cf65b9c3815c9103b1ef8ff6bc02711b6d709de +f101c46cf83e532fd33034abccba496bf395ef10c161af003211614d6581d5eb diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index f43057a16e..7d5da2ce27 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -88,6 +88,11 @@ #include "sqlite3recover.h" #define ISSPACE(X) isspace((unsigned char)(X)) #define ISDIGIT(X) isdigit((unsigned char)(X)) +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FLEXARRAY +#else +# define FLEXARRAY 1 +#endif #ifdef __unix__ @@ -129,7 +134,7 @@ struct Blob { int id; /* Id of this Blob */ int seq; /* Sequence number */ int sz; /* Size of this Blob in bytes */ - unsigned char a[]; /* Blob content. Extra space allocated as needed. */ + unsigned char a[FLEXARRAY]; /* Blob content. Allocated as needed. */ }; /* Size in bytes of a Blob object sufficient to store N byte of content */ From 42db4d043e8a1f63d809cf6031ba2ab6208f47fd Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 15 Mar 2025 23:42:32 +0000 Subject: [PATCH 26/38] Enhance the fuzzcheck testing tool with new command-line options: --brief, and --slice M N. FossilOrigin-Name: e64132723db0c4f2b9a58932a93beb1671e42006eebc1aeaa8f320e717043051 --- manifest | 13 ++++----- manifest.uuid | 2 +- test/fuzzcheck.c | 74 +++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 3abd0f8344..dc9851a03a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\suse\sof\sthe\sC99\sflexible\sarray\sfeature,\swhen\savailable,\sso\sthat\nthe\s-fsanitize=bounds-strict\soption\scan\sbe\sused,\swhen\savailable.\n[forum:/forumpost/311dbf9a1cadfae6|Forum\sthread\s311dbf9a1c]. -D 2025-03-15T19:55:19.890 +C Enhance\sthe\sfuzzcheck\stesting\stool\swith\snew\scommand-line\soptions:\n--brief,\sand\s--slice\sM\sN. +D 2025-03-15T23:42:32.735 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -1278,7 +1278,7 @@ F test/fuzz3.test 70ba57260364b83e964707b9d4b5625284239768ab907dd387c740c0370ce3 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c e94511f5f5b8c134c83535b4799004b85ead69ed8375d74e9af90e41549a82ad +F test/fuzzcheck.c 19f8af47a5c4ee2c3943fdee270f1f14e3d83fe968a9737a7557fb4e3c06efc1 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -2213,9 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2dd5b6895a3b23c2b9cbf0c1c1e802faf8f2b41ef60819eea25d609755266e64 f101c46cf83e532fd33034abccba496bf395ef10c161af003211614d6581d5eb -R cffcee5ac5b13fdb08ee088881970bf2 -T +closed f101c46cf83e532fd33034abccba496bf395ef10c161af003211614d6581d5eb +P d4307a0d43f42e96ec06ad2c1d8d0f5c8ecae759bae8231b1998633089809f49 +R b41c223f32c59ab49558d35f05a2bfa9 U drh -Z 5f005a89032955065b39a2e143d5c279 +Z 01a995e61d39142e747ab20e203c7c6f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d6e3fced3a..ed9e64db29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4307a0d43f42e96ec06ad2c1d8d0f5c8ecae759bae8231b1998633089809f49 +e64132723db0c4f2b9a58932a93beb1671e42006eebc1aeaa8f320e717043051 diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 7d5da2ce27..09898d7b35 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -391,6 +391,21 @@ static void renderDbSqlForCLI( if( nSql>0 && zSql[nSql-1]!='\n' ) fprintf(out, "\n"); } +/* +** Find the tail (the last component) of a pathname. +*/ +static const char *pathTail(const char *zPath){ + const char *zTail = zPath; + while( zPath[0] ){ + if( zPath[0]=='/' && zPath[1]!=0 ) zTail = &zPath[1]; +#ifndef __unix__ + if( zPath[0]=='\\' && zPath[1]!=0 ) zTail = &zPath[1]; +#endif + zPath++; + } + return zTail; +} + /* ** Read the complete content of a file into memory. Add a 0x00 terminator ** and return a pointer to the result. @@ -1847,6 +1862,7 @@ static void showHelp(void){ "Read databases and SQL scripts from SOURCE-DB and execute each script against\n" "each database, checking for crashes and memory leaks.\n" "Options:\n" +" --brief Output only a summary of results at the end\n" " --cell-size-check Set the PRAGMA cell_size_check=ON\n" " --dbid M..N Use only the databases where dbid between M and N\n" " \"M..\" for M and afterwards. Just \"M\" for M only\n" @@ -1873,6 +1889,7 @@ static void showHelp(void){ " --result-trace Show the results of each SQL command\n" " --script Output CLI script instead of running tests\n" " --skip N Skip the first N test cases\n" +" --slice M N Run only the M-th out of each group of N tests\n" " --spinner Use a spinner to show progress\n" " --sqlid M..N Use only SQL where sqlid between M..N\n" " \"M..\" for M and afterwards. Just \"M\" for M only\n" @@ -1887,6 +1904,7 @@ static void showHelp(void){ int main(int argc, char **argv){ sqlite3_int64 iBegin; /* Start time of this program */ int quietFlag = 0; /* True if --quiet or -q */ + int briefFlag = 0; /* Output summary report at the end */ int verboseFlag = 0; /* True if --verbose or -v */ char *zInsSql = 0; /* SQL statement for --load-db or --load-sql */ int iFirstInsArg = 0; /* First argv[] for --load-db or --load-sql */ @@ -1934,6 +1952,8 @@ int main(int argc, char **argv){ int nV; /* How much to increase verbosity with -vvvv */ sqlite3_int64 tmStart; /* Start of each test */ int iEstTime = 0; /* LPF for the time-to-go */ + int iSliceSz = 0; /* Divide the test space into this many pieces */ + int iSliceIdx = 0; /* Only run the piece with this index */ sqlite3_config(SQLITE_CONFIG_URI,1); registerOomSimulator(); @@ -1954,6 +1974,12 @@ int main(int argc, char **argv){ if( z[0]=='-' ){ z++; if( z[0]=='-' ) z++; + if( strcmp(z,"brief")==0 ){ + briefFlag = 1; + quietFlag = 1; + verboseFlag = 1; + eVerbosity = 0; + }else if( strcmp(z,"cell-size-check")==0 ){ cellSzCkFlag = 1; }else @@ -2044,6 +2070,7 @@ int main(int argc, char **argv){ g.uRandom = atoi(argv[++i]); }else if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){ + briefFlag = 0; quietFlag = 1; verboseFlag = 0; eVerbosity = 0; @@ -2062,12 +2089,19 @@ int main(int argc, char **argv){ if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); nSkip = atoi(argv[++i]); }else + if( strcmp(z,"slice")==0 ){ + if( i>=argc-2 ) fatalError("missing arguments on %s", argv[i]); + iSliceIdx = integerValue(argv[++i]); + iSliceSz = integerValue(argv[++i]); + /* --slice implices --brief */ + briefFlag = 1; + quietFlag = 1; + verboseFlag = 1; + eVerbosity = 0; + }else if( strcmp(z,"spinner")==0 ){ bSpinner = 1; }else - if( strcmp(z,"timer")==0 ){ - bTimer = 1; - }else if( strcmp(z,"sqlid")==0 ){ const char *zDotDot; if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); @@ -2093,10 +2127,14 @@ int main(int argc, char **argv){ fatalError("timeout is not available on non-unix systems"); #endif }else + if( strcmp(z,"timer")==0 ){ + bTimer = 1; + }else if( strcmp(z,"vdbe-debug")==0 ){ bVdbeDebug = 1; }else if( strcmp(z,"verbose")==0 ){ + briefFlag = 0; quietFlag = 0; verboseFlag++; eVerbosity++; @@ -2163,6 +2201,12 @@ int main(int argc, char **argv){ fatalError("cannot import into more than one database"); } } + if( iSliceSz<=iSliceIdx + || iSliceSz<=0 + || iSliceIdx<0 + ){ + iSliceSz = iSliceIdx = 0; + } /* Process each source database separately */ for(iSrcDb=0; iSrcDbpNext){ tmStart = timeOfDay(); if( isDbSql(pSql->a, pSql->sz) ){ + if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ + nTest++; + continue; + } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d",pSql->id); if( bScript ){ /* No progress output */ @@ -2510,6 +2558,10 @@ int main(int argc, char **argv){ for(pDb=g.pFirstDb; pDb; pDb=pDb->pNext){ int openFlags; const char *zVfs = "inmem"; + if( iSliceSz>0 && (nTest%iSliceSz)!=iSliceIdx ){ + nTest++; + continue; + } sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d,dbid=%d", pSql->id, pDb->id); if( bScript ){ @@ -2616,7 +2668,20 @@ int main(int argc, char **argv){ } } } - if( bScript ){ + if( briefFlag ){ + sqlite3_int64 iElapse = timeOfDay() - iBegin; + if( iSliceSz>0 ){ + printf("%s %s: slice %d/%d of %d tests, %d.%03d seconds\n", + pathTail(argv[0]), pathTail(g.zDbFile), + iSliceIdx, iSliceSz, nTest, + (int)(iElapse/1000), (int)(iElapse%1000)); + }else{ + printf("%s %s: 0 errors, %d tests, %d.%03d seconds\n", + pathTail(argv[0]), pathTail(g.zDbFile), nTest, + (int)(iElapse/1000), (int)(iElapse%1000)); + } + iBegin = timeOfDay(); + }else if( bScript ){ /* No progress output */ }else if( bSpinner ){ int nTotal = g.nDb*g.nSql; @@ -2634,6 +2699,7 @@ int main(int argc, char **argv){ } /* End loop over all source databases */ + if( !quietFlag && !bScript ){ sqlite3_int64 iElapse = timeOfDay() - iBegin; if( g.nInvariant ){ From fc293f7c0e926501b914c63d5e2e4eb900a4833b Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 16 Mar 2025 00:13:29 +0000 Subject: [PATCH 27/38] Rework the run-fuzzcheck makefile target so that it better exploit parallelism. Test case "make -j16 run-fuzzcheck FUZZDB=20250222.db" went from 596 seconds down to 107 seconds. FossilOrigin-Name: 18bda13e197e4b4ec7464b3e70012f71edc05f73d8b14bb48bad452f81c7e185 --- main.mk | 135 ++++++++++++++++++++++++++++++++++++++++++++++++-- manifest | 12 ++--- manifest.uuid | 2 +- 3 files changed, 138 insertions(+), 11 deletions(-) diff --git a/main.mk b/main.mk index 4f88a062b4..3b4ce443e5 100644 --- a/main.mk +++ b/main.mk @@ -2192,11 +2192,138 @@ xbin: fuzzcheck-ubsan$(T.exe) # # FUZZDB=test/fuzzdata*.db make run-fuzzcheck # -run-fuzzcheck: fuzzcheck$(T.exe) fuzzcheck-asan$(T.exe) fuzzcheck-ubsan$(T.exe) +# The original rules for this target were like this: +# +# run-fuzzcheck: fuzzcheck$(T.exe) fuzzcheck-asan$(T.exe) fuzzcheck-ubsan$(T.exe) +# @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi +# ./fuzzcheck$(T.exe) --spinner $(FUZZDB) +# ./fuzzcheck-asan$(T.exe) --spinner $(FUZZDB) +# ./fuzzcheck-ubsan$(T.exe) --spinner $(FUZZDB) +# +# What follows is a decomposition of these rules in a way that allows make +# to run things in parallel when using the -jN option. +# +run-fuzzcheck: run-fuzzcheck-n0 +run-fuzzcheck-n0: fuzzcheck$(T.exe) @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi - ./fuzzcheck$(T.exe) --spinner $(FUZZDB) - ./fuzzcheck-asan$(T.exe) --spinner $(FUZZDB) - ./fuzzcheck-ubsan$(T.exe) --spinner $(FUZZDB) + ./fuzzcheck$(T.exe) --slice 0 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-n1 +run-fuzzcheck-n1: fuzzcheck$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck$(T.exe) --slice 1 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-n2 +run-fuzzcheck-n2: fuzzcheck$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck$(T.exe) --slice 2 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-n3 +run-fuzzcheck-n3: fuzzcheck$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck$(T.exe) --slice 3 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-n4 +run-fuzzcheck-n4: fuzzcheck$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck$(T.exe) --slice 4 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-n5 +run-fuzzcheck-n5: fuzzcheck$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck$(T.exe) --slice 5 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-n6 +run-fuzzcheck-n6: fuzzcheck$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck$(T.exe) --slice 6 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-n7 +run-fuzzcheck-n7: fuzzcheck$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck$(T.exe) --slice 7 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-n8 +run-fuzzcheck-n8: fuzzcheck$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck$(T.exe) --slice 8 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-n9 +run-fuzzcheck-n9: fuzzcheck$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck$(T.exe) --slice 9 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a0 +run-fuzzcheck-a0: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 0 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a1 +run-fuzzcheck-a1: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 1 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a2 +run-fuzzcheck-a2: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 2 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a3 +run-fuzzcheck-a3: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 3 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a4 +run-fuzzcheck-a4: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 4 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a5 +run-fuzzcheck-a5: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 5 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a6 +run-fuzzcheck-a6: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 6 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a7 +run-fuzzcheck-a7: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 7 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a8 +run-fuzzcheck-a8: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 8 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-a9 +run-fuzzcheck-a9: fuzzcheck-asan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-asan$(T.exe) --slice 9 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u0 +run-fuzzcheck-u0: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 0 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u1 +run-fuzzcheck-u1: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 1 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u2 +run-fuzzcheck-u2: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 2 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u3 +run-fuzzcheck-u3: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 3 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u4 +run-fuzzcheck-u4: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 4 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u5 +run-fuzzcheck-u5: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 5 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u6 +run-fuzzcheck-u6: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 6 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u7 +run-fuzzcheck-u7: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 7 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u8 +run-fuzzcheck-u8: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 8 10 $(FUZZDB) +run-fuzzcheck: run-fuzzcheck-u9 +run-fuzzcheck-u9: fuzzcheck-ubsan$(T.exe) + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + ./fuzzcheck-ubsan$(T.exe) --slice 9 10 $(FUZZDB) + ossshell$(T.exe): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h $(T.link) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ diff --git a/manifest b/manifest index dc9851a03a..fa23eaa811 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sfuzzcheck\stesting\stool\swith\snew\scommand-line\soptions:\n--brief,\sand\s--slice\sM\sN. -D 2025-03-15T23:42:32.735 +C Rework\sthe\srun-fuzzcheck\smakefile\starget\sso\sthat\sit\sbetter\sexploit\sparallelism.\nTest\scase\s"make\s-j16\srun-fuzzcheck\sFUZZDB=20250222.db"\nwent\sfrom\s596\sseconds\sdown\sto\s107\sseconds. +D 2025-03-16T00:13:29.453 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -705,7 +705,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk e15a567c0bcc1aed49d53944c92a30705e77d21be087cbc0f7359cf1bddb31c8 +F main.mk ec71d4fda51578c017ea922cc28cb71395e097bd58900943fd31599933299230 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d4307a0d43f42e96ec06ad2c1d8d0f5c8ecae759bae8231b1998633089809f49 -R b41c223f32c59ab49558d35f05a2bfa9 +P e64132723db0c4f2b9a58932a93beb1671e42006eebc1aeaa8f320e717043051 +R a0d6184ee2c59df7a27bc937eaa19775 U drh -Z 01a995e61d39142e747ab20e203c7c6f +Z 4d1b91d0df9093e02464f9ab269aee1c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ed9e64db29..d21cad483e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e64132723db0c4f2b9a58932a93beb1671e42006eebc1aeaa8f320e717043051 +18bda13e197e4b4ec7464b3e70012f71edc05f73d8b14bb48bad452f81c7e185 From bafddb17d7d3109f4bc2d48c4a07765e410310c8 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Mar 2025 11:24:32 +0000 Subject: [PATCH 28/38] Consolidate some much-duplicated run-fuzzcheck recipe code in main.mk. FossilOrigin-Name: c0d9b9fad3a2f23941927f1be2abded3bde2f2b04f7a5f3cc0a54a978020ebaa --- main.mk | 61 ++++++++++++++++++++++++++------------------------- manifest | 14 ++++++------ manifest.uuid | 2 +- 3 files changed, 39 insertions(+), 38 deletions(-) diff --git a/main.mk b/main.mk index 3b4ce443e5..ff25719f5f 100644 --- a/main.mk +++ b/main.mk @@ -2203,125 +2203,126 @@ xbin: fuzzcheck-ubsan$(T.exe) # What follows is a decomposition of these rules in a way that allows make # to run things in parallel when using the -jN option. # +FUZZDB_CHECK = @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi run-fuzzcheck: run-fuzzcheck-n0 run-fuzzcheck-n0: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 0 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n1 run-fuzzcheck-n1: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 1 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n2 run-fuzzcheck-n2: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 2 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n3 run-fuzzcheck-n3: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 3 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n4 run-fuzzcheck-n4: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 4 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n5 run-fuzzcheck-n5: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 5 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n6 run-fuzzcheck-n6: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 6 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n7 run-fuzzcheck-n7: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 7 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n8 run-fuzzcheck-n8: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 8 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n9 run-fuzzcheck-n9: fuzzcheck$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck$(T.exe) --slice 9 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a0 run-fuzzcheck-a0: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 0 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a1 run-fuzzcheck-a1: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 1 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a2 run-fuzzcheck-a2: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 2 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a3 run-fuzzcheck-a3: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 3 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a4 run-fuzzcheck-a4: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 4 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a5 run-fuzzcheck-a5: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 5 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a6 run-fuzzcheck-a6: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 6 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a7 run-fuzzcheck-a7: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 7 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a8 run-fuzzcheck-a8: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 8 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a9 run-fuzzcheck-a9: fuzzcheck-asan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-asan$(T.exe) --slice 9 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u0 run-fuzzcheck-u0: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 0 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u1 run-fuzzcheck-u1: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 1 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u2 run-fuzzcheck-u2: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 2 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u3 run-fuzzcheck-u3: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 3 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u4 run-fuzzcheck-u4: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 4 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u5 run-fuzzcheck-u5: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 5 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u6 run-fuzzcheck-u6: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 6 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u7 run-fuzzcheck-u7: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 7 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u8 run-fuzzcheck-u8: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 8 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u9 run-fuzzcheck-u9: fuzzcheck-ubsan$(T.exe) - @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi + $(FUZZDB_CHECK) ./fuzzcheck-ubsan$(T.exe) --slice 9 10 $(FUZZDB) diff --git a/manifest b/manifest index fa23eaa811..c40a6b30a1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rework\sthe\srun-fuzzcheck\smakefile\starget\sso\sthat\sit\sbetter\sexploit\sparallelism.\nTest\scase\s"make\s-j16\srun-fuzzcheck\sFUZZDB=20250222.db"\nwent\sfrom\s596\sseconds\sdown\sto\s107\sseconds. -D 2025-03-16T00:13:29.453 +C Consolidate\ssome\smuch-duplicated\srun-fuzzcheck\srecipe\scode\sin\smain.mk. +D 2025-03-16T11:24:32.667 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -705,7 +705,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk ec71d4fda51578c017ea922cc28cb71395e097bd58900943fd31599933299230 +F main.mk 49dc86f98d28724ffab580159ed46a1c2658d5c01e227794ce8e48128a420967 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e64132723db0c4f2b9a58932a93beb1671e42006eebc1aeaa8f320e717043051 -R a0d6184ee2c59df7a27bc937eaa19775 -U drh -Z 4d1b91d0df9093e02464f9ab269aee1c +P 18bda13e197e4b4ec7464b3e70012f71edc05f73d8b14bb48bad452f81c7e185 +R e37b35226abab9636091409f51115d3e +U stephan +Z a039519926ce716e9a22d933e09747f3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d21cad483e..15d7b90e00 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18bda13e197e4b4ec7464b3e70012f71edc05f73d8b14bb48bad452f81c7e185 +c0d9b9fad3a2f23941927f1be2abded3bde2f2b04f7a5f3cc0a54a978020ebaa From 539442300c35e465c7e583949169a99392e8a241 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Mar 2025 12:27:21 +0000 Subject: [PATCH 29/38] Configure-internal doc cleanups. No functional changes. FossilOrigin-Name: be3a2e631100b711996b9524a54fc604966513a62d83fc916270a6226da7adab --- autosetup/proj.tcl | 11 +++++++++-- autosetup/sqlite-config.tcl | 19 ++++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index f367b01567..e00710b5d4 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1039,8 +1039,15 @@ proc proj-check-soname {{libname "libfoo.so.0"}} { # Checks whether CC supports -fsanitize=X, where X is each entry of # the given list of flags. If any of those flags are supported, it # returns the string "-fsanitize=X..." where X... is a comma-separated -# list of all supported flags. If none of the given options are -# supported then it returns an empty string. +# list of all flags from the original set which are supported. If none +# of the given options are supported then it returns an empty string. +# +# Example: +# +# set f [proj-check-fsanitize {address bounds-check just-testing}] +# +# Will, on many systems, resolve to "-fsanitize=address,bounds-check", +# but may also resolve to "-fsanitize=address". proc proj-check-fsanitize {{opts {address bounds-strict}}} { set sup {} foreach opt $opts { diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 3644b4872a..111d822c22 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -1,6 +1,6 @@ # This file holds functions for autosetup which are specific to the # sqlite build tree. They are in this file, instead of auto.def, so -# that they can be reused in the TEA sub-tree. This file requires +# that they can be reused in the autoconf sub-tree. This file requires # functions from proj.tcl. if {[string first " " $autosetup(srcdir)] != -1} { @@ -24,7 +24,6 @@ use system ; # Will output "Host System" and "Build System" lines if {"--help" ni $::argv} { msg-result "Source dir = $::autosetup(srcdir)" msg-result "Build dir = $::autosetup(builddir)" - use cc cc-db cc-shared cc-lib pkg-config } @@ -445,7 +444,7 @@ proc sqlite-configure-finalize {} { # top-level build and the "autoconf" build, but it's not intended to # be a catch-all dumping ground for such. proc sqlite-post-options-init {} { - define PACKAGE_NAME "sqlite" + define PACKAGE_NAME sqlite define PACKAGE_URL {https://sqlite.org} define PACKAGE_BUGREPORT [get-define PACKAGE_URL]/forum define PACKAGE_STRING "[get-define PACKAGE_NAME] [get-define PACKAGE_VERSION]" @@ -1285,7 +1284,9 @@ proc sqlite-handle-icu {} { if {[opt-bool icu-collations]} { msg-result "Enabling ICU collations." sqlite-add-feature-flag -shell -DSQLITE_ENABLE_ICU_COLLATIONS - # Recall that shell.c builds with sqlite3.c + # Recall that shell.c builds with sqlite3.c except in the case + # of --disable-static-shell, a combination we do not + # specifically attempt to account for. } } elseif {[opt-bool icu-collations]} { proj-warn "ignoring --enable-icu-collations because neither --with-icu-ldflags nor --with-icu-config provided any linker flags" @@ -1428,7 +1429,7 @@ proc sqlite-handle-dll-basename {} { # The name of the import library is [define]d in SQLITE_OUT_IMPLIB. # # If the configure flag --out-implib is not used (or programmatically -# set) then this is a no-op (but see [sqliet-handle-env-quirks]). If +# set) then this is a no-op (but see [sqlite-handle-env-quirks]). If # that flag is used but the capability is not available, a fatal error # is triggered. # @@ -1507,7 +1508,7 @@ proc sqlite-env-is-unix-on-windows {{envTuple ""}} { # # [define]s SQLITE_DLL_INSTALL_RULES to a symbolic name suffix for a # set of "make install" rules to use for installation of the DLL -# deliverable. The makefile is tasked with with providing rules named +# deliverable. The makefile is tasked with providing rules named # install-dll-NAME which runs the installation for that set, as well # as providing a rule named install-dll which resolves to # install-dll-NAME (perhaps indirectly, depending on whether the DLL @@ -1519,13 +1520,13 @@ proc sqlite-env-is-unix-on-windows {{envTuple ""}} { # # On platforms where an "import library" is conventionally used but # --out-implib was not explicitly used, automatically add that flag. -# This conventionally applies to the "Unix on Windows" environments -# like msys and cygwin. +# This conventionally applies only to the "Unix on Windows" +# environments like msys and cygwin. # # 3) --dll-basename: # # On the same platforms addressed by --out-implib, if --dll-basename -# is not specified, --dll-basename=auto is implied. +# is not explicitly specified, --dll-basename=auto is implied. proc sqlite-handle-env-quirks {} { set instName unix-generic; # name of installation rules set set autoDll 0; # true if --out-implib/--dll-basename should be implied diff --git a/manifest b/manifest index c40a6b30a1..d5e77ae0e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Consolidate\ssome\smuch-duplicated\srun-fuzzcheck\srecipe\scode\sin\smain.mk. -D 2025-03-16T11:24:32.667 +C Configure-internal\sdoc\scleanups.\sNo\sfunctional\schanges. +D 2025-03-16T12:27:21.085 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -49,8 +49,8 @@ F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1d F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl bacaf1ed827067942a6d33f2a5c95bd649ceacae2a8ddc584d0f74456fb9167e -F autosetup/sqlite-config.tcl a7f4d093d63bc1da9ec3d44f392f377ce4c86aa7e48532ae51619e55558f5fbe +F autosetup/proj.tcl 5832513884aab068ab67fc8081a06793749312bce85509f27a822ad1e4df62f0 +F autosetup/sqlite-config.tcl a650eced0ad414bca6335bbd33136dfd0d0dc4dc7c279e7836fcdc471771eb44 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 18bda13e197e4b4ec7464b3e70012f71edc05f73d8b14bb48bad452f81c7e185 -R e37b35226abab9636091409f51115d3e +P c0d9b9fad3a2f23941927f1be2abded3bde2f2b04f7a5f3cc0a54a978020ebaa +R 3e2bab33a1215d3d1df21710dcb6fd5c U stephan -Z a039519926ce716e9a22d933e09747f3 +Z 677359532c348140e3ec1f8ae1a81268 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 15d7b90e00..8d58c317d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0d9b9fad3a2f23941927f1be2abded3bde2f2b04f7a5f3cc0a54a978020ebaa +be3a2e631100b711996b9524a54fc604966513a62d83fc916270a6226da7adab From ab689657943f08ec31d1b73cc8e60bfbb2bd5ad3 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Mar 2025 13:09:21 +0000 Subject: [PATCH 30/38] Add --asan-fsanitize=... configure flag to the canonical build to optionally set -fsantize flags for the fuzzcheck-asan tool. Teach proj-check-fsanitiz to fail for flags which the compiler emits any warning for, for reasons described in its comments. FossilOrigin-Name: 013730e9b92af39cb7fd2871df9b4bc81b8990f918892bd79370704421672da0 --- Makefile.in | 2 +- auto.def | 5 ++++- autosetup/proj.tcl | 5 ++++- autosetup/sqlite-config.tcl | 4 ++++ main.mk | 4 ++-- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- 7 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Makefile.in b/Makefile.in index 91135edba3..6a2ba323a0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -136,7 +136,7 @@ HAVE_WASI_SDK = @HAVE_WASI_SDK@ libsqlite3.DLL.install-rules = @SQLITE_DLL_INSTALL_RULES@ # -fsanitize flags for the fuzzcheck-asap app -CFLAGS.fuzzcheck.fsanitize = @CFLAGS_FUZZCHECK_FSANITIZE@ +CFLAGS.fuzzcheck-asan.fsanitize = @CFLAGS_ASAN_FSANITIZE@ T.cc.sqlite = $(T.cc) @TARGET_DEBUG@ diff --git a/auto.def b/auto.def index dcc7371ec0..ee852dddf5 100644 --- a/auto.def +++ b/auto.def @@ -46,7 +46,10 @@ sqlite-configure canonical { define LINK_TOOLS_DYNAMICALLY [proj-opt-was-provided dynlink-tools] - define CFLAGS_FUZZCHECK_FSANITIZE [proj-check-fsanitize {address bounds-strict}] + if {[set fsan [join [opt-val asan-fsanitize] ","]] in {auto ""}} { + set fsan address,bounds-strict + } + define CFLAGS_ASAN_FSANITIZE [proj-check-fsanitize [split $fsan ", "]] sqlite-handle-tcl sqlite-handle-emsdk diff --git a/autosetup/proj.tcl b/autosetup/proj.tcl index e00710b5d4..ba5427b6a5 100644 --- a/autosetup/proj.tcl +++ b/autosetup/proj.tcl @@ -1051,7 +1051,10 @@ proc proj-check-soname {{libname "libfoo.so.0"}} { proc proj-check-fsanitize {{opts {address bounds-strict}}} { set sup {} foreach opt $opts { - cc-with {} { + # -nooutput is used because -fsanitize=hwaddress will otherwise + # pass this test on x86_64, but then warn at build time that + # "hwaddress is not supported for this target". + cc-with {-nooutput 1} { if {[cc-check-flags "-fsanitize=$opt"]} { lappend sup $opt } diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 111d822c22..899d8a507e 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -337,6 +337,10 @@ proc sqlite-configure {buildMode configScript} { => {Enable #line macros in the amalgamation} dynlink-tools => {Dynamically link libsqlite3 to certain tools which normally statically embed it} + asan-fsanitize:=auto + => {Comma- or space-separated list of -fsanitize flags for use with the + fuzzcheck-asan tool. Only those which the compiler claims to support + will actually be used. May be provided multiple times.} } {*} { dump-defines=0 diff --git a/main.mk b/main.mk index ff25719f5f..97a747566b 100644 --- a/main.mk +++ b/main.mk @@ -2169,10 +2169,10 @@ fuzzy: fuzzcheck$(T.exe) xbin: fuzzcheck$(T.exe) # -fsanitize=... flags for fuzzcheck-asan. -CFLAGS.fuzzcheck.fsanitize ?= -fsanitize=address +CFLAGS.fuzzcheck-asan.fsanitize ?= -fsanitize=address fuzzcheck-asan$(T.exe): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) - $(T.link) -o $@ $(CFLAGS.fuzzcheck.fsanitize) $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ + $(T.link) -o $@ $(CFLAGS.fuzzcheck-asan.fsanitize) $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) \ sqlite3.c $(LDFLAGS.libsqlite3) fuzzy: fuzzcheck-asan$(T.exe) xbin: fuzzcheck-asan$(T.exe) diff --git a/manifest b/manifest index d5e77ae0e7..6fe3756f0f 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Configure-internal\sdoc\scleanups.\sNo\sfunctional\schanges. -D 2025-03-16T12:27:21.085 +C Add\s--asan-fsanitize=...\sconfigure\sflag\sto\sthe\scanonical\sbuild\sto\soptionally\sset\s-fsantize\sflags\sfor\sthe\sfuzzcheck-asan\stool.\sTeach\sproj-check-fsanitiz\sto\sfail\sfor\sflags\swhich\sthe\scompiler\semits\sany\swarning\sfor,\sfor\sreasons\sdescribed\sin\sits\scomments. +D 2025-03-16T13:09:21.699 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d -F Makefile.in 2788f5a3c36e26817707003170871936d44f46c5b45f8cbefe07c9f1a51a8988 +F Makefile.in 2c4d3c20e42ddd7596432c8d45feeaf709f93b37279e274ea413034912a4f840 F Makefile.linux-generic bd3e3cacd369821a6241d4ea1967395c962dfe3057e38cb0a435cee0e8b789d0 F Makefile.msc ef04498c7e227a0f459b105bb4952f26cc985d1d6340a367e62d5a79c4689dfb F README.md a953c0cffd6e4f2501a306c00ee2b6e1e6630c25031e094629307fe99dd003d1 @@ -14,7 +14,7 @@ F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 -F auto.def 619383263dfd0ee31df6a9d3e9828f65943d9fbcd274084046680641e5de53cb +F auto.def 23f0e7eb5eff4cf922963e667ed630793ed60300df59ef9d93c87a23e31c7232 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac F autoconf/Makefile.in 6c98c82f52aa27a5c586080cf7c61c811174c2b6d8b8de33fd657d78d541dd7d F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2038dcd8b @@ -49,8 +49,8 @@ F autosetup/cc-shared.tcl 4f024e94a47f427ba61de1739f6381ef0080210f9fae89112d5c1d F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e45f F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 5832513884aab068ab67fc8081a06793749312bce85509f27a822ad1e4df62f0 -F autosetup/sqlite-config.tcl a650eced0ad414bca6335bbd33136dfd0d0dc4dc7c279e7836fcdc471771eb44 +F autosetup/proj.tcl bdf0489d4ce8110fc1d4a09b1e2e274e50dd51711637b55c7c63a6a7ecec2aa5 +F autosetup/sqlite-config.tcl 3dddc1f949acfc1cad9faf13ece5095ce5ee34dea96f50aeb96aa91641c3a08c F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 @@ -705,7 +705,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 49dc86f98d28724ffab580159ed46a1c2658d5c01e227794ce8e48128a420967 +F main.mk 11b4da311dedf3bf0301999b307c117fe1b5e4fdc260cb35105d235db0fec13b F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2213,8 +2213,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c0d9b9fad3a2f23941927f1be2abded3bde2f2b04f7a5f3cc0a54a978020ebaa -R 3e2bab33a1215d3d1df21710dcb6fd5c +P be3a2e631100b711996b9524a54fc604966513a62d83fc916270a6226da7adab +R 699bb2f6a1869a511409b94aecd9b45c U stephan -Z 677359532c348140e3ec1f8ae1a81268 +Z b465670c31108cfd99bca4e96066f044 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8d58c317d5..fec909f33f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be3a2e631100b711996b9524a54fc604966513a62d83fc916270a6226da7adab +013730e9b92af39cb7fd2871df9b4bc81b8990f918892bd79370704421672da0 From 1774ec3ad0e4d450cc1026a0370f7c53e7fb9705 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 17 Mar 2025 14:59:55 +0000 Subject: [PATCH 31/38] Add support for the --with-wasi-sdk configure flag to the autoconf build. FossilOrigin-Name: 44880fa3f0748604ef50b942c28390e041138759efea1d076dfcaa1da48970cb --- autoconf/Makefile.in | 10 +++++++--- autoconf/auto.def | 3 ++- autosetup/sqlite-config.tcl | 9 +++++---- manifest | 17 ++++++++--------- manifest.uuid | 2 +- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/autoconf/Makefile.in b/autoconf/Makefile.in index e59864a20b..0fa9065be8 100644 --- a/autoconf/Makefile.in +++ b/autoconf/Makefile.in @@ -50,6 +50,7 @@ CC = @CC@ ENABLE_LIB_SHARED = @ENABLE_LIB_SHARED@ ENABLE_LIB_STATIC = @ENABLE_LIB_STATIC@ +HAVE_WASI_SDK = @HAVE_WASI_SDK@ CFLAGS = @CFLAGS@ @CPPFLAGS@ # @@ -236,11 +237,14 @@ sqlite3$(T.exe): $(TOP)/shell.c $(sqlite3-shell-deps.$(ENABLE_STATIC_SHELL)) $(CFLAGS) $(CFLAGS.readline) $(CFLAGS.icu) \ $(LDFLAGS) $(LDFLAGS.readline) -all: sqlite3$(T.exe) +sqlite3$(T.exe)-1: +sqlite3$(T.exe)-0: sqlite3$(T.exe) +all: sqlite3$(T.exe)-$(HAVE_WASI_SDK) -install-shell: sqlite3$(T.exe) $(install-dir.bin) +install-shell-0: sqlite3$(T.exe) $(install-dir.bin) $(INSTALL.strip) sqlite3$(T.exe) "$(install-dir.bin)" -install: install-shell +install-shell-1: +install: install-shell-$(HAVE_WASI_SDK) install-headers: $(TOP)/sqlite3.h $(install-dir.include) $(INSTALL.noexec) $(TOP)/sqlite3.h $(TOP)/sqlite3ext.h "$(install-dir.include)" diff --git a/autoconf/auto.def b/autoconf/auto.def index 3ba900d957..5d7ff83913 100644 --- a/autoconf/auto.def +++ b/autoconf/auto.def @@ -5,6 +5,7 @@ # "autoconf" bundle of the SQLite project. use sqlite-config sqlite-configure autoconf { - sqlite-check-common-bins + sqlite-check-common-bins ;# must come before [sqlite-handle-wasi-sdk] + sqlite-handle-wasi-sdk ;# must run relatively early, as it changes the environment sqlite-check-common-system-deps } diff --git a/autosetup/sqlite-config.tcl b/autosetup/sqlite-config.tcl index 899d8a507e..42b4382cfa 100644 --- a/autosetup/sqlite-config.tcl +++ b/autosetup/sqlite-config.tcl @@ -258,11 +258,11 @@ proc sqlite-configure {buildMode configScript} { # Options for exotic/alternative build modes alternative-builds { - {canonical} { - # Potential TODO: add --with-wasi-sdk support to the autoconf - # build + {*} { with-wasi-sdk:=/opt/wasi-sdk => {Top-most dir of the wasi-sdk for a WASI build} + } + {canonical} { with-emsdk:=auto => {Top-most dir of the Emscripten SDK installation. @@ -1656,7 +1656,8 @@ proc sqlite-handle-wasi-sdk {} { tcl threadsafe } { - if {[opt-bool $opt]} { + if {[proj-opt-exists $opt] && [opt-bool $opt]} { + # -^^^^ distinguish between canonical and autoconf builds msg-result " --disable-$opt" proj-opt-set $opt 0 } diff --git a/manifest b/manifest index ea7d2d3b75..f9c0490429 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\slong-standing\sfilename\sdigest\scomputation\sbug\sin\sthe\sOPFS\sSAHPool\sVFS\swhich\scaused\sall\sVFS-stored\sfilenames\sto\shave\sa\sdigest\svalue\sof\s0.\sSee\s[/forumpost/042d53c928382021]\sand\sfor\sfull\sdetails. -D 2025-03-16T14:05:42.209 +C Add\ssupport\sfor\sthe\s--with-wasi-sdk\sconfigure\sflag\sto\sthe\sautoconf\sbuild. +D 2025-03-17T14:59:55.092 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -16,11 +16,11 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F art/sqlite370.svg 40b7e2fe8aac3add5d56dd86ab8d427a4eca5bcb3fe4f8946cb3794e1821d531 F auto.def 23f0e7eb5eff4cf922963e667ed630793ed60300df59ef9d93c87a23e31c7232 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.in 6c98c82f52aa27a5c586080cf7c61c811174c2b6d8b8de33fd657d78d541dd7d +F autoconf/Makefile.in b499f790938d5334a5e22fa9a91af603661e714dea40cd5fb60d03f500b3ae4f F autoconf/Makefile.msc 5bc67d3912444c40c6f96d003e5c90663e51abb83d204a520110b1b2038dcd8b F autoconf/README.first f1d3876e9a7852c22f275a6f06814e64934cecbc0b5b9617d64849094c1fd136 F autoconf/README.txt 1a32296d8bbdd67110c79d224c92c05545a0b5bd0c272950025fe3c7c7b49580 -F autoconf/auto.def 8d81c1d728d8462a9b6c1ca0714013bbb097aee0ae5e79309d7939cead98e295 +F autoconf/auto.def 42d239bda4feffe1cf8a431dae35f83d100f2c17ed4b189edeb12f067bd4fa90 F autoconf/tea/Makefile.in ba0556fee8da09c066bad85a4457904e46ee2c2eabaa309c0e83a78f2f151a8e F autoconf/tea/README.txt 6c396709b45eb2b3be0ae6dc7e40a140d231962e3a2354da6c4dd48b1d9999bc F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 @@ -50,7 +50,7 @@ F autosetup/cc.tcl c0fcc50ca91deff8741e449ddad05bcd08268bc31177e613a6343bbd1fd3e F autosetup/jimsh0.c a57c16e65dcffc9c76e496757cb3f7fb47e01ecbd1631a0a5e01751fc856f049 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba F autosetup/proj.tcl bdf0489d4ce8110fc1d4a09b1e2e274e50dd51711637b55c7c63a6a7ecec2aa5 -F autosetup/sqlite-config.tcl 3dddc1f949acfc1cad9faf13ece5095ce5ee34dea96f50aeb96aa91641c3a08c +F autosetup/sqlite-config.tcl a2eb8234de355233787ad7bc926d8a622ef9f97594749374c4360b63c37fc223 F autosetup/system.tcl 51d4be76cd9a9074704b584e5c9cbba616202c8468cf9ba8a4f8294a7ab1dba9 F configure 9a00b21dfd13757bbfb8d89b30660a89ec1f8f3a79402b8f9f9b6fc475c3303a x F contrib/sqlitecon.tcl eb4c6578e08dd353263958da0dc620f8400b869a50d06e271ab0be85a51a08d3 @@ -2215,9 +2215,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 013730e9b92af39cb7fd2871df9b4bc81b8990f918892bd79370704421672da0 500f2e6ec74b4c0e4ac0365ba4e0d81ed6df8dd09dc0f8af65d294c3453f8865 -R 9266ed4d88e1ccecb16602d4a66164a0 -T +closed 500f2e6ec74b4c0e4ac0365ba4e0d81ed6df8dd09dc0f8af65d294c3453f8865 Closed\sby\sintegrate-merge. +P 493cbe74504e8eb1ca8f2edf49fdab6bebc7fe36ffab06932a4b8c5a4eea86cd +R 2a00f59ca343d92d1ab593996ae581fb U stephan -Z 8dfe9f7d9c954162ba84353f5fc2343c +Z c372334b201c7cc2e21fcf33dbf843ab # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3d3dbbdd32..7e8796ece6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -493cbe74504e8eb1ca8f2edf49fdab6bebc7fe36ffab06932a4b8c5a4eea86cd +44880fa3f0748604ef50b942c28390e041138759efea1d076dfcaa1da48970cb From 7b99cd60632a5bfcb790e55757a69a0e20b3433e Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 17 Mar 2025 15:13:47 +0000 Subject: [PATCH 32/38] Prevent integer overflow when parsing NEAR queries in FTS5. FossilOrigin-Name: 1a5283d7dab210badb8a33eac29f44dc8c1c210ffb5ef84f43e348170aa406a6 --- ext/fts5/fts5_expr.c | 3 ++- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index d7574aec52..0a9b08ed15 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2015,7 +2015,8 @@ void sqlite3Fts5ParseSetDistance( ); return; } - nNear = nNear * 10 + (p->p[i] - '0'); + if( nNear<214748363 ) nNear = nNear * 10 + (p->p[i] - '0'); + /* ^^^^^^^^^^^^^^^--- Prevent integer overflow */ } }else{ nNear = FTS5_DEFAULT_NEARDIST; diff --git a/manifest b/manifest index f9c0490429..2b0493d4f9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sthe\s--with-wasi-sdk\sconfigure\sflag\sto\sthe\sautoconf\sbuild. -D 2025-03-17T14:59:55.092 +C Prevent\sinteger\soverflow\swhen\sparsing\sNEAR\squeries\sin\sFTS5. +D 2025-03-17T15:13:47.985 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -110,7 +110,7 @@ F ext/fts5/fts5Int.h bffbd0acdcdf509899681f4e1cfeef1c955030acd9fe15ff9082410f80c F ext/fts5/fts5_aux.c da4a7a9a11ec15c6df0699d908915a209bcde48f0b04101461316b59f71abffb F ext/fts5/fts5_buffer.c f1e6d0324d7c55329d340673befc26681a372a4d36086caa8d1ec7d7c53066c7 F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 -F ext/fts5/fts5_expr.c 887a611b34094c828ff5fb19bbc50a6b1bbfd28791db01b0c8bf722e3c9f437a +F ext/fts5/fts5_expr.c be9e5f7f11d87e7bd3680832c93c13050fe351994b5052b0215c2ef40312c23a F ext/fts5/fts5_hash.c a6266cedd801ab7964fa9e74ebcdda6d30ec6a96107fa24148ec6b7b5b80f6e0 F ext/fts5/fts5_index.c d171f2a507abccb3d524bf461b01f0d3971a9bf221be622ac7c671a991cb62ee F ext/fts5/fts5_main.c 57933c18efe1058d8871199875c7a59744dabc3904f3aefbf9ff4a4e11fc79e2 @@ -2215,8 +2215,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 493cbe74504e8eb1ca8f2edf49fdab6bebc7fe36ffab06932a4b8c5a4eea86cd -R 2a00f59ca343d92d1ab593996ae581fb -U stephan -Z c372334b201c7cc2e21fcf33dbf843ab +P 44880fa3f0748604ef50b942c28390e041138759efea1d076dfcaa1da48970cb +R 62b06ea4ad92e5bc3b483d586cc748b3 +U drh +Z 243e630e7be3fc1ae898e3d108d054eb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7e8796ece6..5b483f0349 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -44880fa3f0748604ef50b942c28390e041138759efea1d076dfcaa1da48970cb +1a5283d7dab210badb8a33eac29f44dc8c1c210ffb5ef84f43e348170aa406a6 From 17df9cd909e06800986a4dc34f3f42bc863f57ec Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 18 Mar 2025 10:28:56 +0000 Subject: [PATCH 33/38] Update the docs in tool/mkccode.tcl to reflect that it's more generic than it was when the docs were written. Change the shebang line to use /bin/env tclsh instead of a hard-coded tclsh path. FossilOrigin-Name: 9300f7f42dfd143f77fd51aa9e080099540854d36b6997ab1a16be7d77f78d8e --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/mkccode.tcl | 13 ++++++------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 2b0493d4f9..170e6538ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Prevent\sinteger\soverflow\swhen\sparsing\sNEAR\squeries\sin\sFTS5. -D 2025-03-17T15:13:47.985 +C Update\sthe\sdocs\sin\stool/mkccode.tcl\sto\sreflect\sthat\sit's\smore\sgeneric\sthan\sit\swas\swhen\sthe\sdocs\swere\swritten.\sChange\sthe\sshebang\sline\sto\suse\s/bin/env\stclsh\sinstead\sof\sa\shard-coded\stclsh\spath. +D 2025-03-18T10:28:56.725 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -2151,7 +2151,7 @@ F tool/max-limits.c cbb635fbb37ae4d05f240bfb5b5270bb63c54439 F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a1914b176 F tool/mkamalzip.tcl 8aa5ebe7973c8b8774062d34e15fea9815c4cc2ceea3a9b184695f005910876a F tool/mkautoconfamal.sh c5e65fa1c922f2e3b3e4f6cd0331ec7d84bdef085f32cb1c46673cdf95ec8090 -F tool/mkccode.tcl 210159febe0ef0ecbc53c79833500663ceaba0115b2b374405818dc835b5f84b x +F tool/mkccode.tcl c42a8f8cf78f92e83795d5447460dbce7aaf78a3bbf9082f1507dc71a3665f3c x F tool/mkctimec.tcl f76dbfc74cefad8d126384ba3263677939f077bd184fcdf8c592a1daf64f50c3 x F tool/mkkeywordhash.c 6b0be901c47f9ad42215fc995eb2f4384ac49213b1fba395102ec3e999acf559 F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a @@ -2215,8 +2215,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 44880fa3f0748604ef50b942c28390e041138759efea1d076dfcaa1da48970cb -R 62b06ea4ad92e5bc3b483d586cc748b3 -U drh -Z 243e630e7be3fc1ae898e3d108d054eb +P 1a5283d7dab210badb8a33eac29f44dc8c1c210ffb5ef84f43e348170aa406a6 +R a929af68b23ab89a4ef69e835299c074 +U stephan +Z cc5c9a3f1559d25c8e48b1b5fb513517 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5b483f0349..9ff19c6263 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a5283d7dab210badb8a33eac29f44dc8c1c210ffb5ef84f43e348170aa406a6 +9300f7f42dfd143f77fd51aa9e080099540854d36b6997ab1a16be7d77f78d8e diff --git a/tool/mkccode.tcl b/tool/mkccode.tcl index ecafbdadb9..8b4fae82c9 100755 --- a/tool/mkccode.tcl +++ b/tool/mkccode.tcl @@ -1,17 +1,16 @@ -#!/usr/bin/tclsh +#!/bin/env tclsh # -# Use this script to build C-language source code for a program that uses -# tclsqlite.c together with custom TCL scripts and/or C extensions for -# either SQLite or TCL. +# This script is used to amalgamate C source code files into a single +# unit. # # Usage example: # # tclsh mkccode.tcl -DENABLE_FEATURE_XYZ demoapp.c.in >demoapp.c # # The demoapp.c.in file contains a mixture of C code, TCL script, and -# processing directives used by mktclsqliteprog.tcl to build the final C-code -# output file. Most lines of demoapp.c.in are copied straight through into -# the output. The following control directives are recognized: +# processing directives used by mkccode.tcl to build the final C-code +# output file. Most lines of demoapp.c.in are copied straight through +# into the output. The following control directives are recognized: # # BEGIN_STRING # From a49265b71778cc061ea69bb1dda8d12765512468 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 18 Mar 2025 11:36:15 +0000 Subject: [PATCH 34/38] General updates to autosetup/README.md. FossilOrigin-Name: be8ad5cf579662c54b7e7bc492d8ca03b1f9032c5dad9a6b20590ca1ac00840b --- autosetup/README.md | 105 +++++++++++++++++++++++++------------------- manifest | 12 ++--- manifest.uuid | 2 +- 3 files changed, 68 insertions(+), 51 deletions(-) diff --git a/autosetup/README.md b/autosetup/README.md index 2d6cf723c0..a61f94fbda 100644 --- a/autosetup/README.md +++ b/autosetup/README.md @@ -14,7 +14,7 @@ build infrastructure. It is not an [Autosetup][] reference. - Symbolic Names of Feature Flags - Do Not Update Global Shared State - [Updating Autosetup](#updating) - - [Patching Autosetup for Project-local changes](#patching) + - ***[Patching Autosetup for Project-local changes](#patching)*** ------------------------------------------------------------------------ @@ -42,12 +42,13 @@ following files: (e.g. Fossil), and personal projects of SQLite's developers. It is essentially an amalgamation of a decade's worth of autosetup-related utility code. -- [auto.def][]: the primary driver for the `./configure` process. - When we talk about "the configure script," we're referring to - this file. - [sqlite-config.tcl][]: utility code which is too project-specific for `proj.tcl`. We split this out of `auto.def` so that it can be used by both `auto.def` and... +- [auto.def][]: the primary driver for the `./configure` process. + When we talk about "the configure script," we're technically + referring to this file, though it actually contains very little + of the TCL code. - [autoconf/auto.def][]: the main driver script for the "autoconf" bundle's configure script. It is essentially a slightly trimmed-down version of the main `auto.def` file. The `autoconf` dir was ported @@ -61,11 +62,11 @@ Autosetup API Tips This section briefly covers only APIs which are frequently useful in day-to-day maintenance and might not be immediately recognized as such -obvious from a casual perusal of the relevant TCL files. The complete -docs of those with `proj-` prefix can be found in [proj.tcl][] and -those with an `sqlite-` prefix are in [sqlite-config.tcl][]. The -others are scattered around [the TCL files in -./autosetup](/dir/autosetup). +from a casual perusal of the relevant TCL files. The complete docs of +those with `proj-` prefix can be found in [proj.tcl][] and those with +an `sqlite-` prefix are in [sqlite-config.tcl][]. The others are part +of Autosetup's core packages and are scattered around [the TCL files +in ./autosetup](/dir/autosetup). In (mostly) alphabetical order: @@ -83,19 +84,14 @@ In (mostly) alphabetical order: Works like `get-env` but will, if that function finds no match, look for a file named `./.env-$VAR` and, if found, return its trimmed contents. This can be used, e.g., to set a developer's - local preferences for the default `CFLAGS`. - -- **`define-for-opt flag defineName ?checkingMsg? ?yesVal=1? ?noVal=0?`**\ - `[define $defineName]` to either `$yesVal` or `$noVal`, depending on - whether `--$flag` is truthy or not. `$checkingMsg` is a - human-readable description of the check being made, e.g. "enable foo?" - If no `checkingMsg` is provided, the operation is silent.\ - Potential TODO: change the final two args to `-yes` and `-no` - flags. They're rarely needed, though: search [auto.def][] for - `TSTRNNR_OPTS` for an example of where they are used. + local preferences for the default `CFLAGS`.\ + Tip: adding `-O0` to `.env-CFLAGS` reduces rebuild times + considerably at the cost of performance in `make devtest` and the + like. - **`proj-fatal msg`**\ - Emits `$msg` to stderr and exits with non-zero. + Emits `$msg` to stderr and exits with non-zero. Its differences from + autosetup's `user-error` are purely cosmetic. - **`proj-if-opt-truthy flag thenScript ?elseScript?`**\ Evals `thenScript` if the given `--flag` is truthy, else it @@ -117,7 +113,10 @@ In (mostly) alphabetical order: else 0. This distinction can be used to determine, e.g., whether `--with-readline` was provided or whether we're searching for readline by default. In the former case, failure to find it should - be treated as fatal, where in the latter case it's not. + be treated as fatal, where in the latter case it's not.\ + Unlike most functions which deal with `--flags`, this one does not + validate that `$FLAG` is a registered flag so will not fail fatally + if `$FLAG` is not registered as an Autosetup option. - **`proj-val-truthy value`**\ Returns 1 if `$value` is "truthy," See `proj-opt-truthy` for the definition @@ -139,6 +138,15 @@ In (mostly) alphabetical order: The shell-specific counterpart of `sqlite-add-feature-flag` which only adds the given flag(s) to the CLI-shell-specific CFLAGS. +- **`sqlite-configure BUILD-NAME {script}`**\ + This is where all configure `--flags` are defined for all known + build modes ("canonical" or "autoconf"). After processing all flags, + this function runs `$script`, which contains the build-mode-specific + configuration bits, and then runs any finalization bits which are + common to all build modes. The `auto.def` files are intended to contain + exactly two commands: + `use sqlite-config; sqlite-configure BUILD-NAME {script}` + - **`user-notice msg`**\ Queues `$msg` to be sent to stderr, but does not emit it until either `show-notices` is called or the next time autosetup would @@ -152,13 +160,18 @@ In (mostly) alphabetical order: Ensuring TCL Compatibility ======================================================================== -It is important that any TCL files used by the configure process -remain compatible with both [JimTCL][] and the canonical TCL. Though -JimTCL has outstanding compatibility with canonical TCL, it does have -a few corners with incompatibilities, e.g. regular expressions. If a -script runs in JimTCL without using any JimTCL-specific features, then -it's a certainty that it will run in canonical TCL as well. The -opposite, however, is not _always_ the case. +One of the significant benefits of using Autosetup is that (A) this +project uses many TCL scripts in the build process and (B) Autosetup +comes with a TCL interpreter named [JimTCL][]. + +It is important that any TCL files used by the configure process and +makefiles remain compatible with both [JimTCL][] and the canonical +TCL. Though JimTCL has outstanding compatibility with canonical TCL, +it does have a few corners with incompatibilities, e.g. regular +expressions. If a script runs in JimTCL without using any +JimTCL-specific features, then it's a certainty that it will run in +canonical TCL as well. The opposite, however, is not _always_ the +case. When [`./configure`](/file/configure) is run, it goes through a bootstrapping process to find a suitable TCL with which to run the @@ -187,14 +200,17 @@ compatibility across TCL implementations: before looking for a system-level `tclsh`. Be aware, though, that `make distclean` will remove that file. -**Note that `jimsh0` is distinctly different from the `jimsh`** which -gets built for code-generation purposes. The latter requires +**Note that `./jimsh0` is distinctly different from the `./jimsh`** +which gets built for code-generation purposes. The latter requires non-default build flags to enable features which are platform-dependent, most notably to make its `[file normalize]` work. This means, for example, that the configure script and its utility APIs must not use `[file normalize]`, but autosetup provides a TCL-only implementation of `[file-normalize]` (note the dash) for -portable use in the configure script. +portable use in the configure script. Contrariwise, code-generation +scripts invoked via `make` may use `[file normalize]`, as they'll use +`./jimsh` or `tclsh` instead of `./jimsh0`. + Known TCL Incompatibilities ------------------------------------------------------------------------ @@ -221,6 +237,7 @@ A summary of known incompatibilities in JimTCL - `regsub` does not support the `\y` flag. A workaround is demonstrated in [](/info/c2e5dd791cce3ec4). + Design Conventions ======================================================================== @@ -247,8 +264,9 @@ dots are not permitted. The `X.y` convention is used in the makefiles primarily because the person who did the initial port finds that considerably easier on the eyes and fingers. In practice, the `X_Y` form of such exports is used -exactly once in [Makefile.in][], where it's translated into into `X.y` -form for consumption by [Makefile.in][] and [main.mk][]. For example: +exactly once in [Makefile.in][], where it's translated from `@X_Y@` +into into `X.y` form for consumption by [Makefile.in][] and +[main.mk][]. For example: > ``` @@ -265,9 +283,9 @@ of taste, for which there is no accounting.) Do Not Update Global Shared State ------------------------------------------------------------------------ -In both the legacy Autotools-driven build and in common Autosetup -usage, feature tests performed by the configure script may amend -global flags such as `LIBS`, `LDFLAGS`, and `CFLAGS`[^as-cflags]. That's +In both the legacy Autotools-driven build and common Autosetup usage, +feature tests performed by the configure script may amend global flags +such as `LIBS`, `LDFLAGS`, and `CFLAGS`[^as-cflags]. That's appropriate for a makefile which builds a single deliverable, but less so for makefiles which produce multiple deliverables. Drawbacks of that approach include: @@ -275,8 +293,8 @@ that approach include: - It's unlikely that every single deliverable will require the same core set of those flags. - It can be difficult to determine the origin of any given change to - that global state because those changes are hidden behind voodoo performed - outside the immediate visibility of the configure script's + that global state because those changes are hidden behind voodoo + performed outside the immediate visibility of the configure script's maintainer. - It can force the maintainers of the configure script to place tests in a specific order so that the resulting flags get applied at @@ -357,12 +375,11 @@ Patching Autosetup for Project-local Changes Autosetup reserves the flag name **`--debug`** for its own purposes, and its own special handling of `--enable-...` flags makes `--debug` -an alias for `--enable-debug`. As we have a long history of using -`--enable-debug` for this project's own purposes, we patch autosetup -to use the name `--autosetup-debug` in place of `--debug`. That -requires (as of this writing) four small edits in -[](/file/autosetup/autosetup), as demonstrated in [check-in -3296c8d3](/info/3296c8d3). +an alias for `--enable-debug`. As this project has a long history of +using `--enable-debug`, we patch autosetup to use the name +`--autosetup-debug` in place of `--debug`. That requires (as of this +writing) four small edits in [](/file/autosetup/autosetup), as +demonstrated in [check-in 3296c8d3](/info/3296c8d3). If autosetup is upgraded and this patch is _not_ applied the invoking `./configure` will fail loudly because of the declaration of the diff --git a/manifest b/manifest index 170e6538ad..df636785e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdocs\sin\stool/mkccode.tcl\sto\sreflect\sthat\sit's\smore\sgeneric\sthan\sit\swas\swhen\sthe\sdocs\swere\swritten.\sChange\sthe\sshebang\sline\sto\suse\s/bin/env\stclsh\sinstead\sof\sa\shard-coded\stclsh\spath. -D 2025-03-18T10:28:56.725 +C General\supdates\sto\sautosetup/README.md. +D 2025-03-18T11:36:15.450 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -37,7 +37,7 @@ F autoconf/tea/win/rules.vc 94a18c3e453535459b4a643983acca52fb8756e79055bd2ad4b0 F autoconf/tea/win/targets.vc 96a25a1fa6e9e9cfb348fd3760a5395b4ce8acafc8ed10f0412937ec200d5dbd F autosetup/LICENSE 41a26aebdd2cd185d1e2b210f71b7ce234496979f6b35aef2cbf6b80cbed4ce4 F autosetup/README.autosetup a78ff8c4a3d2636a4268736672a74bf14a82f42687fcf0631a70c516075c031e -F autosetup/README.md b306314e8a87ccf873cb5b2a360c4a27bbf841df5b76f3acbd65322cff165476 +F autosetup/README.md f98cc827a162a1da4877e9656d749d414ba3f408d457d30e029afc66590c00c3 F autosetup/autosetup 74a9782b68d07934510190fbd03fc6ad92e63f0ea3b5cbffa5f0bd271ad60f01 x F autosetup/autosetup-config.guess dfa101c5e8220e864d5e9c72a85e87110df60260d36cb951ad0a85d6d9eaa463 x F autosetup/autosetup-config.sub a38fb074d0dece01cf919e9fb534a26011608aa8fa606490864295328526cd73 x @@ -2215,8 +2215,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a5283d7dab210badb8a33eac29f44dc8c1c210ffb5ef84f43e348170aa406a6 -R a929af68b23ab89a4ef69e835299c074 +P 9300f7f42dfd143f77fd51aa9e080099540854d36b6997ab1a16be7d77f78d8e +R a74ff5093dd14720da044ecaa9143a8b U stephan -Z cc5c9a3f1559d25c8e48b1b5fb513517 +Z 7470894fe299edfa712004a6589105cc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9ff19c6263..456e6406f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9300f7f42dfd143f77fd51aa9e080099540854d36b6997ab1a16be7d77f78d8e +be8ad5cf579662c54b7e7bc492d8ca03b1f9032c5dad9a6b20590ca1ac00840b From ccda0f008a9817edc402671baf09ce2741440593 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 18 Mar 2025 12:31:09 +0000 Subject: [PATCH 35/38] Very slight simplification of the run-fuzzcheck rules. FossilOrigin-Name: c858a39fad30c46aec6a1f81b2d4e56c18ecf7f5cb6d2fe4a32c4b3bb1a6ed64 --- main.mk | 93 ++++++++++++++++++--------------------------------- manifest | 12 +++---- manifest.uuid | 2 +- 3 files changed, 39 insertions(+), 68 deletions(-) diff --git a/main.mk b/main.mk index 97a747566b..93cfe4b7c1 100644 --- a/main.mk +++ b/main.mk @@ -2203,126 +2203,97 @@ xbin: fuzzcheck-ubsan$(T.exe) # What follows is a decomposition of these rules in a way that allows make # to run things in parallel when using the -jN option. # -FUZZDB_CHECK = @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi +FUZZDB-check: + @if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi run-fuzzcheck: run-fuzzcheck-n0 -run-fuzzcheck-n0: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n0: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 0 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n1 -run-fuzzcheck-n1: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n1: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 1 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n2 -run-fuzzcheck-n2: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n2: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 2 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n3 -run-fuzzcheck-n3: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n3: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 3 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n4 -run-fuzzcheck-n4: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n4: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 4 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n5 -run-fuzzcheck-n5: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n5: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 5 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n6 -run-fuzzcheck-n6: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n6: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 6 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n7 -run-fuzzcheck-n7: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n7: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 7 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n8 -run-fuzzcheck-n8: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n8: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 8 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-n9 -run-fuzzcheck-n9: fuzzcheck$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-n9: FUZZDB-check fuzzcheck$(T.exe) ./fuzzcheck$(T.exe) --slice 9 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a0 -run-fuzzcheck-a0: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a0: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 0 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a1 -run-fuzzcheck-a1: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a1: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 1 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a2 -run-fuzzcheck-a2: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a2: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 2 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a3 -run-fuzzcheck-a3: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a3: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 3 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a4 -run-fuzzcheck-a4: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a4: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 4 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a5 -run-fuzzcheck-a5: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a5: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 5 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a6 -run-fuzzcheck-a6: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a6: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 6 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a7 -run-fuzzcheck-a7: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a7: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 7 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a8 -run-fuzzcheck-a8: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a8: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 8 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-a9 -run-fuzzcheck-a9: fuzzcheck-asan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-a9: FUZZDB-check fuzzcheck-asan$(T.exe) ./fuzzcheck-asan$(T.exe) --slice 9 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u0 -run-fuzzcheck-u0: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u0: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 0 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u1 -run-fuzzcheck-u1: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u1: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 1 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u2 -run-fuzzcheck-u2: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u2: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 2 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u3 -run-fuzzcheck-u3: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u3: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 3 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u4 -run-fuzzcheck-u4: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u4: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 4 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u5 -run-fuzzcheck-u5: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u5: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 5 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u6 -run-fuzzcheck-u6: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u6: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 6 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u7 -run-fuzzcheck-u7: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u7: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 7 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u8 -run-fuzzcheck-u8: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u8: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 8 10 $(FUZZDB) run-fuzzcheck: run-fuzzcheck-u9 -run-fuzzcheck-u9: fuzzcheck-ubsan$(T.exe) - $(FUZZDB_CHECK) +run-fuzzcheck-u9: FUZZDB-check fuzzcheck-ubsan$(T.exe) ./fuzzcheck-ubsan$(T.exe) --slice 9 10 $(FUZZDB) diff --git a/manifest b/manifest index df636785e9..c88b7cd65c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C General\supdates\sto\sautosetup/README.md. -D 2025-03-18T11:36:15.450 +C Very\sslight\ssimplification\sof\sthe\srun-fuzzcheck\srules. +D 2025-03-18T12:31:09.432 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -707,7 +707,7 @@ F ext/wasm/tests/opfs/sahpool/sahpool-pausing.js f264925cfc82155de38cecb3d204c36 F ext/wasm/tests/opfs/sahpool/sahpool-worker.js bd25a43fc2ab2d1bafd8f2854ad3943ef673f7c3be03e95ecf1612ff6e8e2a61 F ext/wasm/wasmfs.make 68999f5bd8c489239592d59a420f8c627c99169bbd6fa16a404751f757b9f702 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk 11b4da311dedf3bf0301999b307c117fe1b5e4fdc260cb35105d235db0fec13b +F main.mk 7006135b902177f7c8bca2733a6820b72a0c6f5a47412b2a7c86668f0398f1fd F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2215,8 +2215,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9300f7f42dfd143f77fd51aa9e080099540854d36b6997ab1a16be7d77f78d8e -R a74ff5093dd14720da044ecaa9143a8b +P be8ad5cf579662c54b7e7bc492d8ca03b1f9032c5dad9a6b20590ca1ac00840b +R a4bbfb3e7a3979ba53919b4ea1630c0c U stephan -Z 7470894fe299edfa712004a6589105cc +Z 147fd9bb2dc3e368a79a71e178280d71 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 456e6406f4..73ba648652 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be8ad5cf579662c54b7e7bc492d8ca03b1f9032c5dad9a6b20590ca1ac00840b +c858a39fad30c46aec6a1f81b2d4e56c18ecf7f5cb6d2fe4a32c4b3bb1a6ed64 From e435547beb9ffc3f88670e218248801f96956ccc Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 18 Mar 2025 13:52:53 +0000 Subject: [PATCH 36/38] Internal doc touchups in ext/wasm/mkwasmbuilds.c. No functional changes. FossilOrigin-Name: 47d34260e74912eeae704bff7c4314b893af86ee66dd96a1bc6f450d3e290702 --- ext/wasm/mkwasmbuilds.c | 7 ++++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ext/wasm/mkwasmbuilds.c b/ext/wasm/mkwasmbuilds.c index e3cd34b00d..d13302769e 100644 --- a/ext/wasm/mkwasmbuilds.c +++ b/ext/wasm/mkwasmbuilds.c @@ -45,6 +45,7 @@ ** "sqlite3-wasmfs" build, only "esm" (ES6 Module) is legal. */ #define JS_BUILD_MODES vanilla esm bundler-friendly node +/* Separator to help eyeballs find the different sections */ static const char * zBanner = "\n########################################################################\n"; @@ -140,8 +141,8 @@ static void mk_prologue(void){ ** mk_lib_mode(). ** ** Maintenance reminder: do not combine flags within this enum, -** e.g. LIBMODE_BUNDLER_FRIEND=0x02|LIBMODE_ESM, as that will lead to -** breakage in some of the flag checks. +** e.g. LIBMODE_BUNDLER_FRIENDLY=0x02|LIBMODE_ESM, as that will lead +** to breakage in some of the flag checks. */ enum LibModeFlags { /* Indicates an ESM module build. */ @@ -208,7 +209,7 @@ static void mk_pre_post(const char *zName /* build name */, pf("$(eval $(call SQLITE.CALL.C-PP.FILTER,$(extern-post-js.js.in),$(extern-post-js.js.%s-%s)," "$(c-pp.D.%s-%s)))\n", zNM, zNM); - /* Combine flags for use with emcc... */ + /* Combined flags for use with emcc... */ pf("pre-post-common.flags.%s-%s := " "$(pre-post-common.flags) " "--post-js=$(post-js.js.%s-%s) " diff --git a/manifest b/manifest index c88b7cd65c..83f99df4a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Very\sslight\ssimplification\sof\sthe\srun-fuzzcheck\srules. -D 2025-03-18T12:31:09.432 +C Internal\sdoc\stouchups\sin\sext/wasm/mkwasmbuilds.c.\sNo\sfunctional\schanges. +D 2025-03-18T13:52:53.910 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -680,7 +680,7 @@ F ext/wasm/index-dist.html 56132399702b15d70c474c3f1952541e25cb0922942868f70daf1 F ext/wasm/index.html bcaa00eca521b372a6a62c7e7b17a870b0fcdf3e418a5921df1fd61e5344080d F ext/wasm/jaccwabyt/jaccwabyt.js 6e4f26d0edb5c2e7d381b7eff1924832a040a12274afab2d1e1789027e9f6c5c F ext/wasm/jaccwabyt/jaccwabyt.md 1128e3563e7eff90b5a373395251fc76cb32386fad1fea6075b0f34a8f1b9bdf -F ext/wasm/mkwasmbuilds.c 082b7372db68c2d4cd9f55e7cde8eb1b83e9569e520984e6b08cb62606f3bf38 +F ext/wasm/mkwasmbuilds.c 6e0b22002bc520b7af053681571a96d30049a51f7f1389e81c524e8d420f5d40 F ext/wasm/module-symbols.html dc476b403369b26a1a23773e13b80f41b9a49f0825e81435fe3600a7cfbbe337 F ext/wasm/scratchpad-wasmfs.html a3d7388f3c4b263676b58b526846e9d02dfcb4014ff29d3a5040935286af5b96 F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd842231505895eff00dbd57c63 @@ -2215,8 +2215,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P be8ad5cf579662c54b7e7bc492d8ca03b1f9032c5dad9a6b20590ca1ac00840b -R a4bbfb3e7a3979ba53919b4ea1630c0c +P c858a39fad30c46aec6a1f81b2d4e56c18ecf7f5cb6d2fe4a32c4b3bb1a6ed64 +R ea53eb835c0e9a65950d5e9bbfcf794b U stephan -Z 147fd9bb2dc3e368a79a71e178280d71 +Z 336f18ce7773ad9574acc5a45857cc6c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 73ba648652..4147b282f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c858a39fad30c46aec6a1f81b2d4e56c18ecf7f5cb6d2fe4a32c4b3bb1a6ed64 +47d34260e74912eeae704bff7c4314b893af86ee66dd96a1bc6f450d3e290702 From 31fd886576455c071efd9d7b939fa7225ec9a6ec Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 18 Mar 2025 19:21:04 +0000 Subject: [PATCH 37/38] Fix a problem that could occur when the RHS of an IN operator was a compound SELECT featuring an ORDER BY on a subquery that was flattened into one of the component SELECTs introduced by [baa83b460c677c21]. Forum post [/forumpost/1e17219c88]. FossilOrigin-Name: 7101ccd5331e36fd1a539f540e79ce0ce159be76ec422e1d9436eec6f3908c6e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 1 + test/bloom1.test | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 83f99df4a9..7345476bdf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Internal\sdoc\stouchups\sin\sext/wasm/mkwasmbuilds.c.\sNo\sfunctional\schanges. -D 2025-03-18T13:52:53.910 +C Fix\sa\sproblem\sthat\scould\soccur\swhen\sthe\sRHS\sof\san\sIN\soperator\swas\sa\scompound\sSELECT\sfeaturing\san\sORDER\sBY\son\sa\ssubquery\sthat\swas\sflattened\sinto\sone\sof\sthe\scomponent\sSELECTs\sintroduced\sby\s[baa83b460c677c21].\sForum\spost\s[/forumpost/1e17219c88]. +D 2025-03-18T19:21:04.551 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -783,7 +783,7 @@ F src/printf.c 33fc0d7643c848a098afdcb6e1db6de12379d47084b1cd0912cfce1d09345e44 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 20e1fbe8f840ffc0cd835e33f68a802a22e34faa918d7a269f3de242fda02f99 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 -F src/select.c 8c273248e38a8a7286abe3f41c1931cc65eff602fef128acc0cc0484e1b7abb3 +F src/select.c bfe14cdfceba54744b1c6c29099313f5173a0793dfaff0cd484774e9d05dbeab F src/shell.c.in 248050551cad788f8bb4b4728e00d8e36a10130d2d101e55cd51cfee03df91ff F src/sqlite.h.in fd70afd92948cf7cc93f687ac960bad1b0b6fbc436752419eff2fd65a1809380 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -975,7 +975,7 @@ F test/bind2.test 918bc35135f4141809ead7585909cde57d44db90a7a62aef540127148f91aa F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 F test/blob.test e7ac6c7d3a985cc4678c64f325292529a69ae252 -F test/bloom1.test cf613a27054bbaf61c5bfc440a5cfd3ff76798d0695f3fc5e5d1bbc819b8dab1 +F test/bloom1.test 04f3a17df8912bfdc292c41b59d79f93893fe69799f3089a64451f9112f9658f F test/boundary1.tcl 6421b2d920d8b09539503a8673339d32f7609eb1 F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb @@ -2215,8 +2215,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c858a39fad30c46aec6a1f81b2d4e56c18ecf7f5cb6d2fe4a32c4b3bb1a6ed64 -R ea53eb835c0e9a65950d5e9bbfcf794b -U stephan -Z 336f18ce7773ad9574acc5a45857cc6c +P 47d34260e74912eeae704bff7c4314b893af86ee66dd96a1bc6f450d3e290702 +R 4ad094e1728498f359eaedd7a6999953 +U dan +Z 7c0745034fb8db0f32cdbdfb739169c4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4147b282f9..14b3bf9e1c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47d34260e74912eeae704bff7c4314b893af86ee66dd96a1bc6f450d3e290702 +7101ccd5331e36fd1a539f540e79ce0ce159be76ec422e1d9436eec6f3908c6e diff --git a/src/select.c b/src/select.c index eedbd82818..b2f2cc7fb8 100644 --- a/src/select.c +++ b/src/select.c @@ -3247,6 +3247,7 @@ static int multiSelect( multi_select_end: pDest->iSdst = dest.iSdst; pDest->nSdst = dest.nSdst; + pDest->iSDParm2 = dest.iSDParm2; if( pDelete ){ sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete); } diff --git a/test/bloom1.test b/test/bloom1.test index 151f364ae0..f8efcc1846 100644 --- a/test/bloom1.test +++ b/test/bloom1.test @@ -183,6 +183,47 @@ do_execsql_test 4.3 { do_execsql_test 4.4 { SELECT * FROM t0 LEFT JOIN t1 LEFT JOIN t2 ON (b NOTNULL)==(c IN ()) WHERE c; } {xyz {} 7.0} + +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1 (c1); + INSERT INTO t1 VALUES (101); + CREATE TABLE t2 ( x ); + INSERT INTO t2 VALUES(404); +} + +do_execsql_test 5.1 { + SELECT 'val' in ( + select 'val' from ( select 'valueB' from t1 order by 1 ) + union all + select 'val' + ); +} {1} + +do_execsql_test 5.2 { + select * from t2 + where 'val' in ( + select 'val' from ( select 'valueB' from t1 order by 1 ) + union all + select 'val' + ); +} {404} + +do_execsql_test 5.3 { + SELECT subq_1.c_0 as c_0 + FROM ( SELECT 0 as c_0) as subq_1 + WHERE (subq_1.c_0) IN ( + SELECT subq_2.c_0 as c_0 + FROM ( + SELECT 0 as c_0 + FROM t1 as ref_1 + WHERE (ref_1.c1) = (2) + ORDER BY c_0 desc + ) as subq_2 + UNION ALL + SELECT 0 as c_0 + ); +} {0} finish_test From 8db881d0550f664fb4f32a088818553d5fef27e8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 18 Mar 2025 20:15:16 +0000 Subject: [PATCH 38/38] Change the generate_series() table-valued function so that its rowid is just an alias for its value. This allows it to be used as the RHS operand of a RIGHT JOIN. This fixes the issue raised by [forum:/forumpost/1e17219c88|forum post 1e17219c88]. FossilOrigin-Name: 77db4d85e70fbf358ae2321c2601966666bdb4d971d7c113ce30a3e541458ee8 --- ext/misc/series.c | 15 ++++++++------- manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/shell2.test | 12 ++++++------ test/tabfunc01.test | 17 ++++++++++++++--- 5 files changed, 38 insertions(+), 26 deletions(-) diff --git a/ext/misc/series.c b/ext/misc/series.c index 04644dd603..e8188f1c97 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -60,8 +60,7 @@ ** step HIDDEN ** ); ** -** The virtual table also has a rowid, logically equivalent to n+1 where -** "n" is the ascending integer in the aforesaid production definition. +** The virtual table also has a rowid which is an alias for the value. ** ** Function arguments in queries against this virtual table are translated ** into equality constraints against successive hidden columns. In other @@ -276,6 +275,7 @@ static int seriesConnect( int rc; /* Column numbers */ +#define SERIES_COLUMN_ROWID (-1) #define SERIES_COLUMN_VALUE 0 #define SERIES_COLUMN_START 1 #define SERIES_COLUMN_STOP 2 @@ -363,13 +363,11 @@ static int seriesColumn( #endif /* -** Return the rowid for the current row, logically equivalent to n+1 where -** "n" is the ascending integer in the aforesaid production definition. +** The rowid is the same as the value. */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - sqlite3_uint64 n = pCur->ss.uSeqIndexNow; - *pRowid = (sqlite3_int64)((nss.iValueNow; return SQLITE_OK; } @@ -657,7 +655,10 @@ static int seriesBestIndex( continue; } if( pConstraint->iColumniColumn==SERIES_COLUMN_VALUE && pConstraint->usable ){ + if( (pConstraint->iColumn==SERIES_COLUMN_VALUE || + pConstraint->iColumn==SERIES_COLUMN_ROWID) + && pConstraint->usable + ){ switch( op ){ case SQLITE_INDEX_CONSTRAINT_EQ: case SQLITE_INDEX_CONSTRAINT_IS: { diff --git a/manifest b/manifest index 7345476bdf..59199b7f83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sthat\scould\soccur\swhen\sthe\sRHS\sof\san\sIN\soperator\swas\sa\scompound\sSELECT\sfeaturing\san\sORDER\sBY\son\sa\ssubquery\sthat\swas\sflattened\sinto\sone\sof\sthe\scomponent\sSELECTs\sintroduced\sby\s[baa83b460c677c21].\sForum\spost\s[/forumpost/1e17219c88]. -D 2025-03-18T19:21:04.551 +C Change\sthe\sgenerate_series()\stable-valued\sfunction\sso\sthat\sits\srowid\sis\sjust\san\nalias\sfor\sits\svalue.\s\sThis\sallows\sit\sto\sbe\sused\sas\sthe\sRHS\soperand\sof\sa\nRIGHT\sJOIN.\s\sThis\sfixes\sthe\sissue\sraised\sby\n[forum:/forumpost/1e17219c88|forum\spost\s1e17219c88]. +D 2025-03-18T20:15:16.250 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -436,7 +436,7 @@ F ext/misc/regexp.c 388e7f237307c7dfbfb8dde44e097946f6c437801d63f0d7ad63f3320d4e F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c 69e0d2b5d193c67bdfee5baf79c10ec24a61a40212c7ca9c219edf7afa24305f +F ext/misc/series.c 076a4c85dde2ae543d040f1080cdab74ebf3da7f3febfe38e0cd45a2217498bf F ext/misc/sha1.c cb5002148c2661b5946f34561701e9105e9d339b713ec8ac057fd888b196dcb9 F ext/misc/shathree.c fd22d70620f86a0467acfdd3acd8435d5cb54eb1e2d9ff36ae44e389826993df F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 @@ -1641,7 +1641,7 @@ F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8e F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test 573942b8d0e444956445993d5a5275c6912bc49b654441eec0b5e1e735f2e5b7 -F test/shell2.test 01a01f76ed98088ce598794fbf5b359e148271541a8ddbf79d21cc353cc67a24 +F test/shell2.test ac102ebc0a9ec166257600c4ee8bdefec242163afced295f10b004f4af3fc9dd F test/shell3.test db1953a8e59d08e9240b7cc5948878e184f7eb2623591587f8fd1f1a5bd536d8 F test/shell4.test 522fdc628c55eff697b061504fb0a9e4e6dfc5d9087a633ab0f3dd11bcc4f807 F test/shell5.test 0e5f8ce08206b9998a778cfe1989e20e47839153c05af2da29198150172e22fc @@ -1719,7 +1719,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 66d1ea27289c19317bf111744a4d5fda901759945d22ddb5bc964641fc38c185 +F test/tabfunc01.test 76da0509b01b9d12f4e71f8af28ee702d38166a5306bd77a955fb1a370dcd8b5 F test/table.test 7862a00b58b5541511a26757ea9c5c7c3f8298766e98aa099deec703d9c0a8e0 F test/tableapi.test e37c33e6be2276e3a96bb54b00eea7f321277115d10e5b30fdb52a112b432750 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -2215,8 +2215,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 47d34260e74912eeae704bff7c4314b893af86ee66dd96a1bc6f450d3e290702 -R 4ad094e1728498f359eaedd7a6999953 -U dan -Z 7c0745034fb8db0f32cdbdfb739169c4 +P 7101ccd5331e36fd1a539f540e79ce0ce159be76ec422e1d9436eec6f3908c6e +R db7ca198487b77ea1364073bfe0ca044 +U drh +Z c0d28f545eced4435e36308f7af9950b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 14b3bf9e1c..b6decbb386 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7101ccd5331e36fd1a539f540e79ce0ce159be76ec422e1d9436eec6f3908c6e +77db4d85e70fbf358ae2321c2601966666bdb4d971d7c113ce30a3e541458ee8 diff --git a/test/shell2.test b/test/shell2.test index ee5ae4bdd9..3f9fec9efa 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -224,24 +224,24 @@ do_test shell2-1.4.10 { set res [catchcmd :memory: [string trim { SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); - SELECT avg(rowid),min(value),max(value) FROM generate_series( + SELECT avg(value),min(value),max(value) FROM generate_series( -9223372036854775808,9223372036854775807,1085102592571150095); SELECT * FROM generate_series(-9223372036854775808,9223372036854775807, 9223372036854775807); - SELECT value,rowid FROM generate_series(-4611686018427387904, + SELECT value FROM generate_series(-4611686018427387904, 4611686018427387904, 4611686018427387904) ORDER BY value DESC; SELECT * FROM generate_series(0,-2,-1); SELECT * FROM generate_series(0,-2); SELECT * FROM generate_series(0,2) LIMIT 3;}]] } {0 {9223372036854775807 9223372036854775807 -9.5|-9223372036854775808|9223372036854775807 +-0.5|-9223372036854775808|9223372036854775807 -9223372036854775808 -1 9223372036854775806 -4611686018427387904|3 -0|2 --4611686018427387904|1 +4611686018427387904 +0 +-4611686018427387904 0 -1 -2 diff --git a/test/tabfunc01.test b/test/tabfunc01.test index cbf9865eda..c16fb9bec2 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -61,10 +61,10 @@ do_execsql_test tabfunc01-1.8 { } {30 25 20 15 10 5 0} do_execsql_test tabfunc01-1.9 { SELECT rowid, * FROM generate_series(0,32,5) ORDER BY value DESC; -} {7 30 6 25 5 20 4 15 3 10 2 5 1 0} +} {30 30 25 25 20 20 15 15 10 10 5 5 0 0} do_execsql_test tabfunc01-1.10 { SELECT rowid, * FROM generate_series(0,32,5) ORDER BY +value DESC; -} {7 30 6 25 5 20 4 15 3 10 2 5 1 0} +} {30 30 25 25 20 20 15 15 10 10 5 5 0 0} do_execsql_test tabfunc01-1.20 { CREATE VIEW v1(a,b) AS VALUES(1,2),(3,4); @@ -383,7 +383,18 @@ do_execsql_test 1100 { where (ref_3.value) in (select 1); } {1} - +# 2025-03-18 /forumpost/1e17219c88 +# The generate_series() table-valued function is modified so that its +# rowid is always its value. That way it can be used on the RHS of a +# RIGHT JOIN. +# +do_execsql_test 1200 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(value INT); + INSERT INTO t1 VALUES (1),(2),(3); + SELECT t1.value, t2.value + FROM t1 RIGHT JOIN generate_series(1,3,1) AS t2 USING(value); +} {1 1 2 2 3 3} # Free up memory allocations intarray_addr