From dcc121999cb458e36b5aef9808f856a734041622 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Mar 2024 11:14:52 +0000 Subject: [PATCH 01/15] Avoid expanding integer values in columns with real affinity to the full 8-byte representation when editing records as part of a DROP COLUMN command. FossilOrigin-Name: a49296de0061931badaf3db6b965131a78b1c6c21b1eeb62815ea7adf767d0b3 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/alter.c | 5 +++++ test/altertab3.test | 23 +++++++++++++++++++++++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 24f7bb9fb9..ea43bdf8f8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sRAISE()\soperator\sis\snot\sa\sconstant\sexpression\sand\scannot\sparticipate\sin\nthe\sVALUE-as-coroutine\soptimization.\ndbsqlfuzz\s74cf7c9904360322a6c917e4934b127543d1cd51 -D 2024-03-25T20:35:14.251 +C Avoid\sexpanding\sinteger\svalues\sin\scolumns\swith\sreal\saffinity\sto\sthe\sfull\s8-byte\srepresentation\swhen\sediting\srecords\sas\spart\sof\sa\sDROP\sCOLUMN\scommand. +D 2024-03-26T11:14:52.242 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -682,7 +682,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 -F src/alter.c 30c2333b8bb3af71e4eb9adeadee8aa20edb15917ed44b8422e5cd15f3dfcddc +F src/alter.c e1b6782b85dd758f89e5c588e4e3eb82638c2dafc0c857b79a43bb8ec1746fca F src/analyze.c a3df28274e2565ba5656577d7e3fd262169a213e6eb0bd47890e0f0729a4031c F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 @@ -867,7 +867,7 @@ F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd4 F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 F test/altertab2.test 62597b6fd08feaba1b6bfe7d31dac6117c67e06dc9ce9c478a3abe75b5926de0 -F test/altertab3.test e167ce3b8e243b52306c1e40b13eb868f402a969513c063998593862cc643b44 +F test/altertab3.test b331ae34e69594e19605e3297805202d6156fcc8f75379dfd972a2e51cae8721 F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b @@ -2183,8 +2183,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f3c4433f6b33087494c8d38ff9dbb008a1bd30b424ca9aaa51c1956d75a86249 -R 11afa0c0cb35ff6b57f28ca49a3f27d4 -U drh -Z 38f01fde0dacda37a79ba00b3f8d1e29 +P 6a06dc73847716c88d65651d1bf0e002002303881df1389beac884d0032eae08 +R 9f7cc1bf5356d77f08ae122347a7b675 +U dan +Z 96694e9dd3b3c0e4acdff2534541b848 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 04e087afeb..8c514c694f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a06dc73847716c88d65651d1bf0e002002303881df1389beac884d0032eae08 \ No newline at end of file +a49296de0061931badaf3db6b965131a78b1c6c21b1eeb62815ea7adf767d0b3 \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index ec45e14331..c1e0a295a4 100644 --- a/src/alter.c +++ b/src/alter.c @@ -2262,7 +2262,12 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); }else{ + char aff = pTab->aCol[i].affinity; + if( aff==SQLITE_AFF_REAL ){ + pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; + } sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + pTab->aCol[i].affinity = aff; } nField++; } diff --git a/test/altertab3.test b/test/altertab3.test index cb6f00de48..5f5c11b0bb 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -763,4 +763,27 @@ do_execsql_test 30.2 { )} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 31.0 { + CREATE TABLE t1(ii INTEGER PRIMARY KEY, tt INTEGER, rr REAL); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000 + ) + INSERT INTO t1 SELECT NULL, i, 5.0 FROM s; +} + +do_test 31.1 { + set pg [db one {PRAGMA page_count}] + execsql { + ALTER TABLE t1 DROP COLUMN tt; + } + set pg2 [db one {PRAGMA page_count}] + expr $pg==$pg2 +} {1} + +do_execsql_test 31.2 { + SELECT rr FROM t1 LIMIT 1 +} {5.0} + finish_test From ed517a708284b6e00b6ae5f1e3f702bbfcbd32ed Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Mar 2024 18:07:39 +0000 Subject: [PATCH 02/15] Add extra tests to vtabL.test. FossilOrigin-Name: fe209099f5c348f1280b9b827ffbf6a6742ebdd1a23ef189445147894b1832ba --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/vtabL.test | 21 +++++++++++++++++---- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ea43bdf8f8..8b6dfb5fc7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sexpanding\sinteger\svalues\sin\scolumns\swith\sreal\saffinity\sto\sthe\sfull\s8-byte\srepresentation\swhen\sediting\srecords\sas\spart\sof\sa\sDROP\sCOLUMN\scommand. -D 2024-03-26T11:14:52.242 +C Add\sextra\stests\sto\svtabL.test. +D 2024-03-26T18:07:39.608 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1951,7 +1951,7 @@ F test/vtabH.test 2efb5a24b0bb50796b21eca23032cfb77abfa4b0c03938e38ce5897abac404 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtabJ.test a6aef49d558af90fae10565b29501f82a95781cb4f797f2d13e2d19f9b6bc77b F test/vtabK.test 13293177528fada1235c0112db0d187d754af1355c5a39371abd365104e3afbf -F test/vtabL.test 040b9f782a3b41844f2a5c660e6a0bfd298492980c6b4e126d82113c9785cec3 +F test/vtabL.test 49ec7342e8bfcb0d6c3d2443c619f430c609c042d5d7e6ddf52def65d6d1812f F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad @@ -2183,8 +2183,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a06dc73847716c88d65651d1bf0e002002303881df1389beac884d0032eae08 -R 9f7cc1bf5356d77f08ae122347a7b675 +P a49296de0061931badaf3db6b965131a78b1c6c21b1eeb62815ea7adf767d0b3 +R 7ff7efd995047005d6aa8ecbdebb1507 U dan -Z 96694e9dd3b3c0e4acdff2534541b848 +Z ff7145a12bdc41a2553dea79acaa6e0c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8c514c694f..46f1a1149d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a49296de0061931badaf3db6b965131a78b1c6c21b1eeb62815ea7adf767d0b3 \ No newline at end of file +fe209099f5c348f1280b9b827ffbf6a6742ebdd1a23ef189445147894b1832ba \ No newline at end of file diff --git a/test/vtabL.test b/test/vtabL.test index a801481c84..0834655ffd 100644 --- a/test/vtabL.test +++ b/test/vtabL.test @@ -31,10 +31,7 @@ proc vtab_command {method args} { return {} } -do_execsql_test 1.0 { - CREATE TABLE t1(a, b); -} - +breakpoint foreach {tn cts} { 1 {SELECT 123} 2 {SELECT 123, 456} @@ -44,6 +41,8 @@ foreach {tn cts} { 6 {DROP TABLE nosuchtable} 7 {DROP TABLE x1} 8 {DROP TABLE t1} + 9 {CREATE TABLE xyz AS SELECT * FROM sqlite_schema} + 10 {CREATE TABLE xyz AS SELECT 1 AS 'col'} } { set ::create_table_sql $cts do_catchsql_test 1.$tn { @@ -51,5 +50,19 @@ foreach {tn cts} { } {1 {vtable constructor failed: x1}} } +foreach {tn cts} { + 1 {CREATE TABLE IF NOT EXISTS t1(a, b)} + 2 {CREATE TABLE ""(a, b PRIMARY KEY) WITHOUT ROWID} +} { + set ::create_table_sql $cts + execsql { DROP TABLE IF EXISTS x1 } + do_execsql_test 2.$tn.1 { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command); + } + do_execsql_test 2.$tn.2 { + SELECT a, b FROM x1 + } +} + finish_test From 6921a2d2054dcbf26c5f455dded278fac795ea5e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 27 Mar 2024 20:34:14 +0000 Subject: [PATCH 03/15] Update comments in fts5.h. FossilOrigin-Name: 862945d5c432c27377e90d93d64c4655eefcc369d086eb51edef925fb3d80b57 --- ext/fts5/fts5.h | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 250d2ee7e3..551618e718 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -55,8 +55,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken diff --git a/manifest b/manifest index 8b6dfb5fc7..0580f02c1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sto\svtabL.test. -D 2024-03-26T18:07:39.608 +C Update\scomments\sin\sfts5.h. +D 2024-03-27T20:34:14.698 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h ecba24fed7b359b3a53016bb07e411b3b4c9cdf163aa141006536423a63b611e +F ext/fts5/fts5.h 8856e11a5f0269cd346754cea0765efe8089635b80cad3222e8bfdb08cd5348a F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834 F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 @@ -2183,8 +2183,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a49296de0061931badaf3db6b965131a78b1c6c21b1eeb62815ea7adf767d0b3 -R 7ff7efd995047005d6aa8ecbdebb1507 +P fe209099f5c348f1280b9b827ffbf6a6742ebdd1a23ef189445147894b1832ba +R c56763aed4d4e5080faa164bf6334faf U dan -Z ff7145a12bdc41a2553dea79acaa6e0c +Z d4f93415501168b234b57e2165e695ab # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 46f1a1149d..1b105c4bee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe209099f5c348f1280b9b827ffbf6a6742ebdd1a23ef189445147894b1832ba \ No newline at end of file +862945d5c432c27377e90d93d64c4655eefcc369d086eb51edef925fb3d80b57 \ No newline at end of file From cfd4507c830a73dad165ec5db2ae28549f62ae9f Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 28 Mar 2024 10:58:18 +0000 Subject: [PATCH 04/15] Document that the order of an update hook call is unspecied vis-a-vis the final result of the operation which triggers that hook. Doc changes only. FossilOrigin-Name: 3d4b1f0791384d3e531d6757daecf67e5b873954de61f37032474e3ae23cd22b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 6 ++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0580f02c1d..335dcd41b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\scomments\sin\sfts5.h. -D 2024-03-27T20:34:14.698 +C Document\sthat\sthe\sorder\sof\san\supdate\shook\scall\sis\sunspecied\svis-a-vis\sthe\sfinal\sresult\sof\sthe\soperation\swhich\striggers\sthat\shook.\sDoc\schanges\sonly. +D 2024-03-28T10:58:18.871 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -752,7 +752,7 @@ F src/resolve.c eb1860b134fb044fd819c4347105c148d5aac7c6498032be2829e5cc95619b28 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 15a221347789e393b39e7d2d2bd102167979c95a1ce0675bb870b86a24ca6cf4 F src/shell.c.in 0354ca51eee5fbf6af394a7ef9f5ef6823ef45b743db65431f6777e4d5be2199 -F src/sqlite.h.in e9af5761aab316d52e5a5ac11b42d6a25eaaa49352e982c14f1080aab684b5c4 +F src/sqlite.h.in b5fb41b730da29d7cec58c6020afba627baf58a3dca4e30814bd674bc2a6fc86 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 F src/sqliteInt.h f8928f0397d797046396dd9d360a6af8ce6dcb48bd72ea290165b07c8c518744 @@ -2183,8 +2183,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fe209099f5c348f1280b9b827ffbf6a6742ebdd1a23ef189445147894b1832ba -R c56763aed4d4e5080faa164bf6334faf -U dan -Z d4f93415501168b234b57e2165e695ab +P 862945d5c432c27377e90d93d64c4655eefcc369d086eb51edef925fb3d80b57 +R c8b4c3bc800aa223f9c43a96ed885105 +U stephan +Z 8903ace573828f0152a9ba1563932171 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1b105c4bee..8763970a63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -862945d5c432c27377e90d93d64c4655eefcc369d086eb51edef925fb3d80b57 \ No newline at end of file +3d4b1f0791384d3e531d6757daecf67e5b873954de61f37032474e3ae23cd22b \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 6841cf38f7..0a020d691c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6887,6 +6887,12 @@ int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the From 69dbd7a4e772e5f5a59409c45f29dbb038a06adc Mon Sep 17 00:00:00 2001 From: stephan Date: Sat, 30 Mar 2024 14:11:30 +0000 Subject: [PATCH 05/15] Make explicit that sqlite3_keyword_name()'s index is 0-based, per forum request. Doc changes only. FossilOrigin-Name: 090943dc31e7a3af5c11c1c0953cb82ae3ca07ba000189bb85deaecc76921504 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 335dcd41b5..1e6bf524c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Document\sthat\sthe\sorder\sof\san\supdate\shook\scall\sis\sunspecied\svis-a-vis\sthe\sfinal\sresult\sof\sthe\soperation\swhich\striggers\sthat\shook.\sDoc\schanges\sonly. -D 2024-03-28T10:58:18.871 +C Make\sexplicit\sthat\ssqlite3_keyword_name()'s\sindex\sis\s0-based,\sper\sforum\srequest.\sDoc\schanges\sonly. +D 2024-03-30T14:11:30.350 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -752,7 +752,7 @@ F src/resolve.c eb1860b134fb044fd819c4347105c148d5aac7c6498032be2829e5cc95619b28 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 15a221347789e393b39e7d2d2bd102167979c95a1ce0675bb870b86a24ca6cf4 F src/shell.c.in 0354ca51eee5fbf6af394a7ef9f5ef6823ef45b743db65431f6777e4d5be2199 -F src/sqlite.h.in b5fb41b730da29d7cec58c6020afba627baf58a3dca4e30814bd674bc2a6fc86 +F src/sqlite.h.in 32389e0d584551b300d0157881336162c14315a424cbf385c0d65eb7c2e31f7b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 F src/sqliteInt.h f8928f0397d797046396dd9d360a6af8ce6dcb48bd72ea290165b07c8c518744 @@ -2183,8 +2183,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 862945d5c432c27377e90d93d64c4655eefcc369d086eb51edef925fb3d80b57 -R c8b4c3bc800aa223f9c43a96ed885105 +P 3d4b1f0791384d3e531d6757daecf67e5b873954de61f37032474e3ae23cd22b +R 1ec9bcb0364e01dba28a12b65d50cd52 U stephan -Z 8903ace573828f0152a9ba1563932171 +Z 8fd3b1df728a4b7485ff966dda79d536 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8763970a63..a86194fdb9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3d4b1f0791384d3e531d6757daecf67e5b873954de61f37032474e3ae23cd22b \ No newline at end of file +090943dc31e7a3af5c11c1c0953cb82ae3ca07ba000189bb85deaecc76921504 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0a020d691c..d553abb7bf 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8363,7 +8363,7 @@ int sqlite3_test_control(int op, ...); ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns From 9352cfa30b33bc88a422a05564616b5397d54a97 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 1 Apr 2024 15:38:15 +0000 Subject: [PATCH 06/15] Improved comments in the query planner logic that computes the cost for a particular step in a query plan. No code changes. FossilOrigin-Name: 0b2ac2cdc767db764e3ea8bbc33898cac4e1ec27fe8c9b60ce08a1785f921e6d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 21 ++++++++++++++++----- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 1e6bf524c1..4939830a71 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sexplicit\sthat\ssqlite3_keyword_name()'s\sindex\sis\s0-based,\sper\sforum\srequest.\sDoc\schanges\sonly. -D 2024-03-30T14:11:30.350 +C Improved\scomments\sin\sthe\squery\splanner\slogic\sthat\scomputes\sthe\scost\sfor\sa\nparticular\sstep\sin\sa\squery\splan.\s\sNo\scode\schanges. +D 2024-04-01T15:38:15.301 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -835,7 +835,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 11a67988c3a5be2e7ee07c958998f3ad4b30dc491c0add894c2933c752415b16 +F src/where.c a15429ae9d42c7b824b9faad7e8234b006abf0a4526166dc4dfc0cc9cd2c9e6a F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 @@ -2183,8 +2183,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3d4b1f0791384d3e531d6757daecf67e5b873954de61f37032474e3ae23cd22b -R 1ec9bcb0364e01dba28a12b65d50cd52 -U stephan -Z 8fd3b1df728a4b7485ff966dda79d536 +P 090943dc31e7a3af5c11c1c0953cb82ae3ca07ba000189bb85deaecc76921504 +R 27a9be3de32bd888ad6632e320231186 +U drh +Z 13460dee49d687f64f99bd37193c204d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a86194fdb9..3fb4e018f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -090943dc31e7a3af5c11c1c0953cb82ae3ca07ba000189bb85deaecc76921504 \ No newline at end of file +0b2ac2cdc767db764e3ea8bbc33898cac4e1ec27fe8c9b60ce08a1785f921e6d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 5767b44128..cea134d0e2 100644 --- a/src/where.c +++ b/src/where.c @@ -3252,10 +3252,13 @@ static int whereLoopAddBtreeIndex( } } - /* Set rCostIdx to the cost of visiting selected rows in index. Add - ** it to pNew->rRun, which is currently set to the cost of the index - ** seek only. Then, if this is a non-covering index, add the cost of - ** visiting the rows in the main table. */ + /* Set rCostIdx to the estimated cost of visiting selected rows in the + ** index. The estimate is the sum of two values: + ** 1. The cost of doing one search-by-key to find the first matching + ** entry + ** 2. Stepping forward in the index pNew->nOut times to find all + ** additional matching entries. + */ assert( pSrc->pTab->szTabRow>0 ); if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* The pProbe->szIdxRow is low for an IPK table since the interior @@ -3266,7 +3269,15 @@ static int whereLoopAddBtreeIndex( }else{ rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; } - pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx); + + /* Estimate the cost of running the loop. If all data is coming + ** from the index, then this is just the cost of doing the index + ** lookup and scan. But if some data is coming out of the main table, + ** we also have to add in the cost of doing pNew->nOut searches to + ** locate the row in the main table that corresponds to the index entry. + */ + pNew->rRun = rCostIdx; if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } From 7f7d0b19f8d2628466d8e25bbb0f7d10bfb98baf Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 2 Apr 2024 11:44:44 +0000 Subject: [PATCH 07/15] Add a heuristic in between the two solver() passes of the query planner that tries to prevent a very slow query plan in cases where the output row count estimate is imprecise. FossilOrigin-Name: 8018417b0143ea11535f2457bf3e4b3755717c554a17df1076425b4251b5f2c6 --- manifest | 16 +++++--- manifest.uuid | 2 +- src/where.c | 64 ++++++++++++++++++++++++++++- test/whereN.test | 103 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 8 deletions(-) create mode 100644 test/whereN.test diff --git a/manifest b/manifest index 4939830a71..24a90e1c43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\sin\sthe\squery\splanner\slogic\sthat\scomputes\sthe\scost\sfor\sa\nparticular\sstep\sin\sa\squery\splan.\s\sNo\scode\schanges. -D 2024-04-01T15:38:15.301 +C Add\sa\sheuristic\sin\sbetween\sthe\stwo\ssolver()\spasses\sof\sthe\squery\splanner\sthat\ntries\sto\sprevent\sa\svery\sslow\squery\splan\sin\scases\swhere\sthe\soutput\srow\scount\nestimate\sis\simprecise. +D 2024-04-02T11:44:44.648 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -835,7 +835,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c a15429ae9d42c7b824b9faad7e8234b006abf0a4526166dc4dfc0cc9cd2c9e6a +F src/where.c ba9d22fda3850b508e075f8bdc4f4d4f1ff314551597a6f14842bb03aeec0d3a F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 @@ -2017,6 +2017,7 @@ F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08a F test/whereK.test 0270ab7f04ba5436fb9156d31d642a1c82727f4c4bfe5ba90d435c78cf44684a F test/whereL.test 438a397fa883b77bb6361c08a8befa41b52e9cfbe15a2a43715d122f8cfa8649 F test/whereM.test 0dbc9998783458ddcf3cc078ca7c2951d8b2677d472ecf0028f449ed327c0250 +F test/whereN.test 63a3584b71acfb6963416de82f26c6b1644abc5ca6080c76546b9246734c8803 F test/wherefault.test 6cf2a9c5712952d463d3f45ebee7f6caf400984df51a195d884cfb7eb0e837a7 F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3 F test/wherelimit.test afb46397c6d7e964e6e294ba3569864a0c570fe3807afc634236c2b752372f31 @@ -2183,8 +2184,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 090943dc31e7a3af5c11c1c0953cb82ae3ca07ba000189bb85deaecc76921504 -R 27a9be3de32bd888ad6632e320231186 +P 0b2ac2cdc767db764e3ea8bbc33898cac4e1ec27fe8c9b60ce08a1785f921e6d +R bbe24560206aebee12aa3740963fd4bf +T *branch * interstage-heuristic +T *sym-interstage-heuristic * +T -sym-trunk * U drh -Z 13460dee49d687f64f99bd37193c204d +Z f6898799f068ada043c459f74f460b9b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3fb4e018f6..4593d196ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b2ac2cdc767db764e3ea8bbc33898cac4e1ec27fe8c9b60ce08a1785f921e6d \ No newline at end of file +8018417b0143ea11535f2457bf3e4b3755717c554a17df1076425b4251b5f2c6 \ No newline at end of file diff --git a/src/where.c b/src/where.c index cea134d0e2..fe76f2c960 100644 --- a/src/where.c +++ b/src/where.c @@ -5530,7 +5530,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } } - pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ @@ -5538,6 +5537,68 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ return SQLITE_OK; } +/* +** This routine implements a heuristic designed to improve query planning. +** This routine is called in between the first and second call to +** wherePathSolver(). Hence the name "Interstage" "Heuristic". +** +** The first call to wherePathSolver() (hereafter just "solver()") computes +** the best path without regard to the order of the outputs. The second call +** to the solver() builds upon the first call to try to find an alternative +** path that satisfies the ORDER BY clause. +** +** This routine looks at the results of the first solver() run, and for +** every FROM clause term in the resulting query plan that uses an equality +** constraint against an index, disable other WhereLoops for that same +** FROM clause term that would try to do a full-table scan. This prevents +** an index search from being converted into a full-table scan in order to +** satisfy an ORDER BY clause, since even though we might get slightly better +** performance using the full-scan without sorting if the output size +** estimates are very precise, we might also get severe performance +** degradation using the full-scan if the output size estimate is too large. +** It is better to err on the side of caution. +** +** Except, if the first solver() call generated a full-table scan in an outer +** loop then stop this analysis at the first full-scan, since the second +** solver() run might try to swap that full-scan for another in order to +** get the output into the correct order. In other words, we do *not* want +** to inhibit a rewrites like this: +** +** First Solver() Second Solver() +** |-- SCAN t1 |-- SCAN t2 +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** Rather, the purpose of this routine is to inhibit rewrites such as: +** +** First Solver() Second Solver() +** |-- SEARCH t1 |-- SCAN t2 <--- bad! +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** See test cases in test/whereN.test for the real-world query that +** originally provoked this heuristic. +*/ +static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ + int i; + for(i=0; inLevel; i++){ + WhereLoop *p = pWInfo->a[i].pWLoop; + if( p==0 ) break; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue; + if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ + u8 iTab = p->iTab; + WhereLoop *pLoop; + for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){ + if( pLoop->iTab!=iTab ) continue; + if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 ) continue; + pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */ + } + }else{ + break; + } + } +} + /* ** Most queries use only a single table (they are not joins) and have ** simple == constraints against indexed fields. This routine attempts @@ -6320,6 +6381,7 @@ WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ + whereInterstageHeuristic(pWInfo); wherePathSolver(pWInfo, pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } diff --git a/test/whereN.test b/test/whereN.test new file mode 100644 index 0000000000..b9b889fa28 --- /dev/null +++ b/test/whereN.test @@ -0,0 +1,103 @@ +# 2024-04-02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Tests for the whereInterstageHeuristic() routine in the query planner. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix whereN + +# The following is a simplified and "sanitized" version of the original +# real-world query that brought the problem to light. +# +# The issue is a slow query. The answer is correct, but it was taking too +# much time, because it was doing a full table scan rather than an indexed +# lookup. +# +# The problem was that the query planner was overestimating the number of +# output rows. The estimated number of output rows is accurate if the +# DSNAME parameter is "ds-one". In that case, a large fraction of the rows +# in "violation" end up being output. The query planner correctly deduces +# that it is faster to do a full table scan of the large "violation" table +# to avoid the after-query sort that implements the ORDER BY clause. However, +# if the DSNAME is "ds-two", then only a few rows (about 6) are generated, +# and it is much much faster to do an indexed lookup of "violation" followed +# by a sort operation to implement ORDER BY +# +# The problem, of course, is that the query planner has no way of knowing +# in advance how many rows will be generated. The query planner tries to +# estimate a worst case, which is a large number of output rows, and it picks +# the best plan for that case. However, the plan choosen is very inefficient +# when the number of output rows is small. +# +# The whereInterstageHeuristic() routine in the query planner attempts to +# correct this by adjusting the query plan such that it avoids the very bad +# query plan for a small number of rows, at the expense of a slightly less +# efficient plan for a large number of rows. The large number of rows case +# is perhaps 5% slower with the revised plan, but the small number of +# rows case is around 100 times faster. That seems like a good tradeoff. +# +do_execsql_test 1.0 { + CREATE TABLE datasource(dsid INT, name TEXT); + INSERT INTO datasource VALUES(1,'ds-one'),(2,'ds-two'),(3,'ds-three'); + CREATE INDEX ds1 ON datasource(name, dsid); + + CREATE TABLE rule(rid INT, team_id INT, dsid INT); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<9) + INSERT INTO rule(rid,team_id,dsid) SELECT n, 1, 1 FROM c; + WITH RECURSIVE c(n) AS (VALUES(10) UNION ALL SELECT n+1 FROM c WHERE n<24) + INSERT INTO rule(rid,team_id,dsid) SELECT n, 2, 2 FROM c; + CREATE INDEX rule2 ON rule(dsid, rid); + + CREATE TABLE violation(vid INT, rid INT, vx BLOB); + /*** Uncomment to insert actual data + WITH src(rid, cnt) AS (VALUES(1,3586),(2,1343),(3,6505),(5,76230), + (6,740),(7,287794),(8,457),(12,1), + (14,1),(16,1),(17,1),(18,1),(19,1)) + INSERT INTO violation(vid, rid, vx) + SELECT rid*1000000+value, rid, randomblob(15) + FROM src, generate_series(1,cnt); + ***/ + CREATE INDEX v1 ON violation(rid, vid); + CREATE INDEX v2 ON violation(vid); + ANALYZE; + DELETE FROM sqlite_stat1; + DROP TABLE IF EXISTS sqlite_stat4; + INSERT INTO sqlite_stat1 VALUES + ('violation','v2','376661 1'), + ('violation','v1','376661 28974 1'), + ('rule','rule2','24 12 1'), + ('datasource','ds1','3 1 1'); + ANALYZE sqlite_schema; +} +set DSNAME ds-two ;# Only a few rows. Change to "ds-one" for many rows. +do_eqp_test 1.1 { + SELECT count(*), length(group_concat(vx)) FROM ( + SELECT V.* + FROM datasource DS, rule R, violation V + WHERE V.rid=R.rid + AND R.dsid=DS.dsid + AND DS.name=$DSNAME + ORDER BY V.vid desc + ); +} { + QUERY PLAN + |--CO-ROUTINE (subquery-xxxxxx) + | |--SEARCH DS USING COVERING INDEX ds1 (name=?) + | |--SEARCH R USING COVERING INDEX rule2 (dsid=?) + | |--SEARCH V USING INDEX v1 (rid=?) + | `--USE TEMP B-TREE FOR ORDER BY + `--SCAN (subquery-xxxxxx) +} +# ^^^^---- We want to see three SEARCH terms. No SCAN terms. +# The ORDER BY is implemented by a separate sorter pass. + +finish_test From 8ce73ce15b8f02b8e86c8a4d567dce42f40c70ba Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 2 Apr 2024 14:12:29 +0000 Subject: [PATCH 08/15] Fix typos in comments. Provided ".wheretrace" debugging output for the interstage heuristic module. Do omit automatic index loops in the interstage heuristic. FossilOrigin-Name: 186dcae19e249db36de15f295999cff25063b54ee3d5d481cd2ba99b6d13148e --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- src/where.c | 23 +++++++++++++++++++---- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 24a90e1c43..790c61ed9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sheuristic\sin\sbetween\sthe\stwo\ssolver()\spasses\sof\sthe\squery\splanner\sthat\ntries\sto\sprevent\sa\svery\sslow\squery\splan\sin\scases\swhere\sthe\soutput\srow\scount\nestimate\sis\simprecise. -D 2024-04-02T11:44:44.648 +C Fix\stypos\sin\scomments.\s\sProvided\s".wheretrace"\sdebugging\soutput\sfor\sthe\ninterstage\sheuristic\smodule.\s\sDo\somit\sautomatic\sindex\sloops\sin\sthe\ninterstage\sheuristic. +D 2024-04-02T14:12:29.323 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -755,7 +755,7 @@ F src/shell.c.in 0354ca51eee5fbf6af394a7ef9f5ef6823ef45b743db65431f6777e4d5be219 F src/sqlite.h.in 32389e0d584551b300d0157881336162c14315a424cbf385c0d65eb7c2e31f7b F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h f8928f0397d797046396dd9d360a6af8ce6dcb48bd72ea290165b07c8c518744 +F src/sqliteInt.h 170f85c5a9d6189cf9b5dd304f4d40b73a8c9762fb927f1327e8ed2cb99c06b6 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -835,7 +835,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c ba9d22fda3850b508e075f8bdc4f4d4f1ff314551597a6f14842bb03aeec0d3a +F src/where.c 08ffccb5b8df523a484af2ae8593c8f571113475fe76b13360f2dfd74e117b4e F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 @@ -2184,11 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0b2ac2cdc767db764e3ea8bbc33898cac4e1ec27fe8c9b60ce08a1785f921e6d -R bbe24560206aebee12aa3740963fd4bf -T *branch * interstage-heuristic -T *sym-interstage-heuristic * -T -sym-trunk * +P 8018417b0143ea11535f2457bf3e4b3755717c554a17df1076425b4251b5f2c6 +R 9b55b3987ec7e07bd641f4ba8c80fe98 U drh -Z f6898799f068ada043c459f74f460b9b +Z 1d7be0abb1ed0f4f893f7da5731cd389 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4593d196ec..5063c5ac66 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8018417b0143ea11535f2457bf3e4b3755717c554a17df1076425b4251b5f2c6 \ No newline at end of file +186dcae19e249db36de15f295999cff25063b54ee3d5d481cd2ba99b6d13148e \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index dba64c71bd..365ff2c7f5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1156,7 +1156,7 @@ extern u32 sqlite3WhereTrace; ** 0x00000010 Display sqlite3_index_info xBestIndex calls ** 0x00000020 Range an equality scan metrics ** 0x00000040 IN operator decisions -** 0x00000080 WhereLoop cost adjustements +** 0x00000080 WhereLoop cost adjustments ** 0x00000100 ** 0x00000200 Covering index decisions ** 0x00000400 OR optimization diff --git a/src/where.c b/src/where.c index fe76f2c960..a4506602a2 100644 --- a/src/where.c +++ b/src/where.c @@ -5561,15 +5561,15 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** Except, if the first solver() call generated a full-table scan in an outer ** loop then stop this analysis at the first full-scan, since the second ** solver() run might try to swap that full-scan for another in order to -** get the output into the correct order. In other words, we do *not* want -** to inhibit a rewrites like this: +** get the output into the correct order. In other words, we allow a +** rewrite like this: ** ** First Solver() Second Solver() ** |-- SCAN t1 |-- SCAN t2 ** |-- SEARCH t2 `-- SEARCH t1 ** `-- SORT USING B-TREE ** -** Rather, the purpose of this routine is to inhibit rewrites such as: +** The purpose of this routine is to disallow rewrites such as: ** ** First Solver() Second Solver() ** |-- SEARCH t1 |-- SCAN t2 <--- bad! @@ -5581,6 +5581,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ */ static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ int i; +#ifdef WHERETRACE_ENABLED + int once = 0; +#endif for(i=0; inLevel; i++){ WhereLoop *p = pWInfo->a[i].pWLoop; if( p==0 ) break; @@ -5590,7 +5593,19 @@ static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ WhereLoop *pLoop; for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){ if( pLoop->iTab!=iTab ) continue; - if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 ) continue; + if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){ + /* Auto-index and index-constrained loops allowed to remain */ + continue; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x80 ){ + if( once==0 ){ + sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n"); + once = 1; + } + sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC); + } +#endif /* WHERETRACE_ENABLED */ pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */ } }else{ From 307f290b329603e70d1053282a96e5536184b252 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 2 Apr 2024 21:55:45 +0000 Subject: [PATCH 09/15] Add an optional 3rd argument to the icu_load_collation() function that specifies the "strength" of the comparison. FossilOrigin-Name: acddbc489d5231dacf29890d1aa562da499026690f55c256a07d04a3190faacc --- ext/icu/icu.c | 37 +++++++++++++++++++++++++++++++++++-- manifest | 14 +++++++------- manifest.uuid | 2 +- test/icu.test | 18 +++++++++++++++++- 4 files changed, 60 insertions(+), 11 deletions(-) diff --git a/ext/icu/icu.c b/ext/icu/icu.c index e745ab0253..f9d9eef905 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -471,7 +471,7 @@ static void icuLoadCollation( UCollator *pUCollator; /* ICU library collation object */ int rc; /* Return code from sqlite3_create_collation_x() */ - assert(nArg==2); + assert(nArg==2 || nArg==3); (void)nArg; /* Unused parameter */ zLocale = (const char *)sqlite3_value_text(apArg[0]); zName = (const char *)sqlite3_value_text(apArg[1]); @@ -486,7 +486,39 @@ static void icuLoadCollation( return; } assert(p); - + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + int i; + for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; i Date: Thu, 4 Apr 2024 12:36:33 +0000 Subject: [PATCH 10/15] Fix exception propagation when the opfs-sahpool VFS fails to initialize to address report in [forum:be0141c639|forum post be0141c639]. FossilOrigin-Name: cdb3a90055afe371962ec6c0931f1b08797aaeaab0358ef64b1cabda0a855b33 --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index f3664fd4b8..3f4182dacc 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -1272,7 +1272,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return poolUtil; }).catch(async (e)=>{ await thePool.removeVfs().catch(()=>{}); - return e; + throw e; }); }).catch((err)=>{ //error("rejecting promise:",err); diff --git a/manifest b/manifest index d49c515d29..b640e1905b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\soptional\s3rd\sargument\sto\sthe\sicu_load_collation()\sfunction\sthat\nspecifies\sthe\s"strength"\sof\sthe\scomparison. -D 2024-04-02T21:55:45.687 +C Fix\sexception\spropagation\swhen\sthe\sopfs-sahpool\sVFS\sfails\sto\sinitialize\sto\saddress\sreport\sin\s[forum:be0141c639|forum\spost\sbe0141c639]. +D 2024-04-04T12:36:33.682 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -612,7 +612,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 8d9c0562831f62218170a3373468d8a0b7a6503b59 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 196ad83d36ca794e564044788c9d21b964679d63cad865f604da37c4afc9a285 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 5a430874906ff3f4a6ca69aadf0c2aaedc1bb45489b8365bff7e955a83a8d42a +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 8433ee332d5f5e39fb19427fccb7bad7f44aa99b5504daad3343fc128c311e78 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 3c72f1a0e6a7343c8c882d29d01bb440f10be12c844651605b486e76f3d6cc8c F ext/wasm/api/sqlite3-vtab-helper.c-pp.js a2fcbc3fecdd0eea229283584ebc122f29d98194083675dbe5cb2cf3a17fe309 F ext/wasm/api/sqlite3-wasm.c afba6827a49151b564af5cf588a6bbd0401b16ef5cbe3269c66f676fee9ca92c @@ -2183,8 +2183,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0b2ac2cdc767db764e3ea8bbc33898cac4e1ec27fe8c9b60ce08a1785f921e6d -R c7e032bcb5236f4db4e7b2c73488ed57 -U drh -Z bc9a8cdde88480024e55380b78b92154 +P acddbc489d5231dacf29890d1aa562da499026690f55c256a07d04a3190faacc +R f4284681fd828de592cb5ab0ae30f403 +U stephan +Z 66fed09968d914b9f5c12c3e86ad8491 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a90644ab0e..f32fa06d10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -acddbc489d5231dacf29890d1aa562da499026690f55c256a07d04a3190faacc \ No newline at end of file +cdb3a90055afe371962ec6c0931f1b08797aaeaab0358ef64b1cabda0a855b33 \ No newline at end of file From efe9500af7051c259e9564db96c900be778ff65c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 4 Apr 2024 13:07:43 +0000 Subject: [PATCH 11/15] Fix test cases due to an error message change. FossilOrigin-Name: c49ec239eddde6f07f8d528cccd6224c4c80b240dc8f5c974186e3bb08afb83b --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/default.test | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b640e1905b..4b231f8550 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sexception\spropagation\swhen\sthe\sopfs-sahpool\sVFS\sfails\sto\sinitialize\sto\saddress\sreport\sin\s[forum:be0141c639|forum\spost\sbe0141c639]. -D 2024-04-04T12:36:33.682 +C Fix\stest\scases\sdue\sto\san\serror\smessage\schange. +D 2024-04-04T13:07:43.910 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1057,7 +1057,7 @@ F test/dbpagefault.test 35f06cfb2ef100a9b19d25754e8141b9cba9b7daabd4c60fa5af93fc F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef F test/decimal.test ef731887b43ee32ef86e1c8fddb61a40789f988332c029c601dcf2c319277e9e -F test/default.test 830fad7180cdf0e6a06e93acc0403bf73762314a639363314db5674c631b6127 +F test/default.test c7124864cded213a3f118bc7e2e26f34b7c36dfa26cf6945cc8b7f5db1191277 F test/delete.test 2686e1c98d552ef37d79ad55b17b93fe96fad9737786917ce3839767f734c48f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab @@ -2183,8 +2183,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P acddbc489d5231dacf29890d1aa562da499026690f55c256a07d04a3190faacc -R f4284681fd828de592cb5ab0ae30f403 -U stephan -Z 66fed09968d914b9f5c12c3e86ad8491 +P cdb3a90055afe371962ec6c0931f1b08797aaeaab0358ef64b1cabda0a855b33 +R ccb06537668742bb198e005c80e2d348 +U drh +Z 5666fcda290346432865b5fd5e238e7f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f32fa06d10..68f73f7bce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cdb3a90055afe371962ec6c0931f1b08797aaeaab0358ef64b1cabda0a855b33 \ No newline at end of file +c49ec239eddde6f07f8d528cccd6224c4c80b240dc8f5c974186e3bb08afb83b \ No newline at end of file diff --git a/test/default.test b/test/default.test index de67f643b4..192b3d2ff3 100644 --- a/test/default.test +++ b/test/default.test @@ -135,10 +135,10 @@ reset_db do_catchsql_test default-5.1 { CREATE TABLE t1 (a,b DEFAULT(random() NOTNULL IN (RAISE(IGNORE),2,3))); INSERT INTO t1(a) VALUES(1); -} {1 {RAISE() may only be used within a trigger-program}} +} {1 {default value of column [b] is not constant}} do_catchsql_test default-5.2 { CREATE TABLE Table0 (Col0 DEFAULT (RAISE(IGNORE) ) ) ; INSERT INTO Table0 DEFAULT VALUES ; -} {1 {RAISE() may only be used within a trigger-program}} +} {1 {default value of column [Col0] is not constant}} finish_test From 634f43468605b8f53404b7838472dff52639d249 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 4 Apr 2024 13:36:48 +0000 Subject: [PATCH 12/15] Fix an ASAN problem in part of the test harness. No changes to SQLite itself. FossilOrigin-Name: 797cda7ddcceb140330d58892c3e73d28df72b638df00fd48f07dfcba7706c5f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_bestindex.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4b231f8550..9947cad665 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\scases\sdue\sto\san\serror\smessage\schange. -D 2024-04-04T13:07:43.910 +C Fix\san\sASAN\sproblem\sin\spart\sof\sthe\stest\sharness.\s\sNo\schanges\sto\sSQLite\sitself. +D 2024-04-04T13:36:48.714 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -771,7 +771,7 @@ F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5 F src/test_async.c 195ab49da082053fdb0f949c114b806a49ca770a F src/test_autoext.c 915d245e736652a219a907909bb6710f0d587871 F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 -F src/test_bestindex.c f6af1e41cb7901edafb065a8198e4a0192dd42432b642d038965be5e628dec12 +F src/test_bestindex.c 1ee3d64b49ca06a9cb8195fab04f1a0585cafc90d25a2a817caa14e7caab22e7 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 F src/test_config.c 5fa77ee6064ba546e144c4fea870c5ede2c54314616f81485c6a9c4192100c75 @@ -2183,8 +2183,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cdb3a90055afe371962ec6c0931f1b08797aaeaab0358ef64b1cabda0a855b33 -R ccb06537668742bb198e005c80e2d348 +P c49ec239eddde6f07f8d528cccd6224c4c80b240dc8f5c974186e3bb08afb83b +R 7af8cb79e580778082df3d4aa754bb75 U drh -Z 5666fcda290346432865b5fd5e238e7f +Z e0623a1804a9fd12482c3193ee60e8ba # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 68f73f7bce..12ab17b4c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c49ec239eddde6f07f8d528cccd6224c4c80b240dc8f5c974186e3bb08afb83b \ No newline at end of file +797cda7ddcceb140330d58892c3e73d28df72b638df00fd48f07dfcba7706c5f \ No newline at end of file diff --git a/src/test_bestindex.c b/src/test_bestindex.c index 8128530b40..6b6b413950 100644 --- a/src/test_bestindex.c +++ b/src/test_bestindex.c @@ -226,7 +226,7 @@ static int tclConnect( } sqlite3_free(zCmd); - *ppVtab = &pTab->base; + *ppVtab = pTab ? &pTab->base : 0; return rc; } From 7fd9b53308d8bea8a8851fb71e6dbe0db4a73775 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 4 Apr 2024 22:53:09 +0000 Subject: [PATCH 13/15] Optimize sqlite3.oo1.DB.exec() for the rowMode='object' case to avoid converting the object property keys (column names) from native code to JS for each row. This speeds up large data sets considerably and addresses the report in [forum:3632183d2470617d|forum post 3632183d2470617d]. FossilOrigin-Name: 8b41ef8690001eb299f5b7182c28f5318333bff5b505e1d59d6e6f4556b1c759 --- ext/wasm/api/sqlite3-api-oo1.js | 91 ++++++++++++++++++++------------- manifest | 15 +++--- manifest.uuid | 2 +- 3 files changed, 64 insertions(+), 44 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 160d59db5a..425b52eeca 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -433,40 +433,56 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ out.returnVal = ()=>opt.resultRows; } if(opt.callback || opt.resultRows){ - switch((undefined===opt.rowMode) - ? 'array' : opt.rowMode) { - case 'object': out.cbArg = (stmt)=>stmt.get(Object.create(null)); break; - case 'array': out.cbArg = (stmt)=>stmt.get([]); break; - case 'stmt': - if(Array.isArray(opt.resultRows)){ - toss3("exec(): invalid rowMode for a resultRows array: must", - "be one of 'array', 'object',", - "a result column number, or column name reference."); - } - out.cbArg = (stmt)=>stmt; + switch((undefined===opt.rowMode) ? 'array' : opt.rowMode) { + case 'object': + out.cbArg = (stmt,cache)=>{ + if( !cache.columnNames ) cache.columnNames = stmt.getColumnNames([]); + /* https://sqlite.org/forum/forumpost/3632183d2470617d: + conversion of rows to objects (key/val pairs) is + somewhat expensive for large data sets because of the + native-to-JS conversion of the column names. If we + instead cache the names and build objects from that + list of strings, it can run twice as fast. The + difference is not noticeable for small data sets but + becomes human-perceivable when enough rows are + involved. */ + const row = stmt.get([]); + const rv = Object.create(null); + for( const i in cache.columnNames ) rv[cache.columnNames[i]] = row[i]; + return rv; + }; + break; + case 'array': out.cbArg = (stmt)=>stmt.get([]); break; + case 'stmt': + if(Array.isArray(opt.resultRows)){ + toss3("exec(): invalid rowMode for a resultRows array: must", + "be one of 'array', 'object',", + "a result column number, or column name reference."); + } + out.cbArg = (stmt)=>stmt; + break; + default: + if(util.isInt32(opt.rowMode)){ + out.cbArg = (stmt)=>stmt.get(opt.rowMode); break; - default: - if(util.isInt32(opt.rowMode)){ - out.cbArg = (stmt)=>stmt.get(opt.rowMode); - break; - }else if('string'===typeof opt.rowMode - && opt.rowMode.length>1 - && '$'===opt.rowMode[0]){ - /* "$X": fetch column named "X" (case-sensitive!). Prior - to 2022-12-14 ":X" and "@X" were also permitted, but - having so many options is unnecessary and likely to - cause confusion. */ - const $colName = opt.rowMode.substr(1); - out.cbArg = (stmt)=>{ - const rc = stmt.get(Object.create(null))[$colName]; - return (undefined===rc) - ? toss3(capi.SQLITE_NOTFOUND, - "exec(): unknown result column:",$colName) - : rc; - }; - break; - } - toss3("Invalid rowMode:",opt.rowMode); + }else if('string'===typeof opt.rowMode + && opt.rowMode.length>1 + && '$'===opt.rowMode[0]){ + /* "$X": fetch column named "X" (case-sensitive!). Prior + to 2022-12-14 ":X" and "@X" were also permitted, but + having so many options is unnecessary and likely to + cause confusion. */ + const $colName = opt.rowMode.substr(1); + out.cbArg = (stmt)=>{ + const rc = stmt.get(Object.create(null))[$colName]; + return (undefined===rc) + ? toss3(capi.SQLITE_NOTFOUND, + "exec(): unknown result column:",$colName) + : rc; + }; + break; + } + toss3("Invalid rowMode:",opt.rowMode); } } return out; @@ -884,10 +900,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ and names. */) ? 0 : 1; evalFirstResult = false; if(arg.cbArg || resultRows){ + const cbArgCache = Object.create(null) + /* 2nd arg for arg.cbArg, used by (at least) row-to-object + converter */; for(; stmt.step(); stmt._lockedByExec = false){ - if(0===gotColNames++) stmt.getColumnNames(opt.columnNames); + if(0===gotColNames++){ + stmt.getColumnNames(cbArgCache.columnNames = (opt.columnNames || [])); + } stmt._lockedByExec = true; - const row = arg.cbArg(stmt); + const row = arg.cbArg(stmt,cbArgCache); if(resultRows) resultRows.push(row); if(callback && false === callback.call(opt, row, stmt)){ break; diff --git a/manifest b/manifest index e296c36245..457e113cfd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"interstage-heuristic"\sthat\sattempts\sto\savoid\swildly\sinefficient\nqueries\sthat\suse\stable\sscans\sinstead\sof\sindex\slookups\sbecause\sthe\soutput\nrow\sestimates\sare\sinaccurate. -D 2024-04-04T14:26:42.961 +C Optimize\ssqlite3.oo1.DB.exec()\sfor\sthe\srowMode='object'\scase\sto\savoid\sconverting\sthe\sobject\sproperty\skeys\s(column\snames)\sfrom\snative\scode\sto\sJS\sfor\seach\srow.\sThis\sspeeds\sup\slarge\sdata\ssets\sconsiderably\sand\saddresses\sthe\sreport\sin\s[forum:3632183d2470617d|forum\spost\s3632183d2470617d]. +D 2024-04-04T22:53:09.385 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -606,7 +606,7 @@ F ext/wasm/api/post-js-header.js 04dc12c3edd666b64a1b4ef3b6690c88dcc653f26451fd4 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e F ext/wasm/api/sqlite3-api-glue.js 2d35660c52dcb4bb16d00c56553d34e7caa6ad30083938b515e6f9aa0b312fbb -F ext/wasm/api/sqlite3-api-oo1.js 7f3bcf0549ac44cde4b9da0b642d771916738d3f6781fb8a1757c50a91e506c0 +F ext/wasm/api/sqlite3-api-oo1.js 365b3ae01a461dc974796823652ef1ecb1a9fac5df295ee1a78002cc77afb0d8 F ext/wasm/api/sqlite3-api-prologue.js 93a72b07b2a5d964d2edc76a90b439ece49298bd7ba60a1c6ae5d4878213701e F ext/wasm/api/sqlite3-api-worker1.js 8d9c0562831f62218170a3373468d8a0b7a6503b5985e309b69bf71187b525cf F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 @@ -2184,9 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 797cda7ddcceb140330d58892c3e73d28df72b638df00fd48f07dfcba7706c5f 186dcae19e249db36de15f295999cff25063b54ee3d5d481cd2ba99b6d13148e -R 869b12f296c2e2b0f215f1adb6cf5942 -T +closed 186dcae19e249db36de15f295999cff25063b54ee3d5d481cd2ba99b6d13148e -U drh -Z 93e4543ac83acbe7df90dc3896cd40af +P 7bf49e2c54c9f6f336416f01c0e76aaf70f1e2f3fd612232e5a33ae5dabe0900 +R 32cab23d7481e1a388944c536dd694cd +U stephan +Z 96ae2cb4464d7637da1ca6fd97c9b197 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e59021c9b1..4ea31d83a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7bf49e2c54c9f6f336416f01c0e76aaf70f1e2f3fd612232e5a33ae5dabe0900 \ No newline at end of file +8b41ef8690001eb299f5b7182c28f5318333bff5b505e1d59d6e6f4556b1c759 \ No newline at end of file From 65a1d7a28464a62488d4771dcbdb35cafa5a4e39 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 5 Apr 2024 11:23:51 +0000 Subject: [PATCH 14/15] When compiling with SQLITE_ALLOW_ROWID_IN_VIEW, if the RETURNING clause of an UPDATE of a view specifies a rowid, then return NULL for the value of that rowid. dbsqlfuzz 7863696e9e5ec10b29bcf5ab2681cd6c82a78a4a. FossilOrigin-Name: c7896e88850669e18e89d44c4169d4f4a5d4b904bea6ccb2ac64f93b6d348a42 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/update.c | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 457e113cfd..51f9fbab8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\ssqlite3.oo1.DB.exec()\sfor\sthe\srowMode='object'\scase\sto\savoid\sconverting\sthe\sobject\sproperty\skeys\s(column\snames)\sfrom\snative\scode\sto\sJS\sfor\seach\srow.\sThis\sspeeds\sup\slarge\sdata\ssets\sconsiderably\sand\saddresses\sthe\sreport\sin\s[forum:3632183d2470617d|forum\spost\s3632183d2470617d]. -D 2024-04-04T22:53:09.385 +C When\scompiling\swith\sSQLITE_ALLOW_ROWID_IN_VIEW,\sif\sthe\sRETURNING\sclause\sof\nan\sUPDATE\sof\sa\sview\sspecifies\sa\srowid,\sthen\sreturn\sNULL\sfor\sthe\svalue\sof\nthat\srowid.\s\sdbsqlfuzz\s7863696e9e5ec10b29bcf5ab2681cd6c82a78a4a. +D 2024-04-05T11:23:51.958 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -815,7 +815,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68 F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee F src/trigger.c 0905b96b04bb6658509f711a8207287f1315cdbc3df1a1b13ba6483c8e341c81 -F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 +F src/update.c 732404a04d1737ef14bb6ec6b84f74edf28b3c102a92ae46b4855438a710efe7 F src/upsert.c 2e60567a0e9e8520c18671b30712a88dc73534474304af94f32bb5f3ef65ac65 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 4d6d7ebfe6772a1b950c97bbb1d1a72ad4874617ec498ab8aa73b7f5a43e44bb @@ -2184,8 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7bf49e2c54c9f6f336416f01c0e76aaf70f1e2f3fd612232e5a33ae5dabe0900 -R 32cab23d7481e1a388944c536dd694cd -U stephan -Z 96ae2cb4464d7637da1ca6fd97c9b197 +P 8b41ef8690001eb299f5b7182c28f5318333bff5b505e1d59d6e6f4556b1c759 +R f56441a29ebaf6b9cc2fa5b75235bdce +U drh +Z ca20487feeb5219923a08470db600847 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4ea31d83a3..58ff11c4a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b41ef8690001eb299f5b7182c28f5318333bff5b505e1d59d6e6f4556b1c759 \ No newline at end of file +c7896e88850669e18e89d44c4169d4f4a5d4b904bea6ccb2ac64f93b6d348a42 \ No newline at end of file diff --git a/src/update.c b/src/update.c index cd7d73f3f7..b6068caa79 100644 --- a/src/update.c +++ b/src/update.c @@ -921,6 +921,9 @@ void sqlite3Update( } } if( chngRowid==0 && pPk==0 ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( isView ) sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); +#endif sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } } From b411c4d69e1d7df018e9c313e104e973fc79d583 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 5 Apr 2024 13:56:05 +0000 Subject: [PATCH 15/15] Check-in [a9657c87c53c1922] is wrong: the IndexedExpr.bMaybeNullRow flag is required for virtual columns if they are part of an outer join. Add a test case (derived from dbsqlfuzz b9e65e2f110df998f1306571fae7af6c01e4d92b) to prove it. FossilOrigin-Name: 4484ec6d26b31305e31de89bdbae26344d8083a7e7de20861430d31737d9979c --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/where.c | 8 +------- test/joinH.test | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 51f9fbab8a..8b598b2633 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\scompiling\swith\sSQLITE_ALLOW_ROWID_IN_VIEW,\sif\sthe\sRETURNING\sclause\sof\nan\sUPDATE\sof\sa\sview\sspecifies\sa\srowid,\sthen\sreturn\sNULL\sfor\sthe\svalue\sof\nthat\srowid.\s\sdbsqlfuzz\s7863696e9e5ec10b29bcf5ab2681cd6c82a78a4a. -D 2024-04-05T11:23:51.958 +C Check-in\s[a9657c87c53c1922]\sis\swrong:\sthe\sIndexedExpr.bMaybeNullRow\sflag\sis\nrequired\sfor\svirtual\scolumns\sif\sthey\sare\spart\sof\san\souter\sjoin.\s\sAdd\sa\ntest\scase\s(derived\sfrom\sdbsqlfuzz\sb9e65e2f110df998f1306571fae7af6c01e4d92b)\nto\sprove\sit. +D 2024-04-05T13:56:05.021 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -835,7 +835,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 08ffccb5b8df523a484af2ae8593c8f571113475fe76b13360f2dfd74e117b4e +F src/where.c 9982128f010a699560fb5be37633801c7641ac3540f49db43cec878025929a0e F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 @@ -1338,7 +1338,7 @@ F test/joinC.test 1f1a602c2127f55f136e2cbd3bf2d26546614bf8cffe5902ec1ac9c07f87f2 F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be28 F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 -F test/joinH.test d5054173442fdf98260eeb6bb9751daa733b0ae6842fe50dcbd5469945b86985 +F test/joinH.test 55f69e64da74d4eca2235237f3acb657aef181e22e45daa228e35bba865e0255 F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ffa140e @@ -2184,8 +2184,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8b41ef8690001eb299f5b7182c28f5318333bff5b505e1d59d6e6f4556b1c759 -R f56441a29ebaf6b9cc2fa5b75235bdce +P c7896e88850669e18e89d44c4169d4f4a5d4b904bea6ccb2ac64f93b6d348a42 +Q -a9657c87c53c19228a42559c82c54b504a5ad729e407e9e2c7dabcc0c949b261 +R 00fdd25fa9a3c1e58bfe6d44fd3eb812 U drh -Z ca20487feeb5219923a08470db600847 +Z d45c3b35e63ad3a95f4d8ac3c99323f3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 58ff11c4a9..0dad7c2062 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c7896e88850669e18e89d44c4169d4f4a5d4b904bea6ccb2ac64f93b6d348a42 \ No newline at end of file +4484ec6d26b31305e31de89bdbae26344d8083a7e7de20861430d31737d9979c \ No newline at end of file diff --git a/src/where.c b/src/where.c index a4506602a2..a1837b5879 100644 --- a/src/where.c +++ b/src/where.c @@ -5948,16 +5948,10 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( for(i=0; inColumn; i++){ Expr *pExpr; int j = pIdx->aiColumn[i]; - int bMaybeNullRow; if( j==XN_EXPR ){ pExpr = pIdx->aColExpr->a[i].pExpr; - testcase( pTabItem->fg.jointype & JT_LEFT ); - testcase( pTabItem->fg.jointype & JT_RIGHT ); - testcase( pTabItem->fg.jointype & JT_LTORJ ); - bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); - bMaybeNullRow = 0; }else{ continue; } @@ -5989,7 +5983,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; p->iIdxCol = i; - p->bMaybeNullRow = bMaybeNullRow; + p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ p->aff = pIdx->zColAff[i]; } diff --git a/test/joinH.test b/test/joinH.test index e3f979409a..908b93dee5 100644 --- a/test/joinH.test +++ b/test/joinH.test @@ -309,4 +309,36 @@ do_execsql_test 12.3 { SELECT * FROM t1 LEFT JOIN t2 ON true RIGHT JOIN t3 ON d2=e3 WHERE c2 BETWEEN NULL AND a1; } +#------------------------------------------------------------------------- +# 2024-04-05 dbsqlfuzz b9e65e2f110df998f1306571fae7af6c01e4d92b +reset_db +do_execsql_test 13.1 { + CREATE TABLE t1(a INT AS (b), b INT); + INSERT INTO t1(b) VALUES(123); + CREATE TABLE t2(a INT, c INT); + SELECT a FROM t2 NATURAL RIGHT JOIN t1; +} {123} +do_execsql_test 13.2 { + CREATE INDEX t1a ON t1(a); + SELECT a FROM t2 NATURAL RIGHT JOIN t1; +} {123} +# Further tests of the same logic (indexes on expressions +# used by RIGHT JOIN) from check-in ffe23af73fcb324d and +# forum post https://sqlite.org/forum/forumpost/9b491e1debf0b67a. +db null NULL +do_execsql_test 13.3 { + CREATE TABLE t3(a INT, b INT); + CREATE UNIQUE INDEX t3x ON t3(a, a+b); + INSERT INTO t3(a,b) VALUES(1,2),(4,8),(16,32),(4,80),(1,-300); + CREATE TABLE t4(x INT, y INT); + INSERT INTO t4(x,y) SELECT a, b FROM t3; + INSERT INTO t4(x,y) VALUES(99,99); + SELECT a1.a, sum( a1.a+a1.b ) FROM t3 AS a1 RIGHT JOIN t4 ON a=x + GROUP BY a1.a ORDER BY 1; +} {NULL NULL 1 -592 4 192 16 48} +do_execsql_test 13.4 { + SELECT sum( a1.a+a1.b ) FROM t3 AS a1 RIGHT JOIN t3 ON true + GROUP BY a1.a ORDER BY 1; +} {-1480 240 480} + finish_test