diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 250d2ee7e3..551618e718 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -55,8 +55,8 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken diff --git a/ext/icu/icu.c b/ext/icu/icu.c index e745ab0253..f9d9eef905 100644 --- a/ext/icu/icu.c +++ b/ext/icu/icu.c @@ -471,7 +471,7 @@ static void icuLoadCollation( UCollator *pUCollator; /* ICU library collation object */ int rc; /* Return code from sqlite3_create_collation_x() */ - assert(nArg==2); + assert(nArg==2 || nArg==3); (void)nArg; /* Unused parameter */ zLocale = (const char *)sqlite3_value_text(apArg[0]); zName = (const char *)sqlite3_value_text(apArg[1]); @@ -486,7 +486,39 @@ static void icuLoadCollation( return; } assert(p); - + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + int i; + for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; iopt.resultRows; } if(opt.callback || opt.resultRows){ - switch((undefined===opt.rowMode) - ? 'array' : opt.rowMode) { - case 'object': out.cbArg = (stmt)=>stmt.get(Object.create(null)); break; - case 'array': out.cbArg = (stmt)=>stmt.get([]); break; - case 'stmt': - if(Array.isArray(opt.resultRows)){ - toss3("exec(): invalid rowMode for a resultRows array: must", - "be one of 'array', 'object',", - "a result column number, or column name reference."); - } - out.cbArg = (stmt)=>stmt; + switch((undefined===opt.rowMode) ? 'array' : opt.rowMode) { + case 'object': + out.cbArg = (stmt,cache)=>{ + if( !cache.columnNames ) cache.columnNames = stmt.getColumnNames([]); + /* https://sqlite.org/forum/forumpost/3632183d2470617d: + conversion of rows to objects (key/val pairs) is + somewhat expensive for large data sets because of the + native-to-JS conversion of the column names. If we + instead cache the names and build objects from that + list of strings, it can run twice as fast. The + difference is not noticeable for small data sets but + becomes human-perceivable when enough rows are + involved. */ + const row = stmt.get([]); + const rv = Object.create(null); + for( const i in cache.columnNames ) rv[cache.columnNames[i]] = row[i]; + return rv; + }; + break; + case 'array': out.cbArg = (stmt)=>stmt.get([]); break; + case 'stmt': + if(Array.isArray(opt.resultRows)){ + toss3("exec(): invalid rowMode for a resultRows array: must", + "be one of 'array', 'object',", + "a result column number, or column name reference."); + } + out.cbArg = (stmt)=>stmt; + break; + default: + if(util.isInt32(opt.rowMode)){ + out.cbArg = (stmt)=>stmt.get(opt.rowMode); break; - default: - if(util.isInt32(opt.rowMode)){ - out.cbArg = (stmt)=>stmt.get(opt.rowMode); - break; - }else if('string'===typeof opt.rowMode - && opt.rowMode.length>1 - && '$'===opt.rowMode[0]){ - /* "$X": fetch column named "X" (case-sensitive!). Prior - to 2022-12-14 ":X" and "@X" were also permitted, but - having so many options is unnecessary and likely to - cause confusion. */ - const $colName = opt.rowMode.substr(1); - out.cbArg = (stmt)=>{ - const rc = stmt.get(Object.create(null))[$colName]; - return (undefined===rc) - ? toss3(capi.SQLITE_NOTFOUND, - "exec(): unknown result column:",$colName) - : rc; - }; - break; - } - toss3("Invalid rowMode:",opt.rowMode); + }else if('string'===typeof opt.rowMode + && opt.rowMode.length>1 + && '$'===opt.rowMode[0]){ + /* "$X": fetch column named "X" (case-sensitive!). Prior + to 2022-12-14 ":X" and "@X" were also permitted, but + having so many options is unnecessary and likely to + cause confusion. */ + const $colName = opt.rowMode.substr(1); + out.cbArg = (stmt)=>{ + const rc = stmt.get(Object.create(null))[$colName]; + return (undefined===rc) + ? toss3(capi.SQLITE_NOTFOUND, + "exec(): unknown result column:",$colName) + : rc; + }; + break; + } + toss3("Invalid rowMode:",opt.rowMode); } } return out; @@ -884,10 +900,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ and names. */) ? 0 : 1; evalFirstResult = false; if(arg.cbArg || resultRows){ + const cbArgCache = Object.create(null) + /* 2nd arg for arg.cbArg, used by (at least) row-to-object + converter */; for(; stmt.step(); stmt._lockedByExec = false){ - if(0===gotColNames++) stmt.getColumnNames(opt.columnNames); + if(0===gotColNames++){ + stmt.getColumnNames(cbArgCache.columnNames = (opt.columnNames || [])); + } stmt._lockedByExec = true; - const row = arg.cbArg(stmt); + const row = arg.cbArg(stmt,cbArgCache); if(resultRows) resultRows.push(row); if(callback && false === callback.call(opt, row, stmt)){ break; diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index f3664fd4b8..3f4182dacc 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -1272,7 +1272,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ return poolUtil; }).catch(async (e)=>{ await thePool.removeVfs().catch(()=>{}); - return e; + throw e; }); }).catch((err)=>{ //error("rejecting promise:",err); diff --git a/manifest b/manifest index cdaaa36873..fbf9643ea6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ C Merge\sall\srecent\strunk\senhancements\sinto\sthe\sreuse-schema\sbranch. -D 2024-03-26T10:48:32.013 +D 2024-04-05T15:04:01.063 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -93,7 +93,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e -F ext/fts5/fts5.h ecba24fed7b359b3a53016bb07e411b3b4c9cdf163aa141006536423a63b611e +F ext/fts5/fts5.h 8856e11a5f0269cd346754cea0765efe8089635b80cad3222e8bfdb08cd5348a F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880e9834 F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 @@ -249,7 +249,7 @@ F ext/fts5/tool/loadfts5.tcl 95b03429ee6b138645703c6ca192c3ac96eaf093 F ext/fts5/tool/mkfts5c.tcl 3eba8e9bee4221ed165f3304b51b2a74a705f4ec5df3d044573a2be539534af8 F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 -F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 +F ext/icu/icu.c 5c858611fd11d65caf8a04acd836af9193880a724ba75cee63f9da75ce4a469d F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 F ext/intck/intck1.test f3a3cba14b6aeff145ffa5515546dd22f7510dad91512e519f43b92b56514012 F ext/intck/intck2.test d2457c7e5e5b688046d15ebe08a1e1427cc5e7a6dc8d6af215f42e8bcaf67304 @@ -607,13 +607,13 @@ F ext/wasm/api/post-js-header.js 04dc12c3edd666b64a1b4ef3b6690c88dcc653f26451fd4 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e F ext/wasm/api/sqlite3-api-glue.js 2d35660c52dcb4bb16d00c56553d34e7caa6ad30083938b515e6f9aa0b312fbb -F ext/wasm/api/sqlite3-api-oo1.js 7f3bcf0549ac44cde4b9da0b642d771916738d3f6781fb8a1757c50a91e506c0 +F ext/wasm/api/sqlite3-api-oo1.js 365b3ae01a461dc974796823652ef1ecb1a9fac5df295ee1a78002cc77afb0d8 F ext/wasm/api/sqlite3-api-prologue.js 93a72b07b2a5d964d2edc76a90b439ece49298bd7ba60a1c6ae5d4878213701e F ext/wasm/api/sqlite3-api-worker1.js 8d9c0562831f62218170a3373468d8a0b7a6503b5985e309b69bf71187b525cf F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 196ad83d36ca794e564044788c9d21b964679d63cad865f604da37c4afc9a285 F ext/wasm/api/sqlite3-vfs-helper.c-pp.js 3f828cc66758acb40e9c5b4dcfd87fd478a14c8fb7f0630264e6c7fa0e57515d -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 5a430874906ff3f4a6ca69aadf0c2aaedc1bb45489b8365bff7e955a83a8d42a +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 8433ee332d5f5e39fb19427fccb7bad7f44aa99b5504daad3343fc128c311e78 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 3c72f1a0e6a7343c8c882d29d01bb440f10be12c844651605b486e76f3d6cc8c F ext/wasm/api/sqlite3-vtab-helper.c-pp.js a2fcbc3fecdd0eea229283584ebc122f29d98194083675dbe5cb2cf3a17fe309 F ext/wasm/api/sqlite3-wasm.c afba6827a49151b564af5cf588a6bbd0401b16ef5cbe3269c66f676fee9ca92c @@ -683,7 +683,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 acdff36db796e2d00225b911d3047d580cd136547298435426ce9d40347973cc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 -F src/alter.c 47253323822cc2ad54d5fe5695db683fae2f904242e9df55a1bc5406b1293e3a +F src/alter.c 8fd1f87dbf8b49f4e037817640d62d5e28661ad194d68fa10455fe5aada79ee9 F src/analyze.c d4147664724ed16e86e974e1714f893105a8d124bad3acebd30a3c0051553f50 F src/attach.c 2af98700f1a3867a78475aa164571a2fbacce09c681076b01b119ef31b8ef4ac F src/auth.c 19b7ccacae3dfba23fc6f1d0af68134fa216e9040e53b0681b4715445ea030b4 @@ -753,10 +753,10 @@ F src/resolve.c eb1860b134fb044fd819c4347105c148d5aac7c6498032be2829e5cc95619b28 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 15a221347789e393b39e7d2d2bd102167979c95a1ce0675bb870b86a24ca6cf4 F src/shell.c.in 95d92c0162bcfea8a9d9610acbd8a0a861c5e24c15272c957acbe4f39382ddae -F src/sqlite.h.in f0f8f0d2a73dd926f41e38bd44fcfac313a40f91373eced4f96333c6c2d4bec3 +F src/sqlite.h.in 6a187bd8f031cdf4045d13f3b6bc5675183dc2eb826b08404d2c540bfa40e28f F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54 -F src/sqliteInt.h 13c2178e80eb068397dffe6ae659cbc95af34ee69ceeeca0ad709615796db471 +F src/sqliteInt.h f02109c375e41bc0df376ac16502e0127099aec4c43224380e8cc5ca9cd86854 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728 F src/status.c 5028a0afee355aa492f26f0b6a3ec23145caa9261a93164d96cd0b9bf1b2318f F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -772,7 +772,7 @@ F src/test9.c 12e5ba554d2d1cbe0158f6ab3f7ffcd7a86ee4e5 F src/test_async.c 195ab49da082053fdb0f949c114b806a49ca770a F src/test_autoext.c 915d245e736652a219a907909bb6710f0d587871 F src/test_backup.c bf5da90c9926df0a4b941f2d92825a01bbe090a0 -F src/test_bestindex.c f6af1e41cb7901edafb065a8198e4a0192dd42432b642d038965be5e628dec12 +F src/test_bestindex.c 1ee3d64b49ca06a9cb8195fab04f1a0585cafc90d25a2a817caa14e7caab22e7 F src/test_blob.c ae4a0620b478548afb67963095a7417cd06a4ec0a56adb453542203bfdcb31ce F src/test_btree.c 8b2dc8b8848cf3a4db93f11578f075e82252a274 F src/test_config.c 7f2b70b7259fb9f6418a6d266a035acf24842e3528745a2bed4dd252bddf1a32 @@ -817,7 +817,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68 F src/treeview.c c6fc972683fd00f975d8b32a81c1f25d2fb7d4035366bf45c9f5622d3ccd70ee F src/trigger.c fce5a48596e30c9517357f533b5726120906e446e9d0ddaa180c8c60b9e14c01 -F src/update.c 6904814dd62a7a93bbb86d9f1419c7f134a9119582645854ab02b36b676d9f92 +F src/update.c 732404a04d1737ef14bb6ec6b84f74edf28b3c102a92ae46b4855438a710efe7 F src/upsert.c 2e60567a0e9e8520c18671b30712a88dc73534474304af94f32bb5f3ef65ac65 F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e F src/util.c 4d6d7ebfe6772a1b950c97bbb1d1a72ad4874617ec498ab8aa73b7f5a43e44bb @@ -837,7 +837,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452 F src/walker.c 7c7ea0115345851c3da4e04e2e239a29983b61fb5b038b94eede6aba462640e2 -F src/where.c 203c385e42ac53642bf54c7a5d99d289fa4e0b2fef2763f6365656a6dfaf551d +F src/where.c aa2f7b6cffc918e078e0187ab9e1c208b2878f682f37d09e3ef441f535336563 F src/whereInt.h 82a13766f13d1a53b05387c2e60726289ef26404bc7b9b1f7770204d97357fb8 F src/wherecode.c 5d77db30a2a3dd532492ae882de114edba2fae672622056b1c7fd61f5917a8f1 F src/whereexpr.c 7b64295f1d82ad0928df435925dd7bbd5997b44a026153113eace0d9e71ff435 @@ -869,7 +869,7 @@ F test/altermalloc3.test 8040e486368403f2fdd6fc3998258b499bd4cc2f3ddbb5f8f874cd4 F test/alterqf.test 8ec03d776de9c391daa0078ea8f838903bdcfb11dfae4ba3576b48436834ccba F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 F test/altertab2.test 62597b6fd08feaba1b6bfe7d31dac6117c67e06dc9ce9c478a3abe75b5926de0 -F test/altertab3.test e167ce3b8e243b52306c1e40b13eb868f402a969513c063998593862cc643b44 +F test/altertab3.test b331ae34e69594e19605e3297805202d6156fcc8f75379dfd972a2e51cae8721 F test/altertrig.test aacc980b657354fe2d3d4d3a004f07d04ccc1a93e5ef82d68a79088c274ddc6b F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 2fb21d7d64748636384e6cb8998dbf83968caf644c07fcb4f76c18f2e7ede94b @@ -1059,7 +1059,7 @@ F test/dbpagefault.test 35f06cfb2ef100a9b19d25754e8141b9cba9b7daabd4c60fa5af93fc F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef F test/decimal.test ef731887b43ee32ef86e1c8fddb61a40789f988332c029c601dcf2c319277e9e -F test/default.test 830fad7180cdf0e6a06e93acc0403bf73762314a639363314db5674c631b6127 +F test/default.test c7124864cded213a3f118bc7e2e26f34b7c36dfa26cf6945cc8b7f5db1191277 F test/delete.test 2686e1c98d552ef37d79ad55b17b93fe96fad9737786917ce3839767f734c48f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab @@ -1269,7 +1269,7 @@ F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 F test/hook.test 18cae9140fa7f9a6f346e892a3fe3e31b2ca0be1494cd01b918adb74281016a6 F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 -F test/icu.test 716a6b89fbabe5cc63e0cd4c260befb08fd7b9d761f04d43669233292f0753b1 +F test/icu.test 8da7d52cd9722c82f33b0466ed915460cb03c23a38f18a9a2d3ff97da9a4a8c0 F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 F test/in.test d1cad4ededd425568b2e39fb0c31fa9a3772311dd595801ff13ba3912b69bba6 @@ -1340,7 +1340,7 @@ F test/joinC.test 1f1a602c2127f55f136e2cbd3bf2d26546614bf8cffe5902ec1ac9c07f87f2 F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be28 F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 -F test/joinH.test d5054173442fdf98260eeb6bb9751daa733b0ae6842fe50dcbd5469945b86985 +F test/joinH.test 55f69e64da74d4eca2235237f3acb657aef181e22e45daa228e35bba865e0255 F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ffa140e @@ -1961,7 +1961,7 @@ F test/vtabH.test 2efb5a24b0bb50796b21eca23032cfb77abfa4b0c03938e38ce5897abac404 F test/vtabI.test 751b07636700dbdea328e4265b6077ccd6811a3f F test/vtabJ.test a6aef49d558af90fae10565b29501f82a95781cb4f797f2d13e2d19f9b6bc77b F test/vtabK.test 13293177528fada1235c0112db0d187d754af1355c5a39371abd365104e3afbf -F test/vtabL.test 040b9f782a3b41844f2a5c660e6a0bfd298492980c6b4e126d82113c9785cec3 +F test/vtabL.test 49ec7342e8bfcb0d6c3d2443c619f430c609c042d5d7e6ddf52def65d6d1812f F test/vtab_alter.test 736e66fb5ec7b4fee58229aa3ada2f27ec58bc58c00edae4836890c3784c6783 F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bda2b65 F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad @@ -2027,6 +2027,7 @@ F test/whereJ.test fc05e374cc9f2dc204148d6c06822c380ad388895fe97a6d335b94a26a08a F test/whereK.test 0270ab7f04ba5436fb9156d31d642a1c82727f4c4bfe5ba90d435c78cf44684a F test/whereL.test 438a397fa883b77bb6361c08a8befa41b52e9cfbe15a2a43715d122f8cfa8649 F test/whereM.test 0dbc9998783458ddcf3cc078ca7c2951d8b2677d472ecf0028f449ed327c0250 +F test/whereN.test 63a3584b71acfb6963416de82f26c6b1644abc5ca6080c76546b9246734c8803 F test/wherefault.test 6cf2a9c5712952d463d3f45ebee7f6caf400984df51a195d884cfb7eb0e837a7 F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3 F test/wherelimit.test afb46397c6d7e964e6e294ba3569864a0c570fe3807afc634236c2b752372f31 @@ -2193,8 +2194,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c2d4adabb929b6088808b05a2f031d045eb4839b44548a66006dc21e358e71f4 6a06dc73847716c88d65651d1bf0e002002303881df1389beac884d0032eae08 -R 3b0652db047bafda6f9442911e1fb947 +P e469b02205a4f3923c93ec9c4825045aba1cc9e053d6e341efaa850993bc155e 4484ec6d26b31305e31de89bdbae26344d8083a7e7de20861430d31737d9979c +R f4dbba28f9c4664c8d96ddb5bc531b37 U drh -Z 2f230d22a67b67fa1a728aa4b3497b01 +Z ff6215cb807d646da0d3d7fb004c0cec # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 375c1e5a17..c59bf83d31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e469b02205a4f3923c93ec9c4825045aba1cc9e053d6e341efaa850993bc155e \ No newline at end of file +e556f3d3819134022f72e72471976616700c4fe00fc924bfbe713a6132440fdf \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index 0698ea62b0..233b76f5ba 100644 --- a/src/alter.c +++ b/src/alter.c @@ -2262,7 +2262,12 @@ void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); }else{ + char aff = pTab->aCol[i].affinity; + if( aff==SQLITE_AFF_REAL ){ + pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; + } sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + pTab->aCol[i].affinity = aff; } nField++; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 10fa8c735b..60d0aec53c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6888,6 +6888,12 @@ int sqlite3_autovacuum_pages( ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -8358,7 +8364,7 @@ int sqlite3_test_control(int op, ...); ** The sqlite3_keyword_count() interface returns the number of distinct ** keywords understood by SQLite. ** -** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and ** makes *Z point to that keyword expressed as UTF8 and writes the number ** of bytes in the keyword into *L. The string that *Z points to is not ** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8213abb655..2ef96d4a93 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1156,7 +1156,7 @@ extern u32 sqlite3WhereTrace; ** 0x00000010 Display sqlite3_index_info xBestIndex calls ** 0x00000020 Range an equality scan metrics ** 0x00000040 IN operator decisions -** 0x00000080 WhereLoop cost adjustements +** 0x00000080 WhereLoop cost adjustments ** 0x00000100 ** 0x00000200 Covering index decisions ** 0x00000400 OR optimization diff --git a/src/test_bestindex.c b/src/test_bestindex.c index 8128530b40..6b6b413950 100644 --- a/src/test_bestindex.c +++ b/src/test_bestindex.c @@ -226,7 +226,7 @@ static int tclConnect( } sqlite3_free(zCmd); - *ppVtab = &pTab->base; + *ppVtab = pTab ? &pTab->base : 0; return rc; } diff --git a/src/update.c b/src/update.c index cd7d73f3f7..b6068caa79 100644 --- a/src/update.c +++ b/src/update.c @@ -921,6 +921,9 @@ void sqlite3Update( } } if( chngRowid==0 && pPk==0 ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( isView ) sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); +#endif sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } } diff --git a/src/where.c b/src/where.c index 2cf7e7ff2c..b04c451e3a 100644 --- a/src/where.c +++ b/src/where.c @@ -3252,10 +3252,13 @@ static int whereLoopAddBtreeIndex( } } - /* Set rCostIdx to the cost of visiting selected rows in index. Add - ** it to pNew->rRun, which is currently set to the cost of the index - ** seek only. Then, if this is a non-covering index, add the cost of - ** visiting the rows in the main table. */ + /* Set rCostIdx to the estimated cost of visiting selected rows in the + ** index. The estimate is the sum of two values: + ** 1. The cost of doing one search-by-key to find the first matching + ** entry + ** 2. Stepping forward in the index pNew->nOut times to find all + ** additional matching entries. + */ assert( pSrc->pTab->szTabRow>0 ); if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* The pProbe->szIdxRow is low for an IPK table since the interior @@ -3266,7 +3269,15 @@ static int whereLoopAddBtreeIndex( }else{ rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; } - pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx); + + /* Estimate the cost of running the loop. If all data is coming + ** from the index, then this is just the cost of doing the index + ** lookup and scan. But if some data is coming out of the main table, + ** we also have to add in the cost of doing pNew->nOut searches to + ** locate the row in the main table that corresponds to the index entry. + */ + pNew->rRun = rCostIdx; if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } @@ -5519,7 +5530,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } } - pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ @@ -5527,6 +5537,83 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ return SQLITE_OK; } +/* +** This routine implements a heuristic designed to improve query planning. +** This routine is called in between the first and second call to +** wherePathSolver(). Hence the name "Interstage" "Heuristic". +** +** The first call to wherePathSolver() (hereafter just "solver()") computes +** the best path without regard to the order of the outputs. The second call +** to the solver() builds upon the first call to try to find an alternative +** path that satisfies the ORDER BY clause. +** +** This routine looks at the results of the first solver() run, and for +** every FROM clause term in the resulting query plan that uses an equality +** constraint against an index, disable other WhereLoops for that same +** FROM clause term that would try to do a full-table scan. This prevents +** an index search from being converted into a full-table scan in order to +** satisfy an ORDER BY clause, since even though we might get slightly better +** performance using the full-scan without sorting if the output size +** estimates are very precise, we might also get severe performance +** degradation using the full-scan if the output size estimate is too large. +** It is better to err on the side of caution. +** +** Except, if the first solver() call generated a full-table scan in an outer +** loop then stop this analysis at the first full-scan, since the second +** solver() run might try to swap that full-scan for another in order to +** get the output into the correct order. In other words, we allow a +** rewrite like this: +** +** First Solver() Second Solver() +** |-- SCAN t1 |-- SCAN t2 +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** The purpose of this routine is to disallow rewrites such as: +** +** First Solver() Second Solver() +** |-- SEARCH t1 |-- SCAN t2 <--- bad! +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** See test cases in test/whereN.test for the real-world query that +** originally provoked this heuristic. +*/ +static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ + int i; +#ifdef WHERETRACE_ENABLED + int once = 0; +#endif + for(i=0; inLevel; i++){ + WhereLoop *p = pWInfo->a[i].pWLoop; + if( p==0 ) break; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue; + if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ + u8 iTab = p->iTab; + WhereLoop *pLoop; + for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){ + if( pLoop->iTab!=iTab ) continue; + if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){ + /* Auto-index and index-constrained loops allowed to remain */ + continue; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x80 ){ + if( once==0 ){ + sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n"); + once = 1; + } + sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC); + } +#endif /* WHERETRACE_ENABLED */ + pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */ + } + }else{ + break; + } + } +} + /* ** Most queries use only a single table (they are not joins) and have ** simple == constraints against indexed fields. This routine attempts @@ -5861,16 +5948,10 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( for(i=0; inColumn; i++){ Expr *pExpr; int j = pIdx->aiColumn[i]; - int bMaybeNullRow; if( j==XN_EXPR ){ pExpr = pIdx->aColExpr->a[i].pExpr; - testcase( pTabItem->fg.jointype & JT_LEFT ); - testcase( pTabItem->fg.jointype & JT_RIGHT ); - testcase( pTabItem->fg.jointype & JT_LTORJ ); - bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); - bMaybeNullRow = 0; }else{ continue; } @@ -5902,7 +5983,7 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( p->iDataCur = pTabItem->iCursor; p->iIdxCur = iIdxCur; p->iIdxCol = i; - p->bMaybeNullRow = bMaybeNullRow; + p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ p->aff = pIdx->zColAff[i]; } @@ -6309,6 +6390,7 @@ WhereInfo *sqlite3WhereBegin( wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ + whereInterstageHeuristic(pWInfo); wherePathSolver(pWInfo, pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } diff --git a/test/altertab3.test b/test/altertab3.test index cb6f00de48..5f5c11b0bb 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -763,4 +763,27 @@ do_execsql_test 30.2 { )} } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 31.0 { + CREATE TABLE t1(ii INTEGER PRIMARY KEY, tt INTEGER, rr REAL); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000 + ) + INSERT INTO t1 SELECT NULL, i, 5.0 FROM s; +} + +do_test 31.1 { + set pg [db one {PRAGMA page_count}] + execsql { + ALTER TABLE t1 DROP COLUMN tt; + } + set pg2 [db one {PRAGMA page_count}] + expr $pg==$pg2 +} {1} + +do_execsql_test 31.2 { + SELECT rr FROM t1 LIMIT 1 +} {5.0} + finish_test diff --git a/test/default.test b/test/default.test index de67f643b4..192b3d2ff3 100644 --- a/test/default.test +++ b/test/default.test @@ -135,10 +135,10 @@ reset_db do_catchsql_test default-5.1 { CREATE TABLE t1 (a,b DEFAULT(random() NOTNULL IN (RAISE(IGNORE),2,3))); INSERT INTO t1(a) VALUES(1); -} {1 {RAISE() may only be used within a trigger-program}} +} {1 {default value of column [b] is not constant}} do_catchsql_test default-5.2 { CREATE TABLE Table0 (Col0 DEFAULT (RAISE(IGNORE) ) ) ; INSERT INTO Table0 DEFAULT VALUES ; -} {1 {RAISE() may only be used within a trigger-program}} +} {1 {default value of column [Col0] is not constant}} finish_test diff --git a/test/icu.test b/test/icu.test index 644cbb1f08..c1b5653d4f 100644 --- a/test/icu.test +++ b/test/icu.test @@ -149,7 +149,7 @@ ifcapable icu { # 2020-03-19 # The ESCAPE clause on LIKE takes precedence over wildcards # -do_execsql_test idu-6.0 { +do_execsql_test icu-6.0 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(id INTEGER PRIMARY KEY, x TEXT); INSERT INTO t1 VALUES @@ -164,4 +164,20 @@ do_execsql_test icu-6.1 { SELECT id FROM t1 WHERE x LIKE 'abc__' ESCAPE '_'; } {2} +# 2024-04-02 +# Optional 3rd argument to icu_load_collation() that specifies +# the "strength" of comparison. +# +reset_db +do_catchsql_test icu-7.1 { + SELECT icu_load_collation('en_US','error','xyzzy'); +} {1 {unknown collation strength "xyzzy" - should be one of: PRIMARY SECONDARY TERTIARY DEFAULT QUARTERNARY IDENTICAL}} +do_execsql_test icu-7.2 { + SELECT icu_load_collation('en_US','prim','PRIMARY'), + icu_load_collation('en_US','dflt','DEFAULT'); +} {{} {}} +do_execsql_test icu-7.3 { + SELECT char(0x100)=='a', char(0x100)=='a' COLLATE dflt, char(0x100)=='a' COLLATE prim; +} {0 0 1} + finish_test diff --git a/test/joinH.test b/test/joinH.test index e3f979409a..908b93dee5 100644 --- a/test/joinH.test +++ b/test/joinH.test @@ -309,4 +309,36 @@ do_execsql_test 12.3 { SELECT * FROM t1 LEFT JOIN t2 ON true RIGHT JOIN t3 ON d2=e3 WHERE c2 BETWEEN NULL AND a1; } +#------------------------------------------------------------------------- +# 2024-04-05 dbsqlfuzz b9e65e2f110df998f1306571fae7af6c01e4d92b +reset_db +do_execsql_test 13.1 { + CREATE TABLE t1(a INT AS (b), b INT); + INSERT INTO t1(b) VALUES(123); + CREATE TABLE t2(a INT, c INT); + SELECT a FROM t2 NATURAL RIGHT JOIN t1; +} {123} +do_execsql_test 13.2 { + CREATE INDEX t1a ON t1(a); + SELECT a FROM t2 NATURAL RIGHT JOIN t1; +} {123} +# Further tests of the same logic (indexes on expressions +# used by RIGHT JOIN) from check-in ffe23af73fcb324d and +# forum post https://sqlite.org/forum/forumpost/9b491e1debf0b67a. +db null NULL +do_execsql_test 13.3 { + CREATE TABLE t3(a INT, b INT); + CREATE UNIQUE INDEX t3x ON t3(a, a+b); + INSERT INTO t3(a,b) VALUES(1,2),(4,8),(16,32),(4,80),(1,-300); + CREATE TABLE t4(x INT, y INT); + INSERT INTO t4(x,y) SELECT a, b FROM t3; + INSERT INTO t4(x,y) VALUES(99,99); + SELECT a1.a, sum( a1.a+a1.b ) FROM t3 AS a1 RIGHT JOIN t4 ON a=x + GROUP BY a1.a ORDER BY 1; +} {NULL NULL 1 -592 4 192 16 48} +do_execsql_test 13.4 { + SELECT sum( a1.a+a1.b ) FROM t3 AS a1 RIGHT JOIN t3 ON true + GROUP BY a1.a ORDER BY 1; +} {-1480 240 480} + finish_test diff --git a/test/vtabL.test b/test/vtabL.test index a801481c84..0834655ffd 100644 --- a/test/vtabL.test +++ b/test/vtabL.test @@ -31,10 +31,7 @@ proc vtab_command {method args} { return {} } -do_execsql_test 1.0 { - CREATE TABLE t1(a, b); -} - +breakpoint foreach {tn cts} { 1 {SELECT 123} 2 {SELECT 123, 456} @@ -44,6 +41,8 @@ foreach {tn cts} { 6 {DROP TABLE nosuchtable} 7 {DROP TABLE x1} 8 {DROP TABLE t1} + 9 {CREATE TABLE xyz AS SELECT * FROM sqlite_schema} + 10 {CREATE TABLE xyz AS SELECT 1 AS 'col'} } { set ::create_table_sql $cts do_catchsql_test 1.$tn { @@ -51,5 +50,19 @@ foreach {tn cts} { } {1 {vtable constructor failed: x1}} } +foreach {tn cts} { + 1 {CREATE TABLE IF NOT EXISTS t1(a, b)} + 2 {CREATE TABLE ""(a, b PRIMARY KEY) WITHOUT ROWID} +} { + set ::create_table_sql $cts + execsql { DROP TABLE IF EXISTS x1 } + do_execsql_test 2.$tn.1 { + CREATE VIRTUAL TABLE x1 USING tcl(vtab_command); + } + do_execsql_test 2.$tn.2 { + SELECT a, b FROM x1 + } +} + finish_test diff --git a/test/whereN.test b/test/whereN.test new file mode 100644 index 0000000000..b9b889fa28 --- /dev/null +++ b/test/whereN.test @@ -0,0 +1,103 @@ +# 2024-04-02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Tests for the whereInterstageHeuristic() routine in the query planner. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix whereN + +# The following is a simplified and "sanitized" version of the original +# real-world query that brought the problem to light. +# +# The issue is a slow query. The answer is correct, but it was taking too +# much time, because it was doing a full table scan rather than an indexed +# lookup. +# +# The problem was that the query planner was overestimating the number of +# output rows. The estimated number of output rows is accurate if the +# DSNAME parameter is "ds-one". In that case, a large fraction of the rows +# in "violation" end up being output. The query planner correctly deduces +# that it is faster to do a full table scan of the large "violation" table +# to avoid the after-query sort that implements the ORDER BY clause. However, +# if the DSNAME is "ds-two", then only a few rows (about 6) are generated, +# and it is much much faster to do an indexed lookup of "violation" followed +# by a sort operation to implement ORDER BY +# +# The problem, of course, is that the query planner has no way of knowing +# in advance how many rows will be generated. The query planner tries to +# estimate a worst case, which is a large number of output rows, and it picks +# the best plan for that case. However, the plan choosen is very inefficient +# when the number of output rows is small. +# +# The whereInterstageHeuristic() routine in the query planner attempts to +# correct this by adjusting the query plan such that it avoids the very bad +# query plan for a small number of rows, at the expense of a slightly less +# efficient plan for a large number of rows. The large number of rows case +# is perhaps 5% slower with the revised plan, but the small number of +# rows case is around 100 times faster. That seems like a good tradeoff. +# +do_execsql_test 1.0 { + CREATE TABLE datasource(dsid INT, name TEXT); + INSERT INTO datasource VALUES(1,'ds-one'),(2,'ds-two'),(3,'ds-three'); + CREATE INDEX ds1 ON datasource(name, dsid); + + CREATE TABLE rule(rid INT, team_id INT, dsid INT); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<9) + INSERT INTO rule(rid,team_id,dsid) SELECT n, 1, 1 FROM c; + WITH RECURSIVE c(n) AS (VALUES(10) UNION ALL SELECT n+1 FROM c WHERE n<24) + INSERT INTO rule(rid,team_id,dsid) SELECT n, 2, 2 FROM c; + CREATE INDEX rule2 ON rule(dsid, rid); + + CREATE TABLE violation(vid INT, rid INT, vx BLOB); + /*** Uncomment to insert actual data + WITH src(rid, cnt) AS (VALUES(1,3586),(2,1343),(3,6505),(5,76230), + (6,740),(7,287794),(8,457),(12,1), + (14,1),(16,1),(17,1),(18,1),(19,1)) + INSERT INTO violation(vid, rid, vx) + SELECT rid*1000000+value, rid, randomblob(15) + FROM src, generate_series(1,cnt); + ***/ + CREATE INDEX v1 ON violation(rid, vid); + CREATE INDEX v2 ON violation(vid); + ANALYZE; + DELETE FROM sqlite_stat1; + DROP TABLE IF EXISTS sqlite_stat4; + INSERT INTO sqlite_stat1 VALUES + ('violation','v2','376661 1'), + ('violation','v1','376661 28974 1'), + ('rule','rule2','24 12 1'), + ('datasource','ds1','3 1 1'); + ANALYZE sqlite_schema; +} +set DSNAME ds-two ;# Only a few rows. Change to "ds-one" for many rows. +do_eqp_test 1.1 { + SELECT count(*), length(group_concat(vx)) FROM ( + SELECT V.* + FROM datasource DS, rule R, violation V + WHERE V.rid=R.rid + AND R.dsid=DS.dsid + AND DS.name=$DSNAME + ORDER BY V.vid desc + ); +} { + QUERY PLAN + |--CO-ROUTINE (subquery-xxxxxx) + | |--SEARCH DS USING COVERING INDEX ds1 (name=?) + | |--SEARCH R USING COVERING INDEX rule2 (dsid=?) + | |--SEARCH V USING INDEX v1 (rid=?) + | `--USE TEMP B-TREE FOR ORDER BY + `--SCAN (subquery-xxxxxx) +} +# ^^^^---- We want to see three SEARCH terms. No SCAN terms. +# The ORDER BY is implemented by a separate sorter pass. + +finish_test