From e3ec00ccb8ea49dbe8c99fd5e5967dadaddd9852 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 15 Feb 2023 17:53:17 +0000 Subject: [PATCH 001/341] Do not compute result columns of subqueries that are never used. Make those columns NULL instead. This optimization potentially resolves the enhancement request described by [/tktview/baa5bb76c35a124c|ticket baa5bb76c35a124c]. FossilOrigin-Name: 5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 --- manifest | 19 +++++++++------ manifest.uuid | 2 +- src/select.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++ test/with1.test | 2 +- 4 files changed, 78 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 91f4eba3e3..a52c37a9b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stestrunner.tcl\sto\srun\szipvfs\stest\sscripts\son\sunix. -D 2023-02-14T18:09:40.825 +C Do\snot\scompute\sresult\scolumns\sof\ssubqueries\sthat\sare\snever\sused.\s\sMake\sthose\ncolumns\sNULL\sinstead.\s\sThis\soptimization\spotentially\sresolves\sthe\senhancement\nrequest\sdescribed\sby\s[/tktview/baa5bb76c35a124c|ticket\sbaa5bb76c35a124c]. +D 2023-02-15T17:53:17.086 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c c3ce1b49cca2c66c8c88fe7d9e1f3db23590deb4dd631619ad90e1e5d21bcf1f +F src/select.c 49d9336c4108a765831b2483d17cf52e196caaf341cba95646a220ec70ae080f F src/shell.c.in 5cb750dc00bb8b20b020031fd1bb0cb8759a6e567c27aee6b0af31d187984985 F src/sqlite.h.in c060f9e6f646fd817abf273c068666e079fc6e8467ea1d3e8b20f355f82722e9 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1918,7 +1918,7 @@ F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c F test/windowpushd.test d8895d08870b7226f7693665bd292eb177e62ca06799184957b3ca7dc03067df -F test/with1.test 9ad67fdeb2bbd808a5763c9060e214ea232f9b18d846ea3a59756dc38bdc3880 +F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d F test/with2.test a1df41b987198383b9b70bf5e5fda390582e46398653858dbc6ceb24253b28df F test/with3.test e7bf809bf75c1f44f98bca78bc331dbf542002c5227bf53c1261144db4e824c8 F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 @@ -2045,8 +2045,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ecdeef43b27412b0b0b09e09a62ad3a03836a3fc80f2070268090e7ca8f02712 -R 008d808935e4782501905c43f1b48b8b -U dan -Z 0a26d7ad99fa8ab0735a6c8903ff6006 +P e6c8e19ab0d6e7526d4596b75a45bb6becaf3c029690f7e75c016eac803c9990 +R 790a0333fb08765177eb98bd169ddbf7 +T *branch * omit-unused-subquery-columns +T *sym-omit-unused-subquery-columns * +T -sym-trunk * +U drh +Z e4cd68bda867ee6d3fbf83551561c523 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d9053da512..715b2f5ee3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6c8e19ab0d6e7526d4596b75a45bb6becaf3c029690f7e75c016eac803c9990 \ No newline at end of file +5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b0e303066a..4958ddefe9 100644 --- a/src/select.c +++ b/src/select.c @@ -5235,6 +5235,57 @@ static int pushDownWhereTerms( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** Check to see if a subquery contains result-set columns that are +** never used. If it does, change the value of those result-set columns +** to NULL so that they do not cause unnecessary work to compute. +** +** Return the number of column that were changed to NULL. +*/ +static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ + int nCol; + Select *pSub; /* The subquery to be simplified */ + Select *pX; /* For looping over compound elements of pSub */ + Table *pTab; /* The table that describes the subquery */ + int j; /* Column number */ + int nChng = 0; /* Number of columns converted to NULL */ + + assert( pItem!=0 ); + assert( pItem->pTab!=0 ); + pTab = pItem->pTab; + if( pTab->tabFlags & TF_Ephemeral ) return 0; + assert( pItem->pSelect!=0 ); + pSub = pItem->pSelect; + assert( pSub->pEList->nExpr==pTab->nCol ); + for(pX=pSub; pX; pX=pX->pPrior){ + if( pX->pPrior && pX->op!=TK_ALL ){ + /* This optimization does not work for compound subqueries that + ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ + return 0; + } + if( pX->pWin ){ + /* This optimization does not work for subqueries that use window + ** functions. */ + return 0; + } + } + nCol = pTab->nCol; + for(j=0; jcolUsed)!=0 ) continue; + if( MASKBIT(j)==0 ) break; + for(pX=pSub; pX; pX=pX->pPrior) { + Expr *pY = pX->pEList->a[j].pExpr; + if( pY->op==TK_NULL ) continue; + pY->op = TK_NULL; + pX->selFlags |= SF_PushDown; + nChng++; + } + } + return nChng; +} + + /* ** The pFunc is the only aggregate function in the query. Check to see ** if the query is a candidate for the min/max optimization. @@ -7333,6 +7384,20 @@ int sqlite3Select( TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); } + /* Convert unused result columns of the subquery into simple NULL + ** expressions, to avoid unneeded searching and computation. + */ + if( disableUnusedSubqueryResultColumns(pItem) ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, + ("Change unused result columns to NULL for subquery %d:\n", + pSub->selId)); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + } + zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; diff --git a/test/with1.test b/test/with1.test index 91cbc79524..7400a7adf3 100644 --- a/test/with1.test +++ b/test/with1.test @@ -1124,7 +1124,7 @@ do_execsql_test 24.1 { CREATE VIEW v1 AS SELECT max(a), min(b) FROM t1 GROUP BY c; } do_test 24.1 { - set program [db eval {EXPLAIN SELECT 1 FROM v1,v1,v1}] + set program [db eval {EXPLAIN SELECT * FROM v1 AS aa, v1 AS bb, v1 AS cc}] expr [lsearch $program OpenDup]>0 } {1} do_execsql_test 24.2 { From 6945ba787f11eb165ec9f25c359b4c91993643a5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 15 Feb 2023 19:53:08 +0000 Subject: [PATCH 002/341] Fix the subquery result column NULL-ifier so that it correctly handles subquery columns past the 63rd column. FossilOrigin-Name: 77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/select.c | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index a52c37a9b2..38e7b2731d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\scompute\sresult\scolumns\sof\ssubqueries\sthat\sare\snever\sused.\s\sMake\sthose\ncolumns\sNULL\sinstead.\s\sThis\soptimization\spotentially\sresolves\sthe\senhancement\nrequest\sdescribed\sby\s[/tktview/baa5bb76c35a124c|ticket\sbaa5bb76c35a124c]. -D 2023-02-15T17:53:17.086 +C Fix\sthe\ssubquery\sresult\scolumn\sNULL-ifier\sso\sthat\sit\scorrectly\shandles\nsubquery\scolumns\spast\sthe\s63rd\scolumn. +D 2023-02-15T19:53:08.659 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 49d9336c4108a765831b2483d17cf52e196caaf341cba95646a220ec70ae080f +F src/select.c 98984bd8a2f00ae6e2a1c33991617c0d0db68f204ea124cdd853383204499697 F src/shell.c.in 5cb750dc00bb8b20b020031fd1bb0cb8759a6e567c27aee6b0af31d187984985 F src/sqlite.h.in c060f9e6f646fd817abf273c068666e079fc6e8467ea1d3e8b20f355f82722e9 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2045,11 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e6c8e19ab0d6e7526d4596b75a45bb6becaf3c029690f7e75c016eac803c9990 -R 790a0333fb08765177eb98bd169ddbf7 -T *branch * omit-unused-subquery-columns -T *sym-omit-unused-subquery-columns * -T -sym-trunk * +P 5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 +R d380059ff4fc01bc81597a53152668a1 U drh -Z e4cd68bda867ee6d3fbf83551561c523 +Z d76a4bd1d7943a5eace66bda083496dd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 715b2f5ee3..149e343a10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 \ No newline at end of file +77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 4958ddefe9..dc187dfa6a 100644 --- a/src/select.c +++ b/src/select.c @@ -5272,8 +5272,8 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ nCol = pTab->nCol; for(j=0; jcolUsed)!=0 ) continue; - if( MASKBIT(j)==0 ) break; + Bitmask m = jcolUsed)!=0 ) continue; for(pX=pSub; pX; pX=pX->pPrior) { Expr *pY = pX->pEList->a[j].pExpr; if( pY->op==TK_NULL ) continue; From 90b7af77152f2b39ee3de40777487862a83597f3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Feb 2023 15:54:55 +0000 Subject: [PATCH 003/341] Do not perform the omit-unused-subquery-columns optimizations on a subquery that is DISTINCT, as that can lead to incorrect results. FossilOrigin-Name: cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 7 ++++++- test/selectC.test | 5 +++++ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 38e7b2731d..4067e40b48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssubquery\sresult\scolumn\sNULL-ifier\sso\sthat\sit\scorrectly\shandles\nsubquery\scolumns\spast\sthe\s63rd\scolumn. -D 2023-02-15T19:53:08.659 +C Do\snot\sperform\sthe\somit-unused-subquery-columns\soptimizations\son\sa\nsubquery\sthat\sis\sDISTINCT,\sas\sthat\scan\slead\sto\sincorrect\sresults. +D 2023-02-16T15:54:55.738 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 98984bd8a2f00ae6e2a1c33991617c0d0db68f204ea124cdd853383204499697 +F src/select.c a723a35366d7fc695371dd80b2ea4dbc306d5fb75df81d3e6ce7e200794f7622 F src/shell.c.in 5cb750dc00bb8b20b020031fd1bb0cb8759a6e567c27aee6b0af31d187984985 F src/sqlite.h.in c060f9e6f646fd817abf273c068666e079fc6e8467ea1d3e8b20f355f82722e9 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1457,7 +1457,7 @@ F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d F test/select9.test f7586b207ce2304ab80dc93d3146469a28fd4403621dd3a82d06644563d3c812 F test/selectA.test 6aef8b2136a4ac7a3e2e4161d2b8ca7bc6ebe2779de084f9bb66ca9e2323a937 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 -F test/selectC.test fec14c9015ed4ec941508bbc144f30b42e40ac34a4bb33001450369865dd0b75 +F test/selectC.test 38c530b0cc5728b793c3c11f52b52c70290d39822224acd39011c89c1853bd31 F test/selectD.test 6d1909b49970bf92f45ce657505befcef5fc7cbc13544e18103a316d32189bfb F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 @@ -2045,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5dec3cc0225296a043d17f73126d477d90a604f82b3180628176d8f950adbce8 -R d380059ff4fc01bc81597a53152668a1 +P 77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 +R e479bead84e886f8b2c21d0d027a081a U drh -Z d76a4bd1d7943a5eace66bda083496dd +Z 0b93f878d511033e2fa0d6dd3c9e8431 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 149e343a10..3b17642550 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 \ No newline at end of file +cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad \ No newline at end of file diff --git a/src/select.c b/src/select.c index dc187dfa6a..ae320b697b 100644 --- a/src/select.c +++ b/src/select.c @@ -5253,10 +5253,15 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ assert( pItem!=0 ); assert( pItem->pTab!=0 ); pTab = pItem->pTab; - if( pTab->tabFlags & TF_Ephemeral ) return 0; + if( pTab->tabFlags & TF_Ephemeral ){ + return 0; + } assert( pItem->pSelect!=0 ); pSub = pItem->pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); + if( pSub->selFlags & SF_Distinct ){ + return 0; + } for(pX=pSub; pX; pX=pX->pPrior){ if( pX->pPrior && pX->op!=TK_ALL ){ /* This optimization does not work for compound subqueries that diff --git a/test/selectC.test b/test/selectC.test index 63851ca5d7..42fa1d11b8 100644 --- a/test/selectC.test +++ b/test/selectC.test @@ -230,6 +230,11 @@ do_execsql_test selectC-4.2 { select a from (select distinct a, b from t_distinct_bug) } {1 1 1} +do_execsql_test selectC-4.2b { + CREATE VIEW v42b AS SELECT DISTINCT a, b FROM t_distinct_bug; + SELECT a FROM v42b; +} {1 1 1} + do_execsql_test selectC-4.3 { select a, udf() from (select distinct a, b from t_distinct_bug) } {1 1 1 2 1 3} From 7defd20cb0ac85b629ad5b2a7db235464838b193 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Feb 2023 18:04:49 +0000 Subject: [PATCH 004/341] Provide an optimization-disable mask for this optimization. Do not do the optimization if the subquery is an aggregate or is distinct, but allow it to be an ephemeral subquery. Do not omit columns that are used in the ORDER BY of the subquery. FossilOrigin-Name: 6b1a1f374d1372f11f5420d99645b218867100bf070bd3a8885bf5f00c189dff --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 28 ++++++++++++++++++++++------ src/sqliteInt.h | 1 + 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 4067e40b48..75fd2e6d0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sperform\sthe\somit-unused-subquery-columns\soptimizations\son\sa\nsubquery\sthat\sis\sDISTINCT,\sas\sthat\scan\slead\sto\sincorrect\sresults. -D 2023-02-16T15:54:55.738 +C Provide\san\soptimization-disable\smask\sfor\sthis\soptimization.\s\sDo\snot\sdo\sthe\noptimization\sif\sthe\ssubquery\sis\san\saggregate\sor\sis\sdistinct,\sbut\sallow\sit\sto\nbe\san\sephemeral\ssubquery.\s\sDo\snot\somit\scolumns\sthat\sare\sused\sin\sthe\sORDER\sBY\nof\sthe\ssubquery. +D 2023-02-16T18:04:49.549 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,12 +621,12 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c a723a35366d7fc695371dd80b2ea4dbc306d5fb75df81d3e6ce7e200794f7622 +F src/select.c fa66a61608731b4842fb2a2799ddeccbb56b4ec863f8b36d8b08b898ff62067a F src/shell.c.in 5cb750dc00bb8b20b020031fd1bb0cb8759a6e567c27aee6b0af31d187984985 F src/sqlite.h.in c060f9e6f646fd817abf273c068666e079fc6e8467ea1d3e8b20f355f82722e9 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 4dcc36b21dd0c77efeb0bb38b0d31916bc63fb429c8a89d186c945eba0816447 +F src/sqliteInt.h 25f38ade2d93828231d6cd8614f635c0b084bee004360d970542861ad2e3e3d8 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2045,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 77b220a7240425fa83e142ceef78505208e6e38d797070e146b9f7d255f753c7 -R e479bead84e886f8b2c21d0d027a081a +P cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad +R 90e53b2f21c8456ed0ad4d26c75d661a U drh -Z 0b93f878d511033e2fa0d6dd3c9e8431 +Z aa83be3d588d7cfee260c5a1a1ab587b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3b17642550..1e9696bac9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad \ No newline at end of file +6b1a1f374d1372f11f5420d99645b218867100bf070bd3a8885bf5f00c189dff \ No newline at end of file diff --git a/src/select.c b/src/select.c index ae320b697b..b8cae155f3 100644 --- a/src/select.c +++ b/src/select.c @@ -5249,17 +5249,20 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ Table *pTab; /* The table that describes the subquery */ int j; /* Column number */ int nChng = 0; /* Number of columns converted to NULL */ + Bitmask colUsed; /* Columns that may not be NULLed out */ assert( pItem!=0 ); - assert( pItem->pTab!=0 ); - pTab = pItem->pTab; - if( pTab->tabFlags & TF_Ephemeral ){ + if( pItem->fg.isCorrelated || pItem->fg.isCte ){ return 0; } + assert( pItem->pTab!=0 ); + pTab = pItem->pTab; assert( pItem->pSelect!=0 ); pSub = pItem->pSelect; assert( pSub->pEList->nExpr==pTab->nCol ); - if( pSub->selFlags & SF_Distinct ){ + if( (pSub->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ + testcase( pSub->selFlags & SF_Distinct ); + testcase( pSub->selFlags & SF_Aggregate ); return 0; } for(pX=pSub; pX; pX=pX->pPrior){ @@ -5274,11 +5277,22 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ return 0; } } + colUsed = pItem->colUsed; + if( pSub->pOrderBy ){ + ExprList *pList = pSub->pOrderBy; + for(j=0; jnExpr; j++){ + u16 iCol = pList->a[j].u.x.iOrderByCol; + if( iCol>0 ){ + iCol--; + colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + } + } nCol = pTab->nCol; for(j=0; jcolUsed)!=0 ) continue; + if( (m & colUsed)!=0 ) continue; for(pX=pSub; pX; pX=pX->pPrior) { Expr *pY = pX->pEList->a[j].pExpr; if( pY->op==TK_NULL ) continue; @@ -7392,7 +7406,9 @@ int sqlite3Select( /* Convert unused result columns of the subquery into simple NULL ** expressions, to avoid unneeded searching and computation. */ - if( disableUnusedSubqueryResultColumns(pItem) ){ + if( OptimizationEnabled(db, SQLITE_NullUnusedCols) + && disableUnusedSubqueryResultColumns(pItem) + ){ #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x4000 ){ TREETRACE(0x4000,pParse,p, diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9af38e8df4..459af6b4ed 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1839,6 +1839,7 @@ struct sqlite3 { /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ #define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ +#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* From e43d101c83bf30b743d74d07eb63a286c90e0724 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Feb 2023 19:41:39 +0000 Subject: [PATCH 005/341] A few simple test cases for the omit-unused-subquery-column optimization. FossilOrigin-Name: cf8f57c53425d89619ece10edc197d7d28946d3b23fcb4f526330196c76d9cc1 --- manifest | 11 ++--- manifest.uuid | 2 +- test/selectH.test | 110 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 test/selectH.test diff --git a/manifest b/manifest index 75fd2e6d0e..8350761a55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Provide\san\soptimization-disable\smask\sfor\sthis\soptimization.\s\sDo\snot\sdo\sthe\noptimization\sif\sthe\ssubquery\sis\san\saggregate\sor\sis\sdistinct,\sbut\sallow\sit\sto\nbe\san\sephemeral\ssubquery.\s\sDo\snot\somit\scolumns\sthat\sare\sused\sin\sthe\sORDER\sBY\nof\sthe\ssubquery. -D 2023-02-16T18:04:49.549 +C A\sfew\ssimple\stest\scases\sfor\sthe\somit-unused-subquery-column\soptimization. +D 2023-02-16T19:41:39.707 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1462,6 +1462,7 @@ F test/selectD.test 6d1909b49970bf92f45ce657505befcef5fc7cbc13544e18103a316d3218 F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 F test/selectG.test 089f7d3d7e6db91566f00b036cb353107a2cca6220eb1cb264085a836dae8840 +F test/selectH.test a1fed72a83a68ed858db6fcf6d8ddccc4a368645c0ef48dc3ac331d41efe34de F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be F test/sessionfuzz-data1.db 1f8d5def831f19b1c74571037f0d53a588ea49a6c4ca2a028fc0c27ef896dbcb F test/sessionfuzz.c 5eef09af01eeff6f20250ae4c0112c2e576e4d2f2026cc9a49dc5be6886fa6ee @@ -2045,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cc148503db8ef180bce984328da7e84959afadd6a9613c2d03bc1eafeb95dfad -R 90e53b2f21c8456ed0ad4d26c75d661a +P 6b1a1f374d1372f11f5420d99645b218867100bf070bd3a8885bf5f00c189dff +R 1e3c678f9fbc9f1a9649605265913343 U drh -Z aa83be3d588d7cfee260c5a1a1ab587b +Z 7d95cef9a9f7a288c86a6fe2f6e032bb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1e9696bac9..3216d50cee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b1a1f374d1372f11f5420d99645b218867100bf070bd3a8885bf5f00c189dff \ No newline at end of file +cf8f57c53425d89619ece10edc197d7d28946d3b23fcb4f526330196c76d9cc1 \ No newline at end of file diff --git a/test/selectH.test b/test/selectH.test new file mode 100644 index 0000000000..f621fc302c --- /dev/null +++ b/test/selectH.test @@ -0,0 +1,110 @@ +# 2023-02-16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for the omit-unused-subquery-column optimization. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix selectH + +do_execsql_test 1.1 { + CREATE TABLE t1( + c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, + c10, c11, c12, c13, c14, c15, c16, c17, c18, c19, + c20, c21, c22, c23, c24, c25, c26, c27, c28, c29, + c30, c31, c32, c33, c34, c35, c36, c37, c38, c39, + c40, c41, c42, c43, c44, c45, c46, c47, c48, c49, + c50, c51, c52, c53, c54, c55, c56, c57, c58, c59, + c60, c61, c62, c63, c64, c65 + ); + INSERT INTO t1 VALUES( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65 + ); + CREATE INDEX t1c60 ON t1(c60); +} + +# The SQL counter(N) function adjusts the value of the global +# TCL variable ::selectH_cnt by the value N and returns the new +# value. By putting calls to counter(N) as unused columns in a +# view or subquery, we can check to see if the counter gets incremented, +# and if not that means that the unused column was omitted. +# +unset -nocomplain selectH_cnt +set selectH_cnt 0 +proc selectH_counter {amt} { + global selectH_cnt + incr selectH_cnt $amt + return $selectH_cnt +} +db func counter selectH_counter + +do_execsql_test 1.2 { + SELECT DISTINCT c44 FROM ( + SELECT c0 AS a, *, counter(1) FROM t1 + UNION ALL + SELECT c1 AS a, *, counter(1) FROM t1 + ) WHERE c60=60; +} {44} +do_test 1.3 { + set ::selectH_cnt +} {0} + +do_execsql_test 2.1 { + SELECT a FROM ( + SELECT counter(1) AS cnt, c15 AS a, *, c62 AS b FROM t1 + UNION ALL + SELECT counter(1) AS cnt, c16 AS a, *, c61 AS b FROM t1 + ORDER BY b + ); +} {16 15} +do_test 2.2 { + set ::selectH_cnt +} {0} + +do_execsql_test 3.1 { + CREATE VIEW v1 AS + SELECT c16 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c17 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c18 AS a, *, counter(1) AS x FROM t1 + UNION ALL + SELECT c19 AS a, *, counter(1) AS x FROM t1; + SELECT count(*) FROM v1 WHERE c60=60; +} {4} +do_test 3.2 { + set ::selectH_cnt +} {0} +do_execsql_test 3.3 { + SELECT count(a) FROM v1 WHERE c60=60; +} {4} +do_execsql_test 3.4 { + SELECT a FROM v1 WHERE c60=60; +} {16 17 18 19} +do_test 3.5 { + set ::selectH_cnt +} {0} +do_execsql_test 3.6 { + SELECT x FROM v1 WHERE c60=60; +} {1 2 3 4} +do_test 3.7 { + set ::selectH_cnt +} {4} + + +finish_test From 2b83570b5fc07efab77c2b0d93fb1414343e8739 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 21 Feb 2023 18:47:25 +0000 Subject: [PATCH 006/341] Unconditionally disable -sSTRICT_JS flag in wasm build because it no longer works with emsdk 3.1.31+. FossilOrigin-Name: b5e0b69649e6341ffff4e62f9f68abd2a53519f73964559a133f3b708e007e98 --- ext/wasm/GNUmakefile | 13 +++++-------- ext/wasm/fiddle.make | 2 +- manifest | 18 ++++++++---------- manifest.uuid | 2 +- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 7ffd866f24..20dab446da 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -402,13 +402,10 @@ emcc.exportedRuntimeMethods := \ emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY -ifeq (3.1.31,$(emcc.version)) - emcc.jsflags += -sSTRICT_JS=0 - $(warning Disabling -sSTRICT_JS for emcc $(emcc.version): \ - https://github.com/emscripten-core/emscripten/issues/18610) -else - emcc.jsflags += -sSTRICT_JS=1 -endif +emcc.jsflags += -sSTRICT_JS=0 +# STRICT_JS disabled due to: +# https://github.com/emscripten-core/emscripten/issues/18610 +# TL;DR: does not work with MODULARIZE or EXPORT_ES6 as of version 3.1.31. emcc.environment := -sENVIRONMENT=web,worker ######################################################################## # -sINITIAL_MEMORY: How much memory we need to start with is governed @@ -772,7 +769,7 @@ emcc.speedtest1 += -sINITIAL_MEMORY=$(emcc.INITIAL_MEMORY.$(emcc.INITIAL_MEMORY) emcc.speedtest1.common += -sINVOKE_RUN=0 emcc.speedtest1.common += --no-entry emcc.speedtest1.common += -sABORTING_MALLOC -emcc.speedtest1.common += -sSTRICT_JS +emcc.speedtest1.common += -sSTRICT_JS=0 emcc.speedtest1.common += -sMODULARIZE emcc.speedtest1.common += -Wno-limited-postlink-optimizations EXPORTED_FUNCTIONS.speedtest1 := $(abspath $(dir.tmp)/EXPORTED_FUNCTIONS.speedtest1) diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 7facd7e9e5..cbe6ab3518 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -29,7 +29,7 @@ fiddle.emcc-flags = \ --minify 0 \ -sALLOW_TABLE_GROWTH \ -sABORTING_MALLOC \ - -sSTRICT_JS \ + -sSTRICT_JS=0 \ -sENVIRONMENT=web,worker \ -sMODULARIZE \ -sDYNAMIC_EXECUTION=0 \ diff --git a/manifest b/manifest index f17d7340c4..97e15ae883 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.41.0 -D 2023-02-21T18:09:37.622 +C Unconditionally\sdisable\s-sSTRICT_JS\sflag\sin\swasm\sbuild\sbecause\sit\sno\slonger\sworks\swith\semsdk\s3.1.31+. +D 2023-02-21T18:47:25.405 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -467,7 +467,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 5418b4702f4ad0f2162a7e0d128042e8d9219827e3d36978bd2dd6e26ce8f68e +F ext/wasm/GNUmakefile 5a8dc82f5384a3deaaee777ed7eefd3c8cbcd0efe797a66925537d2b975af305 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -508,7 +508,7 @@ F ext/wasm/demo-worker1-promiser.js b99c550763fa792c204e9a7cceadd976004036d9fc3e F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa F ext/wasm/dist.make f55f9c9e1980ea11a59964e59535c66175a17f004d1c2e274522c3366b3a084a -F ext/wasm/fiddle.make 396a0d6e2b3aa63ad448ea71d91676cb9cbbba8f5e6529cf1248d17f29142f47 +F ext/wasm/fiddle.make dbe36b90b8907ae28ecb9c0e9fd8389dbdaecf117ea4fb2ea33864bdfa498a94 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa061ce8776a6e75da8a10 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 @@ -2045,10 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 409a19145e9558686acab730b94c6d53691b6d0efcea1e9239fc7e5863586a85 -R 8cba8b939a0caa0ad935e665db07c75b -T +sym-release * -T +sym-version-3.41.0 * -U drh -Z dc017a46fcafa54656c31865fbc11b74 +P 05941c2a04037fc3ed2ffae11f5d2260706f89431f463518740f72ada350866d +R ba70acd94c3027fbd106bb41377e11ce +U stephan +Z 6286e3ef4f3e7ade2798e4e70d43ed4e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 93e13b11f8..683d6c812f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -05941c2a04037fc3ed2ffae11f5d2260706f89431f463518740f72ada350866d \ No newline at end of file +b5e0b69649e6341ffff4e62f9f68abd2a53519f73964559a133f3b708e007e98 \ No newline at end of file From dbb32606278de2d90fb761cf79edc7d6820ef64e Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 21 Feb 2023 20:08:10 +0000 Subject: [PATCH 007/341] Remove some unnecessary bind() calls in JS code. FossilOrigin-Name: 71215599cd8ebb970500d8366dfac40420a9a6b243ea9d6ba62d5cde28fae3d3 --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 6 +++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 3e3255b0c8..c709e8b1ad 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -119,9 +119,9 @@ const installOpfsVfs = function callee(options){ } const thePromise = new Promise(function(promiseResolve, promiseReject_){ const loggers = { - 0:sqlite3.config.error.bind(console), - 1:sqlite3.config.warn.bind(console), - 2:sqlite3.config.log.bind(console) + 0:sqlite3.config.error, + 1:sqlite3.config.warn, + 2:sqlite3.config.log }; const logImpl = (level,...args)=>{ if(options.verbose>level) loggers[level]("OPFS syncer:",...args); diff --git a/manifest b/manifest index 97e15ae883..00adf2e1b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Unconditionally\sdisable\s-sSTRICT_JS\sflag\sin\swasm\sbuild\sbecause\sit\sno\slonger\sworks\swith\semsdk\s3.1.31+. -D 2023-02-21T18:47:25.405 +C Remove\ssome\sunnecessary\sbind()\scalls\sin\sJS\scode. +D 2023-02-21T20:08:10.896 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -486,7 +486,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9551f04cdfcde354e5a6ccb48951e007d618abb4e9 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 7795b84b66a7a8dedc791340709b310bb497c3c72a80bef364fa2a58e2ddae3f F ext/wasm/api/sqlite3-v-helper.js 6f6c3e390a72e08b0a5b16a0d567d7af3c04d172831853a29d72a6f1dd40ff24 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js ca291837840b3eae3a60810721a7970c98f7c7cd3ee1c879acb7e91f1e3fe65a +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 0dfddd0fcd354817c3d5887decebb8a293cbb926c2639ba09b995a524f1085fb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 76625a70937a8522d014ef686c32db5b53a3ee61850323f5c601d2ac39fe52fe F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js c5ac33e39f21a3481812d7333ca6e18853640d423a01960ca8dbc6e7c5c3c21c @@ -2045,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 05941c2a04037fc3ed2ffae11f5d2260706f89431f463518740f72ada350866d -R ba70acd94c3027fbd106bb41377e11ce +P b5e0b69649e6341ffff4e62f9f68abd2a53519f73964559a133f3b708e007e98 +R dd3f5c20eacca5476bd8e8ef24e7cee2 U stephan -Z 6286e3ef4f3e7ade2798e4e70d43ed4e +Z 4ba8ee1ea4c59526ff8ff4648a8a2201 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 683d6c812f..bd009ded60 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5e0b69649e6341ffff4e62f9f68abd2a53519f73964559a133f3b708e007e98 \ No newline at end of file +71215599cd8ebb970500d8366dfac40420a9a6b243ea9d6ba62d5cde28fae3d3 \ No newline at end of file From bcb3215630380b2c0c4a62d19a13a12a192dfa02 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 22 Feb 2023 20:42:15 +0000 Subject: [PATCH 008/341] Remove the push-down optimization restriction that was added by check-in [1ad41840c5e0fa70] because it is no longer needed after the enhancements to compound query processing in check-in [27655c9353620aa5] This resolves the performance regression reported by [forum:/forumpost/bcc4375032|forum post bcc4375032]. FossilOrigin-Name: aa6bd6dff751223e302575c60f335707e4bb820bff716cab3706b564e18ed7ed --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 10 +--------- test/pushdown.test | 42 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 51 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 00adf2e1b0..4c15df292f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sunnecessary\sbind()\scalls\sin\sJS\scode. -D 2023-02-21T20:08:10.896 +C Remove\sthe\spush-down\soptimization\srestriction\sthat\swas\sadded\sby\ncheck-in\s[1ad41840c5e0fa70]\sbecause\sit\sis\sno\slonger\sneeded\safter\nthe\senhancements\sto\scompound\squery\sprocessing\sin\scheck-in\s[27655c9353620aa5]\nThis\sresolves\sthe\sperformance\sregression\sreported\sby\n[forum:/forumpost/bcc4375032|forum\spost\sbcc4375032]. +D 2023-02-22T20:42:15.004 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c d44e78e103cb52b3bdf8d0b7ac2518b7a4cbaf12237bd9078f6897d230d2516e +F src/select.c f80d98f13fa64201ababf5e80d682c430559cd7ee00ab14ee99a1aeb27ac097d F src/shell.c.in 6f36f5ca05f1bebf74935a7fcf2dce983016e807a09cbd752a673583ad7da087 F src/sqlite.h.in 662a2fa083d093896b92560c871dea6d86792b49dc4bf7b4e8dbeca8e7171488 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1385,7 +1385,7 @@ F test/printf.test 390d0d7fcffc3c4ea3c1bb4cbb267444e32b33b048ae21895f23a291844fe F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test f270b8071c02efc218430e0d388c155e1962eaa1d3a3ab186dd38ad6d7e178a4 +F test/pushdown.test 35c23334e5d29d90b1f8360879352dc84d6d5d08aa434b82e455d015823b7c30 F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test f86b25b33455af0189b4d3fe7bd6e553115e80b2d7ec9bbe9a6b37fce0881bfe @@ -2045,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b5e0b69649e6341ffff4e62f9f68abd2a53519f73964559a133f3b708e007e98 -R dd3f5c20eacca5476bd8e8ef24e7cee2 -U stephan -Z 4ba8ee1ea4c59526ff8ff4648a8a2201 +P 71215599cd8ebb970500d8366dfac40420a9a6b243ea9d6ba62d5cde28fae3d3 +R b7b0698140aa56dfbaf179da7c992bef +U drh +Z 71e4a1ce17f1f7f2e2a274a357385131 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bd009ded60..6c00e3fb60 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71215599cd8ebb970500d8366dfac40420a9a6b243ea9d6ba62d5cde28fae3d3 \ No newline at end of file +aa6bd6dff751223e302575c60f335707e4bb820bff716cab3706b564e18ed7ed \ No newline at end of file diff --git a/src/select.c b/src/select.c index 1b5a1b506e..23099b1397 100644 --- a/src/select.c +++ b/src/select.c @@ -4238,8 +4238,7 @@ static int compoundHasDifferentAffinities(Select *p){ ** query or there are no RIGHT or FULL JOINs in any arm ** of the subquery. (This is a duplicate of condition (27b).) ** (17h) The corresponding result set expressions in all arms of the -** compound must have the same affinity. (See restriction (9) -** on the push-down optimization.) +** compound must have the same affinity. ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -5107,10 +5106,6 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** or EXCEPT, then all of the result set columns for all arms of ** the compound must use the BINARY collating sequence. ** -** (9) If the subquery is a compound, then all arms of the compound must -** have the same affinity. (This is the same as restriction (17h) -** for query flattening.) -** ** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. @@ -5141,9 +5136,6 @@ static int pushDownWhereTerms( if( pSel->pWin ) return 0; /* restriction (6b) */ #endif } - if( compoundHasDifferentAffinities(pSubq) ){ - return 0; /* restriction (9) */ - } if( notUnionAll ){ /* If any of the compound arms are connected using UNION, INTERSECT, ** or EXCEPT, then we must ensure that none of the columns use a diff --git a/test/pushdown.test b/test/pushdown.test index ca816fb213..660827e4e4 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -122,5 +122,45 @@ do_execsql_test 3.4 { } { 2 0 0 } - + +# 2023-02-22 https://sqlite.org/forum/forumpost/bcc4375032 +# Performance regression caused by check-in [1ad41840c5e0fa70] from 2022-11-25. +# That check-in added a new restriction on push-down. The new restriction is +# no longer necessary after check-in [27655c9353620aa5] from 2022-12-14. +# +do_execsql_test 3.5 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INT, b INT, c TEXT, PRIMARY KEY(a,b)) WITHOUT ROWID; + INSERT INTO t1(a,b,c) VALUES + (1,100,'abc'), + (2,200,'def'), + (3,300,'abc'); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(a INT, b INT, c TEXT, PRIMARY KEY(a,b)) WITHOUT ROWID; + INSERT INTO t2(a,b,c) VALUES + (1,110,'efg'), + (2,200,'hij'), + (3,330,'klm'); + CREATE VIEW v3 AS + SELECT a, b, c FROM t1 + UNION ALL + SELECT a, b, 'xyz' FROM t2; + SELECT * FROM v3 WHERE a=2 AND b=200; +} {2 200 def 2 200 xyz} +do_eqp_test 3.6 { + SELECT * FROM v3 WHERE a=2 AND b=200; +} { + QUERY PLAN + |--CO-ROUTINE v3 + | `--COMPOUND QUERY + | |--LEFT-MOST SUBQUERY + | | `--SEARCH t1 USING PRIMARY KEY (a=? AND b=?) + | `--UNION ALL + | `--SEARCH t2 USING PRIMARY KEY (a=? AND b=?) + `--SCAN v3 +} +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# We want both arms of the compound subquery to use the +# primary key. + finish_test From 7b56a0eacdc490f91e710cce59dcadcf2ece08b9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 22 Feb 2023 20:50:51 +0000 Subject: [PATCH 009/341] Update the version number to 3.42.0 to begin the next development cycle. FossilOrigin-Name: 65910216393ed287dbc7e42a2598b6980183c62a64da6572cb691ff4ee23555d --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 371986f6df..476cebe063 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.41.0 +3.42.0 diff --git a/configure b/configure index 56e74f5ecc..29ca76b69e 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.41.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.42.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.41.0' -PACKAGE_STRING='sqlite 3.41.0' +PACKAGE_VERSION='3.42.0' +PACKAGE_STRING='sqlite 3.42.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1470,7 +1470,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.41.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.42.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1535,7 +1535,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.41.0:";; + short | recursive ) echo "Configuration of sqlite 3.42.0:";; esac cat <<\_ACEOF @@ -1665,7 +1665,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.41.0 +sqlite configure 3.42.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2084,7 +2084,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.41.0, which was +It was created by sqlite $as_me 3.42.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -12457,7 +12457,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.41.0, which was +This file was extended by sqlite $as_me 3.42.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12523,7 +12523,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.41.0 +sqlite config.status 3.42.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/manifest b/manifest index 4c15df292f..cec9479716 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\spush-down\soptimization\srestriction\sthat\swas\sadded\sby\ncheck-in\s[1ad41840c5e0fa70]\sbecause\sit\sis\sno\slonger\sneeded\safter\nthe\senhancements\sto\scompound\squery\sprocessing\sin\scheck-in\s[27655c9353620aa5]\nThis\sresolves\sthe\sperformance\sregression\sreported\sby\n[forum:/forumpost/bcc4375032|forum\spost\sbcc4375032]. -D 2023-02-22T20:42:15.004 +C Update\sthe\sversion\snumber\sto\s3.42.0\sto\sbegin\sthe\snext\sdevelopment\scycle. +D 2023-02-22T20:50:51.768 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -7,7 +7,7 @@ F Makefile.in 594f07f8829020a45d825234edf14046544b5bf6bc0057a9d5b83b58686b1c46 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc b0026b61d1242add00c28dbe169271a763fffcb3af6762bcdbeb20c3efe6a69f F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e -F VERSION 413ec94920a487ae32c9a2a8819544d690662d6f7c7ce025c0d0b8a1e74fa9db +F VERSION 17f95ae2fdf21f0e9575eb0b0511ea63f15d71dfff431b21c2b4adbfa70cfbbf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 @@ -33,7 +33,7 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034 F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559 -F configure 6ba84abf7878bb072d460766d29e492028468775d4c09464940343092eb6493c x +F configure 2f2c090e0a1d051bb53741f0c961f5ebb24507a4f599d686ad6d7ff236144609 x F configure.ac 4654d32ac0a0d0b48f1e1e79bdc3d777b723cf2f63c33eb1d7c4ed8b435938e8 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd @@ -2045,8 +2045,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 71215599cd8ebb970500d8366dfac40420a9a6b243ea9d6ba62d5cde28fae3d3 -R b7b0698140aa56dfbaf179da7c992bef +P aa6bd6dff751223e302575c60f335707e4bb820bff716cab3706b564e18ed7ed +R 311ae6e51c5af4c947ddc5fe5e93f886 U drh -Z 71e4a1ce17f1f7f2e2a274a357385131 +Z 558b030af06c9bbcf0267311941c1842 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6c00e3fb60..fe2ab7c19f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa6bd6dff751223e302575c60f335707e4bb820bff716cab3706b564e18ed7ed \ No newline at end of file +65910216393ed287dbc7e42a2598b6980183c62a64da6572cb691ff4ee23555d \ No newline at end of file From af03eb3241c832d7352eedc11701eac55d3fc67e Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 22 Feb 2023 21:47:02 +0000 Subject: [PATCH 010/341] Enable the count-of-view optimization by default. Enhancement request [eaed8e36ce888f1e]. FossilOrigin-Name: a4aacdd323a854d771c8cb1e2e4cfc4fb66b0020cfed23525733603605f5c63b --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/select.c | 4 ---- test/pushdown.test | 21 +++++++++++++++++++++ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 65bf1958f6..05ecf9a686 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scomputing\sthe\svalues\sfor\sunused\sresult-set\scolumns\sin\ssubqueries.\nPerformance\soptimization\srequest\s[baa5bb76c35a124c]. -D 2023-02-22T21:11:34.555 +C Enable\sthe\scount-of-view\soptimization\sby\sdefault.\nEnhancement\srequest\s[eaed8e36ce888f1e]. +D 2023-02-22T21:47:02.308 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 751d2f021cb439abdf3e8e7b74c5d0532ca9793868e4fb4c5a73412aea274edc +F src/select.c 39ec68e3e88064c1ce76eb8d3fc7dbf97a95a54ff0c37e22d94df2e33550da6a F src/shell.c.in 6f36f5ca05f1bebf74935a7fcf2dce983016e807a09cbd752a673583ad7da087 F src/sqlite.h.in 662a2fa083d093896b92560c871dea6d86792b49dc4bf7b4e8dbeca8e7171488 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1385,7 +1385,7 @@ F test/printf.test 390d0d7fcffc3c4ea3c1bb4cbb267444e32b33b048ae21895f23a291844fe F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/pushdown.test 35c23334e5d29d90b1f8360879352dc84d6d5d08aa434b82e455d015823b7c30 +F test/pushdown.test c183fa51f93bb3a604eee9141133e36b5fbef0aac3b0447e464d977a84d6d00a F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quickcheck.test f86b25b33455af0189b4d3fe7bd6e553115e80b2d7ec9bbe9a6b37fce0881bfe @@ -2046,9 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 65910216393ed287dbc7e42a2598b6980183c62a64da6572cb691ff4ee23555d cf8f57c53425d89619ece10edc197d7d28946d3b23fcb4f526330196c76d9cc1 -R 34649150ad57f18afb1178077e5cdae5 -T +closed cf8f57c53425d89619ece10edc197d7d28946d3b23fcb4f526330196c76d9cc1 +P 7c2d3406000dc8ac5a99cc205b036356b67e4b0b94738592ffc5680749696904 +R 5d4e72d33df5d01cbe3901dfb215f87e U drh -Z c744fac449b59dcb5adcaf0108c5c30e +Z 008d21ed5d10af73f833ad6a800d4467 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4b12d53327..a3eae223f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c2d3406000dc8ac5a99cc205b036356b67e4b0b94738592ffc5680749696904 \ No newline at end of file +a4aacdd323a854d771c8cb1e2e4cfc4fb66b0020cfed23525733603605f5c63b \ No newline at end of file diff --git a/src/select.c b/src/select.c index 3095685a8f..0018e7988b 100644 --- a/src/select.c +++ b/src/select.c @@ -6838,7 +6838,6 @@ static void agginfoFree(sqlite3 *db, AggInfo *p){ sqlite3DbFreeNN(db, p); } -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION /* ** Attempt to transform a query of the form ** @@ -6926,7 +6925,6 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ #endif return 1; } -#endif /* SQLITE_COUNTOFVIEW_OPTIMIZATION */ /* ** If any term of pSrc, or any SF_NestedFrom sub-query, is not the same @@ -7315,7 +7313,6 @@ int sqlite3Select( TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) && countOfViewOptimization(pParse, p) ){ @@ -7323,7 +7320,6 @@ int sqlite3Select( pEList = p->pEList; pTabList = p->pSrc; } -#endif /* For each term in the FROM clause, do two things: ** (1) Authorized unreferenced tables diff --git a/test/pushdown.test b/test/pushdown.test index 660827e4e4..b9ee48c4ac 100644 --- a/test/pushdown.test +++ b/test/pushdown.test @@ -163,4 +163,25 @@ do_eqp_test 3.6 { # We want both arms of the compound subquery to use the # primary key. +# The following is a test of the count-of-view optimization. This does +# not have anything to do with push-down. It is here because this is a +# convenient place to put the test. +# +do_execsql_test 3.7 { + SELECT count(*) FROM v3; +} 6 +do_eqp_test 3.8 { + SELECT count(*) FROM v3; +} { + QUERY PLAN + |--SCAN CONSTANT ROW + |--SCALAR SUBQUERY xxxxxx + | `--SCAN t1 + `--SCALAR SUBQUERY xxxxxx + `--SCAN t2 +} +# ^^^^^^^^^^^^^^^^^^^^ +# The query should be converted into: +# SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2) + finish_test From 3c2688d13770ea405d890eb7949a675eb237db20 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 23 Feb 2023 01:52:54 +0000 Subject: [PATCH 011/341] Provide -DHAVE_LOG2=0 and -DHAVE_LOG10=0 compile-time options for use on systems that lack the log2() and log10() standard math library routines, to cause SQLite to substitute its own alternatives. FossilOrigin-Name: 7ee22f95e7a7d8650f961f20e304e56c7813e624f05655d7392ca9347748270f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 05ecf9a686..a376f1ad12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sthe\scount-of-view\soptimization\sby\sdefault.\nEnhancement\srequest\s[eaed8e36ce888f1e]. -D 2023-02-22T21:47:02.308 +C Provide\s-DHAVE_LOG2=0\sand\s-DHAVE_LOG10=0\scompile-time\soptions\sfor\suse\son\ssystems\nthat\slack\sthe\slog2()\sand\slog10()\sstandard\smath\slibrary\sroutines,\sto\scause\nSQLite\sto\ssubstitute\sits\sown\salternatives. +D 2023-02-23T01:52:54.076 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -574,7 +574,7 @@ F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e F src/expr.c 2e5e67e800c9416996df52409746a2f3c7dadf5d38b5a367ce379b239e3f40c8 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 -F src/func.c 64c4cb2eebf6ddb6b4b08dc17790e7152b887f223f8caf0c0fe96da1c5a876ca +F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 F src/global.c e06ff8e0acd85aec13563c9ecb44fbbf38232ccf73594998fd880b92d619594b F src/hash.c c6af5f96a7a76d000f07c5402c48c318c2566beecdee9e78b9d9f60ce7119565 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7c2d3406000dc8ac5a99cc205b036356b67e4b0b94738592ffc5680749696904 -R 5d4e72d33df5d01cbe3901dfb215f87e +P a4aacdd323a854d771c8cb1e2e4cfc4fb66b0020cfed23525733603605f5c63b +R d9f0889f326fe560e4856fa0dae980c0 U drh -Z 008d21ed5d10af73f833ad6a800d4467 +Z c988455137d4c4b07349732581414345 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a3eae223f4..3a0aa55d96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4aacdd323a854d771c8cb1e2e4cfc4fb66b0020cfed23525733603605f5c63b \ No newline at end of file +7ee22f95e7a7d8650f961f20e304e56c7813e624f05655d7392ca9347748270f \ No newline at end of file diff --git a/src/func.c b/src/func.c index c1efd8acf8..045c60613b 100644 --- a/src/func.c +++ b/src/func.c @@ -2161,6 +2161,18 @@ static void ceilingFunc( static double xCeil(double x){ return ceil(x); } static double xFloor(double x){ return floor(x); } +/* +** Some systems do not have log2() and log10() in their standard math +** libraries. +*/ +#if defined(HAVE_LOG10) && HAVE_LOG10==0 +# define log10(X) (0.4342944819032517867*log(X)) +#endif +#if defined(HAVE_LOG2) && HAVE_LOG2==0 +# define log2(X) (1.442695040888963456*log(X)) +#endif + + /* ** Implementation of SQL functions: ** From ad96db8df5e6b80ac8139487c4ae915f9cdea9a0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 23 Feb 2023 14:22:29 +0000 Subject: [PATCH 012/341] Allow the sqlite3_config() interface to be invoked at any time for a few choosen options: SQLITE_CONFIG_LOG, SQLITE_CONFIG_URI, and SQLITE_CONFIG_PCACHE_HDRSZ. This list will likely change before release. FossilOrigin-Name: e1702eb48d13c7c9b7605f1e77242672222c53059edcdc4e9cea59510715822a --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/main.c | 36 +++++++++++++++++++++++++++--------- src/sqlite.h.in | 37 +++++++++++++++++++++++++++++-------- src/sqliteInt.h | 4 ++-- 5 files changed, 70 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index a376f1ad12..e62a2ae789 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Provide\s-DHAVE_LOG2=0\sand\s-DHAVE_LOG10=0\scompile-time\soptions\sfor\suse\son\ssystems\nthat\slack\sthe\slog2()\sand\slog10()\sstandard\smath\slibrary\sroutines,\sto\scause\nSQLite\sto\ssubstitute\sits\sown\salternatives. -D 2023-02-23T01:52:54.076 +C Allow\sthe\ssqlite3_config()\sinterface\sto\sbe\sinvoked\sat\sany\stime\sfor\sa\sfew\nchoosen\soptions:\s\sSQLITE_CONFIG_LOG,\sSQLITE_CONFIG_URI,\sand\nSQLITE_CONFIG_PCACHE_HDRSZ.\s\sThis\slist\swill\slikely\schange\sbefore\srelease. +D 2023-02-23T14:22:29.829 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -584,7 +584,7 @@ F src/insert.c 1b11a2e33ee52db93c02fddac67e39d00161d61b69fac2675b82f2aa68c1b61c F src/json.c c85ed6fce06f43d414b0d7fff64749d43a0dbd1067123ee407bd3a0752454161 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c b04eb648cedc45efe4298e1ef439ac4f0096ae27b5f01accb0a1f49d57789128 -F src/main.c f5ed7b748d7e54b96ac577a9d87af9461f13f092bbe684d473448010333402dc +F src/main.c 437e97a14936763b42e013f02495ef9c0bca6654073321c2bdd844cb4ec441f5 F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -623,10 +623,10 @@ F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 39ec68e3e88064c1ce76eb8d3fc7dbf97a95a54ff0c37e22d94df2e33550da6a F src/shell.c.in 6f36f5ca05f1bebf74935a7fcf2dce983016e807a09cbd752a673583ad7da087 -F src/sqlite.h.in 662a2fa083d093896b92560c871dea6d86792b49dc4bf7b4e8dbeca8e7171488 +F src/sqlite.h.in 3bdc92831511442cad4de5243b619ba8b887bb102bb464bbc5436df7e2e6568a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 25f38ade2d93828231d6cd8614f635c0b084bee004360d970542861ad2e3e3d8 +F src/sqliteInt.h cb7182dcdc9910d5f1352c90762545cc5ffb79c4a47f4ae7c5ee044fdb80423b F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2046,8 +2046,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4aacdd323a854d771c8cb1e2e4cfc4fb66b0020cfed23525733603605f5c63b -R d9f0889f326fe560e4856fa0dae980c0 +P 7ee22f95e7a7d8650f961f20e304e56c7813e624f05655d7392ca9347748270f +R dd36b4ed22c075361723563dc0d19a92 +T *branch * anytime-config +T *sym-anytime-config * +T -sym-trunk * U drh -Z c988455137d4c4b07349732581414345 +Z be923bc0bbf1d0d6df34f343f10e1fcc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3a0aa55d96..ac4288a8be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ee22f95e7a7d8650f961f20e304e56c7813e624f05655d7392ca9347748270f \ No newline at end of file +e1702eb48d13c7c9b7605f1e77242672222c53059edcdc4e9cea59510715822a \ No newline at end of file diff --git a/src/main.c b/src/main.c index eaecb56df9..4ca81a6ffb 100644 --- a/src/main.c +++ b/src/main.c @@ -430,9 +430,23 @@ int sqlite3_config(int op, ...){ va_list ap; int rc = SQLITE_OK; - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + /* sqlite3_config() normally returns SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. Except, a few selected opcodes + ** are allowed. + */ + if( sqlite3GlobalConfig.isInit ){ + static const u64 mAnytimeConfigOption = 0 + | MASKBIT64( SQLITE_CONFIG_LOG ) + | MASKBIT64( SQLITE_CONFIG_URI ) + | MASKBIT64( SQLITE_CONFIG_PCACHE_HDRSZ ) + ; + if( op<0 || op>63 || (MASKBIT64(op) & mAnytimeConfigOption)==0 ){ + return SQLITE_MISUSE_BKPT; + } + testcase( op==SQLITE_CONFIG_LOG ); + testcase( op==SQLITE_CONFIG_URI ); + testcase( op==SQLITE_CONFIG_PCACHE_HDRSZ ); + } va_start(ap, op); switch( op ){ @@ -501,6 +515,7 @@ int sqlite3_config(int op, ...){ break; } case SQLITE_CONFIG_MEMSTATUS: { + assert( !sqlite3GlobalConfig.isInit ); /* Cannot change at runtime */ /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes ** single argument of type int, interpreted as a boolean, which enables ** or disables the collection of memory allocation statistics. */ @@ -624,8 +639,10 @@ int sqlite3_config(int op, ...){ ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); */ typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + LOGFUNC_t xLog = va_arg(ap, LOGFUNC_t); + void *pLogArg = va_arg(ap, void*); + AtomicStore(&sqlite3GlobalConfig.xLog, xLog); + AtomicStore(&sqlite3GlobalConfig.pLogArg, pLogArg); break; } @@ -639,7 +656,8 @@ int sqlite3_config(int op, ...){ ** argument of type int. If non-zero, then URI handling is globally ** enabled. If the parameter is zero, then URI handling is globally ** disabled. */ - sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + int bOpenUri = va_arg(ap, int); + AtomicStore(&sqlite3GlobalConfig.bOpenUri, bOpenUri); break; } @@ -2939,9 +2957,9 @@ int sqlite3ParseUri( assert( *pzErrMsg==0 ); - if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ - || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ - && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || AtomicLoad(&sqlite3GlobalConfig.bOpenUri)) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; int eState; /* Parser state when parsing URI */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 049bc17137..198e209dd8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1655,20 +1655,23 @@ int sqlite3_os_end(void); ** must ensure that no other SQLite interfaces are invoked by other ** threads while sqlite3_config() is running. ** -** The sqlite3_config() interface -** may only be invoked prior to library initialization using -** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. -** ^If sqlite3_config() is called after [sqlite3_initialize()] and before -** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. -** Note, however, that ^sqlite3_config() can be called as part of the -** implementation of an application-defined [sqlite3_os_init()]. -** ** The first argument to sqlite3_config() is an integer ** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments ** vary depending on the [configuration option] ** in the first argument. ** +** For most configuration options, the sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** The exceptional configuration options that may be invoked at any time +** are called "anytime configuration options". +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] with a first argument that is not an anytime +** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. @@ -1776,6 +1779,24 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. ** +** Most of the configuration options for sqlite3_config() +** will only work if invoked prior to [sqlite3_initialize()] or after +** [sqlite3_shutdown()]. The few exceptions to this rule are called +** "anytime configuration options". +** ^Calling [sqlite3_config()] with a first argument that is not an +** anytime configuration option in between calls to [sqlite3_initialize()] and +** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE. +** +** The set of anytime configuration options can change (by insertions +** and/or deletions) from one release of SQLite to the next. +** As of SQLite version 3.42.0, the complete set of anytime configuration +** options is: +**
    +**
  • SQLITE_CONFIG_LOG +**
  • SQLITE_CONFIG_URI +**
  • SQLITE_CONFIG_PCACHE_HDRSZ +**
+** ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications ** should check the return code from [sqlite3_config()] to make sure that diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 459af6b4ed..40afc08dbb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -224,8 +224,8 @@ #endif /* -** WAL mode depends on atomic aligned 32-bit loads and stores in a few -** places. The following macros try to make this explicit. +** A few places in the code require atomic load/store of aligned +** integer values. */ #ifndef __has_extension # define __has_extension(x) 0 /* compatibility with non-clang compilers */ From 648de49c03c496f535304f8bfdf986df051553f8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 23 Feb 2023 14:43:15 +0000 Subject: [PATCH 013/341] Fix harmless compiler warnings and a code indentation error. FossilOrigin-Name: de6c5c6bb49a0beca793cdc0462da2c17bb05f3292e08dd815be29b45b59a8fb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a376f1ad12..e6f9823458 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Provide\s-DHAVE_LOG2=0\sand\s-DHAVE_LOG10=0\scompile-time\soptions\sfor\suse\son\ssystems\nthat\slack\sthe\slog2()\sand\slog10()\sstandard\smath\slibrary\sroutines,\sto\scause\nSQLite\sto\ssubstitute\sits\sown\salternatives. -D 2023-02-23T01:52:54.076 +C Fix\sharmless\scompiler\swarnings\sand\sa\scode\sindentation\serror. +D 2023-02-23T14:43:15.500 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 39ec68e3e88064c1ce76eb8d3fc7dbf97a95a54ff0c37e22d94df2e33550da6a +F src/select.c a6a5cc0bbb97be616a2608cb5d063b6635b920ecd6b7b3445cde88597daf3542 F src/shell.c.in 6f36f5ca05f1bebf74935a7fcf2dce983016e807a09cbd752a673583ad7da087 F src/sqlite.h.in 662a2fa083d093896b92560c871dea6d86792b49dc4bf7b4e8dbeca8e7171488 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4aacdd323a854d771c8cb1e2e4cfc4fb66b0020cfed23525733603605f5c63b -R d9f0889f326fe560e4856fa0dae980c0 +P 7ee22f95e7a7d8650f961f20e304e56c7813e624f05655d7392ca9347748270f +R 6f3dc9114a2c05702b3aef7d84d0887b U drh -Z c988455137d4c4b07349732581414345 +Z ad82adef5ba3f74b5f1d1691fda9e306 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3a0aa55d96..1982ec0f0e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ee22f95e7a7d8650f961f20e304e56c7813e624f05655d7392ca9347748270f \ No newline at end of file +de6c5c6bb49a0beca793cdc0462da2c17bb05f3292e08dd815be29b45b59a8fb \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0018e7988b..1c71dbe548 100644 --- a/src/select.c +++ b/src/select.c @@ -5269,7 +5269,7 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ return 0; } } - colUsed = pItem->colUsed; + colUsed = pItem->colUsed; if( pSub->pOrderBy ){ ExprList *pList = pSub->pOrderBy; for(j=0; jnExpr; j++){ @@ -5282,7 +5282,6 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ } nCol = pTab->nCol; for(j=0; jpPrior) { @@ -7317,7 +7316,6 @@ int sqlite3Select( && countOfViewOptimization(pParse, p) ){ if( db->mallocFailed ) goto select_end; - pEList = p->pEList; pTabList = p->pSrc; } From 4df23a3248c864e7b5781e30434ada5a75b5e5dd Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 23 Feb 2023 21:18:47 +0000 Subject: [PATCH 014/341] Increased precision of floating-point to decimal conversions when the floating point value has no fractional part. [forum:/forumpost/d1387c3979c7f557|Forum post d1387c3979c7f557] FossilOrigin-Name: 18de3a8e6b431a075143631eafdcf0e1b1a21868b417394c365d46d2feca10b5 --- manifest | 14 +++--- manifest.uuid | 2 +- src/printf.c | 127 ++++++++++++++++++++++++++++++++--------------- test/printf.test | 35 +++++++++++++ 4 files changed, 130 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index e6f9823458..5e91b785cb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sand\sa\scode\sindentation\serror. -D 2023-02-23T14:43:15.500 +C Increased\sprecision\sof\sfloating-point\sto\sdecimal\sconversions\swhen\sthe\nfloating\spoint\svalue\shas\sno\sfractional\spart.\n[forum:/forumpost/d1387c3979c7f557|Forum\spost\sd1387c3979c7f557] +D 2023-02-23T21:18:47.560 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -617,7 +617,7 @@ F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 82d5090a35eac75876d3b41f48d06b2370553c9576bf2942233c462e03eb94c9 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 -F src/printf.c ff4b05e38bf928ff1b80d3dda4f977b10fe39ecbfe69c018224c7e5594fb2455 +F src/printf.c d7b8c87d3525ee3eea3c4d291023540c88788e47a65c5b1dcff1b432ec9dbda5 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 @@ -1381,7 +1381,7 @@ F test/pragma4.test ca5e4dfc46adfe490f75d73734f70349d95a199e6510973899e502eef2c8 F test/pragma5.test 7b33fc43e2e41abf17f35fb73f71b49671a380ea92a6c94b6ce530a25f8d9102 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8 F test/prefixes.test b524a1c44bffec225b9aec98bd728480352aa8532ac4c15771fb85e8beef65d9 -F test/printf.test 390d0d7fcffc3c4ea3c1bb4cbb267444e32b33b048ae21895f23a291844fe1da +F test/printf.test 22449e13983e9d94fe9a671db68a086c4c55dea51daad825ef7ebb3d192ee997 F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7ee22f95e7a7d8650f961f20e304e56c7813e624f05655d7392ca9347748270f -R 6f3dc9114a2c05702b3aef7d84d0887b +P de6c5c6bb49a0beca793cdc0462da2c17bb05f3292e08dd815be29b45b59a8fb +R 4769ae3a046ae725cd776ab02bbf2890 U drh -Z ad82adef5ba3f74b5f1d1691fda9e306 +Z bc3a2ae65a6e7b1f83be3e57f2f8ed26 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1982ec0f0e..1b7ffdec93 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de6c5c6bb49a0beca793cdc0462da2c17bb05f3292e08dd815be29b45b59a8fb \ No newline at end of file +18de3a8e6b431a075143631eafdcf0e1b1a21868b417394c365d46d2feca10b5 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 335ad06844..5a1b7761f9 100644 --- a/src/printf.c +++ b/src/printf.c @@ -142,6 +142,20 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** "*val" is a u64. *msd is a divisor used to extract the +** most significant digit of *val. Extract that most significant +** digit and return it. +*/ +static char et_getdigit_int(u64 *val, u64 *msd){ + u64 x = (*val)/(*msd); + *val -= x*(*msd); + if( *msd>=10 ) *msd /= 10; + return '0' + (char)(x & 15); +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + /* ** Set the StrAccum object to an error mode. */ @@ -234,6 +248,8 @@ void sqlite3_str_vappendf( char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ sqlite_uint64 longvalue; /* Value for integer types */ LONGDOUBLE_TYPE realvalue; /* Value for real types */ + sqlite_uint64 msd; /* Divisor to get most-significant-digit + ** of longvalue */ const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ @@ -540,52 +556,72 @@ void sqlite3_str_vappendf( }else{ prefix = flag_prefix; } + exp = 0; if( xtype==etGENERIC && precision>0 ) precision--; testcase( precision>0xfff ); - idx = precision & 0xfff; - rounder = arRound[idx%10]; - while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } - if( xtype==etFLOAT ){ - double rx = (double)realvalue; - sqlite3_uint64 u; - int ex; - memcpy(&u, &rx, sizeof(u)); - ex = -1023 + (int)((u>>52)&0x7ff); - if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; - realvalue += rounder; - } - /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ - exp = 0; - if( sqlite3IsNaN((double)realvalue) ){ - bufpt = "NaN"; - length = 3; - break; - } - if( realvalue>0.0 ){ - LONGDOUBLE_TYPE scale = 1.0; - while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } - while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } - realvalue /= scale; - while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } - while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } - if( exp>350 ){ - bufpt = buf; - buf[0] = prefix; - memcpy(buf+(prefix!=0),"Inf",4); - length = 3+(prefix!=0); + if( realvalue<1.0e+16 + && realvalue==(LONGDOUBLE_TYPE)(longvalue = (u64)realvalue) + ){ + /* Number is a pure integer that can be represented as u64 */ + for(msd=1; msd*10<=longvalue; msd *= 10, exp++){} + if( exp>precision && xtype!=etFLOAT ){ + u64 rnd = msd/2; + int kk = precision; + while( kk-- > 0 && rnd ){ rnd /= 10; } + longvalue += rnd; + } + }else{ + msd = 0; + longvalue = 0; /* To prevent a compiler warning */ + idx = precision & 0xfff; + if( idx>20 ){ + rounder = 0.0; + }else{ + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ + double rx = (double)realvalue; + sqlite3_uint64 u; + int ex; + memcpy(&u, &rx, sizeof(u)); + ex = -1023 + (int)((u>>52)&0x7ff); + if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; + realvalue += rounder; + } + } + if( sqlite3IsNaN((double)realvalue) ){ + bufpt = "NaN"; + length = 3; break; } + + /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ + if( realvalue>0.0 ){ + LONGDOUBLE_TYPE scale = 1.0; + while( realvalue>=1e100*scale && exp<=350){ scale*=1e100;exp+=100;} + while( realvalue>=1e10*scale && exp<=350 ){ scale*=1e10; exp+=10; } + while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } + realvalue /= scale; + while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } + while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } + if( exp>350 ){ + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); + break; + } + if( xtype!=etFLOAT ){ + realvalue += rounder; + if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } + } + } } - bufpt = buf; + /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ - if( xtype!=etFLOAT ){ - realvalue += rounder; - if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } - } if( xtype==etGENERIC ){ flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ @@ -602,6 +638,8 @@ void sqlite3_str_vappendf( }else{ e2 = exp; } + nsd = 16 + flag_altform2*10; + bufpt = buf; { i64 szBufNeeded; /* Size of a temporary buffer needed */ szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; @@ -611,7 +649,6 @@ void sqlite3_str_vappendf( } } zOut = bufpt; - nsd = 16 + flag_altform2*10; flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ if( prefix ){ @@ -620,6 +657,10 @@ void sqlite3_str_vappendf( /* Digits prior to the decimal point */ if( e2<0 ){ *(bufpt++) = '0'; + }else if( msd>0 ){ + for(; e2>=0; e2--){ + *(bufpt++) = et_getdigit_int(&longvalue,&msd); + } }else{ for(; e2>=0; e2--){ *(bufpt++) = et_getdigit(&realvalue,&nsd); @@ -636,8 +677,14 @@ void sqlite3_str_vappendf( *(bufpt++) = '0'; } /* Significant digits after the decimal point */ - while( (precision--)>0 ){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); + if( msd>0 ){ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit_int(&longvalue,&msd); + } + }else{ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } } /* Remove trailing zeros and the "." if no digits follow the "." */ if( flag_rtz && flag_dp ){ diff --git a/test/printf.test b/test/printf.test index 445470fc07..24b78529f6 100644 --- a/test/printf.test +++ b/test/printf.test @@ -3786,4 +3786,39 @@ do_execsql_test printf-16.1 { SELECT printf('%.*g',2147483647,0.01); } {0.01} +# 2023-02-23 https://sqlite.org/forum/forumpost/d1387c3979c7f557 +# Loss of precision when doing floating-point to decimal +# conversions on values that have no factional part. +# +do_execsql_test printf-17.1 { + SELECT format('%!.20g', 13.0); +} 13.0 +do_execsql_test printf-17.2 { + SELECT format('%.3e', 199990000.0); +} 2.000e+08 +do_execsql_test printf-17.3 { + SELECT format('%.3f', 199990000.0); +} 199990000.000 +do_execsql_test printf-17.4 { + SELECT format('%.3g', 199990000.0); +} 2e+08 +do_execsql_test printf-17.5 { + SELECT format('%.4e', 199990000.0); +} 1.9999e+08 +do_execsql_test printf-17.6 { + SELECT format('%.4f', 199990000.0); +} 199990000.0000 +do_execsql_test printf-17.7 { + SELECT format('%.4g', 199990000.0); +} 2e+08 +do_execsql_test printf-17.8 { + SELECT format('%.5e', 199990000.0); +} 1.99990e+08 +do_execsql_test printf-17.9 { + SELECT format('%.5f', 199990000.0); +} 199990000.00000 +do_execsql_test printf-17.10 { + SELECT format('%.5g', 199990000.0); +} 1.9999e+08 + finish_test From ed530c022174b1db212a8336c917c3fcfa539a57 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 23 Feb 2023 22:08:22 +0000 Subject: [PATCH 015/341] Omit unnecessary branches from the enhanced floating-point conversion logic. FossilOrigin-Name: 32b0ba0d27481d85a4b0037ec36befcbffecdec75891eba58569acb160045553 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5e91b785cb..a18b8b9b9e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increased\sprecision\sof\sfloating-point\sto\sdecimal\sconversions\swhen\sthe\nfloating\spoint\svalue\shas\sno\sfractional\spart.\n[forum:/forumpost/d1387c3979c7f557|Forum\spost\sd1387c3979c7f557] -D 2023-02-23T21:18:47.560 +C Omit\sunnecessary\sbranches\sfrom\sthe\senhanced\sfloating-point\sconversion\nlogic. +D 2023-02-23T22:08:22.561 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -617,7 +617,7 @@ F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 82d5090a35eac75876d3b41f48d06b2370553c9576bf2942233c462e03eb94c9 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 -F src/printf.c d7b8c87d3525ee3eea3c4d291023540c88788e47a65c5b1dcff1b432ec9dbda5 +F src/printf.c 74e763868698df16ae24e96558c2a6ab9232ceaa373d326087bf519d2b75708d F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P de6c5c6bb49a0beca793cdc0462da2c17bb05f3292e08dd815be29b45b59a8fb -R 4769ae3a046ae725cd776ab02bbf2890 +P 18de3a8e6b431a075143631eafdcf0e1b1a21868b417394c365d46d2feca10b5 +R b615a8a32d1ccae3d6ecef9da1392398 U drh -Z bc3a2ae65a6e7b1f83be3e57f2f8ed26 +Z 97c2ef6d7f0bcb8f23c758a013f9c50f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1b7ffdec93..346f81dbc0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18de3a8e6b431a075143631eafdcf0e1b1a21868b417394c365d46d2feca10b5 \ No newline at end of file +32b0ba0d27481d85a4b0037ec36befcbffecdec75891eba58569acb160045553 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 5a1b7761f9..49a2789312 100644 --- a/src/printf.c +++ b/src/printf.c @@ -567,7 +567,7 @@ void sqlite3_str_vappendf( if( exp>precision && xtype!=etFLOAT ){ u64 rnd = msd/2; int kk = precision; - while( kk-- > 0 && rnd ){ rnd /= 10; } + while( kk-- > 0 ){ rnd /= 10; } longvalue += rnd; } }else{ @@ -596,7 +596,7 @@ void sqlite3_str_vappendf( } /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ - if( realvalue>0.0 ){ + if( ALWAYS(realvalue>0.0) ){ LONGDOUBLE_TYPE scale = 1.0; while( realvalue>=1e100*scale && exp<=350){ scale*=1e100;exp+=100;} while( realvalue>=1e10*scale && exp<=350 ){ scale*=1e10; exp+=10; } From 4b04ced3e05f46afb0e86631566a1adeb3113935 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Feb 2023 01:08:35 +0000 Subject: [PATCH 016/341] Fix an incorrect optimization that was attempted as part of check-in [18de3a8e6b431a07]. FossilOrigin-Name: f32055e8110a2eac6c9e26d1d1e620f0668bcb475d49d309dc549cea05e1e582 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/printf.c | 24 ++++++++++-------------- test/printf.test | 3 +++ 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index a18b8b9b9e..560f711aaa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sunnecessary\sbranches\sfrom\sthe\senhanced\sfloating-point\sconversion\nlogic. -D 2023-02-23T22:08:22.561 +C Fix\san\sincorrect\soptimization\sthat\swas\sattempted\sas\spart\sof\ncheck-in\s[18de3a8e6b431a07]. +D 2023-02-24T01:08:35.334 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -617,7 +617,7 @@ F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 82d5090a35eac75876d3b41f48d06b2370553c9576bf2942233c462e03eb94c9 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 -F src/printf.c 74e763868698df16ae24e96558c2a6ab9232ceaa373d326087bf519d2b75708d +F src/printf.c 5957a30d107a3916a411cb979b9ee80447dc3717aecc4648a4d8cce0f929e508 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 @@ -1381,7 +1381,7 @@ F test/pragma4.test ca5e4dfc46adfe490f75d73734f70349d95a199e6510973899e502eef2c8 F test/pragma5.test 7b33fc43e2e41abf17f35fb73f71b49671a380ea92a6c94b6ce530a25f8d9102 F test/pragmafault.test 275edaf3161771d37de60e5c2b412627ac94cef11739236bec12ed1258b240f8 F test/prefixes.test b524a1c44bffec225b9aec98bd728480352aa8532ac4c15771fb85e8beef65d9 -F test/printf.test 22449e13983e9d94fe9a671db68a086c4c55dea51daad825ef7ebb3d192ee997 +F test/printf.test 931381fede4f901d5f76275959339502f7d3312492c8df129972487951ff9fd1 F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60 F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 18de3a8e6b431a075143631eafdcf0e1b1a21868b417394c365d46d2feca10b5 -R b615a8a32d1ccae3d6ecef9da1392398 +P 32b0ba0d27481d85a4b0037ec36befcbffecdec75891eba58569acb160045553 +R d1a241fa3d7686eec8ef50c8409ad626 U drh -Z 97c2ef6d7f0bcb8f23c758a013f9c50f +Z 23dd6be15d54b437c81982a9d1edbb20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 346f81dbc0..7e680c44f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32b0ba0d27481d85a4b0037ec36befcbffecdec75891eba58569acb160045553 \ No newline at end of file +f32055e8110a2eac6c9e26d1d1e620f0668bcb475d49d309dc549cea05e1e582 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 49a2789312..45a31ec8dc 100644 --- a/src/printf.c +++ b/src/printf.c @@ -574,20 +574,16 @@ void sqlite3_str_vappendf( msd = 0; longvalue = 0; /* To prevent a compiler warning */ idx = precision & 0xfff; - if( idx>20 ){ - rounder = 0.0; - }else{ - rounder = arRound[idx%10]; - while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } - if( xtype==etFLOAT ){ - double rx = (double)realvalue; - sqlite3_uint64 u; - int ex; - memcpy(&u, &rx, sizeof(u)); - ex = -1023 + (int)((u>>52)&0x7ff); - if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; - realvalue += rounder; - } + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ + double rx = (double)realvalue; + sqlite3_uint64 u; + int ex; + memcpy(&u, &rx, sizeof(u)); + ex = -1023 + (int)((u>>52)&0x7ff); + if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; + realvalue += rounder; } if( sqlite3IsNaN((double)realvalue) ){ bufpt = "NaN"; diff --git a/test/printf.test b/test/printf.test index 24b78529f6..e4beb12dd0 100644 --- a/test/printf.test +++ b/test/printf.test @@ -3820,5 +3820,8 @@ do_execsql_test printf-17.9 { do_execsql_test printf-17.10 { SELECT format('%.5g', 199990000.0); } 1.9999e+08 +do_execsql_test printf-17.11 { + SELECT format('%.30f',1.0000000000000000076e-50); +} 0.000000000000000000000000000000 finish_test From 1fa0f27e81c07c1d224d0f360ff8df137ec4af2f Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Feb 2023 13:25:49 +0000 Subject: [PATCH 017/341] Add the "number_of_cores" TCL command to the testfixture. FossilOrigin-Name: 16ee5a7b5d02cad51a35d5675fcc5bb50a62d1f243bde107fc89fed44ffd6a43 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/test1.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 58e5915b73..669fc532bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Slightly\sfaster\sand\smore\sprecise\sfloating-point\sto\sdecimal\sconversion.\nSee\s[forum:/forumpost/d1387c3979c7f557|forum\sthread\sd1387c3979c7f557]\sfor\ndiscussion. -D 2023-02-24T11:54:40.435 +C Add\sthe\s"number_of_cores"\sTCL\scommand\sto\sthe\stestfixture. +D 2023-02-24T13:25:49.251 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -631,7 +631,7 @@ F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a3 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 8522a04fb9c84faa1d80354430ae0ee9349727a3a4b32e3cfe39b9be8324cabd -F src/test1.c 29ab8aca939818a61bf791e4ff5d45df6da30940c0980dcf502562b925de4e05 +F src/test1.c e23623dc569d2d18d1201d624ec279e9a96f0d9e83bf0620c8680d62b35d4ee7 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -2046,9 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P de6c5c6bb49a0beca793cdc0462da2c17bb05f3292e08dd815be29b45b59a8fb f32055e8110a2eac6c9e26d1d1e620f0668bcb475d49d309dc549cea05e1e582 -R d1a241fa3d7686eec8ef50c8409ad626 -T +closed f32055e8110a2eac6c9e26d1d1e620f0668bcb475d49d309dc549cea05e1e582 +P 6dea6f4738fc7d003183e94f08bd95181f23e165a966882c0bdc64708a05a607 +R ec9d2605f4b41b74f37dda061eea41aa U drh -Z bef52b6fcdf6ca2cb08784252d92c2c0 +Z 7c7b73264f13980fd1dcea1f8e67ede6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aeef2fe36b..5aacf306c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6dea6f4738fc7d003183e94f08bd95181f23e165a966882c0bdc64708a05a607 \ No newline at end of file +16ee5a7b5d02cad51a35d5675fcc5bb50a62d1f243bde107fc89fed44ffd6a43 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index b1f0baafce..39efd09ca0 100644 --- a/src/test1.c +++ b/src/test1.c @@ -16,6 +16,13 @@ #include "sqliteInt.h" #if SQLITE_OS_WIN # include "os_win.h" +# include +#else +# include +# if defined(__APPLE__) +# include +# include +# endif #endif #include "vdbeInt.h" @@ -8639,6 +8646,40 @@ static int SQLITE_TCLAPI test_autovacuum_pages( return TCL_OK; } +/* +** Usage: number_of_cores +** +** Return a guess at the number of available cores available on the +** processor on which this process is running. +*/ +static int SQLITE_TCLAPI guess_number_of_cores( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + unsigned int nCore = 1; +#if SQLITE_OS_WIN + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + nCore = (unsigned int)sysinfo.dwNumberOfProcessors; +#elif defined(__APPLE__) + int nm[2]; + size_t len = 4; + nm[0] = CTL_HW; nm[1] = HW_AVAILCPU; + sysctl(nm, 2, &nCore, &len, NULL, 0); + if( nCore<1 ){ + nm[1] = HW_NCPU; + sysctl(nm, 2, &nCore, &len, NULL, 0); + } +#else + nCore = sysconf(_SC_NPROCESSORS_ONLN); +#endif + if( nCore<=0 ) nCore = 1; + Tcl_SetObjResult(interp, Tcl_NewIntObj((int)nCore)); + return SQLITE_OK; +} + /* ** Register commands with the TCL interpreter. @@ -8939,6 +8980,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "test_write_db", test_write_db, 0 }, { "sqlite3_register_cksumvfs", test_register_cksumvfs, 0 }, { "sqlite3_unregister_cksumvfs", test_unregister_cksumvfs, 0 }, + { "number_of_cores", guess_number_of_cores, 0 }, }; static int bitmask_size = sizeof(Bitmask)*8; static int longdouble_size = sizeof(LONGDOUBLE_TYPE); From acf8905425648fc16a090d237c66f4e42fe7308e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Feb 2023 13:45:51 +0000 Subject: [PATCH 018/341] Adjust testrunner.tcl to make use of the new number_of_cores TCL command available in testfixture, and to use no more than half the available cores. FossilOrigin-Name: e0122d3863ed03e7bb64400d6561cbc824ecb14d228949ffde443069dbb4223a --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/testrunner.tcl | 35 ++++++++++++++++++++++------------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 669fc532bd..7f1d0e3a78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"number_of_cores"\sTCL\scommand\sto\sthe\stestfixture. -D 2023-02-24T13:25:49.251 +C Adjust\stestrunner.tcl\sto\smake\suse\sof\sthe\snew\snumber_of_cores\sTCL\scommand\navailable\sin\stestfixture,\sand\sto\suse\sno\smore\sthan\shalf\sthe\savailable\scores. +D 2023-02-24T13:45:51.487 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1564,7 +1564,7 @@ F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d163 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc F test/tester.tcl 8d54d40a55e12554b4509bc12078b201b233c8e842a7543629094a21b1ba956d -F test/testrunner.tcl e0b36a732956c69ce19370611b8dc69b44e627d5ea8ec2a5455ca3e8a170ddaa +F test/testrunner.tcl 59490f189cac99b16b0376d0cc0a7ecfb753a84b89c9f4c361af337d88db53ac F test/testrunner_data.tcl 8169c68654ac8906833b8a6aadca973358a441ebf88270dd05c153e5f96f76b8 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6dea6f4738fc7d003183e94f08bd95181f23e165a966882c0bdc64708a05a607 -R ec9d2605f4b41b74f37dda061eea41aa +P 16ee5a7b5d02cad51a35d5675fcc5bb50a62d1f243bde107fc89fed44ffd6a43 +R b016d73baa34c9fd497cfc4bbf069952 U drh -Z 7c7b73264f13980fd1dcea1f8e67ede6 +Z 4f8b3d88b5b38f10d362f3547581b15a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5aacf306c1..87f246de9d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16ee5a7b5d02cad51a35d5675fcc5bb50a62d1f243bde107fc89fed44ffd6a43 \ No newline at end of file +e0122d3863ed03e7bb64400d6561cbc824ecb14d228949ffde443069dbb4223a \ No newline at end of file diff --git a/test/testrunner.tcl b/test/testrunner.tcl index a71b055e4c..4e21ce08ab 100644 --- a/test/testrunner.tcl +++ b/test/testrunner.tcl @@ -71,26 +71,32 @@ of sub-processes the test script uses to run tests. # switch. # proc guess_number_of_cores {} { - set ret 4 + if {[catch {number_of_cores} ret]} { + set ret 4 - if {$::tcl_platform(os)=="Darwin"} { - set cmd "sysctl -n hw.logicalcpu" - } else { - set cmd "nproc" - } - catch { - set fd [open "|$cmd" r] - set ret [gets $fd] - close $fd - set ret [expr $ret] + if {$::tcl_platform(os)=="Darwin"} { + set cmd "sysctl -n hw.logicalcpu" + } else { + set cmd "nproc" + } + catch { + set fd [open "|$cmd" r] + set ret [gets $fd] + close $fd + set ret [expr $ret] + } } return $ret } proc default_njob {} { set nCore [guess_number_of_cores] - set nHelper [expr int($nCore*0.75)] - expr $nHelper>0 ? $nHelper : 1 + if {$nCore<=2} { + set nHelper 1 + } else { + set nHelper [expr int($nCore*0.5)] + } + return $nHelper } #------------------------------------------------------------------------- @@ -878,6 +884,9 @@ proc run_testset {} { sqlite3 trdb $TRG(dbname) trdb timeout $TRG(timeout) set tm [lindex [time { make_new_testset }] 0] +if {$TRG(nJob)>1} { + puts "splitting work across $TRG(nJob) cores" +} puts "built testset in [expr $tm/1000]ms.." run_testset trdb close From 70e01f6adc520fadb6ad25d57eca62bc6565a2e1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Feb 2023 15:59:27 +0000 Subject: [PATCH 019/341] Back out the SQLITE_CONFIG_URI option from anytime-config. FossilOrigin-Name: 0de98b8e21e4cbf2438160651585ea23ebea952c5a4e7d88e5eabe5cc49517b2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 2 -- src/sqlite.h.in | 1 - 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 122d952ff3..b3f0b6959f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\senhancements\sfrom\strunk\sinto\sthe\sanytime-config\sbranch. -D 2023-02-24T12:55:24.927 +C Back\sout\sthe\sSQLITE_CONFIG_URI\soption\sfrom\sanytime-config. +D 2023-02-24T15:59:27.989 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -584,7 +584,7 @@ F src/insert.c 1b11a2e33ee52db93c02fddac67e39d00161d61b69fac2675b82f2aa68c1b61c F src/json.c c85ed6fce06f43d414b0d7fff64749d43a0dbd1067123ee407bd3a0752454161 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c b04eb648cedc45efe4298e1ef439ac4f0096ae27b5f01accb0a1f49d57789128 -F src/main.c 437e97a14936763b42e013f02495ef9c0bca6654073321c2bdd844cb4ec441f5 +F src/main.c cb5b7277b525ac8f7663426485332bb4f3c1fd9d07f4051980dcab806b780808 F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -623,7 +623,7 @@ F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c a6a5cc0bbb97be616a2608cb5d063b6635b920ecd6b7b3445cde88597daf3542 F src/shell.c.in 6f36f5ca05f1bebf74935a7fcf2dce983016e807a09cbd752a673583ad7da087 -F src/sqlite.h.in 3bdc92831511442cad4de5243b619ba8b887bb102bb464bbc5436df7e2e6568a +F src/sqlite.h.in 5f308635ad467b50af858f271e403d14f8bcc574c2610f7cfd2d00f5bb37f616 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 F src/sqliteInt.h cb7182dcdc9910d5f1352c90762545cc5ffb79c4a47f4ae7c5ee044fdb80423b @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1702eb48d13c7c9b7605f1e77242672222c53059edcdc4e9cea59510715822a 6dea6f4738fc7d003183e94f08bd95181f23e165a966882c0bdc64708a05a607 -R ee505b1732c05f62c8c588523b4188ee +P 04b2fdf3bc986cd402e8a6c060e1b3989e60770695e2f8a4bc12c43e380a694c +R 4b3d5959e03f91a62bec6935bd48bd0e U drh -Z 37e143e028fb2ad426a2f426402e2ea7 +Z ecaf3b30bb2b8b0cc67541076f53d55b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ae55d54044..c9b9851d30 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04b2fdf3bc986cd402e8a6c060e1b3989e60770695e2f8a4bc12c43e380a694c \ No newline at end of file +0de98b8e21e4cbf2438160651585ea23ebea952c5a4e7d88e5eabe5cc49517b2 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 4ca81a6ffb..9ec313e6fc 100644 --- a/src/main.c +++ b/src/main.c @@ -437,14 +437,12 @@ int sqlite3_config(int op, ...){ if( sqlite3GlobalConfig.isInit ){ static const u64 mAnytimeConfigOption = 0 | MASKBIT64( SQLITE_CONFIG_LOG ) - | MASKBIT64( SQLITE_CONFIG_URI ) | MASKBIT64( SQLITE_CONFIG_PCACHE_HDRSZ ) ; if( op<0 || op>63 || (MASKBIT64(op) & mAnytimeConfigOption)==0 ){ return SQLITE_MISUSE_BKPT; } testcase( op==SQLITE_CONFIG_LOG ); - testcase( op==SQLITE_CONFIG_URI ); testcase( op==SQLITE_CONFIG_PCACHE_HDRSZ ); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 198e209dd8..06737e1c90 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1793,7 +1793,6 @@ struct sqlite3_mem_methods { ** options is: **
    **
  • SQLITE_CONFIG_LOG -**
  • SQLITE_CONFIG_URI **
  • SQLITE_CONFIG_PCACHE_HDRSZ **
** From d8f13f48a8947125bf2a814cbcb3aec90badb1da Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Feb 2023 21:05:12 +0000 Subject: [PATCH 020/341] In the CLI, do not emit warnings about the incorrect usage of sqlite3_config() in Fiddle. FossilOrigin-Name: 847021b402a3e624e22959a70e3e29a4f069c2005534485779cdc018722b5863 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/shell.c.in | 18 +++++++++++++++--- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d36b90a14c..fb778df10c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\ssqlite3_config()\sinterface\sso\sthat\sthe\sSQLITE_CONFIG_LOG\sand\nSQLITE_CONFIG_PCACHE_HDRSZ\sopcodes\scan\sbe\scalled\safter\ssqlite3_initialize().\nEnhancement\srequest\s[0b75886e6d48f7c2]. -D 2023-02-24T16:04:05.096 +C In\sthe\sCLI,\sdo\snot\semit\swarnings\sabout\sthe\sincorrect\susage\sof\ssqlite3_config()\nin\sFiddle. +D 2023-02-24T21:05:12.788 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -622,7 +622,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c a6a5cc0bbb97be616a2608cb5d063b6635b920ecd6b7b3445cde88597daf3542 -F src/shell.c.in 6f36f5ca05f1bebf74935a7fcf2dce983016e807a09cbd752a673583ad7da087 +F src/shell.c.in 22568122b97d77439c2f1d6afed40be528f04dab47c8cd2752b9fb37555a58ed F src/sqlite.h.in 5f308635ad467b50af858f271e403d14f8bcc574c2610f7cfd2d00f5bb37f616 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2046,9 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e0122d3863ed03e7bb64400d6561cbc824ecb14d228949ffde443069dbb4223a 0de98b8e21e4cbf2438160651585ea23ebea952c5a4e7d88e5eabe5cc49517b2 -R 5efb0846a9c368160a576caa15e96a1a -T +closed 0de98b8e21e4cbf2438160651585ea23ebea952c5a4e7d88e5eabe5cc49517b2 +P 7902fb806b33a27932f9ca347246f4ac43091caad0a1536009985b650fd720e2 +R fbf26774a3bcd50bb0fc2166f101abc4 U drh -Z 11ff9f1f4744d1480014019a74924c13 +Z ff696b6fee0d3d6fd70e0e7b389de9b1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f9ab6225fc..f2e3a808f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7902fb806b33a27932f9ca347246f4ac43091caad0a1536009985b650fd720e2 \ No newline at end of file +847021b402a3e624e22959a70e3e29a4f069c2005534485779cdc018722b5863 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index de3b6a73bf..512c3436e3 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11737,9 +11737,11 @@ static void main_init(ShellState *data) { memcpy(data->rowSeparator,SEP_Row, 2); data->showHeader = 0; data->shellFlgs = SHFLG_Lookaside; - verify_uninitialized(); - sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); +#if !defined(SQLITE_SHELL_FIDDLE) + verify_uninitialized(); +#endif + sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); @@ -11919,7 +11921,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** the size of the alternative malloc heap, ** and the first command to execute. */ +#ifndef SQLITE_SHELL_FIDDLE verify_uninitialized(); +#endif for(i=1; i0x7fff0000 ) szHeap = 0x7fff0000; + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #else (void)cmdline_option_value(argc, argv, ++i); @@ -11972,6 +11977,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( sz>0 && n>0 && 0xffffffffffffLL/sz0 && sz>0) ? malloc(n*sz) : 0, sz, n); data.shellFlgs |= SHFLG_Pagecache; @@ -11981,11 +11987,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( sz<0 ) sz = 0; n = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( n<0 ) n = 0; + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; }else if( cli_strcmp(z,"-threadsafe")==0 ){ int n; n = (int)integerValue(cmdline_option_value(argc,argv,++i)); + verify_uninitialized(); switch( n ){ case 0: sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); break; case 2: sqlite3_config(SQLITE_CONFIG_MULTITHREAD); break; @@ -12009,10 +12017,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif }else if( cli_strcmp(z,"-mmap")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); -#ifdef SQLITE_ENABLE_SORTER_REFERENCES +#if defined(SQLITE_ENABLE_SORTER_REFERENCES) }else if( cli_strcmp(z,"-sorterref")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); #endif }else if( cli_strcmp(z,"-vfs")==0 ){ @@ -12050,7 +12060,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ /* no-op - catch this on the second pass */ } } +#ifndef SQLITE_SHELL_FIDDLE verify_uninitialized(); +#endif #ifdef SQLITE_SHELL_INIT_PROC From 96705c1d7a8d135dc2d9863d1cb2584d192c8e89 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Feb 2023 21:23:53 +0000 Subject: [PATCH 021/341] Add the "on" option to the ".log" command in the CLI. Allow ".log on" and ".log off" even in --safe mode. Enable the .log command for fiddle builds. FossilOrigin-Name: 6bba9100ae81466eeb49845c449cbfddf4f82f18b89f55e6ef575cbf66af63af --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 18 +++++++++++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index fb778df10c..eda9046b78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sCLI,\sdo\snot\semit\swarnings\sabout\sthe\sincorrect\susage\sof\ssqlite3_config()\nin\sFiddle. -D 2023-02-24T21:05:12.788 +C Add\sthe\s"on"\soption\sto\sthe\s".log"\scommand\sin\sthe\sCLI.\s\sAllow\s".log\son"\sand\n".log\soff"\seven\sin\s--safe\smode.\s\sEnable\sthe\s.log\scommand\sfor\sfiddle\sbuilds. +D 2023-02-24T21:23:53.130 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -622,7 +622,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c a6a5cc0bbb97be616a2608cb5d063b6635b920ecd6b7b3445cde88597daf3542 -F src/shell.c.in 22568122b97d77439c2f1d6afed40be528f04dab47c8cd2752b9fb37555a58ed +F src/shell.c.in d362b6faacd8f80d69cfcc7905b51912ecde4c7dcfbc32401d5af43768ce0d5e F src/sqlite.h.in 5f308635ad467b50af858f271e403d14f8bcc574c2610f7cfd2d00f5bb37f616 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7902fb806b33a27932f9ca347246f4ac43091caad0a1536009985b650fd720e2 -R fbf26774a3bcd50bb0fc2166f101abc4 +P 847021b402a3e624e22959a70e3e29a4f069c2005534485779cdc018722b5863 +R a03e248570a718d46ef150580e9f4d5a U drh -Z ff696b6fee0d3d6fd70e0e7b389de9b1 +Z b866616686ca4aa5278ef4fb5c394a30 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f2e3a808f6..a520b02a63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -847021b402a3e624e22959a70e3e29a4f069c2005534485779cdc018722b5863 \ No newline at end of file +6bba9100ae81466eeb49845c449cbfddf4f82f18b89f55e6ef575cbf66af63af \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 512c3436e3..bc3994dd17 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4623,8 +4623,10 @@ static const char *(azHelp[]) = { #if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE) ".load FILE ?ENTRY? Load an extension library", #endif -#ifndef SQLITE_SHELL_FIDDLE - ".log FILE|off Turn logging on or off. FILE can be stderr/stdout", +#if !defined(SQLITE_SHELL_FIDDLE) + ".log FILE|on|off Turn logging on or off. FILE can be stderr/stdout", +#else + ".log on|off Turn logging on or off.", #endif ".mode MODE ?OPTIONS? Set output mode", " MODE is one of:", @@ -9113,19 +9115,25 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif -#ifndef SQLITE_SHELL_FIDDLE if( c=='l' && cli_strncmp(azArg[0], "log", n)==0 ){ - failIfSafeMode(p, "cannot run .log in safe mode"); if( nArg!=2 ){ raw_printf(stderr, "Usage: .log FILENAME\n"); rc = 1; }else{ const char *zFile = azArg[1]; + if( p->bSafeMode + && cli_strcmp(zFile,"on")!=0 + && cli_strcmp(zFile,"off")!=0 + ){ + raw_printf(stdout, "cannot set .log to anything other " + "than \"on\" or \"off\"\n"); + zFile = "off"; + } output_file_close(p->pLog); + if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout"; p->pLog = output_file_open(zFile, 0); } }else -#endif if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){ const char *zMode = 0; From 0f8aebbfcf5b4d63208ff336bca5df7ed8f9bbb5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Feb 2023 12:50:54 +0000 Subject: [PATCH 022/341] Performance optimization on sqlite3_vsnprintf(). FossilOrigin-Name: 4430e6e96dbf3d0b464776c132838dd3e456716e0ae5972fc7e0ece13b56ea25 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 16 +++++++++++++--- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index eda9046b78..47f38803ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"on"\soption\sto\sthe\s".log"\scommand\sin\sthe\sCLI.\s\sAllow\s".log\son"\sand\n".log\soff"\seven\sin\s--safe\smode.\s\sEnable\sthe\s.log\scommand\sfor\sfiddle\sbuilds. -D 2023-02-24T21:23:53.130 +C Performance\soptimization\son\ssqlite3_vsnprintf(). +D 2023-02-25T12:50:54.857 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -617,7 +617,7 @@ F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 82d5090a35eac75876d3b41f48d06b2370553c9576bf2942233c462e03eb94c9 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 -F src/printf.c 5957a30d107a3916a411cb979b9ee80447dc3717aecc4648a4d8cce0f929e508 +F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 847021b402a3e624e22959a70e3e29a4f069c2005534485779cdc018722b5863 -R a03e248570a718d46ef150580e9f4d5a +P 6bba9100ae81466eeb49845c449cbfddf4f82f18b89f55e6ef575cbf66af63af +R 50518feb0ca5bebb9a44abf5ec071312 U drh -Z b866616686ca4aa5278ef4fb5c394a30 +Z 75921f7ab1a31bc1aea299fbe5176436 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a520b02a63..af12ce8b5d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6bba9100ae81466eeb49845c449cbfddf4f82f18b89f55e6ef575cbf66af63af \ No newline at end of file +4430e6e96dbf3d0b464776c132838dd3e456716e0ae5972fc7e0ece13b56ea25 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 45a31ec8dc..b222cb2146 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1361,12 +1361,22 @@ char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ return zBuf; } char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ - char *z; + StrAccum acc; va_list ap; + if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf ) zBuf[0] = 0; + return zBuf; + } +#endif + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); va_start(ap,zFormat); - z = sqlite3_vsnprintf(n, zBuf, zFormat, ap); + sqlite3_str_vappendf(&acc, zFormat, ap); va_end(ap); - return z; + zBuf[acc.nChar] = 0; + return zBuf; } /* From 4ee8e9681c0f429acb4b696890dd94b3fa754029 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 25 Feb 2023 15:15:40 +0000 Subject: [PATCH 023/341] Revert (mostly) to earlier CLI response to -echo and SQL at the command line. FossilOrigin-Name: 1c6cfcf6ed5e084e5eb6874a261f3670ab5fb3e5dbcf150f37c58b2f16351ee3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 47f38803ce..b61275760d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\son\ssqlite3_vsnprintf(). -D 2023-02-25T12:50:54.857 +C Revert\s(mostly)\sto\searlier\sCLI\sresponse\sto\s-echo\sand\sSQL\sat\sthe\scommand\sline. +D 2023-02-25T15:15:40.203 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -622,7 +622,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c a6a5cc0bbb97be616a2608cb5d063b6635b920ecd6b7b3445cde88597daf3542 -F src/shell.c.in d362b6faacd8f80d69cfcc7905b51912ecde4c7dcfbc32401d5af43768ce0d5e +F src/shell.c.in cb763c332be668ecba48c85de52df52a86f992867b813460d55c3bccaaf4c0eb F src/sqlite.h.in 5f308635ad467b50af858f271e403d14f8bcc574c2610f7cfd2d00f5bb37f616 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6bba9100ae81466eeb49845c449cbfddf4f82f18b89f55e6ef575cbf66af63af -R 50518feb0ca5bebb9a44abf5ec071312 -U drh -Z 75921f7ab1a31bc1aea299fbe5176436 +P 4430e6e96dbf3d0b464776c132838dd3e456716e0ae5972fc7e0ece13b56ea25 +R 8941207ab3042353d14f49547d95d3a0 +U larrybr +Z 0509b1c3c7307441d1eefcb8a2685f24 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index af12ce8b5d..edb26eb4d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4430e6e96dbf3d0b464776c132838dd3e456716e0ae5972fc7e0ece13b56ea25 \ No newline at end of file +1c6cfcf6ed5e084e5eb6874a261f3670ab5fb3e5dbcf150f37c58b2f16351ee3 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index bc3994dd17..5c8d47e9a0 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -12318,6 +12318,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } }else{ open_db(&data, 0); + echo_group_input(&data, azCmd[i]); rc = shell_exec(&data, azCmd[i], &zErrMsg); if( zErrMsg || rc ){ if( zErrMsg!=0 ){ From 20e09ba6a863835175c3276da305965ec66fd998 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Feb 2023 15:34:09 +0000 Subject: [PATCH 024/341] Remove an unnecessary and redundant corruption checks in defragmentPage(). FossilOrigin-Name: 2dc7342e12e9dac2d8345d045a17caa29c26eaf6f417dd052c7c645eddd6c58c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b61275760d..9c0eef5077 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revert\s(mostly)\sto\searlier\sCLI\sresponse\sto\s-echo\sand\sSQL\sat\sthe\scommand\sline. -D 2023-02-25T15:15:40.203 +C Remove\san\sunnecessary\sand\sredundant\scorruption\schecks\sin\sdefragmentPage(). +D 2023-02-25T15:34:09.046 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -560,7 +560,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 2f012aea074de6319c191cbf5c45034de79bf1c762826e381b1cf75421d8d831 +F src/btree.c ff9634e33c27a77a49d430e32f2aa6a16502632376a6f0b65bcf12e09d7ca4d2 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e F src/build.c c55ab6d1b089ceef57160e840f05f692955ac90944c3d04fcf01d97fd7bfd08d @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4430e6e96dbf3d0b464776c132838dd3e456716e0ae5972fc7e0ece13b56ea25 -R 8941207ab3042353d14f49547d95d3a0 -U larrybr -Z 0509b1c3c7307441d1eefcb8a2685f24 +P 1c6cfcf6ed5e084e5eb6874a261f3670ab5fb3e5dbcf150f37c58b2f16351ee3 +R 6f0a4ee3eb90075db6e6500bcbce2e80 +U drh +Z 9c68161318f8714b2adb3b454cee5391 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index edb26eb4d4..5a09b45c8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c6cfcf6ed5e084e5eb6874a261f3670ab5fb3e5dbcf150f37c58b2f16351ee3 \ No newline at end of file +2dc7342e12e9dac2d8345d045a17caa29c26eaf6f417dd052c7c645eddd6c58c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4fbe0b3dbc..2f6ab428b6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1580,10 +1580,10 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ /* These conditions have already been verified in btreeInitPage() ** if PRAGMA cell_size_check=ON. */ - if( pciCellLast ){ + if( pc>iCellLast ){ return SQLITE_CORRUPT_PAGE(pPage); } - assert( pc>=iCellStart && pc<=iCellLast ); + assert( pc>=0 && pc<=iCellLast ); size = pPage->xCellSize(pPage, &src[pc]); cbrk -= size; if( cbrkusableSize ){ From 2d7829057e3806e232f12b66169158fd8f090266 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Feb 2023 16:03:28 +0000 Subject: [PATCH 025/341] Split out a new variant of cellSizePtr() that applies only to leaf pages, for a small performance increase. FossilOrigin-Name: 22d32eef8741ae4f2aac3869465e5a7d2e33c6bc2425dd8e77f2a746e43687e8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9c0eef5077..ddc9f57b71 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sand\sredundant\scorruption\schecks\sin\sdefragmentPage(). -D 2023-02-25T15:34:09.046 +C Split\sout\sa\snew\svariant\sof\scellSizePtr()\sthat\sapplies\sonly\sto\sleaf\spages,\nfor\sa\ssmall\sperformance\sincrease. +D 2023-02-25T16:03:28.921 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -560,7 +560,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c ff9634e33c27a77a49d430e32f2aa6a16502632376a6f0b65bcf12e09d7ca4d2 +F src/btree.c 04bf3d1c49c9000af0e2d50f04198641fd7dfc6e4eae495cb60b002d38b45e27 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e F src/build.c c55ab6d1b089ceef57160e840f05f692955ac90944c3d04fcf01d97fd7bfd08d @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1c6cfcf6ed5e084e5eb6874a261f3670ab5fb3e5dbcf150f37c58b2f16351ee3 -R 6f0a4ee3eb90075db6e6500bcbce2e80 +P 2dc7342e12e9dac2d8345d045a17caa29c26eaf6f417dd052c7c645eddd6c58c +R 7550d632ae1e4db50e93659ce002df8e U drh -Z 9c68161318f8714b2adb3b454cee5391 +Z 36cbe33dfaff9cae13269d630b44e5c2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5a09b45c8e..32b031f27e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2dc7342e12e9dac2d8345d045a17caa29c26eaf6f417dd052c7c645eddd6c58c \ No newline at end of file +22d32eef8741ae4f2aac3869465e5a7d2e33c6bc2425dd8e77f2a746e43687e8 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2f6ab428b6..8296294dbf 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1328,10 +1328,11 @@ static void btreeParseCell( ** ** cellSizePtrNoPayload() => table internal nodes ** cellSizePtrTableLeaf() => table leaf nodes -** cellSizePtr() => all index nodes & table leaf nodes +** cellSizePtr() => index internal nodes +** cellSizeIdxLeaf() => index leaf nodes */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ u8 *pEnd; /* End mark for a varint */ u32 nSize; /* Size value to return */ @@ -1344,6 +1345,49 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pPage->xParseCell(pPage, pCell, &debuginfo); #endif + assert( pPage->childPtrSize==4 ); + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pItermaxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + assert( nSize>4 ); + }else{ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4 + (u16)(pIter - pCell); + } + assert( nSize==debuginfo.nSize || CORRUPT_DB ); + return (u16)nSize; +} +static u16 cellSizePtrIdxLeaf(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#endif + + assert( pPage->childPtrSize==0 ); nSize = *pIter; if( nSize>=0x80 ){ pEnd = &pIter[8]; @@ -1925,14 +1969,14 @@ static int decodeFlags(MemPage *pPage, int flagByte){ }else if( flagByte==(PTF_ZERODATA | PTF_LEAF) ){ pPage->intKey = 0; pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; + pPage->xCellSize = cellSizePtrIdxLeaf; pPage->xParseCell = btreeParseCellPtrIndex; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ pPage->intKey = 0; pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; + pPage->xCellSize = cellSizePtrIdxLeaf; pPage->xParseCell = btreeParseCellPtrIndex; return SQLITE_CORRUPT_PAGE(pPage); } From 4edbcdd391f472e16f2a1037175f270ca346dedb Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 26 Feb 2023 11:36:35 +0000 Subject: [PATCH 026/341] In the [/info/7c2d3406000dc8ac|omit-unused-subquery-columns optimization], be sure to remove the EP_Skip and EP_Unlikely flags from the result set expressions that get nulled-out. dbsqlfuzz bf1d3ed6e0e0dd8766027797d43db40c776d2b15. FossilOrigin-Name: 21aec65e5e2a01e58dd0bb8c8b9b29b8414373b53353fc7ca80a152fdd27566b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 + test/selectH.test | 8 ++++++++ 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ddc9f57b71..85a637de7b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Split\sout\sa\snew\svariant\sof\scellSizePtr()\sthat\sapplies\sonly\sto\sleaf\spages,\nfor\sa\ssmall\sperformance\sincrease. -D 2023-02-25T16:03:28.921 +C In\sthe\s[/info/7c2d3406000dc8ac|omit-unused-subquery-columns\soptimization],\sbe\nsure\sto\sremove\sthe\sEP_Skip\sand\sEP_Unlikely\sflags\sfrom\sthe\sresult\sset\sexpressions\nthat\sget\snulled-out.\s\sdbsqlfuzz\sbf1d3ed6e0e0dd8766027797d43db40c776d2b15. +D 2023-02-26T11:36:35.402 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c a6a5cc0bbb97be616a2608cb5d063b6635b920ecd6b7b3445cde88597daf3542 +F src/select.c 230dc0601f55ae4909b5f88bc002bfa1f1fb331e51f2c6d670d3effc2ced365e F src/shell.c.in cb763c332be668ecba48c85de52df52a86f992867b813460d55c3bccaaf4c0eb F src/sqlite.h.in 5f308635ad467b50af858f271e403d14f8bcc574c2610f7cfd2d00f5bb37f616 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1462,7 +1462,7 @@ F test/selectD.test 6d1909b49970bf92f45ce657505befcef5fc7cbc13544e18103a316d3218 F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 F test/selectG.test 089f7d3d7e6db91566f00b036cb353107a2cca6220eb1cb264085a836dae8840 -F test/selectH.test a1fed72a83a68ed858db6fcf6d8ddccc4a368645c0ef48dc3ac331d41efe34de +F test/selectH.test 88237ded5925adfb3f27fdafb5428c2ffc4d61e313bceb854e225ffc3ef0d206 F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be F test/sessionfuzz-data1.db 1f8d5def831f19b1c74571037f0d53a588ea49a6c4ca2a028fc0c27ef896dbcb F test/sessionfuzz.c 5eef09af01eeff6f20250ae4c0112c2e576e4d2f2026cc9a49dc5be6886fa6ee @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2dc7342e12e9dac2d8345d045a17caa29c26eaf6f417dd052c7c645eddd6c58c -R 7550d632ae1e4db50e93659ce002df8e +P 22d32eef8741ae4f2aac3869465e5a7d2e33c6bc2425dd8e77f2a746e43687e8 +R cc7aa349e394b75f23cb7073138b42c3 U drh -Z 36cbe33dfaff9cae13269d630b44e5c2 +Z ca0c0ef47bfc20fd4080b324e9bbcac3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 32b031f27e..72554d27ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -22d32eef8741ae4f2aac3869465e5a7d2e33c6bc2425dd8e77f2a746e43687e8 \ No newline at end of file +21aec65e5e2a01e58dd0bb8c8b9b29b8414373b53353fc7ca80a152fdd27566b \ No newline at end of file diff --git a/src/select.c b/src/select.c index 1c71dbe548..a882b973ee 100644 --- a/src/select.c +++ b/src/select.c @@ -5288,6 +5288,7 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ Expr *pY = pX->pEList->a[j].pExpr; if( pY->op==TK_NULL ) continue; pY->op = TK_NULL; + ExprClearProperty(pY, EP_Skip|EP_Unlikely); pX->selFlags |= SF_PushDown; nChng++; } diff --git a/test/selectH.test b/test/selectH.test index f621fc302c..a97679bda2 100644 --- a/test/selectH.test +++ b/test/selectH.test @@ -106,5 +106,13 @@ do_test 3.7 { set ::selectH_cnt } {4} +# 2023-02-25 dbsqlfuzz bf1d3ed6e0e0dd8766027797d43db40c776d2b15 +# +do_execsql_test 4.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + SELECT 1 FROM (SELECT DISTINCT name COLLATE rtrim FROM sqlite_schema + UNION ALL SELECT a FROM t1); +} 1 finish_test From d7ddec765cbc5dd97705fd33c8d49b2e16340806 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 26 Feb 2023 20:57:09 +0000 Subject: [PATCH 027/341] When a table-valued function appears as the right table of a RIGHT JOIN, the argument constraints on the table-valued function should be considered part of the ON clause of the RIGHT JOIN. Fix for the problem reported by [forum:/forumpost/422e635f3beafbf6|forum post 422e635f3beafbf6]. Test cases in TH3. Possibly related to the enhancement at [501609eddf2a46d5]. FossilOrigin-Name: 46639f682975dac6efec4e230aca2c9b127b4fc77e0b465fad38ef4caa9b7a4c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/whereexpr.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 85a637de7b..ddcdbff760 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s[/info/7c2d3406000dc8ac|omit-unused-subquery-columns\soptimization],\sbe\nsure\sto\sremove\sthe\sEP_Skip\sand\sEP_Unlikely\sflags\sfrom\sthe\sresult\sset\sexpressions\nthat\sget\snulled-out.\s\sdbsqlfuzz\sbf1d3ed6e0e0dd8766027797d43db40c776d2b15. -D 2023-02-26T11:36:35.402 +C When\sa\stable-valued\sfunction\sappears\sas\sthe\sright\stable\sof\sa\sRIGHT\sJOIN,\nthe\sargument\sconstraints\son\sthe\stable-valued\sfunction\sshould\sbe\sconsidered\npart\sof\sthe\sON\sclause\sof\sthe\sRIGHT\sJOIN.\s\sFix\sfor\sthe\sproblem\sreported\nby\s[forum:/forumpost/422e635f3beafbf6|forum\spost\s422e635f3beafbf6].\s\sTest\ncases\sin\sTH3.\s\sPossibly\srelated\sto\sthe\senhancement\sat\s[501609eddf2a46d5]. +D 2023-02-26T20:57:09.904 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -709,7 +709,7 @@ F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 38c36bb654b4172148aef3b295bbbc1342b52078cf20174f771c746caa98f69a F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c b82d0d33315e1526904b95155e55e61149c4462147668e1cc4567c812735eff1 -F src/whereexpr.c a60a1f88650c561eded95c5affbe846cda71fac35033e1280f28e47934df33c8 +F src/whereexpr.c 16d1eefd95f69843b45aba6d04fe2b63fc4f51584dff85ae380f5c20718f3c75 F src/window.c 76a27cff9ea2ded0c2c3527187029259440fabcc4cc4c07b11d942c78494a614 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 22d32eef8741ae4f2aac3869465e5a7d2e33c6bc2425dd8e77f2a746e43687e8 -R cc7aa349e394b75f23cb7073138b42c3 +P 21aec65e5e2a01e58dd0bb8c8b9b29b8414373b53353fc7ca80a152fdd27566b +R dd644aa05c12deae01f2f05d2c4bf4e4 U drh -Z ca0c0ef47bfc20fd4080b324e9bbcac3 +Z 73efc57a25cea16beea22ea8fc7c8752 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 72554d27ff..f5d539d572 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21aec65e5e2a01e58dd0bb8c8b9b29b8414373b53353fc7ca80a152fdd27566b \ No newline at end of file +46639f682975dac6efec4e230aca2c9b127b4fc77e0b465fad38ef4caa9b7a4c \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 4f86753bf7..7860480dd8 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1856,7 +1856,7 @@ void sqlite3WhereTabFuncArgs( pRhs = sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); - if( pItem->fg.jointype & (JT_LEFT|JT_LTORJ) ){ + if( pItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ){ joinType = EP_OuterON; }else{ joinType = EP_InnerON; From 9885c76c4c86f4de670ce8b762c4df2ce1e99007 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 27 Feb 2023 07:12:28 +0000 Subject: [PATCH 028/341] Extend wasm build to enable inclusion of client-custom C code, initialized via the SQLITE_EXTRA_INIT mechanism, per discussion in [forum:1e1c04f3ed1bc96b|forum post 1e1c04f3ed1bc96b]. FossilOrigin-Name: 68a52cafff60f19c9c998133d04f192b1e8b23f78b8cee13807d76845ef5e13d --- ext/wasm/GNUmakefile | 35 ++++++++++++-- ext/wasm/api/sqlite3-wasm.c | 90 ++++++++++++++++++----------------- ext/wasm/example_extra_init.c | 22 +++++++++ ext/wasm/fiddle.make | 2 +- manifest | 19 ++++---- manifest.uuid | 2 +- 6 files changed, 113 insertions(+), 57 deletions(-) create mode 100644 ext/wasm/example_extra_init.c diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 20dab446da..f2d6ae9f0b 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -189,6 +189,34 @@ else $(info Development build. Use '$(MAKE) release' for a smaller release build.) endif +######################################################################## +# Adding custom C code via sqlite3_wasm_extra_init.c: +# +# If the canonical build process finds the file +# sqlite3_wasm_extra_init.c in the main wasm build directory, it +# arranges to include that file in the build of sqlite3.wasm and +# defines SQLITE_EXTRA_INIT=sqlite3_wasm_extra_init. +# +# sqlite3_wasm_extra_init() must be a function with this signature: +# +# int sqlite3_wasm_extra_init(const char *) +# +# and the sqlite3 library will call it with an argument of NULL one +# time during sqlite3_initialize(). If it returns non-0, +# initialization of the library will fail. +# +# The filename can be overridden with: +# +# make sqlite3_wasm_extra_init.c=my_custom_stuff.c +# +# See example_extra_init.c for an example implementation. +######################################################################## +sqlite3_wasm_extra_init.c ?= $(wildcard sqlite3_wasm_extra_init.c) +ifneq (,$(sqlite3_wasm_extra_init.c)) + $(info Enabling SQLITE_EXTRA_INIT via $(sqlite3_wasm_extra_init.c).) + SQLITE_OPT += -I. -DSQLITE_WASM_EXTRA_INIT +endif + # bin.version-info = binary to output various sqlite3 version info for # embedding in the JS files and in building the distribution zip file. # It must NOT be in $(dir.tmp) because we need it to survive the @@ -576,6 +604,7 @@ endef # https://github.com/emscripten-core/emscripten/issues/14383 sqlite3.wasm := $(dir.dout)/sqlite3.wasm sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c +sqlite3-wasm.cs := $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) # sqlite3-wasm.o vs sqlite3-wasm.c: building against the latter # (predictably) results in a slightly faster binary. We're close # enough to the target speed requirements that the 500ms makes a @@ -650,12 +679,12 @@ $$(eval $$(call call-make-pre-post,sqlite3,$(1))) emcc.flags.sqlite3.$(1) := $(6) $$(eval $$(call C-PP.FILTER, $$(sqlite3-api.js.in), $(3), $(5))) $(4): $(3) -$(4): $(3) $$(MAKEFILE) $$(sqlite3-wasm.c) $$(EXPORTED_FUNCTIONS.api) $$(pre-post-sqlite3.deps.$(1)) +$(4): $(3) $$(MAKEFILE) $$(sqlite3-wasm.cs) $$(EXPORTED_FUNCTIONS.api) $$(pre-post-sqlite3.deps.$(1)) @echo "Building $$@ ..." $$(emcc.bin) -o $$@ $$(emcc_opt_full) $$(emcc.flags) \ $$(emcc.jsflags) \ $$(pre-post-sqlite3.flags.$(1)) $$(emcc.flags.sqlite3.$(1)) \ - $$(cflags.common) $$(SQLITE_OPT) $$(sqlite3-wasm.c) + $$(cflags.common) $$(SQLITE_OPT) $$(sqlite3-wasm.cs) @$$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(2)) @if [ bundler-friendly = $(1) ]; then \ echo "Patching $(3) for sqlite3.wasm..."; \ @@ -806,7 +835,7 @@ $(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) speedtest1.js := $(dir.dout)/speedtest1.js speedtest1.wasm := $(dir.dout)/speedtest1.wasm cflags.speedtest1 := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM -speedtest1.cses := $(speedtest1.c) $(sqlite3-wasm.c) +speedtest1.cses := $(speedtest1.c) $(sqlite3-wasm.cs) $(eval $(call call-make-pre-post,speedtest1,vanilla)) $(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ $(pre-post-speedtest1.deps.vanilla) \ diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index e7513509ca..bcabe922bf 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -163,6 +163,10 @@ # define SQLITE_USE_URI 1 #endif +#ifdef SQLITE_WASM_EXTRA_INIT +# define SQLITE_EXTRA_INIT sqlite3_wasm_extra_init +#endif + #include #include "sqlite3.c" /* yes, .c instead of .h. */ @@ -171,7 +175,7 @@ #endif /* -** SQLITE_WASM_KEEP is functionally identical to EMSCRIPTEN_KEEPALIVE +** SQLITE_WASM_EXPORT is functionally identical to EMSCRIPTEN_KEEPALIVE ** but is not Emscripten-specific. It explicitly marks functions for ** export into the target wasm file without requiring explicit listing ** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list @@ -193,7 +197,7 @@ ** this writing we are tied to Emscripten for various reasons ** and cannot test the library with other build environments. */ -#define SQLITE_WASM_KEEP __attribute__((used,visibility("default"))) +#define SQLITE_WASM_EXPORT __attribute__((used,visibility("default"))) // See also: //__attribute__((export_name("theExportedName"), used, visibility("default"))) @@ -210,24 +214,24 @@ ** Another option is to malloc() a chunk of our own and call that our ** "stack". */ -SQLITE_WASM_KEEP void * sqlite3_wasm_stack_end(void){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_end(void){ extern void __heap_base /* see https://stackoverflow.com/questions/10038964 */; return &__heap_base; } -SQLITE_WASM_KEEP void * sqlite3_wasm_stack_begin(void){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_begin(void){ extern void __data_end; return &__data_end; } static void * pWasmStackPtr = 0; -SQLITE_WASM_KEEP void * sqlite3_wasm_stack_ptr(void){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_ptr(void){ if(!pWasmStackPtr) pWasmStackPtr = sqlite3_wasm_stack_end(); return pWasmStackPtr; } -SQLITE_WASM_KEEP void sqlite3_wasm_stack_restore(void * p){ +SQLITE_WASM_EXPORT void sqlite3_wasm_stack_restore(void * p){ pWasmStackPtr = p; } -SQLITE_WASM_KEEP void * sqlite3_wasm_stack_alloc(int n){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_stack_alloc(int n){ if(n<=0) return 0; n = (n + 7) & ~7 /* align to 8-byte boundary */; unsigned char * const p = (unsigned char *)sqlite3_wasm_stack_ptr(); @@ -263,14 +267,14 @@ static struct { /* ** Returns the current pstack position. */ -SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_ptr(void){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_pstack_ptr(void){ return PStack.pPos; } /* ** Sets the pstack position poitner to p. Results are undefined if the ** given value did not come from sqlite3_wasm_pstack_ptr(). */ -SQLITE_WASM_KEEP void sqlite3_wasm_pstack_restore(unsigned char * p){ +SQLITE_WASM_EXPORT void sqlite3_wasm_pstack_restore(unsigned char * p){ assert(p>=PStack.pBegin && p<=PStack.pEnd && p>=PStack.pPos); assert(0==(p & 0x7)); if(p>=PStack.pBegin && p<=PStack.pEnd /*&& p>=PStack.pPos*/){ @@ -285,7 +289,7 @@ SQLITE_WASM_KEEP void sqlite3_wasm_pstack_restore(unsigned char * p){ ** JS code from having to do so, and most uses of the pstack will ** call for doing so). */ -SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_alloc(int n){ +SQLITE_WASM_EXPORT void * sqlite3_wasm_pstack_alloc(int n){ if( n<=0 ) return 0; //if( n & 0x7 ) n += 8 - (n & 0x7) /* align to 8-byte boundary */; n = (n + 7) & ~7 /* align to 8-byte boundary */; @@ -298,7 +302,7 @@ SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_alloc(int n){ ** Return the number of bytes left which can be ** sqlite3_wasm_pstack_alloc()'d. */ -SQLITE_WASM_KEEP int sqlite3_wasm_pstack_remaining(void){ +SQLITE_WASM_EXPORT int sqlite3_wasm_pstack_remaining(void){ assert(PStack.pPos >= PStack.pBegin); assert(PStack.pPos <= PStack.pEnd); return (int)(PStack.pPos - PStack.pBegin); @@ -309,7 +313,7 @@ SQLITE_WASM_KEEP int sqlite3_wasm_pstack_remaining(void){ ** any space which is currently allocated. This value is a ** compile-time constant. */ -SQLITE_WASM_KEEP int sqlite3_wasm_pstack_quota(void){ +SQLITE_WASM_EXPORT int sqlite3_wasm_pstack_quota(void){ return (int)(PStack.pEnd - PStack.pBegin); } @@ -327,7 +331,7 @@ SQLITE_WASM_KEEP int sqlite3_wasm_pstack_quota(void){ ** ** Returns err_code. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ if( db!=0 ){ if( 0!=zMsg ){ @@ -349,7 +353,7 @@ struct WasmTestStruct { void (*xFunc)(void*); }; typedef struct WasmTestStruct WasmTestStruct; -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void sqlite3_wasm_test_struct(WasmTestStruct * s){ if(s){ s->v4 *= 2; @@ -377,7 +381,7 @@ void sqlite3_wasm_test_struct(WasmTestStruct * s){ ** buffer is not large enough for the generated JSON and needs to be ** increased. In debug builds that will trigger an assert(). */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT const char * sqlite3_wasm_enum_json(void){ static char aBuffer[1024 * 20] = {0} /* where the JSON goes */; int n = 0, nChildren = 0, nStruct = 0 @@ -1182,7 +1186,7 @@ const char * sqlite3_wasm_enum_json(void){ ** method, SQLITE_MISUSE is returned, else the result of the xDelete() ** call is returned. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ int rc = SQLITE_MISUSE /* ??? */; if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); @@ -1200,7 +1204,7 @@ int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char *zName){ ** defaulting to "main" if zDbName is 0. Returns 0 if no db with the ** given name is open. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT sqlite3_vfs * sqlite3_wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ sqlite3_vfs * pVfs = 0; sqlite3_file_control(pDb, zDbName ? zDbName : "main", @@ -1223,7 +1227,7 @@ sqlite3_vfs * sqlite3_wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ ** Returns 0 on success, an SQLITE_xxx code on error. Returns ** SQLITE_MISUSE if pDb is NULL. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_reset(sqlite3 *pDb){ int rc = SQLITE_MISUSE; if( pDb ){ @@ -1254,7 +1258,7 @@ int sqlite3_wasm_db_reset(sqlite3 *pDb){ ** sqlite3_wasm_db_serialize() is arguably the better way to achieve ** this. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_export_chunked( sqlite3* pDb, int (*xCallback)(unsigned const char *zOut, int n) ){ sqlite3_int64 nSize = 0; @@ -1305,7 +1309,7 @@ int sqlite3_wasm_db_export_chunked( sqlite3* pDb, ** If `*pOut` is not NULL, the caller is responsible for passing it to ** sqlite3_free() to free it. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, unsigned char **pOut, sqlite3_int64 *nOut, unsigned int mFlags ){ @@ -1362,7 +1366,7 @@ int sqlite3_wasm_db_serialize( sqlite3 *pDb, const char *zSchema, ** portability, so that the API can still work in builds where BigInt ** support is disabled or unavailable. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, const char *zFilename, const unsigned char * pData, @@ -1446,7 +1450,7 @@ int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, ** NUL-terminated pointer to that string. It is up to the caller to ** use sqlite3_wasm_pstack_restore() to free the returned pointer. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT char * sqlite3_wasm_kvvfsMakeKeyOnPstack(const char *zClass, const char *zKeyIn){ assert(sqlite3KvvfsMethods.nKeySize>24); @@ -1465,7 +1469,7 @@ char * sqlite3_wasm_kvvfsMakeKeyOnPstack(const char *zClass, ** Returns the pointer to the singleton object which holds the kvvfs ** I/O methods and associated state. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ return &sqlite3KvvfsMethods; } @@ -1480,7 +1484,7 @@ sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ ** sqlite3_vtab_config(), or SQLITE_MISUSE if the 2nd arg is not a ** valid value. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_vtab_config(sqlite3 *pDb, int op, int arg){ switch(op){ case SQLITE_VTAB_DIRECTONLY: @@ -1500,7 +1504,7 @@ int sqlite3_wasm_vtab_config(sqlite3 *pDb, int op, int arg){ ** Wrapper for the variants of sqlite3_db_config() which take ** (int,int*) variadic args. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ switch(op){ case SQLITE_DBCONFIG_ENABLE_FKEY: @@ -1531,7 +1535,7 @@ int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ ** Wrapper for the variants of sqlite3_db_config() which take ** (void*,int,int) variadic args. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int arg3){ switch(op){ case SQLITE_DBCONFIG_LOOKASIDE: @@ -1547,7 +1551,7 @@ int sqlite3_wasm_db_config_pii(sqlite3 *pDb, int op, void * pArg1, int arg2, int ** Wrapper for the variants of sqlite3_db_config() which take ** (const char *) variadic args. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ switch(op){ case SQLITE_DBCONFIG_MAINDBNAME: @@ -1564,7 +1568,7 @@ int sqlite3_wasm_db_config_s(sqlite3 *pDb, int op, const char *zArg){ ** Binding for combinations of sqlite3_config() arguments which take ** a single integer argument. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_config_i(int op, int arg){ return sqlite3_config(op, arg); } @@ -1576,7 +1580,7 @@ int sqlite3_wasm_config_i(int op, int arg){ ** Binding for combinations of sqlite3_config() arguments which take ** two int arguments. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_config_ii(int op, int arg1, int arg2){ return sqlite3_config(op, arg1, arg2); } @@ -1588,7 +1592,7 @@ int sqlite3_wasm_config_ii(int op, int arg1, int arg2){ ** Binding for combinations of sqlite3_config() arguments which take ** a single i64 argument. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_config_j(int op, sqlite3_int64 arg){ return sqlite3_config(op, arg); } @@ -1617,7 +1621,7 @@ int sqlite3_wasm_config_j(int op, sqlite3_int64 arg){ ** Safari-specific quirk covered at ** https://sqlite.org/forum/info/e5b20e1feb37a19a. **/ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void * sqlite3_wasm_ptr_to_sqlite3_free(void){ return (void*)sqlite3_free; } @@ -1647,7 +1651,7 @@ void * sqlite3_wasm_ptr_to_sqlite3_free(void){ ** the virtual FS fails. In builds compiled without SQLITE_ENABLE_WASMFS ** defined, SQLITE_NOTFOUND is returned without side effects. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_init_wasmfs(const char *zMountPoint){ static backend_t pOpfs = 0; if( !zMountPoint || !*zMountPoint ) zMountPoint = "/opfs"; @@ -1667,7 +1671,7 @@ int sqlite3_wasm_init_wasmfs(const char *zMountPoint){ return pOpfs ? 0 : SQLITE_NOMEM; } #else -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_init_wasmfs(const char *zUnused){ //emscripten_console_warn("WASMFS OPFS is not compiled in."); if(zUnused){/*unused*/} @@ -1677,51 +1681,51 @@ int sqlite3_wasm_init_wasmfs(const char *zUnused){ #if SQLITE_WASM_TESTS -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int sqlite3_wasm_test_intptr(int * p){ return *p = *p * 2; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void * sqlite3_wasm_test_voidptr(void * p){ return p; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int64_t sqlite3_wasm_test_int64_max(void){ return (int64_t)0x7fffffffffffffff; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int64_t sqlite3_wasm_test_int64_min(void){ return ~sqlite3_wasm_test_int64_max(); } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int64_t sqlite3_wasm_test_int64_times2(int64_t x){ return x * 2; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void sqlite3_wasm_test_int64_minmax(int64_t * min, int64_t *max){ *max = sqlite3_wasm_test_int64_max(); *min = sqlite3_wasm_test_int64_min(); /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT int64_t sqlite3_wasm_test_int64ptr(int64_t * p){ /*printf("sqlite3_wasm_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ return *p = *p * 2; } -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT void sqlite3_wasm_test_stack_overflow(int recurse){ if(recurse) sqlite3_wasm_test_stack_overflow(recurse); } /* For testing the 'string:dealloc' whwasmutil.xWrap() conversion. */ -SQLITE_WASM_KEEP +SQLITE_WASM_EXPORT char * sqlite3_wasm_test_str_hello(int fail){ char * s = fail ? 0 : (char *)sqlite3_malloc(6); if(s){ @@ -1732,4 +1736,4 @@ char * sqlite3_wasm_test_str_hello(int fail){ } #endif /* SQLITE_WASM_TESTS */ -#undef SQLITE_WASM_KEEP +#undef SQLITE_WASM_EXPORT diff --git a/ext/wasm/example_extra_init.c b/ext/wasm/example_extra_init.c new file mode 100644 index 0000000000..ca70386318 --- /dev/null +++ b/ext/wasm/example_extra_init.c @@ -0,0 +1,22 @@ +/* +** If the canonical build process finds the file +** sqlite3_wasm_extra_init.c in the main wasm build directory, it +** arranges to include that file in the build of sqlite3.wasm and +** defines SQLITE_EXTRA_INIT=sqlite3_wasm_extra_init. +** +** sqlite3_wasm_extra_init() must be a function with this signature: +** +** int sqlite3_wasm_extra_init(const char *) +** +** and the sqlite3 library will call it with an argument of NULL one +** time during sqlite3_initialize(). If it returns non-0, +** initialization of the library will fail. +*/ + +#include "sqlite3.h" +#include + +int sqlite3_wasm_extra_init(const char *z){ + fprintf(stderr,"%s: %s()\n", __FILE__, __func__); + return 0; +} diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index cbe6ab3518..86984c9664 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -52,7 +52,7 @@ $(EXPORTED_FUNCTIONS.fiddle): $(fiddle.EXPORTED_FUNCTIONS.in) $(MAKEFILE.fiddle) fiddle-module.js := $(dir.fiddle)/fiddle-module.js fiddle-module.wasm := $(subst .js,.wasm,$(fiddle-module.js)) -fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.c) +fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.cs) fiddle.SOAP.js := $(dir.fiddle)/$(notdir $(SOAP.js)) $(fiddle.SOAP.js): $(SOAP.js) diff --git a/manifest b/manifest index ddcdbff760..46f6e1da0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\stable-valued\sfunction\sappears\sas\sthe\sright\stable\sof\sa\sRIGHT\sJOIN,\nthe\sargument\sconstraints\son\sthe\stable-valued\sfunction\sshould\sbe\sconsidered\npart\sof\sthe\sON\sclause\sof\sthe\sRIGHT\sJOIN.\s\sFix\sfor\sthe\sproblem\sreported\nby\s[forum:/forumpost/422e635f3beafbf6|forum\spost\s422e635f3beafbf6].\s\sTest\ncases\sin\sTH3.\s\sPossibly\srelated\sto\sthe\senhancement\sat\s[501609eddf2a46d5]. -D 2023-02-26T20:57:09.904 +C Extend\swasm\sbuild\sto\senable\sinclusion\sof\sclient-custom\sC\scode,\sinitialized\svia\sthe\sSQLITE_EXTRA_INIT\smechanism,\sper\sdiscussion\sin\s[forum:1e1c04f3ed1bc96b|forum\spost\s1e1c04f3ed1bc96b]. +D 2023-02-27T07:12:28.112 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -467,7 +467,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 5a8dc82f5384a3deaaee777ed7eefd3c8cbcd0efe797a66925537d2b975af305 +F ext/wasm/GNUmakefile 122607cd0255134ae88bfdf6424f299d955b74bc8a6c48160c54c4ecbc7797b3 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -488,7 +488,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 7795b84b66a7a8dedc791340709b310bb497c F ext/wasm/api/sqlite3-v-helper.js 6f6c3e390a72e08b0a5b16a0d567d7af3c04d172831853a29d72a6f1dd40ff24 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 0dfddd0fcd354817c3d5887decebb8a293cbb926c2639ba09b995a524f1085fb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 76625a70937a8522d014ef686c32db5b53a3ee61850323f5c601d2ac39fe52fe +F ext/wasm/api/sqlite3-wasm.c 223d30c41d811cae8b9f1175fa68f2f1fb3cc056d16ad0def3b0ea5c65757a6c F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js c5ac33e39f21a3481812d7333ca6e18853640d423a01960ca8dbc6e7c5c3c21c F ext/wasm/api/sqlite3-worker1.c-pp.js 77b3835192469e9da23926ec8f78fb0b114a51d048dc54388709ac22b5c5f0a0 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -508,7 +508,8 @@ F ext/wasm/demo-worker1-promiser.js b99c550763fa792c204e9a7cceadd976004036d9fc3e F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa F ext/wasm/dist.make f55f9c9e1980ea11a59964e59535c66175a17f004d1c2e274522c3366b3a084a -F ext/wasm/fiddle.make dbe36b90b8907ae28ecb9c0e9fd8389dbdaecf117ea4fb2ea33864bdfa498a94 +F ext/wasm/example_extra_init.c 32c6ce5211409400ee875634d0071a3e4060b6b1d12417624a7e60a7c9d83c40 +F ext/wasm/fiddle.make 71b686d1f3355c0ce65cd76fb231e209dc30fbd2a3b0fd635bf26df904bffce4 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa061ce8776a6e75da8a10 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 @@ -2046,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 21aec65e5e2a01e58dd0bb8c8b9b29b8414373b53353fc7ca80a152fdd27566b -R dd644aa05c12deae01f2f05d2c4bf4e4 -U drh -Z 73efc57a25cea16beea22ea8fc7c8752 +P 46639f682975dac6efec4e230aca2c9b127b4fc77e0b465fad38ef4caa9b7a4c +R 2852837dfc2447b63257f54764575344 +U stephan +Z 644646fdf69d91f54f8f3e9a045a3c06 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f5d539d572..9c8f628bd6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46639f682975dac6efec4e230aca2c9b127b4fc77e0b465fad38ef4caa9b7a4c \ No newline at end of file +68a52cafff60f19c9c998133d04f192b1e8b23f78b8cee13807d76845ef5e13d \ No newline at end of file From ef22012b6e3d4cc271ca727b525aa1bf5e77cc73 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 27 Feb 2023 07:23:54 +0000 Subject: [PATCH 029/341] Minor doc tweak for the previous commit. FossilOrigin-Name: a8832164d36ea88dc1c5c874c67212800238eab6b1477e5ca92fbf8c77f341b4 --- ext/wasm/example_extra_init.c | 3 ++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/wasm/example_extra_init.c b/ext/wasm/example_extra_init.c index ca70386318..b91d757cd5 100644 --- a/ext/wasm/example_extra_init.c +++ b/ext/wasm/example_extra_init.c @@ -4,7 +4,8 @@ ** arranges to include that file in the build of sqlite3.wasm and ** defines SQLITE_EXTRA_INIT=sqlite3_wasm_extra_init. ** -** sqlite3_wasm_extra_init() must be a function with this signature: +** The C file must define the function sqlite3_wasm_extra_init() with +** this signature: ** ** int sqlite3_wasm_extra_init(const char *) ** diff --git a/manifest b/manifest index 46f6e1da0c..ce050b21c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extend\swasm\sbuild\sto\senable\sinclusion\sof\sclient-custom\sC\scode,\sinitialized\svia\sthe\sSQLITE_EXTRA_INIT\smechanism,\sper\sdiscussion\sin\s[forum:1e1c04f3ed1bc96b|forum\spost\s1e1c04f3ed1bc96b]. -D 2023-02-27T07:12:28.112 +C Minor\sdoc\stweak\sfor\sthe\sprevious\scommit. +D 2023-02-27T07:23:54.876 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -508,7 +508,7 @@ F ext/wasm/demo-worker1-promiser.js b99c550763fa792c204e9a7cceadd976004036d9fc3e F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa F ext/wasm/dist.make f55f9c9e1980ea11a59964e59535c66175a17f004d1c2e274522c3366b3a084a -F ext/wasm/example_extra_init.c 32c6ce5211409400ee875634d0071a3e4060b6b1d12417624a7e60a7c9d83c40 +F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f F ext/wasm/fiddle.make 71b686d1f3355c0ce65cd76fb231e209dc30fbd2a3b0fd635bf26df904bffce4 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa061ce8776a6e75da8a10 @@ -2047,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 46639f682975dac6efec4e230aca2c9b127b4fc77e0b465fad38ef4caa9b7a4c -R 2852837dfc2447b63257f54764575344 +P 68a52cafff60f19c9c998133d04f192b1e8b23f78b8cee13807d76845ef5e13d +R ba04fdd01cb61be6ab30c7f760e622b1 U stephan -Z 644646fdf69d91f54f8f3e9a045a3c06 +Z dd2ceac7f5f21e7a1d89e60cff9cef01 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9c8f628bd6..2c5d346a59 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68a52cafff60f19c9c998133d04f192b1e8b23f78b8cee13807d76845ef5e13d \ No newline at end of file +a8832164d36ea88dc1c5c874c67212800238eab6b1477e5ca92fbf8c77f341b4 \ No newline at end of file From baef8442d3373b321f25b8d034ac33ff228d23e6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 27 Feb 2023 14:48:54 +0000 Subject: [PATCH 030/341] Further to [46639f682975dac6], the parameters of a table valued function that is on the left side of a RIGHT JOIN do not need to be in the ON clause. Add new test cases and tags to associated test cases with the code. FossilOrigin-Name: 18ee689de3d1ae43b05ca52e0b62c49442ebf68a88814a7c679e8856250a4b0d --- manifest | 20 ++++++++-------- manifest.uuid | 2 +- src/whereexpr.c | 5 +++- test/join.test | 56 +++++++++++++++++++++++++++++++++++++-------- test/join8.test | 11 +++++++++ test/tabfunc01.test | 1 + 6 files changed, 74 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index ce050b21c8..e7766bf3ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sdoc\stweak\sfor\sthe\sprevious\scommit. -D 2023-02-27T07:23:54.876 +C Further\sto\s[46639f682975dac6],\sthe\sparameters\sof\sa\stable\svalued\sfunction\sthat\nis\son\sthe\sleft\sside\sof\sa\sRIGHT\sJOIN\sdo\snot\sneed\sto\sbe\sin\sthe\sON\sclause.\s\sAdd\nnew\stest\scases\sand\stags\sto\sassociated\stest\scases\swith\sthe\scode. +D 2023-02-27T14:48:54.802 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -710,7 +710,7 @@ F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 38c36bb654b4172148aef3b295bbbc1342b52078cf20174f771c746caa98f69a F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c b82d0d33315e1526904b95155e55e61149c4462147668e1cc4567c812735eff1 -F src/whereexpr.c 16d1eefd95f69843b45aba6d04fe2b63fc4f51584dff85ae380f5c20718f3c75 +F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 F src/window.c 76a27cff9ea2ded0c2c3527187029259440fabcc4cc4c07b11d942c78494a614 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 @@ -1215,14 +1215,14 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test e32cb9b1491eed682489e2cde33a22a4eb7611fe5aa3b0aa4b275fe27ab3f3ac +F test/join.test ed1daf99958fed1b9f017e56bae2bb6b49339a1ec0b70b9e8f7259960c6bf387 F test/join2.test 466b07233820f5deee66a6c3bf6e4500c8bbf7b83649e67606f5f649c07928c0 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 91f1f4c7d81fd87b58e9ba7cf4a2b5d39e3583b4f8e498a162722a60259c5208 F test/join6.test f809c025fa253f9e150c0e9afd4cef8813257bceeb6f46e04041228c9403cc2c F test/join7.test 2268dcbb54b724391dda3748ea95c60d960607ffeed67885675998e7117697f6 -F test/join8.test 40bdf5612444e986187edc5fd5ea9094cb7975b78cac563a97f1f7aefde34ba6 +F test/join8.test d384d63985e3991c404afccadaf3efd1cdf9cd72680167f80e3cb80b95c18c68 F test/join9.test 9056ddd3b0c0f4f9d658f4521038d9a37dc23ead8ca9a505d0b0db2b6a471e05 F test/joinA.test 7eab225dc1c1ab258a5e62513a4ed7cabbd3db971d59d5d92f4fb6fa14c12f6a F test/joinB.test 1b2ba3fc8568b49411787fccbf540570c148e9b6a53a30f80691cb6268098ded @@ -1552,7 +1552,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 241425ce3998687ab24adba09cb95e8012e17499b84a0ed47e128ab45e588bef +F test/tabfunc01.test 668e5e39108ce84b2f24e402f9382daecc0417edb9c78c6c8f58170584e80c91 F test/table.test eb3463b7add9f16a5bb836badf118cf391b809d09fdccd1f79684600d07ec132 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -2047,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 68a52cafff60f19c9c998133d04f192b1e8b23f78b8cee13807d76845ef5e13d -R ba04fdd01cb61be6ab30c7f760e622b1 -U stephan -Z dd2ceac7f5f21e7a1d89e60cff9cef01 +P a8832164d36ea88dc1c5c874c67212800238eab6b1477e5ca92fbf8c77f341b4 +R b2e782aaeda2460b71d17088eec2e860 +U drh +Z 08a6cb668d0e2c04f5a8fa21f96385ca # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c5d346a59..0b2808ee4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8832164d36ea88dc1c5c874c67212800238eab6b1477e5ca92fbf8c77f341b4 \ No newline at end of file +18ee689de3d1ae43b05ca52e0b62c49442ebf68a88814a7c679e8856250a4b0d \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 7860480dd8..d02c8ccfc7 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -1856,9 +1856,12 @@ void sqlite3WhereTabFuncArgs( pRhs = sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); - if( pItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ){ + if( pItem->fg.jointype & (JT_LEFT|JT_RIGHT) ){ + testcase( pItem->fg.jointype & JT_LEFT ); /* testtag-20230227a */ + testcase( pItem->fg.jointype & JT_RIGHT ); /* testtag-20230227b */ joinType = EP_OuterON; }else{ + testcase( pItem->fg.jointype & JT_LTORJ ); /* testtag-20230227c */ joinType = EP_InnerON; } sqlite3SetJoinExpr(pTerm, pItem->iCursor, joinType); diff --git a/test/join.test b/test/join.test index f0ac0be8b1..d87982e718 100644 --- a/test/join.test +++ b/test/join.test @@ -1038,18 +1038,56 @@ do_execsql_test join-22.10 { } {11} # 2019-12-22 ticket 7929c1efb2d67e98 +# Verification of testtag-20230227a +# +# 2023-02-27 https://sqlite.org/forum/forumpost/422e635f3beafbf6 +# Verification of testtag-20230227a, testtag-20230227b, and testtag-20230227c # reset_db ifcapable vtab { -do_execsql_test join-23.10 { - CREATE TABLE t0(c0); - INSERT INTO t0(c0) VALUES(123); - CREATE VIEW v0(c0) AS SELECT 0 GROUP BY 1; - SELECT t0.c0, v0.c0, vt0.name - FROM v0, t0 LEFT JOIN pragma_table_info('t0') AS vt0 - ON vt0.name LIKE 'c0' - WHERE v0.c0 == 0; -} {123 0 c0} + do_execsql_test join-23.10 { + CREATE TABLE t0(c0); + INSERT INTO t0(c0) VALUES(123); + CREATE VIEW v0(c0) AS SELECT 0 GROUP BY 1; + SELECT t0.c0, v0.c0, vt0.name + FROM v0, t0 LEFT JOIN pragma_table_info('t0') AS vt0 + ON vt0.name LIKE 'c0' + WHERE v0.c0 == 0; + } {123 0 c0} + do_execsql_test join-23.20 { + CREATE TABLE a(value TEXT); + INSERT INTO a(value) SELECT value FROM json_each('["a", "b", null]'); + CREATE TABLE b(value TEXT); + INSERT INTO b(value) SELECT value FROM json_each('["a", "c", null]'); + SELECT a.value, b.value FROM a RIGHT JOIN b ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.21 { + SELECT a.value, b.value FROM b LEFT JOIN a ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.22 { + SELECT a.value, b.value + FROM json_each('["a", "c", null]') AS b + LEFT JOIN + json_each('["a", "b", null]') AS a ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.23 { + SELECT a.value, b.value + FROM json_each('["a", "b", null]') AS a + RIGHT JOIN + json_each('["a", "c", null]') AS b ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.24 { + SELECT a.value, b.value + FROM json_each('["a", "b", null]') AS a + RIGHT JOIN + b ON a.value = b.value; + } {a a {} c {} {}} + do_execsql_test join-23.25 { + SELECT a.value, b.value + FROM a + RIGHT JOIN + json_each('["a", "c", null]') AS b ON a.value = b.value; + } {a a {} c {} {}} } #------------------------------------------------------------------------- diff --git a/test/join8.test b/test/join8.test index 1140c104f0..fc50df32ff 100644 --- a/test/join8.test +++ b/test/join8.test @@ -159,6 +159,7 @@ do_catchsql_test join8-5000 { } {0 {- -}} # 2022-04-29 dbsqlfuzz 19f1102a70cf966ab249de56d944fc20dbebcfcf +# Verification of testtag-20230227b and testtag-20230227c # reset_db do_execsql_test join8-6000 { @@ -181,6 +182,16 @@ do_execsql_test join8-6020 { SELECT value, t1.* FROM json_each('17') NATURAL RIGHT JOIN t1 WHERE (a,b) IN (SELECT rowid, b FROM t1); } {17 1 2} +do_execsql_test join8-6021 { + SELECT value, t1.* FROM json_each('null') NATURAL RIGHT JOIN t1 + WHERE (a,b) IN (SELECT rowid, b FROM t1); +} {{} 1 2} +do_execsql_test join8-6022 { + CREATE TABLE a(key TEXT); + INSERT INTO a(key) VALUES('a'),('b'); + SELECT quote(a.key), b.value + FROM a RIGHT JOIN json_each('["a","c"]') AS b ON a.key=b.value; +} {'a' a NULL c} # Bloom filter usage by RIGHT and FULL JOIN # diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 0a7e3bc245..8b7386bfec 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -270,6 +270,7 @@ do_test tabfunc01-750 { } {5.0 x5 | 7.0 x7 | 13.0 x13 | 17.0 x17 | 23.0 x23 |} # ticket https://www.sqlite.org/src/info/2ae0c599b735d59e +# Verification of testtag-20230227a do_test tabfunc01-751 { db eval { SELECT aa.value, bb.value, '|' From ca4cf12d704b02e55c7317a1b767c2a8e2760681 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 27 Feb 2023 18:55:37 +0000 Subject: [PATCH 031/341] Minor performance optimization in the computation of an affinity string for an index. FossilOrigin-Name: 07334aa17b6ded27c5bd353998d96645a94cdcf32440abb59d127a002cd98ce9 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/insert.c | 72 ++++++++++++++++++++++++++------------------------- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index e7766bf3ba..3f9b6c68df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sto\s[46639f682975dac6],\sthe\sparameters\sof\sa\stable\svalued\sfunction\sthat\nis\son\sthe\sleft\sside\sof\sa\sRIGHT\sJOIN\sdo\snot\sneed\sto\sbe\sin\sthe\sON\sclause.\s\sAdd\nnew\stest\scases\sand\stags\sto\sassociated\stest\scases\swith\sthe\scode. -D 2023-02-27T14:48:54.802 +C Minor\sperformance\soptimization\sin\sthe\scomputation\sof\san\saffinity\sstring\nfor\san\sindex. +D 2023-02-27T18:55:37.933 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -581,7 +581,7 @@ F src/hash.c c6af5f96a7a76d000f07c5402c48c318c2566beecdee9e78b9d9f60ce7119565 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 1b11a2e33ee52db93c02fddac67e39d00161d61b69fac2675b82f2aa68c1b61c +F src/insert.c 7940fce7d4aa855606432d82d20694e17c18a03956f5f5776f2404e2df7c18a8 F src/json.c c85ed6fce06f43d414b0d7fff64749d43a0dbd1067123ee407bd3a0752454161 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c b04eb648cedc45efe4298e1ef439ac4f0096ae27b5f01accb0a1f49d57789128 @@ -2047,8 +2047,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a8832164d36ea88dc1c5c874c67212800238eab6b1477e5ca92fbf8c77f341b4 -R b2e782aaeda2460b71d17088eec2e860 +P 18ee689de3d1ae43b05ca52e0b62c49442ebf68a88814a7c679e8856250a4b0d +R eb05b938ae3801da35ea0ed3c7e2ba02 U drh -Z 08a6cb668d0e2c04f5a8fa21f96385ca +Z 906f03c389d933535de5a426ca60838f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0b2808ee4d..3a4fc9a3f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18ee689de3d1ae43b05ca52e0b62c49442ebf68a88814a7c679e8856250a4b0d \ No newline at end of file +07334aa17b6ded27c5bd353998d96645a94cdcf32440abb59d127a002cd98ce9 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 65d11bee54..3abf715c9e 100644 --- a/src/insert.c +++ b/src/insert.c @@ -70,45 +70,47 @@ void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ +static SQLITE_NOINLINE const char *computeIndexAffStr(sqlite3 *db, Index *pIdx){ + /* The first time a column affinity string for a particular index is + ** required, it is allocated and populated here. It is then stored as + ** a member of the Index structure for subsequent use. + ** + ** The column affinity string will eventually be deleted by + ** sqliteDeleteIndex() when the Index structure itself is cleaned + ** up. + */ + int n; + Table *pTab = pIdx->pTable; + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ - /* The first time a column affinity string for a particular index is - ** required, it is allocated and populated here. It is then stored as - ** a member of the Index structure for subsequent use. - ** - ** The column affinity string will eventually be deleted by - ** sqliteDeleteIndex() when the Index structure itself is cleaned - ** up. - */ - int n; - Table *pTab = pIdx->pTable; - pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); - if( !pIdx->zColAff ){ - sqlite3OomFault(db); - return 0; - } - for(n=0; nnColumn; n++){ - i16 x = pIdx->aiColumn[n]; - char aff; - if( x>=0 ){ - aff = pTab->aCol[x].affinity; - }else if( x==XN_ROWID ){ - aff = SQLITE_AFF_INTEGER; - }else{ - assert( x==XN_EXPR ); - assert( pIdx->bHasExpr ); - assert( pIdx->aColExpr!=0 ); - aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); - } - if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; - pIdx->zColAff[n] = aff; - } - pIdx->zColAff[n] = 0; + sqlite3OomFault(db); + return 0; } - + for(n=0; nnColumn; n++){ + i16 x = pIdx->aiColumn[n]; + char aff; + if( x>=0 ){ + aff = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + aff = SQLITE_AFF_INTEGER; + }else{ + assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + } + if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; + pIdx->zColAff[n] = aff; + } + pIdx->zColAff[n] = 0; return pIdx->zColAff; } +const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ + if( !pIdx->zColAff ) return computeIndexAffStr(db, pIdx); + return pIdx->zColAff; +} + /* ** Compute an affinity string for a table. Space is obtained From b211a4b391957691d07e18fb67c823bda2df27dd Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Feb 2023 11:02:02 +0000 Subject: [PATCH 032/341] Improve the error message generated by the recovery extension if it is used with a non-SQLITE_ENABLE_DBPAGE_VTAB build. FossilOrigin-Name: c1f2a1d55c180fb51a4e203befbbe529bdd23137b26190c50b8f85820450e7fa --- ext/recover/dbdata.c | 6 +++-- ext/recover/recoverbuild.test | 42 +++++++++++++++++++++++++++++++++++ manifest | 17 +++++++------- manifest.uuid | 2 +- src/test1.c | 25 +++++++++++++++++++++ 5 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 ext/recover/recoverbuild.test diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c index da02b754b2..3da9f92321 100644 --- a/ext/recover/dbdata.c +++ b/ext/recover/dbdata.c @@ -809,8 +809,6 @@ static int dbdataFilter( } if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); - }else{ - pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } /* Try to determine the encoding of the db by inspecting the header @@ -819,6 +817,10 @@ static int dbdataFilter( rc = dbdataGetEncoding(pCsr); } + if( rc!=SQLITE_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + } + if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); } diff --git a/ext/recover/recoverbuild.test b/ext/recover/recoverbuild.test new file mode 100644 index 0000000000..9f33731d61 --- /dev/null +++ b/ext/recover/recoverbuild.test @@ -0,0 +1,42 @@ +# 2023 February 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] recover_common.tcl] +set testprefix recoverbuild + + +# The following tests verify that if the recovery extension is used with +# a build that does not support the sqlite_dbpage table, the error message +# is "no such table: sqlite_dbpage", and not something more generic. +# +reset_db +create_null_module db sqlite_dbpage +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(123, 'one hundred and twenty three'); +} + +forcedelete test.db2 +do_test 1.1 { + set R [sqlite3_recover_init db main test.db2] +} {sqlite_recover1} + +do_test 1.2 { + $R run +} {1} + +do_test 1.3 { + list [catch { $R finish } msg] $msg +} {1 {no such table: sqlite_dbpage}} + +finish_test + diff --git a/manifest b/manifest index 3f9b6c68df..0bdbfa7445 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sperformance\soptimization\sin\sthe\scomputation\sof\san\saffinity\sstring\nfor\san\sindex. -D 2023-02-27T18:55:37.933 +C Improve\sthe\serror\smessage\sgenerated\sby\sthe\srecovery\sextension\sif\sit\sis\sused\swith\sa\snon-SQLITE_ENABLE_DBPAGE_VTAB\sbuild. +D 2023-02-28T11:02:02.719 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -365,9 +365,10 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69 F ext/rbu/sqlite3rbu.c 348bb6251e6ec459de102f8b2dd50789a98643ef7a28e56e4c787ac9659c15ea F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 -F ext/recover/dbdata.c dc25628e405c86936c597e28f3e6f56a257029c3034c5ef7f6b10f7c02f41018 +F ext/recover/dbdata.c ac504aa06fab51ade9c0521b96cb6ec21d40619aa5e9a3e14ef9a60978699fd0 F ext/recover/recover1.test 2072993624d5e32fef20ae03b17fc06c02bcb344421fe17bb329b24d2a51e647 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a +F ext/recover/recoverbuild.test a6f05273ff5fe517afd166444597c70cb97033e7f58496433a4428a1ecb5d79f F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49 F ext/recover/recovercorrupt2.test 74bef7dd2d7dd4856f3da21be6e213d27da44827e0f5f0946ca0325b46d163ed @@ -632,7 +633,7 @@ F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a3 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 8522a04fb9c84faa1d80354430ae0ee9349727a3a4b32e3cfe39b9be8324cabd -F src/test1.c e23623dc569d2d18d1201d624ec279e9a96f0d9e83bf0620c8680d62b35d4ee7 +F src/test1.c ad4a8503c4290d37805f107107a503874ff42d8405dd1558fbe2a1a71039816d F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -2047,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 18ee689de3d1ae43b05ca52e0b62c49442ebf68a88814a7c679e8856250a4b0d -R eb05b938ae3801da35ea0ed3c7e2ba02 -U drh -Z 906f03c389d933535de5a426ca60838f +P 07334aa17b6ded27c5bd353998d96645a94cdcf32440abb59d127a002cd98ce9 +R 70a154261a0e3445bc93a596d807722e +U dan +Z 7abce0ed86afcadaaf24935108e839a7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3a4fc9a3f7..6c9dd933af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07334aa17b6ded27c5bd353998d96645a94cdcf32440abb59d127a002cd98ce9 \ No newline at end of file +c1f2a1d55c180fb51a4e203befbbe529bdd23137b26190c50b8f85820450e7fa \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 39efd09ca0..421fad5403 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2391,6 +2391,30 @@ static int SQLITE_TCLAPI vfsCurrentTimeInt64( return TCL_OK; } +/* +** Usage: create_null_module DB NAME +*/ +static int SQLITE_TCLAPI test_create_null_module( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db; + char *zName; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); + + sqlite3_create_module(db, zName, 0, 0); + return TCL_OK; +} + #ifdef SQLITE_ENABLE_SNAPSHOT /* ** Usage: sqlite3_snapshot_get DB DBNAME @@ -8981,6 +9005,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_register_cksumvfs", test_register_cksumvfs, 0 }, { "sqlite3_unregister_cksumvfs", test_unregister_cksumvfs, 0 }, { "number_of_cores", guess_number_of_cores, 0 }, + { "create_null_module", test_create_null_module, 0 }, }; static int bitmask_size = sizeof(Bitmask)*8; static int longdouble_size = sizeof(LONGDOUBLE_TYPE); From 485a92cbb961f3f5e83101141405a74d5d8710e7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Feb 2023 12:31:40 +0000 Subject: [PATCH 033/341] Performance optimization in the varint decoder for the cell parser. FossilOrigin-Name: b2b91c7cb40f1efe800059614e34823411016a3ece3f988e1574aecadd4c3114 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 20 ++++++++++++-------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 0bdbfa7445..ea5fdfaf0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\serror\smessage\sgenerated\sby\sthe\srecovery\sextension\sif\sit\sis\sused\swith\sa\snon-SQLITE_ENABLE_DBPAGE_VTAB\sbuild. -D 2023-02-28T11:02:02.719 +C Performance\soptimization\sin\sthe\svarint\sdecoder\sfor\sthe\scell\sparser. +D 2023-02-28T12:31:40.190 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -562,7 +562,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 04bf3d1c49c9000af0e2d50f04198641fd7dfc6e4eae495cb60b002d38b45e27 +F src/btree.c c547e099f853de61835ff45da8d956a932d02ecf1ffd472a1f2a103b83e6dd40 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e F src/build.c c55ab6d1b089ceef57160e840f05f692955ac90944c3d04fcf01d97fd7bfd08d @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07334aa17b6ded27c5bd353998d96645a94cdcf32440abb59d127a002cd98ce9 -R 70a154261a0e3445bc93a596d807722e -U dan -Z 7abce0ed86afcadaaf24935108e839a7 +P c1f2a1d55c180fb51a4e203befbbe529bdd23137b26190c50b8f85820450e7fa +R b1ce68bb615f7298ac94af8677093307 +U drh +Z da47b6a5dd71cc45cfa11f11f6878a01 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6c9dd933af..6d2abf5ca7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1f2a1d55c180fb51a4e203befbbe529bdd23137b26190c50b8f85820450e7fa \ No newline at end of file +b2b91c7cb40f1efe800059614e34823411016a3ece3f988e1574aecadd4c3114 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 8296294dbf..210845eeb6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1231,27 +1231,31 @@ static void btreeParseCellPtr( iKey = *pIter; if( iKey>=0x80 ){ u8 x; - iKey = ((iKey&0x7f)<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x =*++pIter) & 0x7f); + iKey = (iKey<<7) ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x10204000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<8) | (*++pIter); + iKey = (iKey<<8) ^ 0x8000 ^ (*++pIter); } } } } } + }else{ + iKey ^= 0x204000; } + }else{ + iKey ^= 0x4000; } } pIter++; From f43ffb30b019d279887a2353958c3ec065d556e1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Feb 2023 13:46:01 +0000 Subject: [PATCH 034/341] When an automatic index creates a Bloom filter, show that in the EXPLAIN QUERY PLAN output. FossilOrigin-Name: d7b2ac1c1a31fa4285cf6df0995db7e7705bb6a1bc94850c14c94cc4e3eb239a --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/where.c | 1 + test/autoindex1.test | 2 ++ test/autoindex3.test | 1 + test/with3.test | 1 + test/with6.test | 4 ++++ 7 files changed, 20 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ea5fdfaf0f..e118369f8f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\sthe\svarint\sdecoder\sfor\sthe\scell\sparser. -D 2023-02-28T12:31:40.190 +C When\san\sautomatic\sindex\screates\sa\sBloom\sfilter,\sshow\sthat\sin\sthe\nEXPLAIN\sQUERY\sPLAN\soutput. +D 2023-02-28T13:46:01.925 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -708,7 +708,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 38c36bb654b4172148aef3b295bbbc1342b52078cf20174f771c746caa98f69a +F src/where.c accf653499cf77d8974aeb9089d81bd0e689dca2b9ee2114096147eb58d70f61 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c b82d0d33315e1526904b95155e55e61149c4462147668e1cc4567c812735eff1 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -774,9 +774,9 @@ F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec F test/autoinc.test 997d6f185f138229dc4251583a1d04816423dddc2fc034871a01aeb1d728cb39 -F test/autoindex1.test cdc336e80cfd586c0e09426d58bec412db7527ca22dfabe88eab690e3acbb406 +F test/autoindex1.test b8f093ff5574f14d03645148550612b1a01cd45669b05f5728a6c6fef5ac5bec F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df -F test/autoindex3.test 2d13958a5617e987624a428d7aed91bf51f322b49b476e3573fadec697ce6da5 +F test/autoindex3.test dcd6b2f8bed2be67b131e2e671f892e971d934e24fd00988952d0e0a67e24aa7 F test/autoindex4.test 5df39313526b6f22a26bd119bbd97ca69f28386ab3c671fc10568d921c41eb08 F test/autoindex5.test 2ee94f033b87ca0160e08d81034c507aff8e230df2627f0304fa309b2fee19a3 F test/autovacuum.test 00671369bbf96c6a49989a9425f5b78b94075d6a4b031e5e00000c2c32f365df @@ -1923,10 +1923,10 @@ F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c F test/windowpushd.test d8895d08870b7226f7693665bd292eb177e62ca06799184957b3ca7dc03067df F test/with1.test b93833890e5d2a368e78747f124503a0159aa029b98e9ed4795ebf630b2efd3d F test/with2.test a1df41b987198383b9b70bf5e5fda390582e46398653858dbc6ceb24253b28df -F test/with3.test e7bf809bf75c1f44f98bca78bc331dbf542002c5227bf53c1261144db4e824c8 +F test/with3.test fe15975c0b53c9098a757902a908e3f8d6d80ce47c5363ac600f28a79ef8c0ca F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/with5.test 6248213c41fab36290b5b73aa3f937309dfba337004d9d8434c3fabc8c7d4be8 -F test/with6.test 7afab289442bd0a023c18deef854642932294fa63cdb885a4b4db69e28c5fbf9 +F test/with6.test e097a03e5c898a8cd8f3a2d6a994ec510ea4376b5d484c2b669a41001e7758c8 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 F test/without_rowid1.test a5210b8770dc4736bca4e74bc96588f43025ad03ad6a80f885afd36d9890e217 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c1f2a1d55c180fb51a4e203befbbe529bdd23137b26190c50b8f85820450e7fa -R b1ce68bb615f7298ac94af8677093307 +P b2b91c7cb40f1efe800059614e34823411016a3ece3f988e1574aecadd4c3114 +R ad671f91f6548ed5a58a53003a52602c U drh -Z da47b6a5dd71cc45cfa11f11f6878a01 +Z 2bfb99217fbdf0c560b29d9c5fd7f671 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6d2abf5ca7..4cb695987d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2b91c7cb40f1efe800059614e34823411016a3ece3f988e1574aecadd4c3114 \ No newline at end of file +d7b2ac1c1a31fa4285cf6df0995db7e7705bb6a1bc94850c14c94cc4e3eb239a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 08dd21824c..a0a580b100 100644 --- a/src/where.c +++ b/src/where.c @@ -1031,6 +1031,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "for %s", pTable->zName)); if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel); pLevel->regFilter = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); } diff --git a/test/autoindex1.test b/test/autoindex1.test index 4b290c7961..9c073a6ae9 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -194,6 +194,7 @@ do_eqp_test autoindex1-501 { QUERY PLAN |--SCAN t501 `--CORRELATED LIST SUBQUERY xxxxxx + |--BLOOM FILTER ON t502 (y=?) `--SEARCH t502 USING AUTOMATIC COVERING INDEX (y=?) } do_eqp_test autoindex1-502 { @@ -283,6 +284,7 @@ do_eqp_test autoindex1-600a { | `--CORRELATED SCALAR SUBQUERY xxxxxx | `--SEARCH later USING COVERING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date>? AND owner_change_date?) + |--BLOOM FILTER ON u (b=?) `--SEARCH u USING AUTOMATIC COVERING INDEX (b=?) } diff --git a/test/with3.test b/test/with3.test index 85889453a0..650740dcc1 100644 --- a/test/with3.test +++ b/test/with3.test @@ -109,6 +109,7 @@ ifcapable analyze { | `--RECURSIVE STEP | `--SCAN cnt |--SCAN y1 + |--BLOOM FILTER ON cnt (i=?) `--SEARCH cnt USING AUTOMATIC COVERING INDEX (i=?) }] } diff --git a/test/with6.test b/test/with6.test index 91d64fa297..2a4bfc646d 100644 --- a/test/with6.test +++ b/test/with6.test @@ -363,9 +363,13 @@ do_eqp_test 400 { | |--SEARCH raw USING INDEX sqlite_autoindex_raw_1 (country=? AND date>? AND date Date: Tue, 28 Feb 2023 14:28:54 +0000 Subject: [PATCH 035/341] In the Bloom filter optimization, hash all strings and blobs into the same value, because we do not know if two different strings might compare equal even if they have different byte sequences, due to collating functions. Formerly, the hash of a string or blob was just its length. This could all be improved. Fix for the issue reported by [forum:/forumpost/0846211821|forum post 0846211821]. FossilOrigin-Name: 090304b870419acb5b05205a07fc75830b556928149f76a843cda526f77a6fc0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 3 +-- test/bloom1.test | 16 +++++++++++++++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e118369f8f..3aaadcecc8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\san\sautomatic\sindex\screates\sa\sBloom\sfilter,\sshow\sthat\sin\sthe\nEXPLAIN\sQUERY\sPLAN\soutput. -D 2023-02-28T13:46:01.925 +C In\sthe\sBloom\sfilter\soptimization,\shash\sall\sstrings\sand\sblobs\sinto\sthe\ssame\nvalue,\sbecause\swe\sdo\snot\sknow\sif\stwo\sdifferent\sstrings\smight\scompare\sequal\neven\sif\sthey\shave\sdifferent\sbyte\ssequences,\sdue\sto\scollating\sfunctions.\nFormerly,\sthe\shash\sof\sa\sstring\sor\sblob\swas\sjust\sits\slength.\s\sThis\scould\nall\sbe\simproved.\s\sFix\sfor\sthe\sissue\sreported\sby\n[forum:/forumpost/0846211821|forum\spost\s0846211821]. +D 2023-02-28T14:28:54.933 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -693,7 +693,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 0bf4804f6e9cae872502aa4533a62ba3f6e97ee28523302deb27f9c990b5f44f +F src/vdbe.c b3fd04b0643edd7e0a4356aff6d2cf50f04d0e182e292c3a330d1afffe3100e1 F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 @@ -816,7 +816,7 @@ F test/bind2.test 918bc35135f4141809ead7585909cde57d44db90a7a62aef540127148f91aa F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 F test/blob.test e7ac6c7d3a985cc4678c64f325292529a69ae252 -F test/bloom1.test 2785a190fcc2a5e170e5d38b08aca8ff0f3e3b4a74d47453d6ac1bd355180a6a +F test/bloom1.test 589361c1f20158a8583863738c883f0e73e82d18422c9b4ed9c7068c13c2d310 F test/boundary1.tcl 6421b2d920d8b09539503a8673339d32f7609eb1 F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b2b91c7cb40f1efe800059614e34823411016a3ece3f988e1574aecadd4c3114 -R ad671f91f6548ed5a58a53003a52602c +P d7b2ac1c1a31fa4285cf6df0995db7e7705bb6a1bc94850c14c94cc4e3eb239a +R 2983937b0d34422b7617ca6b30588a0e U drh -Z 2bfb99217fbdf0c560b29d9c5fd7f671 +Z c3cca21d42699bd3e59fea902b259dbf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4cb695987d..e66a052f86 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7b2ac1c1a31fa4285cf6df0995db7e7705bb6a1bc94850c14c94cc4e3eb239a \ No newline at end of file +090304b870419acb5b05205a07fc75830b556928149f76a843cda526f77a6fc0 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index f3853e7fd2..a9febbb6e2 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -683,8 +683,7 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ }else if( p->flags & MEM_Real ){ h += sqlite3VdbeIntValue(p); }else if( p->flags & (MEM_Str|MEM_Blob) ){ - h += p->n; - if( p->flags & MEM_Zero ) h += p->u.nZero; + /* no-op */ } } return h; diff --git a/test/bloom1.test b/test/bloom1.test index 1846e4d63f..87a7c8632c 100644 --- a/test/bloom1.test +++ b/test/bloom1.test @@ -99,6 +99,20 @@ do_eqp_test 2.1 { | |--SCAN transit | `--SEARCH objs USING COVERING INDEX objs_cspo (o=? AND p=?) `--SCAN transit -} +} + +# 2023-02-28 +# https://sqlite.org/forum/forumpost/0846211821 +# +# Bloom filter gives an incorrect result if the collating sequence is +# anything other than binary. +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t0(x TEXT COLLATE rtrim); + INSERT INTO t0(x) VALUES ('a'), ('b'), ('c'); + CREATE VIEW v0(y) AS SELECT DISTINCT x FROM t0; + SELECT count(*) FROM t0, v0 WHERE x='b '; +} 3 finish_test From 4990fc84f1586662caab919ad8eadc052f668572 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Feb 2023 18:06:52 +0000 Subject: [PATCH 036/341] Only use a Bloom filter on an automatic index if one or more of the key columns in the index can take on non-TEXT values. FossilOrigin-Name: 5916705c731604d2e6b51a307cc8d7b67f4c102062bfdfcbc716a2916e0b0d86 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/vdbe.c | 5 ++++- src/where.c | 15 +++++++++++++-- test/autoindex1.test | 1 - test/bloom1.test | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 59 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 3aaadcecc8..5395254fff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sBloom\sfilter\soptimization,\shash\sall\sstrings\sand\sblobs\sinto\sthe\ssame\nvalue,\sbecause\swe\sdo\snot\sknow\sif\stwo\sdifferent\sstrings\smight\scompare\sequal\neven\sif\sthey\shave\sdifferent\sbyte\ssequences,\sdue\sto\scollating\sfunctions.\nFormerly,\sthe\shash\sof\sa\sstring\sor\sblob\swas\sjust\sits\slength.\s\sThis\scould\nall\sbe\simproved.\s\sFix\sfor\sthe\sissue\sreported\sby\n[forum:/forumpost/0846211821|forum\spost\s0846211821]. -D 2023-02-28T14:28:54.933 +C Only\suse\sa\sBloom\sfilter\son\san\sautomatic\sindex\sif\sone\sor\smore\sof\sthe\skey\ncolumns\sin\sthe\sindex\scan\stake\son\snon-TEXT\svalues. +D 2023-02-28T18:06:52.731 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -693,7 +693,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c b3fd04b0643edd7e0a4356aff6d2cf50f04d0e182e292c3a330d1afffe3100e1 +F src/vdbe.c 523fbe2086179b42dfdc07093f592443f0a3e9583d1ff17ef0d03a25777c1347 F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 @@ -708,7 +708,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c accf653499cf77d8974aeb9089d81bd0e689dca2b9ee2114096147eb58d70f61 +F src/where.c 76d2014b1e69935cc9b50a710f08321d63f7827f2cf6fd9f21cde7e909a61a6c F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c b82d0d33315e1526904b95155e55e61149c4462147668e1cc4567c812735eff1 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -774,7 +774,7 @@ F test/auth2.test 9eb7fce9f34bf1f50d3f366fb3e606be5a2000a1 F test/auth3.test 76d20a7fa136d63bcfcf8bcb65c0b1455ed71078d81f22bcd0550d3eb18594ab F test/autoanalyze1.test b9cc3f32a990fa56669b668d237c6d53e983554ae80c0604992e18869a0b2dec F test/autoinc.test 997d6f185f138229dc4251583a1d04816423dddc2fc034871a01aeb1d728cb39 -F test/autoindex1.test b8f093ff5574f14d03645148550612b1a01cd45669b05f5728a6c6fef5ac5bec +F test/autoindex1.test d34caffb0384003ee28eae87679214c029e9be4b332d9649a79e0b94ab70502c F test/autoindex2.test 12ef578928102baaa0dc23ad397601a2f4ecb0df F test/autoindex3.test dcd6b2f8bed2be67b131e2e671f892e971d934e24fd00988952d0e0a67e24aa7 F test/autoindex4.test 5df39313526b6f22a26bd119bbd97ca69f28386ab3c671fc10568d921c41eb08 @@ -816,7 +816,7 @@ F test/bind2.test 918bc35135f4141809ead7585909cde57d44db90a7a62aef540127148f91aa F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 F test/blob.test e7ac6c7d3a985cc4678c64f325292529a69ae252 -F test/bloom1.test 589361c1f20158a8583863738c883f0e73e82d18422c9b4ed9c7068c13c2d310 +F test/bloom1.test ab125229849c085b8d4a6768cc321d330d1ed47b21902f1a2854db832c625768 F test/boundary1.tcl 6421b2d920d8b09539503a8673339d32f7609eb1 F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d7b2ac1c1a31fa4285cf6df0995db7e7705bb6a1bc94850c14c94cc4e3eb239a -R 2983937b0d34422b7617ca6b30588a0e +P 090304b870419acb5b05205a07fc75830b556928149f76a843cda526f77a6fc0 +R 192a8af525d9a67c0614fb52e425206b U drh -Z c3cca21d42699bd3e59fea902b259dbf +Z d906284cbfaa2356b8109e77d4c16f08 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e66a052f86..a57e5825b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -090304b870419acb5b05205a07fc75830b556928149f76a843cda526f77a6fc0 \ No newline at end of file +5916705c731604d2e6b51a307cc8d7b67f4c102062bfdfcbc716a2916e0b0d86 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index a9febbb6e2..61ff0b3cc0 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -683,7 +683,10 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ }else if( p->flags & MEM_Real ){ h += sqlite3VdbeIntValue(p); }else if( p->flags & (MEM_Str|MEM_Blob) ){ - /* no-op */ + /* All strings have the same hash and all blobs have the same hash, + ** though, at least, those hashes are different from each other and + ** from NULL. */ + h += 4093 + (p->flags & (MEM_Str|MEM_Blob)); } } return h; diff --git a/src/where.c b/src/where.c index a0a580b100..f71380592e 100644 --- a/src/where.c +++ b/src/where.c @@ -892,7 +892,8 @@ static SQLITE_NOINLINE void constructAutomaticIndex( char *zNotUsed; /* Extra space on the end of pIdx */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ - u8 sentWarning = 0; /* True if a warnning has been issued */ + u8 sentWarning = 0; /* True if a warning has been issued */ + u8 useBloomFilter = 0; /* True to also add a Bloom filter */ Expr *pPartial = 0; /* Partial Index Expression */ int iContinue = 0; /* Jump here to skip excluded rows */ SrcItem *pTabItem; /* FROM clause term being indexed */ @@ -998,6 +999,16 @@ static SQLITE_NOINLINE void constructAutomaticIndex( assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; n++; + if( ALWAYS(pX->pLeft!=0) + && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT + ){ + /* TUNING: only use a Bloom filter on an automatic index + ** if one or more key columns has the ability to hold numeric + ** values, since strings all have the same hash in the Bloom + ** filter implementation and hence a Bloom filter on a text column + ** is not usually helpful. */ + useBloomFilter = 1; + } } } } @@ -1030,7 +1041,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "for %s", pTable->zName)); - if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){ sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel); pLevel->regFilter = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); diff --git a/test/autoindex1.test b/test/autoindex1.test index 9c073a6ae9..08bd9f74e6 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -284,7 +284,6 @@ do_eqp_test autoindex1-600a { | `--CORRELATED SCALAR SUBQUERY xxxxxx | `--SEARCH later USING COVERING INDEX sqlite_autoindex_flock_owner_1 (flock_no=? AND owner_change_date>? AND owner_change_date Date: Tue, 28 Feb 2023 19:39:59 +0000 Subject: [PATCH 037/341] Add an sqlite3_db_config() option - SQLITE_DBCONFIG_STMT_SCANSTATS - for enabling and disabling the collection of sqlite3_stmt_scanstats() statistics in SQLITE_ENABLE_STMT_SCANSTATUS builds. Collection of statistics is disabled by default. FossilOrigin-Name: 0f5579bef27b84ee855065cfe87703c51e1f9773906a9e0d4e4dafc90bd0e553 --- manifest | 36 +++++++++---------- manifest.uuid | 2 +- src/global.c | 2 +- src/main.c | 1 + src/shell.c.in | 12 +++++-- src/sqlite.h.in | 13 ++++++- src/sqliteInt.h | 9 ++++- src/test1.c | 2 +- src/vdbe.c | 19 ++++++---- src/vdbeaux.c | 84 +++++++++++++++++++++++-------------------- src/where.c | 2 +- src/wherecode.c | 50 +++++++++++++------------- test/scanstatus.test | 8 +++++ test/scanstatus2.test | 10 ++++++ 14 files changed, 155 insertions(+), 95 deletions(-) diff --git a/manifest b/manifest index 5395254fff..c289db4cb6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\suse\sa\sBloom\sfilter\son\san\sautomatic\sindex\sif\sone\sor\smore\sof\sthe\skey\ncolumns\sin\sthe\sindex\scan\stake\son\snon-TEXT\svalues. -D 2023-02-28T18:06:52.731 +C Add\san\ssqlite3_db_config()\soption\s-\sSQLITE_DBCONFIG_STMT_SCANSTATS\s-\sfor\senabling\sand\sdisabling\sthe\scollection\sof\ssqlite3_stmt_scanstats()\sstatistics\sin\sSQLITE_ENABLE_STMT_SCANSTATUS\sbuilds.\sCollection\sof\sstatistics\sis\sdisabled\sby\sdefault. +D 2023-02-28T19:39:59.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -577,7 +577,7 @@ F src/expr.c 2e5e67e800c9416996df52409746a2f3c7dadf5d38b5a367ce379b239e3f40c8 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 -F src/global.c e06ff8e0acd85aec13563c9ecb44fbbf38232ccf73594998fd880b92d619594b +F src/global.c 428d2580a1cdf5dbe1f356d1feab83710ae0cc862ece0fb57bc8259e43838c74 F src/hash.c c6af5f96a7a76d000f07c5402c48c318c2566beecdee9e78b9d9f60ce7119565 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 @@ -586,7 +586,7 @@ F src/insert.c 7940fce7d4aa855606432d82d20694e17c18a03956f5f5776f2404e2df7c18a8 F src/json.c c85ed6fce06f43d414b0d7fff64749d43a0dbd1067123ee407bd3a0752454161 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c b04eb648cedc45efe4298e1ef439ac4f0096ae27b5f01accb0a1f49d57789128 -F src/main.c cb5b7277b525ac8f7663426485332bb4f3c1fd9d07f4051980dcab806b780808 +F src/main.c 834e257234fdf9e74c204c1a130920941618e48b2065747162862773ba5416b5 F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -624,16 +624,16 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 230dc0601f55ae4909b5f88bc002bfa1f1fb331e51f2c6d670d3effc2ced365e -F src/shell.c.in cb763c332be668ecba48c85de52df52a86f992867b813460d55c3bccaaf4c0eb -F src/sqlite.h.in 5f308635ad467b50af858f271e403d14f8bcc574c2610f7cfd2d00f5bb37f616 +F src/shell.c.in a319e2a6cdb166d65a91ad0e69a2e9dea6afd066acbf9adce883a6762b43ec49 +F src/sqlite.h.in 7a2fed8c11df276bc35bc9ac47981cb0a7f9db39b85761a7dd50028fe77d28ce F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h cb7182dcdc9910d5f1352c90762545cc5ffb79c4a47f4ae7c5ee044fdb80423b +F src/sqliteInt.h e40980dd50f31aa57bed8dbb32c8fecf64b3ad9bc235a3abb24a3b95734cafeb F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 8522a04fb9c84faa1d80354430ae0ee9349727a3a4b32e3cfe39b9be8324cabd -F src/test1.c ad4a8503c4290d37805f107107a503874ff42d8405dd1558fbe2a1a71039816d +F src/test1.c 5ca9cb48a5dfef5980417bf5f76594071ae94795cbb8d065f8a1290c3b308ec6 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -693,11 +693,11 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 523fbe2086179b42dfdc07093f592443f0a3e9583d1ff17ef0d03a25777c1347 +F src/vdbe.c 1ddcb8a799d1a4e0c3dcc5c3641923d1343439070b35872c1801a78a0059234e F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 -F src/vdbeaux.c 3f9e3b6585e7434aa11300169dd66ddf0fc963a0c6f7940bdc058335dadeb353 +F src/vdbeaux.c 0f5201346a83a35a08e833c3a03abe626119c07b7361c28bc6a259b98540f1d6 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 0388576b7cf0be13ce14b9e3b8aa90b8a1b923b60321d0242131ae0b5732b43b F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -708,9 +708,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 76d2014b1e69935cc9b50a710f08321d63f7827f2cf6fd9f21cde7e909a61a6c +F src/where.c 6b45dc4c47d636f5adb0d78ed71cefab5ab2a7287fb80c86604a5f7874d2ba22 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c b82d0d33315e1526904b95155e55e61149c4462147668e1cc4567c812735eff1 +F src/wherecode.c 9919e5a22f4b24dd96c49b8981484cbe6bbfcf466ff73ac40a06e1356aa8bf87 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 F src/window.c 76a27cff9ea2ded0c2c3527187029259440fabcc4cc4c07b11d942c78494a614 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1436,8 +1436,8 @@ F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 -F test/scanstatus.test 74391c2c0926994bf0962db6c04c9ff5b95d15a41d2e076fe011b73f92815e70 -F test/scanstatus2.test ca6c258425977e5935f0ff3a8670d9b5d813dd5b83cf0bbe39acfc0fd2b5a0b1 +F test/scanstatus.test 177fe0fea60898b2190328fbe358bafa2637e81856f90fb527a792efaf249c14 +F test/scanstatus2.test 7c367a5c818bad1936f291d33f688a3daa42279d4e989d920660c7002d67d3a2 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 090304b870419acb5b05205a07fc75830b556928149f76a843cda526f77a6fc0 -R 192a8af525d9a67c0614fb52e425206b -U drh -Z d906284cbfaa2356b8109e77d4c16f08 +P 5916705c731604d2e6b51a307cc8d7b67f4c102062bfdfcbc716a2916e0b0d86 +R 62c12b75bd9869fbb782d0697362eb0c +U dan +Z 823edb92d517eed03d3961ba1e99b0c0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a57e5825b0..b5a7d79f5a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5916705c731604d2e6b51a307cc8d7b67f4c102062bfdfcbc716a2916e0b0d86 \ No newline at end of file +0f5579bef27b84ee855065cfe87703c51e1f9773906a9e0d4e4dafc90bd0e553 \ No newline at end of file diff --git a/src/global.c b/src/global.c index 799b1e47c4..b018c50027 100644 --- a/src/global.c +++ b/src/global.c @@ -291,7 +291,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ 0, /* iPrngSeed */ #ifdef SQLITE_DEBUG - {0,0,0,0,0,0} /* aTune */ + {0,0,0,0,0,0}, /* aTune */ #endif }; diff --git a/src/main.c b/src/main.c index 9ec313e6fc..a0e5e65852 100644 --- a/src/main.c +++ b/src/main.c @@ -970,6 +970,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, + { SQLITE_DBCONFIG_STMT_SCANSTATS, SQLITE_StmtScanStats }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ diff --git a/src/shell.c.in b/src/shell.c.in index 5c8d47e9a0..a983ec0c0e 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5416,8 +5416,13 @@ static void open_db(ShellState *p, int openFlags){ } #endif } - if( p->bSafeModePersist && p->db!=0 ){ - sqlite3_set_authorizer(p->db, safeModeAuth, p); + if( p->db!=0 ){ + if( p->bSafeModePersist ){ + sqlite3_set_authorizer(p->db, safeModeAuth, p); + } + sqlite3_db_config( + p->db, SQLITE_DBCONFIG_STMT_SCANSTATS, p->scanstatsOn, (int*)0 + ); } } @@ -9769,6 +9774,9 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ p->scanstatsOn = (u8)booleanValue(azArg[1]); } + sqlite3_db_config( + p->db, SQLITE_DBCONFIG_STMT_SCANSTATS, p->scanstatsOn, (int*)0 + ); #ifndef SQLITE_ENABLE_STMT_SCANSTATUS raw_printf(stderr, "Warning: .scanstats not available in this build.\n"); #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 06737e1c90..0a31ba033d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2456,6 +2456,16 @@ struct sqlite3_mem_methods { ** not considered a bug since SQLite versions 3.3.0 and earlier do not support ** either generated columns or decending indexes. ** + +** [[SQLITE_DBCONFIG_STMT_SCANSTATS]] +**
SQLITE_DBCONFIG_STMT_SCANSTATS +**
The SQLITE_DBCONFIG_STMT_SCANSTATS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is clear (collection of statistics is disabled) +** by default. ** */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -2476,7 +2486,8 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_STMT_SCANSTATS 1018 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1018 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 40afc08dbb..21ad2fa7e1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1757,7 +1757,7 @@ struct sqlite3 { #define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ /* result set is empty */ #define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommit 0x00000400 /* READ UNCOMMITTED in shared-cache */ +#define SQLITE_StmtScanStats 0x00000400 /* Enable stmt_scanstats() counters */ #define SQLITE_NoCkptOnClose 0x00000800 /* No checkpoint on close()/DETACH */ #define SQLITE_ReverseOrder 0x00001000 /* Reverse unordered SELECTs */ #define SQLITE_RecTriggers 0x00002000 /* Enable recursive triggers */ @@ -1783,6 +1783,7 @@ struct sqlite3 { /* DELETE, or UPDATE and return */ /* the count using a callback. */ #define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ +#define SQLITE_ReadUncommit HI(0x00004) /* READ UNCOMMITTED in shared-cache */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG @@ -5581,4 +5582,10 @@ int sqlite3KvvfsInit(void); sqlite3_uint64 sqlite3Hwtime(void); #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define IS_STMT_SCANSTATUS(db) (db->flags & SQLITE_StmtScanStats) +#else +# define IS_STMT_SCANSTATUS(db) 0 +#endif + #endif /* SQLITEINT_H */ diff --git a/src/test1.c b/src/test1.c index 421fad5403..a765a66fa6 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2400,7 +2400,6 @@ static int SQLITE_TCLAPI test_create_null_module( int objc, Tcl_Obj *CONST objv[] ){ - int rc; sqlite3 *db; char *zName; @@ -8314,6 +8313,7 @@ static int SQLITE_TCLAPI test_sqlite3_db_config( { "DQS_DML", SQLITE_DBCONFIG_DQS_DML }, { "DQS_DDL", SQLITE_DBCONFIG_DQS_DDL }, { "LEGACY_FILE_FORMAT", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, + { "STMT_SCANSTATS", SQLITE_DBCONFIG_STMT_SCANSTATS }, }; int i; int v = 0; diff --git a/src/vdbe.c b/src/vdbe.c index 61ff0b3cc0..2bb4964c37 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -736,6 +736,7 @@ int sqlite3VdbeExec( Mem *pOut = 0; /* Output operand */ #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) u64 *pnCycle = 0; + int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0; #endif /*** INSERT STACK UNION HERE ***/ @@ -800,13 +801,17 @@ int sqlite3VdbeExec( assert( pOp>=aOp && pOp<&aOp[p->nOp]); nVmStep++; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + +#if defined(VDBE_PROFILE) pOp->nExec++; pnCycle = &pOp->nCycle; -# ifdef VDBE_PROFILE - if( sqlite3NProfileCnt==0 ) -# endif + if( sqlite3NProfileCnt==0 ) *pnCycle -= sqlite3Hwtime(); +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( bStmtScanStatus ){ + pOp->nExec++; + pnCycle = &pOp->nCycle; *pnCycle -= sqlite3Hwtime(); + } #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -8758,8 +8763,10 @@ default: { /* This is really OP_Noop, OP_Explain */ *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); pnCycle = 0; #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) - *pnCycle += sqlite3Hwtime(); - pnCycle = 0; + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; + } #endif /* The following code adds nothing to the actual functionality diff --git a/src/vdbeaux.c b/src/vdbeaux.c index d04d8f1e17..87fd067322 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -443,10 +443,10 @@ void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ */ int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ int addr = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) +#if !defined(SQLITE_DEBUG) /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. ** But omit them (for performance) during production builds */ - if( pParse->explain==2 ) + if( pParse->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { char *zMsg; @@ -1122,18 +1122,20 @@ void sqlite3VdbeScanStatus( LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); - ScanStatus *aNew; - aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); - if( aNew ){ - ScanStatus *pNew = &aNew[p->nScan++]; - memset(pNew, 0, sizeof(ScanStatus)); - pNew->addrExplain = addrExplain; - pNew->addrLoop = addrLoop; - pNew->addrVisit = addrVisit; - pNew->nEst = nEst; - pNew->zName = sqlite3DbStrDup(p->db, zName); - p->aScan = aNew; + if( IS_STMT_SCANSTATUS(p->db) ){ + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + memset(pNew, 0, sizeof(ScanStatus)); + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } } } @@ -1150,20 +1152,22 @@ void sqlite3VdbeScanStatusRange( int addrStart, int addrEnd ){ - ScanStatus *pScan = 0; - int ii; - for(ii=p->nScan-1; ii>=0; ii--){ - pScan = &p->aScan[ii]; - if( pScan->addrExplain==addrExplain ) break; - pScan = 0; - } - if( pScan ){ - if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; - for(ii=0; iiaAddrRange); ii+=2){ - if( pScan->aAddrRange[ii]==0 ){ - pScan->aAddrRange[ii] = addrStart; - pScan->aAddrRange[ii+1] = addrEnd; - break; + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; + for(ii=0; iiaAddrRange); ii+=2){ + if( pScan->aAddrRange[ii]==0 ){ + pScan->aAddrRange[ii] = addrStart; + pScan->aAddrRange[ii+1] = addrEnd; + break; + } } } } @@ -1180,19 +1184,21 @@ void sqlite3VdbeScanStatusCounters( int addrLoop, int addrVisit ){ - ScanStatus *pScan = 0; - int ii; - for(ii=p->nScan-1; ii>=0; ii--){ - pScan = &p->aScan[ii]; - if( pScan->addrExplain==addrExplain ) break; - pScan = 0; - } - if( pScan ){ - pScan->addrLoop = addrLoop; - pScan->addrVisit = addrVisit; + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + pScan->addrLoop = addrLoop; + pScan->addrVisit = addrVisit; + } } } -#endif +#endif /* defined(SQLITE_ENABLE_STMT_SCANSTATUS) */ /* diff --git a/src/where.c b/src/where.c index f71380592e..8f6bc24be9 100644 --- a/src/where.c +++ b/src/where.c @@ -831,7 +831,7 @@ static void explainAutomaticIndex( int bPartial, /* True if pIdx is a partial index */ int *pAddrExplain /* OUT: Address of OP_Explain */ ){ - if( pParse->explain!=2 ){ + if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){ Table *pTab = pIdx->pTable; const char *zSep = ""; char *zText = 0; diff --git a/src/wherecode.c b/src/wherecode.c index 860acb44cf..f74273cf18 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -111,9 +111,9 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ /* ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was -** defined at compile-time. If it is not a no-op, a single OP_Explain opcode -** is added to the output to describe the table scan strategy in pLevel. +** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG +** was defined at compile-time. If it is not a no-op, a single OP_Explain +** opcode is added to the output to describe the table scan strategy in pLevel. ** ** If an OP_Explain opcode is added to the VM, its address is returned. ** Otherwise, if no OP_Explain is coded, zero is returned. @@ -125,8 +125,8 @@ int sqlite3WhereExplainOneScan( u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ int ret = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( sqlite3ParseToplevel(pParse)->explain==2 ) +#if !defined(SQLITE_DEBUG) + if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { SrcItem *pItem = &pTabList->a[pLevel->iFrom]; @@ -292,27 +292,29 @@ void sqlite3WhereAddScanStatus( WhereLevel *pLvl, /* Level to add scanstatus() entry for */ int addrExplain /* Address of OP_Explain (or 0) */ ){ - const char *zObj = 0; - WhereLoop *pLoop = pLvl->pWLoop; - int wsFlags = pLoop->wsFlags; - int viaCoroutine = 0; + if( IS_STMT_SCANSTATUS( sqlite3VdbeDb(v) ) ){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + int wsFlags = pLoop->wsFlags; + int viaCoroutine = 0; - if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ - zObj = pLoop->u.btree.pIndex->zName; - }else{ - zObj = pSrclist->a[pLvl->iFrom].zName; - viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; - } - sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj - ); - - if( viaCoroutine==0 ){ - if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ - sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; } - if( wsFlags & WHERE_INDEXED ){ - sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); + + if( viaCoroutine==0 ){ + if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } + if( wsFlags & WHERE_INDEXED ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + } } } } diff --git a/test/scanstatus.test b/test/scanstatus.test index fa00a356bc..1eab9a26a4 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -45,6 +45,11 @@ proc do_scanstatus_test {tn res} { uplevel [list do_test $tn [list set {} $ret] [list {*}$res]] } +do_execsql_test 1.0a { SELECT count(*) FROM t1, t2; } 6 +do_scanstatus_test 1.0b { } + +sqlite3_db_config db STMT_SCANSTATS 1 + do_execsql_test 1.1 { SELECT count(*) FROM t1, t2; } 6 do_scanstatus_test 1.2 { nLoop 1 nVisit 2 nEst 1048576.0 zName t1 zExplain {SCAN t1} @@ -94,6 +99,7 @@ do_scanstatus_test 1.10 { # Try a few different types of scans. # reset_db +sqlite3_db_config db STMT_SCANSTATS 1 do_execsql_test 2.1 { CREATE TABLE x1(i INTEGER PRIMARY KEY, j); INSERT INTO x1 VALUES(1, 'one'); @@ -277,6 +283,7 @@ do_scanstatus_test 4.2.2 { # Further tests of different scan types. # reset_db +sqlite3_db_config db STMT_SCANSTATS 1 proc tochar {i} { set alphabet {a b c d e f g h i j k l m n o p q r s t u v w x y z} return [lindex $alphabet [expr $i % [llength $alphabet]]] @@ -365,6 +372,7 @@ do_eqp_test 5.5.1 { } { QUERY PLAN |--SCAN t3 + |--BLOOM FILTER ON t1 (c=?) `--SEARCH t1 USING AUTOMATIC COVERING INDEX (c=?) } do_execsql_test 5.5.2 { diff --git a/test/scanstatus2.test b/test/scanstatus2.test index 497cbe67d8..7b8453f94b 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -19,6 +19,8 @@ ifcapable !scanstatus { return } +sqlite3_db_config db STMT_SCANSTATS 1 + do_execsql_test 1.0 { CREATE TABLE t1(a, b); CREATE TABLE t2(x, y); @@ -141,6 +143,7 @@ QUERY (nCycle=nnn) #------------------------------------------------------------------------- ifcapable fts5 { reset_db + sqlite3_db_config db STMT_SCANSTATS 1 do_execsql_test 2.0 { CREATE VIRTUAL TABLE ft USING fts5(a); INSERT INTO ft VALUES('abc'); @@ -158,6 +161,7 @@ QUERY (nCycle=nnn) #------------------------------------------------------------------------- reset_db +sqlite3_db_config db STMT_SCANSTATS 1 do_execsql_test 3.0 { CREATE TABLE x1(a, b); CREATE TABLE x2(c, d); @@ -173,11 +177,13 @@ do_graph_test 2.1 { QUERY (nCycle=nnn) --SCAN x1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON x2(c, d) (nCycle=nnn) +--BLOOM FILTER ON x2 (c=?) --SEARCH x2 USING AUTOMATIC COVERING INDEX (c=?) (nCycle=nnn) } #------------------------------------------------------------------------- reset_db +sqlite3_db_config db STMT_SCANSTATS 1 do_execsql_test 4.0 { CREATE TABLE rt1 (id INTEGER PRIMARY KEY, x1, x2); CREATE TABLE rt2 (id, x1, x2); @@ -189,6 +195,7 @@ do_graph_test 4.1 { QUERY (nCycle=nnn) --SCAN rt1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON rt2(x1, id, x2) (nCycle=nnn) +--BLOOM FILTER ON rt2 (x1=?) --SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) } @@ -198,6 +205,7 @@ do_graph_test 4.2 { QUERY (nCycle=nnn) --SCAN rt1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON rt2(x1, id) (nCycle=nnn) +--BLOOM FILTER ON rt2 (x1=?) --SEARCH rt2 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) } @@ -215,6 +223,7 @@ do_graph_test 4.4 { QUERY (nCycle=nnn) --SCAN rt1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON rt2(x1, id) WHERE (nCycle=nnn) +--BLOOM FILTER ON rt2 (x1=?) --SEARCH rt2 USING AUTOMATIC PARTIAL COVERING INDEX (x1=?) (nCycle=nnn) } @@ -229,6 +238,7 @@ QUERY (nCycle=nnn) ----USE TEMP B-TREE FOR GROUP BY --SCAN rt1 (nCycle=nnn) --CREATE AUTOMATIC INDEX ON v1(x1, cnt) (nCycle=nnn) +--BLOOM FILTER ON v1 (x1=?) --SEARCH v1 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) } From 06382de2a31eefce2f3cb178446d13230b0ab9fc Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Feb 2023 20:04:01 +0000 Subject: [PATCH 038/341] Change the name of SQLITE_DBCONFIG_STMT_SCANSTATS to SQLITE_DBCONFIG_STMT_SCANSTATUS. FossilOrigin-Name: a63e4a150b505fc309fac847131009ee9965eb1b798ebcb202ec8b52f9189240 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/main.c | 2 +- src/shell.c.in | 4 ++-- src/sqlite.h.in | 10 +++++----- src/sqliteInt.h | 4 ++-- src/test1.c | 2 +- test/scanstatus.test | 6 +++--- test/scanstatus2.test | 8 ++++---- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index c289db4cb6..61abcb8b6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\ssqlite3_db_config()\soption\s-\sSQLITE_DBCONFIG_STMT_SCANSTATS\s-\sfor\senabling\sand\sdisabling\sthe\scollection\sof\ssqlite3_stmt_scanstats()\sstatistics\sin\sSQLITE_ENABLE_STMT_SCANSTATUS\sbuilds.\sCollection\sof\sstatistics\sis\sdisabled\sby\sdefault. -D 2023-02-28T19:39:59.514 +C Change\sthe\sname\sof\sSQLITE_DBCONFIG_STMT_SCANSTATS\sto\sSQLITE_DBCONFIG_STMT_SCANSTATUS. +D 2023-02-28T20:04:01.467 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -586,7 +586,7 @@ F src/insert.c 7940fce7d4aa855606432d82d20694e17c18a03956f5f5776f2404e2df7c18a8 F src/json.c c85ed6fce06f43d414b0d7fff64749d43a0dbd1067123ee407bd3a0752454161 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c b04eb648cedc45efe4298e1ef439ac4f0096ae27b5f01accb0a1f49d57789128 -F src/main.c 834e257234fdf9e74c204c1a130920941618e48b2065747162862773ba5416b5 +F src/main.c e5f5323a33c51403bbe83ccb9f3e40aa94f95fd65dbeb5992a12435e0f6c22f9 F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -624,16 +624,16 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 230dc0601f55ae4909b5f88bc002bfa1f1fb331e51f2c6d670d3effc2ced365e -F src/shell.c.in a319e2a6cdb166d65a91ad0e69a2e9dea6afd066acbf9adce883a6762b43ec49 -F src/sqlite.h.in 7a2fed8c11df276bc35bc9ac47981cb0a7f9db39b85761a7dd50028fe77d28ce +F src/shell.c.in af928afa0376b2e2c962be9a0a3ab716b3983b1adfae10e248ffd40ede03b049 +F src/sqlite.h.in dd0f802db5ce8b2e384cf1e04debe175b38ab5ba2c8e09f00672e8b79a1bb134 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h e40980dd50f31aa57bed8dbb32c8fecf64b3ad9bc235a3abb24a3b95734cafeb +F src/sqliteInt.h 87bcbb5e08d454e4ee615079fe0de311e2d62bea776efb90f366bf876d42d9ad F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 8522a04fb9c84faa1d80354430ae0ee9349727a3a4b32e3cfe39b9be8324cabd -F src/test1.c 5ca9cb48a5dfef5980417bf5f76594071ae94795cbb8d065f8a1290c3b308ec6 +F src/test1.c 74d87bbc29954c239fdcd7c0c3dd472c0eb9761c7289b85389f322233932780a F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -1436,8 +1436,8 @@ F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 -F test/scanstatus.test 177fe0fea60898b2190328fbe358bafa2637e81856f90fb527a792efaf249c14 -F test/scanstatus2.test 7c367a5c818bad1936f291d33f688a3daa42279d4e989d920660c7002d67d3a2 +F test/scanstatus.test 2adde2d8fb0fbae73e790afa724f15ea4c3d6d0809e5de4d00407c5c14836401 +F test/scanstatus2.test b77de449be6ffd7a913a06dd84276dc6eb3517d479126e00ac51cd409c0a0af5 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5916705c731604d2e6b51a307cc8d7b67f4c102062bfdfcbc716a2916e0b0d86 -R 62c12b75bd9869fbb782d0697362eb0c +P 0f5579bef27b84ee855065cfe87703c51e1f9773906a9e0d4e4dafc90bd0e553 +R 9421f5c28d2a68696ef4d0c2d913d8d5 U dan -Z 823edb92d517eed03d3961ba1e99b0c0 +Z d689787b0a2f50878869838fedf1f287 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b5a7d79f5a..d100625216 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f5579bef27b84ee855065cfe87703c51e1f9773906a9e0d4e4dafc90bd0e553 \ No newline at end of file +a63e4a150b505fc309fac847131009ee9965eb1b798ebcb202ec8b52f9189240 \ No newline at end of file diff --git a/src/main.c b/src/main.c index a0e5e65852..e79d676139 100644 --- a/src/main.c +++ b/src/main.c @@ -970,7 +970,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, - { SQLITE_DBCONFIG_STMT_SCANSTATS, SQLITE_StmtScanStats }, + { SQLITE_DBCONFIG_STMT_SCANSTATUS, SQLITE_StmtScanStatus }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ diff --git a/src/shell.c.in b/src/shell.c.in index a983ec0c0e..0a54a3c5a5 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5421,7 +5421,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_set_authorizer(p->db, safeModeAuth, p); } sqlite3_db_config( - p->db, SQLITE_DBCONFIG_STMT_SCANSTATS, p->scanstatsOn, (int*)0 + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 ); } } @@ -9775,7 +9775,7 @@ static int do_meta_command(char *zLine, ShellState *p){ p->scanstatsOn = (u8)booleanValue(azArg[1]); } sqlite3_db_config( - p->db, SQLITE_DBCONFIG_STMT_SCANSTATS, p->scanstatsOn, (int*)0 + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 ); #ifndef SQLITE_ENABLE_STMT_SCANSTATUS raw_printf(stderr, "Warning: .scanstats not available in this build.\n"); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0a31ba033d..03e4862cc8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2457,10 +2457,10 @@ struct sqlite3_mem_methods { ** either generated columns or decending indexes. **
-** [[SQLITE_DBCONFIG_STMT_SCANSTATS]] -**
SQLITE_DBCONFIG_STMT_SCANSTATS -**
The SQLITE_DBCONFIG_STMT_SCANSTATS option is only useful in -** SQLITE_ENABLE_STMT_SCANSTATS builds. In this case, it sets or clears +** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] +**
SQLITE_DBCONFIG_STMT_SCANSTATUS +**
The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears ** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() ** statistics. For statistics to be collected, the flag must be set on ** the database handle both when the SQL statement is prepared and when it @@ -2486,7 +2486,7 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_STMT_SCANSTATS 1018 /* int int* */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ #define SQLITE_DBCONFIG_MAX 1018 /* Largest DBCONFIG */ /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 21ad2fa7e1..e97fed25f1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1757,7 +1757,7 @@ struct sqlite3 { #define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ /* result set is empty */ #define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ -#define SQLITE_StmtScanStats 0x00000400 /* Enable stmt_scanstats() counters */ +#define SQLITE_StmtScanStatus 0x00000400 /* Enable stmt_scanstats() counters */ #define SQLITE_NoCkptOnClose 0x00000800 /* No checkpoint on close()/DETACH */ #define SQLITE_ReverseOrder 0x00001000 /* Reverse unordered SELECTs */ #define SQLITE_RecTriggers 0x00002000 /* Enable recursive triggers */ @@ -5583,7 +5583,7 @@ sqlite3_uint64 sqlite3Hwtime(void); #endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS -# define IS_STMT_SCANSTATUS(db) (db->flags & SQLITE_StmtScanStats) +# define IS_STMT_SCANSTATUS(db) (db->flags & SQLITE_StmtScanStatus) #else # define IS_STMT_SCANSTATUS(db) 0 #endif diff --git a/src/test1.c b/src/test1.c index a765a66fa6..a4cfd28ab6 100644 --- a/src/test1.c +++ b/src/test1.c @@ -8313,7 +8313,7 @@ static int SQLITE_TCLAPI test_sqlite3_db_config( { "DQS_DML", SQLITE_DBCONFIG_DQS_DML }, { "DQS_DDL", SQLITE_DBCONFIG_DQS_DDL }, { "LEGACY_FILE_FORMAT", SQLITE_DBCONFIG_LEGACY_FILE_FORMAT }, - { "STMT_SCANSTATS", SQLITE_DBCONFIG_STMT_SCANSTATS }, + { "STMT_SCANSTATUS", SQLITE_DBCONFIG_STMT_SCANSTATUS }, }; int i; int v = 0; diff --git a/test/scanstatus.test b/test/scanstatus.test index 1eab9a26a4..ab87ad944a 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -48,7 +48,7 @@ proc do_scanstatus_test {tn res} { do_execsql_test 1.0a { SELECT count(*) FROM t1, t2; } 6 do_scanstatus_test 1.0b { } -sqlite3_db_config db STMT_SCANSTATS 1 +sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 1.1 { SELECT count(*) FROM t1, t2; } 6 do_scanstatus_test 1.2 { @@ -99,7 +99,7 @@ do_scanstatus_test 1.10 { # Try a few different types of scans. # reset_db -sqlite3_db_config db STMT_SCANSTATS 1 +sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 2.1 { CREATE TABLE x1(i INTEGER PRIMARY KEY, j); INSERT INTO x1 VALUES(1, 'one'); @@ -283,7 +283,7 @@ do_scanstatus_test 4.2.2 { # Further tests of different scan types. # reset_db -sqlite3_db_config db STMT_SCANSTATS 1 +sqlite3_db_config db STMT_SCANSTATUS 1 proc tochar {i} { set alphabet {a b c d e f g h i j k l m n o p q r s t u v w x y z} return [lindex $alphabet [expr $i % [llength $alphabet]]] diff --git a/test/scanstatus2.test b/test/scanstatus2.test index 7b8453f94b..042128ae14 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -19,7 +19,7 @@ ifcapable !scanstatus { return } -sqlite3_db_config db STMT_SCANSTATS 1 +sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 1.0 { CREATE TABLE t1(a, b); @@ -143,7 +143,7 @@ QUERY (nCycle=nnn) #------------------------------------------------------------------------- ifcapable fts5 { reset_db - sqlite3_db_config db STMT_SCANSTATS 1 + sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 2.0 { CREATE VIRTUAL TABLE ft USING fts5(a); INSERT INTO ft VALUES('abc'); @@ -161,7 +161,7 @@ QUERY (nCycle=nnn) #------------------------------------------------------------------------- reset_db -sqlite3_db_config db STMT_SCANSTATS 1 +sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 3.0 { CREATE TABLE x1(a, b); CREATE TABLE x2(c, d); @@ -183,7 +183,7 @@ QUERY (nCycle=nnn) #------------------------------------------------------------------------- reset_db -sqlite3_db_config db STMT_SCANSTATS 1 +sqlite3_db_config db STMT_SCANSTATUS 1 do_execsql_test 4.0 { CREATE TABLE rt1 (id INTEGER PRIMARY KEY, x1, x2); CREATE TABLE rt2 (id, x1, x2); From 0bdee38969498fb2e86e3e07b773a6f41fc2e3f5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Feb 2023 20:06:30 +0000 Subject: [PATCH 039/341] Updates to speedtest1.c and the speed-check.sh test script so that they work with the new SQLITE_DBCONFIG_STMT_SCANSTATUS control. FossilOrigin-Name: bd02df052e1ef78b5335915a38f3c5e13c3c04ab82fd251aeb42a440d1d39257 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/speedtest1.c | 7 +++++++ tool/speed-check.sh | 3 +++ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 61abcb8b6e..7013d3fa1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sSQLITE_DBCONFIG_STMT_SCANSTATS\sto\sSQLITE_DBCONFIG_STMT_SCANSTATUS. -D 2023-02-28T20:04:01.467 +C Updates\sto\sspeedtest1.c\sand\sthe\sspeed-check.sh\stest\sscript\sso\sthat\sthey\swork\nwith\sthe\snew\sSQLITE_DBCONFIG_STMT_SCANSTATUS\scontrol. +D 2023-02-28T20:06:30.036 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1521,7 +1521,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest1.c 645dbf022337116fcef60ac2579e8d25c94e4475109e2c6f48005f302efe7b09 +F test/speedtest1.c 4a883fe277c289161f84f8c805a38f78bfbd9515eb417071f9bce817c2ca8756 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -2005,7 +2005,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c F tool/showwal.c 4699048f68b6dd7b451011abfff404b8890d5a0b7dab78d2ad50d018116239d5 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl 1b5be34c6223cb1af06da2a10fb77863eb869b1962d055820b0a11cf2336ab45 -F tool/speed-check.sh 9b27e158330a6587e92214b2344cc6fadde514996c0648fc0de7955ef7a79d77 +F tool/speed-check.sh c24c30cddd0ecb6d1d0775411d22f7619f55fa696a305487645e27b1b8fc05a2 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0f5579bef27b84ee855065cfe87703c51e1f9773906a9e0d4e4dafc90bd0e553 -R 9421f5c28d2a68696ef4d0c2d913d8d5 -U dan -Z d689787b0a2f50878869838fedf1f287 +P a63e4a150b505fc309fac847131009ee9965eb1b798ebcb202ec8b52f9189240 +R ea36b32b35459259bf3c7ddf5946c632 +U drh +Z 7b95aa43eab112d9f61ce625acf1b329 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d100625216..2c8dcba418 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a63e4a150b505fc309fac847131009ee9965eb1b798ebcb202ec8b52f9189240 \ No newline at end of file +bd02df052e1ef78b5335915a38f3c5e13c3c04ab82fd251aeb42a440d1d39257 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 12c8d69b5b..1dd3bd5fb3 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -39,6 +39,7 @@ static const char zHelp[] = " --size N Relative test size. Default=100\n" " --strict Use STRICT table where appropriate\n" " --stats Show statistics at the end\n" + " --stmtscanstatus Activate SQLITE_DBCONFIG_STMT_SCANSTATUS\n" " --temp N N from 0 to 9. 0: no temp table. 9: all temp tables\n" " --testset T Run test-set T (main, cte, rtree, orm, fp, debug)\n" " --trace Turn on SQL tracing\n" @@ -100,6 +101,7 @@ static struct Global { int nRepeat; /* Repeat selects this many times */ int doCheckpoint; /* Run PRAGMA wal_checkpoint after each trans */ int nReserve; /* Reserve bytes */ + int stmtScanStatus; /* True to activate Stmt ScanStatus reporting */ int doBigTransactions; /* Enable transactions on tests 410 and 510 */ const char *zWR; /* Might be WITHOUT ROWID */ const char *zNN; /* Might be NOT NULL */ @@ -2391,6 +2393,8 @@ int main(int argc, char **argv){ }else if( strcmp(z,"reserve")==0 ){ ARGC_VALUE_CHECK(1); g.nReserve = atoi(argv[++i]); + }else if( strcmp(z,"stmtscanstatus")==0 ){ + g.stmtScanStatus = 1; }else if( strcmp(z,"without-rowid")==0 ){ if( strstr(g.zWR,"WITHOUT")!=0 ){ /* no-op */ @@ -2474,6 +2478,9 @@ int main(int argc, char **argv){ if( g.nReserve>0 ){ sqlite3_file_control(g.db, 0, SQLITE_FCNTL_RESERVE_BYTES, &g.nReserve); } + if( g.stmtScanStatus ){ + sqlite3_db_config(g.db, SQLITE_DBCONFIG_STMT_SCANSTATUS, 1, 0); + } /* Set database connection options */ sqlite3_create_function(g.db, "random", 0, SQLITE_UTF8, 0, randomFunc, 0, 0); diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 4e070565e0..06cbd35745 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -158,6 +158,9 @@ while test "$1" != ""; do --fp) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp" ;; + --stmtscanstatus) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --stmtscanstatus" + ;; -*) CC_OPTS="$CC_OPTS $1" ;; From f1f4f090c77dc0d36829999a2b57167851a5df2d Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Feb 2023 21:23:46 +0000 Subject: [PATCH 040/341] Activate SQLITE_DBCONFIG_STMT_SCANSTATUS in fuzzcheck. FossilOrigin-Name: 4fe1419ac3161ea8735241b04913593170c636cf3e1583756fe94edd396cd38b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fuzzcheck.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7013d3fa1e..d19318bff0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sspeedtest1.c\sand\sthe\sspeed-check.sh\stest\sscript\sso\sthat\sthey\swork\nwith\sthe\snew\sSQLITE_DBCONFIG_STMT_SCANSTATUS\scontrol. -D 2023-02-28T20:06:30.036 +C Activate\sSQLITE_DBCONFIG_STMT_SCANSTATUS\sin\sfuzzcheck. +D 2023-02-28T21:23:46.457 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1140,7 +1140,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 0b8543d29c6f529cb73937607c8649f685510838e40c6287f06df083433c93b6 +F test/fuzzcheck.c 90d3435c5359029d79b0945110e9c83a529469c03e4380338578c47c5a9ea296 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a63e4a150b505fc309fac847131009ee9965eb1b798ebcb202ec8b52f9189240 -R ea36b32b35459259bf3c7ddf5946c632 +P bd02df052e1ef78b5335915a38f3c5e13c3c04ab82fd251aeb42a440d1d39257 +R 8576dd179593a779804154045c49523d U drh -Z 7b95aa43eab112d9f61ce625acf1b329 +Z 6457a01da04b4d92486c2ec77b55c596 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c8dcba418..73bfb25b5b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bd02df052e1ef78b5335915a38f3c5e13c3c04ab82fd251aeb42a440d1d39257 \ No newline at end of file +4fe1419ac3161ea8735241b04913593170c636cf3e1583756fe94edd396cd38b \ No newline at end of file diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index ee7749d969..65dd2fa722 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1181,6 +1181,7 @@ int runCombinedDbSqlInput( sqlite3_free(aDb); return 1; } + sqlite3_db_config(cx.db, SQLITE_DBCONFIG_STMT_SCANSTATUS, 1, 0); if( bVdbeDebug ){ sqlite3_exec(cx.db, "PRAGMA vdbe_debug=ON", 0, 0, 0); } From 46960e4677ec68a7b4139a9310d5d728bd25ef03 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 1 Mar 2023 13:54:07 +0000 Subject: [PATCH 041/341] Do not attempt to apply the count-of-view optimization to a CTE. dbsqlfuzz ef8623915d843b150c159166ee4548c78cc6895a FossilOrigin-Name: abc3a383636c0346053b5d09d96585f56c64cacb5751673ea3bf339e4955d1cd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 3 ++- test/countofview.test | 10 +++++++++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d19318bff0..41e2b25304 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Activate\sSQLITE_DBCONFIG_STMT_SCANSTATUS\sin\sfuzzcheck. -D 2023-02-28T21:23:46.457 +C Do\snot\sattempt\sto\sapply\sthe\scount-of-view\soptimization\sto\sa\sCTE.\ndbsqlfuzz\sef8623915d843b150c159166ee4548c78cc6895a +D 2023-03-01T13:54:07.942 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -623,7 +623,7 @@ F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 230dc0601f55ae4909b5f88bc002bfa1f1fb331e51f2c6d670d3effc2ced365e +F src/select.c 1d9a7c3b60e1e20bd4b5b6c0df0b2fe0958e04fa01b3dedd87c10468bada62e0 F src/shell.c.in af928afa0376b2e2c962be9a0a3ab716b3983b1adfae10e248ffd40ede03b049 F src/sqlite.h.in dd0f802db5ce8b2e384cf1e04debe175b38ab5ba2c8e09f00672e8b79a1bb134 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -893,7 +893,7 @@ F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 -F test/countofview.test e17d6e6688cf74f22783c9ec6e788c0790ee4fbbaee713affd00b1ac0bb39b86 +F test/countofview.test 5773d192f67bbf271f53ebb1f3292f435c9c7ae91ee0be83f81591c19a983604 F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bd02df052e1ef78b5335915a38f3c5e13c3c04ab82fd251aeb42a440d1d39257 -R 8576dd179593a779804154045c49523d +P 4fe1419ac3161ea8735241b04913593170c636cf3e1583756fe94edd396cd38b +R 8eb90f4d18319eb19774d62b004fc1bc U drh -Z 6457a01da04b4d92486c2ec77b55c596 +Z 423142ebb5d4ee37849ae60a48c54522 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 73bfb25b5b..a6dc47513d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4fe1419ac3161ea8735241b04913593170c636cf3e1583756fe94edd396cd38b \ No newline at end of file +abc3a383636c0346053b5d09d96585f56c64cacb5751673ea3bf339e4955d1cd \ No newline at end of file diff --git a/src/select.c b/src/select.c index a882b973ee..82d195de3e 100644 --- a/src/select.c +++ b/src/select.c @@ -6876,7 +6876,8 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( ExprHasProperty(pExpr, EP_WinFunc) ) return 0;/* Not a window function */ pSub = p->pSrc->a[0].pSelect; if( pSub==0 ) return 0; /* The FROM is a subquery */ - if( pSub->pPrior==0 ) return 0; /* Must be a compound ry */ + if( pSub->pPrior==0 ) return 0; /* Must be a compound */ + if( pSub->selFlags & SF_CopyCte ) return 0; /* Not a CTE */ do{ if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ if( pSub->pWhere ) return 0; /* No WHERE clause */ diff --git a/test/countofview.test b/test/countofview.test index 0ee511ff34..c7783e6dbd 100644 --- a/test/countofview.test +++ b/test/countofview.test @@ -51,6 +51,14 @@ do_execsql_test 2.1 { SELECT count(*) FROM v1 GROUP BY y; } {3 3} - +# 2023-03-01 dbsqlfuzz ef8623915d843b150c159166ee4548c78cc6895a +# count-of-view should not apply to CTEs. +# +proc progress_stop args {return 1} +db progress 1000 progress_stop +do_catchsql_test 3.1 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c) + SELECT count(*) FROM c; +} {1 interrupted} finish_test From fe373482552e7b254cf6d3f94ca734014a58250b Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 1 Mar 2023 15:21:53 +0000 Subject: [PATCH 042/341] Follow-up to [bbaf1f2eb1e1637b]: Make sure subtypes do not cross a subquery boundary even if the function that returned the value with a subtype is buried down inside a larger expression. This fixes a problem identified by [forum:/forumpost/37dd14a538|forum post 37dd14a538]. FossilOrigin-Name: e72661eb680ea707a839cb3d5cf6c7ef03706e7b40af1b84760147e59cd61a50 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 11 +++++++---- test/subtype1.test | 20 +++++++++++++++++++- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 41e2b25304..2a2c8a7fdb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sattempt\sto\sapply\sthe\scount-of-view\soptimization\sto\sa\sCTE.\ndbsqlfuzz\sef8623915d843b150c159166ee4548c78cc6895a -D 2023-03-01T13:54:07.942 +C Follow-up\sto\s[bbaf1f2eb1e1637b]:\sMake\ssure\ssubtypes\sdo\snot\scross\sa\ssubquery\nboundary\seven\sif\sthe\sfunction\sthat\sreturned\sthe\svalue\swith\sa\ssubtype\sis\nburied\sdown\sinside\sa\slarger\sexpression.\s\sThis\sfixes\sa\sproblem\sidentified\nby\s[forum:/forumpost/37dd14a538|forum\spost\s37dd14a538]. +D 2023-03-01T15:21:53.838 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -573,7 +573,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 2e5e67e800c9416996df52409746a2f3c7dadf5d38b5a367ce379b239e3f40c8 +F src/expr.c e3520c28b322d0e06e883c91de15322ddd06a98d5b4564c3273c99da9391d1b8 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -1541,7 +1541,7 @@ F test/subquery.test 3a1a5b600b8d4f504d2a2c61f33db820983dba94a0ef3e4aedca8f0165e F test/subquery2.test 90cf944b9de8204569cf656028391e4af1ccc8c0cc02d4ef38ee3be8de1ffb12 F test/subselect.test 0966aa8e720224dbd6a5e769a3ec2a723e332303 F test/substr.test a673e3763e247e9b5e497a6cacbaf3da2bd8ec8921c0677145c109f2e633f36b -F test/subtype1.test 45c85632abd38f7ea9b33f17448d966d67550f552e0822bab74576814d0d1718 +F test/subtype1.test 7a9c55ed84d4ce551203d18046f925e293d75f69da81bff71aaf2696e4a2a748 F test/superlock.test ec94f0556b6488d97f71c79f9061ae08d9ab8f12 F test/swarmvtab.test 250231404fcac88f61a6c147bb0e3a118ed879278cd3ccb0ae2d3a729e1e8e26 F test/swarmvtab2.test c948cb2fdfc5b01d85e8f6d6504854202dc1a0782ab2a0ed61538f27cbd0aa5c @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4fe1419ac3161ea8735241b04913593170c636cf3e1583756fe94edd396cd38b -R 8eb90f4d18319eb19774d62b004fc1bc +P abc3a383636c0346053b5d09d96585f56c64cacb5751673ea3bf339e4955d1cd +R df8f421f27eca726fd01bc8901795f0a U drh -Z 423142ebb5d4ee37849ae60a48c54522 +Z d4a7374d77ecf078e57dc52b4f6eecbc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a6dc47513d..730ee85f6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -abc3a383636c0346053b5d09d96585f56c64cacb5751673ea3bf339e4955d1cd \ No newline at end of file +e72661eb680ea707a839cb3d5cf6c7ef03706e7b40af1b84760147e59cd61a50 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index ea3c901671..763e961b6e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4719,10 +4719,13 @@ expr_code_doover: return target; } case TK_COLLATE: { - if( !ExprHasProperty(pExpr, EP_Collate) - && ALWAYS(pExpr->pLeft) - && pExpr->pLeft->op==TK_FUNCTION - ){ + if( !ExprHasProperty(pExpr, EP_Collate) ){ + /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called + ** "SOFT-COLLATE" that is added to constraints that are pushed down + ** from outer queries into sub-queries by the push-down optimization. + ** Clear subtypes as subtypes may not cross a subquery boundary. + */ + assert( pExpr->pLeft ); inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); if( inReg!=target ){ sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); diff --git a/test/subtype1.test b/test/subtype1.test index d15f737bc2..4d8e68e6af 100644 --- a/test/subtype1.test +++ b/test/subtype1.test @@ -53,7 +53,25 @@ do_execsql_test subtype1-231 { WITH t4(a) AS NOT MATERIALIZED (SELECT json(1)) SELECT subtype(a) FROM t4; } {0} - +# 2023-03-01 +# https://sqlite.org/forum/forumpost/37dd14a538 +# +# Additional tests to show that subtypes do not traverse subquery boundaries. +# +do_execsql_test subtype1-300 { + CREATE TABLE t0(c0); + INSERT INTO t0 VALUES ('1'); + CREATE VIEW v0(c0) AS SELECT CASE WHEN 1 THEN json_patch('1', '1') END + FROM t0 GROUP BY t0.c0; + SELECT * FROM v0 WHERE json_quote(v0.c0) != '1'; +} {1} +do_execsql_test subtype1-310 { + SELECT *, json_quote(y) FROM (SELECT +json('1') AS y); +} {1 {"1"}} +do_execsql_test subtype1-320 { + SELECT *, json_quote(y) FROM (SELECT +json('1') AS y) + WHERE json_quote(y)='"1"'; +} {1 {"1"}} finish_test From bf901148678bef77f5910c7046053021b611b40b Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 1 Mar 2023 20:23:46 +0000 Subject: [PATCH 043/341] When flattening a view that is the right operand of a LEFT JOIN, using the optimization of check-in [41c27bc0ff1d3135], always insert the TK_IF_NULL_ROW expression nodes, even for TK_COLUMN expressions, as the TK_COLUMN might be a column from an outer query and hence still need to be NULLed out. This fixes the problem described by [forum:/forumpost/26387ea7ef|forum post 26387ea7ef]. FossilOrigin-Name: 198b3e33dcfd74c7ba6abcf789ee81dfed464a50ebf15c8edeff349d36789fca --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 2 +- test/join2.test | 31 +++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2a2c8a7fdb..7b4e3ac074 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Follow-up\sto\s[bbaf1f2eb1e1637b]:\sMake\ssure\ssubtypes\sdo\snot\scross\sa\ssubquery\nboundary\seven\sif\sthe\sfunction\sthat\sreturned\sthe\svalue\swith\sa\ssubtype\sis\nburied\sdown\sinside\sa\slarger\sexpression.\s\sThis\sfixes\sa\sproblem\sidentified\nby\s[forum:/forumpost/37dd14a538|forum\spost\s37dd14a538]. -D 2023-03-01T15:21:53.838 +C When\sflattening\sa\sview\sthat\sis\sthe\sright\soperand\sof\sa\sLEFT\sJOIN,\susing\nthe\soptimization\sof\scheck-in\s[41c27bc0ff1d3135],\salways\sinsert\sthe\nTK_IF_NULL_ROW\sexpression\snodes,\seven\sfor\sTK_COLUMN\sexpressions,\sas\nthe\sTK_COLUMN\smight\sbe\sa\scolumn\sfrom\san\souter\squery\sand\shence\sstill\sneed\nto\sbe\sNULLed\sout.\sThis\sfixes\sthe\sproblem\sdescribed\sby\n[forum:/forumpost/26387ea7ef|forum\spost\s26387ea7ef]. +D 2023-03-01T20:23:46.877 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -623,7 +623,7 @@ F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 1d9a7c3b60e1e20bd4b5b6c0df0b2fe0958e04fa01b3dedd87c10468bada62e0 +F src/select.c f6224c55bc0de16b8559e80015a26f1eced7809a4a0e774efabacca98b3150f6 F src/shell.c.in af928afa0376b2e2c962be9a0a3ab716b3983b1adfae10e248ffd40ede03b049 F src/sqlite.h.in dd0f802db5ce8b2e384cf1e04debe175b38ab5ba2c8e09f00672e8b79a1bb134 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1217,7 +1217,7 @@ F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 F test/join.test ed1daf99958fed1b9f017e56bae2bb6b49339a1ec0b70b9e8f7259960c6bf387 -F test/join2.test 466b07233820f5deee66a6c3bf6e4500c8bbf7b83649e67606f5f649c07928c0 +F test/join2.test 88f4527101710806674d49257439f198fa147a7fcec11afedf51cf4894dc8877 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 91f1f4c7d81fd87b58e9ba7cf4a2b5d39e3583b4f8e498a162722a60259c5208 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P abc3a383636c0346053b5d09d96585f56c64cacb5751673ea3bf339e4955d1cd -R df8f421f27eca726fd01bc8901795f0a +P e72661eb680ea707a839cb3d5cf6c7ef03706e7b40af1b84760147e59cd61a50 +R 5a5a4d6f731a2a4b73a8f17b01f36db2 U drh -Z d4a7374d77ecf078e57dc52b4f6eecbc +Z abb70d1f8b32da91985a98db916a8616 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 730ee85f6b..aed665b910 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e72661eb680ea707a839cb3d5cf6c7ef03706e7b40af1b84760147e59cd61a50 \ No newline at end of file +198b3e33dcfd74c7ba6abcf789ee81dfed464a50ebf15c8edeff349d36789fca \ No newline at end of file diff --git a/src/select.c b/src/select.c index 82d195de3e..359dc8cf30 100644 --- a/src/select.c +++ b/src/select.c @@ -3861,7 +3861,7 @@ static Expr *substExpr( sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ sqlite3 *db = pSubst->pParse->db; - if( pSubst->isOuterJoin && pCopy->op!=TK_COLUMN ){ + if( pSubst->isOuterJoin ){ memset(&ifNullRow, 0, sizeof(ifNullRow)); ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; diff --git a/test/join2.test b/test/join2.test index e549c49343..b3fd265353 100644 --- a/test/join2.test +++ b/test/join2.test @@ -354,5 +354,36 @@ do_execsql_test 9.11 { SELECT ccc, ccc IS NULL AS ddd FROM t1 LEFT JOIN v2; } {{} 1} +# 2023-03-01 https://sqlite.org/forum/forumpost/26387ea7ef +# When flattening a VIEW which is the RHS of a LEFT JOIN, always put +# an TK_IF_NULL_ROW operator on all accesses, even TK_COLUMN nodes, since +# the TK_COLUMN might reference an outer subquery. +# +reset_db +db null NULL +do_execsql_test 10.1 { + CREATE TABLE t1 (x INTEGER); + INSERT INTO t1 VALUES(1); -- Some true value + CREATE TABLE t2 (z TEXT); + INSERT INTO t2 VALUES('some value'); + CREATE TABLE t3(w TEXT); + INSERT INTO t3 VALUES('some other value'); +} +do_execsql_test 10.2 { + SELECT ( + SELECT 1 FROM t2 LEFT JOIN (SELECT x AS v FROM t3) ON 500=v WHERE (v OR FALSE) + ) FROM t1; +} NULL +do_execsql_test 10.3 { + SELECT ( + SELECT 1 FROM t2 LEFT JOIN (SELECT x AS v FROM t3) ON 500=v WHERE (v) + ) FROM t1; +} NULL +optimization_control db all 0 +do_execsql_test 10.4 { + SELECT ( + SELECT 1 FROM t2 LEFT JOIN (SELECT x AS v FROM t3) ON 500=v WHERE (v OR FALSE) + ) FROM t1; +} NULL finish_test From c3d84b4cbd911586dbb12b6d7cd57d9a3e8dccf5 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 2 Mar 2023 05:51:03 +0000 Subject: [PATCH 044/341] Rename some vars in the ext/wasm makefiles for consistency's sake. FossilOrigin-Name: c23589d92cd16b67266d97f4a3d8c0991864dbea30ec074173e1a67466532d21 --- ext/wasm/GNUmakefile | 8 ++++---- ext/wasm/fiddle.make | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index f2d6ae9f0b..e221bc67a9 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -604,7 +604,7 @@ endef # https://github.com/emscripten-core/emscripten/issues/14383 sqlite3.wasm := $(dir.dout)/sqlite3.wasm sqlite3-wasm.c := $(dir.api)/sqlite3-wasm.c -sqlite3-wasm.cs := $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) +sqlite3-wasm.cses := $(sqlite3-wasm.c) $(sqlite3_wasm_extra_init.c) # sqlite3-wasm.o vs sqlite3-wasm.c: building against the latter # (predictably) results in a slightly faster binary. We're close # enough to the target speed requirements that the 500ms makes a @@ -679,12 +679,12 @@ $$(eval $$(call call-make-pre-post,sqlite3,$(1))) emcc.flags.sqlite3.$(1) := $(6) $$(eval $$(call C-PP.FILTER, $$(sqlite3-api.js.in), $(3), $(5))) $(4): $(3) -$(4): $(3) $$(MAKEFILE) $$(sqlite3-wasm.cs) $$(EXPORTED_FUNCTIONS.api) $$(pre-post-sqlite3.deps.$(1)) +$(4): $(3) $$(MAKEFILE) $$(sqlite3-wasm.cses) $$(EXPORTED_FUNCTIONS.api) $$(pre-post-sqlite3.deps.$(1)) @echo "Building $$@ ..." $$(emcc.bin) -o $$@ $$(emcc_opt_full) $$(emcc.flags) \ $$(emcc.jsflags) \ $$(pre-post-sqlite3.flags.$(1)) $$(emcc.flags.sqlite3.$(1)) \ - $$(cflags.common) $$(SQLITE_OPT) $$(sqlite3-wasm.cs) + $$(cflags.common) $$(SQLITE_OPT) $$(sqlite3-wasm.cses) @$$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(2)) @if [ bundler-friendly = $(1) ]; then \ echo "Patching $(3) for sqlite3.wasm..."; \ @@ -835,7 +835,7 @@ $(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) speedtest1.js := $(dir.dout)/speedtest1.js speedtest1.wasm := $(dir.dout)/speedtest1.wasm cflags.speedtest1 := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM -speedtest1.cses := $(speedtest1.c) $(sqlite3-wasm.cs) +speedtest1.cses := $(speedtest1.c) $(sqlite3-wasm.cses) $(eval $(call call-make-pre-post,speedtest1,vanilla)) $(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ $(pre-post-speedtest1.deps.vanilla) \ diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 86984c9664..475fe6dc33 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -52,7 +52,7 @@ $(EXPORTED_FUNCTIONS.fiddle): $(fiddle.EXPORTED_FUNCTIONS.in) $(MAKEFILE.fiddle) fiddle-module.js := $(dir.fiddle)/fiddle-module.js fiddle-module.wasm := $(subst .js,.wasm,$(fiddle-module.js)) -fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.cs) +fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.cses) fiddle.SOAP.js := $(dir.fiddle)/$(notdir $(SOAP.js)) $(fiddle.SOAP.js): $(SOAP.js) diff --git a/manifest b/manifest index 7b4e3ac074..276cc9cac4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sflattening\sa\sview\sthat\sis\sthe\sright\soperand\sof\sa\sLEFT\sJOIN,\susing\nthe\soptimization\sof\scheck-in\s[41c27bc0ff1d3135],\salways\sinsert\sthe\nTK_IF_NULL_ROW\sexpression\snodes,\seven\sfor\sTK_COLUMN\sexpressions,\sas\nthe\sTK_COLUMN\smight\sbe\sa\scolumn\sfrom\san\souter\squery\sand\shence\sstill\sneed\nto\sbe\sNULLed\sout.\sThis\sfixes\sthe\sproblem\sdescribed\sby\n[forum:/forumpost/26387ea7ef|forum\spost\s26387ea7ef]. -D 2023-03-01T20:23:46.877 +C Rename\ssome\svars\sin\sthe\sext/wasm\smakefiles\sfor\sconsistency's\ssake. +D 2023-03-02T05:51:03.913 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -468,7 +468,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 122607cd0255134ae88bfdf6424f299d955b74bc8a6c48160c54c4ecbc7797b3 +F ext/wasm/GNUmakefile d5f0eac7de53614381de422053e3fdb20d4315f7a821276359c26771ac0f8ea0 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -510,7 +510,7 @@ F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2 F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa F ext/wasm/dist.make f55f9c9e1980ea11a59964e59535c66175a17f004d1c2e274522c3366b3a084a F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle.make 71b686d1f3355c0ce65cd76fb231e209dc30fbd2a3b0fd635bf26df904bffce4 +F ext/wasm/fiddle.make 6f8bfd2e8c4f911b01a0edc26b4d7f6670253003a71b84d1f288482226dccbbd F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa061ce8776a6e75da8a10 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e72661eb680ea707a839cb3d5cf6c7ef03706e7b40af1b84760147e59cd61a50 -R 5a5a4d6f731a2a4b73a8f17b01f36db2 -U drh -Z abb70d1f8b32da91985a98db916a8616 +P 198b3e33dcfd74c7ba6abcf789ee81dfed464a50ebf15c8edeff349d36789fca +R d3512c3ac02aab42fe267e18e5b38131 +U stephan +Z 8800c626ace8a11c90d3b1467c2d3267 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aed665b910..9dcbdeda2f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -198b3e33dcfd74c7ba6abcf789ee81dfed464a50ebf15c8edeff349d36789fca \ No newline at end of file +c23589d92cd16b67266d97f4a3d8c0991864dbea30ec074173e1a67466532d21 \ No newline at end of file From c355f6c839f3df778eae3892a6bdd1b099566930 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 2 Mar 2023 06:58:55 +0000 Subject: [PATCH 045/341] Resolve a parallel build timing issue when building sqlite3.c/h from ext/wasm. For the time being, do not add sqlite3_wasm_extra_init.c to fiddle.wasm because it can cause duplicate definitions of extensions which are already built into the shell (a better resolution for this conflict is pending). No longer add sqlite3_wasm_extra_init.c to speedtest1.wasm because it's useless there. FossilOrigin-Name: 75fdd5b83b4c527d25649b0d08841e3dc7d4d8109c1c97b2195b303538ced73d --- ext/wasm/GNUmakefile | 21 ++++++++++++--------- ext/wasm/fiddle.make | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index e221bc67a9..13205a6220 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -171,8 +171,10 @@ SQLITE_OPT = \ -DSQLITE_USE_URI=1 \ -DSQLITE_WASM_ENABLE_C_TESTS -$(sqlite3.c) $(sqlite3.h): +.NOTPARALLEL: $(sqlite3.h) +$(sqlite3.h): $(MAKE) -C $(dir.top) sqlite3.c +$(sqlite3.c): $(sqlite3.h) .PHONY: clean distclean clean: @@ -212,9 +214,10 @@ endif # See example_extra_init.c for an example implementation. ######################################################################## sqlite3_wasm_extra_init.c ?= $(wildcard sqlite3_wasm_extra_init.c) +cflags.wasm_extra_init := ifneq (,$(sqlite3_wasm_extra_init.c)) $(info Enabling SQLITE_EXTRA_INIT via $(sqlite3_wasm_extra_init.c).) - SQLITE_OPT += -I. -DSQLITE_WASM_EXTRA_INIT + cflags.wasm_extra_init := -DSQLITE_WASM_EXTRA_INIT endif # bin.version-info = binary to output various sqlite3 version info for @@ -684,7 +687,7 @@ $(4): $(3) $$(MAKEFILE) $$(sqlite3-wasm.cses) $$(EXPORTED_FUNCTIONS.api) $$(pre- $$(emcc.bin) -o $$@ $$(emcc_opt_full) $$(emcc.flags) \ $$(emcc.jsflags) \ $$(pre-post-sqlite3.flags.$(1)) $$(emcc.flags.sqlite3.$(1)) \ - $$(cflags.common) $$(SQLITE_OPT) $$(sqlite3-wasm.cses) + $$(cflags.common) $$(SQLITE_OPT) $$(cflags.wasm_extra_init) $$(sqlite3-wasm.cses) @$$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(2)) @if [ bundler-friendly = $(1) ]; then \ echo "Patching $(3) for sqlite3.wasm..."; \ @@ -727,11 +730,11 @@ $(eval $(call SETUP_LIB_BUILD_MODE,bundler-friendly,1,\ # ######################################################################## ######################################################################## -# We have to ensure that we do not build both $(sqlite3*.*js) in -# parallel because both result in the creation of $(sqlite3.wasm). We -# have no way to build just the .mjs file without also building the -# .wasm file because the generated .mjs file has to include info about -# the imports needed by the wasm file, so they have to be built +# We have to ensure that we do not build $(sqlite3*.*js) in parallel +# because they all result in the creation of $(sqlite3.wasm). We have +# no way to build just a .[m]js file without also building the .wasm +# file because the generated .[m]js file has to include info about the +# imports needed by the wasm file, so they have to be built # together. i.e. we're building $(sqlite3.wasm) multiple times, but # that's unavoidable (and harmless, just a waste of build time). $(sqlite3.wasm): $(sqlite3.js) @@ -835,7 +838,7 @@ $(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) speedtest1.js := $(dir.dout)/speedtest1.js speedtest1.wasm := $(dir.dout)/speedtest1.wasm cflags.speedtest1 := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM -speedtest1.cses := $(speedtest1.c) $(sqlite3-wasm.cses) +speedtest1.cses := $(speedtest1.c) $(sqlite3-wasm.c) $(eval $(call call-make-pre-post,speedtest1,vanilla)) $(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ $(pre-post-speedtest1.deps.vanilla) \ diff --git a/ext/wasm/fiddle.make b/ext/wasm/fiddle.make index 475fe6dc33..cbe6ab3518 100644 --- a/ext/wasm/fiddle.make +++ b/ext/wasm/fiddle.make @@ -52,7 +52,7 @@ $(EXPORTED_FUNCTIONS.fiddle): $(fiddle.EXPORTED_FUNCTIONS.in) $(MAKEFILE.fiddle) fiddle-module.js := $(dir.fiddle)/fiddle-module.js fiddle-module.wasm := $(subst .js,.wasm,$(fiddle-module.js)) -fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.cses) +fiddle.cses := $(dir.top)/shell.c $(sqlite3-wasm.c) fiddle.SOAP.js := $(dir.fiddle)/$(notdir $(SOAP.js)) $(fiddle.SOAP.js): $(SOAP.js) diff --git a/manifest b/manifest index 276cc9cac4..408ede3bd6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\ssome\svars\sin\sthe\sext/wasm\smakefiles\sfor\sconsistency's\ssake. -D 2023-03-02T05:51:03.913 +C Resolve\sa\sparallel\sbuild\stiming\sissue\swhen\sbuilding\ssqlite3.c/h\sfrom\sext/wasm.\sFor\sthe\stime\sbeing,\sdo\snot\sadd\ssqlite3_wasm_extra_init.c\sto\sfiddle.wasm\sbecause\sit\scan\scause\sduplicate\sdefinitions\sof\sextensions\swhich\sare\salready\sbuilt\sinto\sthe\sshell\s(a\sbetter\sresolution\sfor\sthis\sconflict\sis\spending).\sNo\slonger\sadd\ssqlite3_wasm_extra_init.c\sto\sspeedtest1.wasm\sbecause\sit's\suseless\sthere. +D 2023-03-02T06:58:55.482 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -468,7 +468,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile d5f0eac7de53614381de422053e3fdb20d4315f7a821276359c26771ac0f8ea0 +F ext/wasm/GNUmakefile 08fb7a6892acfe8801b547920edf3b914c7045a6bc8bfd4b46c0f2d6c0b6ecbc F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -510,7 +510,7 @@ F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2 F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa F ext/wasm/dist.make f55f9c9e1980ea11a59964e59535c66175a17f004d1c2e274522c3366b3a084a F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f -F ext/wasm/fiddle.make 6f8bfd2e8c4f911b01a0edc26b4d7f6670253003a71b84d1f288482226dccbbd +F ext/wasm/fiddle.make dbe36b90b8907ae28ecb9c0e9fd8389dbdaecf117ea4fb2ea33864bdfa498a94 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa061ce8776a6e75da8a10 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 198b3e33dcfd74c7ba6abcf789ee81dfed464a50ebf15c8edeff349d36789fca -R d3512c3ac02aab42fe267e18e5b38131 +P c23589d92cd16b67266d97f4a3d8c0991864dbea30ec074173e1a67466532d21 +R 5ee5dda65b1a5bf82b60b07edf018dd9 U stephan -Z 8800c626ace8a11c90d3b1467c2d3267 +Z a83a6cf6b033e8a3c3dc80f4ea680e3d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9dcbdeda2f..30f2966370 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c23589d92cd16b67266d97f4a3d8c0991864dbea30ec074173e1a67466532d21 \ No newline at end of file +75fdd5b83b4c527d25649b0d08841e3dc7d4d8109c1c97b2195b303538ced73d \ No newline at end of file From 3f23ce664e84366c3e7c7fcf675c480e2d3988af Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 2 Mar 2023 13:49:50 +0000 Subject: [PATCH 046/341] When flattening the right operand of a LEFT JOIN (check-in [41c27bc0ff1d3135]), ensure that the OP_IfNullRow opcode does not NULL-out a subquery result that was computed within OP_Once. This fixes the problem problem reported by [forum:/forumpost/402f05296d|forum post 402f05296d]. FossilOrigin-Name: 8fe13f7a5e5eb798189acb25a608df7a94c2f5cc83463331a048b779c7890c82 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 20 +++++++++++++------- test/join2.test | 18 ++++++++++++++++++ 4 files changed, 40 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 408ede3bd6..e20c907935 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Resolve\sa\sparallel\sbuild\stiming\sissue\swhen\sbuilding\ssqlite3.c/h\sfrom\sext/wasm.\sFor\sthe\stime\sbeing,\sdo\snot\sadd\ssqlite3_wasm_extra_init.c\sto\sfiddle.wasm\sbecause\sit\scan\scause\sduplicate\sdefinitions\sof\sextensions\swhich\sare\salready\sbuilt\sinto\sthe\sshell\s(a\sbetter\sresolution\sfor\sthis\sconflict\sis\spending).\sNo\slonger\sadd\ssqlite3_wasm_extra_init.c\sto\sspeedtest1.wasm\sbecause\sit's\suseless\sthere. -D 2023-03-02T06:58:55.482 +C When\sflattening\sthe\sright\soperand\sof\sa\sLEFT\sJOIN\n(check-in\s[41c27bc0ff1d3135]),\sensure\sthat\sthe\sOP_IfNullRow\sopcode\sdoes\snot\nNULL-out\sa\ssubquery\sresult\sthat\swas\scomputed\swithin\sOP_Once.\s\sThis\sfixes\nthe\sproblem\sproblem\sreported\sby\n[forum:/forumpost/402f05296d|forum\spost\s402f05296d]. +D 2023-03-02T13:49:50.187 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -573,7 +573,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c e3520c28b322d0e06e883c91de15322ddd06a98d5b4564c3273c99da9391d1b8 +F src/expr.c 8f9d5c20cf412d231b485bae592c78ff1906ce4b8e6b0f185f07441bd4070e72 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -1217,7 +1217,7 @@ F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 F test/join.test ed1daf99958fed1b9f017e56bae2bb6b49339a1ec0b70b9e8f7259960c6bf387 -F test/join2.test 88f4527101710806674d49257439f198fa147a7fcec11afedf51cf4894dc8877 +F test/join2.test c378a2c59db8da13a265481c9aeab08d854c524f56a0eda12fa7bc535bfbebb1 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 91f1f4c7d81fd87b58e9ba7cf4a2b5d39e3583b4f8e498a162722a60259c5208 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c23589d92cd16b67266d97f4a3d8c0991864dbea30ec074173e1a67466532d21 -R 5ee5dda65b1a5bf82b60b07edf018dd9 -U stephan -Z a83a6cf6b033e8a3c3dc80f4ea680e3d +P 75fdd5b83b4c527d25649b0d08841e3dc7d4d8109c1c97b2195b303538ced73d +R 5cc223eb1f7166aa4b37bea1933f39eb +U drh +Z 278b20e1d47d5f647839601276b5762c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 30f2966370..f82e15b88f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75fdd5b83b4c527d25649b0d08841e3dc7d4d8109c1c97b2195b303538ced73d \ No newline at end of file +8fe13f7a5e5eb798189acb25a608df7a94c2f5cc83463331a048b779c7890c82 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 763e961b6e..b688e5b401 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4833,16 +4833,22 @@ expr_code_doover: break; } } - addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); - /* Temporarily disable factoring of constant expressions, since - ** even though expressions may appear to be constant, they are not - ** really constant because they originate from the right-hand side - ** of a LEFT JOIN. */ - pParse->okConstFactor = 0; + addrINR = sqlite3VdbeAddOp3(v, OP_IfNullRow, pExpr->iTable, 0, target); + /* The OP_IfNullRow opcode above can overwrite the result register with + ** NULL. So we have to ensure that the result register is not a value + ** that is suppose to be a constant. Two defenses are needed: + ** (1) Temporarily disable factoring of constant expressions + ** (2) Make sure the computed value really is stored in register + ** "target" and not someplace else. + */ + pParse->okConstFactor = 0; /* note (1) above */ inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); pParse->okConstFactor = okConstFactor; + if( inReg!=target ){ /* note (2) above */ + sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); + inReg = target; + } sqlite3VdbeJumpHere(v, addrINR); - sqlite3VdbeChangeP3(v, addrINR, inReg); break; } diff --git a/test/join2.test b/test/join2.test index b3fd265353..39cabf8809 100644 --- a/test/join2.test +++ b/test/join2.test @@ -386,4 +386,22 @@ do_execsql_test 10.4 { ) FROM t1; } NULL +# 2023-03-02 https://sqlite.org/forum/forumpost/402f05296d +# +# The TK_IF_NULL_ROW expression node must ensure that it does not overwrite +# the result register of an OP_Once subroutine. +# +optimization_control db all 1 +do_execsql_test 11.1 { + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + CREATE TABLE t1(x TEXT, y INTEGER); + INSERT INTO t1(x,y) VALUES(NULL,-2),(NULL,1),('0',2); + CREATE TABLE t2(z INTEGER); + INSERT INTO t2(z) VALUES(2),(-2); + CREATE VIEW t3 AS SELECT z, (SELECT count(*) FROM t1) AS w FROM t2; + SELECT * FROM t1 LEFT JOIN t3 ON y=z; +} {NULL -2 -2 3 NULL 1 NULL NULL 0 2 2 3} + finish_test From 0b23765b6f23368a5664d879f1d65ee4631c6ce5 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Mar 2023 10:42:23 +0000 Subject: [PATCH 047/341] Remove unnecessary call to sqlite3_dbdata_init() from shell.c.in. FossilOrigin-Name: c4d083a3aeeee69342d41b93a1393855871b0e4e7bfdb5fcc2973138018f248b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 3 --- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e20c907935..b42d3253ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sflattening\sthe\sright\soperand\sof\sa\sLEFT\sJOIN\n(check-in\s[41c27bc0ff1d3135]),\sensure\sthat\sthe\sOP_IfNullRow\sopcode\sdoes\snot\nNULL-out\sa\ssubquery\sresult\sthat\swas\scomputed\swithin\sOP_Once.\s\sThis\sfixes\nthe\sproblem\sproblem\sreported\sby\n[forum:/forumpost/402f05296d|forum\spost\s402f05296d]. -D 2023-03-02T13:49:50.187 +C Remove\sunnecessary\scall\sto\ssqlite3_dbdata_init()\sfrom\sshell.c.in. +D 2023-03-03T10:42:23.243 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -624,7 +624,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f6224c55bc0de16b8559e80015a26f1eced7809a4a0e774efabacca98b3150f6 -F src/shell.c.in af928afa0376b2e2c962be9a0a3ab716b3983b1adfae10e248ffd40ede03b049 +F src/shell.c.in fde8844845550e7f9332132e2af57e68fe1c37090986483445c5204ee9599e59 F src/sqlite.h.in dd0f802db5ce8b2e384cf1e04debe175b38ab5ba2c8e09f00672e8b79a1bb134 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75fdd5b83b4c527d25649b0d08841e3dc7d4d8109c1c97b2195b303538ced73d -R 5cc223eb1f7166aa4b37bea1933f39eb -U drh -Z 278b20e1d47d5f647839601276b5762c +P 8fe13f7a5e5eb798189acb25a608df7a94c2f5cc83463331a048b779c7890c82 +R d5de0af7f0990abbeebbc14d0ffe309d +U dan +Z eb40422026ceae79ea2f6e432eef0cbe # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f82e15b88f..704e139b27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8fe13f7a5e5eb798189acb25a608df7a94c2f5cc83463331a048b779c7890c82 \ No newline at end of file +c4d083a3aeeee69342d41b93a1393855871b0e4e7bfdb5fcc2973138018f248b \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 0a54a3c5a5..c2da3dbb38 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5325,9 +5325,6 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); #endif -#if SQLITE_SHELL_HAVE_RECOVER - sqlite3_dbdata_init(p->db, 0, 0); -#endif #ifdef SQLITE_HAVE_ZLIB if( !p->bSafeModePersist ){ sqlite3_zipfile_init(p->db, 0, 0); From dc81902445ae7f2446fc9f3575cb2c2b78b18341 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 3 Mar 2023 15:12:46 +0000 Subject: [PATCH 048/341] Do not use an expression index on a generated column if generated column has the wrong affinity. dbsqlfuzz 65f5eb57f8859344d5f1f33e08c77ee12960ed83 FossilOrigin-Name: e95439119ac200cb47d0e277622f41ee7986b364487cd252b485ce5fa030d70f --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/build.c | 1 + src/expr.c | 11 +++++++++++ src/sqliteInt.h | 1 + src/where.c | 3 +++ test/gencol1.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 73 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index b42d3253ab..a727bbc7e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunnecessary\scall\sto\ssqlite3_dbdata_init()\sfrom\sshell.c.in. -D 2023-03-03T10:42:23.243 +C Do\snot\suse\san\sexpression\sindex\son\sa\sgenerated\scolumn\sif\sgenerated\scolumn\nhas\sthe\swrong\saffinity.\s\sdbsqlfuzz\s65f5eb57f8859344d5f1f33e08c77ee12960ed83 +D 2023-03-03T15:12:46.365 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -565,7 +565,7 @@ F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca F src/btree.c c547e099f853de61835ff45da8d956a932d02ecf1ffd472a1f2a103b83e6dd40 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e -F src/build.c c55ab6d1b089ceef57160e840f05f692955ac90944c3d04fcf01d97fd7bfd08d +F src/build.c f305be161a5609ece7b156f7ccebc92cc6a73915e8c2b4f33efdb07f79fcdae8 F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d @@ -573,7 +573,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 8f9d5c20cf412d231b485bae592c78ff1906ce4b8e6b0f185f07441bd4070e72 +F src/expr.c 399c10566b94ded9b565c01382026bfa0a7350782d4e9aa4194af1ac05f8af31 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -628,7 +628,7 @@ F src/shell.c.in fde8844845550e7f9332132e2af57e68fe1c37090986483445c5204ee9599e5 F src/sqlite.h.in dd0f802db5ce8b2e384cf1e04debe175b38ab5ba2c8e09f00672e8b79a1bb134 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 87bcbb5e08d454e4ee615079fe0de311e2d62bea776efb90f366bf876d42d9ad +F src/sqliteInt.h 34034353de20c29e04295c3486bbf22b2a47842c887f65f7176530b4c35fdd17 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -708,7 +708,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 6b45dc4c47d636f5adb0d78ed71cefab5ab2a7287fb80c86604a5f7874d2ba22 +F src/where.c 201f03ca60f318b6bd1cb1977af93b67ff3679acea72eeec69e3f27e678b7e59 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 9919e5a22f4b24dd96c49b8981484cbe6bbfcf466ff73ac40a06e1356aa8bf87 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -1154,7 +1154,7 @@ F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc F test/fuzzinvariants.c a153253600b2b33a7d5710d40e89b2ac1373a1912517867fb995a45b2d67dcb8 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c -F test/gencol1.test cc0dbb0ee116e5602e18ea7d47f2a0f76b26e09a823b7c36ef254370c2b0f3c1 +F test/gencol1.test e40c94c0d3485c37c23ec8ff42dbb4464cef3f3e41d96ef1abf9c4f2b97936c9 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/having.test a89236dd8d55aa50c4805f82ac9daf64d477a44d712d8209c118978d0ca21ec9 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8fe13f7a5e5eb798189acb25a608df7a94c2f5cc83463331a048b779c7890c82 -R d5de0af7f0990abbeebbc14d0ffe309d -U dan -Z eb40422026ceae79ea2f6e432eef0cbe +P c4d083a3aeeee69342d41b93a1393855871b0e4e7bfdb5fcc2973138018f248b +R cdfe0cff0583f003de0ff68f6f8d954b +U drh +Z 472d47c518c944918dc4fe82959825c1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 704e139b27..486367d02e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c4d083a3aeeee69342d41b93a1393855871b0e4e7bfdb5fcc2973138018f248b \ No newline at end of file +e95439119ac200cb47d0e277622f41ee7986b364487cd252b485ce5fa030d70f \ No newline at end of file diff --git a/src/build.c b/src/build.c index 0390e321f2..57762ebc2b 100644 --- a/src/build.c +++ b/src/build.c @@ -2012,6 +2012,7 @@ void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType){ ** turn it into one by adding a unary "+" operator. */ pExpr = sqlite3PExpr(pParse, TK_UPLUS, pExpr, 0); } + pExpr->affExpr = pCol->affinity; sqlite3ColumnSetExpr(pParse, pTab, pCol, pExpr); pExpr = 0; goto generated_done; diff --git a/src/expr.c b/src/expr.c index b688e5b401..6135da786c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4133,6 +4133,7 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( IndexedExpr *p; Vdbe *v; for(p=pParse->pIdxEpr; p; p=p->pIENext){ + u8 exprAff; int iDataCur = p->iDataCur; if( iDataCur<0 ) continue; if( pParse->iSelfTab ){ @@ -4140,6 +4141,16 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup( iDataCur = -1; } if( sqlite3ExprCompare(0, pExpr, p->pExpr, iDataCur)!=0 ) continue; + assert( p->aff>=SQLITE_AFF_BLOB && p->aff<=SQLITE_AFF_NUMERIC ); + exprAff = sqlite3ExprAffinity(pExpr); + if( (exprAff<=SQLITE_AFF_BLOB && p->aff!=SQLITE_AFF_BLOB) + || (exprAff==SQLITE_AFF_TEXT && p->aff!=SQLITE_AFF_TEXT) + || (exprAff>=SQLITE_AFF_NUMERIC && p->aff!=SQLITE_AFF_NUMERIC) + ){ + /* Affinity mismatch on a generated column */ + continue; + } + v = pParse->pVdbe; assert( v!=0 ); if( p->bMaybeNullRow ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e97fed25f1..3a7be541ae 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3662,6 +3662,7 @@ struct IndexedExpr { int iIdxCur; /* The index cursor */ int iIdxCol; /* The index column that contains value of pExpr */ u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */ + u8 aff; /* Affinity of the pExpr expression */ IndexedExpr *pIENext; /* Next in a list of all indexed expressions */ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS const char *zIdxName; /* Name of index, used only for bytecode comments */ diff --git a/src/where.c b/src/where.c index 8f6bc24be9..d7dbd57909 100644 --- a/src/where.c +++ b/src/where.c @@ -5706,6 +5706,9 @@ static SQLITE_NOINLINE void whereAddIndexedExpr( p->iIdxCur = iIdxCur; p->iIdxCol = i; p->bMaybeNullRow = bMaybeNullRow; + if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ + p->aff = pIdx->zColAff[i]; + } #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS p->zIdxName = pIdx->zName; #endif diff --git a/test/gencol1.test b/test/gencol1.test index ee0ebc53f5..266eee18b4 100644 --- a/test/gencol1.test +++ b/test/gencol1.test @@ -615,4 +615,49 @@ do_execsql_test gencol1-22.1 { AND (y.a=2 OR (x.b LIKE '2*' AND y.a=x.b)); } {2 2 2 2} + +# 2023-03-02 dbsqlfuzz 65f5eb57f8859344d5f1f33e08c77ee12960ed83 +# +set typelist {ANY INT REAL BLOB TEXT {}} +set cnt 0 +foreach t1 $typelist { + foreach t2 $typelist { + incr cnt + db eval " + DROP TABLE IF EXISTS t1; + CREATE TABLE t1( + x $t1, + a $t2 AS (x) VIRTUAL, + b BLOB AS (x) VIRTUAL + ); + CREATE INDEX x2 ON t1(a); + INSERT INTO t1(x) VALUES(NULL),('1'),(2),(3.5),('xyz'); + " + set x1 [lsort [db eval {SELECT typeof(b) FROM t1}]] + do_test gencol1-23.1.$cnt { + lsort [db eval {SELECT typeof(b) FROM t1 INDEXED BY x2}] + } $x1 + } +} +do_execsql_test gencol1-23.2 { + DROP TABLE t1; + CREATE TABLE t1( + x, + a INT AS (x) VIRTUAL, + b BLOB AS (x) VIRTUAL + ); + CREATE INDEX x2 ON t1(a); + INSERT INTO t1(x) VALUES(NULL),('1'),('xyz'),(2),(3.5); + SELECT quote(a) FROM t1 INDEXED BY x2; +} {NULL 1 2 3.5 'xyz'} +do_execsql_test gencol1-23.3 { + EXPLAIN SELECT a FROM t1 INDEXED BY x2; +} {~/Column 0/} +# ^^^^^^^^---- verfies that x2 acts like a covering index +do_execsql_test gencol1-23.4 { + EXPLAIN SELECT b FROM t1 INDEXED BY x2; +} {/Column 0/} +# ^^^^^^^^^^--- Must reference the original table in this case because +# of the different datatype on column b. + finish_test From 04337898635b3048691b795f2f1922b3fda2ddc9 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Mar 2023 16:25:18 +0000 Subject: [PATCH 049/341] When it is known when preparing a statement that X cannot be NULL, transform the expression (X IS NULL) to integer value 1 instead of 'true'. This is because under some circumstances, "Y IS TRUE" may not be equivalent to "Y IS 1". FossilOrigin-Name: cc4bb05b3653e9502b95ea6fe0bfb77feebc11285b66e1dde4c7b945928efbf1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/resolve.c | 12 ++++-------- test/window1.test | 16 ++++++++++++++++ 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index a727bbc7e5..885ee87a21 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\san\sexpression\sindex\son\sa\sgenerated\scolumn\sif\sgenerated\scolumn\nhas\sthe\swrong\saffinity.\s\sdbsqlfuzz\s65f5eb57f8859344d5f1f33e08c77ee12960ed83 -D 2023-03-03T15:12:46.365 +C When\sit\sis\sknown\swhen\spreparing\sa\sstatement\sthat\sX\scannot\sbe\sNULL,\stransform\sthe\sexpression\s(X\sIS\sNULL)\sto\sinteger\svalue\s1\sinstead\sof\s'true'.\sThis\sis\sbecause\sunder\ssome\scircumstances,\s"Y\sIS\sTRUE"\smay\snot\sbe\sequivalent\sto\s"Y\sIS\s1". +D 2023-03-03T16:25:18.921 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c d62c5665279cc7485f9d45b5e20911cc7b19c203f268321a90d05d74f4725750 +F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f6224c55bc0de16b8559e80015a26f1eced7809a4a0e774efabacca98b3150f6 F src/shell.c.in fde8844845550e7f9332132e2af57e68fe1c37090986483445c5204ee9599e59 @@ -1898,7 +1898,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 08bac934238ff66987635ea3367c5aed5b4c088c414441beaba426e68a8257c1 +F test/window1.test 5ad2f3e2aec3f2dc5cf4a66998d42a455b8ebd57ec173c0329f6b8788bcae0aa F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c4d083a3aeeee69342d41b93a1393855871b0e4e7bfdb5fcc2973138018f248b -R cdfe0cff0583f003de0ff68f6f8d954b -U drh -Z 472d47c518c944918dc4fe82959825c1 +P e95439119ac200cb47d0e277622f41ee7986b364487cd252b485ce5fa030d70f +R 832f5112357909076c536cf7b9bea8b8 +U dan +Z 3d9689df10ad73a0297a8e055e1db782 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 486367d02e..e32a6d4fc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e95439119ac200cb47d0e277622f41ee7986b364487cd252b485ce5fa030d70f \ No newline at end of file +cc4bb05b3653e9502b95ea6fe0bfb77feebc11285b66e1dde4c7b945928efbf1 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 9677f9de91..dfb3434396 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -947,14 +947,10 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ testcase( ExprHasProperty(pExpr, EP_OuterON) ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( pExpr->op==TK_NOTNULL ){ - pExpr->u.zToken = "true"; - ExprSetProperty(pExpr, EP_IsTrue); - }else{ - pExpr->u.zToken = "false"; - ExprSetProperty(pExpr, EP_IsFalse); - } - pExpr->op = TK_TRUEFALSE; + pExpr->u.iValue = (pExpr->op==TK_NOTNULL); + pExpr->flags |= EP_IntValue; + pExpr->op = TK_INTEGER; + for(i=0, p=pNC; p && ipNext, i++){ p->nRef = anRef[i]; } diff --git a/test/window1.test b/test/window1.test index 0f22829847..471ac2dce8 100644 --- a/test/window1.test +++ b/test/window1.test @@ -2210,4 +2210,20 @@ do_execsql_test 72.1 { SELECT * FROM v1 WHERE true; } {1 0} +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 72.0 { + CREATE TABLE t0(c0); + INSERT INTO t0(c0) VALUES (0); + CREATE VIEW v0(c0) AS SELECT TOTAL(0) OVER (PARTITION BY t0.c0) FROM t0; +} +do_execsql_test 72.1 { + SELECT COUNT(*) FROM ( + SELECT TOTAL(0) OVER (PARTITION BY t0.c0) FROM t0 + ) + WHERE ('1' IS NOT ('abcde' NOTNULL)); +} {1} + + finish_test From 1b9db7f32de952d7c88a87b8251b005ee08f3d65 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 3 Mar 2023 18:35:00 +0000 Subject: [PATCH 050/341] Enhance PRAGMA integrity_check so that it can detect when there are extra bytes at the end of an index record, which might cause OP_IdxRowid to malfunction. dbsqlfuzz c1aa3986534d5feab8d21f28b3c1712df2ef358ba. Test case in TH3. FossilOrigin-Name: f418bdd627e84e7d494f730d7124d8f4846ebcde031f5b2498685c9aceebb3c8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pragma.c | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 885ee87a21..2a63a4cbbf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sit\sis\sknown\swhen\spreparing\sa\sstatement\sthat\sX\scannot\sbe\sNULL,\stransform\sthe\sexpression\s(X\sIS\sNULL)\sto\sinteger\svalue\s1\sinstead\sof\s'true'.\sThis\sis\sbecause\sunder\ssome\scircumstances,\s"Y\sIS\sTRUE"\smay\snot\sbe\sequivalent\sto\s"Y\sIS\s1". -D 2023-03-03T16:25:18.921 +C Enhance\sPRAGMA\sintegrity_check\sso\sthat\sit\scan\sdetect\swhen\sthere\sare\sextra\nbytes\sat\sthe\send\sof\san\sindex\srecord,\swhich\smight\scause\sOP_IdxRowid\sto\nmalfunction.\s\sdbsqlfuzz\sc1aa3986534d5feab8d21f28b3c1712df2ef358ba.\s\sTest\scase\nin\sTH3. +D 2023-03-03T18:35:00.504 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -616,7 +616,7 @@ F src/parse.y 960d2da92a23f8ba2ca22748a51bd75ee2c575564f2cbc59f119640e7f5b4c5d F src/pcache.c f4268f7f73c6a3db12ce22fd25bc68dc42315d19599414ab1207d7cf32f79197 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc -F src/pragma.c 82d5090a35eac75876d3b41f48d06b2370553c9576bf2942233c462e03eb94c9 +F src/pragma.c e698baae96396cac8ff55afef1c0b84632a4b825548bf98f0c4fd1e0a90ed4bc F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e95439119ac200cb47d0e277622f41ee7986b364487cd252b485ce5fa030d70f -R 832f5112357909076c536cf7b9bea8b8 -U dan -Z 3d9689df10ad73a0297a8e055e1db782 +P cc4bb05b3653e9502b95ea6fe0bfb77feebc11285b66e1dde4c7b945928efbf1 +R e3264926b184a1150ff1e42cc58b6e34 +U drh +Z 4bf4a9f38f38c1a43669629bffba2cbf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e32a6d4fc1..d79799f154 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc4bb05b3653e9502b95ea6fe0bfb77feebc11285b66e1dde4c7b945928efbf1 \ No newline at end of file +f418bdd627e84e7d494f730d7124d8f4846ebcde031f5b2498685c9aceebb3c8 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 522a12d331..01c2d486e7 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1987,6 +1987,23 @@ void sqlite3Pragma( jmp4 = integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, jmp2); + /* The OP_IdxRowid opcode is an optimized version of OP_Column + ** that extracts the rowid off the end of the index record. + ** But it only works correctly if index record does not have + ** any extra bytes at the end. Verify that this is the case. */ + if( HasRowid(pTab) ){ + int jmp7; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur+j, 3); + jmp7 = sqlite3VdbeAddOp3(v, OP_Eq, 3, 0, r1+pIdx->nColumn-1); + VdbeCoverage(v); + sqlite3VdbeLoadString(v, 3, + "rowid not at end-of-record for row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " of index "); + sqlite3VdbeGoto(v, jmp5-1); + sqlite3VdbeJumpHere(v, jmp7); + } + /* Any indexed columns with non-BINARY collations must still hold ** the exact same text value as the table. */ label6 = 0; From 294872382b33b54b961d76958a2f589b3b007f13 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 3 Mar 2023 19:43:39 +0000 Subject: [PATCH 051/341] Follow-up to [e95439119ac200cb] to fix a case where a pointer is NULL due to OOM. FossilOrigin-Name: 2535bc8c256a7642a6ac00ebfd3393beb93da94394c13b886c3ddd20d114aa3c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2a63a4cbbf..7512043e79 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sPRAGMA\sintegrity_check\sso\sthat\sit\scan\sdetect\swhen\sthere\sare\sextra\nbytes\sat\sthe\send\sof\san\sindex\srecord,\swhich\smight\scause\sOP_IdxRowid\sto\nmalfunction.\s\sdbsqlfuzz\sc1aa3986534d5feab8d21f28b3c1712df2ef358ba.\s\sTest\scase\nin\sTH3. -D 2023-03-03T18:35:00.504 +C Follow-up\sto\s[e95439119ac200cb]\sto\sfix\sa\scase\swhere\sa\spointer\sis\sNULL\sdue\sto\nOOM. +D 2023-03-03T19:43:39.157 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -565,7 +565,7 @@ F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca F src/btree.c c547e099f853de61835ff45da8d956a932d02ecf1ffd472a1f2a103b83e6dd40 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e -F src/build.c f305be161a5609ece7b156f7ccebc92cc6a73915e8c2b4f33efdb07f79fcdae8 +F src/build.c 542a04532ae810a58366c47453de4fc1ca9fa6e6b2d920b6d639cdd1a1831309 F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cc4bb05b3653e9502b95ea6fe0bfb77feebc11285b66e1dde4c7b945928efbf1 -R e3264926b184a1150ff1e42cc58b6e34 +P f418bdd627e84e7d494f730d7124d8f4846ebcde031f5b2498685c9aceebb3c8 +R 58ecda2fbd94b7383314d0e1826c5d00 U drh -Z 4bf4a9f38f38c1a43669629bffba2cbf +Z 97fb9baddea4583a0ed72a707ad43573 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d79799f154..8703c3c275 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f418bdd627e84e7d494f730d7124d8f4846ebcde031f5b2498685c9aceebb3c8 \ No newline at end of file +2535bc8c256a7642a6ac00ebfd3393beb93da94394c13b886c3ddd20d114aa3c \ No newline at end of file diff --git a/src/build.c b/src/build.c index 57762ebc2b..c91e7bdd6a 100644 --- a/src/build.c +++ b/src/build.c @@ -2012,7 +2012,7 @@ void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType){ ** turn it into one by adding a unary "+" operator. */ pExpr = sqlite3PExpr(pParse, TK_UPLUS, pExpr, 0); } - pExpr->affExpr = pCol->affinity; + if( pExpr ) pExpr->affExpr = pCol->affinity; sqlite3ColumnSetExpr(pParse, pTab, pCol, pExpr); pExpr = 0; goto generated_done; From fe5261592593ce9eabfb57a32fbe17dfc9deec0f Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 3 Mar 2023 19:56:19 +0000 Subject: [PATCH 052/341] Fix a vdbe-coverage macro added by [f418bdd627e84e7d]. FossilOrigin-Name: 77f559d2647615379fed55ced5d69ae90515273e59811b92171bdd3089c90a22 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7512043e79..bad839105c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Follow-up\sto\s[e95439119ac200cb]\sto\sfix\sa\scase\swhere\sa\spointer\sis\sNULL\sdue\sto\nOOM. -D 2023-03-03T19:43:39.157 +C Fix\sa\svdbe-coverage\smacro\sadded\sby\s[f418bdd627e84e7d]. +D 2023-03-03T19:56:19.090 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -616,7 +616,7 @@ F src/parse.y 960d2da92a23f8ba2ca22748a51bd75ee2c575564f2cbc59f119640e7f5b4c5d F src/pcache.c f4268f7f73c6a3db12ce22fd25bc68dc42315d19599414ab1207d7cf32f79197 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc -F src/pragma.c e698baae96396cac8ff55afef1c0b84632a4b825548bf98f0c4fd1e0a90ed4bc +F src/pragma.c c0c6e0735436e0e9e8d490f82959ddc9475d438e9f808a51953a4d43e30cea51 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f418bdd627e84e7d494f730d7124d8f4846ebcde031f5b2498685c9aceebb3c8 -R 58ecda2fbd94b7383314d0e1826c5d00 +P 2535bc8c256a7642a6ac00ebfd3393beb93da94394c13b886c3ddd20d114aa3c +R 69d00902044004b8ef9c714f6e03d77e U drh -Z 97fb9baddea4583a0ed72a707ad43573 +Z fc0cac54f8cfafa8b58dffee6508db4a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8703c3c275..17f9dc480d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2535bc8c256a7642a6ac00ebfd3393beb93da94394c13b886c3ddd20d114aa3c \ No newline at end of file +77f559d2647615379fed55ced5d69ae90515273e59811b92171bdd3089c90a22 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 01c2d486e7..cd8d51c735 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1995,7 +1995,7 @@ void sqlite3Pragma( int jmp7; sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur+j, 3); jmp7 = sqlite3VdbeAddOp3(v, OP_Eq, 3, 0, r1+pIdx->nColumn-1); - VdbeCoverage(v); + VdbeCoverageNeverNull(v); sqlite3VdbeLoadString(v, 3, "rowid not at end-of-record for row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); From 3410e550dcc82f2bcceac2cf01174621ab6e9c2a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Mar 2023 21:17:12 +0000 Subject: [PATCH 053/341] Make the SQLITE_DBCONFIG_STMT_SCANSTATUS option on by default. FossilOrigin-Name: 5a09191186bc03b374e0c0d029e1a15208c6b845bc2f5f5f9f6a8a882809d9f3 --- ext/rtree/rtree6.test | 2 ++ manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/main.c | 3 +++ src/shell.c.in | 1 + src/sqlite.h.in | 2 +- test/fts3join.test | 1 + test/scanstatus.test | 17 ++++++++++------- 8 files changed, 31 insertions(+), 21 deletions(-) diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index b6dfe992a6..1cbb2c6e87 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -104,6 +104,7 @@ do_eqp_test rtree6.2.4.1 { } { QUERY PLAN |--SCAN t1 VIRTUAL TABLE INDEX 2:C0E1 + |--BLOOM FILTER ON t2 (v=?) `--SEARCH t2 USING AUTOMATIC COVERING INDEX (v=?) } do_eqp_test rtree6.2.4.2 { @@ -111,6 +112,7 @@ do_eqp_test rtree6.2.4.2 { } { QUERY PLAN |--SCAN t1 VIRTUAL TABLE INDEX 2:C0E1 + |--BLOOM FILTER ON t2 (v=?) `--SEARCH t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?) } diff --git a/manifest b/manifest index bad839105c..9ee00ad069 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\svdbe-coverage\smacro\sadded\sby\s[f418bdd627e84e7d]. -D 2023-03-03T19:56:19.090 +C Make\sthe\sSQLITE_DBCONFIG_STMT_SCANSTATUS\soption\son\sby\sdefault. +D 2023-03-03T21:17:12.539 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -400,7 +400,7 @@ F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e130 F ext/rtree/rtree3.test 272594f88c344e973864008bbe4c71fd3a41a264c097d568593ee7886d83d409 F ext/rtree/rtree4.test 304de65d484540111b896827e4261815e5dca4ce28eeecd58be648cd73452c4b F ext/rtree/rtree5.test 49c9041d713d54560b315c2c7ef7207ee287eba1b20f8266968a06f2e55d3142 -F ext/rtree/rtree6.test 9ce3691c1aac43070a9f194f0ebf54372db346c5a82241fd11b525ed53ce9f3f +F ext/rtree/rtree6.test 2f5ffc69670395c1a84fad7924e2d49e82a25460c5293fb1e54e1aa906f04945 F ext/rtree/rtree7.test c8fb2e555b128dd0f0bdb520c61380014f497f8a23c40f2e820acc9f9e4fdce5 F ext/rtree/rtree8.test 2d99006a1386663978c9e1df167554671e4f711c419175b39f332719deb1ce0e F ext/rtree/rtree9.test fd3c9384ef8aabbc127b3878764070398f136eebc551cd20484b570f2cc1956a @@ -586,7 +586,7 @@ F src/insert.c 7940fce7d4aa855606432d82d20694e17c18a03956f5f5776f2404e2df7c18a8 F src/json.c c85ed6fce06f43d414b0d7fff64749d43a0dbd1067123ee407bd3a0752454161 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c b04eb648cedc45efe4298e1ef439ac4f0096ae27b5f01accb0a1f49d57789128 -F src/main.c e5f5323a33c51403bbe83ccb9f3e40aa94f95fd65dbeb5992a12435e0f6c22f9 +F src/main.c 50b4f14242c2949fd1f3e83aacf088ed2f1eb16d8c1369ee0156957ea070916e F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -624,8 +624,8 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f6224c55bc0de16b8559e80015a26f1eced7809a4a0e774efabacca98b3150f6 -F src/shell.c.in fde8844845550e7f9332132e2af57e68fe1c37090986483445c5204ee9599e59 -F src/sqlite.h.in dd0f802db5ce8b2e384cf1e04debe175b38ab5ba2c8e09f00672e8b79a1bb134 +F src/shell.c.in 10088fb7a12d9e9b2453b1df32f80f68c6618f998044f82619d769e6f32f72ca +F src/sqlite.h.in 36882eee65db22e2fdb68782e7fb9b9ea1dd0488d8e2918b3567915ddb3c9611 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 F src/sqliteInt.h 34034353de20c29e04295c3486bbf22b2a47842c887f65f7176530b4c35fdd17 @@ -1083,7 +1083,7 @@ F test/fts3fault.test f4e1342acfe6d216a001490e8cd52afac1f9ffe4a11bbcdcb296129a45 F test/fts3fault2.test 7b2741e5095367238380b0fcdb837f36c24484c7a5f353659b387df63cf039ec F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3fuzz001.test e3c7b0ce9b04cc02281dcc96812a277f02df03cd7dc082055d87e11eb18aaf56 -F test/fts3join.test ee25def5e763ea8879c19e74f862d5191410ccc7259338887a3685e97f512662 +F test/fts3join.test 1a4d786539b2b79a41c28ef2ac22cacd92a8ee830249b68a7dee4a020848e3bb F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 F test/fts3matchinfo.test aa66cc50615578b30f6df9984819ae5b702511cf8a94251ec7c594096a703a4a F test/fts3matchinfo2.test 00144e841704b8debfcdf6097969cd9f2a1cf759e2203cda42583648f2e6bf58 @@ -1436,7 +1436,7 @@ F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 -F test/scanstatus.test 2adde2d8fb0fbae73e790afa724f15ea4c3d6d0809e5de4d00407c5c14836401 +F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e F test/scanstatus2.test b77de449be6ffd7a913a06dd84276dc6eb3517d479126e00ac51cd409c0a0af5 F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2535bc8c256a7642a6ac00ebfd3393beb93da94394c13b886c3ddd20d114aa3c -R 69d00902044004b8ef9c714f6e03d77e -U drh -Z fc0cac54f8cfafa8b58dffee6508db4a +P 77f559d2647615379fed55ced5d69ae90515273e59811b92171bdd3089c90a22 +R b0649eadff654072848a0b28a23fc664 +U dan +Z 3ba2511e2bb55a3e7ecb463946dabd68 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 17f9dc480d..abed977a94 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77f559d2647615379fed55ced5d69ae90515273e59811b92171bdd3089c90a22 \ No newline at end of file +5a09191186bc03b374e0c0d029e1a15208c6b845bc2f5f5f9f6a8a882809d9f3 \ No newline at end of file diff --git a/src/main.c b/src/main.c index e79d676139..23010d5fc4 100644 --- a/src/main.c +++ b/src/main.c @@ -3364,6 +3364,9 @@ static int openDatabase( #endif #if defined(SQLITE_DEFAULT_LEGACY_ALTER_TABLE) | SQLITE_LegacyAlter +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) + | SQLITE_StmtScanStatus #endif ; sqlite3HashInit(&db->aCollSeq); diff --git a/src/shell.c.in b/src/shell.c.in index c2da3dbb38..944f655622 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5309,6 +5309,7 @@ static void open_db(ShellState *p, int openFlags){ } exit(1); } + sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0); #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 03e4862cc8..9dd023821c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2464,7 +2464,7 @@ struct sqlite3_mem_methods { ** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() ** statistics. For statistics to be collected, the flag must be set on ** the database handle both when the SQL statement is prepared and when it -** is stepped. The flag is clear (collection of statistics is disabled) +** is stepped. The flag is set (collection of statistics is enabled) ** by default. ** */ diff --git a/test/fts3join.test b/test/fts3join.test index 3333b1ab42..cbd08b63f2 100644 --- a/test/fts3join.test +++ b/test/fts3join.test @@ -100,6 +100,7 @@ do_eqp_test 4.2 { |--MATERIALIZE rr | `--SCAN ft4 VIRTUAL TABLE INDEX 3: |--SCAN t4 + |--BLOOM FILTER ON rr (docid=?) `--SEARCH rr USING AUTOMATIC COVERING INDEX (docid=?) LEFT-JOIN } diff --git a/test/scanstatus.test b/test/scanstatus.test index ab87ad944a..549e7fd3c8 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -45,17 +45,20 @@ proc do_scanstatus_test {tn res} { uplevel [list do_test $tn [list set {} $ret] [list {*}$res]] } -do_execsql_test 1.0a { SELECT count(*) FROM t1, t2; } 6 -do_scanstatus_test 1.0b { } - -sqlite3_db_config db STMT_SCANSTATUS 1 - -do_execsql_test 1.1 { SELECT count(*) FROM t1, t2; } 6 -do_scanstatus_test 1.2 { +do_execsql_test 1.1a { SELECT count(*) FROM t1, t2; } 6 +do_scanstatus_test 1.1b { nLoop 1 nVisit 2 nEst 1048576.0 zName t1 zExplain {SCAN t1} nLoop 2 nVisit 6 nEst 1048576.0 zName t2 zExplain {SCAN t2} } +sqlite3_db_config db STMT_SCANSTATUS 0 + +do_execsql_test 1.2a { SELECT count(*) FROM t1, t2; } 6 +do_scanstatus_test 1.2b { +} + +sqlite3_db_config db STMT_SCANSTATUS 1 + do_execsql_test 1.3 { ANALYZE; SELECT count(*) FROM t1, t2; From c8c7243d6ee9c6231beadc73b572f10c6b4043e9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 4 Mar 2023 12:57:07 +0000 Subject: [PATCH 054/341] Show the output value from OP_AggFinal when doing byte-code tracing. FossilOrigin-Name: 35f10a06ba81b8a526249729828fff06214e9cf95de418a9081f19d8d69fd657 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9ee00ad069..419bf01dfe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\sSQLITE_DBCONFIG_STMT_SCANSTATUS\soption\son\sby\sdefault. -D 2023-03-03T21:17:12.539 +C Show\sthe\soutput\svalue\sfrom\sOP_AggFinal\swhen\sdoing\sbyte-code\stracing. +D 2023-03-04T12:57:07.246 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -693,7 +693,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 1ddcb8a799d1a4e0c3dcc5c3641923d1343439070b35872c1801a78a0059234e +F src/vdbe.c 0cf4c72a9e0eb614afc19c9c4ca9c8a919c97c0866934a70dac7c2f689a4edf8 F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 77f559d2647615379fed55ced5d69ae90515273e59811b92171bdd3089c90a22 -R b0649eadff654072848a0b28a23fc664 -U dan -Z 3ba2511e2bb55a3e7ecb463946dabd68 +P 5a09191186bc03b374e0c0d029e1a15208c6b845bc2f5f5f9f6a8a882809d9f3 +R c0239d2f4805d5cbeacaa15b77d624c8 +U drh +Z 586551561f2528489f27ae7250514313 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index abed977a94..c2fac0e8e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a09191186bc03b374e0c0d029e1a15208c6b845bc2f5f5f9f6a8a882809d9f3 \ No newline at end of file +35f10a06ba81b8a526249729828fff06214e9cf95de418a9081f19d8d69fd657 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 2bb4964c37..dfe8070de1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -7625,6 +7625,7 @@ case OP_AggFinal: { } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); + REGISTER_TRACE((int)(pMem-aMem), pMem); break; } From 89b3cbed6788f899d02b27d27f5199d777d53db3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 4 Mar 2023 15:36:51 +0000 Subject: [PATCH 055/341] Fix to check-in [b9190d3da70c4171] - the agg-with-indexed-expr optimization requested by ticket [99378177930f87bd] - that can cause an incorrect answer if an aggregate subquery has a GROUP BY clause, and that GROUP BY contains a term that is not in the result set, and the outer query makes use of expression indexes. Problem reported by [forum:/forumpost/a68313d054|forum post a68313d054]. FossilOrigin-Name: e06973876993926fd56181281d04b8dd504c689abf883fa21a5721cc1d478ea8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 4 +++- test/tkt-99378177930f87bd.test | 15 +++++++++++++++ 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 419bf01dfe..cfba0ecda5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Show\sthe\soutput\svalue\sfrom\sOP_AggFinal\swhen\sdoing\sbyte-code\stracing. -D 2023-03-04T12:57:07.246 +C Fix\sto\scheck-in\s[b9190d3da70c4171]\s-\sthe\sagg-with-indexed-expr\soptimization\nrequested\sby\sticket\s[99378177930f87bd]\s-\sthat\scan\scause\san\sincorrect\sanswer\nif\san\saggregate\ssubquery\shas\sa\sGROUP\sBY\sclause,\sand\sthat\sGROUP\sBY\scontains\sa\nterm\sthat\sis\snot\sin\sthe\sresult\sset,\sand\sthe\souter\squery\smakes\suse\sof\sexpression\nindexes.\s\sProblem\sreported\sby\n[forum:/forumpost/a68313d054|forum\spost\sa68313d054]. +D 2023-03-04T15:36:51.218 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -623,7 +623,7 @@ F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c f6224c55bc0de16b8559e80015a26f1eced7809a4a0e774efabacca98b3150f6 +F src/select.c 2f38b8424d204f79180acc6adf84cf3a35a5176e33bc6b009f9a9f080316d4bb F src/shell.c.in 10088fb7a12d9e9b2453b1df32f80f68c6618f998044f82619d769e6f32f72ca F src/sqlite.h.in 36882eee65db22e2fdb68782e7fb9b9ea1dd0488d8e2918b3567915ddb3c9611 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1615,7 +1615,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 -F test/tkt-99378177930f87bd.test 2f07020a82ed1c56bdad60a8a6ef508b2f8a1fb056300b7ec650cbd9975b46bf +F test/tkt-99378177930f87bd.test 28530bf9903dcd7743185ce78b1c02b1f9ba09fe4fa77a70ecbd0af83fe3353c F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a09191186bc03b374e0c0d029e1a15208c6b845bc2f5f5f9f6a8a882809d9f3 -R c0239d2f4805d5cbeacaa15b77d624c8 +P 35f10a06ba81b8a526249729828fff06214e9cf95de418a9081f19d8d69fd657 +R 7369d7a38af682b682af1a066f3ec500 U drh -Z 586551561f2528489f27ae7250514313 +Z a08315d4647510add4b36d411a52c34f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c2fac0e8e2..d60f674cb2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35f10a06ba81b8a526249729828fff06214e9cf95de418a9081f19d8d69fd657 \ No newline at end of file +e06973876993926fd56181281d04b8dd504c689abf883fa21a5721cc1d478ea8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 359dc8cf30..2906a4895c 100644 --- a/src/select.c +++ b/src/select.c @@ -6439,10 +6439,12 @@ static void optimizeAggregateUseOfIndexedExpr( NameContext *pNC /* Name context used to resolve agg-func args */ ){ assert( pAggInfo->iFirstReg==0 ); + assert( pSelect!=0 ); + assert( pSelect->pGroupBy!=0 ); pAggInfo->nColumn = pAggInfo->nAccumulator; if( ALWAYS(pAggInfo->nSortingColumn>0) ){ if( pAggInfo->nColumn==0 ){ - pAggInfo->nSortingColumn = 0; + pAggInfo->nSortingColumn = pSelect->pGroupBy->nExpr; }else{ pAggInfo->nSortingColumn = pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test index 868f36c3a5..ff73529bbd 100644 --- a/test/tkt-99378177930f87bd.test +++ b/test/tkt-99378177930f87bd.test @@ -176,4 +176,19 @@ do_execsql_test tkt-99378-310 { ); } {1 2} +# 2023-03-04 https://sqlite.org/forum/forumpost/a68313d054 +# +do_execsql_test tkt-99378-400 { + DROP TABLE t1; + CREATE TABLE t0(w); + INSERT INTO t0(w) VALUES(1); + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(1); + CREATE INDEX t1x ON t1(x > 0); + CREATE VIEW t2(y) AS SELECT avg(w) FROM t0 GROUP BY w>1; + CREATE VIEW t3(z) AS SELECT count(*) FROM t2 WHERE y BETWEEN 0 and 0; + SELECT count(*) FROM t1 NOT INDEXED WHERE (SELECT z FROM t3); + SELECT count(*) FROM t1 INDEXED BY t1x WHERE (SELECT z FROM t3); +} {0 0} + finish_test From 32b3444f1518d02898ea0b8cbd5a73c7a142cb2a Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 5 Mar 2023 07:33:11 +0000 Subject: [PATCH 056/341] Correct rendering of error messages in demo-worker1.js. Remove some stray EOL whitespace. FossilOrigin-Name: 1d5d515ad97cf61bd679f8c1afc607815c079583fe80264b591c6ef18f56fb8b --- ext/wasm/demo-worker1-promiser.js | 4 ++-- ext/wasm/demo-worker1.js | 4 ++-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ext/wasm/demo-worker1-promiser.js b/ext/wasm/demo-worker1-promiser.js index ef955403c5..e8d3d4e5ac 100644 --- a/ext/wasm/demo-worker1-promiser.js +++ b/ext/wasm/demo-worker1-promiser.js @@ -9,7 +9,7 @@ * May you share freely, never taking more than you give. *********************************************************************** - + Demonstration of the sqlite3 Worker API #1 Promiser: a Promise-based proxy for for the sqlite3 Worker #1 API. */ @@ -81,7 +81,7 @@ }); logHtml('', "Sending 'open' message and waiting for its response before continuing..."); - + await wtest('open', { filename: dbFilename, simulateError: 0 /* if true, fail the 'open' */, diff --git a/ext/wasm/demo-worker1.js b/ext/wasm/demo-worker1.js index cc63f3a7cc..f70179c5e8 100644 --- a/ext/wasm/demo-worker1.js +++ b/ext/wasm/demo-worker1.js @@ -62,7 +62,7 @@ return this.queue.shift(); } }; - + const testCount = ()=>{ logHtml("","Total test count:",T.counter+". Total time =",(performance.now() - startTime),"ms"); }; @@ -74,7 +74,7 @@ (ev.workerRespondTime - ev.workerReceivedTime),"ms.", "Round-trip event time =", (performance.now() - ev.departureTime),"ms.", - (evd.errorClass ? ev.message : "")//, JSON.stringify(evd) + (evd.errorClass ? evd.message : "")//, JSON.stringify(evd) ); }; diff --git a/manifest b/manifest index cfba0ecda5..3dc995f40e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sto\scheck-in\s[b9190d3da70c4171]\s-\sthe\sagg-with-indexed-expr\soptimization\nrequested\sby\sticket\s[99378177930f87bd]\s-\sthat\scan\scause\san\sincorrect\sanswer\nif\san\saggregate\ssubquery\shas\sa\sGROUP\sBY\sclause,\sand\sthat\sGROUP\sBY\scontains\sa\nterm\sthat\sis\snot\sin\sthe\sresult\sset,\sand\sthe\souter\squery\smakes\suse\sof\sexpression\nindexes.\s\sProblem\sreported\sby\n[forum:/forumpost/a68313d054|forum\spost\sa68313d054]. -D 2023-03-04T15:36:51.218 +C Correct\srendering\sof\serror\smessages\sin\sdemo-worker1.js.\sRemove\ssome\sstray\sEOL\swhitespace. +D 2023-03-05T07:33:11.574 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -505,9 +505,9 @@ F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb0 F ext/wasm/demo-jsstorage.html 409c4be4af5f207fb2877160724b91b33ea36a3cd8c204e8da1acb828ffe588e F ext/wasm/demo-jsstorage.js 44e3ae7ec2483b6c511384c3c290beb6f305c721186bcf5398ca4e00004a06b8 F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98ab22f5786620b3354ed15f -F ext/wasm/demo-worker1-promiser.js b99c550763fa792c204e9a7cceadd976004036d9fc3e22fab7051712e30d207d +F ext/wasm/demo-worker1-promiser.js 51b02509a109e82f623fb4c900c8b48b9a77cc13fbd038396f9a083b86593ae3 F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d -F ext/wasm/demo-worker1.js a619adffc98b75b66c633b00f747b856449a134a9a0357909287d80a182d70fa +F ext/wasm/demo-worker1.js 2c7794d8bc4ab9ecf9cdc2c15de940b11a006942226e441ea41edd458dfc0a26 F ext/wasm/dist.make f55f9c9e1980ea11a59964e59535c66175a17f004d1c2e274522c3366b3a084a F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f F ext/wasm/fiddle.make dbe36b90b8907ae28ecb9c0e9fd8389dbdaecf117ea4fb2ea33864bdfa498a94 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 35f10a06ba81b8a526249729828fff06214e9cf95de418a9081f19d8d69fd657 -R 7369d7a38af682b682af1a066f3ec500 -U drh -Z a08315d4647510add4b36d411a52c34f +P e06973876993926fd56181281d04b8dd504c689abf883fa21a5721cc1d478ea8 +R 68e1c9d17126465fe92388ca20cce3d9 +U stephan +Z 9437532fe63139b2ab32f5bb03c117ba # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d60f674cb2..ed905062a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e06973876993926fd56181281d04b8dd504c689abf883fa21a5721cc1d478ea8 \ No newline at end of file +1d5d515ad97cf61bd679f8c1afc607815c079583fe80264b591c6ef18f56fb8b \ No newline at end of file From eac6e8cb6a177500422de07f75a089cad162c5f2 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 5 Mar 2023 07:44:23 +0000 Subject: [PATCH 057/341] Rename sqlite3-worker1-bundler-friendly.js to sqlite3-worker1-bundler-friendly.mjs and refactor it to work as an ES6 module, based on feedback in [forum post a255f89c2eadf4c4|forum:a255f89c2eadf4c4]. FossilOrigin-Name: af312b131457743d98b84137bd51d9ba60e0daf0bd8f5a66f05956ca35ab68fb --- ext/wasm/GNUmakefile | 2 +- ext/wasm/api/sqlite3-worker1-promiser.c-pp.js | 18 +++++++++++++++--- ext/wasm/api/sqlite3-worker1.c-pp.js | 12 +++++------- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 30 insertions(+), 20 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 13205a6220..a7dd79db18 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -752,7 +752,7 @@ sqlite3-worker1.js.in := $(dir.api)/sqlite3-worker1.c-pp.js sqlite3-worker1-promiser.js.in := $(dir.api)/sqlite3-worker1-promiser.c-pp.js sqlite3-worker1.js := $(dir.dout)/sqlite3-worker1.js sqlite3-worker1-promiser.js := $(dir.dout)/sqlite3-worker1-promiser.js -sqlite3-worker1-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-bundler-friendly.js +sqlite3-worker1-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-bundler-friendly.mjs sqlite3-worker1-promiser-bundler-friendly.js := $(dir.dout)/sqlite3-worker1-promiser-bundler-friendly.js $(eval $(call C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1.js))) $(eval $(call C-PP.FILTER,$(sqlite3-worker1.js.in),$(sqlite3-worker1-bundler-friendly.js),\ diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index 0f1ae39eac..86ac279edd 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -114,7 +114,7 @@ by all client code except that which tests this API. The `row` property contains the row result in the form implied by the `rowMode` option (defaulting to `'array'`). The `rowNumber` is a - 1-based integer value incremented by 1 on each call into th + 1-based integer value incremented by 1 on each call into the callback. At the end of the result set, the same event is fired with @@ -122,6 +122,15 @@ the end of the result set has been reached. Note that the rows arrive via worker-posted messages, with all the implications of that. + + Notable shortcomings: + + - This API was not designed with ES6 modules in mind. Neither Firefox + nor Safari support, as of March 2023, the {type:"module"} flag to the + Worker constructor, so that particular usage is not something we're going + to target for the time being: + + https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker */ self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ // Inspired by: https://stackoverflow.com/a/52439530 @@ -160,7 +169,7 @@ self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ if(msgHandler && msgHandler.onrow){ msgHandler.onrow(ev); return; - } + } if(config.onunhandled) config.onunhandled(arguments[0]); else err("sqlite3Worker1Promiser() unhandled worker message:",ev); return; @@ -239,7 +248,10 @@ self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ self.sqlite3Worker1Promiser.defaultConfig = { worker: function(){ //#if target=es6-bundler-friendly - return new Worker("sqlite3-worker1.js"); + return new Worker("sqlite3-worker1-bundler-friendly.mjs",{ + type: 'module' /* Noting that neither Firefox nor Safari suppor this, + as of this writing. */ + }); //#else let theJs = "sqlite3-worker1.js"; if(this.currentScript){ diff --git a/ext/wasm/api/sqlite3-worker1.c-pp.js b/ext/wasm/api/sqlite3-worker1.c-pp.js index 9e9c3ac426..906a6ce0da 100644 --- a/ext/wasm/api/sqlite3-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1.c-pp.js @@ -31,11 +31,11 @@ - `sqlite3.dir`, if set, treats the given directory name as the directory from which `sqlite3.js` will be loaded. */ -"use strict"; -(()=>{ //#if target=es6-bundler-friendly - importScripts('sqlite3.js'); +import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs'; //#else +"use strict"; +{ const urlParams = new URL(self.location.href).searchParams; let theJs = 'sqlite3.js'; if(urlParams.has('sqlite3.dir')){ @@ -43,8 +43,6 @@ } //console.warn("worker1 theJs =",theJs); importScripts(theJs); +} //#endif - sqlite3InitModule().then((sqlite3)=>{ - sqlite3.initWorker1API(); - }); -})(); +sqlite3InitModule().then(sqlite3 => sqlite3.initWorker1API()); diff --git a/manifest b/manifest index 3dc995f40e..49d8aeeb2d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\srendering\sof\serror\smessages\sin\sdemo-worker1.js.\sRemove\ssome\sstray\sEOL\swhitespace. -D 2023-03-05T07:33:11.574 +C Rename\ssqlite3-worker1-bundler-friendly.js\sto\ssqlite3-worker1-bundler-friendly.mjs\sand\srefactor\sit\sto\swork\sas\san\sES6\smodule,\sbased\son\sfeedback\sin\s[forum\spost\sa255f89c2eadf4c4|forum:a255f89c2eadf4c4]. +D 2023-03-05T07:44:23.375 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -468,7 +468,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 08fb7a6892acfe8801b547920edf3b914c7045a6bc8bfd4b46c0f2d6c0b6ecbc +F ext/wasm/GNUmakefile 7bc0f80ccc1e82ef2c2dd597ac98e8c5a2d3094ee06dc4b05d5bbe82fed03143 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -490,8 +490,8 @@ F ext/wasm/api/sqlite3-v-helper.js 6f6c3e390a72e08b0a5b16a0d567d7af3c04d17283185 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 0dfddd0fcd354817c3d5887decebb8a293cbb926c2639ba09b995a524f1085fb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 223d30c41d811cae8b9f1175fa68f2f1fb3cc056d16ad0def3b0ea5c65757a6c -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js c5ac33e39f21a3481812d7333ca6e18853640d423a01960ca8dbc6e7c5c3c21c -F ext/wasm/api/sqlite3-worker1.c-pp.js 77b3835192469e9da23926ec8f78fb0b114a51d048dc54388709ac22b5c5f0a0 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js f17d5a51dbf804c37999c9814dd068017e5336e09a04012e50d9486fb895c2e4 +F ext/wasm/api/sqlite3-worker1.c-pp.js 51f32a719880c9c5142f11a6cdf232f30dcc4387e361eeb5855fcf7caaa8c318 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c27df8fdac93e F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e06973876993926fd56181281d04b8dd504c689abf883fa21a5721cc1d478ea8 -R 68e1c9d17126465fe92388ca20cce3d9 +P 1d5d515ad97cf61bd679f8c1afc607815c079583fe80264b591c6ef18f56fb8b +R 91c22a1ad29afe8245a97e3ad469bc33 U stephan -Z 9437532fe63139b2ab32f5bb03c117ba +Z dc276f42e7adb34e8efd79eda6e2d27e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ed905062a9..89fc7b418a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d5d515ad97cf61bd679f8c1afc607815c079583fe80264b591c6ef18f56fb8b \ No newline at end of file +af312b131457743d98b84137bd51d9ba60e0daf0bd8f5a66f05956ca35ab68fb \ No newline at end of file From f30da226e3461eb11e67795ad7a28fa8f5a371a3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 6 Mar 2023 19:04:39 +0000 Subject: [PATCH 058/341] Add SQLITE_DBCONFIG_REVERSE_SCANORDER for direct C-language access to the "PRAGMA reverse_unordered_selects" setting. FossilOrigin-Name: 83e84531b46814aea6dad1ce8283cb9f6b90ad52badb60b875ea8f66e4ac0925 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/main.c | 1 + src/shell.c.in | 2 ++ src/sqlite.h.in | 19 +++++++++++++++---- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 49d8aeeb2d..d174bd6ab1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\ssqlite3-worker1-bundler-friendly.js\sto\ssqlite3-worker1-bundler-friendly.mjs\sand\srefactor\sit\sto\swork\sas\san\sES6\smodule,\sbased\son\sfeedback\sin\s[forum\spost\sa255f89c2eadf4c4|forum:a255f89c2eadf4c4]. -D 2023-03-05T07:44:23.375 +C Add\sSQLITE_DBCONFIG_REVERSE_SCANORDER\sfor\sdirect\sC-language\saccess\sto\sthe\n"PRAGMA\sreverse_unordered_selects"\ssetting. +D 2023-03-06T19:04:39.903 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -586,7 +586,7 @@ F src/insert.c 7940fce7d4aa855606432d82d20694e17c18a03956f5f5776f2404e2df7c18a8 F src/json.c c85ed6fce06f43d414b0d7fff64749d43a0dbd1067123ee407bd3a0752454161 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c b04eb648cedc45efe4298e1ef439ac4f0096ae27b5f01accb0a1f49d57789128 -F src/main.c 50b4f14242c2949fd1f3e83aacf088ed2f1eb16d8c1369ee0156957ea070916e +F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -624,8 +624,8 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 2f38b8424d204f79180acc6adf84cf3a35a5176e33bc6b009f9a9f080316d4bb -F src/shell.c.in 10088fb7a12d9e9b2453b1df32f80f68c6618f998044f82619d769e6f32f72ca -F src/sqlite.h.in 36882eee65db22e2fdb68782e7fb9b9ea1dd0488d8e2918b3567915ddb3c9611 +F src/shell.c.in db8355bd674fb3ef335f2ca07d7ad8a7512570104eee3ba520cb259c17912b0a +F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 F src/sqliteInt.h 34034353de20c29e04295c3486bbf22b2a47842c887f65f7176530b4c35fdd17 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1d5d515ad97cf61bd679f8c1afc607815c079583fe80264b591c6ef18f56fb8b -R 91c22a1ad29afe8245a97e3ad469bc33 -U stephan -Z dc276f42e7adb34e8efd79eda6e2d27e +P af312b131457743d98b84137bd51d9ba60e0daf0bd8f5a66f05956ca35ab68fb +R ad7078f08d1f30e27457cbb0b25465a3 +U drh +Z f646de84c6f7cf615b430f5fcccad8cd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 89fc7b418a..238d5abcd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af312b131457743d98b84137bd51d9ba60e0daf0bd8f5a66f05956ca35ab68fb \ No newline at end of file +83e84531b46814aea6dad1ce8283cb9f6b90ad52badb60b875ea8f66e4ac0925 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 23010d5fc4..3f03a5c1ea 100644 --- a/src/main.c +++ b/src/main.c @@ -971,6 +971,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, { SQLITE_DBCONFIG_STMT_SCANSTATUS, SQLITE_StmtScanStatus }, + { SQLITE_DBCONFIG_REVERSE_SCANORDER, SQLITE_ReverseOrder }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ diff --git a/src/shell.c.in b/src/shell.c.in index 944f655622..b76e44f45b 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -8169,6 +8169,8 @@ static int do_meta_command(char *zLine, ShellState *p){ { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE }, { "reset_database", SQLITE_DBCONFIG_RESET_DATABASE }, + { "reverse_scanorder", SQLITE_DBCONFIG_REVERSE_SCANORDER }, + { "stmt_scanstatus", SQLITE_DBCONFIG_STMT_SCANSTATUS }, { "trigger_eqp", SQLITE_DBCONFIG_TRIGGER_EQP }, { "trusted_schema", SQLITE_DBCONFIG_TRUSTED_SCHEMA }, { "writable_schema", SQLITE_DBCONFIG_WRITABLE_SCHEMA }, diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9dd023821c..ec63e52c75 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2456,7 +2456,7 @@ struct sqlite3_mem_methods { ** not considered a bug since SQLite versions 3.3.0 and earlier do not support ** either generated columns or decending indexes. **
- +** ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] **
SQLITE_DBCONFIG_STMT_SCANSTATUS **
The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in @@ -2465,7 +2465,17 @@ struct sqlite3_mem_methods { ** statistics. For statistics to be collected, the flag must be set on ** the database handle both when the SQL statement is prepared and when it ** is stepped. The flag is set (collection of statistics is enabled) -** by default. +** by default.
+** +** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] +**
SQLITE_DBCONFIG_REVERSE_SCANORDER +**
The SQLITE_DBCONFIG_REVERSE_SCANORDER option change the default order +** in which tables and indexes are scanned so that the scans start at the end +** and work toward the beginning rather than starting at the beginning and +** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. This configuration option +** is useful for application testing.
+** ** */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -2486,8 +2496,9 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1018 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1080 /* int int* */ +#define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes From 51a31a3b9cdb4cab6f9c57764f3f216f06f2911f Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 6 Mar 2023 21:38:38 +0000 Subject: [PATCH 059/341] Cause CLI to fail noisily when deserialize option used for non-seekable "file". FossilOrigin-Name: 24bd7e82471925987d924188ce0f80ed4f282b10ea1022e42881a7f529814eb9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 22 ++++++++++++++++------ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d174bd6ab1..c702837d55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_DBCONFIG_REVERSE_SCANORDER\sfor\sdirect\sC-language\saccess\sto\sthe\n"PRAGMA\sreverse_unordered_selects"\ssetting. -D 2023-03-06T19:04:39.903 +C Cause\sCLI\sto\sfail\snoisily\swhen\sdeserialize\soption\sused\sfor\snon-seekable\s"file". +D 2023-03-06T21:38:38.217 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -624,7 +624,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 2f38b8424d204f79180acc6adf84cf3a35a5176e33bc6b009f9a9f080316d4bb -F src/shell.c.in db8355bd674fb3ef335f2ca07d7ad8a7512570104eee3ba520cb259c17912b0a +F src/shell.c.in d64f0d3822aa94508285f2bcce185b12ace06f37c953371cb0f0d835c171753c F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P af312b131457743d98b84137bd51d9ba60e0daf0bd8f5a66f05956ca35ab68fb -R ad7078f08d1f30e27457cbb0b25465a3 -U drh -Z f646de84c6f7cf615b430f5fcccad8cd +P 83e84531b46814aea6dad1ce8283cb9f6b90ad52badb60b875ea8f66e4ac0925 +R 3c62467c095e93d2ca950eedb1046af5 +U larrybr +Z f621c03954d1f84da6a769a9d8f7abaa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 238d5abcd3..befb930ec1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83e84531b46814aea6dad1ce8283cb9f6b90ad52badb60b875ea8f66e4ac0925 \ No newline at end of file +24bd7e82471925987d924188ce0f80ed4f282b10ea1022e42881a7f529814eb9 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index b76e44f45b..6b9e2b3545 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4894,16 +4894,27 @@ static char *readFile(const char *zName, int *pnByte){ long nIn; size_t nRead; char *pBuf; + int rc; if( in==0 ) return 0; - fseek(in, 0, SEEK_END); + rc = fseek(in, 0, SEEK_END); + if( rc!=0 ){ + raw_printf(stderr, "Error: '%s' not seekable\n", zName); + fclose(in); + return 0; + } nIn = ftell(in); rewind(in); pBuf = sqlite3_malloc64( nIn+1 ); - if( pBuf==0 ){ fclose(in); return 0; } + if( pBuf==0 ){ + raw_printf(stderr, "Error: out of memory\n"); + fclose(in); + return 0; + } nRead = fread(pBuf, nIn, 1, in); fclose(in); if( nRead!=1 ){ sqlite3_free(pBuf); + raw_printf(stderr, "Error: cannot read '%s'\n", zName); return 0; } pBuf[nIn] = 0; @@ -5398,9 +5409,9 @@ static void open_db(ShellState *p, int openFlags){ aData = (unsigned char*)readFile(zDbFilename, &nData); }else{ aData = readHexDb(p, &nData); - if( aData==0 ){ - return; - } + } + if( aData==0 ){ + return; } rc = sqlite3_deserialize(p->db, "main", aData, nData, nData, SQLITE_DESERIALIZE_RESIZEABLE | @@ -8039,7 +8050,6 @@ static int do_meta_command(char *zLine, ShellState *p){ raw_printf(stderr, "Usage: .check GLOB-PATTERN\n"); rc = 2; }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ - raw_printf(stderr, "Error: cannot read 'testcase-out.txt'\n"); rc = 2; }else if( testcase_glob(azArg[1],zRes)==0 ){ utf8_printf(stderr, From 5b6ba9b250c4c31f8fb117e1f42086bfa5327ec7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 6 Mar 2023 23:38:44 +0000 Subject: [PATCH 060/341] Improvements to query invariant testing such that it uses the new SQLITE_DBCONFIG_REVERSE_SCANORDER opcode to sqlite3_db_config() to make more accurate judgements about when a query is ambiguous, and hence when query invariant testing is approprate. FossilOrigin-Name: be9ab292cd14889b1c9648b47138260b33fe5be282ff2d90653b1387885a7d02 --- Makefile.in | 40 +++++++++++++++++++++++++++++----------- Makefile.msc | 27 +++++++++++++++++++++++++++ manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/fuzzcheck.c | 6 ++++-- test/fuzzinvariants.c | 40 +++++++++++++++++++--------------------- 6 files changed, 89 insertions(+), 44 deletions(-) diff --git a/Makefile.in b/Makefile.in index 2a71bd2c5c..f278b053e6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -602,17 +602,35 @@ SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC FUZZERSHELL_OPT = FUZZCHECK_OPT += -I$(TOP)/test FUZZCHECK_OPT += -I$(TOP)/ext/recover -FUZZCHECK_OPT += -DSQLITE_OMIT_LOAD_EXTENSION -FUZZCHECK_OPT += -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000 -FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000 -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4 -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS3_PARENTHESIS -FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS5 -FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE -FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY -FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB -FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPT += \ + -DSQLITE_OSS_FUZZ \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_BYTECODE_VTAB \ + -DSQLITE_ENABLE_DESERIALIZE \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_FTS3_PARENTHESIS \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_GEOPOLY \ + -DSQLITE_ENABLE_MATH_FUNCTIONS \ + -DSQLITE_ENABLE_MEMSYS5 \ + -DSQLITE_ENABLE_NORMALIZE \ + -DSQLITE_ENABLE_OFFSET_SQL_FUNC \ + -DSQLITE_ENABLE_PREUPDATE_HOOK \ + -DSQLITE_ENABLE_RTREE \ + -DSQLITE_ENABLE_SESSION \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \ + -DSQLITE_ENABLE_STAT4 \ + -DSQLITE_ENABLE_STMT_SCANSTATUS \ + -DSQLITE_MAX_MEMORY=50000000 \ + -DSQLITE_MAX_MMAP_SIZE=0 \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \ + -DSQLITE_PRIVATE="" + FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c diff --git a/Makefile.msc b/Makefile.msc index 5528b8e722..a8b5d3083b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1671,6 +1671,33 @@ FUZZERSHELL_COMPILE_OPTS = FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DESERIALIZE +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS3_PARENTHESIS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MATH_FUNCTIONS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_NORMALIZE +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PREUPDATE_HOOK +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_SESSION +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMTVTAB +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STAT4 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE="" + FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION diff --git a/manifest b/manifest index d174bd6ab1..c7f872c26f 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Add\sSQLITE_DBCONFIG_REVERSE_SCANORDER\sfor\sdirect\sC-language\saccess\sto\sthe\n"PRAGMA\sreverse_unordered_selects"\ssetting. -D 2023-03-06T19:04:39.903 +C Improvements\sto\squery\sinvariant\stesting\ssuch\sthat\sit\suses\sthe\snew\nSQLITE_DBCONFIG_REVERSE_SCANORDER\sopcode\sto\ssqlite3_db_config()\sto\smake\smore\naccurate\sjudgements\sabout\swhen\sa\squery\sis\sambiguous,\sand\shence\swhen\squery\ninvariant\stesting\sis\sapproprate. +D 2023-03-06T23:38:44.071 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 594f07f8829020a45d825234edf14046544b5bf6bc0057a9d5b83b58686b1c46 +F Makefile.in 73076ae60a143476b6890b5c3b88848fce19ac8caefc3ff5ae97f72694dc946d F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc b0026b61d1242add00c28dbe169271a763fffcb3af6762bcdbeb20c3efe6a69f +F Makefile.msc e3a7faf7da80b891edcc0378eb9b4b1274ae080ee1fd1337c649bfe9af9aa456 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e F VERSION 17f95ae2fdf21f0e9575eb0b0511ea63f15d71dfff431b21c2b4adbfa70cfbbf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1140,7 +1140,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 90d3435c5359029d79b0945110e9c83a529469c03e4380338578c47c5a9ea296 +F test/fuzzcheck.c a3d4967ba9a56b13e097f613409178cfb20796d291ad1d7d532c6677fd5909b9 F test/fuzzdata1.db 3e86d9cf5aea68ddb8e27c02d7dfdaa226347426c7eb814918e4d95475bf8517 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1152,7 +1152,7 @@ F test/fuzzdata8.db f6c2f2af4deaaae0ddb3310d509c2659990794aa653dc501b80a0534c349 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c a153253600b2b33a7d5710d40e89b2ac1373a1912517867fb995a45b2d67dcb8 +F test/fuzzinvariants.c 3ec82ac4277a0c62918f39e7a35893d6fb5ed57c465a938ad18e8e7a69527615 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/gencol1.test e40c94c0d3485c37c23ec8ff42dbb4464cef3f3e41d96ef1abf9c4f2b97936c9 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P af312b131457743d98b84137bd51d9ba60e0daf0bd8f5a66f05956ca35ab68fb -R ad7078f08d1f30e27457cbb0b25465a3 +P 83e84531b46814aea6dad1ce8283cb9f6b90ad52badb60b875ea8f66e4ac0925 +R f88f50a654bf2b41e280183e1005cba0 U drh -Z f646de84c6f7cf615b430f5fcccad8cd +Z 7f97178f4b5f9f04d04c9231829a35aa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 238d5abcd3..3624529277 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83e84531b46814aea6dad1ce8283cb9f6b90ad52badb60b875ea8f66e4ac0925 \ No newline at end of file +be9ab292cd14889b1c9648b47138260b33fe5be282ff2d90653b1387885a7d02 \ No newline at end of file diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 65dd2fa722..9f509306b7 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -1001,12 +1001,14 @@ static int recoverSqlCb(void *pCtx, const char *zSql){ */ static int recoverDatabase(sqlite3 *db){ int rc; /* Return code from this routine */ + const char *zRecoveryDb = ""; /* Name of "recovery" database */ const char *zLAF = "lost_and_found"; /* Name of "lost_and_found" table */ int bFreelist = 1; /* True to scan the freelist */ int bRowids = 1; /* True to restore ROWID values */ - sqlite3_recover *p; /* The recovery object */ + sqlite3_recover *p = 0; /* The recovery object */ p = sqlite3_recover_init_sql(db, "main", recoverSqlCb, 0); + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); @@ -1038,7 +1040,7 @@ static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){ printf("RUNNING-SQL: [%s]\n", zSql); fflush(stdout); } - (*pBtsFlags) &= ~BTS_BADPRAGMA; + (*pBtsFlags) &= BTS_BADPRAGMA; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc==SQLITE_OK ){ int nRow = 0; diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 883f8cdfc9..0a5f607148 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -115,6 +115,8 @@ int fuzz_invariant( if( rc==SQLITE_DONE ){ /* No matching output row found */ sqlite3_stmt *pCk = 0; + int iOrigRSO; + /* This is not a fault if the database file is corrupt, because anything ** can happen with a corrupt database file */ @@ -136,28 +138,24 @@ int fuzz_invariant( } sqlite3_finalize(pCk); - if( sqlite3_strlike("%group%by%",sqlite3_sql(pStmt),0)==0 ){ - /* - ** If there is a GROUP BY clause, it might not cover every term in the - ** output. And then non-covered terms can take on a value from any - ** row in the result set. This can cause differing answers. - */ - goto not_a_fault; + /* + ** If inverting the scan order also results in a miss, assume that the + ** query is ambiguous and do not report a fault. + */ + sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, -1, &iOrigRSO); + sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, !iOrigRSO, 0); + sqlite3_prepare_v2(db, sqlite3_sql(pStmt), -1, &pCk, 0); + sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, iOrigRSO, 0); + while( (rc = sqlite3_step(pCk))==SQLITE_ROW ){ + for(i=0; i=nCol ) break; } - - if( sqlite3_strlike("%limit%)%order%by%", sqlite3_sql(pTestStmt),0)==0 ){ - /* crash-89bd6a6f8c6166e9a4c5f47b3e70b225f69b76c6 - ** Original statement is: - ** - ** SELECT a,b,c* FROM t1 LIMIT 1%5<4 - ** - ** When running: - ** - ** SELECT * FROM (...) ORDER BY 1 - ** - ** A different subset of the rows come out - */ - goto not_a_fault; + sqlite3_finalize(pCk); + if( rc==SQLITE_DONE ){ + sqlite3_finalize(pTestStmt); + return SQLITE_DONE; } /* The original sameValue() comparison assumed a collating sequence From c105b3eccf20e2d525f1f81b6898440f21766d65 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 7 Mar 2023 02:24:43 +0000 Subject: [PATCH 061/341] Fix a couple minor spacing issues in the MSVC makefile. FossilOrigin-Name: 46b3ac6d1fdd9207cdc511d445bb4c33d11102d6e4eb43f119293d62bb7008ff --- Makefile.msc | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index a8b5d3083b..63207301df 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1526,7 +1526,7 @@ TESTSRC = \ $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_test.c \ $(TOP)\ext\rbu\test_rbu.c \ - $(TOP)\ext\session\test_session.c + $(TOP)\ext\session\test_session.c # Statically linked extensions. # @@ -1684,7 +1684,7 @@ FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MATH_FUNCTIONS FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_NORMALIZE -FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC +FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PREUPDATE_HOOK FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_SESSION diff --git a/manifest b/manifest index af7fe43591..a3d2e1dc29 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Repair\san\sunintential\sfork. -D 2023-03-06T23:39:19.905 +C Fix\sa\scouple\sminor\sspacing\sissues\sin\sthe\sMSVC\smakefile. +D 2023-03-07T02:24:43.740 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 73076ae60a143476b6890b5c3b88848fce19ac8caefc3ff5ae97f72694dc946d F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc e3a7faf7da80b891edcc0378eb9b4b1274ae080ee1fd1337c649bfe9af9aa456 +F Makefile.msc 492eace1ce32ec464792ae60a9b67225ddb68ef742f1a44d25f47745c9c59d3d F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e F VERSION 17f95ae2fdf21f0e9575eb0b0511ea63f15d71dfff431b21c2b4adbfa70cfbbf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P be9ab292cd14889b1c9648b47138260b33fe5be282ff2d90653b1387885a7d02 24bd7e82471925987d924188ce0f80ed4f282b10ea1022e42881a7f529814eb9 -R ef635fb3d47a6aeb141c223bc41368c2 -U drh -Z 6a26991404e5e106a1611ca3013f4b88 +P 8b524c849f9844ea624cfe8968225200a57cefa8b7a17ebac2153c0cb3b657d7 +R 85b16b20b35bf622a1d3c52ce74239c4 +U mistachkin +Z da1b337cb038339974cdea1e6ad4a420 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5a6e82c941..58bb32e28a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b524c849f9844ea624cfe8968225200a57cefa8b7a17ebac2153c0cb3b657d7 \ No newline at end of file +46b3ac6d1fdd9207cdc511d445bb4c33d11102d6e4eb43f119293d62bb7008ff \ No newline at end of file From 7272f6d64d88da451f18aa515db06ed6f2b459d4 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 7 Mar 2023 12:59:20 +0000 Subject: [PATCH 062/341] In the JS sqlite3.vfs/vtab utility APIs, use a local reference to StructBinder instead of sqlite3.StructBinder, as that object is removed from the sqlite3 namespace during the final steps of API initialization. Based on feedback from [forum:d19d96183badca70|forum post d19d96183badca70]. FossilOrigin-Name: 0d89885d28b44b1858117a72a180841f4f5f44bcc574fc59a116ca3526325932 --- ext/wasm/api/sqlite3-v-helper.js | 11 +++++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index 10be8ebce4..1fe91c1dee 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -19,6 +19,9 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss3; const vfs = Object.create(null), vtab = Object.create(null); + const StructBinder = sqlite3.StructBinder + /* we require a local alias b/c StructBinder is removed from the sqlite3 + object during the final steps of the API cleanup. */; sqlite3.vfs = vfs; sqlite3.vtab = vtab; @@ -112,7 +115,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const installMethod = function callee( tgt, name, func, applyArgcCheck = callee.installMethodArgcCheck ){ - if(!(tgt instanceof sqlite3.StructBinder.StructType)){ + if(!(tgt instanceof StructBinder.StructType)){ toss("Usage error: target object is-not-a StructType."); }else if(!(func instanceof Function) && !wasm.isPtr(func)){ toss("Usage errror: expecting a Function or WASM pointer to one."); @@ -132,7 +135,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; /* An ondispose() callback for use with - sqlite3.StructBinder-created types. */ + StructBinder-created types. */ callee.removeFuncList = function(){ if(this.ondispose.__removeFuncList){ this.ondispose.__removeFuncList.forEach( @@ -221,7 +224,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ and the first is an object, it's instead equivalent to calling installMethods(this,...arguments). */ - sqlite3.StructBinder.StructType.prototype.installMethod = function callee( + StructBinder.StructType.prototype.installMethod = function callee( name, func, applyArgcCheck = installMethod.installMethodArgcCheck ){ return (arguments.length < 3 && name && 'object'===typeof name) @@ -233,7 +236,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Equivalent to calling installMethods() with a first argument of this object. */ - sqlite3.StructBinder.StructType.prototype.installMethods = function( + StructBinder.StructType.prototype.installMethods = function( methods, applyArgcCheck = installMethod.installMethodArgcCheck ){ return installMethods(this, methods, applyArgcCheck); diff --git a/manifest b/manifest index a3d2e1dc29..a61773815d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sminor\sspacing\sissues\sin\sthe\sMSVC\smakefile. -D 2023-03-07T02:24:43.740 +C In\sthe\sJS\ssqlite3.vfs/vtab\sutility\sAPIs,\suse\sa\slocal\sreference\sto\sStructBinder\sinstead\sof\ssqlite3.StructBinder,\sas\sthat\sobject\sis\sremoved\sfrom\sthe\ssqlite3\snamespace\sduring\sthe\sfinal\ssteps\sof\sAPI\sinitialization.\sBased\son\sfeedback\sfrom\s[forum:d19d96183badca70|forum\spost\sd19d96183badca70]. +D 2023-03-07T12:59:20.234 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -486,7 +486,7 @@ F ext/wasm/api/sqlite3-api-prologue.js 5cc817b67a774bfa3c47d4c2fa484b10b24b5529a F ext/wasm/api/sqlite3-api-worker1.js 9551f04cdfcde354e5a6ccb48951e007d618abb4e95758297b7fd44ccffdf89f F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 7795b84b66a7a8dedc791340709b310bb497c3c72a80bef364fa2a58e2ddae3f -F ext/wasm/api/sqlite3-v-helper.js 6f6c3e390a72e08b0a5b16a0d567d7af3c04d172831853a29d72a6f1dd40ff24 +F ext/wasm/api/sqlite3-v-helper.js 2b7c8b26293127a69138d7474521ee3ae6f1e714dfebe6cfb4632e25b448e86d F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 0dfddd0fcd354817c3d5887decebb8a293cbb926c2639ba09b995a524f1085fb F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 223d30c41d811cae8b9f1175fa68f2f1fb3cc056d16ad0def3b0ea5c65757a6c @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8b524c849f9844ea624cfe8968225200a57cefa8b7a17ebac2153c0cb3b657d7 -R 85b16b20b35bf622a1d3c52ce74239c4 -U mistachkin -Z da1b337cb038339974cdea1e6ad4a420 +P 46b3ac6d1fdd9207cdc511d445bb4c33d11102d6e4eb43f119293d62bb7008ff +R 4a31f99bfa30395371cae1302bc1f756 +U stephan +Z 0f22e03fc38467c39e3c17df73bf7790 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 58bb32e28a..74e3d0c1e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46b3ac6d1fdd9207cdc511d445bb4c33d11102d6e4eb43f119293d62bb7008ff \ No newline at end of file +0d89885d28b44b1858117a72a180841f4f5f44bcc574fc59a116ca3526325932 \ No newline at end of file From 4214cc32ba2082b90dde97495058c45920235bf4 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 7 Mar 2023 19:12:06 +0000 Subject: [PATCH 063/341] Replace use of 'self' in JS code with 'globalThis', as that works in browsers and node environments. Avoid using globalThis.location if it's not set (e.g. in node). Based on feedback in [forum:ac7a94d4f77db235|forum post ac7a94d4f77db235]. Minor JS build tweaks. FossilOrigin-Name: dbbe8f25e58738c10b6192d41f1e3886983871f17631cbc45ce626d3f05a6e26 --- ext/wasm/GNUmakefile | 12 +++--- ext/wasm/api/extern-post-js.c-pp.js | 26 +++++------ ext/wasm/api/pre-js.c-pp.js | 6 +-- ext/wasm/api/sqlite3-api-cleanup.js | 16 +++---- ext/wasm/api/sqlite3-api-glue.js | 10 ++--- ext/wasm/api/sqlite3-api-oo1.js | 4 +- ext/wasm/api/sqlite3-api-prologue.js | 43 +++++++++---------- ext/wasm/api/sqlite3-api-worker1.js | 10 ++--- ext/wasm/api/sqlite3-opfs-async-proxy.js | 24 +++++------ ext/wasm/api/sqlite3-v-helper.js | 2 +- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 38 ++++++++-------- ext/wasm/api/sqlite3-worker1-promiser.c-pp.js | 14 +++--- ext/wasm/api/sqlite3-worker1.c-pp.js | 4 +- ext/wasm/common/whwasmutil.js | 10 ++--- ext/wasm/jaccwabyt/jaccwabyt.js | 2 +- manifest | 40 ++++++++--------- manifest.uuid | 2 +- 17 files changed, 134 insertions(+), 129 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index a7dd79db18..ae8a7d98e8 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -437,7 +437,7 @@ emcc.jsflags += -sSTRICT_JS=0 # STRICT_JS disabled due to: # https://github.com/emscripten-core/emscripten/issues/18610 # TL;DR: does not work with MODULARIZE or EXPORT_ES6 as of version 3.1.31. -emcc.environment := -sENVIRONMENT=web,worker +emcc.environment := -sENVIRONMENT=web,worker,node ######################################################################## # -sINITIAL_MEMORY: How much memory we need to start with is governed # at least in part by whether -sALLOW_MEMORY_GROWTH is enabled. If so, @@ -450,9 +450,9 @@ emcc.environment := -sENVIRONMENT=web,worker # such test results are inconsistent due to browser internals which # are opaque to us. emcc.jsflags += -sALLOW_MEMORY_GROWTH -emcc.INITIAL_MEMORY.128 := 13107200 +emcc.INITIAL_MEMORY.128 := 134217728 emcc.INITIAL_MEMORY.96 := 100663296 -emcc.INITIAL_MEMORY.64 := 64225280 +emcc.INITIAL_MEMORY.64 := 67108864 emcc.INITIAL_MEMORY.32 := 33554432 emcc.INITIAL_MEMORY.16 := 16777216 emcc.INITIAL_MEMORY.8 := 8388608 @@ -469,6 +469,8 @@ emcc.jsflags += -sSTACK_SIZE=512KB # ^^^ ACHTUNG: emsdk 3.1.27 reduced the default stack size from 5MB to # a mere 64KB, which leads to silent memory corruption via the kvvfs # VFS, which requires twice that for its xRead() and xWrite() methods. +# 2023-03: those methods have since been adapted to use a malloc()'d +# buffer. ######################################################################## # $(sqlite3.js.init-func) is the name Emscripten assigns our exported # module init/load function. This symbol name is hard-coded in @@ -537,7 +539,7 @@ emcc.jsflags += -sLLD_REPORT_UNDEFINED $(sqlite3-api-build-version.js): $(bin.version-info) $(MAKEFILE) @echo "Making $@..." @{ \ - echo 'self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){'; \ + echo 'globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){'; \ echo -n ' sqlite3.version = '; \ $(bin.version-info) --json; \ echo ';'; \ @@ -666,7 +668,7 @@ pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) define SETUP_LIB_BUILD_MODE $(info Setting up build [$(1)]: $(4)) c-pp.D.$(1) := $(5) -pre-js.js.$(1) := $$(dir.api)/pre-js.$(1).js +pre-js.js.$(1) := $$(dir.tmp)/pre-js.$(1).js $$(eval $$(call C-PP.FILTER,$$(pre-js.js.in),$$(pre-js.js.$(1)),$$(c-pp.D.$(1)))) post-js.js.$(1) := $$(dir.tmp)/post-js.$(1).js $$(eval $$(call C-PP.FILTER,$$(post-js.js.in),$$(post-js.js.$(1)),$$(c-pp.D.$(1)))) diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index a577a63e1e..14d4349912 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -28,7 +28,7 @@ const toExportForESM = for non-ES6 Module cases but wrong for ES6 modules because those resolve this symbol differently. */ sqlite3InitModule; if(!originalInit){ - throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); + throw new Error("Expecting globalThis.sqlite3InitModule to be defined by the Emscripten build."); } /** We need to add some state which our custom Module.locateFile() @@ -41,11 +41,13 @@ const toExportForESM = into the global scope and delete it when sqlite3InitModule() is called. */ - const initModuleState = self.sqlite3InitModuleState = Object.assign(Object.create(null),{ + const initModuleState = globalThis.sqlite3InitModuleState = Object.assign(Object.create(null),{ moduleScript: self?.document?.currentScript, isWorker: ('undefined' !== typeof WorkerGlobalScope), - location: self.location, - urlParams: new URL(self.location.href).searchParams + location: globalThis.location, + urlParams: globalThis?.location?.href + ? new URL(globalThis.location.href).searchParams + : new URLSearchParams() }); initModuleState.debugModule = initModuleState.urlParams.has('sqlite3.debugModule') @@ -60,14 +62,14 @@ const toExportForESM = initModuleState.sqlite3Dir = li.join('/') + '/'; } - self.sqlite3InitModule = function ff(...args){ + globalThis.sqlite3InitModule = function ff(...args){ //console.warn("Using replaced sqlite3InitModule()",self.location); return originalInit(...args).then((EmscriptenModule)=>{ - if(self.window!==self && + if('undefined'!==typeof WorkerGlobalScope && (EmscriptenModule['ENVIRONMENT_IS_PTHREAD'] || EmscriptenModule['_pthread_self'] || 'function'===typeof threadAlert - || self.location.pathname.endsWith('.worker.js') + || globalThis?.location?.pathname?.endsWith?.('.worker.js') )){ /** Workaround for wasmfs-generated worker, which calls this routine from each individual thread and requires that its @@ -88,10 +90,10 @@ const toExportForESM = throw e; }); }; - self.sqlite3InitModule.ready = originalInit.ready; + globalThis.sqlite3InitModule.ready = originalInit.ready; - if(self.sqlite3InitModuleState.moduleScript){ - const sim = self.sqlite3InitModuleState; + if(globalThis.sqlite3InitModuleState.moduleScript){ + const sim = globalThis.sqlite3InitModuleState; let src = sim.moduleScript.src.split('/'); src.pop(); sim.scriptDir = src.join('/') + '/'; @@ -99,7 +101,7 @@ const toExportForESM = initModuleState.debugModule('sqlite3InitModuleState =',initModuleState); if(0){ console.warn("Replaced sqlite3InitModule()"); - console.warn("self.location.href =",self.location.href); + console.warn("globalThis.location.href =",globalThis.location.href); if('undefined' !== typeof document){ console.warn("document.currentScript.src =", document?.currentScript?.src); @@ -119,7 +121,7 @@ const toExportForESM = /* AMD modules get injected in a way we cannot override, so we can't handle those here. */ //#endif // !target=es6-module - return self.sqlite3InitModule /* required for ESM */; + return globalThis.sqlite3InitModule /* required for ESM */; })(); //#if target=es6-module export default toExportForESM; diff --git a/ext/wasm/api/pre-js.c-pp.js b/ext/wasm/api/pre-js.c-pp.js index a25c7ce774..878f3e0546 100644 --- a/ext/wasm/api/pre-js.c-pp.js +++ b/ext/wasm/api/pre-js.c-pp.js @@ -6,12 +6,12 @@ */ // See notes in extern-post-js.js -const sqlite3InitModuleState = self.sqlite3InitModuleState +const sqlite3InitModuleState = globalThis.sqlite3InitModuleState || Object.assign(Object.create(null),{ debugModule: ()=>{} }); -delete self.sqlite3InitModuleState; -sqlite3InitModuleState.debugModule('self.location =',self.location); +delete globalThis.sqlite3InitModuleState; +sqlite3InitModuleState.debugModule('globalThis.location =',globalThis.location); //#ifnot target=es6-bundler-friendly /** diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index 7c23f8f894..d38b401bf7 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -25,7 +25,7 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build exports: Module['asm'], memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */ }, - self.sqlite3ApiConfig || {} + globalThis.sqlite3ApiConfig || {} ); /** @@ -33,29 +33,29 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build sqlite3ApiBootstrap(). That decision will be revisited at some point, as we really want client code to be able to call this to configure certain parts. Clients may modify - self.sqlite3ApiBootstrap.defaultConfig to tweak the default + globalThis.sqlite3ApiBootstrap.defaultConfig to tweak the default configuration used by a no-args call to sqlite3ApiBootstrap(), but must have first loaded their WASM module in order to be able to provide the necessary configuration state. */ - //console.warn("self.sqlite3ApiConfig = ",self.sqlite3ApiConfig); - self.sqlite3ApiConfig = SABC; + //console.warn("globalThis.sqlite3ApiConfig = ",globalThis.sqlite3ApiConfig); + globalThis.sqlite3ApiConfig = SABC; let sqlite3; try{ - sqlite3 = self.sqlite3ApiBootstrap(); + sqlite3 = globalThis.sqlite3ApiBootstrap(); }catch(e){ console.error("sqlite3ApiBootstrap() error:",e); throw e; }finally{ - delete self.sqlite3ApiBootstrap; - delete self.sqlite3ApiConfig; + delete globalThis.sqlite3ApiBootstrap; + delete globalThis.sqlite3ApiConfig; } Module.sqlite3 = sqlite3 /* Needed for customized sqlite3InitModule() to be able to pass the sqlite3 object off to the client. */; }else{ console.warn("This is not running in an Emscripten module context, so", - "self.sqlite3ApiBootstrap() is _not_ being called due to lack", + "globalThis.sqlite3ApiBootstrap() is _not_ being called due to lack", "of config info for the WASM environment.", "It must be called manually."); } diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 7db23bacc9..1cb00b9419 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -16,13 +16,13 @@ initializes the main API pieces so that the downstream components (e.g. sqlite3-api-oo1.js) have all that they need. */ -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; const toss3 = sqlite3.SQLite3Error.toss; const capi = sqlite3.capi, wasm = sqlite3.wasm, util = sqlite3.util; - self.WhWasmUtilInstaller(wasm); - delete self.WhWasmUtilInstaller; + globalThis.WhWasmUtilInstaller(wasm); + delete globalThis.WhWasmUtilInstaller; if(0){ /** @@ -605,7 +605,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** Install JS<->C struct bindings for the non-opaque struct types we need... */ - sqlite3.StructBinder = self.Jaccwabyt({ + sqlite3.StructBinder = globalThis.Jaccwabyt({ heap: 0 ? wasm.memory : wasm.heap8u, alloc: wasm.alloc, dealloc: wasm.dealloc, @@ -613,7 +613,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ memberPrefix: /* Never change this: this prefix is baked into any amount of code and client-facing docs. */ '$' }); - delete self.Jaccwabyt; + delete globalThis.Jaccwabyt; {// wasm.xWrap() bindings... diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index 914497602e..ac6678c88e 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -12,9 +12,9 @@ This file contains the so-called OO #1 API wrapper for the sqlite3 WASM build. It requires that sqlite3-api-glue.js has already run - and it installs its deliverable as self.sqlite3.oo1. + and it installs its deliverable as globalThis.sqlite3.oo1. */ -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = (...args)=>{throw new Error(args.join(' '))}; const toss3 = (...args)=>{throw new sqlite3.SQLite3Error(...args)}; diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index b08ad7a7ce..1fd5770992 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -29,7 +29,7 @@ exposed by this API. It is intended to be called one time at the end of the API amalgamation process, passed configuration details for the current environment, and then optionally be removed from - the global object using `delete self.sqlite3ApiBootstrap`. + the global object using `delete globalThis.sqlite3ApiBootstrap`. This function is not intended for client-level use. It is intended for use in creating bundles configured for specific WASM @@ -58,7 +58,7 @@ WASM-exported memory. - `bigIntEnabled`: true if BigInt support is enabled. Defaults to - true if `self.BigInt64Array` is available, else false. Some APIs + true if `globalThis.BigInt64Array` is available, else false. Some APIs will throw exceptions if called without BigInt support, as BigInt is required for marshalling C-side int64 into and out of JS. (Sidebar: it is technically possible to add int64 support via @@ -100,8 +100,8 @@ */ 'use strict'; -self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( - apiConfig = (self.sqlite3ApiConfig || sqlite3ApiBootstrap.defaultConfig) +globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( + apiConfig = (globalThis.sqlite3ApiConfig || sqlite3ApiBootstrap.defaultConfig) ){ if(sqlite3ApiBootstrap.sqlite3){ /* already initalized */ console.warn("sqlite3ApiBootstrap() called multiple times.", @@ -117,7 +117,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( -sWASM_BIGINT=1, else it will not. */ return !!Module.HEAPU64; } - return !!self.BigInt64Array; + return !!globalThis.BigInt64Array; })(), debug: console.debug.bind(console), warn: console.warn.bind(console), @@ -772,7 +772,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( isBindableTypedArray, isInt32, isSQLableTypedArray, isTypedArray, typedArrayToString, - isUIThread: ()=>(self.window===self && !!self.document), + isUIThread: ()=>(globalThis.window===globalThis && !!globalThis.document), // is this true for ESM?: 'undefined'===typeof WorkerGlobalScope isSharedTypedArray, toss: function(...args){throw new Error(args.join(' '))}, @@ -1203,9 +1203,9 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( console.error("sqlite3_wasmfs_opfs_dir() can no longer work due "+ "to incompatible WASMFS changes. It will be removed."); if(!pdir - || !self.FileSystemHandle - || !self.FileSystemDirectoryHandle - || !self.FileSystemFileHandle){ + || !globalThis.FileSystemHandle + || !globalThis.FileSystemDirectoryHandle + || !globalThis.FileSystemFileHandle){ return __wasmfsOpfsDir = ""; } try{ @@ -1461,8 +1461,8 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( const rc = Object.create(null); rc.prefix = 'kvvfs-'+which; rc.stores = []; - if('session'===which || ""===which) rc.stores.push(self.sessionStorage); - if('local'===which || ""===which) rc.stores.push(self.localStorage); + if('session'===which || ""===which) rc.stores.push(globalThis.sessionStorage); + if('local'===which || ""===which) rc.stores.push(globalThis.localStorage); return rc; }; @@ -1962,7 +1962,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( return sqlite3; }/*sqlite3ApiBootstrap()*/; /** - self.sqlite3ApiBootstrap.initializers is an internal detail used by + globalThis.sqlite3ApiBootstrap.initializers is an internal detail used by the various pieces of the sqlite3 API's amalgamation process. It must not be modified by client code except when plugging such code into the amalgamation process. @@ -1980,14 +1980,14 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( utilized until the whwasmutil.js part is plugged in via sqlite3-api-glue.js. */ -self.sqlite3ApiBootstrap.initializers = []; +globalThis.sqlite3ApiBootstrap.initializers = []; /** - self.sqlite3ApiBootstrap.initializersAsync is an internal detail + globalThis.sqlite3ApiBootstrap.initializersAsync is an internal detail used by the sqlite3 API's amalgamation process. It must not be modified by client code except when plugging such code into the amalgamation process. - The counterpart of self.sqlite3ApiBootstrap.initializers, + The counterpart of globalThis.sqlite3ApiBootstrap.initializers, specifically for initializers which are asynchronous. All entries in this list must be either async functions, non-async functions which return a Promise, or a Promise. Each function in the list is called @@ -1999,10 +1999,10 @@ self.sqlite3ApiBootstrap.initializers = []; This list is not processed until the client calls sqlite3.asyncPostInit(). This means, for example, that intializers - added to self.sqlite3ApiBootstrap.initializers may push entries to + added to globalThis.sqlite3ApiBootstrap.initializers may push entries to this list. */ -self.sqlite3ApiBootstrap.initializersAsync = []; +globalThis.sqlite3ApiBootstrap.initializersAsync = []; /** Client code may assign sqlite3ApiBootstrap.defaultConfig an object-type value before calling sqlite3ApiBootstrap() (without @@ -2012,13 +2012,12 @@ self.sqlite3ApiBootstrap.initializersAsync = []; an environment-suitable configuration without having to define a new global-scope symbol. */ -self.sqlite3ApiBootstrap.defaultConfig = Object.create(null); +globalThis.sqlite3ApiBootstrap.defaultConfig = Object.create(null); /** Placeholder: gets installed by the first call to - self.sqlite3ApiBootstrap(). However, it is recommended that the + globalThis.sqlite3ApiBootstrap(). However, it is recommended that the caller of sqlite3ApiBootstrap() capture its return value and delete - self.sqlite3ApiBootstrap after calling it. It returns the same + globalThis.sqlite3ApiBootstrap after calling it. It returns the same value which will be stored here. */ -self.sqlite3ApiBootstrap.sqlite3 = undefined; - +globalThis.sqlite3ApiBootstrap.sqlite3 = undefined; diff --git a/ext/wasm/api/sqlite3-api-worker1.js b/ext/wasm/api/sqlite3-api-worker1.js index f82be6cd09..5120ed006a 100644 --- a/ext/wasm/api/sqlite3-api-worker1.js +++ b/ext/wasm/api/sqlite3-api-worker1.js @@ -313,7 +313,7 @@ options.columnNames may be populated by the call to db.exec(). */ -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3.initWorker1API = function(){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; @@ -382,10 +382,10 @@ sqlite3.initWorker1API = function(){ */ post: function(msg,xferList){ if(xferList && xferList.length){ - self.postMessage( msg, Array.from(xferList) ); + globalThis.postMessage( msg, Array.from(xferList) ); xferList.length = 0; }else{ - self.postMessage(msg); + globalThis.postMessage(msg); } }, /** Map of DB IDs to DBs. */ @@ -589,7 +589,7 @@ sqlite3.initWorker1API = function(){ } }/*wMsgHandler*/; - self.onmessage = async function(ev){ + globalThis.onmessage = async function(ev){ ev = ev.data; let result, dbId = ev.dbId, evType = ev.type; const arrivalTime = performance.now(); @@ -637,6 +637,6 @@ sqlite3.initWorker1API = function(){ result: result }, wState.xfer); }; - self.postMessage({type:'sqlite3-api',result:'worker1-ready'}); + globalThis.postMessage({type:'sqlite3-api',result:'worker1-ready'}); }.bind({self, sqlite3}); }); diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js index 1456ae08d2..ddcad8f61c 100644 --- a/ext/wasm/api/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -50,10 +50,10 @@ const wPost = (type,...args)=>postMessage({type, payload:args}); const installAsyncProxy = function(self){ const toss = function(...args){throw new Error(args.join(' '))}; - if(self.window === self){ + if(globalThis.window === globalThis){ toss("This code cannot run from the main thread.", "Load it as a Worker from a separate Worker."); - }else if(!navigator.storage.getDirectory){ + }else if(!navigator?.storage?.getDirectory){ toss("This API requires navigator.storage.getDirectory."); } @@ -106,8 +106,8 @@ const installAsyncProxy = function(self){ w += m.wait; m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; } - console.log(self.location.href, - "metrics for",self.location.href,":\n", + console.log(globalThis?.location?.href, + "metrics for",globalThis?.location?.href,":\n", metrics, "\nTotal of",n,"op(s) for",t,"ms", "approx",w,"ms spent waiting on OPFS APIs."); @@ -843,7 +843,7 @@ const installAsyncProxy = function(self){ navigator.storage.getDirectory().then(function(d){ state.rootDir = d; - self.onmessage = function({data}){ + globalThis.onmessage = function({data}){ switch(data.type){ case 'opfs-async-init':{ /* Receive shared state from synchronous partner */ @@ -880,17 +880,17 @@ const installAsyncProxy = function(self){ wPost('opfs-async-loaded'); }).catch((e)=>error("error initializing OPFS asyncer:",e)); }/*installAsyncProxy()*/; -if(!self.SharedArrayBuffer){ +if(!globalThis.SharedArrayBuffer){ wPost('opfs-unavailable', "Missing SharedArrayBuffer API.", "The server must emit the COOP/COEP response headers to enable that."); -}else if(!self.Atomics){ +}else if(!globalThis.Atomics){ wPost('opfs-unavailable', "Missing Atomics API.", "The server must emit the COOP/COEP response headers to enable that."); -}else if(!self.FileSystemHandle || - !self.FileSystemDirectoryHandle || - !self.FileSystemFileHandle || - !self.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator.storage.getDirectory){ +}else if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ wPost('opfs-unavailable',"Missing required OPFS APIs."); }else{ installAsyncProxy(self); diff --git a/ext/wasm/api/sqlite3-v-helper.js b/ext/wasm/api/sqlite3-v-helper.js index 1fe91c1dee..80ab7c5b04 100644 --- a/ext/wasm/api/sqlite3-v-helper.js +++ b/ext/wasm/api/sqlite3-v-helper.js @@ -15,7 +15,7 @@ with its virtual table counterpart, sqlite3.vtab. */ 'use strict'; -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const wasm = sqlite3.wasm, capi = sqlite3.capi, toss = sqlite3.util.toss3; const vfs = Object.create(null), vtab = Object.create(null); diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index c709e8b1ad..09f7f8bdb5 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -18,7 +18,7 @@ after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js. */ 'use strict'; -self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /** installOpfsVfs() returns a Promise which, on success, installs an sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs @@ -76,23 +76,23 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `opfs` property, containing several OPFS-specific utilities. */ const installOpfsVfs = function callee(options){ - if(!self.SharedArrayBuffer - || !self.Atomics){ + if(!globalThis.SharedArrayBuffer + || !globalThis.Atomics){ return Promise.reject( new Error("Cannot install OPFS: Missing SharedArrayBuffer and/or Atomics. "+ "The server must emit the COOP/COEP response headers to enable those. "+ "See https://sqlite.org/wasm/doc/trunk/persistence.md#coop-coep") ); - }else if(self.window===self && self.document){ + }else if('undefined'===typeof WorkerGlobalScope){ return Promise.reject( new Error("The OPFS sqlite3_vfs cannot run in the main thread "+ "because it requires Atomics.wait().") ); - }else if(!self.FileSystemHandle || - !self.FileSystemDirectoryHandle || - !self.FileSystemFileHandle || - !self.FileSystemFileHandle.prototype.createSyncAccessHandle || - !navigator.storage.getDirectory){ + }else if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ return Promise.reject( new Error("Missing required OPFS APIs.") ); @@ -100,7 +100,7 @@ const installOpfsVfs = function callee(options){ if(!options || 'object'!==typeof options){ options = Object.create(null); } - const urlParams = new URL(self.location.href).searchParams; + const urlParams = new URL(globalThis.location.href).searchParams; if(undefined===options.verbose){ options.verbose = urlParams.has('opfs-verbose') ? (+urlParams.get('opfs-verbose') || 2) : 1; @@ -112,7 +112,7 @@ const installOpfsVfs = function callee(options){ options.proxyUri = callee.defaultProxyUri; } - //sqlite3.config.warn("OPFS options =",options,self.location); + //sqlite3.config.warn("OPFS options =",options,globalThis.location); if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); @@ -149,11 +149,11 @@ const installOpfsVfs = function callee(options){ Returns true if _this_ thread has access to the OPFS APIs. */ const thisThreadHasOPFS = ()=>{ - return self.FileSystemHandle && - self.FileSystemDirectoryHandle && - self.FileSystemFileHandle && - self.FileSystemFileHandle.prototype.createSyncAccessHandle && - navigator.storage.getDirectory; + return globalThis.FileSystemHandle && + globalThis.FileSystemDirectoryHandle && + globalThis.FileSystemFileHandle && + globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle && + navigator?.storage?.getDirectory; }; /** @@ -171,8 +171,8 @@ const installOpfsVfs = function callee(options){ m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; } - sqlite3.config.log(self.location.href, - "metrics for",self.location.href,":",metrics, + sqlite3.config.log(globalThis.location.href, + "metrics for",globalThis.location.href,":",metrics, "\nTotal of",n,"op(s) for",t, "ms (incl. "+w+" ms of waiting on the async side)"); sqlite3.config.log("Serialization metrics:",metrics.s11n); @@ -1311,7 +1311,7 @@ const installOpfsVfs = function callee(options){ }/*installOpfsVfs()*/; installOpfsVfs.defaultProxyUri = "sqlite3-opfs-async-proxy.js"; -self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ +globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ try{ let proxyJs = installOpfsVfs.defaultProxyUri; if(sqlite3.scriptInfo.sqlite3Dir){ diff --git a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js index 86ac279edd..48a74d4727 100644 --- a/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -132,7 +132,7 @@ https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker */ -self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ +globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ // Inspired by: https://stackoverflow.com/a/52439530 if(1===arguments.length && 'function'===typeof arguments[0]){ const f = config; @@ -245,7 +245,7 @@ self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ return p; }; }/*sqlite3Worker1Promiser()*/; -self.sqlite3Worker1Promiser.defaultConfig = { +globalThis.sqlite3Worker1Promiser.defaultConfig = { worker: function(){ //#if target=es6-bundler-friendly return new Worker("sqlite3-worker1-bundler-friendly.mjs",{ @@ -259,17 +259,17 @@ self.sqlite3Worker1Promiser.defaultConfig = { src.pop(); theJs = src.join('/')+'/' + theJs; //sqlite3.config.warn("promiser currentScript, theJs =",this.currentScript,theJs); - }else{ - //sqlite3.config.warn("promiser self.location =",self.location); - const urlParams = new URL(self.location.href).searchParams; + }else if(globalThis.location){ + //sqlite3.config.warn("promiser globalThis.location =",globalThis.location); + const urlParams = new URL(globalThis.location.href).searchParams; if(urlParams.has('sqlite3.dir')){ theJs = urlParams.get('sqlite3.dir') + '/' + theJs; } } - return new Worker(theJs + self.location.search); + return new Worker(theJs + globalThis.location.search); //#endif }.bind({ - currentScript: self?.document?.currentScript + currentScript: globalThis?.document?.currentScript }), onerror: (...args)=>console.error('worker1 promiser error',...args) }; diff --git a/ext/wasm/api/sqlite3-worker1.c-pp.js b/ext/wasm/api/sqlite3-worker1.c-pp.js index 906a6ce0da..f260422309 100644 --- a/ext/wasm/api/sqlite3-worker1.c-pp.js +++ b/ext/wasm/api/sqlite3-worker1.c-pp.js @@ -36,7 +36,9 @@ import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs'; //#else "use strict"; { - const urlParams = new URL(self.location.href).searchParams; + const urlParams = globalThis.location + ? new URL(self.location.href).searchParams + : new URLSearchParams(); let theJs = 'sqlite3.js'; if(urlParams.has('sqlite3.dir')){ theJs = urlParams.get('sqlite3.dir') + '/' + theJs; diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index e50210206f..4899799412 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -45,8 +45,8 @@ Intended usage: ``` - self.WhWasmUtilInstaller(appObject); - delete self.WhWasmUtilInstaller; + globalThis.WhWasmUtilInstaller(appObject); + delete globalThis.WhWasmUtilInstaller; ``` Its global-scope symbol is intended only to provide an easy way to @@ -171,7 +171,7 @@ https://fossil.wanderinghorse.net/r/jaccwabbyt/file/common/whwasmutil.js */ -self.WhWasmUtilInstaller = function(target){ +globalThis.WhWasmUtilInstaller = function(target){ 'use strict'; if(undefined===target.bigIntEnabled){ target.bigIntEnabled = !!self['BigInt64Array']; @@ -2194,7 +2194,7 @@ self.WhWasmUtilInstaller = function(target){ Error handling is up to the caller, who may attach a `catch()` call to the promise. */ -self.WhWasmUtilInstaller.yawl = function(config){ +globalThis.WhWasmUtilInstaller.yawl = function(config){ const wfetch = ()=>fetch(config.uri, {credentials: 'same-origin'}); const wui = this; const finalThen = function(arg){ @@ -2240,4 +2240,4 @@ self.WhWasmUtilInstaller.yawl = function(config){ .then(finalThen); }; return loadWasm; -}.bind(self.WhWasmUtilInstaller)/*yawl()*/; +}.bind(globalThis.WhWasmUtilInstaller)/*yawl()*/; diff --git a/ext/wasm/jaccwabyt/jaccwabyt.js b/ext/wasm/jaccwabyt/jaccwabyt.js index d4ec719fb5..bde7d051e8 100644 --- a/ext/wasm/jaccwabyt/jaccwabyt.js +++ b/ext/wasm/jaccwabyt/jaccwabyt.js @@ -19,7 +19,7 @@ */ 'use strict'; -self.Jaccwabyt = function StructBinderFactory(config){ +globalThis.Jaccwabyt = function StructBinderFactory(config){ /* ^^^^ it is recommended that clients move that object into wherever they'd like to have it and delete the self-held copy ("self" being the global window or worker object). This API does not require the diff --git a/manifest b/manifest index a61773815d..e54b24d081 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sJS\ssqlite3.vfs/vtab\sutility\sAPIs,\suse\sa\slocal\sreference\sto\sStructBinder\sinstead\sof\ssqlite3.StructBinder,\sas\sthat\sobject\sis\sremoved\sfrom\sthe\ssqlite3\snamespace\sduring\sthe\sfinal\ssteps\sof\sAPI\sinitialization.\sBased\son\sfeedback\sfrom\s[forum:d19d96183badca70|forum\spost\sd19d96183badca70]. -D 2023-03-07T12:59:20.234 +C Replace\suse\sof\s'self'\sin\sJS\scode\swith\s'globalThis',\sas\sthat\sworks\sin\sbrowsers\sand\snode\senvironments.\sAvoid\susing\sglobalThis.location\sif\sit's\snot\sset\s(e.g.\sin\snode).\sBased\son\sfeedback\sin\s[forum:ac7a94d4f77db235|forum\spost\sac7a94d4f77db235].\sMinor\sJS\sbuild\stweaks. +D 2023-03-07T19:12:06.088 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -468,37 +468,37 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 7bc0f80ccc1e82ef2c2dd597ac98e8c5a2d3094ee06dc4b05d5bbe82fed03143 +F ext/wasm/GNUmakefile 6c0c0e9d3b6ef2090f230bc47f8da5d9614fbdefacb72e8bafcc9a41d035605b F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 77a2f1f2fc60a35def7455dffc8d3f2c56385d6ac5c6cecc60fa938252ea2c54 -F ext/wasm/api/extern-post-js.c-pp.js 44a3a169f55a8dba42cf688954b2625b9b9e6174f2ff02d4918a2ca8c3beab7f +F ext/wasm/api/extern-post-js.c-pp.js 5c4997d3442756e4e4819303fa4a7045de4a2a7b79c3ad6c26cdcf1d9141fac6 F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 -F ext/wasm/api/pre-js.c-pp.js 9ece5de1bb0509f0a8a360712fcc9c1291b9516c0be5bd66acedd6edbcec37a1 -F ext/wasm/api/sqlite3-api-cleanup.js 2d63eb84267a1d15ce002e083d6396a521471da8af3afa76846d50f39a54d65e -F ext/wasm/api/sqlite3-api-glue.js 0a93e58aabf52b32ddccbb107a1fd4552f2505e103ab63396c4d0a0743704785 -F ext/wasm/api/sqlite3-api-oo1.js 9b50c188513c70438a497914089cfeac79b6ac2d73501775538f9e467325ea15 -F ext/wasm/api/sqlite3-api-prologue.js 5cc817b67a774bfa3c47d4c2fa484b10b24b5529a66094b35546f3ebba1ef646 -F ext/wasm/api/sqlite3-api-worker1.js 9551f04cdfcde354e5a6ccb48951e007d618abb4e95758297b7fd44ccffdf89f +F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 +F ext/wasm/api/sqlite3-api-cleanup.js cc21e3486da748463e02bbe51e2464c6ac136587cdfd5aa00cd0b5385f6ca808 +F ext/wasm/api/sqlite3-api-glue.js 32091c2730ecef8f5795c3527d3db6b7bbf054d12e27311ce2da59db559b2e0e +F ext/wasm/api/sqlite3-api-oo1.js 2691a34a741015127b210954a1b9586764d3ff0c8a20f00fd15c00f339ecc79f +F ext/wasm/api/sqlite3-api-prologue.js df8646e4f92b8b09cef255da3530e11dc264a2e8d53b0e78daa2ee04f99c584d +F ext/wasm/api/sqlite3-api-worker1.js 838d756ef059c1f9cfd96f5c3fd3579f1c9be1216efbd1c0a8c086264f96d307 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 -F ext/wasm/api/sqlite3-opfs-async-proxy.js 7795b84b66a7a8dedc791340709b310bb497c3c72a80bef364fa2a58e2ddae3f -F ext/wasm/api/sqlite3-v-helper.js 2b7c8b26293127a69138d7474521ee3ae6f1e714dfebe6cfb4632e25b448e86d -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 0dfddd0fcd354817c3d5887decebb8a293cbb926c2639ba09b995a524f1085fb +F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f +F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 26f6240170d415726d9cfe2fa7a0163e153775e1a74fa91c9ba5446502c71097 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 223d30c41d811cae8b9f1175fa68f2f1fb3cc056d16ad0def3b0ea5c65757a6c -F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js f17d5a51dbf804c37999c9814dd068017e5336e09a04012e50d9486fb895c2e4 -F ext/wasm/api/sqlite3-worker1.c-pp.js 51f32a719880c9c5142f11a6cdf232f30dcc4387e361eeb5855fcf7caaa8c318 +F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc +F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 F ext/wasm/batch-runner.js 0dad6a02ad796f1003d3b7048947d275c4d6277f63767b8e685c27df8fdac93e F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25b4 F ext/wasm/common/SqliteTestUtil.js d8bf97ecb0705a2299765c8fc9e11b1a5ac7f10988bbf375a6558b7ca287067b F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css 0ff15602a3ab2bad8aef2c3bd120c7ee3fd1c2054ad2ace7e214187ae68d926f -F ext/wasm/common/whwasmutil.js cad510071533dbe47787bce53f2e0dcab5b05d2dde1dbe477d8fb04e00d4e8c4 +F ext/wasm/common/whwasmutil.js 749a1f81f85835e9a384e9706f2a955a7158f2b8cc9da33f41105cac7775a305 F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js ebae30756585bca655b4ab2553ec9236a87c23ad24fc8652115dcedb06d28df6 @@ -517,7 +517,7 @@ F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e F ext/wasm/index.html dd900891844caebd9cadbddd704f66bd841d7c12fd69ce5af490e2c10fb49f45 -F ext/wasm/jaccwabyt/jaccwabyt.js 06f2ef1ad640c26c593def3d960336e9bb789819b920516480895c38ed5f58fa +F ext/wasm/jaccwabyt/jaccwabyt.js 8287c0537fa0750414edbe75ce64668a81c8716df5ec4c3e6bb4f11bd1c36031 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10 F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 46b3ac6d1fdd9207cdc511d445bb4c33d11102d6e4eb43f119293d62bb7008ff -R 4a31f99bfa30395371cae1302bc1f756 +P 0d89885d28b44b1858117a72a180841f4f5f44bcc574fc59a116ca3526325932 +R 422bfcde04e8717cdbd839c41655b87a U stephan -Z 0f22e03fc38467c39e3c17df73bf7790 +Z b769421e5ef0157ab70f4e43f4db6aad # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 74e3d0c1e2..38d9047603 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d89885d28b44b1858117a72a180841f4f5f44bcc574fc59a116ca3526325932 \ No newline at end of file +dbbe8f25e58738c10b6192d41f1e3886983871f17631cbc45ce626d3f05a6e26 \ No newline at end of file From 416e9c1908f748d1a4c533c6e3b2d7f1a5b025cc Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 7 Mar 2023 19:23:36 +0000 Subject: [PATCH 064/341] Improve how sqlite3.initWorker1API() determines whether it's running in a Worker thread. Based on feedback in [forum:ac7a94d4f77db235|forum post ac7a94d4f77db235]. FossilOrigin-Name: 2f712b836a0dafd0b2ee6ba4eaa9caa35e49375c7ad0562477e0f2d3b086ec03 --- ext/wasm/api/sqlite3-api-worker1.js | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-worker1.js b/ext/wasm/api/sqlite3-api-worker1.js index 5120ed006a..d1c63c96ee 100644 --- a/ext/wasm/api/sqlite3-api-worker1.js +++ b/ext/wasm/api/sqlite3-api-worker1.js @@ -317,7 +317,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ sqlite3.initWorker1API = function(){ 'use strict'; const toss = (...args)=>{throw new Error(args.join(' '))}; - if('function' !== typeof importScripts){ + if(!(globalThis.WorkerGlobalScope instanceof Function)){ toss("initWorker1API() must be run from a Worker thread."); } const self = this.self; diff --git a/manifest b/manifest index e54b24d081..fff8092263 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\suse\sof\s'self'\sin\sJS\scode\swith\s'globalThis',\sas\sthat\sworks\sin\sbrowsers\sand\snode\senvironments.\sAvoid\susing\sglobalThis.location\sif\sit's\snot\sset\s(e.g.\sin\snode).\sBased\son\sfeedback\sin\s[forum:ac7a94d4f77db235|forum\spost\sac7a94d4f77db235].\sMinor\sJS\sbuild\stweaks. -D 2023-03-07T19:12:06.088 +C Improve\show\ssqlite3.initWorker1API()\sdetermines\swhether\sit's\srunning\sin\sa\sWorker\sthread.\sBased\son\sfeedback\sin\s[forum:ac7a94d4f77db235|forum\spost\sac7a94d4f77db235]. +D 2023-03-07T19:23:36.571 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -483,7 +483,7 @@ F ext/wasm/api/sqlite3-api-cleanup.js cc21e3486da748463e02bbe51e2464c6ac136587cd F ext/wasm/api/sqlite3-api-glue.js 32091c2730ecef8f5795c3527d3db6b7bbf054d12e27311ce2da59db559b2e0e F ext/wasm/api/sqlite3-api-oo1.js 2691a34a741015127b210954a1b9586764d3ff0c8a20f00fd15c00f339ecc79f F ext/wasm/api/sqlite3-api-prologue.js df8646e4f92b8b09cef255da3530e11dc264a2e8d53b0e78daa2ee04f99c584d -F ext/wasm/api/sqlite3-api-worker1.js 838d756ef059c1f9cfd96f5c3fd3579f1c9be1216efbd1c0a8c086264f96d307 +F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d660194ddfc51eab1c5b9bf F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d89885d28b44b1858117a72a180841f4f5f44bcc574fc59a116ca3526325932 -R 422bfcde04e8717cdbd839c41655b87a +P dbbe8f25e58738c10b6192d41f1e3886983871f17631cbc45ce626d3f05a6e26 +R 88242176303c533a7f2325aa7f3c6c50 U stephan -Z b769421e5ef0157ab70f4e43f4db6aad +Z f63a46c4cc22a22c8918188d2a3e0439 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 38d9047603..1dceb579ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dbbe8f25e58738c10b6192d41f1e3886983871f17631cbc45ce626d3f05a6e26 \ No newline at end of file +2f712b836a0dafd0b2ee6ba4eaa9caa35e49375c7ad0562477e0f2d3b086ec03 \ No newline at end of file From bacb8aeb3502017b907729a0774785af28887407 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 7 Mar 2023 19:39:18 +0000 Subject: [PATCH 065/341] A proposed change to [44135d6ea84f7ba6] that retains the historical datatype ("INT", not "NUM") for a table created as follows: "CREATE TABLE t1 AS SELECT CAST(123 AS INT) AS value;". FossilOrigin-Name: a0e54fe2058a4ac1e794b2491e424c60dfa42c6781b1cfd83c5db65c28c11c71 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/select.c | 5 +++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a61773815d..aee9392ccb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sJS\ssqlite3.vfs/vtab\sutility\sAPIs,\suse\sa\slocal\sreference\sto\sStructBinder\sinstead\sof\ssqlite3.StructBinder,\sas\sthat\sobject\sis\sremoved\sfrom\sthe\ssqlite3\snamespace\sduring\sthe\sfinal\ssteps\sof\sAPI\sinitialization.\sBased\son\sfeedback\sfrom\s[forum:d19d96183badca70|forum\spost\sd19d96183badca70]. -D 2023-03-07T12:59:20.234 +C A\sproposed\schange\sto\s[44135d6ea84f7ba6]\sthat\sretains\sthe\nhistorical\sdatatype\s("INT",\snot\s"NUM")\sfor\sa\stable\screated\sas\sfollows:\n"CREATE\sTABLE\st1\sAS\sSELECT\sCAST(123\sAS\sINT)\sAS\svalue;". +D 2023-03-07T19:39:18.271 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -623,7 +623,7 @@ F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 2f38b8424d204f79180acc6adf84cf3a35a5176e33bc6b009f9a9f080316d4bb +F src/select.c 444a4a243c6fac42ecdb6682400be8de448682f261e853445a44e92b3d552d11 F src/shell.c.in d64f0d3822aa94508285f2bcce185b12ace06f37c953371cb0f0d835c171753c F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2048,8 +2048,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 46b3ac6d1fdd9207cdc511d445bb4c33d11102d6e4eb43f119293d62bb7008ff -R 4a31f99bfa30395371cae1302bc1f756 -U stephan -Z 0f22e03fc38467c39e3c17df73bf7790 +P 0d89885d28b44b1858117a72a180841f4f5f44bcc574fc59a116ca3526325932 +R beb23f11e03187a80d89cee523b2f1b3 +T *branch * flexnum-proposed-fix +T *sym-flexnum-proposed-fix * +T -sym-trunk * +U drh +Z 22391db7c3b1c9e948d466993fddf217 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 74e3d0c1e2..0e591cb9a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d89885d28b44b1858117a72a180841f4f5f44bcc574fc59a116ca3526325932 \ No newline at end of file +a0e54fe2058a4ac1e794b2491e424c60dfa42c6781b1cfd83c5db65c28c11c71 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2906a4895c..bdad6bfae4 100644 --- a/src/select.c +++ b/src/select.c @@ -2332,8 +2332,6 @@ void sqlite3SubqueryColumnTypes( pCol->affinity = sqlite3ExprAffinity(p); if( pCol->affinity<=SQLITE_AFF_NONE ){ pCol->affinity = aff; - }else if( pCol->affinity>=SQLITE_AFF_NUMERIC && p->op==TK_CAST ){ - pCol->affinity = SQLITE_AFF_FLEXNUM; } if( pCol->affinity>=SQLITE_AFF_TEXT && pSelect->pNext ){ int m = 0; @@ -2347,6 +2345,9 @@ void sqlite3SubqueryColumnTypes( if( pCol->affinity>=SQLITE_AFF_NUMERIC && (m&0x02)!=0 ){ pCol->affinity = SQLITE_AFF_BLOB; } + if( pCol->affinity>=SQLITE_AFF_NUMERIC && p->op==TK_CAST ){ + pCol->affinity = SQLITE_AFF_FLEXNUM; + } } zType = columnType(&sNC, p, 0, 0, 0); if( zType==0 || pCol->affinity!=sqlite3AffinityType(zType, 0) ){ From cad225d6fccd58eeb00bacd9d548fdfa21721924 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 7 Mar 2023 23:47:38 +0000 Subject: [PATCH 066/341] Fix a bug introduced 4 days ago by [e95439119ac200cb]: do not set the Expr.affExpr field of a generated column expression if the expression is a RAISE() function, as affExpr has a different meaning for RAISE. [forum:/forumpost/b312e075b5|Forum post b312e075b5]. FossilOrigin-Name: 1096b5a7cc8104db01f8820ace47020baad2f12e6711e3a7b4514ed1becc7b66 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 2 +- test/gencol1.test | 9 +++++++++ 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index fff8092263..862003bd55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\show\ssqlite3.initWorker1API()\sdetermines\swhether\sit's\srunning\sin\sa\sWorker\sthread.\sBased\son\sfeedback\sin\s[forum:ac7a94d4f77db235|forum\spost\sac7a94d4f77db235]. -D 2023-03-07T19:23:36.571 +C Fix\sa\sbug\sintroduced\s4\sdays\sago\sby\s[e95439119ac200cb]:\sdo\snot\sset\sthe\nExpr.affExpr\sfield\sof\sa\sgenerated\scolumn\sexpression\sif\sthe\sexpression\sis\sa\nRAISE()\sfunction,\sas\saffExpr\shas\sa\sdifferent\smeaning\sfor\sRAISE.\n[forum:/forumpost/b312e075b5|Forum\spost\sb312e075b5]. +D 2023-03-07T23:47:38.041 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -565,7 +565,7 @@ F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca F src/btree.c c547e099f853de61835ff45da8d956a932d02ecf1ffd472a1f2a103b83e6dd40 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e -F src/build.c 542a04532ae810a58366c47453de4fc1ca9fa6e6b2d920b6d639cdd1a1831309 +F src/build.c 4fed662d383527c808d85f53b9c544ead425ac8b9c7cb38501a64e3797921d41 F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d @@ -1154,7 +1154,7 @@ F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc F test/fuzzinvariants.c 3ec82ac4277a0c62918f39e7a35893d6fb5ed57c465a938ad18e8e7a69527615 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c -F test/gencol1.test e40c94c0d3485c37c23ec8ff42dbb4464cef3f3e41d96ef1abf9c4f2b97936c9 +F test/gencol1.test aef8b0670abd4b1ae4cae786b15a43758d86f6cd9f12b381d45d96bb51e597c9 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 F test/having.test a89236dd8d55aa50c4805f82ac9daf64d477a44d712d8209c118978d0ca21ec9 F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dbbe8f25e58738c10b6192d41f1e3886983871f17631cbc45ce626d3f05a6e26 -R 88242176303c533a7f2325aa7f3c6c50 -U stephan -Z f63a46c4cc22a22c8918188d2a3e0439 +P 2f712b836a0dafd0b2ee6ba4eaa9caa35e49375c7ad0562477e0f2d3b086ec03 +R 7932af26b4c2f8946f0c8d4e43a0814e +U drh +Z 65b72ddc0fa6d0ebce5a1655fb619090 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1dceb579ae..0dd62a5775 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f712b836a0dafd0b2ee6ba4eaa9caa35e49375c7ad0562477e0f2d3b086ec03 \ No newline at end of file +1096b5a7cc8104db01f8820ace47020baad2f12e6711e3a7b4514ed1becc7b66 \ No newline at end of file diff --git a/src/build.c b/src/build.c index c91e7bdd6a..6e3b7b81f8 100644 --- a/src/build.c +++ b/src/build.c @@ -2012,7 +2012,7 @@ void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType){ ** turn it into one by adding a unary "+" operator. */ pExpr = sqlite3PExpr(pParse, TK_UPLUS, pExpr, 0); } - if( pExpr ) pExpr->affExpr = pCol->affinity; + if( pExpr && pExpr->op!=TK_RAISE ) pExpr->affExpr = pCol->affinity; sqlite3ColumnSetExpr(pParse, pTab, pCol, pExpr); pExpr = 0; goto generated_done; diff --git a/test/gencol1.test b/test/gencol1.test index 266eee18b4..f3fbb0dfba 100644 --- a/test/gencol1.test +++ b/test/gencol1.test @@ -660,4 +660,13 @@ do_execsql_test gencol1-23.4 { # ^^^^^^^^^^--- Must reference the original table in this case because # of the different datatype on column b. +# 2023-03-07 https://sqlite.org/forum/forumpost/b312e075b5 +# +do_execsql_test gencol1-23.5 { + CREATE TABLE v0(c1 INT, c2 AS (RAISE(IGNORE))); +} +do_catchsql_test gencol1-23.6 { + SELECT * FROM v0; +} {1 {RAISE() may only be used within a trigger-program}} + finish_test From 217e860053262181d175c5e209bec7d0da194c9f Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 8 Mar 2023 00:47:53 +0000 Subject: [PATCH 067/341] Fix a problem in the count-of-view optimization that can lead to incorrect bytecode. dbsqlfuzz 23d782160b71c3f8f535ccb2da313dfc8eb8c631. FossilOrigin-Name: f45009533a79a59b302598ee2545ef787c51d0128f4e1dca60dd83589f660619 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 + test/countofview.test | 13 +++++++++++++ 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 862003bd55..7bfa0a5377 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sintroduced\s4\sdays\sago\sby\s[e95439119ac200cb]:\sdo\snot\sset\sthe\nExpr.affExpr\sfield\sof\sa\sgenerated\scolumn\sexpression\sif\sthe\sexpression\sis\sa\nRAISE()\sfunction,\sas\saffExpr\shas\sa\sdifferent\smeaning\sfor\sRAISE.\n[forum:/forumpost/b312e075b5|Forum\spost\sb312e075b5]. -D 2023-03-07T23:47:38.041 +C Fix\sa\sproblem\sin\sthe\scount-of-view\soptimization\sthat\scan\slead\sto\sincorrect\nbytecode.\s\sdbsqlfuzz\s23d782160b71c3f8f535ccb2da313dfc8eb8c631. +D 2023-03-08T00:47:53.498 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -623,7 +623,7 @@ F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 2f38b8424d204f79180acc6adf84cf3a35a5176e33bc6b009f9a9f080316d4bb +F src/select.c 0bbc1eb159d8b54fd356de70f778bd1c758c2ccf4fabfca498b2a56481aff2b1 F src/shell.c.in d64f0d3822aa94508285f2bcce185b12ace06f37c953371cb0f0d835c171753c F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -893,7 +893,7 @@ F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 -F test/countofview.test 5773d192f67bbf271f53ebb1f3292f435c9c7ae91ee0be83f81591c19a983604 +F test/countofview.test 5ecb4ab5449683c5eaa315620fc17838c0f2a58001f3979fcdf9f56cd6af83f5 F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651 @@ -2048,8 +2048,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2f712b836a0dafd0b2ee6ba4eaa9caa35e49375c7ad0562477e0f2d3b086ec03 -R 7932af26b4c2f8946f0c8d4e43a0814e +P 1096b5a7cc8104db01f8820ace47020baad2f12e6711e3a7b4514ed1becc7b66 +R 132c7e609c38a7be8ca466782d538c7f U drh -Z 65b72ddc0fa6d0ebce5a1655fb619090 +Z 36cb7a4789e4e3ff0efced0434fbf8d1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0dd62a5775..c0bc28f12c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1096b5a7cc8104db01f8820ace47020baad2f12e6711e3a7b4514ed1becc7b66 \ No newline at end of file +f45009533a79a59b302598ee2545ef787c51d0128f4e1dca60dd83589f660619 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2906a4895c..efa8b301fd 100644 --- a/src/select.c +++ b/src/select.c @@ -6868,6 +6868,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ if( p->pWhere ) return 0; if( p->pGroupBy ) return 0; + if( p->pOrderBy ) return 0; pExpr = p->pEList->a[0].pExpr; if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */ assert( ExprUseUToken(pExpr) ); diff --git a/test/countofview.test b/test/countofview.test index c7783e6dbd..bc5633f7a5 100644 --- a/test/countofview.test +++ b/test/countofview.test @@ -61,4 +61,17 @@ do_catchsql_test 3.1 { SELECT count(*) FROM c; } {1 interrupted} +# 2023-03-07 dbsqlfuzz 23d782160b71c3f8f535ccb2da313dfc8eb8c631 +# +do_execsql_test 4.1 { + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(4,'four'); + CREATE TABLE t2(c INTEGER PRIMARY KEY, d TEXT); + CREATE VIEW t3 AS SELECT a, b FROM t1 UNION ALL SELECT c, d FROM t2; + SELECT count(*) FROM t3 ORDER BY sum(a); +} 1 + finish_test From 248c1abdf4c61bf0761994db40c214272a092c7c Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 8 Mar 2023 10:05:42 +0000 Subject: [PATCH 068/341] Extend wasm build to support a custom sqlite3.c to support building against sqlite3-see.c. The JS code now binds the SEE-specific functions if it detects an SEE build. FossilOrigin-Name: dd8612c8adbaf9d06bf0d7319b9afc9bd8ca3d0fcfa1cb591a7a2fcb86480048 --- ext/wasm/GNUmakefile | 46 ++++++++++++++++----- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see | 5 +++ ext/wasm/api/sqlite3-api-glue.js | 9 ++++ ext/wasm/api/sqlite3-wasm.c | 29 ++++++++++--- manifest | 19 +++++---- manifest.uuid | 2 +- 6 files changed, 85 insertions(+), 25 deletions(-) create mode 100644 ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index ae8a7d98e8..250b8be0c7 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -145,8 +145,27 @@ ifeq (,$(wildcard $(dir.tmp))) dir._tmp := $(shell mkdir -p $(dir.tmp)) endif -sqlite3.c := $(dir.top)/sqlite3.c +######################################################################## +# Set up sqlite3.c and sqlite3.h... +# +# To build with SEE (https://sqlite.org/see), either put sqlite3-see.c +# in the top of this build tree or pass +# sqlite3.c=PATH_TO_sqlite3-see.c to the build. Note that only +# encryption modules with no 3rd-party dependencies will currently +# work here: AES256-OFB, AES128-OFB, and AES128-CCM. Not +# coincidentally, those 3 modules are included in the sqlite3-see.c +# bundle. +# +# A custom sqlite3.c must not have any spaces in its name. +sqlite3.canonical.c := $(dir.top)/sqlite3.c +sqlite3.c ?= $(firstword $(wildcard $(dir.top)/sqlite3-see.c) $(sqlite3.canonical.c)) sqlite3.h := $(dir.top)/sqlite3.h +ifeq (,$(shell grep sqlite3_activate_see $(sqlite3.c) 2>/dev/null)) + SQLITE_C_IS_SEE := 0 +else + SQLITE_C_IS_SEE := 1 + $(info This is an SEE build.) +endif # Most SQLITE_OPT flags are set in sqlite3-wasm.c but we need them # made explicit here for building speedtest1.c. SQLITE_OPT = \ @@ -169,7 +188,8 @@ SQLITE_OPT = \ -DSQLITE_OS_KV_OPTIONAL=1 \ '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"' \ -DSQLITE_USE_URI=1 \ - -DSQLITE_WASM_ENABLE_C_TESTS + -DSQLITE_WASM_ENABLE_C_TESTS \ + -DSQLITE_C=$(sqlite3.c) .NOTPARALLEL: $(sqlite3.h) $(sqlite3.h): @@ -226,7 +246,7 @@ endif # cleanup process for the dist build to work properly. bin.version-info := $(dir.wasm)/version-info $(bin.version-info): $(dir.wasm)/version-info.c $(sqlite3.h) $(MAKEFILE) - $(CC) -O0 -I$(dir.top) -o $@ $< + $(CC) -O0 -I$(dir $(sqlite3.c)) -o $@ $< DISTCLEAN_FILES += $(bin.version-info) # bin.stripcomments is used for stripping C/C++-style comments from JS @@ -284,7 +304,7 @@ endef ######################################################################## # cflags.common = C compiler flags for all builds -cflags.common := -I. -I.. -I$(dir.top) +cflags.common := -I. -I$(dir $(sqlite3.c)) # emcc.WASM_BIGINT = 1 for BigInt (C int64) support, else 0. The API # disables certain features if BigInt is not enabled and such builds # _are not tested_ on any regular basis. @@ -327,10 +347,14 @@ emcc_opt_full := $(emcc_opt) -g3 # EXPORTED_FUNCTIONS.* = files for use with Emscripten's # -sEXPORTED_FUNCTION flag. -EXPORTED_FUNCTIONS.api.in := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) +EXPORTED_FUNCTIONS.api.main := $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-api) +EXPORTED_FUNCTIONS.api.in := $(EXPORTED_FUNCTIONS.api.main) +ifeq (1,$(SQLITE_C_IS_SEE)) + EXPORTED_FUNCTIONS.api.in += $(abspath $(dir.api)/EXPORTED_FUNCTIONS.sqlite3-see) +endif EXPORTED_FUNCTIONS.api := $(dir.tmp)/EXPORTED_FUNCTIONS.api -$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(MAKEFILE) - cp $(EXPORTED_FUNCTIONS.api.in) $@ +$(EXPORTED_FUNCTIONS.api): $(EXPORTED_FUNCTIONS.api.in) $(sqlite3.c) $(MAKEFILE) + cat $(EXPORTED_FUNCTIONS.api.in) > $@ # sqlite3-license-version.js = generated JS file with the license # header and version info. @@ -834,9 +858,9 @@ speedtest1.exit-runtime1 := -sEXIT_RUNTIME=1 # -sEXIT_RUNTIME=1 but we need EXIT_RUNTIME=0 for the worker-based app # which runs speedtest1 multiple times. -$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api) +$(EXPORTED_FUNCTIONS.speedtest1): $(EXPORTED_FUNCTIONS.api.main) @echo "Making $@ ..." - @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api); } > $@ + @{ echo _wasm_main; cat $(EXPORTED_FUNCTIONS.api.main); } > $@ speedtest1.js := $(dir.dout)/speedtest1.js speedtest1.wasm := $(dir.dout)/speedtest1.wasm cflags.speedtest1 := $(cflags.common) -DSQLITE_SPEEDTEST1_WASM @@ -847,9 +871,11 @@ $(speedtest1.js): $(MAKEFILE) $(speedtest1.cses) \ $(EXPORTED_FUNCTIONS.speedtest1) @echo "Building $@ ..." $(emcc.bin) \ - $(emcc.speedtest1) $(emcc.speedtest1.common) \ + $(emcc.speedtest1) -I$(dir $(sqlite3.canonical.c)) \ + $(emcc.speedtest1.common) \ $(cflags.speedtest1) $(pre-post-speedtest1.flags.vanilla) \ $(SQLITE_OPT) \ + -USQLITE_C -DSQLITE_C=$(sqlite3.canonical.c) \ $(speedtest1.exit-runtime0) \ -o $@ $(speedtest1.cses) -lm $(maybe-wasm-strip) $(speedtest1.wasm) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see new file mode 100644 index 0000000000..83f3a97dbc --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see @@ -0,0 +1,5 @@ +_sqlite3_key +_sqlite3_key_v2 +_sqlite3_rekey +_sqlite3_rekey_v2 +_sqlite3_activate_see diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index 1cb00b9419..f444ec975c 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -327,6 +327,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); } + if(wasm.exports.sqlite3_activate_see instanceof Function){ + wasm.bindingSignatures.push( + ["sqlite3_key", "int", "sqlite3*", "string", "int"], + ["sqlite3_key_v2","int","sqlite3*","string","*","int"], + ["sqlite3_rekey", "int", "sqlite3*", "string", "int"], + ["sqlite3_rekey_v2", "int", "sqlite3*", "string", "*", "int"], + ["sqlite3_activate_see", undefined, "string"] + ); + } /** Functions which require BigInt (int64) support are separated from the others because we need to conditionally bind them or apply diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index bcabe922bf..fd074541ad 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -168,11 +168,6 @@ #endif #include -#include "sqlite3.c" /* yes, .c instead of .h. */ - -#if defined(__EMSCRIPTEN__) -# include -#endif /* ** SQLITE_WASM_EXPORT is functionally identical to EMSCRIPTEN_KEEPALIVE @@ -201,6 +196,30 @@ // See also: //__attribute__((export_name("theExportedName"), used, visibility("default"))) +/* +** Which sqlite3.c we're using needs to be configurable to enable +** building against a custom copy, e.g. the SEE variant. Note that we +** #include the .c file, rather than the header, so that the WASM +** extensions have access to private API internals. +** +** The caveat here is that custom variants need to account for +** exporting any necessary symbols (e.g. sqlite3_activate_see()). We +** cannot export them from here using SQLITE_WASM_EXPORT because that +** attribute (apparently) has to be part of the function definition. +*/ +#ifndef SQLITE_C +# define SQLITE_C sqlite3.c /* yes, .c instead of .h. */ +#endif +#define INC__STRINGIFY_(f) #f +#define INC__STRINGIFY(f) INC__STRINGIFY_(f) +#include INC__STRINGIFY(SQLITE_C) +#undef INC__STRINGIFY_ +#undef INC__STRINGIFY +#undef SQLITE_C + +#if defined(__EMSCRIPTEN__) +# include +#endif #if 0 /* diff --git a/manifest b/manifest index 7bfa0a5377..3f8746bfab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\scount-of-view\soptimization\sthat\scan\slead\sto\sincorrect\nbytecode.\s\sdbsqlfuzz\s23d782160b71c3f8f535ccb2da313dfc8eb8c631. -D 2023-03-08T00:47:53.498 +C Extend\swasm\sbuild\sto\ssupport\sa\scustom\ssqlite3.c\sto\ssupport\sbuilding\sagainst\ssqlite3-see.c.\sThe\sJS\scode\snow\sbinds\sthe\sSEE-specific\sfunctions\sif\sit\sdetects\san\sSEE\sbuild. +D 2023-03-08T10:05:42.118 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -468,10 +468,11 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 6c0c0e9d3b6ef2090f230bc47f8da5d9614fbdefacb72e8bafcc9a41d035605b +F ext/wasm/GNUmakefile 97f01bb84b0b745e2ba642e12ab24a682e369538e8d68b6e67bdbe502450fc6c F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 77a2f1f2fc60a35def7455dffc8d3f2c56385d6ac5c6cecc60fa938252ea2c54 F ext/wasm/api/extern-post-js.c-pp.js 5c4997d3442756e4e4819303fa4a7045de4a2a7b79c3ad6c26cdcf1d9141fac6 @@ -480,7 +481,7 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js cc21e3486da748463e02bbe51e2464c6ac136587cdfd5aa00cd0b5385f6ca808 -F ext/wasm/api/sqlite3-api-glue.js 32091c2730ecef8f5795c3527d3db6b7bbf054d12e27311ce2da59db559b2e0e +F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 2691a34a741015127b210954a1b9586764d3ff0c8a20f00fd15c00f339ecc79f F ext/wasm/api/sqlite3-api-prologue.js df8646e4f92b8b09cef255da3530e11dc264a2e8d53b0e78daa2ee04f99c584d F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d660194ddfc51eab1c5b9bf @@ -489,7 +490,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 26f6240170d415726d9cfe2fa7a0163e153775e1a74fa91c9ba5446502c71097 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 223d30c41d811cae8b9f1175fa68f2f1fb3cc056d16ad0def3b0ea5c65757a6c +F ext/wasm/api/sqlite3-wasm.c 4b09550e1a73f6318b47bdacdbd39045acc23f562fedbdc1d8af1b7473763e7b F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2048,8 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1096b5a7cc8104db01f8820ace47020baad2f12e6711e3a7b4514ed1becc7b66 -R 132c7e609c38a7be8ca466782d538c7f -U drh -Z 36cb7a4789e4e3ff0efced0434fbf8d1 +P f45009533a79a59b302598ee2545ef787c51d0128f4e1dca60dd83589f660619 +R cd7604a48b0beb018a7a99fee097a4b7 +U stephan +Z 52942618230bb2090251a07b83122b0f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c0bc28f12c..38299f9abf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f45009533a79a59b302598ee2545ef787c51d0128f4e1dca60dd83589f660619 \ No newline at end of file +dd8612c8adbaf9d06bf0d7319b9afc9bd8ca3d0fcfa1cb591a7a2fcb86480048 \ No newline at end of file From 7ba63831f81482c67f135dabd2fbdc1ea5332519 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 8 Mar 2023 17:09:32 +0000 Subject: [PATCH 069/341] Small performance improvement in the OP_MakeRecord opcode. FossilOrigin-Name: ca89daef0fcf6cb04aa6fa90dd333d6f2474bf3f458c833d9cd5bd8e59f2a04a --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/vdbe.c | 34 +++++++++++++++++----------------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 54a4471617..acf807f0bf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sto\s[44135d6ea84f7ba6]\sthat\sretains\sthe\shistorical\sdatatype\s("INT",\snot\s"NUM")\sfor\sa\stable\screated\sas\sfollows:\s"CREATE\sTABLE\st1\sAS\sSELECT\sCAST(123\sAS\sINT)\sAS\svalue;".\s\sThe\suse\sof\sFLEXNUM\sonly\soccurs\son\scompound\squeries. -D 2023-03-08T14:28:09.479 +C Small\sperformance\simprovement\sin\sthe\sOP_MakeRecord\sopcode. +D 2023-03-08T17:09:32.272 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -694,7 +694,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 0cf4c72a9e0eb614afc19c9c4ca9c8a919c97c0866934a70dac7c2f689a4edf8 +F src/vdbe.c 80a64921734e74b1b28b321225137ec555b0e15d73f3322c2c0651a8be481114 F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 @@ -2049,9 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dd8612c8adbaf9d06bf0d7319b9afc9bd8ca3d0fcfa1cb591a7a2fcb86480048 a0e54fe2058a4ac1e794b2491e424c60dfa42c6781b1cfd83c5db65c28c11c71 -R 9ae8ce857b1bda3c57c6b1a52ab41ca6 -T +closed a0e54fe2058a4ac1e794b2491e424c60dfa42c6781b1cfd83c5db65c28c11c71 +P 6d5b5896261c62a7e130b47416ee8c25793859a2afcb1646c257600537a5b71b +R 1b5d34199b6078f6cf9502fa50e8d5f3 U drh -Z 7b6226a524a3a9d94d5cfc19b85e131d +Z 1c424b0c5c8b9aa32605259b67546e86 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 229403f5af..da2cfe9570 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d5b5896261c62a7e130b47416ee8c25793859a2afcb1646c257600537a5b71b \ No newline at end of file +ca89daef0fcf6cb04aa6fa90dd333d6f2474bf3f458c833d9cd5bd8e59f2a04a \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index dfe8070de1..d224ee2bed 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3379,23 +3379,7 @@ case OP_MakeRecord: { pRec = pLast; do{ assert( memIsValid(pRec) ); - if( pRec->flags & MEM_Null ){ - if( pRec->flags & MEM_Zero ){ - /* Values with MEM_Null and MEM_Zero are created by xColumn virtual - ** table methods that never invoke sqlite3_result_xxxxx() while - ** computing an unchanging column value in an UPDATE statement. - ** Give such values a special internal-use-only serial-type of 10 - ** so that they can be passed through to xUpdate and have - ** a true sqlite3_value_nochange(). */ -#ifndef SQLITE_ENABLE_NULL_TRIM - assert( pOp->p5==OPFLAG_NOCHNG_MAGIC || CORRUPT_DB ); -#endif - pRec->uTemp = 10; - }else{ - pRec->uTemp = 0; - } - nHdr++; - }else if( pRec->flags & (MEM_Int|MEM_IntReal) ){ + if( pRec->flags & (MEM_Int|MEM_IntReal) ){ /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ i64 i = pRec->u.i; u64 uu; @@ -3445,6 +3429,22 @@ case OP_MakeRecord: { pRec->uTemp = 6; } } + }else if( pRec->flags & MEM_Null ){ + if( pRec->flags & MEM_Zero ){ + /* Values with MEM_Null and MEM_Zero are created by xColumn virtual + ** table methods that never invoke sqlite3_result_xxxxx() while + ** computing an unchanging column value in an UPDATE statement. + ** Give such values a special internal-use-only serial-type of 10 + ** so that they can be passed through to xUpdate and have + ** a true sqlite3_value_nochange(). */ +#ifndef SQLITE_ENABLE_NULL_TRIM + assert( pOp->p5==OPFLAG_NOCHNG_MAGIC || CORRUPT_DB ); +#endif + pRec->uTemp = 10; + }else{ + pRec->uTemp = 0; + } + nHdr++; }else if( pRec->flags & MEM_Real ){ nHdr++; nData += 8; From 975f2062dad5425b2384265dfca372d39f200c0f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Mar 2023 18:03:04 +0000 Subject: [PATCH 070/341] Add the SQLITE_CHANGESETAPPLY_IGNORENOOP flag, which may be passed to sqlite3changeset_apply_v2() to have it ignore changes that would be no-ops if applied to the database (e.g. deleting a row that has already been deleted), instead of considering them conflicts. FossilOrigin-Name: cb023fe28560ce0f8c2fd48042553fcdb9db81eba9552be75165de0d46a2645c --- ext/session/session2.test | 4 +- ext/session/sessionG.test | 18 +-- ext/session/sessionH.test | 2 +- ext/session/session_common.tcl | 40 ++++- ext/session/sessionat.test | 12 +- ext/session/sessionbig.test | 4 +- ext/session/sessionfault.test | 2 +- ext/session/sessionfault2.test | 2 +- ext/session/sessionnoop.test | 287 ++++++++++++++++----------------- ext/session/sessionstat1.test | 12 +- ext/session/sqlite3session.c | 131 +++++++++++++-- ext/session/sqlite3session.h | 14 ++ ext/session/test_session.c | 23 ++- manifest | 38 ++--- manifest.uuid | 2 +- 15 files changed, 364 insertions(+), 227 deletions(-) diff --git a/ext/session/session2.test b/ext/session/session2.test index 806687745e..207b98740e 100644 --- a/ext/session/session2.test +++ b/ext/session/session2.test @@ -191,7 +191,7 @@ do_common_sql { } foreach {tn sql} [string map {%T1% t1 %T2% t2 %T3% t3 %T4% t4} $set_of_tests] { - do_then_apply_sql $sql + do_then_apply_sql -ignorenoop $sql do_test 2.$tn { compare_db db db2 } {} } @@ -598,7 +598,7 @@ do_common_sql { INSERT INTO t1 SELECT NULL, 0, 0, 0, 0, 0 FROM s } -do_then_apply_sql { +do_then_apply_sql -ignorenoop { UPDATE t1 SET f=f+1 WHERE a=1; UPDATE t1 SET e=e+1 WHERE a=2; UPDATE t1 SET e=e+1, f=f+1 WHERE a=3; diff --git a/ext/session/sessionG.test b/ext/session/sessionG.test index 58ea17d2ee..1ebcc926a5 100644 --- a/ext/session/sessionG.test +++ b/ext/session/sessionG.test @@ -34,7 +34,7 @@ do_test 1.0 { INSERT INTO t1 VALUES(2, 'two'); INSERT INTO t1 VALUES(3, 'three'); } - do_then_apply_sql { + do_then_apply_sql -ignorenoop { DELETE FROM t1 WHERE a=1; INSERT INTO t1 VALUES(4, 'one'); } @@ -42,7 +42,7 @@ do_test 1.0 { } {} do_test 1.1 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { DELETE FROM t1 WHERE a=4; INSERT INTO t1 VALUES(1, 'one'); } @@ -51,7 +51,7 @@ do_test 1.1 { do_test 1.2 { execsql { INSERT INTO t1 VALUES(5, 'five') } db2 - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1 VALUES(11, 'eleven'); INSERT INTO t1 VALUES(12, 'five'); } @@ -82,7 +82,7 @@ do_test 2.2.1 { # It is not possible to apply the changeset generated by the following # SQL, as none of the three updated rows may be updated as part of the # first pass. - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=0 WHERE a=1; UPDATE t1 SET b=1 WHERE a=2; UPDATE t1 SET b=2 WHERE a=3; @@ -109,7 +109,7 @@ do_test 3.1 { } {} do_test 3.3 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=4 WHERE a=3; UPDATE t1 SET b=3 WHERE a=2; UPDATE t1 SET b=2 WHERE a=1; @@ -118,7 +118,7 @@ do_test 3.3 { } {} do_test 3.4 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=1 WHERE a=1; UPDATE t1 SET b=2 WHERE a=2; UPDATE t1 SET b=3 WHERE a=3; @@ -148,7 +148,7 @@ do_test 4.1 { } {} do_test 4.2 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=4 WHERE a=3; UPDATE t1 SET b=3 WHERE a=2; UPDATE t1 SET b=2 WHERE a=1; @@ -161,7 +161,7 @@ do_test 4.2 { } {} do_test 4.3 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t1 SET b=1 WHERE a=1; UPDATE t1 SET b=2 WHERE a=2; UPDATE t1 SET b=3 WHERE a=3; @@ -191,7 +191,7 @@ do_execsql_test -db db2 5.0.2 { } do_test 5.1 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t2 VALUES(4, 5, 6); INSERT INTO t3 VALUES(7, 8, 9); diff --git a/ext/session/sessionH.test b/ext/session/sessionH.test index 8ba23119c8..e1b12571c6 100644 --- a/ext/session/sessionH.test +++ b/ext/session/sessionH.test @@ -25,7 +25,7 @@ do_test 1.0 { do_common_sql { CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)); } - do_then_apply_sql { + do_then_apply_sql -ignorenoop { WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERe i<10000 ) diff --git a/ext/session/session_common.tcl b/ext/session/session_common.tcl index c52ac457c0..22e427f2be 100644 --- a/ext/session/session_common.tcl +++ b/ext/session/session_common.tcl @@ -112,20 +112,52 @@ proc patchset_from_sql {sql {dbname main}} { return $patchset } -proc do_then_apply_sql {sql {dbname main}} { - proc xConflict args { return "OMIT" } +# Usage: do_then_apply_sql ?-ignorenoop? SQL ?DBNAME? +# +proc do_then_apply_sql {args} { + + set bIgnoreNoop 0 + set a1 [lindex $args 0] + if {[string length $a1]>1 && [string first $a1 -ignorenoop]==0} { + set bIgnoreNoop 1 + set args [lrange $args 1 end] + } + + if {[llength $args]!=1 && [llength $args]!=2} { + error "usage: do_then_apply_sql ?-ignorenoop? SQL ?DBNAME?" + } + + set sql [lindex $args 0] + if {[llength $args]==1} { + set dbname main + } else { + set dbname [lindex $args 1] + } + + set ::n_conflict 0 + proc xConflict args { incr ::n_conflict ; return "OMIT" } set rc [catch { sqlite3session S db $dbname db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { S attach $name } db eval $sql - sqlite3changeset_apply db2 [S changeset] xConflict + set ::changeset [S changeset] + sqlite3changeset_apply db2 $::changeset xConflict } msg] catch { S delete } - if {$rc} {error $msg} + + if {$bIgnoreNoop} { + set nSave $::n_conflict + set ::n_conflict 0 + proc xConflict args { incr ::n_conflict ; return "OMIT" } + sqlite3changeset_apply_v2 -ignorenoop db2 $::changeset xConflict + if {$::n_conflict!=$nSave} { + error "-ignorenoop problem ($::n_conflict $nSave)..." + } + } } proc do_iterator_test {tn tbl_list sql res} { diff --git a/ext/session/sessionat.test b/ext/session/sessionat.test index e3f9e31ed7..e14901e8b5 100644 --- a/ext/session/sessionat.test +++ b/ext/session/sessionat.test @@ -110,7 +110,7 @@ eval [string map [list %WR% $trailing] { CREATE TABLE t3(a, b, c DEFAULT 'D', PRIMARY KEY(b)) %WR%; } do_test $tn.3.2 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t3 VALUES(1, 2); INSERT INTO t3 VALUES(3, 4); INSERT INTO t3 VALUES(5, 6); @@ -118,7 +118,7 @@ eval [string map [list %WR% $trailing] { db2 eval {SELECT * FROM t3} } {1 2 D 3 4 D 5 6 D} do_test $tn.3.3 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t3 SET a=45 WHERE b=4; DELETE FROM t3 WHERE a=5; }; @@ -253,7 +253,7 @@ eval [string map [list %WR% $trailing] { CREATE TABLE t8(a PRIMARY KEY, b, c, d DEFAULT 'D', e DEFAULT 'E'); } - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t8 VALUES(1, 2, 3); INSERT INTO t8 VALUES(4, 5, 6); } @@ -264,7 +264,7 @@ eval [string map [list %WR% $trailing] { SELECT * FROM t8 } {1 2 3 D E 4 5 6 D E} - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t8 SET c=45 WHERE a=4; } do_execsql_test $tn.7.3.1 { @@ -282,7 +282,7 @@ eval [string map [list %WR% $trailing] { do_execsql_test -db db2 $tn.8.1 { CREATE TABLE t9(a PRIMARY KEY, b, c, d, e, f, g, h, i, j, k, l); } - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t9 VALUES(1, 2, 3, 4, 5, 6, 7, 8); } do_then_apply_sql { @@ -291,7 +291,7 @@ eval [string map [list %WR% $trailing] { do_execsql_test -db db2 $tn.8.2 { SELECT * FROM t9 } {1 2 3 4 5 6 7 450 {} {} {} {}} - do_then_apply_sql { + do_then_apply_sql -ignorenoop { UPDATE t9 SET h=NULL } do_execsql_test -db db2 $tn.8.2 { diff --git a/ext/session/sessionbig.test b/ext/session/sessionbig.test index 80ce00a0f7..462e21f61f 100644 --- a/ext/session/sessionbig.test +++ b/ext/session/sessionbig.test @@ -43,7 +43,7 @@ do_execsql_test -db db2 1.1 { } do_test 1.2 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); @@ -71,7 +71,7 @@ do_test 1.3 { do_test 1.4 { set rc [catch { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); INSERT INTO t1(b) VALUES( zeroblob(100*1000*1000) ); diff --git a/ext/session/sessionfault.test b/ext/session/sessionfault.test index be6c4568ce..96e966b41d 100644 --- a/ext/session/sessionfault.test +++ b/ext/session/sessionfault.test @@ -44,7 +44,7 @@ do_faultsim_test 1.1 -faults oom-* -prep { faultsim_restore_and_reopen sqlite3 db2 test.db2 } -body { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO t1 VALUES('a string value', 8, 9); UPDATE t1 SET c = 10 WHERE a = 1; DELETE FROM t1 WHERE a = 4; diff --git a/ext/session/sessionfault2.test b/ext/session/sessionfault2.test index dd00eaa1c8..a2dc39e437 100644 --- a/ext/session/sessionfault2.test +++ b/ext/session/sessionfault2.test @@ -132,7 +132,7 @@ do_faultsim_test 1.1 -faults oom-* -prep { faultsim_restore_and_reopen sqlite3 db2 test.db2 } -body { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { INSERT INTO sqlite_stat1 VALUES('x', 'y', 45); UPDATE sqlite_stat1 SET stat = 123 WHERE tbl='t1' AND idx='i1'; UPDATE sqlite_stat1 SET stat = 456 WHERE tbl='t2'; diff --git a/ext/session/sessionnoop.test b/ext/session/sessionnoop.test index 16c60b7abf..5549773440 100644 --- a/ext/session/sessionnoop.test +++ b/ext/session/sessionnoop.test @@ -1,4 +1,4 @@ -# 2021 Februar 20 +# 2011 March 07 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -20,166 +20,159 @@ ifcapable !session {finish_test; return} set testprefix sessionnoop -#------------------------------------------------------------------------- -# Test plan: -# -# 1.*: Test that concatenating changesets cannot produce a noop UPDATE. -# 2.*: Test that rebasing changesets cannot produce a noop UPDATE. -# 3.*: Test that sqlite3changeset_apply() ignores noop UPDATE changes. -# - -do_execsql_test 1.0 { - CREATE TABLE t1(a PRIMARY KEY, b, c, d); - INSERT INTO t1 VALUES(1, 1, 1, 1); - INSERT INTO t1 VALUES(2, 2, 2, 2); - INSERT INTO t1 VALUES(3, 3, 3, 3); -} - -proc do_concat_test {tn sql1 sql2 res} { - uplevel [list do_test $tn [subst -nocommands { - set C1 [changeset_from_sql {$sql1}] - set C2 [changeset_from_sql {$sql2}] - set C3 [sqlite3changeset_concat [set C1] [set C2]] - set got [list] - sqlite3session_foreach elem [set C3] { lappend got [set elem] } - set got - }] [list {*}$res]] -} - -do_concat_test 1.1 { - UPDATE t1 SET c=c+1; -} { - UPDATE t1 SET c=c-1; +foreach {tn wo} { + 1 "" + 2 " WITHOUT ROWID " } { + reset_db + eval [string map [list %WO% $wo] { +do_execsql_test $tn.1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c) %WO%; + INSERT INTO t1 VALUES('a', 'A', 'AAA'); + INSERT INTO t1 VALUES('b', 'B', 'BBB'); + INSERT INTO t1 VALUES('c', 'C', 'CCC'); + INSERT INTO t1 VALUES('d', 'D', 'DDD'); + INSERT INTO t1 VALUES('e', 'E', 'EEE'); } -#------------------------------------------------------------------------- -reset_db -do_execsql_test 2.0 { - CREATE TABLE t1(a PRIMARY KEY, b, c); - INSERT INTO t1 VALUES(1, 1, 1); - INSERT INTO t1 VALUES(2, 2, 2); - INSERT INTO t1 VALUES(3, 3, 3); +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_execsql_test -db db2 $tn.1.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c) %WO%; + INSERT INTO t1 VALUES('a', 'A', 'AAA'); + INSERT INTO t1 VALUES('b', 'B', '123'); + INSERT INTO t1 VALUES('c', 'C', 'CCC'); + INSERT INTO t1 VALUES('e', 'E', 'EEE'); + INSERT INTO t1 VALUES('f', 'F', 'FFF'); } -proc do_rebase_test {tn sql_local sql_remote conflict_res expected} { - proc xConflict {args} [list return $conflict_res] +set C [changeset_from_sql { + UPDATE t1 SET c='123' WHERE a='b'; + DELETE FROM t1 WHERE a='d'; + INSERT INTO t1 VALUES('f', 'F', 'FFF'); +}] - uplevel [list \ - do_test $tn [subst -nocommands { - execsql BEGIN - set c_remote [changeset_from_sql {$sql_remote}] - execsql ROLLBACK - execsql BEGIN - set c_local [changeset_from_sql {$sql_local}] - set base [sqlite3changeset_apply_v2 db [set c_remote] xConflict] - execsql ROLLBACK - - sqlite3rebaser_create R - R config [set base] - set res [list] - sqlite3session_foreach elem [R rebase [set c_local]] { - lappend res [set elem] - } - R delete - set res - }] [list {*}$expected] - ] +set ::conflict_list [list] +proc xConflict {args} { + lappend ::conflict_list $args + return "OMIT" } +do_test $tn.1.2 { + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} [list {*}{ + {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 123}} + {INSERT t1 CONFLICT {t f t F t FFF} {t f t F t FFF}} + {DELETE t1 NOTFOUND {t d t D t DDD}} +}] +do_test $tn.1.3 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} [list {*}{ + {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 123}} + {INSERT t1 CONFLICT {t f t F t FFF} {t f t F t FFF}} + {DELETE t1 NOTFOUND {t d t D t DDD}} +}] -do_rebase_test 2.1 { - UPDATE t1 SET c=2 WHERE a=1; -- local -} { - UPDATE t1 SET c=3 WHERE a=1; -- remote -} OMIT { - {UPDATE t1 0 X.. {i 1 {} {} i 3} {{} {} {} {} i 2}} -} - -do_rebase_test 2.2 { - UPDATE t1 SET c=2 WHERE a=1; -- local -} { - UPDATE t1 SET c=3 WHERE a=1; -- remote -} REPLACE { -} - -do_rebase_test 2.3.1 { - UPDATE t1 SET c=4 WHERE a=1; -- local -} { - UPDATE t1 SET c=4 WHERE a=1 -- remote -} OMIT { - {UPDATE t1 0 X.. {i 1 {} {} i 4} {{} {} {} {} i 4}} -} - -do_rebase_test 2.3.2 { - UPDATE t1 SET c=5 WHERE a=1; -- local -} { - UPDATE t1 SET c=5 WHERE a=1 -- remote -} REPLACE { -} - -#------------------------------------------------------------------------- -# -reset_db -do_execsql_test 3.0 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - INSERT INTO t1 VALUES(1, 1, 1); - INSERT INTO t1 VALUES(2, 2, 2); - INSERT INTO t1 VALUES(3, 3, 3); - INSERT INTO t1 VALUES(4, 4, 4); -} - -# Arg $pkstr contains one character for each column in the table. An -# "X" for PK column, or a "." for a non-PK. -# -proc mk_tbl_header {name pkstr} { - set ret [binary format H2c 54 [string length $pkstr]] - foreach i [split $pkstr {}] { - if {$i=="X"} { - append ret [binary format H2 01] - } else { - if {$i!="."} {error "bad pkstr: $pkstr ($i)"} - append ret [binary format H2 00] - } - } - append ret $name - append ret [binary format H2 00] - set ret -} - -proc mk_update_change {args} { - set ret [binary format H2H2 17 00] - foreach a $args { - if {$a==""} { - append ret [binary format H2 00] - } else { - append ret [binary format H2W 01 $a] - } - } - set ret -} - -proc xConflict {args} { return "ABORT" } -do_test 3.1 { - set C [mk_tbl_header t1 X..] - append C [mk_update_change 1 {} 1 {} {} 500] - append C [mk_update_change 2 {} {} {} {} {}] - append C [mk_update_change 3 3 {} {} 600 {}] - append C [mk_update_change 4 {} {} {} {} {}] - - sqlite3changeset_apply_v2 db $C xConflict +do_test $tn.1.4 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list } {} -do_execsql_test 3.2 { - SELECT * FROM t1 -} { - 1 1 500 - 2 2 2 - 3 600 3 - 4 4 4 + +do_execsql_test -db db2 1.5 { + UPDATE t1 SET b='G' WHERE a='f'; + UPDATE t1 SET c='456' WHERE a='b'; } +do_test $tn.1.6 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} [list {*}{ + {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 456}} + {INSERT t1 CONFLICT {t f t F t FFF} {t f t G t FFF}} +}] + +db2 close + +#-------------------------------------------------------------------------- + +reset_db +forcedelete test.db2 +sqlite3 db2 test.db2 +do_execsql_test $tn.2.0 { + CREATE TABLE t1(a PRIMARY KEY, b) %WO%; +} +do_execsql_test -db db2 $tn.2.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c DEFAULT 'val') %WO%; +} + +do_test $tn.2.2 { + do_then_apply_sql -ignorenoop { + INSERT INTO t1 VALUES(1, 2); + } + do_then_apply_sql -ignorenoop { + UPDATE t1 SET b=2 WHERE a=1 + } +} {} +}] +} + +db2 close + +#------------------------------------------------------------------------- +reset_db +forcedelete test.db2 +do_execsql_test 3.0 { + CREATE TABLE xyz(a, b, c, PRIMARY KEY(a, b), UNIQUE(c)); + ANALYZE; + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO xyz SELECT i, i, i FROM s; + VACUUM INTO 'test.db2'; +} + +set C [changeset_from_sql { ANALYZE }] +sqlite3 db2 test.db2 + +set ::conflict_list [list] +proc xConflict {args} { lappend ::conflict_list $args ; return "OMIT" } +do_test 3.1 { + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} {} + +do_test 3.2 { + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} {} + +do_test 3.3 { + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} [list {*}{ + {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}} {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}}} + {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_2 t {100 1}} {t xyz t sqlite_autoindex_xyz_2 t {100 1}}} +}] + +do_execsql_test -db db2 3.4 { + UPDATE sqlite_stat1 SET stat='200 1 1' WHERE idx='sqlite_autoindex_xyz_1'; +} + +do_test 3.5 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} [list {*}{ + {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}} {t xyz t sqlite_autoindex_xyz_1 t {200 1 1}}} +}] diff --git a/ext/session/sessionstat1.test b/ext/session/sessionstat1.test index 774899d96b..2757d60440 100644 --- a/ext/session/sessionstat1.test +++ b/ext/session/sessionstat1.test @@ -82,7 +82,7 @@ do_test 2.0 { } {} do_test 2.1 { - do_then_apply_sql { + do_then_apply_sql -ignorenoop { WITH s(i) AS ( SELECT 0 UNION ALL SELECT i+1 FROM s WHERE (i+1)<32 ) @@ -100,7 +100,7 @@ do_execsql_test -db db2 2.2 { } do_test 2.3 { - do_then_apply_sql { DROP INDEX t1c } + do_then_apply_sql -ignorenoop { DROP INDEX t1c } } {} do_execsql_test -db db2 2.4 { @@ -111,7 +111,7 @@ do_execsql_test -db db2 2.4 { } do_test 2.3 { - do_then_apply_sql { DROP TABLE t1 } + do_then_apply_sql -ignorenoop { DROP TABLE t1 } } {} do_execsql_test -db db2 2.4 { @@ -153,16 +153,16 @@ do_execsql_test 3.2 { } {t1 null 4} do_test 3.3 { execsql { DELETE FROM sqlite_stat1 } - do_then_apply_sql { ANALYZE } + do_then_apply_sql -ignorenoop { ANALYZE } execsql { SELECT * FROM sqlite_stat1 } db2 } {t1 null 4} do_test 3.4 { execsql { INSERT INTO t1 VALUES(5,5,5) } - do_then_apply_sql { ANALYZE } + do_then_apply_sql -ignorenoop { ANALYZE } execsql { SELECT * FROM sqlite_stat1 } db2 } {t1 null 5} do_test 3.5 { - do_then_apply_sql { DROP TABLE t1 } + do_then_apply_sql -ignorenoop { DROP TABLE t1 } execsql { SELECT * FROM sqlite_stat1 } db2 } {} diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index a3f28abe9e..225d0316a8 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -2113,9 +2113,10 @@ static void sessionAppendStr( int *pRc ){ int nStr = sqlite3Strlen30(zStr); - if( 0==sessionBufferGrow(p, nStr, pRc) ){ + if( 0==sessionBufferGrow(p, nStr+1, pRc) ){ memcpy(&p->aBuf[p->nBuf], zStr, nStr); p->nBuf += nStr; + p->aBuf[p->nBuf] = 0x00; } } @@ -2137,6 +2138,27 @@ static void sessionAppendInteger( sessionAppendStr(p, aBuf, pRc); } +static void sessionAppendPrintf( + SessionBuffer *p, /* Buffer to append to */ + int *pRc, + const char *zFmt, + ... +){ + if( *pRc==SQLITE_OK ){ + char *zApp = 0; + va_list ap; + va_start(ap, zFmt); + zApp = sqlite3_vmprintf(zFmt, ap); + if( zApp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sessionAppendStr(p, zApp, pRc); + } + va_end(ap); + sqlite3_free(zApp); + } +} + /* ** This function is a no-op if *pRc is other than SQLITE_OK when it is ** called. Otherwise, append the string zStr enclosed in quotes (") and @@ -2151,7 +2173,7 @@ static void sessionAppendIdent( const char *zStr, /* String to quote, escape and append */ int *pRc /* IN/OUT: Error code */ ){ - int nStr = sqlite3Strlen30(zStr)*2 + 2 + 1; + int nStr = sqlite3Strlen30(zStr)*2 + 2 + 2; if( 0==sessionBufferGrow(p, nStr, pRc) ){ char *zOut = (char *)&p->aBuf[p->nBuf]; const char *zIn = zStr; @@ -2162,6 +2184,7 @@ static void sessionAppendIdent( } *zOut++ = '"'; p->nBuf = (int)((u8 *)zOut - p->aBuf); + p->aBuf[p->nBuf] = 0x00; } } @@ -2386,10 +2409,17 @@ static int sessionAppendDelete( ** Formulate and prepare a SELECT statement to retrieve a row from table ** zTab in database zDb based on its primary key. i.e. ** -** SELECT * FROM zDb.zTab WHERE pk1 = ? AND pk2 = ? AND ... +** SELECT *, FROM zDb.zTab WHERE (pk1, pk2,...) IS (?1, ?2,...) +** +** where is: +** +** 1 AND (?A OR ?1 IS ) AND ... +** +** for each non-pk . */ static int sessionSelectStmt( sqlite3 *db, /* Database handle */ + int bIgnoreNoop, const char *zDb, /* Database name */ const char *zTab, /* Table name */ int nCol, /* Number of columns in table */ @@ -2399,8 +2429,51 @@ static int sessionSelectStmt( ){ int rc = SQLITE_OK; char *zSql = 0; + const char *zSep = ""; + const char *zCols = "*"; int nSql = -1; + int i; + SessionBuffer nooptest = {0, 0, 0}; + SessionBuffer pkfield = {0, 0, 0}; + SessionBuffer pkvar = {0, 0, 0}; + + sessionAppendStr(&nooptest, ", 1", &rc); + + if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){ + sessionAppendStr(&nooptest, " AND (?6 OR ?3 IS stat)", &rc); + sessionAppendStr(&pkfield, "tbl, idx", &rc); + sessionAppendStr(&pkvar, + "?1, (CASE WHEN ?2=X'' THEN NULL ELSE ?2 END)", &rc + ); + zCols = "tbl, ?2, stat"; + }else{ + for(i=0; izDb, zName, nCol, azCol, abPK, &pSel); + db, 0, pSession->zDb, zName, nCol, azCol, abPK, &pSel + ); } nNoop = buf.nBuf; @@ -3782,6 +3859,7 @@ struct SessionApplyCtx { SessionBuffer rebase; /* Rebase information (if any) here */ u8 bRebaseStarted; /* If table header is already in rebase */ u8 bRebase; /* True to collect rebase information */ + u8 bIgnoreNoop; /* True to ignore no-op conflicts */ }; /* Number of prepared UPDATE statements to cache. */ @@ -4032,8 +4110,9 @@ static int sessionSelectRow( const char *zTab, /* Table name */ SessionApplyCtx *p /* Session changeset-apply context */ ){ - return sessionSelectStmt( - db, "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect); + return sessionSelectStmt(db, p->bIgnoreNoop, + "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect + ); } /* @@ -4192,20 +4271,33 @@ static int sessionBindRow( */ static int sessionSeekToRow( sqlite3_changeset_iter *pIter, /* Changeset iterator */ - u8 *abPK, /* Primary key flags array */ - sqlite3_stmt *pSelect /* SELECT statement from sessionSelectRow() */ + SessionApplyCtx *p ){ + sqlite3_stmt *pSelect = p->pSelect; int rc; /* Return code */ int nCol; /* Number of columns in table */ int op; /* Changset operation (SQLITE_UPDATE etc.) */ const char *zDummy; /* Unused */ + sqlite3_clear_bindings(pSelect); sqlite3changeset_op(pIter, &zDummy, &nCol, &op, 0); rc = sessionBindRow(pIter, op==SQLITE_INSERT ? sqlite3changeset_new : sqlite3changeset_old, - nCol, abPK, pSelect + nCol, p->abPK, pSelect ); + if( op!=SQLITE_DELETE && p->bIgnoreNoop ){ + int ii; + for(ii=0; rc==SQLITE_OK && iiabPK[ii]==0 ){ + sqlite3_value *pVal = 0; + sqlite3changeset_new(pIter, ii, &pVal); + sqlite3_bind_int(pSelect, ii+1+nCol, (pVal==0)); + if( pVal ) rc = sessionBindValue(pSelect, ii+1, pVal); + } + } + } + if( rc==SQLITE_OK ){ rc = sqlite3_step(pSelect); if( rc!=SQLITE_ROW ) rc = sqlite3_reset(pSelect); @@ -4320,16 +4412,22 @@ static int sessionConflictHandler( /* Bind the new.* PRIMARY KEY values to the SELECT statement. */ if( pbReplace ){ - rc = sessionSeekToRow(pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); }else{ rc = SQLITE_OK; } if( rc==SQLITE_ROW ){ /* There exists another row with the new.* primary key. */ - pIter->pConflict = p->pSelect; - res = xConflict(pCtx, eType, pIter); - pIter->pConflict = 0; + if( p->bIgnoreNoop + && sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) + ){ + res = SQLITE_CHANGESET_OMIT; + }else{ + pIter->pConflict = p->pSelect; + res = xConflict(pCtx, eType, pIter); + pIter->pConflict = 0; + } rc = sqlite3_reset(p->pSelect); }else if( rc==SQLITE_OK ){ if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){ @@ -4437,7 +4535,7 @@ static int sessionApplyOneOp( sqlite3_step(p->pDelete); rc = sqlite3_reset(p->pDelete); - if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 && p->bIgnoreNoop==0 ){ rc = sessionConflictHandler( SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry ); @@ -4494,7 +4592,7 @@ static int sessionApplyOneOp( /* Check if there is a conflicting row. For sqlite_stat1, this needs ** to be done using a SELECT, as there is no PRIMARY KEY in the ** database schema to throw an exception if a duplicate is inserted. */ - rc = sessionSeekToRow(pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); if( rc==SQLITE_ROW ){ rc = SQLITE_CONSTRAINT; sqlite3_reset(p->pSelect); @@ -4671,6 +4769,7 @@ static int sessionChangesetApply( memset(&sApply, 0, sizeof(sApply)); sApply.bRebase = (ppRebase && pnRebase); sApply.bInvertConstraints = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + sApply.bIgnoreNoop = !!(flags & SQLITE_CHANGESETAPPLY_IGNORENOOP); sqlite3_mutex_enter(sqlite3_db_mutex(db)); if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){ rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index 10d0133901..f3a98d2ecd 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -1243,9 +1243,23 @@ int sqlite3changeset_apply_v2( ** Invert the changeset before applying it. This is equivalent to inverting ** a changeset using sqlite3changeset_invert() before applying it. It is ** an error to specify this flag with a patchset. +** +**
SQLITE_CHANGESETAPPLY_IGNORENOOP
+** Do not invoke the conflict handler callback for any changes that +** would not actually modify the database even if they were applied. +** Specifically, this means that the conflict handler is not invoked +** for: +**
    +**
  • a delete change if the row being deleted cannot be found, +**
  • an update change if the modified fields are already set to +** their new values in the conflicting row, or +**
  • an insert change if all fields of the conflicting row match +** the row being inserted. +**
*/ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 #define SQLITE_CHANGESETAPPLY_INVERT 0x0002 +#define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 /* ** CAPI3REF: Constants Passed To The Conflict Handler diff --git a/ext/session/test_session.c b/ext/session/test_session.c index 242e0fb0fb..f10afe03f6 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -793,32 +793,31 @@ static int SQLITE_TCLAPI testSqlite3changesetApply( memset(&sStr, 0, sizeof(sStr)); sStr.nStream = test_tcl_integer(interp, SESSION_STREAM_TCL_VAR); - /* Check for the -nosavepoint flag */ + /* Check for the -nosavepoint, -invert or -ignorenoop switches */ if( bV2 ){ - if( objc>1 ){ + while( objc>1 ){ const char *z1 = Tcl_GetString(objv[1]); int n = strlen(z1); if( n>1 && n<=12 && 0==sqlite3_strnicmp("-nosavepoint", z1, n) ){ flags |= SQLITE_CHANGESETAPPLY_NOSAVEPOINT; - objc--; - objv++; } - } - if( objc>1 ){ - const char *z1 = Tcl_GetString(objv[1]); - int n = strlen(z1); - if( n>1 && n<=7 && 0==sqlite3_strnicmp("-invert", z1, n) ){ + else if( n>2 && n<=7 && 0==sqlite3_strnicmp("-invert", z1, n) ){ flags |= SQLITE_CHANGESETAPPLY_INVERT; - objc--; - objv++; } + else if( n>2 && n<=11 && 0==sqlite3_strnicmp("-ignorenoop", z1, n) ){ + flags |= SQLITE_CHANGESETAPPLY_IGNORENOOP; + }else{ + break; + } + objc--; + objv++; } } if( objc!=4 && objc!=5 ){ const char *zMsg; if( bV2 ){ - zMsg = "?-nosavepoint? ?-inverse? " + zMsg = "?-nosavepoint? ?-inverse? ?-ignorenoop? " "DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?"; }else{ zMsg = "DB CHANGESET CONFLICT-SCRIPT ?FILTER-SCRIPT?"; diff --git a/manifest b/manifest index acf807f0bf..57192a45d7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sin\sthe\sOP_MakeRecord\sopcode. -D 2023-03-08T17:09:32.272 +C Add\sthe\sSQLITE_CHANGESETAPPLY_IGNORENOOP\sflag,\swhich\smay\sbe\spassed\sto\ssqlite3changeset_apply_v2()\sto\shave\sit\signore\schanges\sthat\swould\sbe\sno-ops\sif\sapplied\sto\sthe\sdatabase\s(e.g.\sdeleting\sa\srow\sthat\shas\salready\sbeen\sdeleted),\sinstead\sof\sconsidering\sthem\sconflicts. +D 2023-03-08T18:03:04.122 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -432,7 +432,7 @@ F ext/session/changeset.c 7a1e6a14c7e92d36ca177e92e88b5281acd709f3b726298dc34ec0 F ext/session/changesetfuzz.c 227076ab0ae4447d742c01ee88a564da6478bbf26b65108bf8fac9cd8b0b24aa F ext/session/changesetfuzz1.test 2e1b90d888fbf0eea5e1bd2f1e527a48cc85f8e0ff75df1ec4e320b21f580b3a F ext/session/session1.test e94f764fbfb672147c0ef7026b195988133b371dc8cf9e52423eba6cad69717e -F ext/session/session2.test 7f53d755d921e0baf815c4258348e0ed460dfd8a772351bca5ad3ccbb1dc786e +F ext/session/session2.test ee83bb973b9ce17ccce4db931cdcdae65eb40bbb22089b2fe6aa4f6be3b9303f F ext/session/session3.test ce9ce3dfa489473987f899e9f6a0f2db9bde3479 F ext/session/session4.test 6778997065b44d99c51ff9cece047ff9244a32856b328735ae27ddef68979c40 F ext/session/session5.test 716bc6fafd625ce60dfa62ae128971628c1a1169 @@ -445,25 +445,25 @@ F ext/session/sessionC.test f8a5508bc059ae646e5ec9bdbca66ad24bc92fe99fda5790ac57 F ext/session/sessionD.test 4f91d0ca8afc4c3969c72c9f0b5ea9527e21de29039937d0d973f821e8470724 F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d064bce83111d F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401 -F ext/session/sessionG.test 3828b944cd1285f4379340fd36f8b64c464fc84df6ff3ccbc95578fd87140b9c -F ext/session/sessionH.test b17afdbd3b8f17e9bab91e235acf167cf35485db2ab2df0ea8893fbb914741a4 -F ext/session/session_common.tcl f613174665456b2d916ae8df3e5735092a1c1712f36f46840172e9a01e8cc53e +F ext/session/sessionG.test 3efe388282d641b65485b5462e67851002cd91a282dc95b685d085eb8efdad0a +F ext/session/sessionH.test 71bbff6b1abb2c4ac62b84dee53273c37e0b21e5fde3aed80929403e091ef859 +F ext/session/session_common.tcl db0dda567c75950604072251744e9a6ad5795a3009963c44eb8510f23a8cda64 F ext/session/session_speed_test.c dcf0ef58d76b70c8fbd9eab3be77cf9deb8bc1638fed8be518b62d6cbdef88b3 -F ext/session/sessionat.test 46fd847f6ed194ebb7ebef9fe68b2e2ec88d9c2383a6846cddc5604b35f1d4ae -F ext/session/sessionbig.test 890ade19e3f80f3d3a3e83821ff79c5e2af906a67ecb5450879f0015cadf101e +F ext/session/sessionat.test 00c8badb35e43a2f12a716d2734a44d614ff62361979b6b85419035bc04b45ee +F ext/session/sessionbig.test 47c381e7acfabeef17d98519a3080d69151723354d220afa2053852182ca7adf F ext/session/sessiondiff.test ad13dd65664bae26744e1f18eb3cbd5588349b7e9118851d8f9364248d67bcec -F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7 -F ext/session/sessionfault2.test dd593f80b6b4786f7adfe83c5939620bc505559770cc181332da26f29cddd7bb +F ext/session/sessionfault.test 573bf027fb870d57bd4e7cf50822a3e4b17b2b923407438747aaa918dec57a09 +F ext/session/sessionfault2.test b0d6a7c1d7398a7e800d84657404909c7d385965ea8576dc79ed344c46fbf41c F ext/session/sessioninvert.test 04075517a9497a80d39c495ba6b44f3982c7371129b89e2c52219819bc105a25 F ext/session/sessionmem.test f2a735db84a3e9e19f571033b725b0b2daf847f3f28b1da55a0c1a4e74f1de09 -F ext/session/sessionnoop.test a9366a36a95ef85f8a3687856ebef46983df399541174cb1ede2ee53b8011bc7 +F ext/session/sessionnoop.test 5c9a882219e54711c98dccd2fd81392f189a59325e4fb5d8ed25e33a0c2f0ba2 F ext/session/sessionrebase.test ccfa716b23bd1d3b03217ee58cfd90c78d4b99f53e6a9a2f05e82363b9142810 F ext/session/sessionsize.test 6f644aff31c7f1e4871e9ff3542766e18da68fc7e587b83a347ea9820a002dd8 -F ext/session/sessionstat1.test 218d351cf9fcd6648f125a26b607b140310160184723c2666091b54450a68fb5 +F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c 13bdc093416cd284d4075328dd8599eb59bcedc23a21d561a15d78805c5866bf -F ext/session/sqlite3session.h 0907de79bc13a2e3af30a6dc29acc60792a3eaf7d33d44cf52500d0f3c2b2171 -F ext/session/test_session.c 2de472b4d7e62e85ca1992094612725e2450a77dbf7523db64de94197812462e +F ext/session/sqlite3session.c 1795263b72c1a17e48e95a131a69543af3fa31aa8e81271c7c5cb0911f063604 +F ext/session/sqlite3session.h c367c3043dbb57f69cca35258ebbeadb24e8738980b1a1ae1e281c1b0fac3989 +F ext/session/test_session.c b55a669a2150eb7c491b8b42c69a3eed9bc895cf5fea371a2c813b9618f72163 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb @@ -2049,8 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6d5b5896261c62a7e130b47416ee8c25793859a2afcb1646c257600537a5b71b -R 1b5d34199b6078f6cf9502fa50e8d5f3 -U drh -Z 1c424b0c5c8b9aa32605259b67546e86 +P ca89daef0fcf6cb04aa6fa90dd333d6f2474bf3f458c833d9cd5bd8e59f2a04a +R b858992430ed0de53c8f23b82b0d78e8 +U dan +Z efdd331472cbc1f3fc97c7776e84ccdd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index da2cfe9570..e6427110e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca89daef0fcf6cb04aa6fa90dd333d6f2474bf3f458c833d9cd5bd8e59f2a04a \ No newline at end of file +cb023fe28560ce0f8c2fd48042553fcdb9db81eba9552be75165de0d46a2645c \ No newline at end of file From 99ace8263426ee9001d3994e6a8275d4fd8b8fc3 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 8 Mar 2023 18:05:57 +0000 Subject: [PATCH 071/341] Export the new SQLITE_CHANGESETAPPLY_IGNORENOOP flag to JS. FossilOrigin-Name: ac7359b2633ead74a53b2796fe038ca285aacad45b45980db2a76a87154e66e3 --- ext/wasm/api/sqlite3-wasm.c | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index fd074541ad..649959a8ff 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -495,6 +495,7 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_CHANGESETSTART_INVERT); DefInt(SQLITE_CHANGESETAPPLY_NOSAVEPOINT); DefInt(SQLITE_CHANGESETAPPLY_INVERT); + DefInt(SQLITE_CHANGESETAPPLY_IGNORENOOP); DefInt(SQLITE_CHANGESET_DATA); DefInt(SQLITE_CHANGESET_NOTFOUND); diff --git a/manifest b/manifest index 57192a45d7..7293a6e16c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_CHANGESETAPPLY_IGNORENOOP\sflag,\swhich\smay\sbe\spassed\sto\ssqlite3changeset_apply_v2()\sto\shave\sit\signore\schanges\sthat\swould\sbe\sno-ops\sif\sapplied\sto\sthe\sdatabase\s(e.g.\sdeleting\sa\srow\sthat\shas\salready\sbeen\sdeleted),\sinstead\sof\sconsidering\sthem\sconflicts. -D 2023-03-08T18:03:04.122 +C Export\sthe\snew\sSQLITE_CHANGESETAPPLY_IGNORENOOP\sflag\sto\sJS. +D 2023-03-08T18:05:57.506 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -490,7 +490,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 26f6240170d415726d9cfe2fa7a0163e153775e1a74fa91c9ba5446502c71097 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 4b09550e1a73f6318b47bdacdbd39045acc23f562fedbdc1d8af1b7473763e7b +F ext/wasm/api/sqlite3-wasm.c b32e40678d34f9787f7ebef07a4156c157ef4fce31c7349f0f6bac36b5df86d1 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2049,8 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ca89daef0fcf6cb04aa6fa90dd333d6f2474bf3f458c833d9cd5bd8e59f2a04a -R b858992430ed0de53c8f23b82b0d78e8 -U dan -Z efdd331472cbc1f3fc97c7776e84ccdd +P cb023fe28560ce0f8c2fd48042553fcdb9db81eba9552be75165de0d46a2645c +R a9df85e2031c92ccc881b96d87614853 +U stephan +Z fbd009d6ed5e486e6195915a41c1fb94 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e6427110e8..7c59ec33b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb023fe28560ce0f8c2fd48042553fcdb9db81eba9552be75165de0d46a2645c \ No newline at end of file +ac7359b2633ead74a53b2796fe038ca285aacad45b45980db2a76a87154e66e3 \ No newline at end of file From 07d1ef97a1891acff9a249f62d294a26089be622 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 8 Mar 2023 22:48:03 +0000 Subject: [PATCH 072/341] Backout the OP_MakeRecord optimization as it does not work. FossilOrigin-Name: 25017312d0d476d9cd5a39835748ee26c2ea482e163264ce2f9843ac627276d6 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/vdbe.c | 34 +++++++++++++++++----------------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 7293a6e16c..dad39608e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Export\sthe\snew\sSQLITE_CHANGESETAPPLY_IGNORENOOP\sflag\sto\sJS. -D 2023-03-08T18:05:57.506 +C Backout\sthe\sOP_MakeRecord\soptimization\sas\sit\sdoes\snot\swork. +D 2023-03-08T22:48:03.746 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -694,7 +694,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 80a64921734e74b1b28b321225137ec555b0e15d73f3322c2c0651a8be481114 +F src/vdbe.c 0cf4c72a9e0eb614afc19c9c4ca9c8a919c97c0866934a70dac7c2f689a4edf8 F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 @@ -2049,8 +2049,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb023fe28560ce0f8c2fd48042553fcdb9db81eba9552be75165de0d46a2645c -R a9df85e2031c92ccc881b96d87614853 -U stephan -Z fbd009d6ed5e486e6195915a41c1fb94 +P ac7359b2633ead74a53b2796fe038ca285aacad45b45980db2a76a87154e66e3 +Q -ca89daef0fcf6cb04aa6fa90dd333d6f2474bf3f458c833d9cd5bd8e59f2a04a +R 5b614ce6b236c8311c427bbe81e4374c +U drh +Z 739ce1ec9014df3e668a3263b66c8258 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7c59ec33b1..752eaba01f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac7359b2633ead74a53b2796fe038ca285aacad45b45980db2a76a87154e66e3 \ No newline at end of file +25017312d0d476d9cd5a39835748ee26c2ea482e163264ce2f9843ac627276d6 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index d224ee2bed..dfe8070de1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3379,7 +3379,23 @@ case OP_MakeRecord: { pRec = pLast; do{ assert( memIsValid(pRec) ); - if( pRec->flags & (MEM_Int|MEM_IntReal) ){ + if( pRec->flags & MEM_Null ){ + if( pRec->flags & MEM_Zero ){ + /* Values with MEM_Null and MEM_Zero are created by xColumn virtual + ** table methods that never invoke sqlite3_result_xxxxx() while + ** computing an unchanging column value in an UPDATE statement. + ** Give such values a special internal-use-only serial-type of 10 + ** so that they can be passed through to xUpdate and have + ** a true sqlite3_value_nochange(). */ +#ifndef SQLITE_ENABLE_NULL_TRIM + assert( pOp->p5==OPFLAG_NOCHNG_MAGIC || CORRUPT_DB ); +#endif + pRec->uTemp = 10; + }else{ + pRec->uTemp = 0; + } + nHdr++; + }else if( pRec->flags & (MEM_Int|MEM_IntReal) ){ /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ i64 i = pRec->u.i; u64 uu; @@ -3429,22 +3445,6 @@ case OP_MakeRecord: { pRec->uTemp = 6; } } - }else if( pRec->flags & MEM_Null ){ - if( pRec->flags & MEM_Zero ){ - /* Values with MEM_Null and MEM_Zero are created by xColumn virtual - ** table methods that never invoke sqlite3_result_xxxxx() while - ** computing an unchanging column value in an UPDATE statement. - ** Give such values a special internal-use-only serial-type of 10 - ** so that they can be passed through to xUpdate and have - ** a true sqlite3_value_nochange(). */ -#ifndef SQLITE_ENABLE_NULL_TRIM - assert( pOp->p5==OPFLAG_NOCHNG_MAGIC || CORRUPT_DB ); -#endif - pRec->uTemp = 10; - }else{ - pRec->uTemp = 0; - } - nHdr++; }else if( pRec->flags & MEM_Real ){ nHdr++; nData += 8; From 8c26e6fa168da6c2bdd5c0f74e4014cb2b165dfe Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 8 Mar 2023 23:05:18 +0000 Subject: [PATCH 073/341] Fix a possible NULL pointer dereference due to the sqlite3_interrupt() enhancement at [bd8fa10e59f58886]. Reported by [forum:/forumpost/f5a2b1db87|forum post f5a2b1db87]. FossilOrigin-Name: 84417bbd144b2197c9930a520feb94b59053957c190be79f8deaaaebca68ecf1 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/trigger.c | 2 +- test/returning1.test | 19 +++++++++++++++---- 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index dad39608e9..7548c98b51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Backout\sthe\sOP_MakeRecord\soptimization\sas\sit\sdoes\snot\swork. -D 2023-03-08T22:48:03.746 +C Fix\sa\spossible\sNULL\spointer\sdereference\sdue\sto\sthe\ssqlite3_interrupt()\nenhancement\sat\s[bd8fa10e59f58886].\s\sReported\sby\n[forum:/forumpost/f5a2b1db87|forum\spost\sf5a2b1db87]. +D 2023-03-08T23:05:18.525 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -688,7 +688,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c fccf3b8c517c1f55cb380c1522febe6921fcb2bd800c16c78cab571d0eb0ccbd -F src/trigger.c 5e68b790f022b8dafbfb0eb244786512a95c9575fc198719d2557d73e5795858 +F src/trigger.c 6072c531d9bcc3980528150a1b03fda2e85a08c10023fafb42f93ffd68607ffe F src/update.c f118e51768d2c1309e3c81e9f91141b22b8a1339cbc5969b1b2d810feaa25b22 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 @@ -1407,7 +1407,7 @@ F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2 F test/releasetest_data.tcl b550dd1b122a9c969df794d05ea272df535f10ff1a245062e7ba080822378016 F test/resetdb.test 54c06f18bc832ac6d6319e5ab23d5c8dd49fdbeec7c696d791682a8006bd5fc3 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test 3ef7b264598b3292be0cdb028e4acb7524c5fd409b33b78449f894dfd68db334 +F test/returning1.test ee72bece38ea39ef82a219d8f6619e5c1c15835ea127cd7e6e30012cde462dfe F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f @@ -2049,9 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac7359b2633ead74a53b2796fe038ca285aacad45b45980db2a76a87154e66e3 -Q -ca89daef0fcf6cb04aa6fa90dd333d6f2474bf3f458c833d9cd5bd8e59f2a04a -R 5b614ce6b236c8311c427bbe81e4374c +P 25017312d0d476d9cd5a39835748ee26c2ea482e163264ce2f9843ac627276d6 +R 04992e5ef08ba70fad9283f737a9de1e U drh -Z 739ce1ec9014df3e668a3263b66c8258 +Z f2ca12f92c53ce29bf6ab3953a4fa6eb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 752eaba01f..ac3cc4a4f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25017312d0d476d9cd5a39835748ee26c2ea482e163264ce2f9843ac627276d6 \ No newline at end of file +84417bbd144b2197c9930a520feb94b59053957c190be79f8deaaaebca68ecf1 \ No newline at end of file diff --git a/src/trigger.c b/src/trigger.c index 02d8540237..f83c719a33 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -983,7 +983,7 @@ static void codeReturningTrigger( } sqlite3ExprListDelete(db, sSelect.pEList); pNew = sqlite3ExpandReturning(pParse, pReturning->pReturnEL, pTab); - if( !db->mallocFailed ){ + if( pParse->nErr==0 ){ NameContext sNC; memset(&sNC, 0, sizeof(sNC)); if( pReturning->nRetCol==0 ){ diff --git a/test/returning1.test b/test/returning1.test index 326af800e1..0d97b70201 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -212,13 +212,13 @@ do_execsql_test 10.2 { END; } -do_catchsql_test 10.3 { +do_catchsql_test 10.3a { INSERT INTO t1(a, b) VALUES(1234, 5678) RETURNING rowid; -} {1 {no such column: rowid}} +} {1 {no such column: new.rowid}} -do_catchsql_test 10.3 { +do_catchsql_test 10.3b { UPDATE t1 SET a='z' WHERE b='y' RETURNING rowid; -} {1 {no such column: rowid}} +} {1 {no such column: new.rowid}} do_execsql_test 10.4 { SELECT * FROM log; @@ -408,4 +408,15 @@ do_execsql_test 17.0 { UPDATE bug SET x=NULL WHERE id = 20 RETURNING quote(x), x IS NULL; } {NULL 1} +# 2023-03-08 https://sqlite.org/forum/forumpost/f5a2b1db87 +# NULL pointer dereference following an error. +# +do_execsql_test 18.0 { + CREATE TABLE v0(c1 INT); + CREATE VIEW view_2(c1) AS SELECT CASE WHEN c1 COLLATE TRUE THEN TRUE ELSE TRUE END FROM v0; +} +do_catchsql_test 18.1 { + INSERT INTO view_2 DEFAULT VALUES RETURNING *; +} {1 {no such collation sequence: TRUE}} + finish_test From fcdfd4716e3f3f16ea94aa7d626aef66d7a43483 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 9 Mar 2023 08:51:04 +0000 Subject: [PATCH 074/341] Experimental addition of sqlite3-node.mjs, for node.js, based on feedback from [forum:ac7a94d4f77db235|forum post ac7a94d4f77db235] and related off-list discussions. Build changes only - no code changes. FossilOrigin-Name: a5db97fa17d15711ab19af70595912d342ffa326a9b7029d4deb2194ae72a9f0 --- ext/wasm/GNUmakefile | 63 ++++++++++++++++++++++++++++++++------------ ext/wasm/dist.make | 3 ++- manifest | 16 +++++------ manifest.uuid | 2 +- 4 files changed, 57 insertions(+), 27 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 250b8be0c7..a99513bfa8 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -44,7 +44,7 @@ # 1) Consolidate the code generation for sqlite3*.*js into a script # which generates the makefile code, rather than using $(call) and # $(eval), or at least centralize the setup of the numerous vars -# related to each build variant (vanilla, esm, bundler-friendly). +# related to each build variant $(JS_BUILD_MODES). # SHELL := $(shell which bash 2>/dev/null) MAKEFILE := $(lastword $(MAKEFILE_LIST)) @@ -52,7 +52,9 @@ CLEAN_FILES := DISTCLEAN_FILES := ./--dummy-- default: all release: oz - +# JS_BUILD_MODES exists solely to reduce repetition in documentation +# below. +JS_BUILD_MODES := vanilla esm bunder-friendly node # Emscripten SDK home dir and related binaries... EMSDK_HOME ?= $(word 1,$(wildcard $(HOME)/emsdk $(HOME)/src/emsdk)) emcc.bin ?= $(word 1,$(wildcard $(EMSDK_HOME)/upstream/emscripten/emcc) $(shell which emcc)) @@ -461,7 +463,16 @@ emcc.jsflags += -sSTRICT_JS=0 # STRICT_JS disabled due to: # https://github.com/emscripten-core/emscripten/issues/18610 # TL;DR: does not work with MODULARIZE or EXPORT_ES6 as of version 3.1.31. -emcc.environment := -sENVIRONMENT=web,worker,node + +# -sENVIRONMENT values for the various build modes: +emcc.environment.vanilla := web,worker +emcc.environment.bundler-friendly := $(emcc.environment.vanilla) +emcc.environment.esm := $(emcc.environment.vanilla) +emcc.environment.node := node +# Note that adding "node" to the list for the other builds causes +# Emscripten to generate code which confuses node: it cannot reliably +# determine whether the build is for a browser or for node. + ######################################################################## # -sINITIAL_MEMORY: How much memory we need to start with is governed # at least in part by whether -sALLOW_MEMORY_GROWTH is enabled. If so, @@ -605,10 +616,11 @@ $(post-js.js.in): $(post-jses.js) $(MAKEFILE) ######################################################################## # call-make-pre-post is a $(call)able which creates rules for # pre-js-$(1).js. $1 = the base name of the JS file on whose behalf -# this pre-js is for (one of: sqlite3, sqlite3-wasmfs). $2 is the build -# mode: one of (vanilla, esm, bundler-friendly). This sets up -# --[extern-][pre/post]-js flags in $(pre-post-$(1).flags.$(2)) and -# dependencies in $(pre-post-$(1).deps.$(2)). +# this pre-js is for (one of: sqlite3, sqlite3-wasmfs). $2 is the +# build mode: one of $(JS_BUILD_MODES). This +# sets up --[extern-][pre/post]-js flags in +# $(pre-post-$(1).flags.$(2)) and dependencies in +# $(pre-post-$(1).deps.$(2)). define call-make-pre-post pre-post-$(1).flags.$(2) ?= $$(dir.tmp)/pre-js-$(1)-$(2).js: $$(pre-js.js.$(2)) $$(MAKEFILE) @@ -681,14 +693,16 @@ pre-post-common.flags := \ pre-post-jses.deps.common := $(extern-pre-js.js) $(sqlite3-license-version.js) ######################################################################## # SETUP_LIB_BUILD_MODE is a $(call)'able which sets up numerous pieces -# for one of the build modes (vanilla, esm, bundler-friendly). +# for one of the build modes. # -# $1 = build mode name +# $1 = build mode name: one of $(JS_BUILD_MODES) # $2 = 1 for ESM build mode, else 0 # $3 = resulting sqlite-api JS/MJS file # $4 = resulting JS/MJS file # $5 = -D... flags for $(bin.c-pp) -# $6 = emcc -sXYZ flags +# $6 = emcc -sXYZ flags (CURRENTLY UNUSED - was factored out) +# +# emcc.environment.$(1) must be set to a value for the -sENVIRONMENT flag. define SETUP_LIB_BUILD_MODE $(info Setting up build [$(1)]: $(4)) c-pp.D.$(1) := $(5) @@ -707,19 +721,21 @@ pre-post-jses.deps.$(1) := $$(pre-post-jses.deps.common) \ $$(eval $$(call call-make-pre-post,sqlite3,$(1))) emcc.flags.sqlite3.$(1) := $(6) $$(eval $$(call C-PP.FILTER, $$(sqlite3-api.js.in), $(3), $(5))) -$(4): $(3) $(4): $(3) $$(MAKEFILE) $$(sqlite3-wasm.cses) $$(EXPORTED_FUNCTIONS.api) $$(pre-post-sqlite3.deps.$(1)) @echo "Building $$@ ..." $$(emcc.bin) -o $$@ $$(emcc_opt_full) $$(emcc.flags) \ $$(emcc.jsflags) \ + -sENVIRONMENT=$$(emcc.environment.$(1)) \ $$(pre-post-sqlite3.flags.$(1)) $$(emcc.flags.sqlite3.$(1)) \ $$(cflags.common) $$(SQLITE_OPT) $$(cflags.wasm_extra_init) $$(sqlite3-wasm.cses) @$$(call SQLITE3.xJS.ESM-EXPORT-DEFAULT,$(2)) - @if [ bundler-friendly = $(1) ]; then \ - echo "Patching $(3) for sqlite3.wasm..."; \ - rm -f $$(dir.dout)/sqlite3-bundler-friendly.wasm; \ - sed -i -e 's/sqlite3-bundler-friendly.wasm/sqlite3.wasm/g' $$@ || exit $$$$?; \ - fi + @case $(1) in \ + bundler-friendly|node) \ + echo "Patching $(3) for sqlite3.wasm..."; \ + rm -f $$(dir.dout)/sqlite3-$(1).wasm; \ + sed -i -e 's/sqlite3-$(1).wasm/sqlite3.wasm/g' $$@ || exit $$$$?; \ + ;; \ + esac chmod -x $$(sqlite3.wasm) $$(maybe-wasm-strip) $$(sqlite3.wasm) @ls -la $@ $$(sqlite3.wasm) @@ -735,6 +751,8 @@ sqlite3-api.mjs := $(dir.dout)/sqlite3-api.mjs sqlite3.mjs := $(dir.dout)/sqlite3.mjs sqlite3-api-bundler-friendly.mjs := $(dir.dout)/sqlite3-api-bundler-friendly.mjs sqlite3-bundler-friendly.mjs := $(dir.dout)/sqlite3-bundler-friendly.mjs +sqlite3-api-node.mjs := $(dir.dout)/sqlite3-api-node.mjs +sqlite3-node.mjs := $(dir.dout)/sqlite3-node.mjs # Maintenance reminder: careful not to introduce spaces around args $1, $2 #$(info $(call SETUP_LIB_BUILD_MODE,vanilla,0, $(sqlite3-api.js), $(sqlite3.js))) $(eval $(call SETUP_LIB_BUILD_MODE,vanilla,0, $(sqlite3-api.js), $(sqlite3.js))) @@ -742,7 +760,10 @@ $(eval $(call SETUP_LIB_BUILD_MODE,esm,1, $(sqlite3-api.mjs), $(sqlite3.mjs), \ -Dtarget=es6-module, -sEXPORT_ES6 -sUSE_ES6_IMPORT_META)) $(eval $(call SETUP_LIB_BUILD_MODE,bundler-friendly,1,\ $(sqlite3-api-bundler-friendly.mjs),$(sqlite3-bundler-friendly.mjs),\ - $(c-pp.D.esm) -Dtarget=es6-bundler-friendly, $(emcc.flags.sqlite3.esm))) + $(c-pp.D.esm) -Dtarget=es6-bundler-friendly)) +$(eval $(call SETUP_LIB_BUILD_MODE,node,1,\ + $(sqlite3-api-node.mjs),$(sqlite3-node.mjs),\ + $(c-pp.D.bundler-friendly) -Dtarget=node)) # The various -D... values used by *.c-pp.js include: # # -Dtarget=es6-module: for all ESM module builds @@ -754,6 +775,13 @@ $(eval $(call SETUP_LIB_BUILD_MODE,bundler-friendly,1,\ # as string literals so that bundlers' static-analysis tools can # find those files and include them in their bundles. # +# -Dtarget=es6-module -Dtarget=es6-bundler-friendly -Dtarget=node: is +# intended for use by node.js for node.js, as opposed to by +# node.js on behalf of a browser. Mixing -sENVIRONMENT=web and +# -sENVIRONMENT=node leads to ambiguity and confusion on node's +# part, as it's unable to reliably determine whether the target is +# a browser or node. +# ######################################################################## ######################################################################## # We have to ensure that we do not build $(sqlite3*.*js) in parallel @@ -766,6 +794,7 @@ $(eval $(call SETUP_LIB_BUILD_MODE,bundler-friendly,1,\ $(sqlite3.wasm): $(sqlite3.js) $(sqlite3.mjs): $(sqlite3.js) $(sqlite3-bundler-friendly.mjs): $(sqlite3.mjs) +$(sqlite3-node.mjs): $(sqlite3.mjs) CLEAN_FILES += $(sqlite3.wasm) ######################################################################## diff --git a/ext/wasm/dist.make b/ext/wasm/dist.make index 7073c24b78..3f99ad5a5b 100644 --- a/ext/wasm/dist.make +++ b/ext/wasm/dist.make @@ -70,7 +70,8 @@ STRIP_K1.js := $(sqlite3-worker1.js) $(sqlite3-worker1-promiser.js) \ $(sqlite3-worker1-bundler-friendly.js) $(sqlite3-worker1-promiser-bundler-friendly.js) # STRIP_K2.js = list of JS files which need to be passed through # $(bin.stripcomments) with two -k flags. -STRIP_K2.js := $(sqlite3.js) $(sqlite3.mjs) $(sqlite3-bundler-friendly.mjs) +STRIP_K2.js := $(sqlite3.js) $(sqlite3.mjs) \ + $(sqlite3-bundler-friendly.mjs) $(sqlite3-node.mjs) ######################################################################## # dist: create the end-user deliverable archive. # diff --git a/manifest b/manifest index 7548c98b51..a9a799a9c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spossible\sNULL\spointer\sdereference\sdue\sto\sthe\ssqlite3_interrupt()\nenhancement\sat\s[bd8fa10e59f58886].\s\sReported\sby\n[forum:/forumpost/f5a2b1db87|forum\spost\sf5a2b1db87]. -D 2023-03-08T23:05:18.525 +C Experimental\saddition\sof\ssqlite3-node.mjs,\sfor\snode.js,\sbased\son\sfeedback\sfrom\s[forum:ac7a94d4f77db235|forum\spost\sac7a94d4f77db235]\sand\srelated\soff-list\sdiscussions.\sBuild\schanges\sonly\s-\sno\scode\schanges. +D 2023-03-09T08:51:04.021 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -468,7 +468,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile 97f01bb84b0b745e2ba642e12ab24a682e369538e8d68b6e67bdbe502450fc6c +F ext/wasm/GNUmakefile 38700d5074af690f004e4e5f3533164ab49693b9d0832929c4ecf97a0bc09494 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -509,7 +509,7 @@ F ext/wasm/demo-worker1-promiser.html 1de7c248c7c2cfd4a5783d2aa154bce62d74c6de98 F ext/wasm/demo-worker1-promiser.js 51b02509a109e82f623fb4c900c8b48b9a77cc13fbd038396f9a083b86593ae3 F ext/wasm/demo-worker1.html 2c178c1890a2beb5a5fecb1453e796d067a4b8d3d2a04d65ca2eb1ab2c68ef5d F ext/wasm/demo-worker1.js 2c7794d8bc4ab9ecf9cdc2c15de940b11a006942226e441ea41edd458dfc0a26 -F ext/wasm/dist.make f55f9c9e1980ea11a59964e59535c66175a17f004d1c2e274522c3366b3a084a +F ext/wasm/dist.make 451fb1b732257849f6e898d2a862512a0401500ed369ef53bdfeddf9c77bc3b9 F ext/wasm/example_extra_init.c 2347cd69d19d839ef4e5e77b7855103a7fe3ef2af86f2e8c95839afd8b05862f F ext/wasm/fiddle.make dbe36b90b8907ae28ecb9c0e9fd8389dbdaecf117ea4fb2ea33864bdfa498a94 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f @@ -2049,8 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 25017312d0d476d9cd5a39835748ee26c2ea482e163264ce2f9843ac627276d6 -R 04992e5ef08ba70fad9283f737a9de1e -U drh -Z f2ca12f92c53ce29bf6ab3953a4fa6eb +P 84417bbd144b2197c9930a520feb94b59053957c190be79f8deaaaebca68ecf1 +R 9c34fcd2bec4c96e1de9e3017b1c7eba +U stephan +Z 735017807a470df527691c6295a81ba6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ac3cc4a4f2..77410f6e6a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84417bbd144b2197c9930a520feb94b59053957c190be79f8deaaaebca68ecf1 \ No newline at end of file +a5db97fa17d15711ab19af70595912d342ffa326a9b7029d4deb2194ae72a9f0 \ No newline at end of file From 1211e1d112cd55421322d98d6fba64afbff83b5d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 9 Mar 2023 15:08:06 +0000 Subject: [PATCH 075/341] Fix countofview.test so that it works with SQLITE_OMIT_PROGRESS_CALLBACK builds. FossilOrigin-Name: 2fc7c3fcee05c2a251ceb3666f3f6e9014cfe6e2f8570b72c43f251067e6b252 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/countofview.test | 14 ++++++++------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a9a799a9c8..7eedebd097 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\saddition\sof\ssqlite3-node.mjs,\sfor\snode.js,\sbased\son\sfeedback\sfrom\s[forum:ac7a94d4f77db235|forum\spost\sac7a94d4f77db235]\sand\srelated\soff-list\sdiscussions.\sBuild\schanges\sonly\s-\sno\scode\schanges. -D 2023-03-09T08:51:04.021 +C Fix\scountofview.test\sso\sthat\sit\sworks\swith\sSQLITE_OMIT_PROGRESS_CALLBACK\sbuilds. +D 2023-03-09T15:08:06.155 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -894,7 +894,7 @@ F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 -F test/countofview.test 5ecb4ab5449683c5eaa315620fc17838c0f2a58001f3979fcdf9f56cd6af83f5 +F test/countofview.test f9025dba8a9bb673a1f3ce25405900f630f81592c226546242df1ef1e62d0192 F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651 @@ -2049,8 +2049,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 84417bbd144b2197c9930a520feb94b59053957c190be79f8deaaaebca68ecf1 -R 9c34fcd2bec4c96e1de9e3017b1c7eba -U stephan -Z 735017807a470df527691c6295a81ba6 +P a5db97fa17d15711ab19af70595912d342ffa326a9b7029d4deb2194ae72a9f0 +R 3676734d6a6a7d185c9f818259772398 +U dan +Z dc232d0f3fec0eb69d0d02fdfd511211 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 77410f6e6a..30323296f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a5db97fa17d15711ab19af70595912d342ffa326a9b7029d4deb2194ae72a9f0 \ No newline at end of file +2fc7c3fcee05c2a251ceb3666f3f6e9014cfe6e2f8570b72c43f251067e6b252 \ No newline at end of file diff --git a/test/countofview.test b/test/countofview.test index bc5633f7a5..7e94fa5ee7 100644 --- a/test/countofview.test +++ b/test/countofview.test @@ -54,12 +54,14 @@ do_execsql_test 2.1 { # 2023-03-01 dbsqlfuzz ef8623915d843b150c159166ee4548c78cc6895a # count-of-view should not apply to CTEs. # -proc progress_stop args {return 1} -db progress 1000 progress_stop -do_catchsql_test 3.1 { - WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c) - SELECT count(*) FROM c; -} {1 interrupted} +ifcapable progress { + proc progress_stop args {return 1} + db progress 1000 progress_stop + do_catchsql_test 3.1 { + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c) + SELECT count(*) FROM c; + } {1 interrupted} +} # 2023-03-07 dbsqlfuzz 23d782160b71c3f8f535ccb2da313dfc8eb8c631 # From bad7d5041a2bb71f3d3776ef153e472cf9975ce6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 9 Mar 2023 16:11:43 +0000 Subject: [PATCH 076/341] Reinstate some test cases accidentally removed by [cb023fe28560ce0f]. FossilOrigin-Name: 870de61f8ef8781f2f9969b012f5c1cb95b6bce1a9a4dcaf02945b7846c3aa83 --- ext/session/sessionnoop.test | 289 +++++++++++++++++----------------- ext/session/sessionnoop2.test | 180 +++++++++++++++++++++ manifest | 13 +- manifest.uuid | 2 +- 4 files changed, 336 insertions(+), 148 deletions(-) create mode 100644 ext/session/sessionnoop2.test diff --git a/ext/session/sessionnoop.test b/ext/session/sessionnoop.test index 5549773440..16c60b7abf 100644 --- a/ext/session/sessionnoop.test +++ b/ext/session/sessionnoop.test @@ -1,4 +1,4 @@ -# 2011 March 07 +# 2021 Februar 20 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -20,159 +20,166 @@ ifcapable !session {finish_test; return} set testprefix sessionnoop -foreach {tn wo} { - 1 "" - 2 " WITHOUT ROWID " +#------------------------------------------------------------------------- +# Test plan: +# +# 1.*: Test that concatenating changesets cannot produce a noop UPDATE. +# 2.*: Test that rebasing changesets cannot produce a noop UPDATE. +# 3.*: Test that sqlite3changeset_apply() ignores noop UPDATE changes. +# + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d); + INSERT INTO t1 VALUES(1, 1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3, 3); +} + +proc do_concat_test {tn sql1 sql2 res} { + uplevel [list do_test $tn [subst -nocommands { + set C1 [changeset_from_sql {$sql1}] + set C2 [changeset_from_sql {$sql2}] + set C3 [sqlite3changeset_concat [set C1] [set C2]] + set got [list] + sqlite3session_foreach elem [set C3] { lappend got [set elem] } + set got + }] [list {*}$res]] +} + +do_concat_test 1.1 { + UPDATE t1 SET c=c+1; +} { + UPDATE t1 SET c=c-1; } { - reset_db - eval [string map [list %WO% $wo] { -do_execsql_test $tn.1.0 { - CREATE TABLE t1(a PRIMARY KEY, b, c) %WO%; - INSERT INTO t1 VALUES('a', 'A', 'AAA'); - INSERT INTO t1 VALUES('b', 'B', 'BBB'); - INSERT INTO t1 VALUES('c', 'C', 'CCC'); - INSERT INTO t1 VALUES('d', 'D', 'DDD'); - INSERT INTO t1 VALUES('e', 'E', 'EEE'); } -forcedelete test.db2 -sqlite3 db2 test.db2 - -do_execsql_test -db db2 $tn.1.1 { - CREATE TABLE t1(a PRIMARY KEY, b, c) %WO%; - INSERT INTO t1 VALUES('a', 'A', 'AAA'); - INSERT INTO t1 VALUES('b', 'B', '123'); - INSERT INTO t1 VALUES('c', 'C', 'CCC'); - INSERT INTO t1 VALUES('e', 'E', 'EEE'); - INSERT INTO t1 VALUES('f', 'F', 'FFF'); -} - -set C [changeset_from_sql { - UPDATE t1 SET c='123' WHERE a='b'; - DELETE FROM t1 WHERE a='d'; - INSERT INTO t1 VALUES('f', 'F', 'FFF'); -}] - - -set ::conflict_list [list] -proc xConflict {args} { - lappend ::conflict_list $args - return "OMIT" -} -do_test $tn.1.2 { - sqlite3changeset_apply_v2 db2 $C xConflict - set ::conflict_list -} [list {*}{ - {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 123}} - {INSERT t1 CONFLICT {t f t F t FFF} {t f t F t FFF}} - {DELETE t1 NOTFOUND {t d t D t DDD}} -}] -do_test $tn.1.3 { - set ::conflict_list [list] - sqlite3changeset_apply_v2 db2 $C xConflict - set ::conflict_list -} [list {*}{ - {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 123}} - {INSERT t1 CONFLICT {t f t F t FFF} {t f t F t FFF}} - {DELETE t1 NOTFOUND {t d t D t DDD}} -}] - -do_test $tn.1.4 { - set ::conflict_list [list] - sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict - set ::conflict_list -} {} - -do_execsql_test -db db2 1.5 { - UPDATE t1 SET b='G' WHERE a='f'; - UPDATE t1 SET c='456' WHERE a='b'; -} - -do_test $tn.1.6 { - set ::conflict_list [list] - sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict - set ::conflict_list -} [list {*}{ - {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 456}} - {INSERT t1 CONFLICT {t f t F t FFF} {t f t G t FFF}} -}] - -db2 close - -#-------------------------------------------------------------------------- - -reset_db -forcedelete test.db2 -sqlite3 db2 test.db2 -do_execsql_test $tn.2.0 { - CREATE TABLE t1(a PRIMARY KEY, b) %WO%; -} -do_execsql_test -db db2 $tn.2.1 { - CREATE TABLE t1(a PRIMARY KEY, b, c DEFAULT 'val') %WO%; -} - -do_test $tn.2.2 { - do_then_apply_sql -ignorenoop { - INSERT INTO t1 VALUES(1, 2); - } - do_then_apply_sql -ignorenoop { - UPDATE t1 SET b=2 WHERE a=1 - } -} {} - - -}] -} - -db2 close - #------------------------------------------------------------------------- reset_db -forcedelete test.db2 +do_execsql_test 2.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); +} + +proc do_rebase_test {tn sql_local sql_remote conflict_res expected} { + proc xConflict {args} [list return $conflict_res] + + uplevel [list \ + do_test $tn [subst -nocommands { + execsql BEGIN + set c_remote [changeset_from_sql {$sql_remote}] + execsql ROLLBACK + + execsql BEGIN + set c_local [changeset_from_sql {$sql_local}] + set base [sqlite3changeset_apply_v2 db [set c_remote] xConflict] + execsql ROLLBACK + + sqlite3rebaser_create R + R config [set base] + set res [list] + sqlite3session_foreach elem [R rebase [set c_local]] { + lappend res [set elem] + } + R delete + set res + }] [list {*}$expected] + ] +} + +do_rebase_test 2.1 { + UPDATE t1 SET c=2 WHERE a=1; -- local +} { + UPDATE t1 SET c=3 WHERE a=1; -- remote +} OMIT { + {UPDATE t1 0 X.. {i 1 {} {} i 3} {{} {} {} {} i 2}} +} + +do_rebase_test 2.2 { + UPDATE t1 SET c=2 WHERE a=1; -- local +} { + UPDATE t1 SET c=3 WHERE a=1; -- remote +} REPLACE { +} + +do_rebase_test 2.3.1 { + UPDATE t1 SET c=4 WHERE a=1; -- local +} { + UPDATE t1 SET c=4 WHERE a=1 -- remote +} OMIT { + {UPDATE t1 0 X.. {i 1 {} {} i 4} {{} {} {} {} i 4}} +} + +do_rebase_test 2.3.2 { + UPDATE t1 SET c=5 WHERE a=1; -- local +} { + UPDATE t1 SET c=5 WHERE a=1 -- remote +} REPLACE { +} + +#------------------------------------------------------------------------- +# +reset_db do_execsql_test 3.0 { - CREATE TABLE xyz(a, b, c, PRIMARY KEY(a, b), UNIQUE(c)); - ANALYZE; - WITH s(i) AS ( - VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100 - ) - INSERT INTO xyz SELECT i, i, i FROM s; - VACUUM INTO 'test.db2'; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + INSERT INTO t1 VALUES(4, 4, 4); } -set C [changeset_from_sql { ANALYZE }] -sqlite3 db2 test.db2 +# Arg $pkstr contains one character for each column in the table. An +# "X" for PK column, or a "." for a non-PK. +# +proc mk_tbl_header {name pkstr} { + set ret [binary format H2c 54 [string length $pkstr]] + foreach i [split $pkstr {}] { + if {$i=="X"} { + append ret [binary format H2 01] + } else { + if {$i!="."} {error "bad pkstr: $pkstr ($i)"} + append ret [binary format H2 00] + } + } + append ret $name + append ret [binary format H2 00] + set ret +} -set ::conflict_list [list] -proc xConflict {args} { lappend ::conflict_list $args ; return "OMIT" } +proc mk_update_change {args} { + set ret [binary format H2H2 17 00] + foreach a $args { + if {$a==""} { + append ret [binary format H2 00] + } else { + append ret [binary format H2W 01 $a] + } + } + set ret +} + +proc xConflict {args} { return "ABORT" } do_test 3.1 { - sqlite3changeset_apply_v2 db2 $C xConflict - set ::conflict_list + set C [mk_tbl_header t1 X..] + append C [mk_update_change 1 {} 1 {} {} 500] + append C [mk_update_change 2 {} {} {} {} {}] + append C [mk_update_change 3 3 {} {} 600 {}] + append C [mk_update_change 4 {} {} {} {} {}] + + sqlite3changeset_apply_v2 db $C xConflict } {} - -do_test 3.2 { - sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict - set ::conflict_list -} {} - -do_test 3.3 { - sqlite3changeset_apply_v2 db2 $C xConflict - set ::conflict_list -} [list {*}{ - {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}} {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}}} - {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_2 t {100 1}} {t xyz t sqlite_autoindex_xyz_2 t {100 1}}} -}] - -do_execsql_test -db db2 3.4 { - UPDATE sqlite_stat1 SET stat='200 1 1' WHERE idx='sqlite_autoindex_xyz_1'; +do_execsql_test 3.2 { + SELECT * FROM t1 +} { + 1 1 500 + 2 2 2 + 3 600 3 + 4 4 4 } -do_test 3.5 { - set ::conflict_list [list] - sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict - set ::conflict_list -} [list {*}{ - {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}} {t xyz t sqlite_autoindex_xyz_1 t {200 1 1}}} -}] + + diff --git a/ext/session/sessionnoop2.test b/ext/session/sessionnoop2.test new file mode 100644 index 0000000000..5549773440 --- /dev/null +++ b/ext/session/sessionnoop2.test @@ -0,0 +1,180 @@ +# 2011 March 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionnoop + +foreach {tn wo} { + 1 "" + 2 " WITHOUT ROWID " +} { + reset_db + eval [string map [list %WO% $wo] { +do_execsql_test $tn.1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c) %WO%; + INSERT INTO t1 VALUES('a', 'A', 'AAA'); + INSERT INTO t1 VALUES('b', 'B', 'BBB'); + INSERT INTO t1 VALUES('c', 'C', 'CCC'); + INSERT INTO t1 VALUES('d', 'D', 'DDD'); + INSERT INTO t1 VALUES('e', 'E', 'EEE'); +} + +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_execsql_test -db db2 $tn.1.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c) %WO%; + INSERT INTO t1 VALUES('a', 'A', 'AAA'); + INSERT INTO t1 VALUES('b', 'B', '123'); + INSERT INTO t1 VALUES('c', 'C', 'CCC'); + INSERT INTO t1 VALUES('e', 'E', 'EEE'); + INSERT INTO t1 VALUES('f', 'F', 'FFF'); +} + +set C [changeset_from_sql { + UPDATE t1 SET c='123' WHERE a='b'; + DELETE FROM t1 WHERE a='d'; + INSERT INTO t1 VALUES('f', 'F', 'FFF'); +}] + + +set ::conflict_list [list] +proc xConflict {args} { + lappend ::conflict_list $args + return "OMIT" +} +do_test $tn.1.2 { + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} [list {*}{ + {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 123}} + {INSERT t1 CONFLICT {t f t F t FFF} {t f t F t FFF}} + {DELETE t1 NOTFOUND {t d t D t DDD}} +}] +do_test $tn.1.3 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} [list {*}{ + {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 123}} + {INSERT t1 CONFLICT {t f t F t FFF} {t f t F t FFF}} + {DELETE t1 NOTFOUND {t d t D t DDD}} +}] + +do_test $tn.1.4 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} {} + +do_execsql_test -db db2 1.5 { + UPDATE t1 SET b='G' WHERE a='f'; + UPDATE t1 SET c='456' WHERE a='b'; +} + +do_test $tn.1.6 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} [list {*}{ + {UPDATE t1 DATA {t b {} {} t BBB} {{} {} {} {} t 123} {t b t B t 456}} + {INSERT t1 CONFLICT {t f t F t FFF} {t f t G t FFF}} +}] + +db2 close + +#-------------------------------------------------------------------------- + +reset_db +forcedelete test.db2 +sqlite3 db2 test.db2 +do_execsql_test $tn.2.0 { + CREATE TABLE t1(a PRIMARY KEY, b) %WO%; +} +do_execsql_test -db db2 $tn.2.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c DEFAULT 'val') %WO%; +} + +do_test $tn.2.2 { + do_then_apply_sql -ignorenoop { + INSERT INTO t1 VALUES(1, 2); + } + do_then_apply_sql -ignorenoop { + UPDATE t1 SET b=2 WHERE a=1 + } +} {} + + +}] +} + +db2 close + +#------------------------------------------------------------------------- +reset_db +forcedelete test.db2 +do_execsql_test 3.0 { + CREATE TABLE xyz(a, b, c, PRIMARY KEY(a, b), UNIQUE(c)); + ANALYZE; + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO xyz SELECT i, i, i FROM s; + VACUUM INTO 'test.db2'; +} + +set C [changeset_from_sql { ANALYZE }] +sqlite3 db2 test.db2 + +set ::conflict_list [list] +proc xConflict {args} { lappend ::conflict_list $args ; return "OMIT" } +do_test 3.1 { + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} {} + +do_test 3.2 { + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} {} + +do_test 3.3 { + sqlite3changeset_apply_v2 db2 $C xConflict + set ::conflict_list +} [list {*}{ + {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}} {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}}} + {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_2 t {100 1}} {t xyz t sqlite_autoindex_xyz_2 t {100 1}}} +}] + +do_execsql_test -db db2 3.4 { + UPDATE sqlite_stat1 SET stat='200 1 1' WHERE idx='sqlite_autoindex_xyz_1'; +} + +do_test 3.5 { + set ::conflict_list [list] + sqlite3changeset_apply_v2 -ignorenoop db2 $C xConflict + set ::conflict_list +} [list {*}{ + {INSERT sqlite_stat1 CONFLICT {t xyz t sqlite_autoindex_xyz_1 t {100 1 1}} {t xyz t sqlite_autoindex_xyz_1 t {200 1 1}}} +}] + + + +finish_test + diff --git a/manifest b/manifest index 7eedebd097..395629ab8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scountofview.test\sso\sthat\sit\sworks\swith\sSQLITE_OMIT_PROGRESS_CALLBACK\sbuilds. -D 2023-03-09T15:08:06.155 +C Reinstate\ssome\stest\scases\saccidentally\sremoved\sby\s[cb023fe28560ce0f]. +D 2023-03-09T16:11:43.576 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -456,7 +456,8 @@ F ext/session/sessionfault.test 573bf027fb870d57bd4e7cf50822a3e4b17b2b9234074387 F ext/session/sessionfault2.test b0d6a7c1d7398a7e800d84657404909c7d385965ea8576dc79ed344c46fbf41c F ext/session/sessioninvert.test 04075517a9497a80d39c495ba6b44f3982c7371129b89e2c52219819bc105a25 F ext/session/sessionmem.test f2a735db84a3e9e19f571033b725b0b2daf847f3f28b1da55a0c1a4e74f1de09 -F ext/session/sessionnoop.test 5c9a882219e54711c98dccd2fd81392f189a59325e4fb5d8ed25e33a0c2f0ba2 +F ext/session/sessionnoop.test a9366a36a95ef85f8a3687856ebef46983df399541174cb1ede2ee53b8011bc7 +F ext/session/sessionnoop2.test 5c9a882219e54711c98dccd2fd81392f189a59325e4fb5d8ed25e33a0c2f0ba2 F ext/session/sessionrebase.test ccfa716b23bd1d3b03217ee58cfd90c78d4b99f53e6a9a2f05e82363b9142810 F ext/session/sessionsize.test 6f644aff31c7f1e4871e9ff3542766e18da68fc7e587b83a347ea9820a002dd8 F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c @@ -2049,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a5db97fa17d15711ab19af70595912d342ffa326a9b7029d4deb2194ae72a9f0 -R 3676734d6a6a7d185c9f818259772398 +P 2fc7c3fcee05c2a251ceb3666f3f6e9014cfe6e2f8570b72c43f251067e6b252 +R 9fdb5fa7fb7a117f6c9f957b7fddcae2 U dan -Z dc232d0f3fec0eb69d0d02fdfd511211 +Z 5974dbe29f84c67ef0a243c359b4746e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 30323296f9..118682aaf8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2fc7c3fcee05c2a251ceb3666f3f6e9014cfe6e2f8570b72c43f251067e6b252 \ No newline at end of file +870de61f8ef8781f2f9969b012f5c1cb95b6bce1a9a4dcaf02945b7846c3aa83 \ No newline at end of file From 434205a2c152516b3363ca6ca1542415e72be88c Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 9 Mar 2023 22:09:13 +0000 Subject: [PATCH 077/341] Replace a lingering use of 'self' with 'globalThis' in JS code, for node compatibility. FossilOrigin-Name: 7e3782b5aa07621db95c2dc25b25ae21da988c9876d537b78ea289a83c75b06b --- ext/wasm/api/extern-post-js.c-pp.js | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/wasm/api/extern-post-js.c-pp.js b/ext/wasm/api/extern-post-js.c-pp.js index 14d4349912..927bf64f9e 100644 --- a/ext/wasm/api/extern-post-js.c-pp.js +++ b/ext/wasm/api/extern-post-js.c-pp.js @@ -42,7 +42,7 @@ const toExportForESM = is called. */ const initModuleState = globalThis.sqlite3InitModuleState = Object.assign(Object.create(null),{ - moduleScript: self?.document?.currentScript, + moduleScript: globalThis?.document?.currentScript, isWorker: ('undefined' !== typeof WorkerGlobalScope), location: globalThis.location, urlParams: globalThis?.location?.href @@ -63,7 +63,7 @@ const toExportForESM = } globalThis.sqlite3InitModule = function ff(...args){ - //console.warn("Using replaced sqlite3InitModule()",self.location); + //console.warn("Using replaced sqlite3InitModule()",globalThis.location); return originalInit(...args).then((EmscriptenModule)=>{ if('undefined'!==typeof WorkerGlobalScope && (EmscriptenModule['ENVIRONMENT_IS_PTHREAD'] diff --git a/manifest b/manifest index 395629ab8d..c4c70314b1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reinstate\ssome\stest\scases\saccidentally\sremoved\sby\s[cb023fe28560ce0f]. -D 2023-03-09T16:11:43.576 +C Replace\sa\slingering\suse\sof\s'self'\swith\s'globalThis'\sin\sJS\scode,\sfor\snode\scompatibility. +D 2023-03-09T22:09:13.397 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -476,7 +476,7 @@ F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2 F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-see fb29e62082a658f0d81102488414d422c393c4b20cc2f685b216bc566237957b F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 77a2f1f2fc60a35def7455dffc8d3f2c56385d6ac5c6cecc60fa938252ea2c54 -F ext/wasm/api/extern-post-js.c-pp.js 5c4997d3442756e4e4819303fa4a7045de4a2a7b79c3ad6c26cdcf1d9141fac6 +F ext/wasm/api/extern-post-js.c-pp.js 393ab78b807da94096eae1a68bfddb999a2299936a185d910162fe87a57a9a3a F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e7a4d11d43c5c8f41 F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2fc7c3fcee05c2a251ceb3666f3f6e9014cfe6e2f8570b72c43f251067e6b252 -R 9fdb5fa7fb7a117f6c9f957b7fddcae2 -U dan -Z 5974dbe29f84c67ef0a243c359b4746e +P 870de61f8ef8781f2f9969b012f5c1cb95b6bce1a9a4dcaf02945b7846c3aa83 +R adc352d96ddebf0239be405076193c61 +U stephan +Z e97aa026ed23206198357485225fe5b1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 118682aaf8..9e837ad49e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -870de61f8ef8781f2f9969b012f5c1cb95b6bce1a9a4dcaf02945b7846c3aa83 \ No newline at end of file +7e3782b5aa07621db95c2dc25b25ae21da988c9876d537b78ea289a83c75b06b \ No newline at end of file From 17bbbde537536dd05ddee2d19ad430613d7640f9 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 10 Mar 2023 11:57:23 +0000 Subject: [PATCH 078/341] Export SQLITE_FCNTL_RESET_CACHE to JS. FossilOrigin-Name: 6195cfc86b15614b8db0e0dc5cc79b8d1acaf483f0131c8526992dc8ca075630 --- ext/wasm/api/sqlite3-wasm.c | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 649959a8ff..a3b349e5ff 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -638,6 +638,7 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_FCNTL_CKPT_START); DefInt(SQLITE_FCNTL_EXTERNAL_READER); DefInt(SQLITE_FCNTL_CKSM_FILE); + DefInt(SQLITE_FCNTL_RESET_CACHE); } _DefGroup; DefGroup(flock) { diff --git a/manifest b/manifest index c4c70314b1..9704d5a64b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\sa\slingering\suse\sof\s'self'\swith\s'globalThis'\sin\sJS\scode,\sfor\snode\scompatibility. -D 2023-03-09T22:09:13.397 +C Export\sSQLITE_FCNTL_RESET_CACHE\sto\sJS. +D 2023-03-10T11:57:23.758 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 26f6240170d415726d9cfe2fa7a0163e153775e1a74fa91c9ba5446502c71097 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c b32e40678d34f9787f7ebef07a4156c157ef4fce31c7349f0f6bac36b5df86d1 +F ext/wasm/api/sqlite3-wasm.c e04e863be7c3a58aa144a463770c1b68ce69f7cfc9e1a5d6397a67877acaf006 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 870de61f8ef8781f2f9969b012f5c1cb95b6bce1a9a4dcaf02945b7846c3aa83 -R adc352d96ddebf0239be405076193c61 +P 7e3782b5aa07621db95c2dc25b25ae21da988c9876d537b78ea289a83c75b06b +R bc96caed61337068c801c690a60c3eac U stephan -Z e97aa026ed23206198357485225fe5b1 +Z ab2e7fad50105ec607d48449b7ca6e1c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9e837ad49e..a262ce9617 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e3782b5aa07621db95c2dc25b25ae21da988c9876d537b78ea289a83c75b06b \ No newline at end of file +6195cfc86b15614b8db0e0dc5cc79b8d1acaf483f0131c8526992dc8ca075630 \ No newline at end of file From 5f708cc83202076c47a6204f3160849c1b7177a2 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Mar 2023 13:36:19 +0000 Subject: [PATCH 079/341] Fix a problem with the fts5 snippet() function that shows up when snippets just 1 token in length are requested. FossilOrigin-Name: 96d5116d17cbf3de6be38952881c6ddf6a226ddfd7649197bdb2df39c42b5d8e --- ext/fts5/fts5_aux.c | 8 +++++--- ext/fts5/test/fts5af.test | 30 ++++++++++++++++++++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 77f6d5baba..b178f47334 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -163,7 +163,7 @@ static int fts5HighlightCb( if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; iPos = p->iPos++; - if( p->iRangeEnd>0 ){ + if( p->iRangeEnd>=0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; } @@ -175,7 +175,7 @@ static int fts5HighlightCb( } if( iPos==p->iter.iEnd ){ - if( p->iRangeEnd && p->iter.iStartiRangeStart ){ + if( p->iRangeEnd>=0 && p->iter.iStartiRangeStart ){ fts5HighlightAppend(&rc, p, p->zOpen, -1); } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); @@ -186,7 +186,7 @@ static int fts5HighlightCb( } } - if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ + if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; if( iPos>=p->iter.iStart && iPositer.iEnd ){ @@ -221,6 +221,7 @@ static void fts5HighlightFunction( memset(&ctx, 0, sizeof(HighlightContext)); ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + ctx.iRangeEnd = -1; rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); if( ctx.zIn ){ @@ -406,6 +407,7 @@ static void fts5SnippetFunction( iCol = sqlite3_value_int(apVal[0]); ctx.zOpen = fts5ValueToText(apVal[1]); ctx.zClose = fts5ValueToText(apVal[2]); + ctx.iRangeEnd = -1; zEllips = fts5ValueToText(apVal[3]); nToken = sqlite3_value_int(apVal[4]); diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index a3ff330ef3..3d79295092 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -193,4 +193,34 @@ do_execsql_test 5.6 { } ;# foreach_detail_mode +reset_db +do_execsql_test 6.0 { + CREATE VIRTUAL TABLE t1 USING fts5(colA, colB); + INSERT INTO t1 VALUES('A B C', 'D E F'); +} + +do_execsql_test 6.1 { + SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'B'; +} {{A B C} {D E F} ...[B]...} +breakpoint +do_execsql_test 6.2 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'B'; +} {{A B C} {D E F} {D E...}} +do_execsql_test 6.3 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'B'; +} {{A B C} {D E F} {D...}} + +do_execsql_test 6.1 { + SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'A'; +} {{A B C} {D E F} [A]...} +breakpoint +do_execsql_test 6.2 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'A'; +} {{A B C} {D E F} {D E...}} +do_execsql_test 6.3 { + SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'A'; +} {{A B C} {D E F} {D...}} + + + finish_test diff --git a/manifest b/manifest index 9704d5a64b..fa653f3ab4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Export\sSQLITE_FCNTL_RESET_CACHE\sto\sJS. -D 2023-03-10T11:57:23.758 +C Fix\sa\sproblem\swith\sthe\sfts5\ssnippet()\sfunction\sthat\sshows\sup\swhen\ssnippets\sjust\s1\stoken\sin\slength\sare\srequested. +D 2023-03-10T13:36:19.548 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -87,7 +87,7 @@ F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a0 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a F ext/fts5/fts5Int.h c0d46e399e345e35985b72a1c1af025973bfaa5b1e3563b0ce3bb0ce144a7ca3 -F ext/fts5/fts5_aux.c f558e1fb9f0f86a4f7489e258c162e1f947de5ff2709087fbb465fddb7092f98 +F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 501e7d3566bc92766b0e11c0109a7c5a6146bc41144195459af5422f6c2078aa F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 @@ -110,7 +110,7 @@ F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de F ext/fts5/test/fts5ae.test 1142d16d9cc193894dc13cc8f9c7a8a21411ac61b5567a878514df6f9f0d7bb7 -F ext/fts5/test/fts5af.test bea75184c0e63631b552c20ebe4a631699f357e00a2059c92538f7aeece8291e +F ext/fts5/test/fts5af.test 2329b6c6e6e9243371022dd9439892ff1850b590ae9ce073e3a83eefaf993a81 F ext/fts5/test/fts5ag.test 7816f25a0707578f08145ab539fc0ca025f8951e788b28a6a18a06b2099469dd F ext/fts5/test/fts5ah.test 2f047dfe89dc8611fa53e3d8bfc453b79cff037aa423c8d171e91e645745aa2c F ext/fts5/test/fts5ai.test bc97e4758cc93e06bf851d61c98fdf4e8b8f8315ee28a84fb15f916360856414 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7e3782b5aa07621db95c2dc25b25ae21da988c9876d537b78ea289a83c75b06b -R bc96caed61337068c801c690a60c3eac -U stephan -Z ab2e7fad50105ec607d48449b7ca6e1c +P 6195cfc86b15614b8db0e0dc5cc79b8d1acaf483f0131c8526992dc8ca075630 +R 0cbc439df8a4bc1455473d9bc06be876 +U dan +Z 82bb6cdaa19cf778ce3d38362f6d2ead # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a262ce9617..55453226f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6195cfc86b15614b8db0e0dc5cc79b8d1acaf483f0131c8526992dc8ca075630 \ No newline at end of file +96d5116d17cbf3de6be38952881c6ddf6a226ddfd7649197bdb2df39c42b5d8e \ No newline at end of file From a9fffc179f038435903a6e81fe703848eabf6639 Mon Sep 17 00:00:00 2001 From: larrybr Date: Fri, 10 Mar 2023 20:54:44 +0000 Subject: [PATCH 080/341] Give CLI a no-more-options option. (--) FossilOrigin-Name: 0822788752621f6bf6af44b420b594ddd352634b3b0ed0eb835abea34b45817a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c.in | 11 ++++++++--- test/shell1.test | 10 ++++++++++ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index fa653f3ab4..ae1b495b4e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sthe\sfts5\ssnippet()\sfunction\sthat\sshows\sup\swhen\ssnippets\sjust\s1\stoken\sin\slength\sare\srequested. -D 2023-03-10T13:36:19.548 +C Give\sCLI\sa\sno-more-options\soption.\s(--) +D 2023-03-10T20:54:44.804 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c d4dd7843ce2fe0b886c3970b34a12330369fbfc7c86a5fece5a8ae02ae157038 -F src/shell.c.in d64f0d3822aa94508285f2bcce185b12ace06f37c953371cb0f0d835c171753c +F src/shell.c.in c289de5d93ea2531bc8158e64ea8b6278ab87f953d185a811f4a7ae3ea310d84 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -1482,7 +1482,7 @@ F test/sharedA.test 64bdd21216dda2c6a3bd3475348ccdc108160f34682c97f2f51c19fc0e21 F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8ee707 F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test ab88e763854ea4734796067e20e258589b4d5bb9e77d70d8c4c8e99cf77c8b64 +F test/shell1.test 291c5c4b313adbe44d847af78b730a3bbfa1598c450e09ffe5601170bf7f28e7 F test/shell2.test 35c0c19d3198ee7669a748c1aedcce27a776ee575cc76128f8fcf665b79672f7 F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6195cfc86b15614b8db0e0dc5cc79b8d1acaf483f0131c8526992dc8ca075630 -R 0cbc439df8a4bc1455473d9bc06be876 -U dan -Z 82bb6cdaa19cf778ce3d38362f6d2ead +P 96d5116d17cbf3de6be38952881c6ddf6a226ddfd7649197bdb2df39c42b5d8e +R 3e992775c349664a49a877a29c086ae2 +U larrybr +Z 54fdda75138049cd04c8f980315f5c7b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 55453226f6..abe4a589a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96d5116d17cbf3de6be38952881c6ddf6a226ddfd7649197bdb2df39c42b5d8e \ No newline at end of file +0822788752621f6bf6af44b420b594ddd352634b3b0ed0eb835abea34b45817a \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 6b9e2b3545..0709f7d004 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11844,6 +11844,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int warnInmemoryDb = 0; int readStdin = 1; int nCmd = 0; + int nOptsEnd = argc; char **azCmd = 0; const char *zVfs = 0; /* Value of -vfs command-line option */ #if !SQLITE_SHELL_IS_UTF8 @@ -11953,7 +11954,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ for(i=1; inOptsEnd ){ if( data.aAuxDb->zDbFilename==0 ){ data.aAuxDb->zDbFilename = z; }else{ @@ -11965,9 +11966,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ shell_check_oom(azCmd); azCmd[nCmd-1] = z; } + continue; } if( z[1]=='-' ) z++; - if( cli_strcmp(z,"-separator")==0 + if( cli_strcmp(z, "-")==0 ){ + nOptsEnd = i; + continue; + }else if( cli_strcmp(z,"-separator")==0 || cli_strcmp(z,"-nullvalue")==0 || cli_strcmp(z,"-newline")==0 || cli_strcmp(z,"-cmd")==0 @@ -12152,7 +12157,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ */ for(i=1; i=nOptsEnd ) continue; if( z[1]=='-' ){ z++; } if( cli_strcmp(z,"-init")==0 ){ i++; diff --git a/test/shell1.test b/test/shell1.test index fbbda0348d..7a32c7b1bb 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -175,6 +175,16 @@ do_test shell1-1.16.1 { set x [catchcmd "-version test.db" ""] } {/3.[0-9.]+ 20\d\d-[01]\d-\d\d \d\d:\d\d:\d\d [0-9a-f]+/} +# Handle no-more-options option +forcedelete ./--db +do_test shell1-1.17.1 { + catchcmd {-- --db "CREATE TABLE T(c1);"} +} {0 {}} +do_test shell1-1.17.2 { + catchcmd {-- --db "SELECT name from sqlite_schema;"} +} {0 T} +forcedelete ./--db + #---------------------------------------------------------------------------- # Test cases shell1-2.*: Basic "dot" command token parsing. # From 30ebcf2ecb06884a2e1030c38889f4b016d83384 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 10 Mar 2023 21:27:59 +0000 Subject: [PATCH 081/341] Fix a typo in a comment. No code changes. FossilOrigin-Name: 76acc075402aac2d14d8279b8095008a842522a0985fdf89200da4339757a40b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ae1b495b4e..30cc60c9cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Give\sCLI\sa\sno-more-options\soption.\s(--) -D 2023-03-10T20:54:44.804 +C Fix\sa\stypo\sin\sa\scomment.\s\sNo\scode\schanges. +D 2023-03-10T21:27:59.458 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 399c10566b94ded9b565c01382026bfa0a7350782d4e9aa4194af1ac05f8af31 +F src/expr.c 0fc6ebb38658d1a08fdd53582f0d2716f72e63d6a49b3c0c669be659545f6142 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 96d5116d17cbf3de6be38952881c6ddf6a226ddfd7649197bdb2df39c42b5d8e -R 3e992775c349664a49a877a29c086ae2 -U larrybr -Z 54fdda75138049cd04c8f980315f5c7b +P 0822788752621f6bf6af44b420b594ddd352634b3b0ed0eb835abea34b45817a +R db9363d6a19c6dbc5cb6cb577fb0dab9 +U drh +Z fc73c7e44cdd2d405b203191c21c667e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index abe4a589a8..aaf566c2e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0822788752621f6bf6af44b420b594ddd352634b3b0ed0eb835abea34b45817a \ No newline at end of file +76acc075402aac2d14d8279b8095008a842522a0985fdf89200da4339757a40b \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 6135da786c..dc90956b3f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2650,7 +2650,7 @@ int sqlite3IsRowid(const char *z){ ** pX is the RHS of an IN operator. If pX is a SELECT statement ** that can be simplified to a direct table access, then return ** a pointer to the SELECT statement. If pX is not a SELECT statement, -** or if the SELECT statement needs to be manifested into a transient +** or if the SELECT statement needs to be materialized into a transient ** table, then return NULL. */ #ifndef SQLITE_OMIT_SUBQUERY From 20f2917881685e92ff53a66d1727d887b27f6f91 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 11 Mar 2023 00:15:41 +0000 Subject: [PATCH 082/341] CLI help to reflect no-more-options option FossilOrigin-Name: 30d95a12eb8b1cfc8ed11d3ed68cd713993ba34efa2fe623c18cfe41f14df1d9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 5 +++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 30cc60c9cf..7476f15705 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\scomment.\s\sNo\scode\schanges. -D 2023-03-10T21:27:59.458 +C CLI\shelp\sto\sreflect\sno-more-options\soption +D 2023-03-11T00:15:41.078 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c d4dd7843ce2fe0b886c3970b34a12330369fbfc7c86a5fece5a8ae02ae157038 -F src/shell.c.in c289de5d93ea2531bc8158e64ea8b6278ab87f953d185a811f4a7ae3ea310d84 +F src/shell.c.in 7ade4506b907025a0d19e31f85d1200bfbd1eed48e94cea013b669d3efe5073c F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0822788752621f6bf6af44b420b594ddd352634b3b0ed0eb835abea34b45817a -R db9363d6a19c6dbc5cb6cb577fb0dab9 -U drh -Z fc73c7e44cdd2d405b203191c21c667e +P 76acc075402aac2d14d8279b8095008a842522a0985fdf89200da4339757a40b +R ff346326dc1d1e08fe0a4af372c66c16 +U larrybr +Z 0fb7cbabf6bd8e54781f43c8092f8b04 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aaf566c2e4..89bdf2aef6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76acc075402aac2d14d8279b8095008a842522a0985fdf89200da4339757a40b \ No newline at end of file +30d95a12eb8b1cfc8ed11d3ed68cd713993ba34efa2fe623c18cfe41f14df1d9 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 0709f7d004..33bf0fad83 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11668,6 +11668,7 @@ static void process_sqliterc( */ static const char zOptions[] = #if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) + " -- treat none of arguments following as options\n" " -A ARGS... run \".archive ARGS\" and exit\n" #endif " -append append the database to the end of the file\n" @@ -11729,9 +11730,9 @@ static const char zOptions[] = ; static void usage(int showDetail){ utf8_printf(stderr, - "Usage: %s [OPTIONS] FILENAME [SQL]\n" + "Usage: %s [OPTIONS] [FILENAME] [SQL]\n" "FILENAME is the name of an SQLite database. A new database is created\n" - "if the file does not previously exist.\n", Argv0); + "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ utf8_printf(stderr, "OPTIONS include:\n%s", zOptions); }else{ From 7802389398c46b584713f9626ba5468e3f317f81 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 11 Mar 2023 12:27:12 +0000 Subject: [PATCH 083/341] Allow functions named using keywords "CROSS", "FULL", "INNER", "LEFT", "NATURAL", "OUTER", and "RIGHT". FossilOrigin-Name: eeac3d5ec93e9cf8613d764b700b6e39a3ffb39b3953ce1e6a2f84dd986f9e86 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/parse.y | 19 ++++++++++--------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 30cc60c9cf..637e14b97a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\scomment.\s\sNo\scode\schanges. -D 2023-03-10T21:27:59.458 +C Allow\sfunctions\snamed\susing\skeywords\s"CROSS",\s"FULL",\s"INNER",\s"LEFT",\s\n"NATURAL",\s"OUTER",\sand\s"RIGHT". +D 2023-03-11T12:27:12.895 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -614,7 +614,7 @@ F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 39af8ff7c73a991f61f4d1e3a6f8f98b1c8e29144723507822774cac5e6ee0b5 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 -F src/parse.y 960d2da92a23f8ba2ca22748a51bd75ee2c575564f2cbc59f119640e7f5b4c5d +F src/parse.y 424e49ed8fc6c907920db9be5a13a75ed43811e1ea8dd21b0fa9ef97f083dc6b F src/pcache.c f4268f7f73c6a3db12ce22fd25bc68dc42315d19599414ab1207d7cf32f79197 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc @@ -2050,8 +2050,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0822788752621f6bf6af44b420b594ddd352634b3b0ed0eb835abea34b45817a -R db9363d6a19c6dbc5cb6cb577fb0dab9 +P 76acc075402aac2d14d8279b8095008a842522a0985fdf89200da4339757a40b +R 0a1fb32029faefee149cc3206232c91a +T *branch * functions-named-left +T *sym-functions-named-left * +T -sym-trunk * U drh -Z fc73c7e44cdd2d405b203191c21c667e +Z 8d90c0869867550dc2ef5c4600461bb5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aaf566c2e4..18424331b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76acc075402aac2d14d8279b8095008a842522a0985fdf89200da4339757a40b \ No newline at end of file +eeac3d5ec93e9cf8613d764b700b6e39a3ffb39b3953ce1e6a2f84dd986f9e86 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 760cb114ad..73508b046b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -296,17 +296,19 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);} // %token_class id ID|INDEXED. - // And "ids" is an identifer-or-string. // %token_class ids ID|STRING. +// An identifier or a join-keyword +// +%token_class idj ID|INDEXED|JOIN_KW. + // The name of a column or table can be any of the following: // %type nm {Token} -nm(A) ::= id(A). +nm(A) ::= idj(A). nm(A) ::= STRING(A). -nm(A) ::= JOIN_KW(A). // A typetoken is really zero or more tokens that form a type name such // as can be found after the column name in a CREATE TABLE statement. @@ -1084,8 +1086,7 @@ idlist(A) ::= nm(Y). expr(A) ::= term(A). expr(A) ::= LP expr(X) RP. {A = X;} -expr(A) ::= id(X). {A=tokenExpr(pParse,TK_ID,X); /*A-overwrites-X*/} -expr(A) ::= JOIN_KW(X). {A=tokenExpr(pParse,TK_ID,X); /*A-overwrites-X*/} +expr(A) ::= idj(X). {A=tokenExpr(pParse,TK_ID,X); /*A-overwrites-X*/} expr(A) ::= nm(X) DOT nm(Y). { Expr *temp1 = tokenExpr(pParse,TK_ID,X); Expr *temp2 = tokenExpr(pParse,TK_ID,Y); @@ -1138,19 +1139,19 @@ expr(A) ::= CAST LP expr(E) AS typetoken(T) RP. { %endif SQLITE_OMIT_CAST -expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP. { +expr(A) ::= idj(X) LP distinct(D) exprlist(Y) RP. { A = sqlite3ExprFunction(pParse, Y, &X, D); } -expr(A) ::= id(X) LP STAR RP. { +expr(A) ::= idj(X) LP STAR RP. { A = sqlite3ExprFunction(pParse, 0, &X, 0); } %ifndef SQLITE_OMIT_WINDOWFUNC -expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP filter_over(Z). { +expr(A) ::= idj(X) LP distinct(D) exprlist(Y) RP filter_over(Z). { A = sqlite3ExprFunction(pParse, Y, &X, D); sqlite3WindowAttach(pParse, A, Z); } -expr(A) ::= id(X) LP STAR RP filter_over(Z). { +expr(A) ::= idj(X) LP STAR RP filter_over(Z). { A = sqlite3ExprFunction(pParse, 0, &X, 0); sqlite3WindowAttach(pParse, A, Z); } From 80511f32f7e71062026edd471913ef0455563964 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 11 Mar 2023 23:21:21 +0000 Subject: [PATCH 084/341] The check-in at [198b3e33dcfd74c7] caused a performance regression for some queries, which is here fixed. Problem reported by [forum:/forumpost/b405033490fa56d9|forum post b405033490fa56d9]. FossilOrigin-Name: dc9f025dc43cb8008e7d8d644175d8b2d084e602a1513803c40c513d1e99fea4 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/select.c | 4 +++- test/join2.test | 24 ++++++++++++++++++++++++ test/notnull2.test | 18 ------------------ test/tester.tcl | 23 +++++++++++++++++++++++ 6 files changed, 61 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 7476f15705..0409e43fbb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C CLI\shelp\sto\sreflect\sno-more-options\soption -D 2023-03-11T00:15:41.078 +C The\scheck-in\sat\s[198b3e33dcfd74c7]\scaused\sa\sperformance\sregression\sfor\ssome\nqueries,\swhich\sis\shere\sfixed.\s\sProblem\sreported\sby\n[forum:/forumpost/b405033490fa56d9|forum\spost\sb405033490fa56d9]. +D 2023-03-11T23:21:21.156 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -625,7 +625,7 @@ F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c d4dd7843ce2fe0b886c3970b34a12330369fbfc7c86a5fece5a8ae02ae157038 +F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa F src/shell.c.in 7ade4506b907025a0d19e31f85d1200bfbd1eed48e94cea013b669d3efe5073c F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1219,7 +1219,7 @@ F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 F test/join.test ed1daf99958fed1b9f017e56bae2bb6b49339a1ec0b70b9e8f7259960c6bf387 -F test/join2.test c378a2c59db8da13a265481c9aeab08d854c524f56a0eda12fa7bc535bfbebb1 +F test/join2.test 8561fe82ce434ac96de91544072e578dc2cadddf2d9bc9cd802f866a9b92502e F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 F test/join5.test 91f1f4c7d81fd87b58e9ba7cf4a2b5d39e3583b4f8e498a162722a60259c5208 @@ -1337,7 +1337,7 @@ F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 796c7b7157f55c93b4e672b724e9c923a6fc6aa72ac419379a623e2350472e22 F test/notnull.test a37b663d5bb728d66fc182016613fb8e4a0a4bbf3d75b8876a7527f7d4ed3f18 -F test/notnull2.test 8e1aa4f311df37f9d2cd4f5563eea955b8028c692151404e9cc896420a01a3dc +F test/notnull2.test 1ee4acbd614d3cf5f1c4a52f5af7fc771b82352f1a51a86afeaa02c9df1d82ef F test/notnullfault.test fc4bb7845582a2b3db376001ef49118393b1b11abe0d24adb03db057ee2b73d5 F test/null.test b7ff206a1c60fe01aa2abd33ef9ea83c93727d993ca8a613de86e925c9f2bc6f F test/nulls1.test 7a5e4346ee4285034100b4cd20e6784f16a9d6c927e44ecdf10034086bbee9c9 @@ -1567,7 +1567,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 8d54d40a55e12554b4509bc12078b201b233c8e842a7543629094a21b1ba956d +F test/tester.tcl 68454ef88508c196d19e8694daa27bff7107a91857799eaa12f417188ae53ede F test/testrunner.tcl 59490f189cac99b16b0376d0cc0a7ecfb753a84b89c9f4c361af337d88db53ac F test/testrunner_data.tcl 8169c68654ac8906833b8a6aadca973358a441ebf88270dd05c153e5f96f76b8 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 76acc075402aac2d14d8279b8095008a842522a0985fdf89200da4339757a40b -R ff346326dc1d1e08fe0a4af372c66c16 -U larrybr -Z 0fb7cbabf6bd8e54781f43c8092f8b04 +P 30d95a12eb8b1cfc8ed11d3ed68cd713993ba34efa2fe623c18cfe41f14df1d9 +R 79587856954d09f32d5a8e1654d18654 +U drh +Z 3f25d641e3ecb8ffdac28f15dbcf5caf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 89bdf2aef6..bcb371e0b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30d95a12eb8b1cfc8ed11d3ed68cd713993ba34efa2fe623c18cfe41f14df1d9 \ No newline at end of file +dc9f025dc43cb8008e7d8d644175d8b2d084e602a1513803c40c513d1e99fea4 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 7e85925246..ddaa8c06b5 100644 --- a/src/select.c +++ b/src/select.c @@ -3862,7 +3862,9 @@ static Expr *substExpr( sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ sqlite3 *db = pSubst->pParse->db; - if( pSubst->isOuterJoin ){ + if( pSubst->isOuterJoin + && (pCopy->op!=TK_COLUMN || pCopy->iTable!=pSubst->iNewTable) + ){ memset(&ifNullRow, 0, sizeof(ifNullRow)); ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; diff --git a/test/join2.test b/test/join2.test index 39cabf8809..15e76f965d 100644 --- a/test/join2.test +++ b/test/join2.test @@ -404,4 +404,28 @@ do_execsql_test 11.1 { SELECT * FROM t1 LEFT JOIN t3 ON y=z; } {NULL -2 -2 3 NULL 1 NULL NULL 0 2 2 3} +# 2023-03-11 https://sqlite.org/forum/forumpost/b405033490fa56d9 +# The fix that test 11.1 above checks also caused a performance regression. +# This test case verifies that the performance regression has been resolved. +# +do_execsql_test 12.1 { + DROP TABLE t1; + DROP TABLE t2; + DROP VIEW t3; + CREATE TABLE t1(a INTEGER PRIMARY KEY); + WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<100) + INSERT INTO t1(a) SELECT n FROM c; + CREATE VIEW t2(b) AS SELECT a FROM t1; +} +do_vmstep_test 12.2 { + SELECT * FROM t1 LEFT JOIN t2 ON a=b LIMIT 10 OFFSET 98; +} 2000 {99 99 100 100} +do_eqp_test 12.3 { + SELECT * FROM t1 LEFT JOIN t2 ON a=b LIMIT 10 OFFSET 98; +} { + QUERY PLAN + |--SCAN t1 + `--SEARCH t1 USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN +} + finish_test diff --git a/test/notnull2.test b/test/notnull2.test index 12fffe28ce..7f68086810 100644 --- a/test/notnull2.test +++ b/test/notnull2.test @@ -28,24 +28,6 @@ do_execsql_test 1.0 { INSERT INTO t2 SELECT * FROM t1; } -proc do_vmstep_test {tn sql nstep {res {}}} { - uplevel [list do_execsql_test $tn.0 $sql $res] - - set vmstep [db status vmstep] - if {[string range $nstep 0 0]=="+"} { - set body "if {$vmstep<$nstep} { - error \"got $vmstep, expected more than [string range $nstep 1 end]\" - }" - } else { - set body "if {$vmstep>$nstep} { - error \"got $vmstep, expected less than $nstep\" - }" - } - - # set name "$tn.vmstep=$vmstep,expect=$nstep" - set name "$tn.1" - uplevel [list do_test $name $body {}] -} do_vmstep_test 1.1.1 { SELECT * FROM t1 LEFT JOIN t2 WHERE a=c AND d IS NULL; diff --git a/test/tester.tcl b/test/tester.tcl index 4658590cf4..021830aa95 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -949,6 +949,29 @@ proc normalize_list {L} { set L2 } +# Run SQL and verify that the number of "vmsteps" required is greater +# than or less than some constant. +# +proc do_vmstep_test {tn sql nstep {res {}}} { + uplevel [list do_execsql_test $tn.0 $sql $res] + + set vmstep [db status vmstep] + if {[string range $nstep 0 0]=="+"} { + set body "if {$vmstep<$nstep} { + error \"got $vmstep, expected more than [string range $nstep 1 end]\" + }" + } else { + set body "if {$vmstep>$nstep} { + error \"got $vmstep, expected less than $nstep\" + }" + } + + # set name "$tn.vmstep=$vmstep,expect=$nstep" + set name "$tn.1" + uplevel [list do_test $name $body {}] +} + + # Either: # # do_execsql_test TESTNAME SQL ?RES? From bf7602a7d891ba4ceca74679403ebecbfe38c6a2 Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 13 Mar 2023 16:08:44 +0000 Subject: [PATCH 085/341] Include CLI's tip for -- in all builds. Better show optionality of its non-option arguments. FossilOrigin-Name: 9e2c771daad485d923751fdee0316b195bb49ec9f865f2e39565906edeaa5cf8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0409e43fbb..22d26facf4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\scheck-in\sat\s[198b3e33dcfd74c7]\scaused\sa\sperformance\sregression\sfor\ssome\nqueries,\swhich\sis\shere\sfixed.\s\sProblem\sreported\sby\n[forum:/forumpost/b405033490fa56d9|forum\spost\sb405033490fa56d9]. -D 2023-03-11T23:21:21.156 +C Include\sCLI's\stip\sfor\s--\sin\sall\sbuilds.\sBetter\sshow\soptionality\sof\sits\snon-option\sarguments. +D 2023-03-13T16:08:44.440 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa -F src/shell.c.in 7ade4506b907025a0d19e31f85d1200bfbd1eed48e94cea013b669d3efe5073c +F src/shell.c.in 3c62d4b5278a3d267a40aee82fefdd9324b59449bee983859040577838e6ae50 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 30d95a12eb8b1cfc8ed11d3ed68cd713993ba34efa2fe623c18cfe41f14df1d9 -R 79587856954d09f32d5a8e1654d18654 -U drh -Z 3f25d641e3ecb8ffdac28f15dbcf5caf +P dc9f025dc43cb8008e7d8d644175d8b2d084e602a1513803c40c513d1e99fea4 +R 8d1fcb1706ba67304c10514cf5da8048 +U larrybr +Z e91cad60d50b03b49ea4a130f780deed # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bcb371e0b4..cb22a1be62 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc9f025dc43cb8008e7d8d644175d8b2d084e602a1513803c40c513d1e99fea4 \ No newline at end of file +9e2c771daad485d923751fdee0316b195bb49ec9f865f2e39565906edeaa5cf8 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 33bf0fad83..892c3a5aa9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11667,8 +11667,8 @@ static void process_sqliterc( ** Show available command line options */ static const char zOptions[] = + " -- treat no subsequent arguments as options\n" #if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) - " -- treat none of arguments following as options\n" " -A ARGS... run \".archive ARGS\" and exit\n" #endif " -append append the database to the end of the file\n" @@ -11730,7 +11730,7 @@ static const char zOptions[] = ; static void usage(int showDetail){ utf8_printf(stderr, - "Usage: %s [OPTIONS] [FILENAME] [SQL]\n" + "Usage: %s [OPTIONS] [FILENAME [SQL]]\n" "FILENAME is the name of an SQLite database. A new database is created\n" "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ From cdfb518f6f4909e320d26285e4408f86b6e6a6f4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 14 Mar 2023 20:08:39 +0000 Subject: [PATCH 086/341] Fix Bloom filters on an expression index. [forum:/forumpost/2e427099d5|forum post 2e427099d5] and [forum:/forumpost/d47a0e8e3a|forum post d47a0e8e3a]. FossilOrigin-Name: c028fb669a5ae34dbaf50fffab1ae49bc568b994435cf02e145d24da3cfb48d7 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 1 + src/where.c | 8 ++++++-- test/bloom1.test | 40 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 56 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 22d26facf4..f8d977ee84 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Include\sCLI's\stip\sfor\s--\sin\sall\sbuilds.\sBetter\sshow\soptionality\sof\sits\snon-option\sarguments. -D 2023-03-13T16:08:44.440 +C Fix\sBloom\sfilters\son\san\sexpression\sindex.\s\n[forum:/forumpost/2e427099d5|forum\spost\s2e427099d5]\sand\n[forum:/forumpost/d47a0e8e3a|forum\spost\sd47a0e8e3a]. +D 2023-03-14T20:08:39.468 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 0fc6ebb38658d1a08fdd53582f0d2716f72e63d6a49b3c0c669be659545f6142 +F src/expr.c 55df0e3312a36efe597e48f299da390a83584d3371a311a35e58e537f462b4ff F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -710,7 +710,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 201f03ca60f318b6bd1cb1977af93b67ff3679acea72eeec69e3f27e678b7e59 +F src/where.c 9f0ffd5f4cb2f6ff09da2e805d95e859babf601383a5d7f527cbe036765f72f5 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 9919e5a22f4b24dd96c49b8981484cbe6bbfcf466ff73ac40a06e1356aa8bf87 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -818,7 +818,7 @@ F test/bind2.test 918bc35135f4141809ead7585909cde57d44db90a7a62aef540127148f91aa F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 F test/blob.test e7ac6c7d3a985cc4678c64f325292529a69ae252 -F test/bloom1.test ab125229849c085b8d4a6768cc321d330d1ed47b21902f1a2854db832c625768 +F test/bloom1.test cf613a27054bbaf61c5bfc440a5cfd3ff76798d0695f3fc5e5d1bbc819b8dab1 F test/boundary1.tcl 6421b2d920d8b09539503a8673339d32f7609eb1 F test/boundary1.test 66d7f4706ccdb42d58eafdb081de07b0eb42d77b F test/boundary2.tcl e34ef4e930cf1083150d4d2c603e146bd3b76bcb @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dc9f025dc43cb8008e7d8d644175d8b2d084e602a1513803c40c513d1e99fea4 -R 8d1fcb1706ba67304c10514cf5da8048 -U larrybr -Z e91cad60d50b03b49ea4a130f780deed +P 9e2c771daad485d923751fdee0316b195bb49ec9f865f2e39565906edeaa5cf8 +R c3393138ebe1c3b0466080fa5ffa80b2 +U drh +Z f74a80a854a5c1a7493bae1be34b47cd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cb22a1be62..fad5367d7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9e2c771daad485d923751fdee0316b195bb49ec9f865f2e39565906edeaa5cf8 \ No newline at end of file +c028fb669a5ae34dbaf50fffab1ae49bc568b994435cf02e145d24da3cfb48d7 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index dc90956b3f..223ebf1cbd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3867,6 +3867,7 @@ void sqlite3ExprCodeGetColumnOfTable( Column *pCol; assert( v!=0 ); assert( pTab!=0 ); + assert( iCol!=XN_EXPR ); if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); VdbeComment((v, "%s.rowid", pTab->zName)); diff --git a/src/where.c b/src/where.c index d7dbd57909..727c77948e 100644 --- a/src/where.c +++ b/src/where.c @@ -1135,6 +1135,10 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( Vdbe *v = pParse->pVdbe; /* VDBE under construction */ WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */ int iCur; /* Cursor for table getting the filter */ + IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */ + + saved_pIdxEpr = pParse->pIdxEpr; + pParse->pIdxEpr = 0; assert( pLoop!=0 ); assert( v!=0 ); @@ -1191,9 +1195,8 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( int r1 = sqlite3GetTempRange(pParse, n); int jj; for(jj=0; jjaiColumn[jj]; assert( pIdx->pTable==pItem->pTab ); - sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); } sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); sqlite3ReleaseTempRange(pParse, r1, n); @@ -1224,6 +1227,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( } }while( iLevel < pWInfo->nLevel ); sqlite3VdbeJumpHere(v, addrOnce); + pParse->pIdxEpr = saved_pIdxEpr; } diff --git a/test/bloom1.test b/test/bloom1.test index 11d4cd3cb2..151f364ae0 100644 --- a/test/bloom1.test +++ b/test/bloom1.test @@ -145,6 +145,44 @@ do_eqp_test 3.4 { `--SEARCH t1 USING AUTOMATIC PARTIAL COVERING INDEX (x=?) } - +# 2023-03-14 +# https://sqlite.org/forum/forumpost/d47a0e8e3a +# https://sqlite.org/forum/forumpost/2e427099d5 +# +# Both reports are for the same problem - using a Bloom filter on an +# expression index can cause issues. +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(x TEXT, y INT, z TEXT); + INSERT INTO t1(rowid,x,y,z) VALUES(12,'aa','bb','aa'); + CREATE INDEX i1x ON t1(1 IS true,z); + CREATE TABLE t0(x TEXT); + INSERT INTO t0(rowid,x) VALUES(4,'aa'); + ANALYZE sqlite_schema; + INSERT INTO sqlite_stat1 VALUES('t0',NULL,'20'); + INSERT INTO sqlite_stat1 VALUES('t1','i1x','18 18 2'); + ANALYZE sqlite_schema; +} +do_execsql_test 4.2 { + SELECT * FROM t0 NATURAL JOIN t1 WHERE z=t1.x; +} {aa bb aa} +do_execsql_test 4.3 { + DROP TABLE t0; + CREATE TABLE t0(a TEXT); + INSERT INTO t0 VALUES ('xyz'); + CREATE INDEX t0x ON t0(a IS FALSE) WHERE false; + DROP TABLE t1; + CREATE TABLE t1(b INT); + INSERT INTO t1 VALUES('aaa'),('bbb'),('ccc'),('ddd'),(NULL); + CREATE TABLE t2(c REAL); + INSERT INTO t2 VALUES(7); + ANALYZE; + CREATE INDEX t2x ON t2(true IN ()); +} +do_execsql_test 4.4 { + SELECT * FROM t0 LEFT JOIN t1 LEFT JOIN t2 ON (b NOTNULL)==(c IN ()) WHERE c; +} {xyz {} 7.0} + finish_test From eb5d71ed58ce1b5b9a68c928f48063e4e3969467 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Mar 2023 13:53:47 +0000 Subject: [PATCH 087/341] Fix a broken assert() in the recovery extension. FossilOrigin-Name: 4c4e66f293d7768cceb875a936ca0f4cd910473e20b9910698cc1e1ce221a7d4 --- ext/recover/dbdata.c | 3 +-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c index 3da9f92321..15b73cad95 100644 --- a/ext/recover/dbdata.c +++ b/ext/recover/dbdata.c @@ -750,8 +750,7 @@ static int dbdataGetEncoding(DbdataCursor *pCsr){ int nPg1 = 0; u8 *aPg1 = 0; rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); - assert( rc!=SQLITE_OK || nPg1==0 || nPg1>=512 ); - if( rc==SQLITE_OK && nPg1>0 ){ + if( rc==SQLITE_OK && nPg1>=(56+4) ){ pCsr->enc = get_uint32(&aPg1[56]); } sqlite3_free(aPg1); diff --git a/manifest b/manifest index f8d977ee84..bce8e76843 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sBloom\sfilters\son\san\sexpression\sindex.\s\n[forum:/forumpost/2e427099d5|forum\spost\s2e427099d5]\sand\n[forum:/forumpost/d47a0e8e3a|forum\spost\sd47a0e8e3a]. -D 2023-03-14T20:08:39.468 +C Fix\sa\sbroken\sassert()\sin\sthe\srecovery\sextension. +D 2023-03-15T13:53:47.255 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -365,7 +365,7 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69 F ext/rbu/sqlite3rbu.c 348bb6251e6ec459de102f8b2dd50789a98643ef7a28e56e4c787ac9659c15ea F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 -F ext/recover/dbdata.c ac504aa06fab51ade9c0521b96cb6ec21d40619aa5e9a3e14ef9a60978699fd0 +F ext/recover/dbdata.c ab5e6ee837717c5cec505c34bba9fb188cc405f95157d681f8df8fe927c82324 F ext/recover/recover1.test 2072993624d5e32fef20ae03b17fc06c02bcb344421fe17bb329b24d2a51e647 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverbuild.test a6f05273ff5fe517afd166444597c70cb97033e7f58496433a4428a1ecb5d79f @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9e2c771daad485d923751fdee0316b195bb49ec9f865f2e39565906edeaa5cf8 -R c3393138ebe1c3b0466080fa5ffa80b2 -U drh -Z f74a80a854a5c1a7493bae1be34b47cd +P c028fb669a5ae34dbaf50fffab1ae49bc568b994435cf02e145d24da3cfb48d7 +R 08e4ffb83c4b2d94021f24c51d8dc054 +U dan +Z 71596409b5637be1db41be73621dc119 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fad5367d7f..48c0e6fd55 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c028fb669a5ae34dbaf50fffab1ae49bc568b994435cf02e145d24da3cfb48d7 \ No newline at end of file +4c4e66f293d7768cceb875a936ca0f4cd910473e20b9910698cc1e1ce221a7d4 \ No newline at end of file From ffcad5893aa40c52629d02851b05de20cee60f0d Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 15 Mar 2023 17:58:51 +0000 Subject: [PATCH 088/341] Disallow the one-pass optimization for DELETE if the WHERE clause contains a subquery. Fix for the problem reported by [forum:/forumpost/e61252062c9d286d|forum post e61252062c9d286d]. This fix is more restrictive than necessary. It could be relaxed if the subquery does not involve the table that is the subject of the DELETE. FossilOrigin-Name: 73f0036f045bf37193b6e87ae45b578c5831614c530488257c69666178da3aa5 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/delete.c | 2 +- src/resolve.c | 2 +- src/sqliteInt.h | 2 +- test/delete.test | 21 +++++++++++++++++++++ 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index bce8e76843..ea75ce9552 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbroken\sassert()\sin\sthe\srecovery\sextension. -D 2023-03-15T13:53:47.255 +C Disallow\sthe\sone-pass\soptimization\sfor\sDELETE\sif\sthe\sWHERE\sclause\scontains\na\ssubquery.\s\sFix\sfor\sthe\sproblem\sreported\sby\n[forum:/forumpost/e61252062c9d286d|forum\spost\se61252062c9d286d].\s\sThis\sfix\nis\smore\srestrictive\sthan\snecessary.\s\sIt\scould\sbe\srelaxed\sif\sthe\ssubquery\sdoes\nnot\sinvolve\sthe\stable\sthat\sis\sthe\ssubject\sof\sthe\sDELETE. +D 2023-03-15T17:58:51.689 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -574,7 +574,7 @@ F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef -F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e +F src/delete.c 201fe0763c52783d205c8c13cdd9d55c1bd5cb21c1f036753f99103b43284b90 F src/expr.c 55df0e3312a36efe597e48f299da390a83584d3371a311a35e58e537f462b4ff F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 @@ -623,14 +623,14 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c 4233c3030341bf1a21cea90890e6b3d3531721acc62ede147e899d36ffad8238 +F src/resolve.c 6a0253379cc15b3f80321362a61f487a8ef7cd2487fe62e1eb2317b3f871c61f F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa F src/shell.c.in 3c62d4b5278a3d267a40aee82fefdd9324b59449bee983859040577838e6ae50 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 34034353de20c29e04295c3486bbf22b2a47842c887f65f7176530b4c35fdd17 +F src/sqliteInt.h 3c96ed2651cdc5ebedf1cd5e1d75ceb2dac8416b48e03fa1fb053d46f5a39c27 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -928,7 +928,7 @@ F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef F test/decimal.test fcf403fd5585f47342234e153c4a4338cd737b8e0884ac66fc484df47dbcf1a7 F test/default.test 9687cfb16717e4b8238c191697c98be88c0b16e568dd5368cd9284154097ef50 -F test/delete.test 31832b0c45ecb51a54348c68db173be462985901e6ed7f403d6d7a8f70ab4ef0 +F test/delete.test f66581a22960dae89b05f14755a576d935dcb94aa3aeacc0005b29bdbaa9949c F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab F test/delete4.test 51fafebe9503a40796d1aae1565c60524cada720e50eecac01b7fd0419d9ea0b @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c028fb669a5ae34dbaf50fffab1ae49bc568b994435cf02e145d24da3cfb48d7 -R 08e4ffb83c4b2d94021f24c51d8dc054 -U dan -Z 71596409b5637be1db41be73621dc119 +P 4c4e66f293d7768cceb875a936ca0f4cd910473e20b9910698cc1e1ce221a7d4 +R 934d92c186df99d22e77975c034c41b7 +U drh +Z 54012e32e8462771957ea3ef7763a915 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 48c0e6fd55..cea5ff8945 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c4e66f293d7768cceb875a936ca0f4cd910473e20b9910698cc1e1ce221a7d4 \ No newline at end of file +73f0036f045bf37193b6e87ae45b578c5831614c530488257c69666178da3aa5 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 704a3c7110..22ef7ab658 100644 --- a/src/delete.c +++ b/src/delete.c @@ -483,7 +483,7 @@ void sqlite3DeleteFrom( #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; - if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; + if( sNC.ncFlags & NC_Subquery ) bComplex = 1; wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ diff --git a/src/resolve.c b/src/resolve.c index dfb3434396..4b36ecca34 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1252,8 +1252,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); - pNC->ncFlags |= NC_VarSelect; } + pNC->ncFlags |= NC_Subquery; } break; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3a7be541ae..95fda3bbb3 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3343,7 +3343,7 @@ struct NameContext { #define NC_HasAgg 0x000010 /* One or more aggregate functions seen */ #define NC_IdxExpr 0x000020 /* True if resolving columns of CREATE INDEX */ #define NC_SelfRef 0x00002e /* Combo: PartIdx, isCheck, GenCol, and IdxExpr */ -#define NC_VarSelect 0x000040 /* A correlated subquery has been seen */ +#define NC_Subquery 0x000040 /* A subquery has been seen */ #define NC_UEList 0x000080 /* True if uNC.pEList is used */ #define NC_UAggInfo 0x000100 /* True if uNC.pAggInfo is used */ #define NC_UUpsert 0x000200 /* True if uNC.pUpsert is used */ diff --git a/test/delete.test b/test/delete.test index a448e52dd2..cdf6798ea6 100644 --- a/test/delete.test +++ b/test/delete.test @@ -416,4 +416,25 @@ do_execsql_test delete-11.1 { } {6 2 12 4 18 6 19 23 20 40} +# 2023-03-15 +# https://sqlite.org/forum/forumpost/e61252062c9d286d +# +# When the WHERE clause of a DELETE statement contains a subquery +# which uses the table that is being deleted from and there is a +# short-circuit operator of some kind in the WHERE clause such that +# the subquery might not run right away, then the subquery might +# run after one or more rows have been deleted, which can change +# the result of the subquery, and result in the wrong answer. +# +reset_db +do_execsql_test delete-12.0 { + CREATE TABLE t0(vkey INTEGER, pkey INTEGER,c1 INTEGER); + INSERT INTO t0 VALUES(2,1,-20),(2,2,NULL),(2,3,0),(8,4,95); + DELETE FROM t0 WHERE NOT ( + (t0.vkey <= t0.c1) AND + (t0.vkey <> (SELECT vkey FROM t0 ORDER BY vkey LIMIT 1 OFFSET 2)) + ); + SELECT * FROM t0; +} {8 4 95} + finish_test From 5531316dd030b320c5653454bc2dcec23bf261b2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Mar 2023 09:07:46 +0000 Subject: [PATCH 089/341] Remove a NEVER() from btreeNext() that dbsqlfuzz 460aa158f9a2c41145831cc924296cde1f312b3f found could sometimes be reached. I will find a way to test that branch later. FossilOrigin-Name: 1dffeffe150fff6777cf26bc960326ca4e4190d25d05bb066b082d1e1c8eb374 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ea75ce9552..100d6872ee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disallow\sthe\sone-pass\soptimization\sfor\sDELETE\sif\sthe\sWHERE\sclause\scontains\na\ssubquery.\s\sFix\sfor\sthe\sproblem\sreported\sby\n[forum:/forumpost/e61252062c9d286d|forum\spost\se61252062c9d286d].\s\sThis\sfix\nis\smore\srestrictive\sthan\snecessary.\s\sIt\scould\sbe\srelaxed\sif\sthe\ssubquery\sdoes\nnot\sinvolve\sthe\stable\sthat\sis\sthe\ssubject\sof\sthe\sDELETE. -D 2023-03-15T17:58:51.689 +C Remove\sa\sNEVER()\sfrom\sbtreeNext()\sthat\sdbsqlfuzz\s460aa158f9a2c41145831cc924296cde1f312b3f\nfound\scould\ssometimes\sbe\sreached.\s\sI\swill\sfind\sa\sway\sto\stest\sthat\sbranch\nlater. +D 2023-03-16T09:07:46.644 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c c547e099f853de61835ff45da8d956a932d02ecf1ffd472a1f2a103b83e6dd40 +F src/btree.c 9192990bb78806df33ac680df47cb4100b5dbe5d298c3d0abffc82cd31ca6634 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e F src/build.c 4fed662d383527c808d85f53b9c544ead425ac8b9c7cb38501a64e3797921d41 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4c4e66f293d7768cceb875a936ca0f4cd910473e20b9910698cc1e1ce221a7d4 -R 934d92c186df99d22e77975c034c41b7 +P 73f0036f045bf37193b6e87ae45b578c5831614c530488257c69666178da3aa5 +R 416ca2e38e88837ddad74173b91b3e6d U drh -Z 54012e32e8462771957ea3ef7763a915 +Z 476a952efa83c3a1b612460afea7cd08 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cea5ff8945..7a6660030a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -73f0036f045bf37193b6e87ae45b578c5831614c530488257c69666178da3aa5 \ No newline at end of file +1dffeffe150fff6777cf26bc960326ca4e4190d25d05bb066b082d1e1c8eb374 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 210845eeb6..918388a64a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6132,7 +6132,7 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ pPage = pCur->pPage; idx = ++pCur->ix; - if( NEVER(!pPage->isInit) || sqlite3FaultSim(412) ){ + if( !pPage->isInit || sqlite3FaultSim(412) ){ return SQLITE_CORRUPT_BKPT; } From 7d5a5491215eb48acba22a81067aae2661bb2fd6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Mar 2023 10:17:30 +0000 Subject: [PATCH 090/341] Do not use the one-pass optimization on an UPDATE if there is a subquery in the WHERE clause, since if the subquery is hidden behind a short-circuit operator, the subquery might not be evaluated until after one or more rows have been updated. Fix for the problem reported by [forum:/forumpost/0007d1fdb1|forum post 0007d1fdb1]. This is the same problem that was fixed by [73f0036f045bf371] only for UPDATE instead of DELETE. FossilOrigin-Name: 2c56b984a0bd3be5ec326a2109ea7b8f1d4ef63c8fc325caac9663cf2479eaff --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/update.c | 20 +++++++++++++++----- test/corruptL.test | 7 +++++-- test/delete.test | 3 +++ test/update.test | 33 +++++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 100d6872ee..e55697af41 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sNEVER()\sfrom\sbtreeNext()\sthat\sdbsqlfuzz\s460aa158f9a2c41145831cc924296cde1f312b3f\nfound\scould\ssometimes\sbe\sreached.\s\sI\swill\sfind\sa\sway\sto\stest\sthat\sbranch\nlater. -D 2023-03-16T09:07:46.644 +C Do\snot\suse\sthe\sone-pass\soptimization\son\san\sUPDATE\sif\sthere\sis\sa\ssubquery\nin\sthe\sWHERE\sclause,\ssince\sif\sthe\ssubquery\sis\shidden\sbehind\sa\sshort-circuit\noperator,\sthe\ssubquery\smight\snot\sbe\sevaluated\suntil\safter\sone\sor\smore\srows\nhave\sbeen\supdated.\s\sFix\sfor\sthe\sproblem\sreported\sby\n[forum:/forumpost/0007d1fdb1|forum\spost\s0007d1fdb1].\s\sThis\sis\sthe\ssame\nproblem\sthat\swas\sfixed\sby\s[73f0036f045bf371]\sonly\sfor\sUPDATE\sinstead\sof\nDELETE. +D 2023-03-16T10:17:30.566 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -690,7 +690,7 @@ F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c fccf3b8c517c1f55cb380c1522febe6921fcb2bd800c16c78cab571d0eb0ccbd F src/trigger.c 6072c531d9bcc3980528150a1b03fda2e85a08c10023fafb42f93ffd68607ffe -F src/update.c f118e51768d2c1309e3c81e9f91141b22b8a1339cbc5969b1b2d810feaa25b22 +F src/update.c 76664e1beae86e8e961983ebe19a4ee9ebd7e26683ead2b288ba08f81fc7ba4e F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f @@ -890,7 +890,7 @@ F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 F test/corruptI.test a17bbf54fdde78d43cf3cc34b0057719fd4a173a3d824285b67dc5257c064c7b F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 F test/corruptK.test 5b4212fe346699831c5ad559a62c54e11c0611bdde1ea8423a091f9c01aa32af -F test/corruptL.test 9d1a0055c8db19baccd12f22ac36a33ec7d63afb59e82eb30835aea8f89b94df +F test/corruptL.test b42978028afc5eefc8b51d8d7cd6a9344ba7362d7ed4511ee2070f56e06d5a1c F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c3e067 F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 @@ -928,7 +928,7 @@ F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef F test/decimal.test fcf403fd5585f47342234e153c4a4338cd737b8e0884ac66fc484df47dbcf1a7 F test/default.test 9687cfb16717e4b8238c191697c98be88c0b16e568dd5368cd9284154097ef50 -F test/delete.test f66581a22960dae89b05f14755a576d935dcb94aa3aeacc0005b29bdbaa9949c +F test/delete.test 2686e1c98d552ef37d79ad55b17b93fe96fad9737786917ce3839767f734c48f F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab F test/delete4.test 51fafebe9503a40796d1aae1565c60524cada720e50eecac01b7fd0419d9ea0b @@ -1774,7 +1774,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 3674e9f2a3f1fbbfd4772ac74b7a97090d0f77d2 F test/unixexcl.test d936ba2b06794018e136418addd59a2354eeae97 F test/unordered.test 0edaf3411d300693bca595897c5201421c6c5ec787990a1dfe2f7f60ae93f1e2 -F test/update.test eb7f4eb172ce270e51bb67d7867521f33a63635bb671e261bbafccaef3bd6db2 +F test/update.test 90772ede84cfc779fc3d31884a84ec4c74deb501eeb09a1c6c91c03d8e94c0d8 F test/update2.test 67455bc61fcbcf96923c45b3bc4f87bc72be7d67575ad35f134906148c7b06d3 F test/upfrom1.tcl 8859d9d437f03b44174c4524a7a734a391fd4526fcff65be08285dafc9dc9041 F test/upfrom1.test 8cb06689e99cd707d884faa16da0e8eb26ff658bb01c47ddf72fadade666e6e1 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 73f0036f045bf37193b6e87ae45b578c5831614c530488257c69666178da3aa5 -R 416ca2e38e88837ddad74173b91b3e6d +P 1dffeffe150fff6777cf26bc960326ca4e4190d25d05bb066b082d1e1c8eb374 +R 5410b321a0481083a3d313d81510e3fc U drh -Z 476a952efa83c3a1b612460afea7cd08 +Z eea9af84fd6b98bd366cd12ed13a97d2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7a6660030a..7742930f8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1dffeffe150fff6777cf26bc960326ca4e4190d25d05bb066b082d1e1c8eb374 \ No newline at end of file +2c56b984a0bd3be5ec326a2109ea7b8f1d4ef63c8fc325caac9663cf2479eaff \ No newline at end of file diff --git a/src/update.c b/src/update.c index a4e57d1d42..bd1ddb1a30 100644 --- a/src/update.c +++ b/src/update.c @@ -727,12 +727,22 @@ void sqlite3Update( /* Begin the database scan. ** ** Do not consider a single-pass strategy for a multi-row update if - ** there are any triggers or foreign keys to process, or rows may - ** be deleted as a result of REPLACE conflict handling. Any of these - ** things might disturb a cursor being used to scan through the table - ** or index, causing a single-pass approach to malfunction. */ + ** there is anything that might disrupt the cursor being used to do + ** the UPDATE: + ** (1) This is a nested UPDATE + ** (2) There are triggers + ** (3) There are FOREIGN KEY constraints + ** (4) There are REPLACE conflict handlers + ** (5) There are subqueries in the WHERE clause + */ flags = WHERE_ONEPASS_DESIRED; - if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){ + if( !pParse->nested + && !pTrigger + && !hasFK + && !chngKey + && !bReplace + && (sNC.ncFlags & NC_Subquery)==0 + ){ flags |= WHERE_ONEPASS_MULTIROW; } pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,0,0,flags,iIdxCur); diff --git a/test/corruptL.test b/test/corruptL.test index 3a841a8199..65d9821b35 100644 --- a/test/corruptL.test +++ b/test/corruptL.test @@ -1269,11 +1269,14 @@ do_test 15.0 { }]} {} extra_schema_checks 0 -do_execsql_test 15.1 { +do_catchsql_test 15.1 { PRAGMA cell_size_check = 0; UPDATE c1 SET c= NOT EXISTS(SELECT 1 FROM c1 ORDER BY (SELECT 1 FROM c1 ORDER BY a)) +10 WHERE d BETWEEN 4 AND 7; -} {} +} {1 {database disk image is malformed}} extra_schema_checks 1 +do_execsql_test 15.2 { + PRAGMA integrity_check; +} {/in database main/} #------------------------------------------------------------------------- reset_db diff --git a/test/delete.test b/test/delete.test index cdf6798ea6..214bae6f70 100644 --- a/test/delete.test +++ b/test/delete.test @@ -426,6 +426,9 @@ do_execsql_test delete-11.1 { # run after one or more rows have been deleted, which can change # the result of the subquery, and result in the wrong answer. # +# Similar problem for UPDATE tested by update-21.4 +# https://sqlite.org/forum/forumpost/0007d1fdb1 +# reset_db do_execsql_test delete-12.0 { CREATE TABLE t0(vkey INTEGER, pkey INTEGER,c1 INTEGER); diff --git a/test/update.test b/test/update.test index 66efd10ec8..47b6029bee 100644 --- a/test/update.test +++ b/test/update.test @@ -732,4 +732,37 @@ do_execsql_test update-20.30 { PRAGMA integrity_check; } {ok} +# 2023-03-16 https://sqlite.org/forum/forumpost/0007d1fdb1 +# A subquery in the WHERE clause of an UPDATE and behind a +# short-circuit evaluation caused problems because multi-row +# single-pass was selected. +# +# Similar problem for DELETE tested by delete-12.0. +# https://sqlite.org/src/info/73f0036f045bf371 +# +reset_db +do_execsql_test update-21.1 { + CREATE TABLE t1 (vkey INTEGER, c5 INTEGER); + INSERT INTO t1 VALUES(3,NULL),(6,-54); +} +db null NULL +do_execsql_test update-21.2 { + BEGIN; + UPDATE t1 SET vkey = 100 WHERE c5 is null; + SELECT * FROM t1 ORDER BY vkey, c5; + ROLLBACK; +} {6 -54 100 NULL} +do_execsql_test update-21.3 { + BEGIN; + UPDATE t1 SET vkey = 100 WHERE NOT (-10*(select min(vkey) from t1) >= c5); + SELECT * FROM t1 ORDER BY vkey, c5; + ROLLBACK; +} {3 NULL 6 -54} +do_execsql_test update-21.4 { + BEGIN; + UPDATE t1 SET vkey = 100 WHERE c5 is null OR NOT (-10*(select min(vkey) from t1) >= c5); + SELECT * FROM t1 ORDER BY vkey, c5; + ROLLBACK; +} {6 -54 100 NULL} + finish_test From db8c6aa03ab5a6dd7cd0614a3bc6a0371df1f558 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Mar 2023 11:50:44 +0000 Subject: [PATCH 091/341] Update the tracing output for the query-invariant checker such that it shows the SQL that is run to verify that a found query-invariant discrepency is valid. Changes to testing logic only. FossilOrigin-Name: 8f45ad27403e971d88ec62e674c03f82eb19df0b43aa58d20b3400cdb6611d90 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fuzzinvariants.c | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e55697af41..a2da194766 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sthe\sone-pass\soptimization\son\san\sUPDATE\sif\sthere\sis\sa\ssubquery\nin\sthe\sWHERE\sclause,\ssince\sif\sthe\ssubquery\sis\shidden\sbehind\sa\sshort-circuit\noperator,\sthe\ssubquery\smight\snot\sbe\sevaluated\suntil\safter\sone\sor\smore\srows\nhave\sbeen\supdated.\s\sFix\sfor\sthe\sproblem\sreported\sby\n[forum:/forumpost/0007d1fdb1|forum\spost\s0007d1fdb1].\s\sThis\sis\sthe\ssame\nproblem\sthat\swas\sfixed\sby\s[73f0036f045bf371]\sonly\sfor\sUPDATE\sinstead\sof\nDELETE. -D 2023-03-16T10:17:30.566 +C Update\sthe\stracing\soutput\sfor\sthe\squery-invariant\schecker\ssuch\sthat\sit\sshows\nthe\sSQL\sthat\sis\srun\sto\sverify\sthat\sa\sfound\squery-invariant\sdiscrepency\sis\nvalid.\s\sChanges\sto\stesting\slogic\sonly. +D 2023-03-16T11:50:44.645 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1154,7 +1154,7 @@ F test/fuzzdata8.db f6c2f2af4deaaae0ddb3310d509c2659990794aa653dc501b80a0534c349 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c 3ec82ac4277a0c62918f39e7a35893d6fb5ed57c465a938ad18e8e7a69527615 +F test/fuzzinvariants.c cc9291de9be93207442561ec9a0c07d2c236389c47c4965a6f90bc0fbf4ab6bd F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/gencol1.test aef8b0670abd4b1ae4cae786b15a43758d86f6cd9f12b381d45d96bb51e597c9 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1dffeffe150fff6777cf26bc960326ca4e4190d25d05bb066b082d1e1c8eb374 -R 5410b321a0481083a3d313d81510e3fc +P 2c56b984a0bd3be5ec326a2109ea7b8f1d4ef63c8fc325caac9663cf2479eaff +R b0e423029bdbdf0c07b87c0b38c5e2f1 U drh -Z eea9af84fd6b98bd366cd12ed13a97d2 +Z cc7b3cfd55b0f4c7215af58b26a1c7fb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7742930f8e..8e76e35b28 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c56b984a0bd3be5ec326a2109ea7b8f1d4ef63c8fc325caac9663cf2479eaff \ No newline at end of file +8f45ad27403e971d88ec62e674c03f82eb19df0b43aa58d20b3400cdb6611d90 \ No newline at end of file diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 0a5f607148..73472e0895 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -126,6 +126,12 @@ int fuzz_invariant( sqlite3_finalize(pTestStmt); return rc; } + if( eVerbosity>=2 ){ + char *zSql = sqlite3_expanded_sql(pCk); + printf("invariant-validity-check #1:\n%s\n", zSql); + sqlite3_free(zSql); + } + rc = sqlite3_step(pCk); if( rc!=SQLITE_ROW || sqlite3_column_text(pCk, 0)==0 @@ -146,6 +152,11 @@ int fuzz_invariant( sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, !iOrigRSO, 0); sqlite3_prepare_v2(db, sqlite3_sql(pStmt), -1, &pCk, 0); sqlite3_db_config(db, SQLITE_DBCONFIG_REVERSE_SCANORDER, iOrigRSO, 0); + if( eVerbosity>=2 ){ + char *zSql = sqlite3_expanded_sql(pCk); + printf("invariant-validity-check #2:\n%s\n", zSql); + sqlite3_free(zSql); + } while( (rc = sqlite3_step(pCk))==SQLITE_ROW ){ for(i=0; i=2 ){ + char *zSql = sqlite3_expanded_sql(pCk); + printf("invariant-validity-check #3:\n%s\n", zSql); + sqlite3_free(zSql); + } + sqlite3_reset(pTestStmt); while( (rc = sqlite3_step(pTestStmt))==SQLITE_ROW ){ for(i=0; i=2 ){ + char *zSql = sqlite3_expanded_sql(pCk); + printf("invariant-validity-check #4:\n%s\n", zSql); + sqlite3_free(zSql); + } sqlite3_bind_pointer(pCk, 1, pStmt, "stmt-pointer", 0); rc = sqlite3_step(pCk); } From 7a9c235aaf3b8ecd63a77d59744ccb5c39d84158 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Mar 2023 12:28:48 +0000 Subject: [PATCH 092/341] Additional debug/test output from the query invariant checker showing the row-number that is being checked. FossilOrigin-Name: e4b6eb58e65f7fa0c92768f42bc820614169b7c2f37ded391b866ddb0d894de0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fuzzinvariants.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a2da194766..42eebc38f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\stracing\soutput\sfor\sthe\squery-invariant\schecker\ssuch\sthat\sit\sshows\nthe\sSQL\sthat\sis\srun\sto\sverify\sthat\sa\sfound\squery-invariant\sdiscrepency\sis\nvalid.\s\sChanges\sto\stesting\slogic\sonly. -D 2023-03-16T11:50:44.645 +C Additional\sdebug/test\soutput\sfrom\sthe\squery\sinvariant\schecker\sshowing\sthe\nrow-number\sthat\sis\sbeing\schecked. +D 2023-03-16T12:28:48.259 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1154,7 +1154,7 @@ F test/fuzzdata8.db f6c2f2af4deaaae0ddb3310d509c2659990794aa653dc501b80a0534c349 F test/fuzzer1.test 3d4c4b7e547aba5e5511a2991e3e3d07166cfbb8 F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test f64c4aef4c9e9edf1d6dc0d3f1e65dcc81e67c996403c88d14f09b74807a42bc -F test/fuzzinvariants.c cc9291de9be93207442561ec9a0c07d2c236389c47c4965a6f90bc0fbf4ab6bd +F test/fuzzinvariants.c b34530e8431f2cf3591eff588fc7684d6fdef466916fb46141c8c5374a3d8099 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/gencol1.test aef8b0670abd4b1ae4cae786b15a43758d86f6cd9f12b381d45d96bb51e597c9 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2c56b984a0bd3be5ec326a2109ea7b8f1d4ef63c8fc325caac9663cf2479eaff -R b0e423029bdbdf0c07b87c0b38c5e2f1 +P 8f45ad27403e971d88ec62e674c03f82eb19df0b43aa58d20b3400cdb6611d90 +R 1be156af5539063c8f0cdb78fcf0c4ec U drh -Z cc7b3cfd55b0f4c7215af58b26a1c7fb +Z cd1b805566b97c55f9fb9cc640849ea7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8e76e35b28..d27699e924 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f45ad27403e971d88ec62e674c03f82eb19df0b43aa58d20b3400cdb6611d90 \ No newline at end of file +e4b6eb58e65f7fa0c92768f42bc820614169b7c2f37ded391b866ddb0d894de0 \ No newline at end of file diff --git a/test/fuzzinvariants.c b/test/fuzzinvariants.c index 73472e0895..5d473f19f3 100644 --- a/test/fuzzinvariants.c +++ b/test/fuzzinvariants.c @@ -103,7 +103,7 @@ int fuzz_invariant( } if( eVerbosity>=2 ){ char *zSql = sqlite3_expanded_sql(pTestStmt); - printf("invariant-sql #%d:\n%s\n", iCnt, zSql); + printf("invariant-sql row=%d #%d:\n%s\n", iRow, iCnt, zSql); sqlite3_free(zSql); } while( (rc = sqlite3_step(pTestStmt))==SQLITE_ROW ){ From b816ca9994e03a8bc829b49452b8158a731e81a9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 16 Mar 2023 20:54:29 +0000 Subject: [PATCH 093/341] Correctly handle SELECT DISTINCT ... ORDER BY when all of the result set terms are constant and there are more result set terms than ORDER BY terms. Fix for these tickets: [c36cdb4afd504dc1], [4051a7f931d9ba24], [d6fd512f50513ab7]. FossilOrigin-Name: 12ad822d9b827777526ca5ed5bf3e678d600294fc9b5c25482dfff2a021328a4 --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/where.c | 4 ++++ test/distinct.test | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 42eebc38f5..7fa302f49d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Additional\sdebug/test\soutput\sfrom\sthe\squery\sinvariant\schecker\sshowing\sthe\nrow-number\sthat\sis\sbeing\schecked. -D 2023-03-16T12:28:48.259 +C Correctly\shandle\sSELECT\sDISTINCT\s...\sORDER\sBY\swhen\sall\sof\sthe\sresult\sset\sterms\nare\sconstant\sand\sthere\sare\smore\sresult\sset\sterms\sthan\sORDER\sBY\sterms.\nFix\sfor\sthese\stickets:\s[c36cdb4afd504dc1],\s[4051a7f931d9ba24],\n[d6fd512f50513ab7]. +D 2023-03-16T20:54:29.173 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -710,7 +710,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 9f0ffd5f4cb2f6ff09da2e805d95e859babf601383a5d7f527cbe036765f72f5 +F src/where.c f369f3423da9c0d71f7206345e6e775aa06997b74bdb70934ed4677187a72eb8 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 9919e5a22f4b24dd96c49b8981484cbe6bbfcf466ff73ac40a06e1356aa8bf87 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -937,7 +937,7 @@ F test/descidx1.test edc8adee58d491b06c7157c50364eaf1c3605c9c19f8093cb1ea2b6184f F test/descidx2.test a0ba347037ff3b811f4c6ceca5fd0f9d5d72e74e59f2d9de346a9d2f6ad78298 F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a1999b926 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test a7687c2fb50c93f6a486936c51439a93221c6e1188f9bc7b27b3ec26f9c58b1e +F test/distinct.test 691c9e850b0d0b56b66e7e235453198cb4cf0760e324b7403d3c5abbeab0a014 F test/distinct2.test cd1d15a4a2abf579298f7161e821ed50c0119136fe0424db85c52cf0adc230d1 F test/distinctagg.test 14ec5026e684eddd414c61c08692b43773e224ac92efbed6ec08c6994bc39723 F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8f45ad27403e971d88ec62e674c03f82eb19df0b43aa58d20b3400cdb6611d90 -R 1be156af5539063c8f0cdb78fcf0c4ec +P e4b6eb58e65f7fa0c92768f42bc820614169b7c2f37ded391b866ddb0d894de0 +R 4d12677b2569d68bf087d08fc5281334 U drh -Z cd1b805566b97c55f9fb9cc640849ea7 +Z f57ce3e1c7e1fa81dd8ab238bc06fc2a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d27699e924..9371192f9e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4b6eb58e65f7fa0c92768f42bc820614169b7c2f37ded391b866ddb0d894de0 \ No newline at end of file +12ad822d9b827777526ca5ed5bf3e678d600294fc9b5c25482dfff2a021328a4 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 727c77948e..f707fab12b 100644 --- a/src/where.c +++ b/src/where.c @@ -5299,6 +5299,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } + if( pWInfo->pSelect->pOrderBy + && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){ + pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr; + } }else{ pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ diff --git a/test/distinct.test b/test/distinct.test index 446f85bb8e..760b2341f5 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -348,5 +348,61 @@ foreach {tn idx} { } } +# 2023-03-16 +# https://sqlite.org/forum/forumpost/16ce2bb7a639e29b +# ticket c36cdb4afd504dc1 +# ticket 4051a7f931d9ba24 +# ticket d6fd512f50513ab7 +# +do_execsql_test 10.1 { + SELECT DISTINCT + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 + ORDER BY + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x'; +} {1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1} +do_execsql_test 10.2 { + EXPLAIN + SELECT DISTINCT + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1 + ORDER BY + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x','x','x','x','x','x','x', + 'x','x','x','x'; +} {/0 Init 0 /} +do_execsql_test 10.3 { + EXPLAIN CREATE TABLE t2 AS SELECT DISTINCT ':memory:', 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 ORDER BY '%J%j%w%s', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', '%J%j%w%s', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 42e-300, 'unixepoch', 'unixepoch', 'unixepoch' LIMIT 0xda; +} {/0 Init 0/} +do_execsql_test 10.4 { + DROP TABLE IF EXISTS t0; + CREATE TABLE t0 AS SELECT DISTINCT 0xda, 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 0xda-0xda-42e-300, 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0', 'lit0' ORDER BY '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%Y-%m-%d', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', 'lit0', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', 'auto', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', ':memory:', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', '%%', ''; + SELECT count(*) FROM t0; +} {1} +do_execsql_test 10.5 { + DROP TABLE IF EXISTS t2; + CREATE TABLE t2 AS SELECT DISTINCT ':memory:', 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 0.0*7/0, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 ORDER BY '%J%j%w%s', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', '%J%j%w%s', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 'unixepoch', 42e-300, 'unixepoch', 'unixepoch', 'unixepoch' LIMIT 0xda; + SELECT count(*) FROM t2; +} {1} finish_test From a84ead119e291cb980e3d18135a0c28d82cf2317 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 17 Mar 2023 00:01:32 +0000 Subject: [PATCH 094/341] Fix assert() statements that would (incorrectly) fire if an IF NOT EXISTS trigger that already exists contained two or more RETURNING clauses. Tickets [89d259d45b855a0d] and [d15b3a4ea901ef0d]. FossilOrigin-Name: 648899e4ded72cac6cc24bccf7ebfc709ee7309a003452b21fd6ab0ba20c34b8 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/build.c | 5 +++-- src/sqliteInt.h | 3 +++ src/trigger.c | 1 + test/returning1.test | 19 +++++++++++++++++++ 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 7fa302f49d..177214cf96 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correctly\shandle\sSELECT\sDISTINCT\s...\sORDER\sBY\swhen\sall\sof\sthe\sresult\sset\sterms\nare\sconstant\sand\sthere\sare\smore\sresult\sset\sterms\sthan\sORDER\sBY\sterms.\nFix\sfor\sthese\stickets:\s[c36cdb4afd504dc1],\s[4051a7f931d9ba24],\n[d6fd512f50513ab7]. -D 2023-03-16T20:54:29.173 +C Fix\sassert()\sstatements\sthat\swould\s(incorrectly)\sfire\sif\san\sIF\sNOT\sEXISTS\ntrigger\sthat\salready\sexists\scontained\stwo\sor\smore\sRETURNING\sclauses.\nTickets\s[89d259d45b855a0d]\sand\s[d15b3a4ea901ef0d]. +D 2023-03-17T00:01:32.145 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -567,7 +567,7 @@ F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca F src/btree.c 9192990bb78806df33ac680df47cb4100b5dbe5d298c3d0abffc82cd31ca6634 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e -F src/build.c 4fed662d383527c808d85f53b9c544ead425ac8b9c7cb38501a64e3797921d41 +F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d @@ -630,7 +630,7 @@ F src/shell.c.in 3c62d4b5278a3d267a40aee82fefdd9324b59449bee983859040577838e6ae5 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 3c96ed2651cdc5ebedf1cd5e1d75ceb2dac8416b48e03fa1fb053d46f5a39c27 +F src/sqliteInt.h ee87a8982f59897ea0594df986bb3ebbe971b16fa0248c9c4940ae18beda7deb F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -689,7 +689,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c fccf3b8c517c1f55cb380c1522febe6921fcb2bd800c16c78cab571d0eb0ccbd -F src/trigger.c 6072c531d9bcc3980528150a1b03fda2e85a08c10023fafb42f93ffd68607ffe +F src/trigger.c f34367fad4df451b5dfe63fcc1d384fd16e40077e42092b1c3682dedeef5a7e3 F src/update.c 76664e1beae86e8e961983ebe19a4ee9ebd7e26683ead2b288ba08f81fc7ba4e F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 @@ -1408,7 +1408,7 @@ F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2 F test/releasetest_data.tcl b550dd1b122a9c969df794d05ea272df535f10ff1a245062e7ba080822378016 F test/resetdb.test 54c06f18bc832ac6d6319e5ab23d5c8dd49fdbeec7c696d791682a8006bd5fc3 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test ee72bece38ea39ef82a219d8f6619e5c1c15835ea127cd7e6e30012cde462dfe +F test/returning1.test c78efa2829d5f44165fcc7959961b298ae005d7775f09e5542bb2815bbaee692 F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e4b6eb58e65f7fa0c92768f42bc820614169b7c2f37ded391b866ddb0d894de0 -R 4d12677b2569d68bf087d08fc5281334 +P 12ad822d9b827777526ca5ed5bf3e678d600294fc9b5c25482dfff2a021328a4 +R 8a6f90bdd13ff84285687c30c2f9dc79 U drh -Z f57ce3e1c7e1fa81dd8ab238bc06fc2a +Z e93c4e476e40be14bed110c238138789 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9371192f9e..9040daca11 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -12ad822d9b827777526ca5ed5bf3e678d600294fc9b5c25482dfff2a021328a4 \ No newline at end of file +648899e4ded72cac6cc24bccf7ebfc709ee7309a003452b21fd6ab0ba20c34b8 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 6e3b7b81f8..eecfa4705b 100644 --- a/src/build.c +++ b/src/build.c @@ -1451,7 +1451,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){ if( pParse->pNewTrigger ){ sqlite3ErrorMsg(pParse, "cannot use RETURNING in a trigger"); }else{ - assert( pParse->bReturning==0 ); + assert( pParse->bReturning==0 || pParse->ifNotExists ); } pParse->bReturning = 1; pRet = sqlite3DbMallocZero(db, sizeof(*pRet)); @@ -1477,7 +1477,8 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){ pRet->retTStep.pTrig = &pRet->retTrig; pRet->retTStep.pExprList = pList; pHash = &(db->aDb[1].pSchema->trigHash); - assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 || pParse->nErr ); + assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 + || pParse->nErr || pParse->ifNotExists ); if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig) ==&pRet->retTrig ){ sqlite3OomFault(db); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 95fda3bbb3..8cb0198a29 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3714,6 +3714,9 @@ struct Parse { u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ +#endif +#ifdef SQLITE_DEBUG + u8 ifNotExists; /* Might be true if IF NOT EXISTS. Assert()s only */ #endif int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ diff --git a/src/trigger.c b/src/trigger.c index f83c719a33..d179d747af 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -202,6 +202,7 @@ void sqlite3BeginTrigger( }else{ assert( !db->init.busy ); sqlite3CodeVerifySchema(pParse, iDb); + VVA_ONLY( pParse->ifNotExists = 1; ) } goto trigger_cleanup; } diff --git a/test/returning1.test b/test/returning1.test index 0d97b70201..5ab44f2cb7 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -419,4 +419,23 @@ do_catchsql_test 18.1 { INSERT INTO view_2 DEFAULT VALUES RETURNING *; } {1 {no such collation sequence: TRUE}} +# 2023-03-16 +# https://sqlite.org/forum/forumpost/c99d6e0329 +# ticket d15b3a4ea901ef0d +# ticket 89d259d45b855a0d +# +# A RETURNING clause on an IF NOT EXISTS trigger does not generate +# an error if the trigger already exists. +# +do_execsql_test 19.0 { + DROP TABLE IF EXISTS t1;CREATE TABLE t1(a); + CREATE TRIGGER r1 AFTER UPDATE ON t1 BEGIN VALUES(0); END; +} {} +do_catchsql_test 19.1 { + CREATE TRIGGER IF NOT EXISTS r1 AFTER DELETE ON t1 BEGIN + INSERT INTO t1(a) VALUES (1) RETURNING FALSE; + INSERT INTO t1(a) VALUES (2) RETURNING TRUE; + END; +} {0 {}} + finish_test From 14e8299aa0b46e3853097a32638aa21b6bec9df6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 17 Mar 2023 00:42:27 +0000 Subject: [PATCH 095/341] Add safety margin on the CLI input buffer for tickets [33aa4c0de8a62e33], [b97e6c5e6a91d97f], [2971fbe3f993e95a], and [2971fbe3f993e95a]. FossilOrigin-Name: 741af08af1b93406a120580379d13e514524af627da5387ecfa6e442d004bfdd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 177214cf96..f9a814aac4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sassert()\sstatements\sthat\swould\s(incorrectly)\sfire\sif\san\sIF\sNOT\sEXISTS\ntrigger\sthat\salready\sexists\scontained\stwo\sor\smore\sRETURNING\sclauses.\nTickets\s[89d259d45b855a0d]\sand\s[d15b3a4ea901ef0d]. -D 2023-03-17T00:01:32.145 +C Add\ssafety\smargin\son\sthe\sCLI\sinput\sbuffer\sfor\stickets\n[33aa4c0de8a62e33],\s[b97e6c5e6a91d97f],\s[2971fbe3f993e95a],\sand\n[2971fbe3f993e95a]. +D 2023-03-17T00:42:27.613 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 6a0253379cc15b3f80321362a61f487a8ef7cd2487fe62e1eb2317b3f871c61f F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa -F src/shell.c.in 3c62d4b5278a3d267a40aee82fefdd9324b59449bee983859040577838e6ae50 +F src/shell.c.in 55c54f1a48d7c78a0fca98a685ec6809b0ce8782169d441d7dd2c891dc07cfc3 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 12ad822d9b827777526ca5ed5bf3e678d600294fc9b5c25482dfff2a021328a4 -R 8a6f90bdd13ff84285687c30c2f9dc79 +P 648899e4ded72cac6cc24bccf7ebfc709ee7309a003452b21fd6ab0ba20c34b8 +R 669516456aeda41d2a3ee4cbf4f0ac18 U drh -Z e93c4e476e40be14bed110c238138789 +Z 314e43e240f0cb240b68efb4946e48cc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9040daca11..7805e4d19b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -648899e4ded72cac6cc24bccf7ebfc709ee7309a003452b21fd6ab0ba20c34b8 \ No newline at end of file +741af08af1b93406a120580379d13e514524af627da5387ecfa6e442d004bfdd \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 892c3a5aa9..1f49a50565 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11465,9 +11465,9 @@ static int process_input(ShellState *p){ } /* No single-line dispositions remain; accumulate line(s). */ nLine = strlen(zLine); - if( nSql+nLine+2>=nAlloc ){ + if( nSql+nLine+8>=nAlloc ){ /* Grow buffer by half-again increments when big. */ - nAlloc = nSql+(nSql>>1)+nLine+100; + nAlloc = nSql+(nSql>>1)+nLine+104; zSql = realloc(zSql, nAlloc); shell_check_oom(zSql); } From 3f22b9ee2c0aff784f24a11e3b35159519f61c30 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 17 Mar 2023 10:43:05 +0000 Subject: [PATCH 096/341] Ensure that an error does not delete the Table object out from under the xConstruct method of a virtual table. dbsqlfuzz 7cc8804a1c6d4e3d554d79096e6ea75a7c1c7d2d FossilOrigin-Name: df4928c92b0db77d0a40d7b492b609db191252e2f87bca63d000e4fe2e206293 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vtab.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f9a814aac4..0c12280f4a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssafety\smargin\son\sthe\sCLI\sinput\sbuffer\sfor\stickets\n[33aa4c0de8a62e33],\s[b97e6c5e6a91d97f],\s[2971fbe3f993e95a],\sand\n[2971fbe3f993e95a]. -D 2023-03-17T00:42:27.613 +C Ensure\sthat\san\serror\sdoes\snot\sdelete\sthe\sTable\sobject\sout\sfrom\sunder\nthe\sxConstruct\smethod\sof\sa\svirtual\stable.\ndbsqlfuzz\s7cc8804a1c6d4e3d554d79096e6ea75a7c1c7d2d +D 2023-03-17T10:43:05.828 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -705,7 +705,7 @@ F src/vdbemem.c 0388576b7cf0be13ce14b9e3b8aa90b8a1b923b60321d0242131ae0b5732b43b F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac -F src/vtab.c a39f6ed161f16a84ff445af9dba4776b2cf0898be08a33bee8e1128a66c0074b +F src/vtab.c 4516e9fecff8d4f353132be0468bc6424b102937c50c52e01fd9a0a9e9e34ca5 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 648899e4ded72cac6cc24bccf7ebfc709ee7309a003452b21fd6ab0ba20c34b8 -R 669516456aeda41d2a3ee4cbf4f0ac18 +P 741af08af1b93406a120580379d13e514524af627da5387ecfa6e442d004bfdd +R c2d5a5d6b87ec41a79e446c5372dacc4 U drh -Z 314e43e240f0cb240b68efb4946e48cc +Z 5cbbee7a44b99c1cb81f73e037d66431 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7805e4d19b..c828c35aae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -741af08af1b93406a120580379d13e514524af627da5387ecfa6e442d004bfdd \ No newline at end of file +df4928c92b0db77d0a40d7b492b609db191252e2f87bca63d000e4fe2e206293 \ No newline at end of file diff --git a/src/vtab.c b/src/vtab.c index 2dd92762b7..58452dfc5d 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -610,7 +610,9 @@ static int vtabCallConstructor( sCtx.pPrior = db->pVtabCtx; sCtx.bDeclared = 0; db->pVtabCtx = &sCtx; + pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); assert( sCtx.pTab==pTab ); From 66086eb6a6c55a59d0529074bbf9564b64014370 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Mar 2023 14:18:39 +0000 Subject: [PATCH 097/341] Fix a potential buffer overread in the recovery extension. FossilOrigin-Name: 0b3b5bf9597615589a1d045aaa697c13550553ee4fe4b9008a8e51415b6fe96a --- ext/recover/dbdata.c | 6 +++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c index 15b73cad95..4100202354 100644 --- a/ext/recover/dbdata.c +++ b/ext/recover/dbdata.c @@ -512,10 +512,14 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); if( rc!=SQLITE_OK ) return rc; - if( pCsr->aPage ) break; + if( pCsr->aPage && pCsr->nPage>=256 ) break; + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; } + + assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); } diff --git a/manifest b/manifest index 0c12280f4a..21149d9fac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\san\serror\sdoes\snot\sdelete\sthe\sTable\sobject\sout\sfrom\sunder\nthe\sxConstruct\smethod\sof\sa\svirtual\stable.\ndbsqlfuzz\s7cc8804a1c6d4e3d554d79096e6ea75a7c1c7d2d -D 2023-03-17T10:43:05.828 +C Fix\sa\spotential\sbuffer\soverread\sin\sthe\srecovery\sextension. +D 2023-03-17T14:18:39.265 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -365,7 +365,7 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69 F ext/rbu/sqlite3rbu.c 348bb6251e6ec459de102f8b2dd50789a98643ef7a28e56e4c787ac9659c15ea F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 -F ext/recover/dbdata.c ab5e6ee837717c5cec505c34bba9fb188cc405f95157d681f8df8fe927c82324 +F ext/recover/dbdata.c 501227169803bfafb34cd8ec7fffda6e64c1a282ac33315d3a096076c5928feb F ext/recover/recover1.test 2072993624d5e32fef20ae03b17fc06c02bcb344421fe17bb329b24d2a51e647 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverbuild.test a6f05273ff5fe517afd166444597c70cb97033e7f58496433a4428a1ecb5d79f @@ -2050,8 +2050,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 741af08af1b93406a120580379d13e514524af627da5387ecfa6e442d004bfdd -R c2d5a5d6b87ec41a79e446c5372dacc4 -U drh -Z 5cbbee7a44b99c1cb81f73e037d66431 +P df4928c92b0db77d0a40d7b492b609db191252e2f87bca63d000e4fe2e206293 +R 70e8a82bdecc07447025e06299f230f8 +U dan +Z e6d83c6fc661a51988a2c81a05d958c9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c828c35aae..db9230734c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df4928c92b0db77d0a40d7b492b609db191252e2f87bca63d000e4fe2e206293 \ No newline at end of file +0b3b5bf9597615589a1d045aaa697c13550553ee4fe4b9008a8e51415b6fe96a \ No newline at end of file From 2eacecfe51c3d8be32eaf2061e306ee5ccfb3110 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 17 Mar 2023 19:07:48 +0000 Subject: [PATCH 098/341] Add test cases for functions named the same as join keywords. FossilOrigin-Name: 94944b239ce674d984c88ef6029b0260a972f1b25f01614b559ca07c3ebaf8f5 --- manifest | 14 +++++------ manifest.uuid | 2 +- test/func8.test | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 test/func8.test diff --git a/manifest b/manifest index 637e14b97a..bf15510d06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sfunctions\snamed\susing\skeywords\s"CROSS",\s"FULL",\s"INNER",\s"LEFT",\s\n"NATURAL",\s"OUTER",\sand\s"RIGHT". -D 2023-03-11T12:27:12.895 +C Add\stest\scases\sfor\sfunctions\snamed\sthe\ssame\sas\sjoin\skeywords. +D 2023-03-17T19:07:48.724 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1135,6 +1135,7 @@ F test/func4.test 2285fb5792d593fef442358763f0fd9de806eda47dbc7a5934df57ffdc484c F test/func5.test 863e6d1bd0013d09c17236f8a13ea34008dd857d87d85a13a673960e4c25d82a F test/func6.test 9cc9b1f43b435af34fe1416eb1e318c8920448ea7a6962f2121972f5215cb9b0 F test/func7.test adbfc910385a6ffd15dc47be3c619ef070c542fcb7488964badb17b2d9a4d080 +F test/func8.test c4e2ecacf9f16e47a245e7a25fbabcc7e78f9c7c41a80f158527cdfdc6dd299d F test/fuzz-oss1.test 514dcabb24687818ea949fa6760229eaacad74ca70157743ef36d35bbe01ffb0 F test/fuzz.test 4608c1310cff4c3014a84bcced6278139743e080046e5f6784b0de7b069371d8 F test/fuzz2.test 76dc35b32b6d6f965259508508abce75a6c4d7e1 @@ -2050,11 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 76acc075402aac2d14d8279b8095008a842522a0985fdf89200da4339757a40b -R 0a1fb32029faefee149cc3206232c91a -T *branch * functions-named-left -T *sym-functions-named-left * -T -sym-trunk * +P eeac3d5ec93e9cf8613d764b700b6e39a3ffb39b3953ce1e6a2f84dd986f9e86 +R f66f9e3cb0d1817a5dd25fbf43695e68 U drh -Z 8d90c0869867550dc2ef5c4600461bb5 +Z 2a027c2b3d756e8d338ef759071dc4ed # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 18424331b8..7354bd054c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eeac3d5ec93e9cf8613d764b700b6e39a3ffb39b3953ce1e6a2f84dd986f9e86 \ No newline at end of file +94944b239ce674d984c88ef6029b0260a972f1b25f01614b559ca07c3ebaf8f5 \ No newline at end of file diff --git a/test/func8.test b/test/func8.test new file mode 100644 index 0000000000..348dfb7f66 --- /dev/null +++ b/test/func8.test @@ -0,0 +1,64 @@ +# 2023-03-17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Test cases for SQL functions with names that are the same as join +# keywords: CROSS FULL INNER LEFT NATURAL OUTER RIGHT +# +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +proc joinx {args} {return [join $args -]} +db func cross {joinx cross} +db func full {joinx full} +db func inner {joinx inner} +db func left {joinx left} +db func natural {joinx natural} +db func outer {joinx outer} +db func right {joinx right} +do_execsql_test func8-100 { + CREATE TABLE cross(cross,full,inner,left,natural,outer,right); + CREATE TABLE full(cross,full,inner,left,natural,outer,right); + CREATE TABLE inner(cross,full,inner,left,natural,outer,right); + CREATE TABLE left(cross,full,inner,left,natural,outer,right); + CREATE TABLE natural(cross,full,inner,left,natural,outer,right); + CREATE TABLE outer(cross,full,inner,left,natural,outer,right); + CREATE TABLE right(cross,full,inner,left,natural,outer,right); + INSERT INTO cross VALUES(1,2,3,4,5,6,7); + INSERT INTO full VALUES(1,2,3,4,5,6,7); + INSERT INTO inner VALUES(1,2,3,4,5,6,7); + INSERT INTO left VALUES(1,2,3,4,5,6,7); + INSERT INTO natural VALUES(1,2,3,4,5,6,7); + INSERT INTO outer VALUES(1,2,3,4,5,6,7); + INSERT INTO right VALUES(1,2,3,4,5,6,7); +} +do_execsql_test func8-110 { + SELECT cross(cross,full,inner,left,natural,outer,right) FROM cross; +} cross-1-2-3-4-5-6-7 +do_execsql_test func8-120 { + SELECT full(cross,full,inner,left,natural,outer,right) FROM full; +} full-1-2-3-4-5-6-7 +do_execsql_test func8-130 { + SELECT inner(cross,full,inner,left,natural,outer,right) FROM inner; +} inner-1-2-3-4-5-6-7 +do_execsql_test func8-140 { + SELECT left(cross,full,inner,left,natural,outer,right) FROM left; +} left-1-2-3-4-5-6-7 +do_execsql_test func8-150 { + SELECT natural(cross,full,inner,left,natural,outer,right) FROM natural; +} natural-1-2-3-4-5-6-7 +do_execsql_test func8-160 { + SELECT outer(cross,full,inner,left,natural,outer,right) FROM outer; +} outer-1-2-3-4-5-6-7 +do_execsql_test func8-170 { + SELECT right(cross,full,inner,left,natural,outer,right) FROM right; +} right-1-2-3-4-5-6-7 + +finish_test From 667b5cc985d68f0f63e67a2888d0ec36c83a1688 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 17 Mar 2023 20:31:24 +0000 Subject: [PATCH 099/341] Fix json rendering so that it shows positive and negative infinity as 9.0e+999 and -9.0e+999 respectively. FossilOrigin-Name: efce4690a52592c4f5fc7d023eebe771b0e615bb03c0fe203493f853b28e8f85 --- manifest | 20 +++++++++++--------- manifest.uuid | 2 +- src/json.c | 5 ++++- src/printf.c | 15 ++++++++++----- test/json101.test | 12 ++++++++++++ 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 33375936fa..f7577f6b35 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\sname\sfunctions\susing\sone\sof\sthe\sjoin\skeywords\slike\nCROSS\sFULL\sINNER\sLEFT\sNATURAL\sOUTER\sRIGHT. -D 2023-03-17T19:18:17.276 +C Fix\sjson\srendering\sso\sthat\sit\sshows\spositive\sand\snegative\sinfinity\sas\n9.0e+999\sand\s-9.0e+999\srespectively. +D 2023-03-17T20:31:24.056 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 7940fce7d4aa855606432d82d20694e17c18a03956f5f5776f2404e2df7c18a8 -F src/json.c c85ed6fce06f43d414b0d7fff64749d43a0dbd1067123ee407bd3a0752454161 +F src/json.c 9e3a6acf9e4fa8bb6c84a38538ed9c2f9ad09f106910ef0e9cc68e19c8dc4fc8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c b04eb648cedc45efe4298e1ef439ac4f0096ae27b5f01accb0a1f49d57789128 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -621,7 +621,7 @@ F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c c0c6e0735436e0e9e8d490f82959ddc9475d438e9f808a51953a4d43e30cea51 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 -F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 +F src/printf.c 8c16d3783e87d5899c23a2736106dca11645b50e2c8606766d2d1446de278a0c F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 6a0253379cc15b3f80321362a61f487a8ef7cd2487fe62e1eb2317b3f871c61f F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 @@ -1241,7 +1241,7 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test 9f6337e7423d2c1774d1d485c919fc4e11c8f7e423a2f8831f73471983432108 +F test/json101.test c7707ee623c57a24845ef260d4388a6fade1eb294e450caadd7f1d9ced19dea7 F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6a0ac F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 @@ -2051,9 +2051,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0b3b5bf9597615589a1d045aaa697c13550553ee4fe4b9008a8e51415b6fe96a 94944b239ce674d984c88ef6029b0260a972f1b25f01614b559ca07c3ebaf8f5 -R 07b8484e41d6b78cbc774ca07208b7eb -T +closed 94944b239ce674d984c88ef6029b0260a972f1b25f01614b559ca07c3ebaf8f5 +P 0910b1925e97f7ae4dae86894c9e2f54273c85115e19d0d9bff1280ffee35eed +R d887fd54d882d8b9d5d596a32a4dfb61 +T *branch * numeric-only-json +T *sym-numeric-only-json * +T -sym-trunk * U drh -Z 4e9f7dbd3bbd0c5da8cb618454aab138 +Z b44e31c25af126076d4adb070d9c2558 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 91c00e53a6..def72dff24 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0910b1925e97f7ae4dae86894c9e2f54273c85115e19d0d9bff1280ffee35eed \ No newline at end of file +efce4690a52592c4f5fc7d023eebe771b0e615bb03c0fe203493f853b28e8f85 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 67448421b5..9783a3898a 100644 --- a/src/json.c +++ b/src/json.c @@ -307,8 +307,11 @@ static void jsonAppendValue( jsonAppendRaw(p, "null", 4); break; } - case SQLITE_INTEGER: case SQLITE_FLOAT: { + jsonPrintf(100, p, "%!0.15g", sqlite3_value_double(pValue)); + break; + } + case SQLITE_INTEGER: { const char *z = (const char*)sqlite3_value_text(pValue); u32 n = (u32)sqlite3_value_bytes(pValue); jsonAppendRaw(p, z, n); diff --git a/src/printf.c b/src/printf.c index b222cb2146..d59855917e 100644 --- a/src/printf.c +++ b/src/printf.c @@ -601,11 +601,16 @@ void sqlite3_str_vappendf( while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } if( exp>350 ){ - bufpt = buf; - buf[0] = prefix; - memcpy(buf+(prefix!=0),"Inf",4); - length = 3+(prefix!=0); - break; + if( flag_zeropad ){ + realvalue = 9.0; + exp = 999; + }else{ + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); + break; + } } if( xtype!=etFLOAT ){ realvalue += rounder; diff --git a/test/json101.test b/test/json101.test index 135c988e97..596a084968 100644 --- a/test/json101.test +++ b/test/json101.test @@ -885,4 +885,16 @@ do_execsql_test json-19.3 { SELECT * FROM t1; } {} +# 2023-03-17 positive and negative infinities +# +do_execsql_test json-20.1 { + SELECT json_object('a',2e370,'b',-3e380); +} {{{"a":9.0e+999,"b":-9.0e+999}}} +do_execsql_test json-20.2 { + SELECT json_object('a',2e370,'b',-3e380)->>'a'; +} Inf +do_execsql_test json-20.3 { + SELECT json_object('a',2e370,'b',-3e380)->>'b'; +} {-Inf} + finish_test From 9b532d1e1b1eb25e5d3a4179f8d4aaf75b54956f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 18 Mar 2023 16:12:27 +0000 Subject: [PATCH 100/341] Avoid a buffer overread in fts3 that could occur when processing a corrupt record. FossilOrigin-Name: 02ac2297abee6af64c8df230b42b07f21cff4565d7e315860b2396a7c0c556ca --- ext/fts3/fts3_write.c | 8 +++++--- manifest | 15 +++++++-------- manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 6a727eaf5f..393f8a8717 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2667,16 +2667,18 @@ static int fts3MsrBufferData( char *pList, i64 nList ){ - if( nList>pMsr->nBuffer ){ + if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){ char *pNew; - pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, pMsr->nBuffer); + int nNew = nList*2 + FTS3_NODE_PADDING; + pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew); if( !pNew ) return SQLITE_NOMEM; pMsr->aBuffer = pNew; + pMsr->nBuffer = nNew; } assert( nList>0 ); memcpy(pMsr->aBuffer, pList, nList); + memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING); return SQLITE_OK; } diff --git a/manifest b/manifest index 33375936fa..a56838f0a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\sname\sfunctions\susing\sone\sof\sthe\sjoin\skeywords\slike\nCROSS\sFULL\sINNER\sLEFT\sNATURAL\sOUTER\sRIGHT. -D 2023-03-17T19:18:17.276 +C Avoid\sa\sbuffer\soverread\sin\sfts3\sthat\scould\soccur\swhen\sprocessing\sa\scorrupt\srecord. +D 2023-03-18T16:12:27.555 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -76,7 +76,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c c1de4ae28356ad98ccb8b2e3388a7fdcce7607b5523738c9afb6275dab765154 F ext/fts3/fts3_unicode.c de426ff05c1c2e7bce161cf6b706638419c3a1d9c2667de9cb9dc0458c18e226 F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c 4fb644df0ff840267e47a724286c7a1fa5540273a7ce15756dd5913a101ec302 +F ext/fts3/fts3_write.c 33d2d0db4dd4e7a7a7e9a7f790414293277f9e7682a2fd9d61c713bfc37cd8b6 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674 @@ -2051,9 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0b3b5bf9597615589a1d045aaa697c13550553ee4fe4b9008a8e51415b6fe96a 94944b239ce674d984c88ef6029b0260a972f1b25f01614b559ca07c3ebaf8f5 -R 07b8484e41d6b78cbc774ca07208b7eb -T +closed 94944b239ce674d984c88ef6029b0260a972f1b25f01614b559ca07c3ebaf8f5 -U drh -Z 4e9f7dbd3bbd0c5da8cb618454aab138 +P 0910b1925e97f7ae4dae86894c9e2f54273c85115e19d0d9bff1280ffee35eed +R 0669622949fcc9c4cc476c1d0c95dee1 +U dan +Z 7914734e12c594e04b43e5b8d992dc23 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 91c00e53a6..fea3d9d247 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0910b1925e97f7ae4dae86894c9e2f54273c85115e19d0d9bff1280ffee35eed \ No newline at end of file +02ac2297abee6af64c8df230b42b07f21cff4565d7e315860b2396a7c0c556ca \ No newline at end of file From 2ff65699401b9cd67e5c246cbdf0e174a7004931 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 19 Mar 2023 21:48:55 +0000 Subject: [PATCH 101/341] Increase the size of ref-count values in the pager layer to 64-bits, to avoid any reasonable possiblity of overflowing the counters. There is a performance and memory penality for this. [forum:/forumpost/b741f15a35|Forum post b741f15a35]. FossilOrigin-Name: 6c5d99a813e8498c2ea91383ac8886a43eaf47023919c03900e893640c03778d --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pcache.c | 8 ++++---- src/pcache.h | 6 +++--- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index a56838f0a0..138c8790b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\sbuffer\soverread\sin\sfts3\sthat\scould\soccur\swhen\sprocessing\sa\scorrupt\srecord. -D 2023-03-18T16:12:27.555 +C Increase\sthe\ssize\sof\sref-count\svalues\sin\sthe\spager\slayer\sto\s64-bits,\sto\savoid\nany\sreasonable\spossiblity\sof\soverflowing\sthe\scounters.\nThere\sis\sa\sperformance\sand\smemory\spenality\sfor\sthis.\n[forum:/forumpost/b741f15a35|Forum\spost\sb741f15a35]. +D 2023-03-19T21:48:55.585 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -615,8 +615,8 @@ F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 39af8ff7c73a991f61f4d1e3a6f8f98b1c8e29144723507822774cac5e6ee0b5 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y 424e49ed8fc6c907920db9be5a13a75ed43811e1ea8dd21b0fa9ef97f083dc6b -F src/pcache.c f4268f7f73c6a3db12ce22fd25bc68dc42315d19599414ab1207d7cf32f79197 -F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 +F src/pcache.c 842410539b544e12d5fccfcf29890782f46a58f227a77bc0bd76243799662c0c +F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c c0c6e0735436e0e9e8d490f82959ddc9475d438e9f808a51953a4d43e30cea51 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0910b1925e97f7ae4dae86894c9e2f54273c85115e19d0d9bff1280ffee35eed -R 0669622949fcc9c4cc476c1d0c95dee1 -U dan -Z 7914734e12c594e04b43e5b8d992dc23 +P 02ac2297abee6af64c8df230b42b07f21cff4565d7e315860b2396a7c0c556ca +R 786303851e34199f284746d4fa5c1e0a +U drh +Z 90a20a841f2fa0f322839e6c9c925028 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fea3d9d247..a0d05a322e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -02ac2297abee6af64c8df230b42b07f21cff4565d7e315860b2396a7c0c556ca \ No newline at end of file +6c5d99a813e8498c2ea91383ac8886a43eaf47023919c03900e893640c03778d \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index 0407e06b2f..153628dc9a 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -41,7 +41,7 @@ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRefSum; /* Sum of ref counts over all pages */ + i64 nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ int szSpill; /* Size before spilling occurs */ int szPage; /* Size of every page in this cache */ @@ -71,7 +71,7 @@ struct PCache { unsigned char *a; int j; pPg = (PgHdr*)pLower->pExtra; - printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); + printf("%3lld: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); a = (unsigned char *)pLower->pBuf; for(j=0; j<12; j++) printf("%02x", a[j]); printf(" ptr %p\n", pPg); @@ -815,14 +815,14 @@ PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ ** This is not the total number of pages referenced, but the sum of the ** reference count for all pages. */ -int sqlite3PcacheRefCount(PCache *pCache){ +i64 sqlite3PcacheRefCount(PCache *pCache){ return pCache->nRefSum; } /* ** Return the number of references to the page supplied as an argument. */ -int sqlite3PcachePageRefcount(PgHdr *p){ +i64 sqlite3PcachePageRefcount(PgHdr *p){ return p->nRef; } diff --git a/src/pcache.h b/src/pcache.h index eb55396afa..f945dab1a4 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -40,7 +40,7 @@ struct PgHdr { ** private to pcache.c and should not be accessed by other modules. ** pCache is grouped with the public elements for efficiency. */ - i16 nRef; /* Number of users of this page */ + i64 nRef; /* Number of users of this page */ PgHdr *pDirtyNext; /* Next element in list of dirty pages */ PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ /* NB: pDirtyNext and pDirtyPrev are undefined if the @@ -121,12 +121,12 @@ void sqlite3PcacheClearSyncFlags(PCache *); void sqlite3PcacheClear(PCache*); /* Return the total number of outstanding page references */ -int sqlite3PcacheRefCount(PCache*); +i64 sqlite3PcacheRefCount(PCache*); /* Increment the reference count of an existing page */ void sqlite3PcacheRef(PgHdr*); -int sqlite3PcachePageRefcount(PgHdr*); +i64 sqlite3PcachePageRefcount(PgHdr*); /* Return the total number of pages stored in the cache */ int sqlite3PcachePagecount(PCache*); From 0d53e8ed543e9501026b895b8a02a7822d333b99 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 20 Mar 2023 00:48:40 +0000 Subject: [PATCH 102/341] When reporting errors in the CLI, ignore the output of sqlite3_error_offset() if the value returned is clearly out-of-range. One of two lines of defense against [33aa4c0de8a62e33]. FossilOrigin-Name: 26adbb80f51b9219ab2a3593496458b74a567ba5ee46b5154f056e389cfc09f7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 138c8790b9..05483f4ab4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\ssize\sof\sref-count\svalues\sin\sthe\spager\slayer\sto\s64-bits,\sto\savoid\nany\sreasonable\spossiblity\sof\soverflowing\sthe\scounters.\nThere\sis\sa\sperformance\sand\smemory\spenality\sfor\sthis.\n[forum:/forumpost/b741f15a35|Forum\spost\sb741f15a35]. -D 2023-03-19T21:48:55.585 +C When\sreporting\serrors\sin\sthe\sCLI,\signore\sthe\soutput\sof\ssqlite3_error_offset()\nif\sthe\svalue\sreturned\sis\sclearly\sout-of-range.\s\sOne\sof\stwo\slines\sof\sdefense\nagainst\s[33aa4c0de8a62e33]. +D 2023-03-20T00:48:40.933 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 6a0253379cc15b3f80321362a61f487a8ef7cd2487fe62e1eb2317b3f871c61f F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa -F src/shell.c.in 55c54f1a48d7c78a0fca98a685ec6809b0ce8782169d441d7dd2c891dc07cfc3 +F src/shell.c.in b47c3489777c6bb7b30c049bef3ef925528c19fbb53d980d772d0b8e8841db8f F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 02ac2297abee6af64c8df230b42b07f21cff4565d7e315860b2396a7c0c556ca -R 786303851e34199f284746d4fa5c1e0a +P 6c5d99a813e8498c2ea91383ac8886a43eaf47023919c03900e893640c03778d +R 9c89c66108b961928ba91967c3305114 U drh -Z 90a20a841f2fa0f322839e6c9c925028 +Z 117402f7ae1dac33496e02b8aa6ae706 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a0d05a322e..13bd45e197 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c5d99a813e8498c2ea91383ac8886a43eaf47023919c03900e893640c03778d \ No newline at end of file +26adbb80f51b9219ab2a3593496458b74a567ba5ee46b5154f056e389cfc09f7 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1f49a50565..8e2ad1cb23 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -2774,6 +2774,7 @@ static char *shell_error_context(const char *zSql, sqlite3 *db){ if( db==0 || zSql==0 || (iOffset = sqlite3_error_offset(db))<0 + || iOffset>=strlen(zSql) ){ return sqlite3_mprintf(""); } From 715963f1ba3c173a7145858e5ee2461fc8431660 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 20 Mar 2023 00:53:56 +0000 Subject: [PATCH 103/341] Expression errors in generated columns should not generate non-negative sqlite3_error_offset() returns. Second of two defenses against [33aa4c0de8a62e33]. FossilOrigin-Name: 2adb4e0dda471ffc9b2a5cbad01b6c2bc6091b7526a8c8b19944cb2d4a5111e7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 05483f4ab4..5ca8b0eb91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sreporting\serrors\sin\sthe\sCLI,\signore\sthe\soutput\sof\ssqlite3_error_offset()\nif\sthe\svalue\sreturned\sis\sclearly\sout-of-range.\s\sOne\sof\stwo\slines\sof\sdefense\nagainst\s[33aa4c0de8a62e33]. -D 2023-03-20T00:48:40.933 +C Expression\serrors\sin\sgenerated\scolumns\sshould\snot\sgenerate\snon-negative\nsqlite3_error_offset()\sreturns.\s\sSecond\sof\stwo\sdefenses\sagainst\n[33aa4c0de8a62e33]. +D 2023-03-20T00:53:56.312 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 201fe0763c52783d205c8c13cdd9d55c1bd5cb21c1f036753f99103b43284b90 -F src/expr.c 55df0e3312a36efe597e48f299da390a83584d3371a311a35e58e537f462b4ff +F src/expr.c 46aa5d02c6815c8addb0ea42fffaf6eaefeb68fdf0ebe26dd53da1a87b8e0737 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6c5d99a813e8498c2ea91383ac8886a43eaf47023919c03900e893640c03778d -R 9c89c66108b961928ba91967c3305114 +P 26adbb80f51b9219ab2a3593496458b74a567ba5ee46b5154f056e389cfc09f7 +R a948b20149315f9dc540cd3e08c48bb0 U drh -Z 117402f7ae1dac33496e02b8aa6ae706 +Z bb36e2b1c27e531b9e4a73caa47bce20 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 13bd45e197..ea71c8424e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -26adbb80f51b9219ab2a3593496458b74a567ba5ee46b5154f056e389cfc09f7 \ No newline at end of file +2adb4e0dda471ffc9b2a5cbad01b6c2bc6091b7526a8c8b19944cb2d4a5111e7 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 223ebf1cbd..ac6f538727 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3841,6 +3841,7 @@ void sqlite3ExprCodeGeneratedColumn( Vdbe *v = pParse->pVdbe; assert( v!=0 ); assert( pParse->iSelfTab!=0 ); + if( pParse->nErr ) return; if( pParse->iSelfTab>0 ){ iAddr = sqlite3VdbeAddOp3(v, OP_IfNullRow, pParse->iSelfTab-1, 0, regOut); }else{ @@ -3851,6 +3852,7 @@ void sqlite3ExprCodeGeneratedColumn( sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); } if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); + pParse->db->errByteOffset = -1; } #endif /* SQLITE_OMIT_GENERATED_COLUMNS */ From 6263cefcf5d236f8d1ed1a6b997ed858b6c8dc4b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 20 Mar 2023 01:55:39 +0000 Subject: [PATCH 104/341] A better fix for the sqlite3_error_offset() problem on generated columns. FossilOrigin-Name: 770b3e67c89ca16b08a9234acb626917a26ea414b3bc0340f9601644cb7504f2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 5ca8b0eb91..606ad4918a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expression\serrors\sin\sgenerated\scolumns\sshould\snot\sgenerate\snon-negative\nsqlite3_error_offset()\sreturns.\s\sSecond\sof\stwo\sdefenses\sagainst\n[33aa4c0de8a62e33]. -D 2023-03-20T00:53:56.312 +C A\sbetter\sfix\sfor\sthe\ssqlite3_error_offset()\sproblem\son\sgenerated\scolumns. +D 2023-03-20T01:55:39.149 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 201fe0763c52783d205c8c13cdd9d55c1bd5cb21c1f036753f99103b43284b90 -F src/expr.c 46aa5d02c6815c8addb0ea42fffaf6eaefeb68fdf0ebe26dd53da1a87b8e0737 +F src/expr.c e02117f1d698d781dc5990364bec01ef19688f084e14b5ea468a972c3b26903b F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 26adbb80f51b9219ab2a3593496458b74a567ba5ee46b5154f056e389cfc09f7 -R a948b20149315f9dc540cd3e08c48bb0 +P 2adb4e0dda471ffc9b2a5cbad01b6c2bc6091b7526a8c8b19944cb2d4a5111e7 +R 6c29c6313761fba8300293fe905fb541 U drh -Z bb36e2b1c27e531b9e4a73caa47bce20 +Z ca1ef3f48c098aade28ded216a6d7df7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ea71c8424e..b0bf79293a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2adb4e0dda471ffc9b2a5cbad01b6c2bc6091b7526a8c8b19944cb2d4a5111e7 \ No newline at end of file +770b3e67c89ca16b08a9234acb626917a26ea414b3bc0340f9601644cb7504f2 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index ac6f538727..0a52e62309 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3839,10 +3839,10 @@ void sqlite3ExprCodeGeneratedColumn( ){ int iAddr; Vdbe *v = pParse->pVdbe; + int nErr = pParse->nErr; assert( v!=0 ); assert( pParse->iSelfTab!=0 ); - if( pParse->nErr ) return; - if( pParse->iSelfTab>0 ){ + if( pParse->iSelfTab>0 ){ iAddr = sqlite3VdbeAddOp3(v, OP_IfNullRow, pParse->iSelfTab-1, 0, regOut); }else{ iAddr = 0; @@ -3852,7 +3852,7 @@ void sqlite3ExprCodeGeneratedColumn( sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); } if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); - pParse->db->errByteOffset = -1; + if( pParse->nErr>nErr ) pParse->db->errByteOffset = -1; } #endif /* SQLITE_OMIT_GENERATED_COLUMNS */ From f3ddaaa89a12d4431461bfdb3c35c74341ed3c65 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 20 Mar 2023 10:43:52 +0000 Subject: [PATCH 105/341] Back out the extra margin added to the input buffer of the CLI, as it is not needed. FossilOrigin-Name: ac8d1e5de5da35fa93c3f9241a7817285920431c66f4e4593d4d5c2db9f2b960 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/shell.c.in | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 606ad4918a..55f4c5fe5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sbetter\sfix\sfor\sthe\ssqlite3_error_offset()\sproblem\son\sgenerated\scolumns. -D 2023-03-20T01:55:39.149 +C Back\sout\sthe\sextra\smargin\sadded\sto\sthe\sinput\sbuffer\sof\sthe\sCLI,\sas\sit\sis\snot\nneeded. +D 2023-03-20T10:43:52.546 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 6a0253379cc15b3f80321362a61f487a8ef7cd2487fe62e1eb2317b3f871c61f F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa -F src/shell.c.in b47c3489777c6bb7b30c049bef3ef925528c19fbb53d980d772d0b8e8841db8f +F src/shell.c.in 7394f84ace7bcff18c686b09ff7a1c02ffc5fb8fb851b3b2be128762ee71589e F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2051,8 +2051,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2adb4e0dda471ffc9b2a5cbad01b6c2bc6091b7526a8c8b19944cb2d4a5111e7 -R 6c29c6313761fba8300293fe905fb541 +P 770b3e67c89ca16b08a9234acb626917a26ea414b3bc0340f9601644cb7504f2 +Q -741af08af1b93406a120580379d13e514524af627da5387ecfa6e442d004bfdd +R 6fce8a37edd3eccfb817f05a356bd030 U drh -Z ca1ef3f48c098aade28ded216a6d7df7 +Z 0c2311a8849988977130c5a153101397 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b0bf79293a..ce4eb878f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -770b3e67c89ca16b08a9234acb626917a26ea414b3bc0340f9601644cb7504f2 \ No newline at end of file +ac8d1e5de5da35fa93c3f9241a7817285920431c66f4e4593d4d5c2db9f2b960 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 8e2ad1cb23..199a6df9dd 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11466,9 +11466,9 @@ static int process_input(ShellState *p){ } /* No single-line dispositions remain; accumulate line(s). */ nLine = strlen(zLine); - if( nSql+nLine+8>=nAlloc ){ + if( nSql+nLine+2>=nAlloc ){ /* Grow buffer by half-again increments when big. */ - nAlloc = nSql+(nSql>>1)+nLine+104; + nAlloc = nSql+(nSql>>1)+nLine+100; zSql = realloc(zSql, nAlloc); shell_check_oom(zSql); } From bb130c9223397ab9b29f2c4cc6de28e5623e0f0e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Mar 2023 14:59:19 +0000 Subject: [PATCH 106/341] Fix a problem causing a cursor to retain an out-of-date cell-info cache when processing a DISTINCT query on values that are identical according to their collation sequence, but different on disk. [forum:/forumpost/e132e6cde44fb505|Forum post e123e6cde4]. FossilOrigin-Name: 1b3abc1daeac29714256b5a1d5a07a75dc986f1089054a8bee44a00583b7383a --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/btree.c | 2 +- test/distinct2.test | 12 ++++++++++++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 55f4c5fe5b..aa2e57ba8f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Back\sout\sthe\sextra\smargin\sadded\sto\sthe\sinput\sbuffer\sof\sthe\sCLI,\sas\sit\sis\snot\nneeded. -D 2023-03-20T10:43:52.546 +C Fix\sa\sproblem\scausing\sa\scursor\sto\sretain\san\sout-of-date\scell-info\scache\swhen\sprocessing\sa\sDISTINCT\squery\son\svalues\sthat\sare\sidentical\saccording\sto\stheir\scollation\ssequence,\sbut\sdifferent\son\sdisk.\s[forum:/forumpost/e132e6cde44fb505|Forum\spost\se123e6cde4]. +D 2023-03-20T14:59:19.115 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 9192990bb78806df33ac680df47cb4100b5dbe5d298c3d0abffc82cd31ca6634 +F src/btree.c 8c04a7ed24642970158ae9fbf0d146d9625615094f84a4a24e0a6ed2d9f3fd62 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -938,7 +938,7 @@ F test/descidx2.test a0ba347037ff3b811f4c6ceca5fd0f9d5d72e74e59f2d9de346a9d2f6ad F test/descidx3.test 953c831df7ea219c73826dfbf2f6ee02d95040725aa88ccb4fa43d1a1999b926 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 691c9e850b0d0b56b66e7e235453198cb4cf0760e324b7403d3c5abbeab0a014 -F test/distinct2.test cd1d15a4a2abf579298f7161e821ed50c0119136fe0424db85c52cf0adc230d1 +F test/distinct2.test bb71cc7b5e58e895787f9910a788c254f679928d324732d063fe9bc202ecbe71 F test/distinctagg.test 14ec5026e684eddd414c61c08692b43773e224ac92efbed6ec08c6994bc39723 F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52 F test/e_blobclose.test 692fc02a058476c2222a63d97e3f3b2b809c1842e5525ded7f854d540ac2e075 @@ -2051,9 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 770b3e67c89ca16b08a9234acb626917a26ea414b3bc0340f9601644cb7504f2 -Q -741af08af1b93406a120580379d13e514524af627da5387ecfa6e442d004bfdd -R 6fce8a37edd3eccfb817f05a356bd030 -U drh -Z 0c2311a8849988977130c5a153101397 +P ac8d1e5de5da35fa93c3f9241a7817285920431c66f4e4593d4d5c2db9f2b960 +R ba20ce5c356b9d123e09b3173425d3de +U dan +Z f96e7e21af2385632fea87093dba304b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ce4eb878f2..bb80796c5b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac8d1e5de5da35fa93c3f9241a7817285920431c66f4e4593d4d5c2db9f2b960 \ No newline at end of file +1b3abc1daeac29714256b5a1d5a07a75dc986f1089054a8bee44a00583b7383a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 918388a64a..2fbceb7d2a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9261,6 +9261,7 @@ int sqlite3BtreeInsert( assert( szNew==pPage->xCellSize(pPage, newCell) ); assert( szNew <= MX_CELL_SIZE(p->pBt) ); idx = pCur->ix; + pCur->info.nSize = 0; if( loc==0 ){ CellInfo info; assert( idx>=0 ); @@ -9333,7 +9334,6 @@ int sqlite3BtreeInsert( ** larger than the largest existing key, it is possible to insert the ** row without seeking the cursor. This can be a big performance boost. */ - pCur->info.nSize = 0; if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); pCur->curFlags &= ~(BTCF_ValidNKey); diff --git a/test/distinct2.test b/test/distinct2.test index 46eace6f0f..958d9634fd 100644 --- a/test/distinct2.test +++ b/test/distinct2.test @@ -301,4 +301,16 @@ do_execsql_test 3030 { {} 1 a } +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 4010 { + CREATE TABLE t1(a, b COLLATE RTRIM); + INSERT INTO t1 VALUES(1, ''), (2, ' '), (3, ' '); +} +do_execsql_test 4020 { + SELECT b FROM t1 UNION SELECT 1; +} {1 { }} + finish_test From 1eca5b5a4011efd2ac5a89396e812d5b3119be38 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 20 Mar 2023 18:35:48 +0000 Subject: [PATCH 107/341] Minor change to btreeNext() to facilitate coverage testing. FossilOrigin-Name: 20b3ef04d8c79e281e32676d57c7a8569fac9e782ca24337691d44d383eff7bb --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index aa2e57ba8f..3fd3923e33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\sa\scursor\sto\sretain\san\sout-of-date\scell-info\scache\swhen\sprocessing\sa\sDISTINCT\squery\son\svalues\sthat\sare\sidentical\saccording\sto\stheir\scollation\ssequence,\sbut\sdifferent\son\sdisk.\s[forum:/forumpost/e132e6cde44fb505|Forum\spost\se123e6cde4]. -D 2023-03-20T14:59:19.115 +C Minor\schange\sto\sbtreeNext()\sto\sfacilitate\scoverage\stesting. +D 2023-03-20T18:35:48.209 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 8c04a7ed24642970158ae9fbf0d146d9625615094f84a4a24e0a6ed2d9f3fd62 +F src/btree.c 47def1b43b10fcffd927ef6e17c9382a39c898af9eb98318460253ddb5e4a07f F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac8d1e5de5da35fa93c3f9241a7817285920431c66f4e4593d4d5c2db9f2b960 -R ba20ce5c356b9d123e09b3173425d3de -U dan -Z f96e7e21af2385632fea87093dba304b +P 1b3abc1daeac29714256b5a1d5a07a75dc986f1089054a8bee44a00583b7383a +R 6331ee6de9d8828fe1972f245b77c00c +U drh +Z 11eb1b62cd83892815efd7aae8753438 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bb80796c5b..8d262b4141 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b3abc1daeac29714256b5a1d5a07a75dc986f1089054a8bee44a00583b7383a \ No newline at end of file +20b3ef04d8c79e281e32676d57c7a8569fac9e782ca24337691d44d383eff7bb \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2fbceb7d2a..e90d0c1f1c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6132,7 +6132,8 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ pPage = pCur->pPage; idx = ++pCur->ix; - if( !pPage->isInit || sqlite3FaultSim(412) ){ + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ return SQLITE_CORRUPT_BKPT; } From 29fbee66c93582c714b1535cb56ea708510d08c3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 21 Mar 2023 11:13:53 +0000 Subject: [PATCH 108/341] Fix a valgrind error and potential buffer overread when handling a corrupt database. FossilOrigin-Name: b1e0cd6444d1f710e58129723b285cf1321679fa920fc2841492dcb489ab8b9d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 8 +++++++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3fd3923e33..21407ac24f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schange\sto\sbtreeNext()\sto\sfacilitate\scoverage\stesting. -D 2023-03-20T18:35:48.209 +C Fix\sa\svalgrind\serror\sand\spotential\sbuffer\soverread\swhen\shandling\sa\scorrupt\sdatabase. +D 2023-03-21T11:13:53.102 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -710,7 +710,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c f369f3423da9c0d71f7206345e6e775aa06997b74bdb70934ed4677187a72eb8 +F src/where.c db043f9b2da776105e04b0796f42034745a8bb59433d5b7dae798a4417987e52 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 9919e5a22f4b24dd96c49b8981484cbe6bbfcf466ff73ac40a06e1356aa8bf87 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1b3abc1daeac29714256b5a1d5a07a75dc986f1089054a8bee44a00583b7383a -R 6331ee6de9d8828fe1972f245b77c00c -U drh -Z 11eb1b62cd83892815efd7aae8753438 +P 20b3ef04d8c79e281e32676d57c7a8569fac9e782ca24337691d44d383eff7bb +R a3001c5c51434121b500669f922f717c +U dan +Z 9c6312542876d7b19be169e17e21a1f1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8d262b4141..10c02662b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20b3ef04d8c79e281e32676d57c7a8569fac9e782ca24337691d44d383eff7bb \ No newline at end of file +b1e0cd6444d1f710e58129723b285cf1321679fa920fc2841492dcb489ab8b9d \ No newline at end of file diff --git a/src/where.c b/src/where.c index f707fab12b..9de72d76de 100644 --- a/src/where.c +++ b/src/where.c @@ -1526,6 +1526,7 @@ static int whereKeyStats( assert( pRec!=0 ); assert( pIdx->nSample>0 ); assert( pRec->nField>0 ); + /* Do a binary search to find the first sample greater than or equal ** to pRec. If pRec contains a single field, the set of samples to search @@ -1571,7 +1572,12 @@ static int whereKeyStats( ** it is extended to two fields. The duplicates that this creates do not ** cause any problems. */ - nField = MIN(pRec->nField, pIdx->nSample); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nField = pIdx->nKeyCol; + }else{ + nField = pIdx->nColumn; + } + nField = MIN(pRec->nField, nField); iCol = 0; iSample = pIdx->nSample * nField; do{ From 2439fc1f6dda432d45b7e5badd6e88989a687bc2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 21 Mar 2023 11:56:04 +0000 Subject: [PATCH 109/341] Add the fuzzcheck-asan target to the main posix makefile. FossilOrigin-Name: 55e94adddba77c1d6ee6ef0c0cf37f7cdccc3bdc144ceaed675d79cdb25f48a6 --- Makefile.in | 3 +++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Makefile.in b/Makefile.in index f278b053e6..61cb3ef0e1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -697,6 +697,9 @@ fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h fuzzcheck$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) +fuzzcheck-asan$(TEXE): $(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP) + $(LTLINK) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS) + ossshell$(TEXE): $(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \ $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS) diff --git a/manifest b/manifest index 21407ac24f..a8451026ae 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sa\svalgrind\serror\sand\spotential\sbuffer\soverread\swhen\shandling\sa\scorrupt\sdatabase. -D 2023-03-21T11:13:53.102 +C Add\sthe\sfuzzcheck-asan\starget\sto\sthe\smain\sposix\smakefile. +D 2023-03-21T11:56:04.957 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 73076ae60a143476b6890b5c3b88848fce19ac8caefc3ff5ae97f72694dc946d +F Makefile.in 764f2e3e8fb4ae1c8dfe03e65b2b3b01bd1fc57edf78ec2cab3a1301e90e1905 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc 492eace1ce32ec464792ae60a9b67225ddb68ef742f1a44d25f47745c9c59d3d F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 20b3ef04d8c79e281e32676d57c7a8569fac9e782ca24337691d44d383eff7bb -R a3001c5c51434121b500669f922f717c -U dan -Z 9c6312542876d7b19be169e17e21a1f1 +P b1e0cd6444d1f710e58129723b285cf1321679fa920fc2841492dcb489ab8b9d +R 50824eb04e8f08352358f9986fed7b28 +U drh +Z 5768155f673f1bab658f8db6b588a12f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 10c02662b7..bbf3b438db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b1e0cd6444d1f710e58129723b285cf1321679fa920fc2841492dcb489ab8b9d \ No newline at end of file +55e94adddba77c1d6ee6ef0c0cf37f7cdccc3bdc144ceaed675d79cdb25f48a6 \ No newline at end of file From 4e45e01693f78162ddbc3259bbf3c58574d86f2e Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 21 Mar 2023 12:29:50 +0000 Subject: [PATCH 110/341] Add the fuzzcheck-asan.exe target to the MSVC makefile. FossilOrigin-Name: 0901bc02783527090bdaa626d946ee1e92c7c6848f3c77a799e6837f4e7f8d7b --- Makefile.msc | 3 +++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 63207301df..a6b74cdc20 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1816,6 +1816,9 @@ dbfuzz.exe: $(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H) fuzzcheck.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +fuzzcheck-asan.exe: $(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) /fsanitize=address $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + ossshell.exe: $(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) diff --git a/manifest b/manifest index a8451026ae..b2f12a3f53 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Add\sthe\sfuzzcheck-asan\starget\sto\sthe\smain\sposix\smakefile. -D 2023-03-21T11:56:04.957 +C Add\sthe\sfuzzcheck-asan.exe\starget\sto\sthe\sMSVC\smakefile. +D 2023-03-21T12:29:50.863 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 764f2e3e8fb4ae1c8dfe03e65b2b3b01bd1fc57edf78ec2cab3a1301e90e1905 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc 492eace1ce32ec464792ae60a9b67225ddb68ef742f1a44d25f47745c9c59d3d +F Makefile.msc ada3466f8f0112a8baead4d6cc2a99bf544d228958baae12ca35a3ee5755c806 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e F VERSION 17f95ae2fdf21f0e9575eb0b0511ea63f15d71dfff431b21c2b4adbfa70cfbbf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b1e0cd6444d1f710e58129723b285cf1321679fa920fc2841492dcb489ab8b9d -R 50824eb04e8f08352358f9986fed7b28 +P 55e94adddba77c1d6ee6ef0c0cf37f7cdccc3bdc144ceaed675d79cdb25f48a6 +R 3dfde50db61b3d09a5903868e1d360e1 U drh -Z 5768155f673f1bab658f8db6b588a12f +Z 41857f2f862e112dfc52716e5f288973 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bbf3b438db..3d3f7cbb4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55e94adddba77c1d6ee6ef0c0cf37f7cdccc3bdc144ceaed675d79cdb25f48a6 \ No newline at end of file +0901bc02783527090bdaa626d946ee1e92c7c6848f3c77a799e6837f4e7f8d7b \ No newline at end of file From 68e976bcc2bdd6f0f5d583cc07b64969bbba25d0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 21 Mar 2023 14:20:10 +0000 Subject: [PATCH 111/341] Add ALWAYS() on a branch this is always true now due to [84417bbd144b2197]. FossilOrigin-Name: badf7d0e3cfa6efaff4b132cb4ecca79a16e73197c5e275db14ccb5ff938276d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/resolve.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b2f12a3f53..2cf47b2054 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sfuzzcheck-asan.exe\starget\sto\sthe\sMSVC\smakefile. -D 2023-03-21T12:29:50.863 +C Add\sALWAYS()\son\sa\sbranch\sthis\sis\salways\strue\snow\sdue\sto\s[84417bbd144b2197]. +D 2023-03-21T14:20:10.860 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -623,7 +623,7 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 F src/printf.c fb31597cf93200eba9167573094ee312b0d51b2c1998613284ceb2c8c212b492 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c -F src/resolve.c 6a0253379cc15b3f80321362a61f487a8ef7cd2487fe62e1eb2317b3f871c61f +F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa F src/shell.c.in 7394f84ace7bcff18c686b09ff7a1c02ffc5fb8fb851b3b2be128762ee71589e @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 55e94adddba77c1d6ee6ef0c0cf37f7cdccc3bdc144ceaed675d79cdb25f48a6 -R 3dfde50db61b3d09a5903868e1d360e1 +P 0901bc02783527090bdaa626d946ee1e92c7c6848f3c77a799e6837f4e7f8d7b +R 26f3f2ce957685138bbdbbf793825478 U drh -Z 41857f2f862e112dfc52716e5f288973 +Z 9924100eafef345f3f8e1d95add43173 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3d3f7cbb4f..e83182887d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0901bc02783527090bdaa626d946ee1e92c7c6848f3c77a799e6837f4e7f8d7b \ No newline at end of file +badf7d0e3cfa6efaff4b132cb4ecca79a16e73197c5e275db14ccb5ff938276d \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 4b36ecca34..adfcc8dbe9 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -467,7 +467,8 @@ static int lookupName( assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); if( pParse->bReturning ){ if( (pNC->ncFlags & NC_UBaseReg)!=0 - && (zTab==0 || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + && ALWAYS(zTab==0 + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) ){ pExpr->iTable = op!=TK_DELETE; pTab = pParse->pTriggerTab; From 00cd655c8f4416fa8eeddff466ad79dfb55aefb2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 22 Mar 2023 14:51:22 +0000 Subject: [PATCH 112/341] Update the version number for the TEA tarball to 3.42.0, to match the core. FossilOrigin-Name: 03e6918e7f749c3ca1fa66df28b203ff26680f129003e20c71ca246cf7ff34fe --- autoconf/tea/configure.ac | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/autoconf/tea/configure.ac b/autoconf/tea/configure.ac index f6291d2557..e26780e2e9 100644 --- a/autoconf/tea/configure.ac +++ b/autoconf/tea/configure.ac @@ -19,7 +19,7 @@ dnl to configure the system for the local environment. # so that we create the export library with the dll. #----------------------------------------------------------------------- -AC_INIT([sqlite],[3.41.0]) +AC_INIT([sqlite],[3.42.0]) #-------------------------------------------------------------------- # Call TEA_INIT as the first TEA_ macro to set up initial vars. diff --git a/manifest b/manifest index 2cf47b2054..cfbaa56a74 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sALWAYS()\son\sa\sbranch\sthis\sis\salways\strue\snow\sdue\sto\s[84417bbd144b2197]. -D 2023-03-21T14:20:10.860 +C Update\sthe\sversion\snumber\sfor\sthe\sTEA\starball\sto\s3.42.0,\sto\smatch\sthe\score. +D 2023-03-22T14:51:22.647 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -22,7 +22,7 @@ F autoconf/configure.ac ec7fa914c5e74ff212fe879f9bb6918e1234497e05facfb641f30c4d F autoconf/tea/Makefile.in 106a96f2f745d41a0f6193f1de98d7355830b65d45032c18cd7c90295ec24196 F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.ac d1eaa72f4d202ca106f4b2762733709d3f89a877da2e385a57881c4ad0427be7 +F autoconf/tea/configure.ac e6a7f2abf7a94b5dae00713ad6b3bebc59b878e2705607589830792c61d2ca38 F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in b9eb6dd37f64e08e637d576b3c83259814b9cddd78bec4af2e5abfc6c5c750ce @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0901bc02783527090bdaa626d946ee1e92c7c6848f3c77a799e6837f4e7f8d7b -R 26f3f2ce957685138bbdbbf793825478 +P badf7d0e3cfa6efaff4b132cb4ecca79a16e73197c5e275db14ccb5ff938276d +R f9a0e32a5a944935ccdf4d832a3793b8 U drh -Z 9924100eafef345f3f8e1d95add43173 +Z c47bf29e585485f3f7c489971119cf34 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e83182887d..4085c2d2fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -badf7d0e3cfa6efaff4b132cb4ecca79a16e73197c5e275db14ccb5ff938276d \ No newline at end of file +03e6918e7f749c3ca1fa66df28b203ff26680f129003e20c71ca246cf7ff34fe \ No newline at end of file From 8854f30295cb35cb5d6185aed84d2fe6336dcf8a Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 22 Mar 2023 16:01:06 +0000 Subject: [PATCH 113/341] In the CLI, the magic parameter :inf and :nan bind floating point values Infinity and NaN, respectively, as an add to testing SQLite's handling of those quantities. FossilOrigin-Name: c70a61d8fbfb722679398c211aa48ccd84a392a6b59ba70551fe3a9fbab9a6d5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 18 ++++++++++++------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index cfbaa56a74..fe149a5cb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sversion\snumber\sfor\sthe\sTEA\starball\sto\s3.42.0,\sto\smatch\sthe\score. -D 2023-03-22T14:51:22.647 +C In\sthe\sCLI,\sthe\smagic\sparameter\s:inf\sand\s:nan\sbind\sfloating\spoint\svalues\nInfinity\sand\sNaN,\srespectively,\sas\san\sadd\sto\stesting\sSQLite's\shandling\sof\nthose\squantities. +D 2023-03-22T16:01:06.778 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa -F src/shell.c.in 7394f84ace7bcff18c686b09ff7a1c02ffc5fb8fb851b3b2be128762ee71589e +F src/shell.c.in e66360191de42f8b43454886a059e3eb983e6dc0f2bd855dd4ff5244694f61a6 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P badf7d0e3cfa6efaff4b132cb4ecca79a16e73197c5e275db14ccb5ff938276d -R f9a0e32a5a944935ccdf4d832a3793b8 +P 03e6918e7f749c3ca1fa66df28b203ff26680f129003e20c71ca246cf7ff34fe +R 8e977335e4f9dd507a1a79512cd70e26 U drh -Z c47bf29e585485f3f7c489971119cf34 +Z e01204911fdf00ed5178cd1186ef489a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4085c2d2fb..50d888f24e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03e6918e7f749c3ca1fa66df28b203ff26680f129003e20c71ca246cf7ff34fe \ No newline at end of file +c70a61d8fbfb722679398c211aa48ccd84a392a6b59ba70551fe3a9fbab9a6d5 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 199a6df9dd..b97889f938 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -99,6 +99,7 @@ typedef unsigned short int u16; #include #include #include +#include #include "sqlite3.h" typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; @@ -3387,12 +3388,13 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ if( nVar==0 ) return; /* Nothing to do */ if( sqlite3_table_column_metadata(pArg->db, "TEMP", "sqlite_parameters", "key", 0, 0, 0, 0, 0)!=SQLITE_OK ){ - return; /* Parameter table does not exist */ + rc = SQLITE_NOTFOUND; + pQ = 0; + }else{ + rc = sqlite3_prepare_v2(pArg->db, + "SELECT value FROM temp.sqlite_parameters" + " WHERE key=?1", -1, &pQ, 0); } - rc = sqlite3_prepare_v2(pArg->db, - "SELECT value FROM temp.sqlite_parameters" - " WHERE key=?1", -1, &pQ, 0); - if( rc || pQ==0 ) return; for(i=1; i<=nVar; i++){ char zNum[30]; const char *zVar = sqlite3_bind_parameter_name(pStmt, i); @@ -3401,8 +3403,12 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ zVar = zNum; } sqlite3_bind_text(pQ, 1, zVar, -1, SQLITE_STATIC); - if( sqlite3_step(pQ)==SQLITE_ROW ){ + if( rc==SQLITE_OK && pQ && sqlite3_step(pQ)==SQLITE_ROW ){ sqlite3_bind_value(pStmt, i, sqlite3_column_value(pQ, 0)); + }else if( sqlite3_strlike("_NAN", zVar, 0)==0 ){ + sqlite3_bind_double(pStmt, i, NAN); + }else if( sqlite3_strlike("_INF", zVar, 0)==0 ){ + sqlite3_bind_double(pStmt, i, INFINITY); }else{ sqlite3_bind_null(pStmt, i); } From 5e04d26a3d6861fc2968bd05dbe6f4ec58f43821 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 22 Mar 2023 16:37:17 +0000 Subject: [PATCH 114/341] For consistency, the ".mode json" output from the CLI now renders infinity in the same format as the JSON functions. FossilOrigin-Name: abee339d5eae8b63d9579b7ec3a25f18071e675223e0bb5294c733fb618a3a26 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/shell.c.in | 8 ++++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index b032f01988..ed02a96544 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sdouble-to-text\sconversion\srenders\sinfinity\sas\s9e999,\sso\sthat\sJSON\noutput\sis\scompliant\sand\sso\sthat\svalues\scan\sbe\sround-tripped. -D 2023-03-22T16:24:17.609 +C For\sconsistency,\sthe\s".mode\sjson"\soutput\sfrom\sthe\sCLI\snow\srenders\sinfinity\nin\sthe\ssame\sformat\sas\sthe\sJSON\sfunctions. +D 2023-03-22T16:37:17.803 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa -F src/shell.c.in e66360191de42f8b43454886a059e3eb983e6dc0f2bd855dd4ff5244694f61a6 +F src/shell.c.in e92846a40dae4acd20ae6073846d8a54ba7fc84c8299af3b8d63be19491c5ad9 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2051,9 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c70a61d8fbfb722679398c211aa48ccd84a392a6b59ba70551fe3a9fbab9a6d5 efce4690a52592c4f5fc7d023eebe771b0e615bb03c0fe203493f853b28e8f85 -R 877d544aec0d825a0fa74ee8dc5a48fd -T +closed efce4690a52592c4f5fc7d023eebe771b0e615bb03c0fe203493f853b28e8f85 +P b52081d0acd07dc5bdb4951a3e8419866131965260c1e3a4c9b6e673bfe3dfea +R 34b5c89a78ee207c15c2a2a6b01eb9c8 U drh -Z e50ed26df279ddaf1f9de438f8e2a2a0 +Z 12d2d760264ba756a8188ada6c35650c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aced567e8d..b8db01212e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b52081d0acd07dc5bdb4951a3e8419866131965260c1e3a4c9b6e673bfe3dfea \ No newline at end of file +abee339d5eae8b63d9579b7ec3a25f18071e675223e0bb5294c733fb618a3a26 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index b97889f938..340dc13ce2 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -2523,9 +2523,9 @@ static int shell_callback( sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - raw_printf(p->out, "1e999"); + raw_printf(p->out, "9.0e+999"); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-1e999"); + raw_printf(p->out, "-9.0e+999"); }else{ sqlite3_int64 ir = (sqlite3_int64)r; if( r==(double)ir ){ @@ -2569,9 +2569,9 @@ static int shell_callback( sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - raw_printf(p->out, "1e999"); + raw_printf(p->out, "9.0e+999"); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-1e999"); + raw_printf(p->out, "-9.0e+999"); }else{ sqlite3_snprintf(50,z,"%!.20g", r); raw_printf(p->out, "%s", z); From ad99243846e76b23ba7644ec6ff77338e11f250a Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 22 Mar 2023 16:55:35 +0000 Subject: [PATCH 115/341] The floating-point-to-text conversion with the zero-padding option now renders NaN as "null". FossilOrigin-Name: ad59fa17663bda54ec5d4e48ac24e04b87daa70c795d840cd8db382e2dd581b9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 9 +++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ed02a96544..8d94a9b1da 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sconsistency,\sthe\s".mode\sjson"\soutput\sfrom\sthe\sCLI\snow\srenders\sinfinity\nin\sthe\ssame\sformat\sas\sthe\sJSON\sfunctions. -D 2023-03-22T16:37:17.803 +C The\sfloating-point-to-text\sconversion\swith\sthe\szero-padding\soption\snow\nrenders\sNaN\sas\s"null". +D 2023-03-22T16:55:35.088 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c c0c6e0735436e0e9e8d490f82959ddc9475d438e9f808a51953a4d43e30cea51 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 -F src/printf.c 8c16d3783e87d5899c23a2736106dca11645b50e2c8606766d2d1446de278a0c +F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b52081d0acd07dc5bdb4951a3e8419866131965260c1e3a4c9b6e673bfe3dfea -R 34b5c89a78ee207c15c2a2a6b01eb9c8 +P abee339d5eae8b63d9579b7ec3a25f18071e675223e0bb5294c733fb618a3a26 +R cb06c74e184cc2909674761792344766 U drh -Z 12d2d760264ba756a8188ada6c35650c +Z a8c6d294d393530f190c714127c209f2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b8db01212e..2b896cefb0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -abee339d5eae8b63d9579b7ec3a25f18071e675223e0bb5294c733fb618a3a26 \ No newline at end of file +ad59fa17663bda54ec5d4e48ac24e04b87daa70c795d840cd8db382e2dd581b9 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index d59855917e..65e539befd 100644 --- a/src/printf.c +++ b/src/printf.c @@ -586,8 +586,13 @@ void sqlite3_str_vappendf( realvalue += rounder; } if( sqlite3IsNaN((double)realvalue) ){ - bufpt = "NaN"; - length = 3; + if( flag_zeropad ){ + bufpt = "null"; + length = 4; + }else{ + bufpt = "NaN"; + length = 3; + } break; } From 2255d89f0ab6972a26b8e3dd0e013b58af22c65c Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 22 Mar 2023 19:57:19 +0000 Subject: [PATCH 116/341] Internal cleanups in JS code. No functional changes. FossilOrigin-Name: 8fbdf7d10400c4f54fc3d8bc823f97818de860deeeed35ab6ad717260cd301e0 --- ext/wasm/api/sqlite3-api-prologue.js | 41 ++++++++++++++-------------- manifest | 14 +++++----- manifest.uuid | 2 +- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 1fd5770992..61c8a4c62e 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1538,7 +1538,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( Returns capi.SQLITE_MISUSE if op is not a valid operation ID. */ - capi.sqlite3_db_config = function f(pDb, op, ...args){ + capi.sqlite3_db_config = function(pDb, op, ...args){ if(!this.s){ this.s = wasm.xWrap('sqlite3_wasm_db_config_s','int', ['sqlite3*', 'int', 'string:static'] @@ -1548,31 +1548,30 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( this.ip = wasm.xWrap('sqlite3_wasm_db_config_ip','int', ['sqlite3*', 'int', 'int','*']); } - const c = capi; switch(op){ - case c.SQLITE_DBCONFIG_ENABLE_FKEY: - case c.SQLITE_DBCONFIG_ENABLE_TRIGGER: - case c.SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: - case c.SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: - case c.SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: - case c.SQLITE_DBCONFIG_ENABLE_QPSG: - case c.SQLITE_DBCONFIG_TRIGGER_EQP: - case c.SQLITE_DBCONFIG_RESET_DATABASE: - case c.SQLITE_DBCONFIG_DEFENSIVE: - case c.SQLITE_DBCONFIG_WRITABLE_SCHEMA: - case c.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: - case c.SQLITE_DBCONFIG_DQS_DML: - case c.SQLITE_DBCONFIG_DQS_DDL: - case c.SQLITE_DBCONFIG_ENABLE_VIEW: - case c.SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: - case c.SQLITE_DBCONFIG_TRUSTED_SCHEMA: + case capi.SQLITE_DBCONFIG_ENABLE_FKEY: + case capi.SQLITE_DBCONFIG_ENABLE_TRIGGER: + case capi.SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER: + case capi.SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION: + case capi.SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE: + case capi.SQLITE_DBCONFIG_ENABLE_QPSG: + case capi.SQLITE_DBCONFIG_TRIGGER_EQP: + case capi.SQLITE_DBCONFIG_RESET_DATABASE: + case capi.SQLITE_DBCONFIG_DEFENSIVE: + case capi.SQLITE_DBCONFIG_WRITABLE_SCHEMA: + case capi.SQLITE_DBCONFIG_LEGACY_ALTER_TABLE: + case capi.SQLITE_DBCONFIG_DQS_DML: + case capi.SQLITE_DBCONFIG_DQS_DDL: + case capi.SQLITE_DBCONFIG_ENABLE_VIEW: + case capi.SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: + case capi.SQLITE_DBCONFIG_TRUSTED_SCHEMA: return this.ip(pDb, op, args[0], args[1] || 0); - case c.SQLITE_DBCONFIG_LOOKASIDE: + case capi.SQLITE_DBCONFIG_LOOKASIDE: return this.pii(pDb, op, args[0], args[1], args[2]); - case c.SQLITE_DBCONFIG_MAINDBNAME: + case capi.SQLITE_DBCONFIG_MAINDBNAME: return this.s(pDb, op, args[0]); default: - return c.SQLITE_MISUSE; + return capi.SQLITE_MISUSE; } }.bind(Object.create(null)); diff --git a/manifest b/manifest index 8d94a9b1da..d22b460b97 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sfloating-point-to-text\sconversion\swith\sthe\szero-padding\soption\snow\nrenders\sNaN\sas\s"null". -D 2023-03-22T16:55:35.088 +C Internal\scleanups\sin\sJS\scode.\sNo\sfunctional\schanges. +D 2023-03-22T19:57:19.911 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -484,7 +484,7 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af F ext/wasm/api/sqlite3-api-cleanup.js cc21e3486da748463e02bbe51e2464c6ac136587cdfd5aa00cd0b5385f6ca808 F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 2691a34a741015127b210954a1b9586764d3ff0c8a20f00fd15c00f339ecc79f -F ext/wasm/api/sqlite3-api-prologue.js df8646e4f92b8b09cef255da3530e11dc264a2e8d53b0e78daa2ee04f99c584d +F ext/wasm/api/sqlite3-api-prologue.js 461ffa5a95f4c1935b3970a58790d3cdca62b16e9b9a6a8d993a2a47d7561f51 F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d660194ddfc51eab1c5b9bf F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P abee339d5eae8b63d9579b7ec3a25f18071e675223e0bb5294c733fb618a3a26 -R cb06c74e184cc2909674761792344766 -U drh -Z a8c6d294d393530f190c714127c209f2 +P ad59fa17663bda54ec5d4e48ac24e04b87daa70c795d840cd8db382e2dd581b9 +R 3410d1e0b2bdb9b7c1b919caab60a1a5 +U stephan +Z 8aac156f6abc9f9b4761ce1a56e3e4e0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2b896cefb0..d34e83057f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad59fa17663bda54ec5d4e48ac24e04b87daa70c795d840cd8db382e2dd581b9 \ No newline at end of file +8fbdf7d10400c4f54fc3d8bc823f97818de860deeeed35ab6ad717260cd301e0 \ No newline at end of file From c03dbc5a249f8222098000d1bad2e6a0d602cb5b Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 23 Mar 2023 10:54:07 +0000 Subject: [PATCH 117/341] The attempt to bring STAT4 up to 100% MC/DC at [55a26c67ed4a3a93] and at [168fa2fb22b8c1ad] are incorrect. Back them out and replace them with a simple NEVER() macro. Error reported by [forum:/forumpost/dc4854437b|forum post dc4854437b]. FossilOrigin-Name: 5992370a89f8de7a6e941130b381f85d369856dbdb7860405e6fb17dad2293df --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbemem.c | 7 +------ test/analyzeE.test | 12 ++++++++++++ 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index d22b460b97..c21f9a0e05 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Internal\scleanups\sin\sJS\scode.\sNo\sfunctional\schanges. -D 2023-03-22T19:57:19.911 +C The\sattempt\sto\sbring\sSTAT4\sup\sto\s100%\sMC/DC\sat\s[55a26c67ed4a3a93]\sand\nat\s[168fa2fb22b8c1ad]\sare\sincorrect.\s\sBack\sthem\sout\sand\sreplace\sthem\swith\na\ssimple\sNEVER()\smacro.\s\sError\sreported\sby\n[forum:/forumpost/dc4854437b|forum\spost\sdc4854437b]. +D 2023-03-23T10:54:07.499 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -701,7 +701,7 @@ F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 F src/vdbeaux.c 0f5201346a83a35a08e833c3a03abe626119c07b7361c28bc6a259b98540f1d6 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd -F src/vdbemem.c 0388576b7cf0be13ce14b9e3b8aa90b8a1b923b60321d0242131ae0b5732b43b +F src/vdbemem.c 656a51f3e99d7123ef852413c0249101718f3f3138e66be37ba2f16dcf7ce43e F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -753,7 +753,7 @@ F test/analyze8.test 29ef237d8a59b39cc31c3310134fefe96a690b195e3deed5ecb65283908 F test/analyze9.test 30e1cb99336045a384a11d97900720184333c88174b3b89bc07444ea39e7df19 F test/analyzeC.test 1111830ad355d29a294a5dda654dd5f6a8622c6a223a4f7b7b3d091df7a7a42b F test/analyzeD.test 485f621cfd2ef0a8f8ac79672586651bfa495bd899db50461bb4b558400ab3c1 -F test/analyzeE.test 69d130f9ba78c9853dcd5a18317e81f462a72d704cec0c4c30afb220213acd29 +F test/analyzeE.test 3e35783c15180c475bf0e1d69089ea8cb17f4af44fbca0938ccd476af8ac6aa8 F test/analyzeF.test 40b5cc3ad7b10e81020d7ca86f1417647ecfae7477cfd88acc5aa7ae1068f949 F test/analyzeG.test 623be33038c49648872746c8dd8b23b5792c08fef173c55e82f1b12fca259852 F test/analyzer1.test 459fa02c445ddbf0101a3bad47b34290a35f2e49 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ad59fa17663bda54ec5d4e48ac24e04b87daa70c795d840cd8db382e2dd581b9 -R 3410d1e0b2bdb9b7c1b919caab60a1a5 -U stephan -Z 8aac156f6abc9f9b4761ce1a56e3e4e0 +P 8fbdf7d10400c4f54fc3d8bc823f97818de860deeeed35ab6ad717260cd301e0 +R b1afaa2171dcf7fcb417d71a6c9fd64f +U drh +Z cd2206df50c0abaeae5a80127570c62a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d34e83057f..c65144d6e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8fbdf7d10400c4f54fc3d8bc823f97818de860deeeed35ab6ad717260cd301e0 \ No newline at end of file +5992370a89f8de7a6e941130b381f85d369856dbdb7860405e6fb17dad2293df \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index be52062d55..02588707de 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1519,16 +1519,11 @@ static int valueFromFunction( }else{ sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); assert( rc==SQLITE_OK ); - assert( enc==pVal->enc - || (pVal->flags & MEM_Str)==0 - || db->mallocFailed ); -#if 0 /* Not reachable except after a prior failure */ rc = sqlite3VdbeChangeEncoding(pVal, enc); - if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){ + if( NEVER(rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal)) ){ rc = SQLITE_TOOBIG; pCtx->pParse->nErr++; } -#endif } value_from_function_out: diff --git a/test/analyzeE.test b/test/analyzeE.test index 733b79367f..9f65b7a320 100644 --- a/test/analyzeE.test +++ b/test/analyzeE.test @@ -239,4 +239,16 @@ do_execsql_test analyzeE-4.11 { SELECT * FROM t1 WHERE a<1900 AND c=123 } {/SCAN t1/} +# 2023-03-23 https://sqlite.org/forum/forumpost/dc4854437b +# +reset_db +do_execsql_test analyzeE-5.0 { + PRAGMA encoding = 'UTF-16'; + CREATE TABLE t0 (c1 TEXT); + INSERT INTO t0 VALUES (''); + CREATE INDEX i0 ON t0(c1); + ANALYZE; + SELECT * FROM t0 WHERE t0.c1 BETWEEN '' AND (ABS('')); +} {{}} + finish_test From 28fe02cd5124730c719dfc0f8210a392dbfc7367 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Mar 2023 16:57:21 +0000 Subject: [PATCH 118/341] Fix byte-code register allocation in ANALYZE for STAT4 when there multiple indexes with differing numbers of columns. [forum:/forumpost/bc39e531e5|forum post bc39e531e5]. FossilOrigin-Name: 2bf5413dc2c19d5feb32e5b01aa9b990ec2f74f45f5ca0dca15215f8c9dbc9b9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 25 +++++++++++++++++++++++-- test/analyzeE.test | 26 ++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c21f9a0e05..5fd3858c5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sattempt\sto\sbring\sSTAT4\sup\sto\s100%\sMC/DC\sat\s[55a26c67ed4a3a93]\sand\nat\s[168fa2fb22b8c1ad]\sare\sincorrect.\s\sBack\sthem\sout\sand\sreplace\sthem\swith\na\ssimple\sNEVER()\smacro.\s\sError\sreported\sby\n[forum:/forumpost/dc4854437b|forum\spost\sdc4854437b]. -D 2023-03-23T10:54:07.499 +C Fix\sbyte-code\sregister\sallocation\sin\sANALYZE\sfor\sSTAT4\swhen\sthere\smultiple\nindexes\swith\sdiffering\snumbers\sof\scolumns.\n[forum:/forumpost/bc39e531e5|forum\spost\sbc39e531e5]. +D 2023-03-24T16:57:21.318 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -558,7 +558,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 3ca2f449c890f8b86ec9e06f0c4fccf0648941c3308a16904cb2852227db83f7 -F src/analyze.c b597c382f23b19cce563211181e84b7e8edddd6871d5f630bbadedb57e562806 +F src/analyze.c 26e68072d62d0a28b2dd8619f76be876db9b4ff6e8ad1fd4ed3ac56b1cb045c4 F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 @@ -753,7 +753,7 @@ F test/analyze8.test 29ef237d8a59b39cc31c3310134fefe96a690b195e3deed5ecb65283908 F test/analyze9.test 30e1cb99336045a384a11d97900720184333c88174b3b89bc07444ea39e7df19 F test/analyzeC.test 1111830ad355d29a294a5dda654dd5f6a8622c6a223a4f7b7b3d091df7a7a42b F test/analyzeD.test 485f621cfd2ef0a8f8ac79672586651bfa495bd899db50461bb4b558400ab3c1 -F test/analyzeE.test 3e35783c15180c475bf0e1d69089ea8cb17f4af44fbca0938ccd476af8ac6aa8 +F test/analyzeE.test 8770de3834bb9aea1c892f4d155a5bd9578c23c0c3f199dc6f1b5d597b5bf65c F test/analyzeF.test 40b5cc3ad7b10e81020d7ca86f1417647ecfae7477cfd88acc5aa7ae1068f949 F test/analyzeG.test 623be33038c49648872746c8dd8b23b5792c08fef173c55e82f1b12fca259852 F test/analyzer1.test 459fa02c445ddbf0101a3bad47b34290a35f2e49 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8fbdf7d10400c4f54fc3d8bc823f97818de860deeeed35ab6ad717260cd301e0 -R b1afaa2171dcf7fcb417d71a6c9fd64f +P 5992370a89f8de7a6e941130b381f85d369856dbdb7860405e6fb17dad2293df +R 77e3e44c3db2ed26a66a9624f00b38b1 U drh -Z cd2206df50c0abaeae5a80127570c62a +Z cb6078c6d4f127cf1058bdefbeb157d4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c65144d6e1..b1168025de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5992370a89f8de7a6e941130b381f85d369856dbdb7860405e6fb17dad2293df \ No newline at end of file +2bf5413dc2c19d5feb32e5b01aa9b990ec2f74f45f5ca0dca15215f8c9dbc9b9 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index f3356ea3ca..2eaabd6be4 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -994,8 +994,11 @@ static void analyzeOneTable( int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ int regPrev = iMem; /* MUST BE LAST (see below) */ +#ifdef SQLITE_ENABLE_STAT4 + int doOnce = 1; /* Flag for a one-time computation */ +#endif #ifdef SQLITE_ENABLE_PREUPDATE_HOOK - Table *pStat1 = 0; + Table *pStat1 = 0; #endif pParse->nMem = MAX(pParse->nMem, iMem); @@ -1276,7 +1279,25 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - pParse->nMem = MAX(pParse->nMem, regCol+nCol); + if( doOnce ){ + int mxCol = nCol; + Index *pX; + + /* Compute the maximum number of columns in any index */ + for(pX=pTab->pIndex; pX; pX=pX->pNext){ + int nColX; /* Number of columns in pX */ + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pX) ){ + nColX = pX->nKeyCol; + }else{ + nColX = pX->nColumn; + } + if( nColX>mxCol ) mxCol = nColX; + } + + /* Allocate space to compute results for the largest index */ + pParse->nMem = MAX(pParse->nMem, regCol+mxCol); + doOnce = 0; + } addrNext = sqlite3VdbeCurrentAddr(v); callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); diff --git a/test/analyzeE.test b/test/analyzeE.test index 9f65b7a320..9b5a27b87a 100644 --- a/test/analyzeE.test +++ b/test/analyzeE.test @@ -251,4 +251,30 @@ do_execsql_test analyzeE-5.0 { SELECT * FROM t0 WHERE t0.c1 BETWEEN '' AND (ABS('')); } {{}} +# 2023-03-24 https://sqlite.org/forum/forumpost/bc39e531e5 +# +reset_db +do_execsql_test analyzeE-6.0 { + CREATE TABLE t1(x); + CREATE INDEX i1 ON t1(x,x,x,x,x||2); + CREATE INDEX i2 ON t1(1<2); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t1(x) SELECT x FROM c; + ANALYZE; +} {} +do_execsql_test analyzeE-6.1 { + SELECT count(*)>1 FROM sqlite_stat4 WHERE idx='i2' AND neq='1000 1'; +} 1 +do_execsql_test analyzeE-6.2 { + SELECT count(*) FROM sqlite_stat4 WHERE idx='i2' AND neq<>'1000 1'; +} 0 +do_execsql_test analyzeE-6.3 { + SELECT count(*)>1 FROM sqlite_stat4 WHERE idx='i1' AND neq='1 1 1 1 1 1'; +} 1 +do_execsql_test analyzeE-6.4 { + SELECT count(*) FROM sqlite_stat4 WHERE idx='i1' AND neq<>'1 1 1 1 1 1'; +} 0 + + + finish_test From 4a1c8ed725ba0beb1da5942dca6b332b25d79e67 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 24 Mar 2023 17:15:15 +0000 Subject: [PATCH 119/341] Add a JS test which checks for the issue described in [forum post 895425b49a|forum:895425b49a]. FossilOrigin-Name: 98d30400e4721b1d48ff601698ced146052654f8c6de7c014d4d239bb2dbef43 --- ext/wasm/tester1.c-pp.js | 12 ++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 42d1167b60..539c239849 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1463,6 +1463,18 @@ self.sqlite3InitModule = sqlite3InitModule; T.assert(e instanceof sqlite3.SQLite3Error) .assert(0==e.message.indexOf('Cannot prepare empty')); } + + counter = 0; + db.exec({ + // Check for https://sqlite.org/forum/forumpost/895425b49a + sql: "pragma table_info('t')", + rowMode: 'object', + callback: function(row){ + ++counter; + T.assert(row.name==='a' || row.name==='b'); + } + }); + T.assert(2===counter); })/*setup table T*/ //////////////////////////////////////////////////////////////////// diff --git a/manifest b/manifest index 5fd3858c5a..d436877932 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sbyte-code\sregister\sallocation\sin\sANALYZE\sfor\sSTAT4\swhen\sthere\smultiple\nindexes\swith\sdiffering\snumbers\sof\scolumns.\n[forum:/forumpost/bc39e531e5|forum\spost\sbc39e531e5]. -D 2023-03-24T16:57:21.318 +C Add\sa\sJS\stest\swhich\schecks\sfor\sthe\sissue\sdescribed\sin\s[forum\spost\s895425b49a|forum:895425b49a]. +D 2023-03-24T17:15:15.541 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -535,7 +535,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html 258d08f1ba9cc2d455958751e26be833893cf9ff7853e9436e593e1f778a386b F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 9844c675bd1f2353deabd7847d10c4fa55ff78a5c773073a239197d186123de7 +F ext/wasm/tester1.c-pp.js 942e31e5acacdea8f7e5c134c9c23433d56ca95e9f033708556aead1e11a6e6c F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5992370a89f8de7a6e941130b381f85d369856dbdb7860405e6fb17dad2293df -R 77e3e44c3db2ed26a66a9624f00b38b1 -U drh -Z cb6078c6d4f127cf1058bdefbeb157d4 +P 2bf5413dc2c19d5feb32e5b01aa9b990ec2f74f45f5ca0dca15215f8c9dbc9b9 +R 9fe10dfde164fd7e9d2d141aa7003a01 +U stephan +Z 3260419241ad58efe4624da079c64fb7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b1168025de..a0cde4908e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2bf5413dc2c19d5feb32e5b01aa9b990ec2f74f45f5ca0dca15215f8c9dbc9b9 \ No newline at end of file +98d30400e4721b1d48ff601698ced146052654f8c6de7c014d4d239bb2dbef43 \ No newline at end of file From d23924d1ea4c9ffded38a73eb0f1b5cae52e22dc Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Mar 2023 19:17:25 +0000 Subject: [PATCH 120/341] Fix a problem in cursor-hints for WITHOUT ROWID tables used in a RIGHT JOIN. [forum:/forumpost/591006b1cc|Forum post 591006b1cc]. FossilOrigin-Name: 221fdcec964f8317b2c23e926cc23799615cd3b4239a8a9ff87a83588d05bc64 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 2 +- test/cursorhint.test | 9 +++++++++ 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d436877932..5bfae257eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sJS\stest\swhich\schecks\sfor\sthe\sissue\sdescribed\sin\s[forum\spost\s895425b49a|forum:895425b49a]. -D 2023-03-24T17:15:15.541 +C Fix\sa\sproblem\sin\scursor-hints\sfor\sWITHOUT\sROWID\stables\sused\sin\sa\sRIGHT\sJOIN.\n[forum:/forumpost/591006b1cc|Forum\spost\s591006b1cc]. +D 2023-03-24T19:17:25.884 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -710,7 +710,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c db043f9b2da776105e04b0796f42034745a8bb59433d5b7dae798a4417987e52 +F src/where.c 2bf0f0e28a7093b8f3fa661c5474993413bfac5cc55939288856952e73d9d140 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 9919e5a22f4b24dd96c49b8981484cbe6bbfcf466ff73ac40a06e1356aa8bf87 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -911,7 +911,7 @@ F test/createtab.test 85cdfdae5c3de331cd888d6c66e1aba575b47c2e3c3cc4a1d6f5414069 F test/cse.test 00b3aea44b16828833c94fbe92475fd6977583fcb064ae0bc590986812b38d0c F test/csv01.test 2ab5514005fd308995c8910bc313e47f0368b94213b9d6c27f9a2da78796a091 F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c4773 -F test/cursorhint.test 0175e4404181ace3ceca8b114eb0a98eae600d565aa4e2705abbe6614c7fe201 +F test/cursorhint.test 096c8a210b31934a1964d45d7fb2aedf1e09e91773b0942abd039285b9953906 F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 118e04db8c8b4efeb885542b4918c7b869a34c460a6bebbfe927dfd75706b80d @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2bf5413dc2c19d5feb32e5b01aa9b990ec2f74f45f5ca0dca15215f8c9dbc9b9 -R 9fe10dfde164fd7e9d2d141aa7003a01 -U stephan -Z 3260419241ad58efe4624da079c64fb7 +P 98d30400e4721b1d48ff601698ced146052654f8c6de7c014d4d239bb2dbef43 +R 6146bc2919cc9016e0146c0a3f2b25ac +U drh +Z 04567d784fa8abfc6c05842f7b980df9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a0cde4908e..bed544cb06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98d30400e4721b1d48ff601698ced146052654f8c6de7c014d4d239bb2dbef43 \ No newline at end of file +221fdcec964f8317b2c23e926cc23799615cd3b4239a8a9ff87a83588d05bc64 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9de72d76de..dd2b8b73c4 100644 --- a/src/where.c +++ b/src/where.c @@ -6238,7 +6238,7 @@ WhereInfo *sqlite3WhereBegin( assert( n<=pTab->nCol ); } #ifdef SQLITE_ENABLE_CURSOR_HINTS - if( pLoop->u.btree.pIndex!=0 ){ + if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); }else #endif diff --git a/test/cursorhint.test b/test/cursorhint.test index a3397b8673..16e26fecd4 100644 --- a/test/cursorhint.test +++ b/test/cursorhint.test @@ -159,4 +159,13 @@ do_test 4.6desc { } } {AND(AND(EQ(c0,22),GE(c1,10)),LE(c1,20))} +# 2023-03-24 https://sqlite.org/forum/forumpost/591006b1cc +# +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(x TEXT PRIMARY KEY) WITHOUT ROWID; + CREATE VIEW t2 AS SELECT 0 FROM t1 WHERE x>='a' OR x='1'; + SELECT * FROM t2 RIGHT JOIN t1 ON true; +} + finish_test From b3ad5444d0a123ea0af7da9e158d2d0f49c9cd00 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Mar 2023 20:35:56 +0000 Subject: [PATCH 121/341] Fix the handling of indexed expressions in an outer query that appear as corelated values inside an aggregate function within a subquery. [forum:/forumpost/79cf371080|Forum post 79cf371080]. FossilOrigin-Name: d8259877eaa962e6f90675790d3770ef02bd1a5d86d319cd8c1834710df844c4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 3 ++- test/indexexpr2.test | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5bfae257eb..810a58357a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\scursor-hints\sfor\sWITHOUT\sROWID\stables\sused\sin\sa\sRIGHT\sJOIN.\n[forum:/forumpost/591006b1cc|Forum\spost\s591006b1cc]. -D 2023-03-24T19:17:25.884 +C Fix\sthe\shandling\sof\sindexed\sexpressions\sin\san\souter\squery\sthat\sappear\sas\ncorelated\svalues\sinside\san\saggregate\sfunction\swithin\sa\ssubquery.\n[forum:/forumpost/79cf371080|Forum\spost\s79cf371080]. +D 2023-03-24T20:35:56.400 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -625,7 +625,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 64a4b48a14f5d19a5ddc4afde37a435aebb92113f52653f8fe9e69fc3e9ff9aa +F src/select.c 83812448cf5a645f37d79c43fa92d38d7a9f7790d95f7ac09ac02b4064810bd2 F src/shell.c.in e92846a40dae4acd20ae6073846d8a54ba7fc84c8299af3b8d63be19491c5ad9 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1195,7 +1195,7 @@ F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a91 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 F test/indexexpr1.test b2a15637dcbae7fd8d7e2fc51f74ac4feaf5510130ee2089a5ec5bd1ef7270e1 -F test/indexexpr2.test 2c7abe3c48f8aaa5a448615ab4d13df3662185d28419c00999670834a3f0b484 +F test/indexexpr2.test 8a4cf83e12b041334c5ef36b9a3932451808378352c24d65049b31f3737a82b9 F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 4e3f0de67aac3c5be1f4aaedbcea11638f1b5cdc9a3115be14d19aa9db7623c6 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 98d30400e4721b1d48ff601698ced146052654f8c6de7c014d4d239bb2dbef43 -R 6146bc2919cc9016e0146c0a3f2b25ac +P 221fdcec964f8317b2c23e926cc23799615cd3b4239a8a9ff87a83588d05bc64 +R 5ccb0dc2f189cd11515b977033218bf1 U drh -Z 04567d784fa8abfc6c05842f7b980df9 +Z b958c4a96a3327f8806b22897ddc9447 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bed544cb06..7da91c4c53 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -221fdcec964f8317b2c23e926cc23799615cd3b4239a8a9ff87a83588d05bc64 \ No newline at end of file +d8259877eaa962e6f90675790d3770ef02bd1a5d86d319cd8c1834710df844c4 \ No newline at end of file diff --git a/src/select.c b/src/select.c index ddaa8c06b5..a22aed8524 100644 --- a/src/select.c +++ b/src/select.c @@ -6485,7 +6485,8 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; pAggInfo = pExpr->pAggInfo; - assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + if( pExpr->iAgg>=pAggInfo->nColumn ) return WRC_Continue; + assert( pExpr->iAgg>=0 ); pCol = &pAggInfo->aCol[pExpr->iAgg]; pExpr->op = TK_AGG_COLUMN; pExpr->iTable = pCol->iTable; diff --git a/test/indexexpr2.test b/test/indexexpr2.test index 8c1171e038..3cc030f86b 100644 --- a/test/indexexpr2.test +++ b/test/indexexpr2.test @@ -372,4 +372,18 @@ foreach {tn expr} { " {1 1} } +# 2023-03-24 https://sqlite.org/forum/forumpost/79cf371080 +# +reset_db +do_execsql_test 9.0 { + CREATE TABLE t1(a INT, b INT); + CREATE INDEX t1x ON t1(a, abs(b)); + CREATE TABLE t2(c INT, d INT); + INSERT INTO t1(a,b) VALUES(4,4),(5,-5),(5,20),(6,6); + INSERT INTO t2(c,d) VALUES(100,1),(200,1),(300,2); + SELECT *, + (SELECT max(c+abs(b)) FROM t2 GROUP BY d ORDER BY d LIMIT 1) AS subq + FROM t1 WHERE a=5; +} {5 -5 205 5 20 220} + finish_test From ed8e12e97bddf094e3435a5c7ea80d1119dc6ca7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Mar 2023 21:24:52 +0000 Subject: [PATCH 122/341] Fix an error in the OP_SeekScan opcode added by check-in [4a43430fd23f8835]. Problem reported by [forum:/forumpost/8cc1dc0fe9|forum post 8cc1dc0fe9]. FossilOrigin-Name: 651a13fcd16f03e89eb6228c9f3250e25910b9bbe2637f627f65ff78f8ba2059 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 + test/skipscan1.test | 10 ++++++++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 810a58357a..af86acbc77 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\shandling\sof\sindexed\sexpressions\sin\san\souter\squery\sthat\sappear\sas\ncorelated\svalues\sinside\san\saggregate\sfunction\swithin\sa\ssubquery.\n[forum:/forumpost/79cf371080|Forum\spost\s79cf371080]. -D 2023-03-24T20:35:56.400 +C Fix\san\serror\sin\sthe\sOP_SeekScan\sopcode\sadded\sby\scheck-in\s[4a43430fd23f8835].\nProblem\sreported\sby\s[forum:/forumpost/8cc1dc0fe9|forum\spost\s8cc1dc0fe9]. +D 2023-03-24T21:24:52.602 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -695,7 +695,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 0cf4c72a9e0eb614afc19c9c4ca9c8a919c97c0866934a70dac7c2f689a4edf8 +F src/vdbe.c 2bb4694bff3c1c4f52fd47e66be30278b5251d62d23b768af7c3b3a0dd7ba901 F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 @@ -1496,7 +1496,7 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 2668e607dcdfa19c52828c09b69685b38da793856582ae31debf79d90c7bbbdc F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test 1a9972e1dc15ca3887f306d3cd9a29679afb382eca0f3539f3b746f3c2ccaf68 +F test/skipscan1.test e03ba5b977da6fd71662a4b0a668f04053bda4b187ff3214db7533e28c732279 F test/skipscan2.test b032ed3e0ba5caa4df6c43ef22c31566aac67783bc031869155989a7ccdb5bd5 F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 0672103fd2c8f96bd114133f356192b35ece45c794fe3677e1d9e5e3104a608e @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 221fdcec964f8317b2c23e926cc23799615cd3b4239a8a9ff87a83588d05bc64 -R 5ccb0dc2f189cd11515b977033218bf1 +P d8259877eaa962e6f90675790d3770ef02bd1a5d86d319cd8c1834710df844c4 +R ceac5b217457e4b5c354099ee0f2a202 U drh -Z b958c4a96a3327f8806b22897ddc9447 +Z c8ed2eeb19fba230a018ea1c48ee7af0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7da91c4c53..d830bec02d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8259877eaa962e6f90675790d3770ef02bd1a5d86d319cd8c1834710df844c4 \ No newline at end of file +651a13fcd16f03e89eb6228c9f3250e25910b9bbe2637f627f65ff78f8ba2059 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index dfe8070de1..4c66d555f3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4973,6 +4973,7 @@ case OP_SeekScan: { /* ncycle */ break; } nStep--; + pC->cacheStatus = CACHE_STALE; rc = sqlite3BtreeNext(pC->uc.pCursor, 0); if( rc ){ if( rc==SQLITE_DONE ){ diff --git a/test/skipscan1.test b/test/skipscan1.test index 94062fb17e..bd5b83d34c 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -419,4 +419,14 @@ do_execsql_test skipscan1-4.10 { AND a <= 10; } {3} +# 2023-03-24 https://sqlite.org/forum/forumpost/8cc1dc0fe9 +# +reset_db +do_execsql_test skipscan1-5.0 { + CREATE TABLE t1(a TEXT, UNIQUE(a,a,a)); + INSERT INTO t1 VALUES (hex(zeroblob(241))),(1),(2),(3); + ANALYZE; + SELECT max(a) FROM t1 WHERE a IN t1; +} {3} + finish_test From 338ff1ceca6890ff068110b125ce1ed837b07660 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Mar 2023 21:35:48 +0000 Subject: [PATCH 123/341] Fix possible integer overflow in bounds checking for the debugging function "shell_int32()" found in the CLI. This change does not affect the core SQLite. [forum:/forumpost/be9c294ee0|Forum post be9c294ee0]. FossilOrigin-Name: 6211471138a654641a4cf4831cfa3b470e06f29a2b77e4d58177c8e065bec11e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index af86acbc77..05049b6247 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sthe\sOP_SeekScan\sopcode\sadded\sby\scheck-in\s[4a43430fd23f8835].\nProblem\sreported\sby\s[forum:/forumpost/8cc1dc0fe9|forum\spost\s8cc1dc0fe9]. -D 2023-03-24T21:24:52.602 +C Fix\spossible\sinteger\soverflow\sin\sbounds\schecking\sfor\sthe\sdebugging\sfunction\n"shell_int32()"\sfound\sin\sthe\sCLI.\s\sThis\schange\sdoes\snot\saffect\sthe\score\nSQLite.\s\s[forum:/forumpost/be9c294ee0|Forum\spost\sbe9c294ee0]. +D 2023-03-24T21:35:48.482 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 83812448cf5a645f37d79c43fa92d38d7a9f7790d95f7ac09ac02b4064810bd2 -F src/shell.c.in e92846a40dae4acd20ae6073846d8a54ba7fc84c8299af3b8d63be19491c5ad9 +F src/shell.c.in 9e99faf8b27757d3a774e7febeae31cdd873249097127c0297089a6352316570 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d8259877eaa962e6f90675790d3770ef02bd1a5d86d319cd8c1834710df844c4 -R ceac5b217457e4b5c354099ee0f2a202 +P 651a13fcd16f03e89eb6228c9f3250e25910b9bbe2637f627f65ff78f8ba2059 +R bcd7de74660ca1aa84d989e9651b4c63 U drh -Z c8ed2eeb19fba230a018ea1c48ee7af0 +Z 1b96d62587bba5c2429601d51964cf5c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d830bec02d..f9da35fff1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -651a13fcd16f03e89eb6228c9f3250e25910b9bbe2637f627f65ff78f8ba2059 \ No newline at end of file +6211471138a654641a4cf4831cfa3b470e06f29a2b77e4d58177c8e065bec11e \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 340dc13ce2..75de1bc2f2 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5130,7 +5130,7 @@ static void shellInt32( pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); iInt = sqlite3_value_int(argv[1]); - if( iInt>=0 && (iInt+1)*4<=nBlob ){ + if( iInt>=0 && iInt<(nBlob/4) ){ const unsigned char *a = &pBlob[iInt*4]; sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) + ((sqlite3_int64)a[1]<<16) From 62e43b25ddfa12f970a6f818d4942aeac8838f65 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 24 Mar 2023 22:17:59 +0000 Subject: [PATCH 124/341] Remove undocumented, vestigial SQL functions in the CLI that were once used for the ".recover" command but are now no longer needed. FossilOrigin-Name: 1ef461aa4e95d254c2c1edebbbfd92ca96d752e04f68ebe70104e8d936d36be9 --- manifest | 12 ++--- manifest.uuid | 2 +- src/shell.c.in | 144 ------------------------------------------------- 3 files changed, 7 insertions(+), 151 deletions(-) diff --git a/manifest b/manifest index 05049b6247..e8e79c11df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\spossible\sinteger\soverflow\sin\sbounds\schecking\sfor\sthe\sdebugging\sfunction\n"shell_int32()"\sfound\sin\sthe\sCLI.\s\sThis\schange\sdoes\snot\saffect\sthe\score\nSQLite.\s\s[forum:/forumpost/be9c294ee0|Forum\spost\sbe9c294ee0]. -D 2023-03-24T21:35:48.482 +C Remove\sundocumented,\svestigial\sSQL\sfunctions\sin\sthe\sCLI\sthat\swere\sonce\sused\nfor\sthe\s".recover"\scommand\sbut\sare\snow\sno\slonger\sneeded. +D 2023-03-24T22:17:59.908 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 83812448cf5a645f37d79c43fa92d38d7a9f7790d95f7ac09ac02b4064810bd2 -F src/shell.c.in 9e99faf8b27757d3a774e7febeae31cdd873249097127c0297089a6352316570 +F src/shell.c.in 3ae753373f8031b4a7d4258293d458cfbe807c69d94c513189b7b2884dc4909e F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 651a13fcd16f03e89eb6228c9f3250e25910b9bbe2637f627f65ff78f8ba2059 -R bcd7de74660ca1aa84d989e9651b4c63 +P 6211471138a654641a4cf4831cfa3b470e06f29a2b77e4d58177c8e065bec11e +R d74b6f5e7ca51ba8ce014f3390bf47fc U drh -Z 1b96d62587bba5c2429601d51964cf5c +Z 0874726345819af3ee3fcb8bd20f97a4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f9da35fff1..87becb1292 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6211471138a654641a4cf4831cfa3b470e06f29a2b77e4d58177c8e065bec11e \ No newline at end of file +1ef461aa4e95d254c2c1edebbbfd92ca96d752e04f68ebe70104e8d936d36be9 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 75de1bc2f2..c61a7432f0 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5110,53 +5110,6 @@ readHexDb_error: } #endif /* SQLITE_OMIT_DESERIALIZE */ -/* -** Scalar function "shell_int32". The first argument to this function -** must be a blob. The second a non-negative integer. This function -** reads and returns a 32-bit big-endian integer from byte -** offset (4*) of the blob. -*/ -static void shellInt32( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *pBlob; - int nBlob; - int iInt; - - UNUSED_PARAMETER(argc); - nBlob = sqlite3_value_bytes(argv[0]); - pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); - iInt = sqlite3_value_int(argv[1]); - - if( iInt>=0 && iInt<(nBlob/4) ){ - const unsigned char *a = &pBlob[iInt*4]; - sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) - + ((sqlite3_int64)a[1]<<16) - + ((sqlite3_int64)a[2]<< 8) - + ((sqlite3_int64)a[3]<< 0); - sqlite3_result_int64(context, iVal); - } -} - -/* -** Scalar function "shell_idquote(X)" returns string X quoted as an identifier, -** using "..." with internal double-quote characters doubled. -*/ -static void shellIdQuote( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zName = (const char*)sqlite3_value_text(argv[0]); - UNUSED_PARAMETER(argc); - if( zName ){ - char *z = sqlite3_mprintf("\"%w\"", zName); - sqlite3_result_text(context, z, -1, sqlite3_free); - } -} - /* ** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X. */ @@ -5171,97 +5124,6 @@ static void shellUSleepFunc( sqlite3_result_int(context, sleep); } -/* -** Scalar function "shell_escape_crnl" used by the .recover command. -** The argument passed to this function is the output of built-in -** function quote(). If the first character of the input is "'", -** indicating that the value passed to quote() was a text value, -** then this function searches the input for "\n" and "\r" characters -** and adds a wrapper similar to the following: -** -** replace(replace(, '\n', char(10), '\r', char(13)); -** -** Or, if the first character of the input is not "'", then a copy -** of the input is returned. -*/ -static void shellEscapeCrnl( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zText = (const char*)sqlite3_value_text(argv[0]); - UNUSED_PARAMETER(argc); - if( zText && zText[0]=='\'' ){ - i64 nText = sqlite3_value_bytes(argv[0]); - i64 i; - char zBuf1[20]; - char zBuf2[20]; - const char *zNL = 0; - const char *zCR = 0; - i64 nCR = 0; - i64 nNL = 0; - - for(i=0; zText[i]; i++){ - if( zNL==0 && zText[i]=='\n' ){ - zNL = unused_string(zText, "\\n", "\\012", zBuf1); - nNL = strlen(zNL); - } - if( zCR==0 && zText[i]=='\r' ){ - zCR = unused_string(zText, "\\r", "\\015", zBuf2); - nCR = strlen(zCR); - } - } - - if( zNL || zCR ){ - i64 iOut = 0; - i64 nMax = (nNL > nCR) ? nNL : nCR; - i64 nAlloc = nMax * nText + (nMax+64)*2; - char *zOut = (char*)sqlite3_malloc64(nAlloc); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - return; - } - - if( zNL && zCR ){ - memcpy(&zOut[iOut], "replace(replace(", 16); - iOut += 16; - }else{ - memcpy(&zOut[iOut], "replace(", 8); - iOut += 8; - } - for(i=0; zText[i]; i++){ - if( zText[i]=='\n' ){ - memcpy(&zOut[iOut], zNL, nNL); - iOut += nNL; - }else if( zText[i]=='\r' ){ - memcpy(&zOut[iOut], zCR, nCR); - iOut += nCR; - }else{ - zOut[iOut] = zText[i]; - iOut++; - } - } - - if( zNL ){ - memcpy(&zOut[iOut], ",'", 2); iOut += 2; - memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; - memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; - } - if( zCR ){ - memcpy(&zOut[iOut], ",'", 2); iOut += 2; - memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; - memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; - } - - sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); - sqlite3_free(zOut); - return; - } - } - - sqlite3_result_value(context, argv[0]); -} - /* Flags for open_db(). ** ** The default behavior of open_db() is to exit(1) if the database fails to @@ -5384,12 +5246,6 @@ static void open_db(ShellState *p, int openFlags){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); - sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, - shellEscapeCrnl, 0, 0); - sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, - shellInt32, 0, 0); - sqlite3_create_function(p->db, "shell_idquote", 1, SQLITE_UTF8, 0, - shellIdQuote, 0, 0); sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0, shellUSleepFunc, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM From 86b67f09e7587249d53fdce35c70ce459bb68ad7 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 25 Mar 2023 01:29:40 +0000 Subject: [PATCH 125/341] Fix CLI non-handling of OOM reported at [forum:/forumpost/6872514e04|Forum post 6872514e04]. FossilOrigin-Name: 6f6a0fd63b13cb827d6a402de01a701eb5b3f92954032ea80e78ec864861a26c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e8e79c11df..8422f9d3e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sundocumented,\svestigial\sSQL\sfunctions\sin\sthe\sCLI\sthat\swere\sonce\sused\nfor\sthe\s".recover"\scommand\sbut\sare\snow\sno\slonger\sneeded. -D 2023-03-24T22:17:59.908 +C Fix\sCLI\snon-handling\sof\sOOM\sreported\sat\s[forum:/forumpost/6872514e04|Forum\spost\s6872514e04]. +D 2023-03-25T01:29:40.329 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 83812448cf5a645f37d79c43fa92d38d7a9f7790d95f7ac09ac02b4064810bd2 -F src/shell.c.in 3ae753373f8031b4a7d4258293d458cfbe807c69d94c513189b7b2884dc4909e +F src/shell.c.in d80b53a9ec0a43ac0157fc54d667cb49fb6dd2229558135530feab2fc04b909d F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6211471138a654641a4cf4831cfa3b470e06f29a2b77e4d58177c8e065bec11e -R d74b6f5e7ca51ba8ce014f3390bf47fc -U drh -Z 0874726345819af3ee3fcb8bd20f97a4 +P 1ef461aa4e95d254c2c1edebbbfd92ca96d752e04f68ebe70104e8d936d36be9 +R fbdb1a010f49bcb705fde9b123ee05a5 +U larrybr +Z 701892f33d4b936af4ab128b00443fae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 87becb1292..bc8d2726a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ef461aa4e95d254c2c1edebbbfd92ca96d752e04f68ebe70104e8d936d36be9 \ No newline at end of file +6f6a0fd63b13cb827d6a402de01a701eb5b3f92954032ea80e78ec864861a26c \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c61a7432f0..594efc4566 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -2089,6 +2089,7 @@ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ int i; for(i=0; i Date: Sat, 25 Mar 2023 02:07:20 +0000 Subject: [PATCH 126/341] New test case to further validate the fix at [221fdcec964f8317]. [forum:/forumpost/d34ad68c36|Forum post d34ad68c36]. FossilOrigin-Name: a6e218a6e1ddd74be6a313b1d336334071747efd4ecd354ed7efe303d09c849c --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/cursorhint.test | 12 ++++++++++++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8422f9d3e1..76fefd2ffb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sCLI\snon-handling\sof\sOOM\sreported\sat\s[forum:/forumpost/6872514e04|Forum\spost\s6872514e04]. -D 2023-03-25T01:29:40.329 +C New\stest\scase\sto\sfurther\svalidate\sthe\sfix\sat\s[221fdcec964f8317].\n[forum:/forumpost/d34ad68c36|Forum\spost\sd34ad68c36]. +D 2023-03-25T02:07:20.189 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -911,7 +911,7 @@ F test/createtab.test 85cdfdae5c3de331cd888d6c66e1aba575b47c2e3c3cc4a1d6f5414069 F test/cse.test 00b3aea44b16828833c94fbe92475fd6977583fcb064ae0bc590986812b38d0c F test/csv01.test 2ab5514005fd308995c8910bc313e47f0368b94213b9d6c27f9a2da78796a091 F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c4773 -F test/cursorhint.test 096c8a210b31934a1964d45d7fb2aedf1e09e91773b0942abd039285b9953906 +F test/cursorhint.test 12e7d03262be8705798fe7b5719956bcda14989f10296a5e8fdc13466b7b75fc F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 118e04db8c8b4efeb885542b4918c7b869a34c460a6bebbfe927dfd75706b80d @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1ef461aa4e95d254c2c1edebbbfd92ca96d752e04f68ebe70104e8d936d36be9 -R fbdb1a010f49bcb705fde9b123ee05a5 -U larrybr -Z 701892f33d4b936af4ab128b00443fae +P 6f6a0fd63b13cb827d6a402de01a701eb5b3f92954032ea80e78ec864861a26c +R e2ed905d54c7918df3ba25117b34f60f +U drh +Z 49399da496c8dc088d55e5249fd1fc3d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bc8d2726a6..14ac5aef42 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f6a0fd63b13cb827d6a402de01a701eb5b3f92954032ea80e78ec864861a26c \ No newline at end of file +a6e218a6e1ddd74be6a313b1d336334071747efd4ecd354ed7efe303d09c849c \ No newline at end of file diff --git a/test/cursorhint.test b/test/cursorhint.test index 16e26fecd4..eab2a2739e 100644 --- a/test/cursorhint.test +++ b/test/cursorhint.test @@ -167,5 +167,17 @@ do_execsql_test 5.0 { CREATE VIEW t2 AS SELECT 0 FROM t1 WHERE x>='a' OR x='1'; SELECT * FROM t2 RIGHT JOIN t1 ON true; } +# Additional test case from https://sqlite.org/forum/forumpost/d34ad68c36?t=c +# which is a different way to acces the same problem. +# +do_execsql_test 5.1 { + CREATE TABLE v1 (c1, PRIMARY KEY( c1 )) WITHOUT ROWID; + CREATE VIEW v2 AS SELECT 0 FROM v1 WHERE c1 IS '' OR c1 > ''; + CREATE VIEW v3 AS SELECT 0 FROM v2 JOIN (v2 RIGHT JOIN v1); + CREATE VIEW v4 AS SELECT 0 FROM v3, v3; + SELECT * FROM v3 JOIN v3 AS a0, v4 AS a1, v4 AS a2, v3 AS a3, + v3 AS a4, v4 AS a5 + ORDER BY 1; +} finish_test From 12b954db7a92212f06171527ebd5ea1862cb55fa Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Mar 2023 03:17:08 +0000 Subject: [PATCH 127/341] Tighter constraints on the maximum length of the filename handed over to sqlite3_load_extension(), due to [forum:/forumpost/a43074729e|forum post a43074729e]. This is a follow-on to [01f3877c7172d522] and [forum:/forumpost/08a0d6d9bf|forum post 08a0d6d9bf]. FossilOrigin-Name: 9f351bdee2a09a4419bb8256a13d1f757b3e00ec26a445523f224fc56d9ae26d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/loadext.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 76fefd2ffb..70ba130f10 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scase\sto\sfurther\svalidate\sthe\sfix\sat\s[221fdcec964f8317].\n[forum:/forumpost/d34ad68c36|Forum\spost\sd34ad68c36]. -D 2023-03-25T02:07:20.189 +C Tighter\sconstraints\son\sthe\smaximum\slength\sof\sthe\sfilename\shanded\sover\nto\ssqlite3_load_extension(),\sdue\sto\n[forum:/forumpost/a43074729e|forum\spost\sa43074729e].\s\sThis\sis\sa\nfollow-on\sto\s[01f3877c7172d522]\sand\n[forum:/forumpost/08a0d6d9bf|forum\spost\s08a0d6d9bf]. +D 2023-03-25T03:17:08.825 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -587,7 +587,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 7940fce7d4aa855606432d82d20694e17c18a03956f5f5776f2404e2df7c18a8 F src/json.c 9e3a6acf9e4fa8bb6c84a38538ed9c2f9ad09f106910ef0e9cc68e19c8dc4fc8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c b04eb648cedc45efe4298e1ef439ac4f0096ae27b5f01accb0a1f49d57789128 +F src/loadext.c fb07842759d6b44d4d4579afba8738eefd3d1895ec1bc9762a570c989e5db75e F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6f6a0fd63b13cb827d6a402de01a701eb5b3f92954032ea80e78ec864861a26c -R e2ed905d54c7918df3ba25117b34f60f +P a6e218a6e1ddd74be6a313b1d336334071747efd4ecd354ed7efe303d09c849c +R 0a897a9ea5887cea19fc58ed2eebee3b U drh -Z 49399da496c8dc088d55e5249fd1fc3d +Z 873f4ab64a73df2a4c85c82566d6faec # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 14ac5aef42..520d070222 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6e218a6e1ddd74be6a313b1d336334071747efd4ecd354ed7efe303d09c849c \ No newline at end of file +9f351bdee2a09a4419bb8256a13d1f757b3e00ec26a445523f224fc56d9ae26d \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 40bde9ce80..8a2eb0c4db 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -585,8 +585,12 @@ static int sqlite3LoadExtension( /* tag-20210611-1. Some dlopen() implementations will segfault if given ** an oversize filename. Most filesystems have a pathname limit of 4K, ** so limit the extension filename length to about twice that. - ** https://sqlite.org/forum/forumpost/08a0d6d9bf */ - if( nMsg>SQLITE_MAX_PATHLEN ) goto extension_not_found; + ** https://sqlite.org/forum/forumpost/08a0d6d9bf + ** + ** Later (2023-03-25): Save an extra 6 bytes for the filename suffix. + ** See https://sqlite.org/forum/forumpost/24083b579d. + */ + if( nMsg>SQLITE_MAX_PATHLEN-6 ) goto extension_not_found; handle = sqlite3OsDlOpen(pVfs, zFile); #if SQLITE_OS_UNIX || SQLITE_OS_WIN From e5895333dde929c9be1baa88ffda7b3471754c04 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Mar 2023 12:27:36 +0000 Subject: [PATCH 128/341] Even tighter bounds on the maximum length of the filename for sqlite3_load_extension(). FossilOrigin-Name: 787291414d2d2082a3c63e7cdd6bec4719f0c8b75ad1355f5026932ecbb28ba8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/loadext.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 70ba130f10..cfe3c95b5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tighter\sconstraints\son\sthe\smaximum\slength\sof\sthe\sfilename\shanded\sover\nto\ssqlite3_load_extension(),\sdue\sto\n[forum:/forumpost/a43074729e|forum\spost\sa43074729e].\s\sThis\sis\sa\nfollow-on\sto\s[01f3877c7172d522]\sand\n[forum:/forumpost/08a0d6d9bf|forum\spost\s08a0d6d9bf]. -D 2023-03-25T03:17:08.825 +C Even\stighter\sbounds\son\sthe\smaximum\slength\sof\sthe\sfilename\sfor\nsqlite3_load_extension(). +D 2023-03-25T12:27:36.520 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -587,7 +587,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 7940fce7d4aa855606432d82d20694e17c18a03956f5f5776f2404e2df7c18a8 F src/json.c 9e3a6acf9e4fa8bb6c84a38538ed9c2f9ad09f106910ef0e9cc68e19c8dc4fc8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa -F src/loadext.c fb07842759d6b44d4d4579afba8738eefd3d1895ec1bc9762a570c989e5db75e +F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d F src/malloc.c 47b82c5daad557d9b963e3873e99c22570fb470719082c6658bf64e3012f7d23 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a6e218a6e1ddd74be6a313b1d336334071747efd4ecd354ed7efe303d09c849c -R 0a897a9ea5887cea19fc58ed2eebee3b +P 9f351bdee2a09a4419bb8256a13d1f757b3e00ec26a445523f224fc56d9ae26d +R 40c0be8beff032c6896e7e0265e6be39 U drh -Z 873f4ab64a73df2a4c85c82566d6faec +Z 073a667407358758809a055e67c88eb4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 520d070222..35667bf093 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f351bdee2a09a4419bb8256a13d1f757b3e00ec26a445523f224fc56d9ae26d \ No newline at end of file +787291414d2d2082a3c63e7cdd6bec4719f0c8b75ad1355f5026932ecbb28ba8 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 8a2eb0c4db..a00df6536f 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -590,14 +590,16 @@ static int sqlite3LoadExtension( ** Later (2023-03-25): Save an extra 6 bytes for the filename suffix. ** See https://sqlite.org/forum/forumpost/24083b579d. */ - if( nMsg>SQLITE_MAX_PATHLEN-6 ) goto extension_not_found; + if( nMsg>SQLITE_MAX_PATHLEN ) goto extension_not_found; handle = sqlite3OsDlOpen(pVfs, zFile); #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; ii Date: Sat, 25 Mar 2023 18:31:24 +0000 Subject: [PATCH 129/341] The fix at [2bf5413dc2c19d5f] was incomplete in that it failed to clear the reusable register cache that might contain registers in the STAT4 buffer region. This additional change corrects the problem. [forum:/forumpost/83cb4a95a0|Forum post 83cb4a95a0]. Test case in TH3. FossilOrigin-Name: 5d554e4d0f59a4309fed40e4fb26c7be42f1d4d55ccdcaaf7b4d445aa3122955 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index cfe3c95b5a..51f18c47ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Even\stighter\sbounds\son\sthe\smaximum\slength\sof\sthe\sfilename\sfor\nsqlite3_load_extension(). -D 2023-03-25T12:27:36.520 +C The\sfix\sat\s[2bf5413dc2c19d5f]\swas\sincomplete\sin\sthat\sit\sfailed\sto\sclear\nthe\sreusable\sregister\scache\sthat\smight\scontain\sregisters\sin\sthe\sSTAT4\nbuffer\sregion.\s\sThis\sadditional\schange\scorrects\sthe\sproblem.\n[forum:/forumpost/83cb4a95a0|Forum\spost\s83cb4a95a0].\s\sTest\scase\sin\sTH3. +D 2023-03-25T18:31:24.391 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -558,7 +558,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 3ca2f449c890f8b86ec9e06f0c4fccf0648941c3308a16904cb2852227db83f7 -F src/analyze.c 26e68072d62d0a28b2dd8619f76be876db9b4ff6e8ad1fd4ed3ac56b1cb045c4 +F src/analyze.c 6a7af762bb64bcb1954db905cf158c3eb3d0d31ff5acc946e7855c0f31728b9d F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9f351bdee2a09a4419bb8256a13d1f757b3e00ec26a445523f224fc56d9ae26d -R 40c0be8beff032c6896e7e0265e6be39 +P 787291414d2d2082a3c63e7cdd6bec4719f0c8b75ad1355f5026932ecbb28ba8 +R 32eca9004805bd405ad340f1dad6e254 U drh -Z 073a667407358758809a055e67c88eb4 +Z 0ff7057a5cb437509cd22493623e26ff # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 35667bf093..7d2293060c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -787291414d2d2082a3c63e7cdd6bec4719f0c8b75ad1355f5026932ecbb28ba8 \ No newline at end of file +5d554e4d0f59a4309fed40e4fb26c7be42f1d4d55ccdcaaf7b4d445aa3122955 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2eaabd6be4..9e0a0c5e6b 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1297,6 +1297,21 @@ static void analyzeOneTable( /* Allocate space to compute results for the largest index */ pParse->nMem = MAX(pParse->nMem, regCol+mxCol); doOnce = 0; +#ifdef SQLITE_DEBUG + /* Verify that setting pParse->nTempReg to zero below really + ** is needed in some cases, in order to excise all temporary + ** registers from the middle of the STAT4 buffer. + ** https://sqlite.org/forum/forumpost/83cb4a95a0 (2023-03-25) + */ + if( pParse->nTempReg>0 ){ + int kk; + for(kk=0; kknTempReg; kk++){ + int regT = pParse->aTempReg[kk]; + testcase( regT>=regCol && regTnTempReg = 0; } addrNext = sqlite3VdbeCurrentAddr(v); From 52786ec185cc80af097aa44ca3a5d46eb5621d81 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Mar 2023 18:41:42 +0000 Subject: [PATCH 130/341] Add the tag-20230325-1 comment that was omitted from the prior check-in. FossilOrigin-Name: a13c01d076d23f0de500e8e6283e803dfc96f0da7509c0d97d598d6b3e7b930b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 51f18c47ff..d8a542c732 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sfix\sat\s[2bf5413dc2c19d5f]\swas\sincomplete\sin\sthat\sit\sfailed\sto\sclear\nthe\sreusable\sregister\scache\sthat\smight\scontain\sregisters\sin\sthe\sSTAT4\nbuffer\sregion.\s\sThis\sadditional\schange\scorrects\sthe\sproblem.\n[forum:/forumpost/83cb4a95a0|Forum\spost\s83cb4a95a0].\s\sTest\scase\sin\sTH3. -D 2023-03-25T18:31:24.391 +C Add\sthe\stag-20230325-1\scomment\sthat\swas\somitted\sfrom\sthe\sprior\scheck-in. +D 2023-03-25T18:41:42.956 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -558,7 +558,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 3ca2f449c890f8b86ec9e06f0c4fccf0648941c3308a16904cb2852227db83f7 -F src/analyze.c 6a7af762bb64bcb1954db905cf158c3eb3d0d31ff5acc946e7855c0f31728b9d +F src/analyze.c d0a84c1de4640485c0752123dc89788b8333438cde8e4e8ec1e046a374d27cec F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 787291414d2d2082a3c63e7cdd6bec4719f0c8b75ad1355f5026932ecbb28ba8 -R 32eca9004805bd405ad340f1dad6e254 +P 5d554e4d0f59a4309fed40e4fb26c7be42f1d4d55ccdcaaf7b4d445aa3122955 +R 7269972d12a2f4677a304c1374486b2d U drh -Z 0ff7057a5cb437509cd22493623e26ff +Z 5e482c20fe6bc94e59979ad542d34a8c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7d2293060c..630df53313 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d554e4d0f59a4309fed40e4fb26c7be42f1d4d55ccdcaaf7b4d445aa3122955 \ No newline at end of file +a13c01d076d23f0de500e8e6283e803dfc96f0da7509c0d97d598d6b3e7b930b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 9e0a0c5e6b..960571b14e 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1311,7 +1311,7 @@ static void analyzeOneTable( } } #endif - pParse->nTempReg = 0; + pParse->nTempReg = 0; /* tag-20230325-1 */ } addrNext = sqlite3VdbeCurrentAddr(v); From 28ae1956583ebd539f6019d5c6be5e6c54a7c3b6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Mar 2023 19:44:25 +0000 Subject: [PATCH 131/341] In the byte-code, when the result of an expression needs to be in a particular register, always use the sqlite3ExprCode() routine because it has the smarts to know whether to use OP_Copy or OP_SCopy. Do not try to OP_SCopy inline because an OP_Copy might be required. Fix for the problem identified by [forum:/forumpost/5522082cfc|forum post 5522082cfc]. FossilOrigin-Name: c104e5c6eeb89575319d9f94f49446142b06912fa8b283c19d46aa2ccddc5bda --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 24 +++++++----------------- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index d8a542c732..5146a1ff25 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\stag-20230325-1\scomment\sthat\swas\somitted\sfrom\sthe\sprior\scheck-in. -D 2023-03-25T18:41:42.956 +C In\sthe\sbyte-code,\swhen\sthe\sresult\sof\san\sexpression\sneeds\sto\sbe\sin\sa\sparticular\nregister,\salways\suse\sthe\ssqlite3ExprCode()\sroutine\sbecause\sit\shas\sthe\ssmarts\nto\sknow\swhether\sto\suse\sOP_Copy\sor\sOP_SCopy.\s\sDo\snot\stry\sto\sOP_SCopy\sinline\nbecause\san\sOP_Copy\smight\sbe\srequired.\s\sFix\sfor\sthe\sproblem\sidentified\sby\n[forum:/forumpost/5522082cfc|forum\spost\s5522082cfc]. +D 2023-03-25T19:44:25.803 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 201fe0763c52783d205c8c13cdd9d55c1bd5cb21c1f036753f99103b43284b90 -F src/expr.c e02117f1d698d781dc5990364bec01ef19688f084e14b5ea468a972c3b26903b +F src/expr.c fc8e484fbd2e512ac2cf0349af2f4da875fbeadd1713b0c241855a9848b035da F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d554e4d0f59a4309fed40e4fb26c7be42f1d4d55ccdcaaf7b4d445aa3122955 -R 7269972d12a2f4677a304c1374486b2d +P a13c01d076d23f0de500e8e6283e803dfc96f0da7509c0d97d598d6b3e7b930b +R e8d8985bf9d367c8fad5202d85b330d3 U drh -Z 5e482c20fe6bc94e59979ad542d34a8c +Z 42699f196a341272ef4b9eed5abfaeeb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 630df53313..a0a545044e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a13c01d076d23f0de500e8e6283e803dfc96f0da7509c0d97d598d6b3e7b930b \ No newline at end of file +c104e5c6eeb89575319d9f94f49446142b06912fa8b283c19d46aa2ccddc5bda \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 0a52e62309..9e2ae0ebd4 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4397,11 +4397,8 @@ expr_code_doover: #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - if( inReg!=target ){ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( inReg==target ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3VdbeAddOp2(v, OP_Cast, target, sqlite3AffinityType(pExpr->u.zToken, 0)); @@ -4740,13 +4737,9 @@ expr_code_doover: ** Clear subtypes as subtypes may not cross a subquery boundary. */ assert( pExpr->pLeft ); - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - if( inReg!=target ){ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } - sqlite3VdbeAddOp1(v, OP_ClrSubtype, inReg); - return inReg; + sqlite3ExprCode(pParse, pExpr->pLeft, target); + sqlite3VdbeAddOp1(v, OP_ClrSubtype, target); + return target; }else{ pExpr = pExpr->pLeft; goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. */ @@ -4856,12 +4849,9 @@ expr_code_doover: ** "target" and not someplace else. */ pParse->okConstFactor = 0; /* note (1) above */ - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( target==inReg ); pParse->okConstFactor = okConstFactor; - if( inReg!=target ){ /* note (2) above */ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } sqlite3VdbeJumpHere(v, addrINR); break; } From de0416171b4f62bdd134ff6d97950f33050ad63a Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Mar 2023 21:01:11 +0000 Subject: [PATCH 132/341] When the left table of a RIGHT JOIN is used inside an aggregate function and the left table employs an index on expressions, then make sure the expressions evaluate to NULL for the cases where the left table should be NULL. Proposed fix for [forum:/forumpost/9b491e1deb|forum post 9b491e1deb]. More testing an analysis needed - there is a FIXME in this check-in. FossilOrigin-Name: 3572b40a7dfc4acc35e72e08e79f64688f8737e57ac89e4d10e6b32bd5178c63 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5146a1ff25..543571e7e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sbyte-code,\swhen\sthe\sresult\sof\san\sexpression\sneeds\sto\sbe\sin\sa\sparticular\nregister,\salways\suse\sthe\ssqlite3ExprCode()\sroutine\sbecause\sit\shas\sthe\ssmarts\nto\sknow\swhether\sto\suse\sOP_Copy\sor\sOP_SCopy.\s\sDo\snot\stry\sto\sOP_SCopy\sinline\nbecause\san\sOP_Copy\smight\sbe\srequired.\s\sFix\sfor\sthe\sproblem\sidentified\sby\n[forum:/forumpost/5522082cfc|forum\spost\s5522082cfc]. -D 2023-03-25T19:44:25.803 +C When\sthe\sleft\stable\sof\sa\sRIGHT\sJOIN\sis\sused\sinside\san\saggregate\sfunction\nand\sthe\sleft\stable\semploys\san\sindex\son\sexpressions,\sthen\smake\ssure\sthe\nexpressions\sevaluate\sto\sNULL\sfor\sthe\scases\swhere\sthe\sleft\stable\sshould\sbe\nNULL.\s\sProposed\sfix\sfor\s[forum:/forumpost/9b491e1deb|forum\spost\s9b491e1deb].\nMore\stesting\san\sanalysis\sneeded\s-\sthere\sis\sa\sFIXME\sin\sthis\scheck-in. +D 2023-03-25T21:01:11.261 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 201fe0763c52783d205c8c13cdd9d55c1bd5cb21c1f036753f99103b43284b90 -F src/expr.c fc8e484fbd2e512ac2cf0349af2f4da875fbeadd1713b0c241855a9848b035da +F src/expr.c c6f7772fd988d86c9e9c34d19e62369cb26de6a783dbc5fe77d8d952e973425f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a13c01d076d23f0de500e8e6283e803dfc96f0da7509c0d97d598d6b3e7b930b -R e8d8985bf9d367c8fad5202d85b330d3 +P c104e5c6eeb89575319d9f94f49446142b06912fa8b283c19d46aa2ccddc5bda +R 2bf850351800b6fc356e18027e4287ef U drh -Z 42699f196a341272ef4b9eed5abfaeeb +Z 3175f533ee39afc8052867b6068f0268 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a0a545044e..11883c7ef5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c104e5c6eeb89575319d9f94f49446142b06912fa8b283c19d46aa2ccddc5bda \ No newline at end of file +3572b40a7dfc4acc35e72e08e79f64688f8737e57ac89e4d10e6b32bd5178c63 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 9e2ae0ebd4..372e598036 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4222,7 +4222,13 @@ expr_code_doover: AggInfo *pAggInfo = pExpr->pAggInfo; struct AggInfo_col *pCol; assert( pAggInfo!=0 ); - assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + assert( pExpr->iAgg>=0 ); + if( pExpr->iAgg>=pAggInfo->nColumn ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + /* FIXME: Need to verify that tests run this opcode + ** Some kind of coverage macro.. VdbeCoverage(v); tag-20230325-2 */ + break; + } pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ return AggInfoColumnReg(pAggInfo, pExpr->iAgg); From 7bb842642c75b8b2664dd1c7b47df65bf4500e9e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Mar 2023 22:37:23 +0000 Subject: [PATCH 133/341] When reading sqlite_stat4 data during query planning, be sure to expand zeroblobs prior to running comparisons. Fix for the issue identified by [forum:/forumpost/5275207102|forum post 5275207102]. FossilOrigin-Name: 5c8dd8dfcaab9c364b3a126ca35880ef57f5cecbe030771e646c934c8cf43709 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbemem.c | 1 + test/analyzeE.test | 16 +++++++++++++++- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5146a1ff25..0f9117b1ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sbyte-code,\swhen\sthe\sresult\sof\san\sexpression\sneeds\sto\sbe\sin\sa\sparticular\nregister,\salways\suse\sthe\ssqlite3ExprCode()\sroutine\sbecause\sit\shas\sthe\ssmarts\nto\sknow\swhether\sto\suse\sOP_Copy\sor\sOP_SCopy.\s\sDo\snot\stry\sto\sOP_SCopy\sinline\nbecause\san\sOP_Copy\smight\sbe\srequired.\s\sFix\sfor\sthe\sproblem\sidentified\sby\n[forum:/forumpost/5522082cfc|forum\spost\s5522082cfc]. -D 2023-03-25T19:44:25.803 +C When\sreading\ssqlite_stat4\sdata\sduring\squery\splanning,\sbe\ssure\sto\sexpand\nzeroblobs\sprior\sto\srunning\scomparisons.\s\sFix\sfor\sthe\sissue\sidentified\nby\s[forum:/forumpost/5275207102|forum\spost\s5275207102]. +D 2023-03-25T22:37:23.202 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -701,7 +701,7 @@ F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 F src/vdbeaux.c 0f5201346a83a35a08e833c3a03abe626119c07b7361c28bc6a259b98540f1d6 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd -F src/vdbemem.c 656a51f3e99d7123ef852413c0249101718f3f3138e66be37ba2f16dcf7ce43e +F src/vdbemem.c cbd2305d6444dc0aa3ab0dbe8f45ca5166da6f5a9bb6402be9739b6443a83fd0 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -753,7 +753,7 @@ F test/analyze8.test 29ef237d8a59b39cc31c3310134fefe96a690b195e3deed5ecb65283908 F test/analyze9.test 30e1cb99336045a384a11d97900720184333c88174b3b89bc07444ea39e7df19 F test/analyzeC.test 1111830ad355d29a294a5dda654dd5f6a8622c6a223a4f7b7b3d091df7a7a42b F test/analyzeD.test 485f621cfd2ef0a8f8ac79672586651bfa495bd899db50461bb4b558400ab3c1 -F test/analyzeE.test 8770de3834bb9aea1c892f4d155a5bd9578c23c0c3f199dc6f1b5d597b5bf65c +F test/analyzeE.test d2ec7921c162cdc33ac8e7eb01f9ebf78100610af7c94c8552bbf551de1fb397 F test/analyzeF.test 40b5cc3ad7b10e81020d7ca86f1417647ecfae7477cfd88acc5aa7ae1068f949 F test/analyzeG.test 623be33038c49648872746c8dd8b23b5792c08fef173c55e82f1b12fca259852 F test/analyzer1.test 459fa02c445ddbf0101a3bad47b34290a35f2e49 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a13c01d076d23f0de500e8e6283e803dfc96f0da7509c0d97d598d6b3e7b930b -R e8d8985bf9d367c8fad5202d85b330d3 +P c104e5c6eeb89575319d9f94f49446142b06912fa8b283c19d46aa2ccddc5bda +R a2c3c428afb2b0766c34c47704a29efd U drh -Z 42699f196a341272ef4b9eed5abfaeeb +Z 5b3601bdf9b325a3bd0226a3a60df613 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a0a545044e..4c444a3bb6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c104e5c6eeb89575319d9f94f49446142b06912fa8b283c19d46aa2ccddc5bda \ No newline at end of file +5c8dd8dfcaab9c364b3a126ca35880ef57f5cecbe030771e646c934c8cf43709 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 02588707de..1b8808ea6f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1587,6 +1587,7 @@ static int valueFromExpr( rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); testcase( rc!=SQLITE_OK ); if( *ppVal ){ + rc = ExpandBlob(*ppVal); sqlite3VdbeMemCast(*ppVal, aff, enc); sqlite3ValueApplyAffinity(*ppVal, affinity, enc); } diff --git a/test/analyzeE.test b/test/analyzeE.test index 9b5a27b87a..2547daa1c8 100644 --- a/test/analyzeE.test +++ b/test/analyzeE.test @@ -275,6 +275,20 @@ do_execsql_test analyzeE-6.4 { SELECT count(*) FROM sqlite_stat4 WHERE idx='i1' AND neq<>'1 1 1 1 1 1'; } 0 - +# 2023-03-25 https://sqlite.org/forum/forumpost/5275207102 +# Correctly expand zeroblobs while processing STAT4 information +# during query planning. +# +reset_db +do_execsql_test analyzeE-7.0 { + CREATE TABLE t1(a TEXT COLLATE binary); + CREATE INDEX t1x ON t1(a); + INSERT INTO t1(a) VALUES(0),('apple'),(NULL),(''),('banana'); + ANALYZE; + SELECT format('(%s)',a) FROM t1 WHERE t1.a > CAST(zeroblob(5) AS TEXT); +} {(0) (apple) (banana)} +do_execsql_test analyzeE-7.1 { + SELECT format('(%s)',a) FROM t1 WHERE t1.a <= CAST(zeroblob(5) AS TEXT); +} {()} finish_test From debc3202c6ac81a4baee951cddfad9a2087ed56e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 25 Mar 2023 23:40:21 +0000 Subject: [PATCH 134/341] Add usage detection to the NULL value generator for the left table of a RIGHT JOIN inside of an aggregate. FossilOrigin-Name: 4d05a009dfb63bcb4173da4d09ccc00c308ce4dd2534a32aeb23c5d8d6a1fd4b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 10 ++++++++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 543571e7e4..20441d9d79 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sthe\sleft\stable\sof\sa\sRIGHT\sJOIN\sis\sused\sinside\san\saggregate\sfunction\nand\sthe\sleft\stable\semploys\san\sindex\son\sexpressions,\sthen\smake\ssure\sthe\nexpressions\sevaluate\sto\sNULL\sfor\sthe\scases\swhere\sthe\sleft\stable\sshould\sbe\nNULL.\s\sProposed\sfix\sfor\s[forum:/forumpost/9b491e1deb|forum\spost\s9b491e1deb].\nMore\stesting\san\sanalysis\sneeded\s-\sthere\sis\sa\sFIXME\sin\sthis\scheck-in. -D 2023-03-25T21:01:11.261 +C Add\susage\sdetection\sto\sthe\sNULL\svalue\sgenerator\sfor\sthe\sleft\stable\sof\sa\nRIGHT\sJOIN\sinside\sof\san\saggregate. +D 2023-03-25T23:40:21.196 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 201fe0763c52783d205c8c13cdd9d55c1bd5cb21c1f036753f99103b43284b90 -F src/expr.c c6f7772fd988d86c9e9c34d19e62369cb26de6a783dbc5fe77d8d952e973425f +F src/expr.c c938990570effc02e065b176702d627a0237ec796263c142868f8007b0b703c0 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c104e5c6eeb89575319d9f94f49446142b06912fa8b283c19d46aa2ccddc5bda -R 2bf850351800b6fc356e18027e4287ef +P 3572b40a7dfc4acc35e72e08e79f64688f8737e57ac89e4d10e6b32bd5178c63 +R c45a71619ca942293d80f31a1a49eea4 U drh -Z 3175f533ee39afc8052867b6068f0268 +Z aa39fec11cd82de6900c756e7906d67c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 11883c7ef5..f9540775bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3572b40a7dfc4acc35e72e08e79f64688f8737e57ac89e4d10e6b32bd5178c63 \ No newline at end of file +4d05a009dfb63bcb4173da4d09ccc00c308ce4dd2534a32aeb23c5d8d6a1fd4b \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 372e598036..cd09c0d711 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4224,9 +4224,15 @@ expr_code_doover: assert( pAggInfo!=0 ); assert( pExpr->iAgg>=0 ); if( pExpr->iAgg>=pAggInfo->nColumn ){ + /* Happens when the left table of a RIGHT JOIN is null and + ** is using an expression index */ sqlite3VdbeAddOp2(v, OP_Null, 0, target); - /* FIXME: Need to verify that tests run this opcode - ** Some kind of coverage macro.. VdbeCoverage(v); tag-20230325-2 */ +#ifdef SQLITE_VDBE_COVERAGE + /* Verify that the OP_Null above is exercised by tests + ** tag-20230325-2 */ + sqlite3VdbeAddOp2(v, OP_NotNull, target, 1); + VdbeCoverageNeverTaken(v); +#endif break; } pCol = &pAggInfo->aCol[pExpr->iAgg]; From 418f947b9818542e8c2ab0a9e5c269adfbdc21f4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 26 Mar 2023 11:54:51 +0000 Subject: [PATCH 135/341] Disable factoring of constant values during ANALYZE. This is a temporary fix for [forum:/forumpost/07de5f6216|forum post 07de5f6216]. The register allocation logic in ANALYZE needs to be completely refactored, but that will take longer. This check-in will serve to resolve the issue until a better fix can be devised. FossilOrigin-Name: c3967d1259f1df969d303394986960bd098e174dcd337e374c9c3c39e0efa466 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/analyze.c | 1 + 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 85c614c489..e84dd218bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sthe\sleft\stable\sof\sa\sRIGHT\sJOIN\sis\sused\sinside\san\saggregate\sfunction\sand\sthe\sleft\stable\semploys\san\sindex\son\sexpressions,\sthen\smake\ssure\sthe\sexpressions\sevaluate\sto\sNULL\sfor\sthe\scases\swhere\sthe\sleft\stable\sshould\sbe\sNULL.\sFix\sfor\n[forum:/forumpost/9b491e1deb|forum\spost\s9b491e1deb]. -D 2023-03-25T23:52:05.666 +C Disable\sfactoring\sof\sconstant\svalues\sduring\sANALYZE.\s\sThis\sis\sa\stemporary\nfix\sfor\s[forum:/forumpost/07de5f6216|forum\spost\s07de5f6216].\s\sThe\sregister\nallocation\slogic\sin\sANALYZE\sneeds\sto\sbe\scompletely\srefactored,\sbut\sthat\swill\ntake\slonger.\s\sThis\scheck-in\swill\sserve\sto\sresolve\sthe\sissue\suntil\sa\sbetter\nfix\scan\sbe\sdevised. +D 2023-03-26T11:54:51.582 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -558,7 +558,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 3ca2f449c890f8b86ec9e06f0c4fccf0648941c3308a16904cb2852227db83f7 -F src/analyze.c d0a84c1de4640485c0752123dc89788b8333438cde8e4e8ec1e046a374d27cec +F src/analyze.c 3449514fb387fbc15d2cead1fb6856d0556bf7491ab5088d7937bf1d48567ef3 F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 @@ -2051,9 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5c8dd8dfcaab9c364b3a126ca35880ef57f5cecbe030771e646c934c8cf43709 4d05a009dfb63bcb4173da4d09ccc00c308ce4dd2534a32aeb23c5d8d6a1fd4b -R f5cc169fe82411f91415c843e9957996 -T +closed 4d05a009dfb63bcb4173da4d09ccc00c308ce4dd2534a32aeb23c5d8d6a1fd4b +P ffe23af73fcb324df988a00be343654ce7078b7208647c4eb779d666b8297e7c +R 23223a4c6ca9776e461c6a724ecec668 U drh -Z d658d6cec90cfd7621a51815fecf3fc5 +Z 8a53fd62f6653c359d7e1076b2f287a9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 84bfa0315c..dbafff546d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffe23af73fcb324df988a00be343654ce7078b7208647c4eb779d666b8297e7c \ No newline at end of file +c3967d1259f1df969d303394986960bd098e174dcd337e374c9c3c39e0efa466 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 960571b14e..086a58494f 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -198,6 +198,7 @@ static void openStatTable( assert( sqlite3BtreeHoldsAllMutexes(db) ); assert( sqlite3VdbeDb(v)==db ); pDb = &db->aDb[iDb]; + pParse->okConstFactor = 0; /* Create new statistic tables if they do not exist, or clear them ** if they do already exist. From aa9192e6aa4da7d6c685bfbbefe0b4f6e55c5b2d Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 26 Mar 2023 16:36:27 +0000 Subject: [PATCH 136/341] Improvements to register allocation, especially in the ANALYZE command. New assert() statements added to help verify that memory allocation is correct, and to help fuzzer find lingering errors. FossilOrigin-Name: 6f8b97f31a4c8552312b4c98432ea356ae54c06d9cc929969f50c3c88360cd7b --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/analyze.c | 26 +++++++++++--------------- src/expr.c | 37 +++++++++++++++++++++++++++++++++++++ src/pragma.c | 6 +++--- src/sqliteInt.h | 2 ++ 6 files changed, 63 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index e84dd218bd..4dc1905e93 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sfactoring\sof\sconstant\svalues\sduring\sANALYZE.\s\sThis\sis\sa\stemporary\nfix\sfor\s[forum:/forumpost/07de5f6216|forum\spost\s07de5f6216].\s\sThe\sregister\nallocation\slogic\sin\sANALYZE\sneeds\sto\sbe\scompletely\srefactored,\sbut\sthat\swill\ntake\slonger.\s\sThis\scheck-in\swill\sserve\sto\sresolve\sthe\sissue\suntil\sa\sbetter\nfix\scan\sbe\sdevised. -D 2023-03-26T11:54:51.582 +C Improvements\sto\sregister\sallocation,\sespecially\sin\sthe\sANALYZE\scommand.\nNew\sassert()\sstatements\sadded\sto\shelp\sverify\sthat\smemory\sallocation\sis\ncorrect,\sand\sto\shelp\sfuzzer\sfind\slingering\serrors. +D 2023-03-26T16:36:27.642 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -558,7 +558,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 3ca2f449c890f8b86ec9e06f0c4fccf0648941c3308a16904cb2852227db83f7 -F src/analyze.c 3449514fb387fbc15d2cead1fb6856d0556bf7491ab5088d7937bf1d48567ef3 +F src/analyze.c 8230fc1f3ddcf5492d258f28afcd36026c0de2f35cbbee22a41eed4baaf26148 F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c 201fe0763c52783d205c8c13cdd9d55c1bd5cb21c1f036753f99103b43284b90 -F src/expr.c c938990570effc02e065b176702d627a0237ec796263c142868f8007b0b703c0 +F src/expr.c 4cc4595f1c589b76f33f884f9c6b5f85ea80d205f1fc5ef9fb3617bf03920f0b F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -618,7 +618,7 @@ F src/parse.y 424e49ed8fc6c907920db9be5a13a75ed43811e1ea8dd21b0fa9ef97f083dc6b F src/pcache.c 842410539b544e12d5fccfcf29890782f46a58f227a77bc0bd76243799662c0c F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc -F src/pragma.c c0c6e0735436e0e9e8d490f82959ddc9475d438e9f808a51953a4d43e30cea51 +F src/pragma.c 20103b19d91551a673e5a35bd41a830b3d1bf2929174825cf6148430562c4533 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 @@ -630,7 +630,7 @@ F src/shell.c.in d80b53a9ec0a43ac0157fc54d667cb49fb6dd2229558135530feab2fc04b909 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h ee87a8982f59897ea0594df986bb3ebbe971b16fa0248c9c4940ae18beda7deb +F src/sqliteInt.h 8dab239c6de1d9b9e8732bbc4ad454f142b4b2781867cab0786597a391489ed4 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ffe23af73fcb324df988a00be343654ce7078b7208647c4eb779d666b8297e7c -R 23223a4c6ca9776e461c6a724ecec668 +P c3967d1259f1df969d303394986960bd098e174dcd337e374c9c3c39e0efa466 +R 83f2cb26cda25754a99646b5b49c48f5 U drh -Z 8a53fd62f6653c359d7e1076b2f287a9 +Z e8ea09f6c5d7f65fd4a1a44c0da01723 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dbafff546d..cde71f3477 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3967d1259f1df969d303394986960bd098e174dcd337e374c9c3c39e0efa466 \ No newline at end of file +6f8b97f31a4c8552312b4c98432ea356ae54c06d9cc929969f50c3c88360cd7b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 086a58494f..b6b59a6b79 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -198,7 +198,6 @@ static void openStatTable( assert( sqlite3BtreeHoldsAllMutexes(db) ); assert( sqlite3VdbeDb(v)==db ); pDb = &db->aDb[iDb]; - pParse->okConstFactor = 0; /* Create new statistic tables if they do not exist, or clear them ** if they do already exist. @@ -1002,7 +1001,8 @@ static void analyzeOneTable( Table *pStat1 = 0; #endif - pParse->nMem = MAX(pParse->nMem, iMem); + sqlite3TouchRegister(pParse, iMem); + assert( sqlite3NoTempsInRange(pParse, regNewRowid, iMem) ); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; @@ -1108,7 +1108,7 @@ static void analyzeOneTable( ** the regPrev array and a trailing rowid (the rowid slot is required ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); + sqlite3TouchRegister(pParse, regPrev+nColTest); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -1296,24 +1296,19 @@ static void analyzeOneTable( } /* Allocate space to compute results for the largest index */ - pParse->nMem = MAX(pParse->nMem, regCol+mxCol); + sqlite3TouchRegister(pParse, regCol+mxCol); doOnce = 0; #ifdef SQLITE_DEBUG - /* Verify that setting pParse->nTempReg to zero below really - ** is needed in some cases, in order to excise all temporary - ** registers from the middle of the STAT4 buffer. + /* Verify that the call to sqlite3ClearTempRegCache() below + ** really is needed. ** https://sqlite.org/forum/forumpost/83cb4a95a0 (2023-03-25) */ - if( pParse->nTempReg>0 ){ - int kk; - for(kk=0; kknTempReg; kk++){ - int regT = pParse->aTempReg[kk]; - testcase( regT>=regCol && regTnTempReg = 0; /* tag-20230325-1 */ + sqlite3ClearTempRegCache(pParse); /* tag-20230325-1 */ + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) ); } + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+nCol) ); addrNext = sqlite3VdbeCurrentAddr(v); callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); @@ -1394,6 +1389,7 @@ static void analyzeDatabase(Parse *pParse, int iDb){ for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + iMem = sqlite3FirstAvailableRegister(pParse, iMem); } loadAnalysis(pParse, iDb); } diff --git a/src/expr.c b/src/expr.c index cd09c0d711..564dfc7fbb 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6635,6 +6635,35 @@ void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nRangeReg = 0; } +/* +** Make sure sufficient registers have been allocated so that +** iReg is a valid register number. +*/ +void sqlite3TouchRegister(Parse *pParse, int iReg){ + if( pParse->nMemnMem = iReg; +} + +/* +** Return the latest reusable register in the set of all registers. +** The value returned is no less than iMin. If any register iMin or +** greater is in permanent use, then return one more than that last +** permanent register. +*/ +int sqlite3FirstAvailableRegister(Parse *pParse, int iMin){ + const ExprList *pList = pParse->pConstExpr; + if( pList ){ + int i; + for(i=0; inExpr; i++){ + if( pList->a[i].u.iConstExprReg>=iMin ){ + iMin = pList->a[i].u.iConstExprReg + 1; + } + } + } + pParse->nTempReg = 0; + pParse->nRangeReg = 0; + return iMin; +} + /* ** Validate that no temporary register falls within the range of ** iFirst..iLast, inclusive. This routine is only call from within assert() @@ -6654,6 +6683,14 @@ int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ return 0; } } + if( pParse->pConstExpr ){ + ExprList *pList = pParse->pConstExpr; + for(i=0; inExpr; i++){ + int iReg = pList->a[i].u.iConstExprReg; + if( iReg==0 ) continue; + if( iReg>=iFirst && iReg<=iLast ) return 0; + } + } return 1; } #endif /* SQLITE_DEBUG */ diff --git a/src/pragma.c b/src/pragma.c index cd8d51c735..4686adfad5 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1524,7 +1524,7 @@ void sqlite3Pragma( zDb = db->aDb[iDb].zDbSName; sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3TouchRegister(pParse, pTab->nCol+regRow); sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); sqlite3VdbeLoadString(v, regResult, pTab->zName); assert( IsOrdinaryTable(pTab) ); @@ -1565,7 +1565,7 @@ void sqlite3Pragma( ** regRow..regRow+n. If any of the child key values are NULL, this ** row cannot cause an FK violation. Jump directly to addrOk in ** this case. */ - if( regRow+pFK->nCol>pParse->nMem ) pParse->nMem = regRow+pFK->nCol; + sqlite3TouchRegister(pParse, regRow + pFK->nCol); for(j=0; jnCol; j++){ int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom; sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j); @@ -1729,7 +1729,7 @@ void sqlite3Pragma( aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); + sqlite3TouchRegister(pParse, 8+mxIdx); sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8cb0198a29..d6560c51bf 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4649,6 +4649,8 @@ void sqlite3ReleaseTempReg(Parse*,int); int sqlite3GetTempRange(Parse*,int); void sqlite3ReleaseTempRange(Parse*,int,int); void sqlite3ClearTempRegCache(Parse*); +void sqlite3TouchRegister(Parse*,int); +int sqlite3FirstAvailableRegister(Parse*,int); #ifdef SQLITE_DEBUG int sqlite3NoTempsInRange(Parse*,int,int); #endif From 3594b2b3031ce0239ddedbd2c2b0367cf51b56ed Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 27 Mar 2023 13:24:02 +0000 Subject: [PATCH 137/341] Do not allow constant factoring during PRAGMA integrity_check, since the constants might be stored in registers that are later reused for other purposes. dbsqlfuzz dc9ab26037cf5ef797d28cd1ae0855ade584216d. Problem discovered by a new assert() statement added in [6f8b97f31a4c8552]. FossilOrigin-Name: 0bba27b78112b2b2271e498f41c437df985aa2faab302ee5b29d9b60003a8379 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pragma.c | 1 + test/pragma.test | 13 +++++++++++++ 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4dc1905e93..fc6c84fa6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sregister\sallocation,\sespecially\sin\sthe\sANALYZE\scommand.\nNew\sassert()\sstatements\sadded\sto\shelp\sverify\sthat\smemory\sallocation\sis\ncorrect,\sand\sto\shelp\sfuzzer\sfind\slingering\serrors. -D 2023-03-26T16:36:27.642 +C Do\snot\sallow\sconstant\sfactoring\sduring\sPRAGMA\sintegrity_check,\ssince\sthe\nconstants\smight\sbe\sstored\sin\sregisters\sthat\sare\slater\sreused\sfor\sother\npurposes.\s\sdbsqlfuzz\sdc9ab26037cf5ef797d28cd1ae0855ade584216d.\s\sProblem\ndiscovered\sby\sa\snew\sassert()\sstatement\sadded\sin\s[6f8b97f31a4c8552]. +D 2023-03-27T13:24:02.826 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -618,7 +618,7 @@ F src/parse.y 424e49ed8fc6c907920db9be5a13a75ed43811e1ea8dd21b0fa9ef97f083dc6b F src/pcache.c 842410539b544e12d5fccfcf29890782f46a58f227a77bc0bd76243799662c0c F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc -F src/pragma.c 20103b19d91551a673e5a35bd41a830b3d1bf2929174825cf6148430562c4533 +F src/pragma.c 367652f5374b2f17761c96b262d9534d89bf089572a1ebba70d426a138c6802a F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 @@ -1379,7 +1379,7 @@ F test/pendingrace.test cbdf0f74bc939fb43cebad64dda7a0b5a3941a10b7e9cc2b596ff3e4 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff F test/permutations.test 8bd6b6db541e2a7f9bb894be99ef5c00526b23762c4a00c574e1cba697495125 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f -F test/pragma.test aeefa47ba5ebbf4ffc6addc223568d2a95dc7ec3ba3e3c4a26b199e805f94514 +F test/pragma.test d21ae02249f2a1813f016efebe616a1916b3f9abd71867da6211a50bfea373ba F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 92a46bbea12322dd94a404f49edcfbfc913a2c98115f0d030a7459bb4712ef31 F test/pragma4.test ca5e4dfc46adfe490f75d73734f70349d95a199e6510973899e502eef2c8b1f8 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c3967d1259f1df969d303394986960bd098e174dcd337e374c9c3c39e0efa466 -R 83f2cb26cda25754a99646b5b49c48f5 +P 6f8b97f31a4c8552312b4c98432ea356ae54c06d9cc929969f50c3c88360cd7b +R da63f53a7775f8aca8c00c917e70bb9d U drh -Z e8ea09f6c5d7f65fd4a1a44c0da01723 +Z f3c893e6519a38082584c0ff29f6d8de # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cde71f3477..2a932831a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f8b97f31a4c8552312b4c98432ea356ae54c06d9cc929969f50c3c88360cd7b \ No newline at end of file +0bba27b78112b2b2271e498f41c437df985aa2faab302ee5b29d9b60003a8379 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 4686adfad5..4138752455 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1694,6 +1694,7 @@ void sqlite3Pragma( if( iDb>=0 && i!=iDb ) continue; sqlite3CodeVerifySchema(pParse, i); + pParse->okConstFactor = 0; /* tag-20230327-1 */ /* Do an integrity check of the B-Tree ** diff --git a/test/pragma.test b/test/pragma.test index 9ab332c3f1..a3e1d3cd65 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -2060,4 +2060,17 @@ ifcapable !has_codec { } {0 {{database disk image is malformed}}} } database_never_corrupt + +# 2023-03-27. Register allocation issue in integrity_check discovered +# by new assert() statements added in [6f8b97f31a4c8552]. +# dbsqlfuzz dc9ab26037cf5ef797d28cd1ae0855ade584216d +# tag-20230327-1 +# +reset_db +do_execsql_test 25.0 { + CREATE TABLE t1(a INT, b AS (a*2) NOT NULL); + CREATE TEMP TABLE t2(a PRIMARY KEY, b, c UNIQUE) WITHOUT ROWID; + CREATE UNIQUE INDEX t2x ON t2(c,b); + PRAGMA integrity_check; +} ok finish_test From fc6c3936aac93d7c47c2780b0254422737a23090 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 27 Mar 2023 13:57:08 +0000 Subject: [PATCH 138/341] Remove a meaningless JS test. Add a timer to the OPFS async-side worker loader in an attempt to catch a browser-specific quirk in which the worker loading silently fails, per discussion in/around [forum post a708c98dcb3ef|forum:a708c98dcb3ef]. FossilOrigin-Name: 4fc1904b8e18c7d41fa65490ced125f1df4f0c22c13de957b24615ed09b3ecb7 --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 25 +++++++++++++++++++++++-- ext/wasm/tester1.c-pp.js | 11 ----------- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 09f7f8bdb5..e9dfcf4171 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -117,7 +117,7 @@ const installOpfsVfs = function callee(options){ if('function' === typeof options.proxyUri){ options.proxyUri = options.proxyUri(); } - const thePromise = new Promise(function(promiseResolve, promiseReject_){ + const thePromise = new Promise(function(promiseResolve_, promiseReject_){ const loggers = { 0:sqlite3.config.error, 1:sqlite3.config.warn, @@ -193,10 +193,16 @@ const installOpfsVfs = function callee(options){ }/*metrics*/; const opfsVfs = new sqlite3_vfs(); const opfsIoMethods = new sqlite3_io_methods(); - const promiseReject = function(err){ + let promiseWasRejected = undefined; + const promiseReject = (err)=>{ + promiseWasRejected = true; opfsVfs.dispose(); return promiseReject_(err); }; + const promiseResolve = (value)=>{ + promiseWasRejected = false; + return promiseResolve_(value); + }; const W = //#if target=es6-bundler-friendly new Worker(new URL("sqlite3-opfs-async-proxy.js", import.meta.url)); @@ -205,6 +211,18 @@ const installOpfsVfs = function callee(options){ //#else new Worker(options.proxyUri); //#endif + setTimeout(()=>{ + /* At attempt to work around a browser-specific quirk in which + the Worker load is failing in such a way that we neither + resolve nor reject it. This workaround gives that resolve/reject + a time limit and rejects if that timer expires. Discussion: + https://sqlite.org/forum/forumpost/a708c98dcb3ef */ + if(undefined===promiseWasRejected){ + promiseReject( + new Error("Timeout while waiting for OPFS async proxy worker.") + ); + } + }, 4000); W._originalOnError = W.onerror /* will be restored later */; W.onerror = function(err){ // The error object doesn't contain any useful info when the @@ -1269,6 +1287,9 @@ const installOpfsVfs = function callee(options){ /*Indicates that the async partner has received the 'init' and has finished initializing, so the real work can begin...*/ + if(true===promiseWasRejected){ + break /* promise was already rejected via timer */; + } try { sqlite3.vfs.installVfs({ io: {struct: opfsIoMethods, methods: ioSyncWrappers}, diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 539c239849..02eeba7a49 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -1189,17 +1189,6 @@ self.sqlite3InitModule = sqlite3InitModule; const stack = wasm.pstack.pointer; try { const [pCur, pHi] = wasm.pstack.allocChunks(2,'i64'); - rc = capi.sqlite3_db_status(this.db, capi.SQLITE_DBSTATUS_LOOKASIDE_USED, - pCur, pHi, 0); - T.assert(0===rc); - if(!wasm.peek32(pCur)){ - rc = capi.sqlite3_db_config(this.db, capi.SQLITE_DBCONFIG_LOOKASIDE, - 0, 4096, 12); - T.assert(0 === rc); - }else{ - console.debug("Cannot test db_config(SQLITE_DBCONFIG_LOOKASIDE)", - "while lookaside memory is in use."); - } wasm.poke32([pCur, pHi], 0); let [vCur, vHi] = wasm.peek32(pCur, pHi); T.assert(0===vCur).assert(0===vHi); diff --git a/manifest b/manifest index fc6c84fa6d..db5fc66988 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sconstant\sfactoring\sduring\sPRAGMA\sintegrity_check,\ssince\sthe\nconstants\smight\sbe\sstored\sin\sregisters\sthat\sare\slater\sreused\sfor\sother\npurposes.\s\sdbsqlfuzz\sdc9ab26037cf5ef797d28cd1ae0855ade584216d.\s\sProblem\ndiscovered\sby\sa\snew\sassert()\sstatement\sadded\sin\s[6f8b97f31a4c8552]. -D 2023-03-27T13:24:02.826 +C Remove\sa\smeaningless\sJS\stest.\sAdd\sa\stimer\sto\sthe\sOPFS\sasync-side\sworker\sloader\sin\san\sattempt\sto\scatch\sa\sbrowser-specific\squirk\sin\swhich\sthe\sworker\sloading\ssilently\sfails,\sper\sdiscussion\sin/around\s[forum\spost\sa708c98dcb3ef|forum:a708c98dcb3ef]. +D 2023-03-27T13:57:08.550 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -489,7 +489,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 26f6240170d415726d9cfe2fa7a0163e153775e1a74fa91c9ba5446502c71097 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 5cb63f847737ded723d0bdf9b3313e8f7deb0bbfd0082dedde540991edde155b F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c e04e863be7c3a58aa144a463770c1b68ce69f7cfc9e1a5d6397a67877acaf006 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc @@ -535,7 +535,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html 258d08f1ba9cc2d455958751e26be833893cf9ff7853e9436e593e1f778a386b F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js 942e31e5acacdea8f7e5c134c9c23433d56ca95e9f033708556aead1e11a6e6c +F ext/wasm/tester1.c-pp.js 587a18db0f794c594eb0fade37c92af3e2370501576bb08dafd8ed7d009b6516 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6f8b97f31a4c8552312b4c98432ea356ae54c06d9cc929969f50c3c88360cd7b -R da63f53a7775f8aca8c00c917e70bb9d -U drh -Z f3c893e6519a38082584c0ff29f6d8de +P 0bba27b78112b2b2271e498f41c437df985aa2faab302ee5b29d9b60003a8379 +R d30a83bc6e415dfd421aa91cf230833b +U stephan +Z fe1a84b137285d5b69dcaac8bd1588ba # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2a932831a6..1a1d620ef5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0bba27b78112b2b2271e498f41c437df985aa2faab302ee5b29d9b60003a8379 \ No newline at end of file +4fc1904b8e18c7d41fa65490ced125f1df4f0c22c13de957b24615ed09b3ecb7 \ No newline at end of file From 3cbf38c7832e234aab1848e32c104fe345e19e4f Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Mar 2023 11:18:04 +0000 Subject: [PATCH 139/341] Fix multiple problems with RETURNING on a DML statement against a view, all inspired by [forum:/forumpost/dc3b92cfa0|forum post dc3b92cfa0]. (1) Do not allow a RETURNING clause to trick the code generator into thinking that the view being updated has an INSTEAD OF trigger. (2) Generate all result columns for a view in a DML statement. (3) The automatic covering index for a view should cover all result columns of the view. FossilOrigin-Name: c8bedef0d61731c29ae34de1594222d15b578f9e2cddbbd5b74fb3059644fe0f --- manifest | 28 ++++++++++++------------ manifest.uuid | 2 +- src/delete.c | 8 ++++--- src/insert.c | 4 ++-- src/sqliteInt.h | 2 +- src/trigger.c | 3 +++ src/update.c | 2 +- src/where.c | 6 +++++- test/returning1.test | 1 + test/window1.test | 51 ++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 84 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index db5fc66988..5e3af5f06b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\smeaningless\sJS\stest.\sAdd\sa\stimer\sto\sthe\sOPFS\sasync-side\sworker\sloader\sin\san\sattempt\sto\scatch\sa\sbrowser-specific\squirk\sin\swhich\sthe\sworker\sloading\ssilently\sfails,\sper\sdiscussion\sin/around\s[forum\spost\sa708c98dcb3ef|forum:a708c98dcb3ef]. -D 2023-03-27T13:57:08.550 +C Fix\smultiple\sproblems\swith\sRETURNING\son\sa\sDML\sstatement\sagainst\sa\sview,\nall\sinspired\sby\s[forum:/forumpost/dc3b92cfa0|forum\spost\sdc3b92cfa0].\n(1)\sDo\snot\sallow\sa\sRETURNING\sclause\sto\strick\sthe\scode\sgenerator\sinto\sthinking\nthat\sthe\sview\sbeing\supdated\shas\san\sINSTEAD\sOF\strigger.\n(2)\sGenerate\sall\sresult\scolumns\sfor\sa\sview\sin\sa\sDML\sstatement.\n(3)\sThe\sautomatic\scovering\sindex\sfor\sa\sview\sshould\scover\sall\sresult\scolumns\nof\sthe\sview. +D 2023-03-28T11:18:04.328 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -574,7 +574,7 @@ F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef -F src/delete.c 201fe0763c52783d205c8c13cdd9d55c1bd5cb21c1f036753f99103b43284b90 +F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 F src/expr.c 4cc4595f1c589b76f33f884f9c6b5f85ea80d205f1fc5ef9fb3617bf03920f0b F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 @@ -584,7 +584,7 @@ F src/hash.c c6af5f96a7a76d000f07c5402c48c318c2566beecdee9e78b9d9f60ce7119565 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 7940fce7d4aa855606432d82d20694e17c18a03956f5f5776f2404e2df7c18a8 +F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 F src/json.c 9e3a6acf9e4fa8bb6c84a38538ed9c2f9ad09f106910ef0e9cc68e19c8dc4fc8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 @@ -630,7 +630,7 @@ F src/shell.c.in d80b53a9ec0a43ac0157fc54d667cb49fb6dd2229558135530feab2fc04b909 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 8dab239c6de1d9b9e8732bbc4ad454f142b4b2781867cab0786597a391489ed4 +F src/sqliteInt.h 2a3331b16b70b0b56c65574bf3e4f749ccf493787ca6550a29dc389cad28bb34 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -689,8 +689,8 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c 1305797eab3542a0896b552c6e7669c972c1468e11e92b370533c1f37a37082b F src/treeview.c fccf3b8c517c1f55cb380c1522febe6921fcb2bd800c16c78cab571d0eb0ccbd -F src/trigger.c f34367fad4df451b5dfe63fcc1d384fd16e40077e42092b1c3682dedeef5a7e3 -F src/update.c 76664e1beae86e8e961983ebe19a4ee9ebd7e26683ead2b288ba08f81fc7ba4e +F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0 +F src/update.c 3f4fb5ad7c9b48d7911974d6579192bb3a6c27f46140b6cbb9139cc8a77b8691 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f @@ -710,7 +710,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 2bf0f0e28a7093b8f3fa661c5474993413bfac5cc55939288856952e73d9d140 +F src/where.c 2bafa1b7d1969d28135e4e7978f578a550d0a5b9005bc5a9180146a3de126880 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 9919e5a22f4b24dd96c49b8981484cbe6bbfcf466ff73ac40a06e1356aa8bf87 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -1409,7 +1409,7 @@ F test/reindex.test cd9d6021729910ece82267b4f5e1b5ac2911a7566c43b43c176a6a4732e2 F test/releasetest_data.tcl b550dd1b122a9c969df794d05ea272df535f10ff1a245062e7ba080822378016 F test/resetdb.test 54c06f18bc832ac6d6319e5ab23d5c8dd49fdbeec7c696d791682a8006bd5fc3 F test/resolver01.test f4022acafda7f4d40eca94dbf16bc5fc4ac30ceb -F test/returning1.test c78efa2829d5f44165fcc7959961b298ae005d7775f09e5542bb2815bbaee692 +F test/returning1.test db532cde29d6aebbc48c6ddc3149b30476f8e69ca7a2c4b53986c7635e6fd8ec F test/returningfault.test ae4c4b5e8745813287a359d9ccdb9d5c883c2e68afb18fb0767937d5de5692a4 F test/rollback.test 06680159bc6746d0f26276e339e3ae2f951c64812468308838e0a3362d911eaa F test/rollback2.test 3f3a4e20401825017df7e7671e9f31b6de5fae5620c2b9b49917f52f8c160a8f @@ -1901,7 +1901,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 5ad2f3e2aec3f2dc5cf4a66998d42a455b8ebd57ec173c0329f6b8788bcae0aa +F test/window1.test 71e51e4d5ed34c2c7427fcfb74056713560aa7f0e8ed49728b858893716fe772 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0bba27b78112b2b2271e498f41c437df985aa2faab302ee5b29d9b60003a8379 -R d30a83bc6e415dfd421aa91cf230833b -U stephan -Z fe1a84b137285d5b69dcaac8bd1588ba +P 4fc1904b8e18c7d41fa65490ced125f1df4f0c22c13de957b24615ed09b3ecb7 +R 5282f7ad32e4fd95877ad9bd0db98c3f +U drh +Z ebe32d8b36560ad06c065336765ee10b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1a1d620ef5..4451ccdc13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4fc1904b8e18c7d41fa65490ced125f1df4f0c22c13de957b24615ed09b3ecb7 \ No newline at end of file +c8bedef0d61731c29ae34de1594222d15b578f9e2cddbbd5b74fb3059644fe0f \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 22ef7ab658..27a554916a 100644 --- a/src/delete.c +++ b/src/delete.c @@ -114,13 +114,15 @@ static int tabIsReadOnly(Parse *pParse, Table *pTab){ ** If pTab is writable but other errors have occurred -> return 1. ** If pTab is writable and no prior errors -> return 0; */ -int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ +int sqlite3IsReadOnly(Parse *pParse, Table *pTab, Trigger *pTrigger){ if( tabIsReadOnly(pParse, pTab) ){ sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); return 1; } #ifndef SQLITE_OMIT_VIEW - if( !viewOk && IsView(pTab) ){ + if( IsView(pTab) + && (pTrigger==0 || (pTrigger->bReturning && pTrigger->pNext==0)) + ){ sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); return 1; } @@ -374,7 +376,7 @@ void sqlite3DeleteFrom( goto delete_from_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto delete_from_cleanup; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); diff --git a/src/insert.c b/src/insert.c index 3abf715c9e..423af77234 100644 --- a/src/insert.c +++ b/src/insert.c @@ -796,7 +796,7 @@ void sqlite3Insert( /* Cannot insert into a read-only table. */ - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto insert_cleanup; } @@ -1243,7 +1243,7 @@ void sqlite3Insert( } /* Copy the new data already generated. */ - assert( pTab->nNVCol>0 ); + assert( pTab->nNVCol>0 || pParse->nErr>0 ); sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1); #ifndef SQLITE_OMIT_GENERATED_COLUMNS diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d6560c51bf..dcf17d9631 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4801,7 +4801,7 @@ Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, Expr*,ExprList*,u32,Expr*); void sqlite3SelectDelete(sqlite3*, Select*); Table *sqlite3SrcListLookup(Parse*, SrcList*); -int sqlite3IsReadOnly(Parse*, Table*, int); +int sqlite3IsReadOnly(Parse*, Table*, Trigger*); void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*); diff --git a/src/trigger.c b/src/trigger.c index d179d747af..bcb2132f0b 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -1453,6 +1453,9 @@ u32 sqlite3TriggerColmask( Trigger *p; assert( isNew==1 || isNew==0 ); + if( IsView(pTab) ){ + return 0xffffffff; + } for(p=pTrigger; p; p=p->pNext){ if( p->op==op && (tr_tm&p->tr_tm) diff --git a/src/update.c b/src/update.c index bd1ddb1a30..c1f1363980 100644 --- a/src/update.c +++ b/src/update.c @@ -408,7 +408,7 @@ void sqlite3Update( if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto update_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto update_cleanup; } diff --git a/src/where.c b/src/where.c index dd2b8b73c4..509d14ff77 100644 --- a/src/where.c +++ b/src/where.c @@ -963,7 +963,11 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** original table changes and the index and table cannot both be used ** if they go out of sync. */ - extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + if( IsView(pTable) ){ + extraCols = ALLBITS; + }else{ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + } mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); diff --git a/test/returning1.test b/test/returning1.test index 5ab44f2cb7..6c098dc256 100644 --- a/test/returning1.test +++ b/test/returning1.test @@ -414,6 +414,7 @@ do_execsql_test 17.0 { do_execsql_test 18.0 { CREATE TABLE v0(c1 INT); CREATE VIEW view_2(c1) AS SELECT CASE WHEN c1 COLLATE TRUE THEN TRUE ELSE TRUE END FROM v0; + CREATE TRIGGER x1 INSTEAD OF INSERT ON view_2 BEGIN SELECT true; END; } do_catchsql_test 18.1 { INSERT INTO view_2 DEFAULT VALUES RETURNING *; diff --git a/test/window1.test b/test/window1.test index 471ac2dce8..b906a761a6 100644 --- a/test/window1.test +++ b/test/window1.test @@ -2225,5 +2225,56 @@ do_execsql_test 72.1 { WHERE ('1' IS NOT ('abcde' NOTNULL)); } {1} +# 2023-03-28 https://sqlite.org/forum/forumpost/dc3b92cfa0 (Song Liu) +# +reset_db +do_execsql_test 73.0 { + CREATE TABLE t1(a INT); + INSERT INTO t1(a) VALUES(1),(2),(4); + CREATE VIEW t2(b,c) AS SELECT * FROM t1 JOIN t1 A ORDER BY sum(0) OVER(PARTITION BY 0); + CREATE TRIGGER x1 INSTEAD OF UPDATE ON t2 BEGIN SELECT true; END; +} +do_execsql_test 73.1 { + SELECT * FROM t2; +} {1 1 1 2 1 4 2 1 2 2 2 4 4 1 4 2 4 4} +do_execsql_test 73.2 { + UPDATE t2 SET c=99 WHERE b=4 RETURNING *; +} {4 99 4 99 4 99} +do_execsql_test 73.3 { + SELECT *, nth_value(15,2) OVER() FROM t2, t1 WHERE b=4; +} { + 4 1 1 15 + 4 2 1 15 + 4 4 1 15 + 4 1 2 15 + 4 2 2 15 + 4 4 2 15 + 4 1 4 15 + 4 2 4 15 + 4 4 4 15 +} +do_execsql_test 73.4 { + UPDATE t2 SET c=nth_value(15,2) OVER() FROM (SELECT * FROM t1) WHERE b=4 RETURNING *; +} { + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 + 4 15 +} +do_execsql_test 73.5 { + DROP TRIGGER x1; +} +do_catchsql_test 73.6 { + UPDATE t2 SET c=99 WHERE b=4 RETURNING *; +} {1 {cannot modify t2 because it is a view}} +do_catchsql_test 73.7 { + UPDATE t2 SET c=nth_value(15,2) OVER() FROM (SELECT * FROM t1) WHERE b=4 RETURNING *; +} {1 {cannot modify t2 because it is a view}} + finish_test From e1be2b7ad97e774aaeeee438ceb0b2ef2a138f6d Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 28 Mar 2023 16:02:28 +0000 Subject: [PATCH 140/341] Fix a weird corner case in aggregate function processing that results from the recent addition of support for index expressions on aggregate queries. [forum:/forumpost/bad532820c|Forum post bad532820c]. FossilOrigin-Name: c34fd9fe1b76e0a5943f014f46141cbe55d41bb1e6980adf9bcb6785a03e7883 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 11 +++++++---- test/window1.test | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 5e3af5f06b..d5fb3e57d7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smultiple\sproblems\swith\sRETURNING\son\sa\sDML\sstatement\sagainst\sa\sview,\nall\sinspired\sby\s[forum:/forumpost/dc3b92cfa0|forum\spost\sdc3b92cfa0].\n(1)\sDo\snot\sallow\sa\sRETURNING\sclause\sto\strick\sthe\scode\sgenerator\sinto\sthinking\nthat\sthe\sview\sbeing\supdated\shas\san\sINSTEAD\sOF\strigger.\n(2)\sGenerate\sall\sresult\scolumns\sfor\sa\sview\sin\sa\sDML\sstatement.\n(3)\sThe\sautomatic\scovering\sindex\sfor\sa\sview\sshould\scover\sall\sresult\scolumns\nof\sthe\sview. -D 2023-03-28T11:18:04.328 +C Fix\sa\sweird\scorner\scase\sin\saggregate\sfunction\sprocessing\sthat\sresults\sfrom\sthe\nrecent\saddition\sof\ssupport\sfor\sindex\sexpressions\son\saggregate\squeries.\n[forum:/forumpost/bad532820c|Forum\spost\sbad532820c]. +D 2023-03-28T16:02:28.157 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 4cc4595f1c589b76f33f884f9c6b5f85ea80d205f1fc5ef9fb3617bf03920f0b +F src/expr.c 98ba98271bbe55661d8630cc60083344b2c1713553e46b69afd7ff9f85a30e29 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -1901,7 +1901,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 71e51e4d5ed34c2c7427fcfb74056713560aa7f0e8ed49728b858893716fe772 +F test/window1.test 67aaef6e01e8d8fb3f89aa440e8c8045dbfd3c6f21628a6d7bad2b786c310d9b F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4fc1904b8e18c7d41fa65490ced125f1df4f0c22c13de957b24615ed09b3ecb7 -R 5282f7ad32e4fd95877ad9bd0db98c3f +P c8bedef0d61731c29ae34de1594222d15b578f9e2cddbbd5b74fb3059644fe0f +R 68fb0cae877919b0f69109342d216ada U drh -Z ebe32d8b36560ad06c065336765ee10b +Z f12fcdd961dcea42a705ad864ed7007e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4451ccdc13..a6c7d49117 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8bedef0d61731c29ae34de1594222d15b578f9e2cddbbd5b74fb3059644fe0f \ No newline at end of file +c34fd9fe1b76e0a5943f014f46141cbe55d41bb1e6980adf9bcb6785a03e7883 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 564dfc7fbb..80bfbadc33 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6286,9 +6286,11 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ int iAgg = pExpr->iAgg; Parse *pParse = pWalker->pParse; sqlite3 *db = pParse->db; + assert( iAgg>=0 ); if( pExpr->op!=TK_AGG_FUNCTION ){ - assert( iAgg>=0 && iAggnColumn ); - if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){ + if( iAggnColumn + && pAggInfo->aCol[iAgg].pCExpr==pExpr + ){ pExpr = sqlite3ExprDup(db, pExpr, 0); if( pExpr ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; @@ -6297,8 +6299,9 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ } }else{ assert( pExpr->op==TK_AGG_FUNCTION ); - assert( iAgg>=0 && iAggnFunc ); - if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ + if( ALWAYS(iAggnFunc) + && pAggInfo->aFunc[iAgg].pFExpr==pExpr + ){ pExpr = sqlite3ExprDup(db, pExpr, 0); if( pExpr ){ pAggInfo->aFunc[iAgg].pFExpr = pExpr; diff --git a/test/window1.test b/test/window1.test index b906a761a6..59a933c4f6 100644 --- a/test/window1.test +++ b/test/window1.test @@ -2276,5 +2276,44 @@ do_catchsql_test 73.7 { UPDATE t2 SET c=nth_value(15,2) OVER() FROM (SELECT * FROM t1) WHERE b=4 RETURNING *; } {1 {cannot modify t2 because it is a view}} +# 2023-03-28 https://sqlite.org/forum/forumpost/bad532820c +# +reset_db +do_execsql_test 74.0 { + CREATE TABLE t1 (a INT, b INT); + CREATE TABLE t2 (c INT, d INT); + CREATE INDEX idx ON t1(abs(a)); + INSERT INTO t1 VALUES(1,2),(3,4); + INSERT INTO t2 VALUES(5,6),(7,8); +} +do_execsql_test 74.1 { + SELECT ( + SELECT count( a ) FROM t2 LIMIT 1 + ) + FROM t1; +} {2} ;# Verified using PG 14.2 +do_execsql_test 74.2 { + SELECT ( + SELECT count( a+c ) FROM t2 LIMIT 1 + ) + FROM t1; +} {2 2} ;# verified on PG 14.2. Crashes PG 9.6! +do_execsql_test 74.3 { + SELECT ( + SELECT count( ( SELECT(sum(0) OVER(ORDER BY c, abs(a))) ) ) + FROM t2 GROUP BY c LIMIT 1 + ) + FROM t1; +} {1 1} ;# verified on PG 14.2 +do_execsql_test 74.4 { + /* Original test case reported in https://sqlite.org/forum/forumpost/bad532820c + CREATE TABLE v0 (c1); + CREATE INDEX i ON v0 (c1, c1=1); + SELECT 0 FROM v0 AS a1 + WHERE (SELECT count((SELECT(sum(0) OVER(PARTITION BY(c1), (a1.c1=1) )))) + FROM v0 + GROUP BY hex(0)) + AND a1.c1=0; +} {} finish_test From df542e0500fadd22b04cc839264aaa146049265b Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 29 Mar 2023 11:36:24 +0000 Subject: [PATCH 141/341] Enhance PRAGMA integrity_check so that it can detect that a NOT NULL column contains a NaN value and report that as an error. dbsqlfuzz f144b642fe6f1a1c196f258ac6e60118a0cb59b2. FossilOrigin-Name: 7638d9755dc90fd353b874d03ed418fa8aaee4440290ff69b1b552eae84e5baa --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pragma.c | 16 +++++++++++++++- src/vdbe.c | 6 ++++++ 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d5fb3e57d7..6c086cad58 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sweird\scorner\scase\sin\saggregate\sfunction\sprocessing\sthat\sresults\sfrom\sthe\nrecent\saddition\sof\ssupport\sfor\sindex\sexpressions\son\saggregate\squeries.\n[forum:/forumpost/bad532820c|Forum\spost\sbad532820c]. -D 2023-03-28T16:02:28.157 +C Enhance\sPRAGMA\sintegrity_check\sso\sthat\sit\scan\sdetect\sthat\sa\sNOT\sNULL\scolumn\ncontains\sa\sNaN\svalue\sand\sreport\sthat\sas\san\serror.\ndbsqlfuzz\sf144b642fe6f1a1c196f258ac6e60118a0cb59b2. +D 2023-03-29T11:36:24.481 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -618,7 +618,7 @@ F src/parse.y 424e49ed8fc6c907920db9be5a13a75ed43811e1ea8dd21b0fa9ef97f083dc6b F src/pcache.c 842410539b544e12d5fccfcf29890782f46a58f227a77bc0bd76243799662c0c F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc -F src/pragma.c 367652f5374b2f17761c96b262d9534d89bf089572a1ebba70d426a138c6802a +F src/pragma.c 26ed2cfdc5c12aa1c707178635709684960288cacc9cff9d491a38ff10e395f1 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 @@ -695,7 +695,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 2bb4694bff3c1c4f52fd47e66be30278b5251d62d23b768af7c3b3a0dd7ba901 +F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8bedef0d61731c29ae34de1594222d15b578f9e2cddbbd5b74fb3059644fe0f -R 68fb0cae877919b0f69109342d216ada +P c34fd9fe1b76e0a5943f014f46141cbe55d41bb1e6980adf9bcb6785a03e7883 +R df22a94c4699a51ccc0a1666db70f0f4 U drh -Z f12fcdd961dcea42a705ad864ed7007e +Z 05ab517810b9c5736ce209544eca4198 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a6c7d49117..3bb06482ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c34fd9fe1b76e0a5943f014f46141cbe55d41bb1e6980adf9bcb6785a03e7883 \ No newline at end of file +7638d9755dc90fd353b874d03ed418fa8aaee4440290ff69b1b552eae84e5baa \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 4138752455..68a1531853 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1880,15 +1880,29 @@ void sqlite3Pragma( labelOk = sqlite3VdbeMakeLabel(pParse); if( pCol->notNull ){ /* (1) NOT NULL columns may not contain a NULL */ + int jmp3; int jmp2 = sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); - sqlite3VdbeChangeP5(v, 0x0f); VdbeCoverage(v); + if( p1<0 ){ + sqlite3VdbeChangeP5(v, 0x0f); /* INT, REAL, TEXT, or BLOB */ + jmp3 = jmp2; + }else{ + sqlite3VdbeChangeP5(v, 0x0d); /* INT, TEXT, or BLOB */ + /* OP_IsType does not detect NaN values in the database file + ** which should be treated as a NULL. So if the header type + ** is REAL, we have to load the actual data using OP_Column + ** to reliably determine if the value is a NULL. */ + sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); + VdbeCoverage(v); + } zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pCol->zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); if( doTypeCheck ){ sqlite3VdbeGoto(v, labelError); sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeJumpHere(v, jmp3); }else{ /* VDBE byte code will fall thru */ } diff --git a/src/vdbe.c b/src/vdbe.c index 4c66d555f3..ba726a7954 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2626,6 +2626,12 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ ** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04. ** SQLITE_BLOB is 0x08. SQLITE_NULL is 0x10. ** +** WARNING: This opcode does not reliably distinguish between NULL and REAL +** when P1>=0. If the database contains a NaN value, this opcode will think +** that the datatype is REAL when it should be NULL. When P1<0 and the value +** is already stored in register P3, then this opcode does reliably +** distinguish between NULL and REAL. The problem only arises then P1>=0. +** ** Take the jump to address P2 if and only if the datatype of the ** value determined by P1 and P3 corresponds to one of the bits in the ** P5 bitmask. From a9fd5f224c72013062a8b34781396d3832c8f05a Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 29 Mar 2023 14:42:11 +0000 Subject: [PATCH 142/341] New #ifdefs to omit code that is unused except under STAT4. FossilOrigin-Name: 09a9b30ba7828a487a33a3ebf8028dfaa147dff67d2724584123b90a88d9814b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 4 ++++ src/expr.c | 2 ++ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6c086cad58..5ea5da352e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sPRAGMA\sintegrity_check\sso\sthat\sit\scan\sdetect\sthat\sa\sNOT\sNULL\scolumn\ncontains\sa\sNaN\svalue\sand\sreport\sthat\sas\san\serror.\ndbsqlfuzz\sf144b642fe6f1a1c196f258ac6e60118a0cb59b2. -D 2023-03-29T11:36:24.481 +C New\s#ifdefs\sto\somit\scode\sthat\sis\sunused\sexcept\sunder\sSTAT4. +D 2023-03-29T14:42:11.580 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -558,7 +558,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 3ca2f449c890f8b86ec9e06f0c4fccf0648941c3308a16904cb2852227db83f7 -F src/analyze.c 8230fc1f3ddcf5492d258f28afcd36026c0de2f35cbbee22a41eed4baaf26148 +F src/analyze.c 01bfd40026632eaae1d93212b684f539c6674cb573535dc90199674cbf7e0cdc F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 98ba98271bbe55661d8630cc60083344b2c1713553e46b69afd7ff9f85a30e29 +F src/expr.c bf74dd13f36dfc57c2a397f3d41cf9ea35ffd17f9baed1db51bd8736b276d4b1 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c34fd9fe1b76e0a5943f014f46141cbe55d41bb1e6980adf9bcb6785a03e7883 -R df22a94c4699a51ccc0a1666db70f0f4 +P 7638d9755dc90fd353b874d03ed418fa8aaee4440290ff69b1b552eae84e5baa +R b188963cb85e43486729f0b1a856004f U drh -Z 05ab517810b9c5736ce209544eca4198 +Z 5386f27a21d49b16511eaa8ce902e6c0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3bb06482ac..1c60389dcf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7638d9755dc90fd353b874d03ed418fa8aaee4440290ff69b1b552eae84e5baa \ No newline at end of file +09a9b30ba7828a487a33a3ebf8028dfaa147dff67d2724584123b90a88d9814b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index b6b59a6b79..6e9d72464c 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1389,7 +1389,11 @@ static void analyzeDatabase(Parse *pParse, int iDb){ for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); +#ifdef SQLITE_ENABLE_STAT4 iMem = sqlite3FirstAvailableRegister(pParse, iMem); +#else + assert( iMem==sqlite3FirstAvailableRegister(pParse,iMem) ); +#endif } loadAnalysis(pParse, iDb); } diff --git a/src/expr.c b/src/expr.c index 80bfbadc33..c17eb0f7ef 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6646,6 +6646,7 @@ void sqlite3TouchRegister(Parse *pParse, int iReg){ if( pParse->nMemnMem = iReg; } +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) /* ** Return the latest reusable register in the set of all registers. ** The value returned is no less than iMin. If any register iMin or @@ -6666,6 +6667,7 @@ int sqlite3FirstAvailableRegister(Parse *pParse, int iMin){ pParse->nRangeReg = 0; return iMin; } +#endif /* SQLITE_ENABLE_STAT4 || SQLITE_DEBUG */ /* ** Validate that no temporary register falls within the range of From 0dc43fd4c8578baa002acd47a7043869d01929e9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 29 Mar 2023 15:16:29 +0000 Subject: [PATCH 143/341] Another #ifdef to omit code that is only used by STAT4. FossilOrigin-Name: 445c75567de9bf9c8075dfc62c60dc4abeefab59c7803a5ef2cc5d69c08d7633 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5ea5da352e..55d58b9c3b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\s#ifdefs\sto\somit\scode\sthat\sis\sunused\sexcept\sunder\sSTAT4. -D 2023-03-29T14:42:11.580 +C Another\s#ifdef\sto\somit\scode\sthat\sis\sonly\sused\sby\sSTAT4. +D 2023-03-29T15:16:29.589 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -701,7 +701,7 @@ F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 F src/vdbeaux.c 0f5201346a83a35a08e833c3a03abe626119c07b7361c28bc6a259b98540f1d6 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd -F src/vdbemem.c cbd2305d6444dc0aa3ab0dbe8f45ca5166da6f5a9bb6402be9739b6443a83fd0 +F src/vdbemem.c db0458d11a51f6cfad2333a41e36a3795be0b2f316d070df5d33543a9ac884ac F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7638d9755dc90fd353b874d03ed418fa8aaee4440290ff69b1b552eae84e5baa -R b188963cb85e43486729f0b1a856004f +P 09a9b30ba7828a487a33a3ebf8028dfaa147dff67d2724584123b90a88d9814b +R 0b2f665677a5709c67b05890c7e30712 U drh -Z 5386f27a21d49b16511eaa8ce902e6c0 +Z 32665a8d3990ec1291de16f5a821ea1c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1c60389dcf..9f5b85ecaf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09a9b30ba7828a487a33a3ebf8028dfaa147dff67d2724584123b90a88d9814b \ No newline at end of file +445c75567de9bf9c8075dfc62c60dc4abeefab59c7803a5ef2cc5d69c08d7633 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 1b8808ea6f..58621de89d 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1587,7 +1587,13 @@ static int valueFromExpr( rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); testcase( rc!=SQLITE_OK ); if( *ppVal ){ +#ifdef SQLITE_ENABLE_STAT4 rc = ExpandBlob(*ppVal); +#else + /* zero-blobs only come from functions, not literal values. And + ** functions are only processed under STAT4 */ + assert( (ppVal[0][0].flags & MEM_Zero)==0 ); +#endif sqlite3VdbeMemCast(*ppVal, aff, enc); sqlite3ValueApplyAffinity(*ppVal, affinity, enc); } From 7b56e978668283abc19d8f1bb00044f10405d92d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 29 Mar 2023 18:54:01 +0000 Subject: [PATCH 144/341] Fix a problem with sqlite3_stmt_scanstatus() from within an SQLITE_TRACE_STMT callback made from within a trigger. FossilOrigin-Name: 1fa78fafa1340de458546526b03cf8b3e9c823913c4225d7c747ad182df5c0fc --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbeapi.c | 31 ++++++++++++++++++++----------- test/scanstatus2.test | 28 ++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 55d58b9c3b..a7828912a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\s#ifdef\sto\somit\scode\sthat\sis\sonly\sused\sby\sSTAT4. -D 2023-03-29T15:16:29.589 +C Fix\sa\sproblem\swith\ssqlite3_stmt_scanstatus()\sfrom\swithin\san\sSQLITE_TRACE_STMT\scallback\smade\sfrom\swithin\sa\strigger. +D 2023-03-29T18:54:01.311 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -698,7 +698,7 @@ F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb -F src/vdbeapi.c 40c47b1528d308a322203de21d2e0d711753257ed9771771b6129214b1d65932 +F src/vdbeapi.c 998d240b3afd78df3581d545240330ca4cc09ecd10d0af730fe364f1c7eb253d F src/vdbeaux.c 0f5201346a83a35a08e833c3a03abe626119c07b7361c28bc6a259b98540f1d6 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c db0458d11a51f6cfad2333a41e36a3795be0b2f316d070df5d33543a9ac884ac @@ -1440,7 +1440,7 @@ F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 F test/scanstatus.test b249328caf4d317e71058006872b8012598a5fa045b30bf24a81eeff650ab49e -F test/scanstatus2.test b77de449be6ffd7a913a06dd84276dc6eb3517d479126e00ac51cd409c0a0af5 +F test/scanstatus2.test 9a00becb1d60d168944f8e2f3585d44d46123aa95c5c7c25563309e1f15f924d F test/schema.test 5dd11c96ba64744de955315d2e4f8992e447533690153b93377dffb2a5ef5431 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 8ed4ae66e082cdd8b1b1f22d8549e1e7a0db4527a8e6ee8b6193053ee1e5c9ce @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 09a9b30ba7828a487a33a3ebf8028dfaa147dff67d2724584123b90a88d9814b -R 0b2f665677a5709c67b05890c7e30712 -U drh -Z 32665a8d3990ec1291de16f5a821ea1c +P 445c75567de9bf9c8075dfc62c60dc4abeefab59c7803a5ef2cc5d69c08d7633 +R cb2f6677b12ee5e73b1a3763b21cb5a6 +U dan +Z 498d074832c5920d3093566d1bedac02 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9f5b85ecaf..a1d2644d92 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -445c75567de9bf9c8075dfc62c60dc4abeefab59c7803a5ef2cc5d69c08d7633 \ No newline at end of file +1fa78fafa1340de458546526b03cf8b3e9c823913c4225d7c747ad182df5c0fc \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 476b6a2adf..42f3bb0b40 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2138,15 +2138,24 @@ int sqlite3_stmt_scanstatus_v2( void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; + VdbeOp *aOp = p->aOp; + int nOp = p->nOp; ScanStatus *pScan; int idx; + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + aOp = pFrame->aOp; + nOp = pFrame->nOp; + } + if( iScan<0 ){ int ii; if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ i64 res = 0; - for(ii=0; iinOp; ii++){ - res += p->aOp[ii].nCycle; + for(ii=0; iiaddrLoop>0 ){ - *(sqlite3_int64*)pOut = p->aOp[pScan->addrLoop].nExec; + *(sqlite3_int64*)pOut = aOp[pScan->addrLoop].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -2180,7 +2189,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_NVISIT: { if( pScan->addrVisit>0 ){ - *(sqlite3_int64*)pOut = p->aOp[pScan->addrVisit].nExec; + *(sqlite3_int64*)pOut = aOp[pScan->addrVisit].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -2202,7 +2211,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_EXPLAIN: { if( pScan->addrExplain ){ - *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + *(const char**)pOut = aOp[ pScan->addrExplain ].p4.z; }else{ *(const char**)pOut = 0; } @@ -2210,7 +2219,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_SELECTID: { if( pScan->addrExplain ){ - *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + *(int*)pOut = aOp[ pScan->addrExplain ].p1; }else{ *(int*)pOut = -1; } @@ -2218,7 +2227,7 @@ int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_PARENTID: { if( pScan->addrExplain ){ - *(int*)pOut = p->aOp[ pScan->addrExplain ].p2; + *(int*)pOut = aOp[ pScan->addrExplain ].p2; }else{ *(int*)pOut = -1; } @@ -2236,18 +2245,18 @@ int sqlite3_stmt_scanstatus_v2( if( iIns==0 ) break; if( iIns>0 ){ while( iIns<=iEnd ){ - res += p->aOp[iIns].nCycle; + res += aOp[iIns].nCycle; iIns++; } }else{ int iOp; - for(iOp=0; iOpnOp; iOp++){ - Op *pOp = &p->aOp[iOp]; + for(iOp=0; iOpp1!=iEnd ) continue; if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ continue; } - res += p->aOp[iOp].nCycle; + res += aOp[iOp].nCycle; } } } diff --git a/test/scanstatus2.test b/test/scanstatus2.test index 042128ae14..fb62af2103 100644 --- a/test/scanstatus2.test +++ b/test/scanstatus2.test @@ -242,6 +242,34 @@ QUERY (nCycle=nnn) --SEARCH v1 USING AUTOMATIC COVERING INDEX (x1=?) (nCycle=nnn) } +#------------------------------------------------------------------------- +reset_db + +ifcapable trace { + do_execsql_test 5.0 { + CREATE TABLE t1(x, y); + CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN + SELECT 1; + END; + INSERT INTO t1 VALUES(1, 2); + } + + proc trace {stmt sql} { + array set A [sqlite3_stmt_scanstatus -flags complex [format %x $stmt] 0] + lappend ::trace_explain $A(zExplain) + } + db trace_v2 trace + + set ::trace_explain [list] + do_execsql_test 5.1 { + DELETE FROM t1 WHERE x=1; + } + + do_test 5.2 { + set ::trace_explain + } {{SCAN t1} {SCAN t1} {SCAN t1}} +} + finish_test From e1afa2bd24288db5f7a6518426e877db6f69bb61 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 29 Mar 2023 21:58:06 +0000 Subject: [PATCH 145/341] Avoid having OP_SeekScan jump over an OP_IdxGT or OP_IdxGE that follows the OP_SeekGE opcode. Fix for [b50528af4468237c]. FossilOrigin-Name: c9c4f287652933eb6262a3419efe7e7288f55d3db7e2ac032eeae025f665d306 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wherecode.c | 13 ++----------- test/seekscan1.test | 5 +++++ 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index a7828912a9..5901ddde84 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\ssqlite3_stmt_scanstatus()\sfrom\swithin\san\sSQLITE_TRACE_STMT\scallback\smade\sfrom\swithin\sa\strigger. -D 2023-03-29T18:54:01.311 +C Avoid\shaving\sOP_SeekScan\sjump\sover\san\sOP_IdxGT\sor\sOP_IdxGE\sthat\sfollows\sthe\sOP_SeekGE\sopcode.\sFix\sfor\s[b50528af4468237c]. +D 2023-03-29T21:58:06.379 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -712,7 +712,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 2bafa1b7d1969d28135e4e7978f578a550d0a5b9005bc5a9180146a3de126880 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c 9919e5a22f4b24dd96c49b8981484cbe6bbfcf466ff73ac40a06e1356aa8bf87 +F src/wherecode.c 76996676015ee5ad228e8dc4d2631c7e397e912865504f3667fc5a01c2a9c866 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 F src/window.c 76a27cff9ea2ded0c2c3527187029259440fabcc4cc4c07b11d942c78494a614 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1450,7 +1450,7 @@ F test/schema6.test e4bd1f23d368695eb9e7b51ef6e02ca0642ea2ab4a52579959826b5e7dce F test/schemafault.test 1936bceca55ac82c5efbcc9fc91a1933e45c8d1e1d106b9a7e56c972a5a2a51e F test/securedel.test 2f70b2449186a1921bd01ec9da407fbfa98c3a7a5521854c300c194b2ff09384 F test/securedel2.test 2d54c28e46eb1fd6902089958b20b1b056c6f1c5 -F test/seekscan1.test d79c97de5bb1dd1fd466687f3014add514fddf8248c57baf51d749c7dfd573d8 +F test/seekscan1.test 31af16e3bb3203d153aea320939c5da97ec44705c2710d153c06a01397d45b09 F test/select1.test 692e84cfa29c405854c69e8a4027183d64c22952866a123fabbce741a379e889 F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56 F test/select3.test 8d04b66df7475275a65f7e4a786d6a724c30bd9929f8ae5bd59c8d3d6e75e6cd @@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 445c75567de9bf9c8075dfc62c60dc4abeefab59c7803a5ef2cc5d69c08d7633 -R cb2f6677b12ee5e73b1a3763b21cb5a6 +P 1fa78fafa1340de458546526b03cf8b3e9c823913c4225d7c747ad182df5c0fc +R 9fb7ab19ce27b13f1101f7c5ac33fc82 U dan -Z 498d074832c5920d3093566d1bedac02 +Z be61addd94b8ce083f9ab537ba38c397 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a1d2644d92..44db875821 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1fa78fafa1340de458546526b03cf8b3e9c823913c4225d7c747ad182df5c0fc \ No newline at end of file +c9c4f287652933eb6262a3419efe7e7288f55d3db7e2ac032eeae025f665d306 \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index f74273cf18..ac93265990 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1927,7 +1927,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** guess. */ addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, (pIdx->aiRowLogEst[0]+9)/10); - if( pRangeStart ){ + if( pRangeStart || pRangeEnd ){ sqlite3VdbeChangeP5(v, 1); sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1); addrSeekScan = 0; @@ -1968,16 +1968,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->p2==0 ); if( pRangeEnd ){ Expr *pRight = pRangeEnd->pExpr->pRight; - if( addrSeekScan ){ - /* For a seek-scan that has a range on the lowest term of the index, - ** we have to make the top of the loop be code that sets the end - ** condition of the range. Otherwise, the OP_SeekScan might jump - ** over that initialization, leaving the range-end value set to the - ** range-start value, resulting in a wrong answer. - ** See ticket 5981a8c041a3c2f3 (2021-11-02). - */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - } + assert( addrSeekScan==0 ); codeExprOrVector(pParse, pRight, regBase+nEq, nTop); whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); if( (pRangeEnd->wtFlags & TERM_VNULL)==0 diff --git a/test/seekscan1.test b/test/seekscan1.test index e68d71449d..0e53c19166 100644 --- a/test/seekscan1.test +++ b/test/seekscan1.test @@ -24,6 +24,7 @@ do_execsql_test 1.0 { ANALYZE; } + do_execsql_test 1.1 { SELECT a,b,c FROM t1 WHERE b IN (234, 345) AND c BETWEEN 6 AND 6.5 AND a='abc' @@ -59,5 +60,9 @@ do_execsql_test 1.4 { abc 345 7 } +do_execsql_test 1.5 { + SELECT a,b,c FROM t1 WHERE b IN (235, 345) AND c<=3 AND a='abc' ORDER BY a, b; +} + finish_test From 8fe84a40c77cde3d7c790f864c4b8d1ae4a30899 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 30 Mar 2023 11:05:36 +0000 Subject: [PATCH 146/341] Fix a crash that could follow an OOM error while processing aggregate functions. FossilOrigin-Name: 804435a2731bd3c26278c47098854b9ee7727a686587f6208e793738fbfc0555 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/expr.c | 1 + test/aggfault.test | 43 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 test/aggfault.test diff --git a/manifest b/manifest index 5901ddde84..1cee4cb5e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\shaving\sOP_SeekScan\sjump\sover\san\sOP_IdxGT\sor\sOP_IdxGE\sthat\sfollows\sthe\sOP_SeekGE\sopcode.\sFix\sfor\s[b50528af4468237c]. -D 2023-03-29T21:58:06.379 +C Fix\sa\scrash\sthat\scould\sfollow\san\sOOM\serror\swhile\sprocessing\saggregate\sfunctions. +D 2023-03-30T11:05:36.177 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c bf74dd13f36dfc57c2a397f3d41cf9ea35ffd17f9baed1db51bd8736b276d4b1 +F src/expr.c e35eb0f324645e172cab367403e17701332a21f7af0f2ef6d0a6de08fb1f4fbe F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -719,6 +719,7 @@ F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 +F test/aggfault.test 0643b110cfec4e7f188087ae481f5538905096281694e4ba910beb1ec226d5fe F test/aggnested.test 7269d07ac879fce161cb26c8fabe65cba5715742fac8a1fccac570dcdaf28f00 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13 @@ -2051,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1fa78fafa1340de458546526b03cf8b3e9c823913c4225d7c747ad182df5c0fc -R 9fb7ab19ce27b13f1101f7c5ac33fc82 +P c9c4f287652933eb6262a3419efe7e7288f55d3db7e2ac032eeae025f665d306 +R e49172473ffe9cb42606ee09290143f6 U dan -Z be61addd94b8ce083f9ab537ba38c397 +Z 6c3e374e9fe9a2c79df9bbcd4176cd8f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 44db875821..d87987b6e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c9c4f287652933eb6262a3419efe7e7288f55d3db7e2ac032eeae025f665d306 \ No newline at end of file +804435a2731bd3c26278c47098854b9ee7727a686587f6208e793738fbfc0555 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c17eb0f7ef..e5a16ca77e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6462,6 +6462,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ tmp.iTable = pIEpr->iIdxCur; tmp.iColumn = pIEpr->iIdxCol; findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + if( pParse->nErr ) return WRC_Abort; pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; pExpr->pAggInfo = pAggInfo; pExpr->iAgg = tmp.iAgg; diff --git a/test/aggfault.test b/test/aggfault.test new file mode 100644 index 0000000000..3b9d957bf7 --- /dev/null +++ b/test/aggfault.test @@ -0,0 +1,43 @@ +# 2023 March 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix aggfault + + +do_execsql_test 1 { + CREATE TABLE t1(x); + CREATE TABLE t1x ON t1(x, x=0); +} +faultsim_save_and_close + +do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { SELECT * FROM sqlite_schema } +} -body { + execsql { + SELECT * FROM t1 AS a1 WHERE ( + SELECT count(x AND 0=a1.x) FROM t1 GROUP BY abs(1) + ) AND x=( + SELECT * FROM t1 AS a1 + WHERE (SELECT count(x IS 1 AND a1.x=0) + FROM t1 + GROUP BY abs(1)) AND x=0 + ); + } +} -test { + faultsim_test_result {0 {}} +} + + +finish_test From 53ed3830753dec80ad9bd4c7f372efe969f152a0 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 30 Mar 2023 12:19:38 +0000 Subject: [PATCH 147/341] Fix an error in new test script test/aggfault.test. FossilOrigin-Name: 8724fe7426da55d19dba7b30e09321ba30c73286513864cb05de32f72e50ee31 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/aggfault.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1cee4cb5e1..a9320129a3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scrash\sthat\scould\sfollow\san\sOOM\serror\swhile\sprocessing\saggregate\sfunctions. -D 2023-03-30T11:05:36.177 +C Fix\san\serror\sin\snew\stest\sscript\stest/aggfault.test. +D 2023-03-30T12:19:38.983 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -719,7 +719,7 @@ F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 -F test/aggfault.test 0643b110cfec4e7f188087ae481f5538905096281694e4ba910beb1ec226d5fe +F test/aggfault.test 777f269d0da5b0c2524c7ff6d99ae9a93db4f1b1839a914dd2a12e3035c29829 F test/aggnested.test 7269d07ac879fce161cb26c8fabe65cba5715742fac8a1fccac570dcdaf28f00 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 2ecb8bbd52416642e41c9081182a8df05d42c75637afd4488aace78cc4b69e13 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c9c4f287652933eb6262a3419efe7e7288f55d3db7e2ac032eeae025f665d306 -R e49172473ffe9cb42606ee09290143f6 +P 804435a2731bd3c26278c47098854b9ee7727a686587f6208e793738fbfc0555 +R b296eb533e950e069bba30b8ded554ba U dan -Z 6c3e374e9fe9a2c79df9bbcd4176cd8f +Z 3a1b614658fa77d50aaf0b29cb8c8a77 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d87987b6e5..e53f3ad814 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -804435a2731bd3c26278c47098854b9ee7727a686587f6208e793738fbfc0555 \ No newline at end of file +8724fe7426da55d19dba7b30e09321ba30c73286513864cb05de32f72e50ee31 \ No newline at end of file diff --git a/test/aggfault.test b/test/aggfault.test index 3b9d957bf7..7c16b039ce 100644 --- a/test/aggfault.test +++ b/test/aggfault.test @@ -17,7 +17,7 @@ set testprefix aggfault do_execsql_test 1 { CREATE TABLE t1(x); - CREATE TABLE t1x ON t1(x, x=0); + CREATE INDEX t1x ON t1(x, x=0); } faultsim_save_and_close From 09db37c022187873bc496c9f7e6630d4511560ca Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Mar 2023 16:08:54 +0000 Subject: [PATCH 148/341] Omit a branch that is no longer needed following [c9c4f287652933eb]. FossilOrigin-Name: 960a488a2db92f5437bc8171cdbed44618d9381c57fd41b2f0e357b6012c80b5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wherecode.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a9320129a3..3fada7c74c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\snew\stest\sscript\stest/aggfault.test. -D 2023-03-30T12:19:38.983 +C Omit\sa\sbranch\sthat\sis\sno\slonger\sneeded\sfollowing\s[c9c4f287652933eb]. +D 2023-03-30T16:08:54.466 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -712,7 +712,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c 2bafa1b7d1969d28135e4e7978f578a550d0a5b9005bc5a9180146a3de126880 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c 76996676015ee5ad228e8dc4d2631c7e397e912865504f3667fc5a01c2a9c866 +F src/wherecode.c 41c17b089082558c1c5496e8a453b1f9a96485b6b355f318440fca2ee4754b2e F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 F src/window.c 76a27cff9ea2ded0c2c3527187029259440fabcc4cc4c07b11d942c78494a614 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 804435a2731bd3c26278c47098854b9ee7727a686587f6208e793738fbfc0555 -R b296eb533e950e069bba30b8ded554ba -U dan -Z 3a1b614658fa77d50aaf0b29cb8c8a77 +P 8724fe7426da55d19dba7b30e09321ba30c73286513864cb05de32f72e50ee31 +R 1624b0eb89a914d1ae752ba25df52cd1 +U drh +Z 7c95356d2ffe255785c95c9a7696e988 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e53f3ad814..1d2da30e59 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8724fe7426da55d19dba7b30e09321ba30c73286513864cb05de32f72e50ee31 \ No newline at end of file +960a488a2db92f5437bc8171cdbed44618d9381c57fd41b2f0e357b6012c80b5 \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index ac93265990..abb582fb7b 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -2002,7 +2002,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff); /* Top of the loop body */ - if( pLevel->p2==0 ) pLevel->p2 = sqlite3VdbeCurrentAddr(v); + pLevel->p2 = sqlite3VdbeCurrentAddr(v); /* Check if the index cursor is past the end of the range. */ if( nConstraint ){ From 567b069e427b7fea8fedfe5fc5efdd74fbece03c Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 30 Mar 2023 19:05:48 +0000 Subject: [PATCH 149/341] Earlier error detection for index expression usage by aggregate functions. dbsqlfuzz 29214ace4e25c98d2ddff8fbcf97afdda23f28b9 FossilOrigin-Name: 8e841e7f025f7205959453875f2d9db36271642045593970a2b2fc20b2f847c3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3fada7c74c..0dac7c6ea6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sa\sbranch\sthat\sis\sno\slonger\sneeded\sfollowing\s[c9c4f287652933eb]. -D 2023-03-30T16:08:54.466 +C Earlier\serror\sdetection\sfor\sindex\sexpression\susage\sby\saggregate\sfunctions.\ndbsqlfuzz\s29214ace4e25c98d2ddff8fbcf97afdda23f28b9 +D 2023-03-30T19:05:48.274 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c e35eb0f324645e172cab367403e17701332a21f7af0f2ef6d0a6de08fb1f4fbe +F src/expr.c 5f2b5f25a90aa4ff1c84d175b1adf790be8f8719f3b38851e2b5b293e0e5e916 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8724fe7426da55d19dba7b30e09321ba30c73286513864cb05de32f72e50ee31 -R 1624b0eb89a914d1ae752ba25df52cd1 +P 960a488a2db92f5437bc8171cdbed44618d9381c57fd41b2f0e357b6012c80b5 +R 4f561ce7d94ebba6771f71291846cad2 U drh -Z 7c95356d2ffe255785c95c9a7696e988 +Z 91fc03feb4d1edcaf09570adbf0b39df # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1d2da30e59..17864e80e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -960a488a2db92f5437bc8171cdbed44618d9381c57fd41b2f0e357b6012c80b5 \ No newline at end of file +8e841e7f025f7205959453875f2d9db36271642045593970a2b2fc20b2f847c3 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index e5a16ca77e..4b536a8435 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6452,6 +6452,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ if( pIEpr==0 ) break; if( NEVER(!ExprUseYTab(pExpr)) ) break; if( pExpr->pAggInfo!=0 ) break; /* Already resolved by outer context */ + if( pParse->nErr ){ return WRC_Abort; } /* If we reach this point, it means that expression pExpr can be ** translated into a reference to an index column as described by @@ -6462,7 +6463,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ tmp.iTable = pIEpr->iIdxCur; tmp.iColumn = pIEpr->iIdxCol; findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); - if( pParse->nErr ) return WRC_Abort; + if( pParse->nErr ){ return WRC_Abort; } + assert( pAggInfo->aCol!=0 ); + assert( tmp.iAggnColumn ); pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; pExpr->pAggInfo = pAggInfo; pExpr->iAgg = tmp.iAgg; From 1839b7990dceafef0caad098a0c881d7a64ba307 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 31 Mar 2023 23:48:59 +0000 Subject: [PATCH 150/341] Omit the the count-of-view optimization if there is a HAVING clause. dbsqlfuzz 6a107e3055bd22afab31cfddabc2d9d54fcbaf69 FossilOrigin-Name: babe2b5e59647ac9db4601e67c25190aac14eb76d5fcb9fa5b3692b955fefd61 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 4 +++- test/countofview.test | 28 ++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0dac7c6ea6..83b46dac1b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Earlier\serror\sdetection\sfor\sindex\sexpression\susage\sby\saggregate\sfunctions.\ndbsqlfuzz\s29214ace4e25c98d2ddff8fbcf97afdda23f28b9 -D 2023-03-30T19:05:48.274 +C Omit\sthe\sthe\scount-of-view\soptimization\sif\sthere\sis\sa\sHAVING\sclause.\ndbsqlfuzz\s6a107e3055bd22afab31cfddabc2d9d54fcbaf69 +D 2023-03-31T23:48:59.808 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -625,7 +625,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 83812448cf5a645f37d79c43fa92d38d7a9f7790d95f7ac09ac02b4064810bd2 +F src/select.c 2fef08c06348c9b490f047926d322b0d58e556dae658f1dac14b9e4b9410fd46 F src/shell.c.in d80b53a9ec0a43ac0157fc54d667cb49fb6dd2229558135530feab2fc04b909d F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -896,7 +896,7 @@ F test/corruptM.test 7d574320e08c1b36caa3e47262061f186367d593a7e305d35f15289cc2c F test/corruptN.test 7c099d153a554001b4fb829c799b01f2ea6276cbc32479131e0db0da4efd9cc4 F test/cost.test b11cdbf9f11ffe8ef99c9881bf390e61fe92baf2182bad1dbe6de59a7295c576 F test/count.test cd4bd531066e8d77ef8fe1e3fc8253d042072e117ccab214b290cf83f1602249 -F test/countofview.test f9025dba8a9bb673a1f3ce25405900f630f81592c226546242df1ef1e62d0192 +F test/countofview.test 4088e461a10ee33e69803c177a69aa1d7bba81a9ffc2df66d76465a22ca7fdfc F test/coveridxscan.test f35c7208dedc4f98e471c569df64c0f95a49f6e072d8dc7c8f99bdee2697de1b F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 960a488a2db92f5437bc8171cdbed44618d9381c57fd41b2f0e357b6012c80b5 -R 4f561ce7d94ebba6771f71291846cad2 +P 8e841e7f025f7205959453875f2d9db36271642045593970a2b2fc20b2f847c3 +R a324206315ce89eead4c9961ac2ace00 U drh -Z 91fc03feb4d1edcaf09570adbf0b39df +Z 53309c46a9911b6ad0f66ef820f68132 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 17864e80e2..1d995d43d7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e841e7f025f7205959453875f2d9db36271642045593970a2b2fc20b2f847c3 \ No newline at end of file +babe2b5e59647ac9db4601e67c25190aac14eb76d5fcb9fa5b3692b955fefd61 \ No newline at end of file diff --git a/src/select.c b/src/select.c index a22aed8524..d97ecb2af6 100644 --- a/src/select.c +++ b/src/select.c @@ -6871,6 +6871,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ if( p->pWhere ) return 0; + if( p->pHaving ) return 0; if( p->pGroupBy ) return 0; if( p->pOrderBy ) return 0; pExpr = p->pEList->a[0].pExpr; @@ -6890,7 +6891,8 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pSub->pWhere ) return 0; /* No WHERE clause */ if( pSub->pLimit ) return 0; /* No LIMIT clause */ if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ - pSub = pSub->pPrior; /* Repeat over compound */ + assert( pSub->pHaving==0 ); /* Due to the previous */ + pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); /* If we reach this point then it is OK to perform the transformation */ diff --git a/test/countofview.test b/test/countofview.test index 7e94fa5ee7..cea6fa4274 100644 --- a/test/countofview.test +++ b/test/countofview.test @@ -76,4 +76,32 @@ do_execsql_test 4.1 { SELECT count(*) FROM t3 ORDER BY sum(a); } 1 +# 2023-03-31 dbsqlfuzz 6a107e3055bd22afab31cfddabc2d9d54fcbaf69 +# Having clauses should disqualify count-of-view +# +reset_db +do_execsql_test 5.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t1 VALUES(1,'one'),(4,'four'); + CREATE TABLE t2(c INTEGER PRIMARY KEY, d TEXT); + INSERT INTO t2 VALUES(2,'two'),(5,'five'); + CREATE VIEW t3 AS SELECT a, b FROM t1 UNION ALL SELECT c, d FROM t2; + SELECT count(*) FROM t3 HAVING count(*)>0; +} 4 +do_execsql_test 5.2 { + SELECT count(*) FROM t3 HAVING count(*)>5; +} {} +do_execsql_test 5.3 { + SELECT count(*) FROM t3 HAVING max(b)>'mmm'; +} 4 +do_execsql_test 5.4 { + SELECT count(*) FROM t3 HAVING min(b)>'mmm'; +} {} +do_execsql_test 5.5 { + SELECT count(*) FROM ( + SELECT a, max(b) FROM t1 HAVING a<100 UNION ALL SELECT c, d FROM t2 + ) +} 3 + + finish_test From 2e89f1cdebfa54a8350b22177dee374e82f6f534 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 1 Apr 2023 12:22:57 +0000 Subject: [PATCH 151/341] In the b-tree module use %u instead of %d to print unsigned quantities such as page numbers and offsets. FossilOrigin-Name: 33ac62d8eec56eb71f20ccd28a5d6e4e8051522feb2db0199abee9e18ce1f64e --- manifest | 12 +++++----- manifest.uuid | 2 +- src/btree.c | 64 +++++++++++++++++++++++++-------------------------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index 83b46dac1b..589edfc74e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\sthe\scount-of-view\soptimization\sif\sthere\sis\sa\sHAVING\sclause.\ndbsqlfuzz\s6a107e3055bd22afab31cfddabc2d9d54fcbaf69 -D 2023-03-31T23:48:59.808 +C In\sthe\sb-tree\smodule\suse\s%u\sinstead\sof\s%d\sto\sprint\sunsigned\squantities\ssuch\nas\spage\snumbers\sand\soffsets. +D 2023-04-01T12:22:57.449 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 47def1b43b10fcffd927ef6e17c9382a39c898af9eb98318460253ddb5e4a07f +F src/btree.c 0fda1fc90c2b240df720c020276e3027e0aa77313dbc667341ba27597557b4d8 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8e841e7f025f7205959453875f2d9db36271642045593970a2b2fc20b2f847c3 -R a324206315ce89eead4c9961ac2ace00 +P babe2b5e59647ac9db4601e67c25190aac14eb76d5fcb9fa5b3692b955fefd61 +R 48deb435a759e04a65c214c6987fb0c1 U drh -Z 53309c46a9911b6ad0f66ef820f68132 +Z 5e57bd2f4f9c8a5c6b6db368295f00fa # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1d995d43d7..e915d72023 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -babe2b5e59647ac9db4601e67c25190aac14eb76d5fcb9fa5b3692b955fefd61 \ No newline at end of file +33ac62d8eec56eb71f20ccd28a5d6e4e8051522feb2db0199abee9e18ce1f64e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e90d0c1f1c..ec0fc07ac5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -136,8 +136,8 @@ sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){ int corruptPageError(int lineno, MemPage *p){ char *zMsg; sqlite3BeginBenignMalloc(); - zMsg = sqlite3_mprintf("database corruption page %d of %s", - (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) + zMsg = sqlite3_mprintf("database corruption page %u of %s", + p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) ); sqlite3EndBenignMalloc(); if( zMsg ){ @@ -1032,7 +1032,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ - TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + TRACE(("PTRMAP_UPDATE: %u->(%u,%u)\n", key, eType, parent)); *pRC= rc = sqlite3PagerWrite(pDbPage); if( rc==SQLITE_OK ){ pPtrmap[offset] = eType; @@ -3846,7 +3846,7 @@ static int relocatePage( if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; /* Move page iDbPage from its current location to page number iFreePage */ - TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", + TRACE(("AUTOVACUUM: Moving %u to free page %u (ptr page %u type %u)\n", iDbPage, iFreePage, iPtrPage, eType)); rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); if( rc!=SQLITE_OK ){ @@ -6396,7 +6396,7 @@ static int allocateBtreePage( memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); *ppPage = pTrunk; pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); }else if( k>(u32)(pBt->usableSize/4 - 2) ){ /* Value of k is out of range. Database corruption */ rc = SQLITE_CORRUPT_PGNO(iTrunk); @@ -6462,7 +6462,7 @@ static int allocateBtreePage( } } pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); #endif }else if( k>0 ){ /* Extract a leaf from the trunk */ @@ -6507,8 +6507,8 @@ static int allocateBtreePage( ){ int noContent; *pPgno = iPage; - TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" - ": %d more free pages\n", + TRACE(("ALLOCATE: %u was leaf %u of %u on trunk %u" + ": %u more free pages\n", *pPgno, closest+1, k, pTrunk->pgno, n-1)); rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc ) goto end_allocate_page; @@ -6564,7 +6564,7 @@ static int allocateBtreePage( ** becomes a new pointer-map page, the second is used by the caller. */ MemPage *pPg = 0; - TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); + TRACE(("ALLOCATE: %u from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ @@ -6587,7 +6587,7 @@ static int allocateBtreePage( releasePage(*ppPage); *ppPage = 0; } - TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); + TRACE(("ALLOCATE: %u from end of file\n", *pPgno)); } assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) ); @@ -6715,7 +6715,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ } rc = btreeSetHasContent(pBt, iPage); } - TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); + TRACE(("FREE-PAGE: %u leaf on trunk page %u\n",pPage->pgno,pTrunk->pgno)); goto freepage_out; } } @@ -6736,7 +6736,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ put4byte(pPage->aData, iTrunk); put4byte(&pPage->aData[4], 0); put4byte(&pPage1->aData[32], iPage); - TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); + TRACE(("FREE-PAGE: %u new trunk page replacing %u\n", pPage->pgno, iTrunk)); freepage_out: if( pPage ){ @@ -8322,7 +8322,7 @@ static int balance_nonroot( ** that page. */ assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); - TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + TRACE(("BALANCE: old: %u(nc=%u) %u(nc=%u) %u(nc=%u)\n", apOld[0]->pgno, apOld[0]->nCell, nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 @@ -8406,8 +8406,8 @@ static int balance_nonroot( } } - TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " - "%d(%d nc=%d) %d(%d nc=%d)\n", + TRACE(("BALANCE: new: %u(%u nc=%u) %u(%u nc=%u) %u(%u nc=%u) " + "%u(%u nc=%u) %u(%u nc=%u)\n", apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, @@ -8652,7 +8652,7 @@ static int balance_nonroot( } assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + TRACE(("BALANCE: finished: old=%u new=%u cells=%u\n", nOld, nNew, b.nCell)); /* Free any old pages that were not reused as new pages. @@ -8737,7 +8737,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); - TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); + TRACE(("BALANCE: copy root %u into %u\n", pRoot->pgno, pChild->pgno)); /* Copy the overflow cells from pRoot to pChild */ memcpy(pChild->aiOvfl, pRoot->aiOvfl, @@ -9235,7 +9235,7 @@ int sqlite3BtreeInsert( if( rc ) return rc; } - TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", + TRACE(("INSERT: table=%u nkey=%lld ndata=%u page=%u %s\n", pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit || CORRUPT_DB ); @@ -10323,11 +10323,11 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ */ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ if( iPage>pCheck->nPage || iPage==0 ){ - checkAppendMsg(pCheck, "invalid page number %d", iPage); + checkAppendMsg(pCheck, "invalid page number %u", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ - checkAppendMsg(pCheck, "2nd reference to page %d", iPage); + checkAppendMsg(pCheck, "2nd reference to page %u", iPage); return 1; } setPageReferenced(pCheck, iPage); @@ -10353,13 +10353,13 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) checkOom(pCheck); - checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); + checkAppendMsg(pCheck, "Failed to read ptrmap key=%u", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ checkAppendMsg(pCheck, - "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", + "Bad ptr map entry key=%u expected=(%u,%u) got=(%u,%u)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } } @@ -10384,7 +10384,7 @@ static void checkList( if( checkRef(pCheck, iPage) ) break; N--; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ - checkAppendMsg(pCheck, "failed to get page %d", iPage); + checkAppendMsg(pCheck, "failed to get page %u", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); @@ -10397,7 +10397,7 @@ static void checkList( #endif if( n>pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, - "freelist leaf count too big on page %d", iPage); + "freelist leaf count too big on page %u", iPage); N--; }else{ for(i=0; i<(int)n; i++){ @@ -10429,7 +10429,7 @@ static void checkList( } if( N && nErrAtStart==pCheck->nErr ){ checkAppendMsg(pCheck, - "%s is %d but should be %d", + "%s is %u but should be %u", isFreeList ? "size" : "overflow list length", expected-N, expected); } @@ -10571,7 +10571,7 @@ static int checkTreePage( hdr = pPage->hdrOffset; /* Set up for cell analysis */ - pCheck->zPfx = "On tree page %u cell %d: "; + pCheck->zPfx = "On tree page %u cell %u: "; contentOffset = get2byteNotZero(&data[hdr+5]); assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ @@ -10615,7 +10615,7 @@ static int checkTreePage( pc = get2byteAligned(pCellIdx); pCellIdx -= 2; if( pcusableSize-4 ){ - checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + checkAppendMsg(pCheck, "Offset %u out of range %u..%u", pc, contentOffset, usableSize-4); doCoverageCheck = 0; continue; @@ -10747,7 +10747,7 @@ static int checkTreePage( */ if( heap[0]==0 && nFrag!=data[hdr+7] ){ checkAppendMsg(pCheck, - "Fragmentation of %d bytes reported as %d on page %u", + "Fragmentation of %u bytes reported as %u on page %u", nFrag, data[hdr+7], iPage); } } @@ -10861,7 +10861,7 @@ int sqlite3BtreeIntegrityCheck( mxInHdr = get4byte(&pBt->pPage1->aData[52]); if( mx!=mxInHdr ){ checkAppendMsg(&sCheck, - "max rootpage (%d) disagrees with header (%d)", + "max rootpage (%u) disagrees with header (%u)", mx, mxInHdr ); } @@ -10892,7 +10892,7 @@ int sqlite3BtreeIntegrityCheck( for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %u is never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain @@ -10900,11 +10900,11 @@ int sqlite3BtreeIntegrityCheck( */ if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %u is never used", i); } if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); + checkAppendMsg(&sCheck, "Pointer map page %u is referenced", i); } #endif } From e5ea81ae85081b4bd9fd802c956a34717275f2d6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 1 Apr 2023 13:14:53 +0000 Subject: [PATCH 152/341] Improved error messages from PRAGMA integrity_check. Identify the root of the tree when a problem is found in a b-tree, making it easier to track the problem to a specific table or index. FossilOrigin-Name: a1cb152e69c7c6cdd99300c91a8104716089de459d9d19e33ef38432aad70908 --- manifest | 26 ++++++++--------- manifest.uuid | 2 +- src/btree.c | 19 +++++++------ src/btreeInt.h | 5 ++-- test/corrupt2.test | 12 ++++---- test/corrupt3.test | 10 +++---- test/corrupt7.test | 4 +-- test/corruptC.test | 4 +-- test/corruptD.test | 2 +- test/pragma.test | 70 +++++++++++++++++++++++----------------------- 10 files changed, 78 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index 589edfc74e..4e72d9d151 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sb-tree\smodule\suse\s%u\sinstead\sof\s%d\sto\sprint\sunsigned\squantities\ssuch\nas\spage\snumbers\sand\soffsets. -D 2023-04-01T12:22:57.449 +C Improved\serror\smessages\sfrom\sPRAGMA\sintegrity_check.\s\sIdentify\sthe\sroot\sof\nthe\stree\swhen\sa\sproblem\sis\sfound\sin\sa\sb-tree,\smaking\sit\seasier\sto\strack\sthe\nproblem\sto\sa\sspecific\stable\sor\sindex. +D 2023-04-01T13:14:53.367 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,9 +564,9 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 0fda1fc90c2b240df720c020276e3027e0aa77313dbc667341ba27597557b4d8 +F src/btree.c ecce2a6cf75ccd1f419f774a026ab102399aeae90f33abb14aed363c4e0f839c F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc -F src/btreeInt.h 06bb2c1a07172d5a1cd27a2a5d617b93b1e976c5873709c31964786f86365a6e +F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e @@ -872,18 +872,18 @@ F test/conflict2.test 5557909ce683b1073982f5d1b61dfb1d41e369533bfdaf003180c5bc87 F test/conflict3.test 81865d9599609aca394fb3b9cd5f561d4729ea5b176bece3644f6ecb540f88ac F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4 F test/corrupt.test d7cb0300e4a297147b6a05e92a1684bc8973635c3bcaa3d66e983c9cbdbf47a3 -F test/corrupt2.test bb50042cf9a1f1023d73af325d47eb02a6bb11e3c52f8812644b220c5d4bca35 -F test/corrupt3.test 2520432b1fbf99994841e69804a3c59fb828183f4d09b85a1631bc7adca17e31 +F test/corrupt2.test 6e0c1e1c2ff4bedde4bc73f16250d74ae5b3d9ece086640ce88b9a94620ba993 +F test/corrupt3.test 6a982535d52c8165654cbc79a043cfd0bf02495a5efbf4754295e056fc548539 F test/corrupt4.test b5ae41607e8d17d9c1f3e94fdb572ce061ed3beeebdb46fb3a348181b8c8a097 F test/corrupt5.test 387be3250795e2a86e6234745558b80efb248a357d0cd8e53bce75c7463f545d F test/corrupt6.test fc6a891716139665dae0073b6945e3670bf92568 -F test/corrupt7.test b036f94bda4b0b23a2919bf717046ce9ecca4543 +F test/corrupt7.test ffa86896fe63a3d00b0a131e1e64f402e4da9f7e5d89609d6501c851e511d73a F test/corrupt8.test 2399dfe40d2c0c63af86706e30f3e6302a8d0516 F test/corrupt9.test 730a3db08d4ab9aa43392ea30d9c2b4879cbff85 F test/corruptA.test 112f4b2ae0b95ebf3ea63718642fb969a93acea557ace3a307234d19c245989b F test/corruptB.test 73a8d6c0b9833697ecf16b63e3c5c05c945b5dec -F test/corruptC.test 74d4498fd25759618b393f1e9cde111de828b88c1848ab320f6c179fd52b5a60 -F test/corruptD.test 33a37ce3ed56a20093ceee778cd2d7109c7085a59f3213d2baede11d952e8e50 +F test/corruptC.test 9cf32275dae3ca33f645afe5d1d3f5ba5ac2af2b0833dfb5282f9dccb6fb81bb +F test/corruptD.test a828c788535946a372a56a750b242cd96287cd823657abe5a73c5e51b91bdd28 F test/corruptE.test 4143791f2dfb443aec5b7fabfa5821e6063eccc3b49b06f212c2f014715fd476 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 @@ -1380,7 +1380,7 @@ F test/pendingrace.test cbdf0f74bc939fb43cebad64dda7a0b5a3941a10b7e9cc2b596ff3e4 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff F test/permutations.test 8bd6b6db541e2a7f9bb894be99ef5c00526b23762c4a00c574e1cba697495125 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f -F test/pragma.test d21ae02249f2a1813f016efebe616a1916b3f9abd71867da6211a50bfea373ba +F test/pragma.test 57a36226218c03cfb381019fe43234b2cefbd8a1f12825514f906a17ccf7991e F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 92a46bbea12322dd94a404f49edcfbfc913a2c98115f0d030a7459bb4712ef31 F test/pragma4.test ca5e4dfc46adfe490f75d73734f70349d95a199e6510973899e502eef2c8b1f8 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P babe2b5e59647ac9db4601e67c25190aac14eb76d5fcb9fa5b3692b955fefd61 -R 48deb435a759e04a65c214c6987fb0c1 +P 33ac62d8eec56eb71f20ccd28a5d6e4e8051522feb2db0199abee9e18ce1f64e +R 46b4ea5505ae8daa6bae8608979d5af1 U drh -Z 5e57bd2f4f9c8a5c6b6db368295f00fa +Z d031925974df47c7e88e1b1118ce617e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e915d72023..5e1238dd89 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -33ac62d8eec56eb71f20ccd28a5d6e4e8051522feb2db0199abee9e18ce1f64e \ No newline at end of file +a1cb152e69c7c6cdd99300c91a8104716089de459d9d19e33ef38432aad70908 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index ec0fc07ac5..1b91120ffd 100644 --- a/src/btree.c +++ b/src/btree.c @@ -10283,7 +10283,8 @@ static void checkAppendMsg( sqlite3_str_append(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ - sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, + pCheck->v0, pCheck->v1, pCheck->v2); } sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap); va_end(ap); @@ -10544,8 +10545,8 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; - pCheck->zPfx = "Page %u: "; - pCheck->v1 = iPage; + pCheck->zPfx = "Tree %u page %u: "; + pCheck->v0 = pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); @@ -10571,7 +10572,7 @@ static int checkTreePage( hdr = pPage->hdrOffset; /* Set up for cell analysis */ - pCheck->zPfx = "On tree page %u cell %u: "; + pCheck->zPfx = "Tree %u page %u cell %u: "; contentOffset = get2byteNotZero(&data[hdr+5]); assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ @@ -10591,7 +10592,7 @@ static int checkTreePage( pgno = get4byte(&data[hdr+8]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - pCheck->zPfx = "On page %u at right child: "; + pCheck->zPfx = "Tree %u page %u right child: "; checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif @@ -10844,7 +10845,7 @@ int sqlite3BtreeIntegrityCheck( /* Check the integrity of the freelist */ if( bCkFreelist ){ - sCheck.zPfx = "Main freelist: "; + sCheck.zPfx = "Freelist: "; checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), get4byte(&pBt->pPage1->aData[36])); sCheck.zPfx = 0; @@ -10892,7 +10893,7 @@ int sqlite3BtreeIntegrityCheck( for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, "Page %u is never used", i); + checkAppendMsg(&sCheck, "Page %u: never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain @@ -10900,11 +10901,11 @@ int sqlite3BtreeIntegrityCheck( */ if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Page %u is never used", i); + checkAppendMsg(&sCheck, "Page %u: never used", i); } if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Pointer map page %u is referenced", i); + checkAppendMsg(&sCheck, "Page %u: pointer map referenced", i); } #endif } diff --git a/src/btreeInt.h b/src/btreeInt.h index 79c3296d04..160a6af18f 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -701,8 +701,9 @@ struct IntegrityCk { int rc; /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */ u32 nStep; /* Number of steps into the integrity_check process */ const char *zPfx; /* Error message prefix */ - Pgno v1; /* Value for first %u substitution in zPfx */ - int v2; /* Value for second %d substitution in zPfx */ + Pgno v0; /* Value for first %u substitution in zPfx (root page) */ + Pgno v1; /* Value for second %u substitution in zPfx (current pg) */ + int v2; /* Value for third %d substitution in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ diff --git a/test/corrupt2.test b/test/corrupt2.test index f97a526ef5..96d28490aa 100644 --- a/test/corrupt2.test +++ b/test/corrupt2.test @@ -99,7 +99,7 @@ do_test corrupt2-1.4 { # of MemPage.nFree catchsql {PRAGMA quick_check} db2 } {0 {{*** in database main *** -Page 1: free space corruption}}} +Tree 1 page 1: free space corruption}}} do_test corrupt2-1.5 { db2 close @@ -120,7 +120,7 @@ do_test corrupt2-1.5 { sqlite3 db2 corrupt.db catchsql {PRAGMA quick_check} db2 } {0 {{*** in database main *** -Page 1: free space corruption}}} +Tree 1 page 1: free space corruption}}} db2 close # Corrupt a database by having 2 indices of the same name: @@ -248,8 +248,8 @@ do_test corrupt2-5.1 { } set result } {{*** in database main *** -On tree page 2 cell 0: 2nd reference to page 10 -Page 4 is never used}} +Tree 11 page 2 cell 0: 2nd reference to page 10 +Page 4: never used}} db2 close @@ -591,7 +591,7 @@ do_test 14.2 { do_execsql_test 14.3 { PRAGMA integrity_check; } {{*** in database main *** -Main freelist: size is 3 but should be 2}} +Freelist: size is 3 but should be 2}} # Use 2 of the free pages on the free-list. # @@ -603,7 +603,7 @@ do_execsql_test 14.4 { do_execsql_test 14.5 { PRAGMA integrity_check; } {{*** in database main *** -Main freelist: size is 1 but should be 0}} +Freelist: size is 1 but should be 0}} finish_test diff --git a/test/corrupt3.test b/test/corrupt3.test index 7a2e174ead..691302f7af 100644 --- a/test/corrupt3.test +++ b/test/corrupt3.test @@ -82,7 +82,7 @@ do_test corrupt3-1.8 { PRAGMA integrity_check } } {0 {{*** in database main *** -On tree page 2 cell 0: 2nd reference to page 3}}} +Tree 2 page 2 cell 0: 2nd reference to page 3}}} # Change the pointer for the first page of the overflow # change to be a non-existant page. @@ -100,8 +100,8 @@ do_test corrupt3-1.10 { PRAGMA integrity_check } } {0 {{*** in database main *** -On tree page 2 cell 0: invalid page number 4 -Page 3 is never used}}} +Tree 2 page 2 cell 0: invalid page number 4 +Page 3: never used}}} do_test corrupt3-1.11 { db close hexio_write test.db 2044 [hexio_render_int32 0] @@ -115,7 +115,7 @@ do_test corrupt3-1.12 { PRAGMA integrity_check } } {0 {{*** in database main *** -On tree page 2 cell 0: overflow list length is 0 but should be 1 -Page 3 is never used}}} +Tree 2 page 2 cell 0: overflow list length is 0 but should be 1 +Page 3: never used}}} finish_test diff --git a/test/corrupt7.test b/test/corrupt7.test index aa66cc7ece..b62515b784 100644 --- a/test/corrupt7.test +++ b/test/corrupt7.test @@ -70,14 +70,14 @@ do_test corrupt7-2.1 { sqlite3 db test.db db eval {PRAGMA integrity_check(1)} } {{*** in database main *** -On tree page 2 cell 15: Offset 65457 out of range 945..1020}} +Tree 2 page 2 cell 15: Offset 65457 out of range 945..1020}} do_test corrupt7-2.2 { db close hexio_write test.db 1062 04 sqlite3 db test.db db eval {PRAGMA integrity_check(1)} } {{*** in database main *** -On tree page 2 cell 15: Offset 1201 out of range 945..1020}} +Tree 2 page 2 cell 15: Offset 1201 out of range 945..1020}} # The code path that was causing the buffer overrun that this test # case was checking for was removed. diff --git a/test/corruptC.test b/test/corruptC.test index a56abeec72..f5733a8186 100644 --- a/test/corruptC.test +++ b/test/corruptC.test @@ -98,7 +98,7 @@ do_test corruptC-2.1 { sqlite3 db test.db catchsql {PRAGMA integrity_check} } {0 {{*** in database main *** -Page 3: free space corruption}}} +Tree 3 page 3: free space corruption}}} # test that a corrupt content offset size is handled (seed 5649) # @@ -165,7 +165,7 @@ do_test corruptC-2.5 { catchsql {BEGIN; UPDATE t2 SET y='abcdef-uvwxyz'; ROLLBACK;} catchsql {PRAGMA integrity_check} } {0 {{*** in database main *** -On tree page 4 cell 19: Extends off end of page} {database disk image is malformed}}} +Tree 4 page 4 cell 19: Extends off end of page} {database disk image is malformed}}} # {0 {{*** in database main *** # Corruption detected in cell 710 on page 4 diff --git a/test/corruptD.test b/test/corruptD.test index eb6ccb3fcd..c35388adfb 100644 --- a/test/corruptD.test +++ b/test/corruptD.test @@ -113,7 +113,7 @@ do_test corruptD-1.1.1 { hexio_write test.db [expr 1024+1] FFFF catchsql { PRAGMA quick_check } } {0 {{*** in database main *** -Page 2: free space corruption}}} +Tree 2 page 2: free space corruption}}} do_test corruptD-1.1.2 { incr_change_counter hexio_write test.db [expr 1024+1] [hexio_render_int32 1021] diff --git a/test/pragma.test b/test/pragma.test index a3e1d3cd65..5b45a74400 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -433,9 +433,9 @@ ifcapable attach { PRAGMA integrity_check } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} do_execsql_test pragma-3.9b { PRAGMA t2.integrity_check=t2; } {{row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} @@ -447,79 +447,79 @@ Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2 PRAGMA integrity_check=1 } } {{*** in database t2 *** -Page 4 is never used}} +Page 4: never used}} do_test pragma-3.11 { execsql { PRAGMA integrity_check=5 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2}} do_test pragma-3.12 { execsql { PRAGMA integrity_check=4 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2}} do_test pragma-3.13 { execsql { PRAGMA integrity_check=3 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used}} +Page 4: never used +Page 5: never used +Page 6: never used}} do_test pragma-3.14 { execsql { PRAGMA integrity_check(2) } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used}} +Page 4: never used +Page 5: never used}} do_test pragma-3.15 { execsql { ATTACH 'testerr.db' AS t3; PRAGMA integrity_check } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2}} do_test pragma-3.16 { execsql { PRAGMA integrity_check(10) } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2}} do_test pragma-3.17 { execsql { PRAGMA integrity_check=8 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** -Page 4 is never used -Page 5 is never used}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2} {row 2 missing from index i2} {wrong # of entries in index i2} {*** in database t3 *** +Page 4: never used +Page 5: never used}} do_test pragma-3.18 { execsql { PRAGMA integrity_check=4 } } {{*** in database t2 *** -Page 4 is never used -Page 5 is never used -Page 6 is never used} {row 1 missing from index i2}} +Page 4: never used +Page 5: never used +Page 6: never used} {row 1 missing from index i2}} } do_test pragma-3.19 { catch {db close} From 54b81e36ffff011aaf5173d7d9ba475819964bc8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 1 Apr 2023 15:51:21 +0000 Subject: [PATCH 153/341] Fix harmless compiler warnings. FossilOrigin-Name: a4fb2864fe01cce9694242a0750623ca47fcecd68f74c4239d3eb5fbf978770a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 2 ++ src/vdbeapi.c | 2 +- src/vdbeaux.c | 6 +++--- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 4e72d9d151..402e7df226 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\serror\smessages\sfrom\sPRAGMA\sintegrity_check.\s\sIdentify\sthe\sroot\sof\nthe\stree\swhen\sa\sproblem\sis\sfound\sin\sa\sb-tree,\smaking\sit\seasier\sto\strack\sthe\nproblem\sto\sa\sspecific\stable\sor\sindex. -D 2023-04-01T13:14:53.367 +C Fix\sharmless\scompiler\swarnings. +D 2023-04-01T15:51:21.968 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -630,7 +630,7 @@ F src/shell.c.in d80b53a9ec0a43ac0157fc54d667cb49fb6dd2229558135530feab2fc04b909 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 2a3331b16b70b0b56c65574bf3e4f749ccf493787ca6550a29dc389cad28bb34 +F src/sqliteInt.h f0edd1cd318a5bc02b2dcc68768bf1c5763628b6e3e75f30e81d0644e17d40aa F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -698,8 +698,8 @@ F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb -F src/vdbeapi.c 998d240b3afd78df3581d545240330ca4cc09ecd10d0af730fe364f1c7eb253d -F src/vdbeaux.c 0f5201346a83a35a08e833c3a03abe626119c07b7361c28bc6a259b98540f1d6 +F src/vdbeapi.c 03a21e617ce787300699cfda7ec12053e74b5194af34f4dd2593e4190d3d20de +F src/vdbeaux.c 2ec444e0be19d2cf671e2f3ae331965f5d9e24625525cc947048ab61625e1a37 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c db0458d11a51f6cfad2333a41e36a3795be0b2f316d070df5d33543a9ac884ac F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 33ac62d8eec56eb71f20ccd28a5d6e4e8051522feb2db0199abee9e18ce1f64e -R 46b4ea5505ae8daa6bae8608979d5af1 +P a1cb152e69c7c6cdd99300c91a8104716089de459d9d19e33ef38432aad70908 +R 1d46d8321fd0682be5d0a8790e197dc3 U drh -Z d031925974df47c7e88e1b1118ce617e +Z 9ba80d4da1e49f126926ac3e65e84ef8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5e1238dd89..0a2dd04d57 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1cb152e69c7c6cdd99300c91a8104716089de459d9d19e33ef38432aad70908 \ No newline at end of file +a4fb2864fe01cce9694242a0750623ca47fcecd68f74c4239d3eb5fbf978770a \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index dcf17d9631..d393853371 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4650,7 +4650,9 @@ int sqlite3GetTempRange(Parse*,int); void sqlite3ReleaseTempRange(Parse*,int,int); void sqlite3ClearTempRegCache(Parse*); void sqlite3TouchRegister(Parse*,int); +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) int sqlite3FirstAvailableRegister(Parse*,int); +#endif #ifdef SQLITE_DEBUG int sqlite3NoTempsInRange(Parse*,int,int); #endif diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 42f3bb0b40..794e2289e8 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -2140,7 +2140,7 @@ int sqlite3_stmt_scanstatus_v2( Vdbe *p = (Vdbe*)pStmt; VdbeOp *aOp = p->aOp; int nOp = p->nOp; - ScanStatus *pScan; + ScanStatus *pScan = 0; int idx; if( p->pFrame ){ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 87fd067322..e92523710f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3644,9 +3644,9 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ #ifdef SQLITE_ENABLE_NORMALIZE sqlite3DbFree(db, p->zNormSql); { - DblquoteStr *pThis, *pNext; - for(pThis=p->pDblStr; pThis; pThis=pNext){ - pNext = pThis->pNextStr; + DblquoteStr *pThis, *pNxt; + for(pThis=p->pDblStr; pThis; pThis=pNxt){ + pNxt = pThis->pNextStr; sqlite3DbFree(db, pThis); } } From 4d5a5a48c524bed2ccff738321ea6d026de386a8 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 1 Apr 2023 16:14:50 +0000 Subject: [PATCH 154/341] Swat grammar nit on README.md FossilOrigin-Name: 715c00e58b774d6a29e2f60fd33a2ecb960307192ae1fdb8ed396f74fd62b444 --- README.md | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7ad42061c1..0df8b58c23 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ and most of the documentation are managed separately. ## Version Control -SQLite sources are managed using the +SQLite sources are managed using [Fossil](https://www.fossil-scm.org/), a distributed version control system that was specifically designed and written to support SQLite development. The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext. diff --git a/manifest b/manifest index 402e7df226..a80374e421 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Fix\sharmless\scompiler\swarnings. -D 2023-04-01T15:51:21.968 +C Swat\sgrammar\snit\son\sREADME.md +D 2023-04-01T16:14:50.138 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F Makefile.in 764f2e3e8fb4ae1c8dfe03e65b2b3b01bd1fc57edf78ec2cab3a1301e90e1905 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc ada3466f8f0112a8baead4d6cc2a99bf544d228958baae12ca35a3ee5755c806 -F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e +F README.md e05bd8fcb45da04ab045c37f79a98654e8aa3b3b8f302cfbba80a0d510df75f7 F VERSION 17f95ae2fdf21f0e9575eb0b0511ea63f15d71dfff431b21c2b4adbfa70cfbbf F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a1cb152e69c7c6cdd99300c91a8104716089de459d9d19e33ef38432aad70908 -R 1d46d8321fd0682be5d0a8790e197dc3 -U drh -Z 9ba80d4da1e49f126926ac3e65e84ef8 +P a4fb2864fe01cce9694242a0750623ca47fcecd68f74c4239d3eb5fbf978770a +R fa814ff747457204df2159749d0b640b +U larrybr +Z e5553ea70d055b6fd82dee42eb351c51 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0a2dd04d57..ea564aa142 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4fb2864fe01cce9694242a0750623ca47fcecd68f74c4239d3eb5fbf978770a \ No newline at end of file +715c00e58b774d6a29e2f60fd33a2ecb960307192ae1fdb8ed396f74fd62b444 \ No newline at end of file From 8d1005a593b9d3f79b629c231180a48dce819371 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 1 Apr 2023 23:29:59 +0000 Subject: [PATCH 155/341] Allow special floating-point value names in JSON: "inf", "-inf", "infinity", "-infinity", "nan", "qnan", and "snan". All are converted into valid JSON values: 9e999, -9e999, or null. Requires the SQLITE_ENABLE_JSON_NAN_INF compile-time option to operate. FossilOrigin-Name: fc8793e5acac7351749e360c6bace5d5a8b3de3aa600ae23e260557db650c461 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/json.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 402e7df226..b14a6fbe48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2023-04-01T15:51:21.968 +C Allow\sspecial\sfloating-point\svalue\snames\sin\sJSON:\s\s"inf",\s"-inf",\s"infinity",\n"-infinity",\s"nan",\s"qnan",\sand\s"snan".\s\sAll\sare\sconverted\sinto\svalid\sJSON\nvalues:\s9e999,\s-9e999,\sor\snull.\s\sRequires\sthe\sSQLITE_ENABLE_JSON_NAN_INF\ncompile-time\soption\sto\soperate. +D 2023-04-01T23:29:59.248 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 9e3a6acf9e4fa8bb6c84a38538ed9c2f9ad09f106910ef0e9cc68e19c8dc4fc8 +F src/json.c 5ac77e419921a0e63dafe07b560e9ce5e9823055e814308559e4d54cc885a967 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2052,8 +2052,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a1cb152e69c7c6cdd99300c91a8104716089de459d9d19e33ef38432aad70908 -R 1d46d8321fd0682be5d0a8790e197dc3 +P a4fb2864fe01cce9694242a0750623ca47fcecd68f74c4239d3eb5fbf978770a +R d4fc5e6b6e165fd35d3ba84e2f999b35 +T *branch * json-nan-inf +T *sym-json-nan-inf * +T -sym-trunk * U drh -Z 9ba80d4da1e49f126926ac3e65e84ef8 +Z 2575536ea35edbedba9ae17b565e4eb0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0a2dd04d57..ca3061fe85 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4fb2864fe01cce9694242a0750623ca47fcecd68f74c4239d3eb5fbf978770a \ No newline at end of file +fc8793e5acac7351749e360c6bace5d5a8b3de3aa600ae23e260557db650c461 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9783a3898a..dbca6d2bb7 100644 --- a/src/json.c +++ b/src/json.c @@ -757,6 +757,27 @@ static int jsonIs4Hex(const char *z){ return 1; } +#ifdef SQLITE_ENABLE_JSON_NAN_INF +/* +** Extra floating-point literals to allow in JSON. +*/ +static const struct NanInfName { + char c1; + char c2; + char n; + char eType; + char nRepl; + char *zMatch; + char *zRepl; +} aNanInfName[] = { + { 'i', 'I', 3, JSON_REAL, 7, "inf", "9.0e999" }, + { 'i', 'I', 8, JSON_REAL, 7, "infinity", "9.0e999" }, + { 'n', 'N', 3, JSON_NULL, 4, "NaN", "null" }, + { 'q', 'Q', 4, JSON_NULL, 4, "QNaN", "null" }, + { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" }, +}; +#endif /* SQLITE_ENABLE_JSON_NAN_INF */ + /* ** Parse a single JSON value which begins at pParse->zJson[i]. Return the ** index of the first character past the end of the value parsed. @@ -902,6 +923,24 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ if( c<'0' || c>'9' ) return -1; continue; } +#ifdef SQLITE_ENABLE_JSON_NAN_INF + /* Non-standard JSON: Allow "-Inf" (in any case) + ** to be understood as floating point literals. */ + if( (c=='i' || c=='I') + && j==i+1 + && z[i]=='-' + && sqlite3StrNICmp(&z[j], "inf",3)==0 + ){ + if( !sqlite3Isalnum(z[j+4]) ){ + jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); + return i+4; + }else if( (sqlite3StrNICmp(&z[j],"infinity",8)==0 && + !sqlite3Isalnum(z[j+5])) ){ + jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); + return i+9; + } + } +#endif break; } if( z[j-1]<'0' ) return -1; @@ -915,6 +954,20 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ }else if( c==0 ){ return 0; /* End of file */ }else{ +#ifdef SQLITE_ENABLE_JSON_NAN_INF + int k, nn; + for(k=0; k Date: Sun, 2 Apr 2023 14:53:59 +0000 Subject: [PATCH 156/341] Clear executable bit on base64.c FossilOrigin-Name: ec1ddbce7db270afc25f71546fa745935ad74bc72920fccfcb37e98649612bf2 --- ext/misc/base64.c | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) mode change 100755 => 100644 ext/misc/base64.c diff --git a/ext/misc/base64.c b/ext/misc/base64.c old mode 100755 new mode 100644 index 69eff61811..4da3f43eb7 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -76,6 +76,7 @@ typedef unsigned char u8; #define U8_TYPEDEF #endif +/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ static const u8 b64DigitValues[128] = { /* HT LF VT FF CR */ ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, diff --git a/manifest b/manifest index a80374e421..340b8d10c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Swat\sgrammar\snit\son\sREADME.md -D 2023-04-01T16:14:50.138 +C Clear\sexecutable\sbit\son\sbase64.c +D 2023-04-02T14:53:59.580 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -264,7 +264,7 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c d43d2b209c8ab70ca3f860104bb353b0f52a1c5462a2466140025c954e4f3ea7 x +F ext/misc/base64.c e83a915fcb94c9332e9a92aa4c3beafe2552bd3da2813fc5fff31918cca0b834 F ext/misc/base85.c 77dfd5813d23ea561d0348f922583888e78f8eaeb2b9a4a28226d092389890b8 F ext/misc/basexx.c 5e859e1820620aa8080fb9145eb47089de426ae808f6abb01a8e12921c3a8e67 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4fb2864fe01cce9694242a0750623ca47fcecd68f74c4239d3eb5fbf978770a -R fa814ff747457204df2159749d0b640b +P 715c00e58b774d6a29e2f60fd33a2ecb960307192ae1fdb8ed396f74fd62b444 +R 76be783a3d8976f5c3106dd2a2179fd6 U larrybr -Z e5553ea70d055b6fd82dee42eb351c51 +Z 7cb6cca24a98be0d56433a58a4384bb2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ea564aa142..74a443f7df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -715c00e58b774d6a29e2f60fd33a2ecb960307192ae1fdb8ed396f74fd62b444 \ No newline at end of file +ec1ddbce7db270afc25f71546fa745935ad74bc72920fccfcb37e98649612bf2 \ No newline at end of file From af3365453375e610bfdce88a1c8984323334477e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 2 Apr 2023 16:43:20 +0000 Subject: [PATCH 157/341] Earlier detection of corruption in sqlite3BtreeDelete(). dbsqlfuzz a4c48c291d6e40157a1b749a05eaa7c7faf5a625. FossilOrigin-Name: 978dc71c388b37740da38c310674315c7d7fe814d1daa16a146b4df71385d1e1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 5 ++++- test/corruptI.test | 7 +------ 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 340b8d10c4..ed71113881 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clear\sexecutable\sbit\son\sbase64.c -D 2023-04-02T14:53:59.580 +C Earlier\sdetection\sof\scorruption\sin\ssqlite3BtreeDelete().\ndbsqlfuzz\sa4c48c291d6e40157a1b749a05eaa7c7faf5a625. +D 2023-04-02T16:43:20.854 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c ecce2a6cf75ccd1f419f774a026ab102399aeae90f33abb14aed363c4e0f839c +F src/btree.c c2ececcbcb1a35bbd9efcbb43a821eaec60cc9796dccbb33f3b0fd4c7ab2f539 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -888,7 +888,7 @@ F test/corruptE.test 4143791f2dfb443aec5b7fabfa5821e6063eccc3b49b06f212c2f014715 F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test adf79b669cbfd19e28c8191a610d083ae53a6d51 F test/corruptH.test 79801d97ec5c2f9f3c87739aa1ec2eb786f96454 -F test/corruptI.test a17bbf54fdde78d43cf3cc34b0057719fd4a173a3d824285b67dc5257c064c7b +F test/corruptI.test 9d8cbf6214e492abe9e822e759b9751ae336cec0a6fe3ff3b37bfbd8ff9c22ca F test/corruptJ.test 4d5ccc4bf959464229a836d60142831ef76a5aa4 F test/corruptK.test 5b4212fe346699831c5ad559a62c54e11c0611bdde1ea8423a091f9c01aa32af F test/corruptL.test b42978028afc5eefc8b51d8d7cd6a9344ba7362d7ed4511ee2070f56e06d5a1c @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 715c00e58b774d6a29e2f60fd33a2ecb960307192ae1fdb8ed396f74fd62b444 -R 76be783a3d8976f5c3106dd2a2179fd6 -U larrybr -Z 7cb6cca24a98be0d56433a58a4384bb2 +P ec1ddbce7db270afc25f71546fa745935ad74bc72920fccfcb37e98649612bf2 +R 1f7f7780d232aaca9814c0eee95933c1 +U drh +Z 4959631f3771aa98a65ccf057f0978b9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 74a443f7df..32b288ce4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec1ddbce7db270afc25f71546fa745935ad74bc72920fccfcb37e98649612bf2 \ No newline at end of file +978dc71c388b37740da38c310674315c7d7fe814d1daa16a146b4df71385d1e1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 1b91120ffd..b2edcd7d49 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7351,7 +7351,7 @@ static int rebuildPage( assert( i(u32)usableSize ){ j = 0; } + if( NEVER(j>(u32)usableSize) ){ j = 0; } memcpy(&pTmp[j], &aData[j], usableSize - j); for(k=0; pCArray->ixNx[k]<=i && ALWAYS(knFree<0 && btreeComputeFreeSpace(pPage) ){ return SQLITE_CORRUPT_BKPT; } + if( pCell<&pPage->aCellIdx[pPage->nCell] ){ + return SQLITE_CORRUPT_BKPT; + } /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must ** be preserved following this delete operation. If the current delete diff --git a/test/corruptI.test b/test/corruptI.test index 12ce446540..65ef376258 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -123,18 +123,13 @@ do_execsql_test 4.0 { set root [db one {SELECT rootpage FROM sqlite_master}] set offset [expr ($root-1) * 65536] -ifcapable oversize_cell_check { - set res {1 {database disk image is malformed}} -} else { - set res {0 {}} -} do_test 4.1 { db close hexio_write test.db [expr $offset + 8 + 2] 0000 hexio_write test.db [expr $offset + 5] 0000 sqlite3 db test.db catchsql { DELETE FROM t1 WHERE a=0 } -} $res +} {1 {database disk image is malformed}} #------------------------------------------------------------------------- From 26cc3a1387ecc236e826d23a806d7e131d5c4f41 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 2 Apr 2023 20:27:02 +0000 Subject: [PATCH 158/341] Fix an off-by-one error in the recognition of -Infinity. FossilOrigin-Name: f7ebf3e6286ddc8cdaa9446235407785d1be2be2d9992e21ef59fcd655f68432 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b14a6fbe48..4f7228c081 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sspecial\sfloating-point\svalue\snames\sin\sJSON:\s\s"inf",\s"-inf",\s"infinity",\n"-infinity",\s"nan",\s"qnan",\sand\s"snan".\s\sAll\sare\sconverted\sinto\svalid\sJSON\nvalues:\s9e999,\s-9e999,\sor\snull.\s\sRequires\sthe\sSQLITE_ENABLE_JSON_NAN_INF\ncompile-time\soption\sto\soperate. -D 2023-04-01T23:29:59.248 +C Fix\san\soff-by-one\serror\sin\sthe\srecognition\sof\s-Infinity. +D 2023-04-02T20:27:02.691 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 5ac77e419921a0e63dafe07b560e9ce5e9823055e814308559e4d54cc885a967 +F src/json.c 442e035cca4f93b22a024851762499ea7156fc2496e1062a21cfa5f9729fe7a8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2052,11 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a4fb2864fe01cce9694242a0750623ca47fcecd68f74c4239d3eb5fbf978770a -R d4fc5e6b6e165fd35d3ba84e2f999b35 -T *branch * json-nan-inf -T *sym-json-nan-inf * -T -sym-trunk * +P fc8793e5acac7351749e360c6bace5d5a8b3de3aa600ae23e260557db650c461 +R e03c2372523f504c7365a6c5a8a2394b U drh -Z 2575536ea35edbedba9ae17b565e4eb0 +Z b04d3ca6b8bf072d9ad0c754d33e8df2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ca3061fe85..cb0b23859b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc8793e5acac7351749e360c6bace5d5a8b3de3aa600ae23e260557db650c461 \ No newline at end of file +f7ebf3e6286ddc8cdaa9446235407785d1be2be2d9992e21ef59fcd655f68432 \ No newline at end of file diff --git a/src/json.c b/src/json.c index dbca6d2bb7..a4ba2e0cfb 100644 --- a/src/json.c +++ b/src/json.c @@ -935,7 +935,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); return i+4; }else if( (sqlite3StrNICmp(&z[j],"infinity",8)==0 && - !sqlite3Isalnum(z[j+5])) ){ + !sqlite3Isalnum(z[j+9])) ){ jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); return i+9; } From c406ecc662794445a0805e0803efcbbb662db48f Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 2 Apr 2023 20:46:24 +0000 Subject: [PATCH 159/341] More off-by-one errors in the new JSON parsing. FossilOrigin-Name: dbc99662087b63c9ed5b398535a6091fc2c5e507907dd1fcb7ad0b6ab3f17144 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4f7228c081..721364a058 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\serror\sin\sthe\srecognition\sof\s-Infinity. -D 2023-04-02T20:27:02.691 +C More\soff-by-one\serrors\sin\sthe\snew\sJSON\sparsing. +D 2023-04-02T20:46:24.359 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -585,7 +585,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 442e035cca4f93b22a024851762499ea7156fc2496e1062a21cfa5f9729fe7a8 +F src/json.c edae65fe1f66ce8b1e7fa6eb036a3d8cf525dacd91d58f12068e80a81ac34f61 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fc8793e5acac7351749e360c6bace5d5a8b3de3aa600ae23e260557db650c461 -R e03c2372523f504c7365a6c5a8a2394b +P f7ebf3e6286ddc8cdaa9446235407785d1be2be2d9992e21ef59fcd655f68432 +R d7e51978d4e1fb261c348646bf7a99ee U drh -Z b04d3ca6b8bf072d9ad0c754d33e8df2 +Z 9992e24fdfdd6a2fbe49a1cdeef61dfd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cb0b23859b..d6013fb531 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7ebf3e6286ddc8cdaa9446235407785d1be2be2d9992e21ef59fcd655f68432 \ No newline at end of file +dbc99662087b63c9ed5b398535a6091fc2c5e507907dd1fcb7ad0b6ab3f17144 \ No newline at end of file diff --git a/src/json.c b/src/json.c index a4ba2e0cfb..aa8bf64c86 100644 --- a/src/json.c +++ b/src/json.c @@ -931,11 +931,11 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ && z[i]=='-' && sqlite3StrNICmp(&z[j], "inf",3)==0 ){ - if( !sqlite3Isalnum(z[j+4]) ){ + if( !sqlite3Isalnum(z[j+3]) ){ jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); return i+4; }else if( (sqlite3StrNICmp(&z[j],"infinity",8)==0 && - !sqlite3Isalnum(z[j+9])) ){ + !sqlite3Isalnum(z[j+8])) ){ jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); return i+9; } From d8c34e3311179b7728ed2f7c32dbb4b299236c14 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 3 Apr 2023 12:33:12 +0000 Subject: [PATCH 160/341] Stronger constraint checking in allocateSpace(). dbsqlfuzz 93d4c9ff5ef7cd29f16e767af1ee71c29ec5a4c0 FossilOrigin-Name: 9e968f4fbce061190f10f31ce9d3eb4fce6706ea6b7e5011bfa1e893d37ca68d --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/btree.c | 5 +++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a93cce208b..e638755a60 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C With\sthe\s-DSQLITE_ENABLE_JSON_NAN_INF\scompile-time\soption,\snon-standard\nJSON\snumeric\svalues\s"Inf",\s"Infinity",\s"-Inf",\s"-Infinity",\s"NaN",\s"QNaN",\nand\s"SNaN"\sare\sall\saccepted.\s\sSQLite\sshould\snever\sgenerate\sthese\svalues,\nbut\sit\swill\saccept\sthat\swith\sthe\sappropriate\scompile-time\soption. -D 2023-04-02T20:56:29.250 +C Stronger\sconstraint\schecking\sin\sallocateSpace().\ndbsqlfuzz\s93d4c9ff5ef7cd29f16e767af1ee71c29ec5a4c0 +D 2023-04-03T12:33:12.512 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c c2ececcbcb1a35bbd9efcbb43a821eaec60cc9796dccbb33f3b0fd4c7ab2f539 +F src/btree.c 023b41e0a4563880812288e32682538013287b7b82e0c24a57b0e26b6f0ef168 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -2052,9 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 978dc71c388b37740da38c310674315c7d7fe814d1daa16a146b4df71385d1e1 dbc99662087b63c9ed5b398535a6091fc2c5e507907dd1fcb7ad0b6ab3f17144 -R 0c80a0a9d7056c4e9cc79fd3f48ab255 -T +closed dbc99662087b63c9ed5b398535a6091fc2c5e507907dd1fcb7ad0b6ab3f17144 +P 0a050e9013331595e13ca9f859180057b59291c70a6cedb6230eefb25956df9e +R b7ae8f88dfdd5f67e5cf7bfa8f601f36 U drh -Z ff807b459bbce1f6e27ef0b5e65a972d +Z a657f98c5955b6184103f4e037cbdd56 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 069e164985..07c70afe43 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a050e9013331595e13ca9f859180057b59291c70a6cedb6230eefb25956df9e \ No newline at end of file +9e968f4fbce061190f10f31ce9d3eb4fce6706ea6b7e5011bfa1e893d37ca68d \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b2edcd7d49..2f3063c3a3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1772,13 +1772,14 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** integer, so a value of 0 is used in its place. */ pTmp = &data[hdr+5]; top = get2byte(pTmp); - assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ if( gap>top ){ if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536; }else{ return SQLITE_CORRUPT_PAGE(pPage); } + }else if( top>(int)pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); } /* If there is enough space between gap and top for one more cell pointer, @@ -7585,7 +7586,7 @@ static int editPage( pData = &aData[get2byteNotZero(&aData[hdr+5])]; if( pDatapPg->aDataEnd ) goto editpage_fail; + if( NEVER(pData>pPg->aDataEnd) ) goto editpage_fail; /* Add cells to the start of the page */ if( iNew Date: Mon, 3 Apr 2023 12:48:51 +0000 Subject: [PATCH 161/341] Make the sqlite_dbdata and sqlite3_dbptr virtual tables accessible to the CLI. FossilOrigin-Name: c0eff02854d469bcdb949c09531760d6f96b49c17375abd1c131d41f38b9b2f0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e638755a60..035730cb61 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Stronger\sconstraint\schecking\sin\sallocateSpace().\ndbsqlfuzz\s93d4c9ff5ef7cd29f16e767af1ee71c29ec5a4c0 -D 2023-04-03T12:33:12.512 +C Make\sthe\ssqlite_dbdata\sand\ssqlite3_dbptr\svirtual\stables\saccessible\sto\nthe\sCLI. +D 2023-04-03T12:48:51.691 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 2fef08c06348c9b490f047926d322b0d58e556dae658f1dac14b9e4b9410fd46 -F src/shell.c.in d80b53a9ec0a43ac0157fc54d667cb49fb6dd2229558135530feab2fc04b909d +F src/shell.c.in 55a17c94e3e240f4d5f01eecff1f6f9bea7f8fd257e95d5ce014eef7b556a6e1 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a050e9013331595e13ca9f859180057b59291c70a6cedb6230eefb25956df9e -R b7ae8f88dfdd5f67e5cf7bfa8f601f36 +P 9e968f4fbce061190f10f31ce9d3eb4fce6706ea6b7e5011bfa1e893d37ca68d +R 8f1852e8d4e7f2a1a2abb9d3705dc108 U drh -Z a657f98c5955b6184103f4e037cbdd56 +Z 2852639c24f122942ec11fcb2efc3386 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 07c70afe43..671854ae15 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9e968f4fbce061190f10f31ce9d3eb4fce6706ea6b7e5011bfa1e893d37ca68d \ No newline at end of file +c0eff02854d469bcdb949c09531760d6f96b49c17375abd1c131d41f38b9b2f0 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 594efc4566..a4bce70582 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5203,6 +5203,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); + sqlite3_dbdata_init(p->db, 0, 0); #ifndef SQLITE_SHELL_FIDDLE sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); From 0669d6ebbfe68609ad7ac73b674a8476b6076d1b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 3 Apr 2023 15:01:37 +0000 Subject: [PATCH 162/341] Add the SQLITE_VTAB_USES_ALL_SCHEMAS option to sqlite3_vtab_config(). Update the sqlite_dbpage, sqlite_dbdata, and sqlite_dbptr virtual tables to make use of that interface. This was formerly handled by the internal sqlite3VtabUsesAllSchemas() routine that was called directly from sqlite_dbpage. But since sqlite_dbdata and sqlite_dbptr are an extension, an external interface to that functionality had to be provided. dbsqlfuzz 1a29c245175a63393b6a78c5b8cab5199939d6a8 FossilOrigin-Name: bcd51abee0b0f82bb5dbb881025a92d55baf9df6adeaf3a305e2e0da96a81d58 --- ext/recover/dbdata.c | 1 + manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/dbpage.c | 2 +- src/sqlite.h.in | 12 +++++++++++- src/sqliteInt.h | 6 ++---- src/vtab.c | 4 ++++ src/where.c | 10 ++++------ 8 files changed, 35 insertions(+), 24 deletions(-) diff --git a/ext/recover/dbdata.c b/ext/recover/dbdata.c index 4100202354..878a61f1d8 100644 --- a/ext/recover/dbdata.c +++ b/ext/recover/dbdata.c @@ -167,6 +167,7 @@ static int dbdataConnect( (void)argc; (void)argv; (void)pzErr; + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); if( rc==SQLITE_OK ){ pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); if( pTab==0 ){ diff --git a/manifest b/manifest index 035730cb61..09464fcaf9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\ssqlite_dbdata\sand\ssqlite3_dbptr\svirtual\stables\saccessible\sto\nthe\sCLI. -D 2023-04-03T12:48:51.691 +C Add\sthe\sSQLITE_VTAB_USES_ALL_SCHEMAS\soption\sto\ssqlite3_vtab_config().\s\sUpdate\nthe\ssqlite_dbpage,\ssqlite_dbdata,\sand\ssqlite_dbptr\svirtual\stables\sto\smake\nuse\sof\sthat\sinterface.\s\sThis\swas\sformerly\shandled\sby\sthe\sinternal\nsqlite3VtabUsesAllSchemas()\sroutine\sthat\swas\scalled\sdirectly\sfrom\ssqlite_dbpage.\nBut\ssince\ssqlite_dbdata\sand\ssqlite_dbptr\sare\san\sextension,\san\sexternal\ninterface\sto\sthat\sfunctionality\shad\sto\sbe\sprovided.\ndbsqlfuzz\s1a29c245175a63393b6a78c5b8cab5199939d6a8 +D 2023-04-03T15:01:37.035 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -365,7 +365,7 @@ F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69 F ext/rbu/sqlite3rbu.c 348bb6251e6ec459de102f8b2dd50789a98643ef7a28e56e4c787ac9659c15ea F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 -F ext/recover/dbdata.c 501227169803bfafb34cd8ec7fffda6e64c1a282ac33315d3a096076c5928feb +F ext/recover/dbdata.c 31d580785cf14eb3c20ed6fbb421a10a66569858f837928e6b326088c38d4c72 F ext/recover/recover1.test 2072993624d5e32fef20ae03b17fc06c02bcb344421fe17bb329b24d2a51e647 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverbuild.test a6f05273ff5fe517afd166444597c70cb97033e7f58496433a4428a1ecb5d79f @@ -572,7 +572,7 @@ F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 20507cc0b0a6c19cd882fcd0eaeda32ae6a4229fb4b024cfdf3183043d9b703d F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 -F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e +F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 F src/expr.c 5f2b5f25a90aa4ff1c84d175b1adf790be8f8719f3b38851e2b5b293e0e5e916 @@ -627,10 +627,10 @@ F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 2fef08c06348c9b490f047926d322b0d58e556dae658f1dac14b9e4b9410fd46 F src/shell.c.in 55a17c94e3e240f4d5f01eecff1f6f9bea7f8fd257e95d5ce014eef7b556a6e1 -F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938 +F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h f0edd1cd318a5bc02b2dcc68768bf1c5763628b6e3e75f30e81d0644e17d40aa +F src/sqliteInt.h 899781baef0d1dd0910524df6350e0ef7e2761131f6e04ec5e34f3b32e262998 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -705,12 +705,12 @@ F src/vdbemem.c db0458d11a51f6cfad2333a41e36a3795be0b2f316d070df5d33543a9ac884ac F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac -F src/vtab.c 4516e9fecff8d4f353132be0468bc6424b102937c50c52e01fd9a0a9e9e34ca5 +F src/vtab.c 4a1b231b5938de0282fbb605eb068ca673a1b6a383130f54d1bcbbac951988ad F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 2bafa1b7d1969d28135e4e7978f578a550d0a5b9005bc5a9180146a3de126880 +F src/where.c 8c348f08017ac6a766a4990816b111c0b8a272ada8b50fd7821bd267b03e18ac F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 41c17b089082558c1c5496e8a453b1f9a96485b6b355f318440fca2ee4754b2e F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9e968f4fbce061190f10f31ce9d3eb4fce6706ea6b7e5011bfa1e893d37ca68d -R 8f1852e8d4e7f2a1a2abb9d3705dc108 +P c0eff02854d469bcdb949c09531760d6f96b49c17375abd1c131d41f38b9b2f0 +R 163596c68b1c70989e5dfc4dd76d338c U drh -Z 2852639c24f122942ec11fcb2efc3386 +Z 23f00d31ae2c07aa6d8ef757c285add5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 671854ae15..6b1852d41b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0eff02854d469bcdb949c09531760d6f96b49c17375abd1c131d41f38b9b2f0 \ No newline at end of file +bcd51abee0b0f82bb5dbb881025a92d55baf9df6adeaf3a305e2e0da96a81d58 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index 17e5f44f59..32a9ce55bf 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -78,6 +78,7 @@ static int dbpageConnect( (void)pzErr; sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); rc = sqlite3_declare_vtab(db, "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); if( rc==SQLITE_OK ){ @@ -161,7 +162,6 @@ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ pIdxInfo->orderByConsumed = 1; } - sqlite3VtabUsesAllSchemas(pIdxInfo); return SQLITE_OK; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ec63e52c75..be86c1b17f 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -9606,18 +9606,28 @@ int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]
SQLITE_VTAB_INNOCUOUS
**
Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a ** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS ** flag unless absolutely necessary. **
+** +** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]
SQLITE_VTAB_USES_ALL_SCHEMAS
+**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** instruct the query planner to begin at least a read transaction on +** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the +** virtual table is used. +**
** */ #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 #define SQLITE_VTAB_INNOCUOUS 2 #define SQLITE_VTAB_DIRECTONLY 3 +#define SQLITE_VTAB_USES_ALL_SCHEMAS 4 /* ** CAPI3REF: Determine The Virtual Table Conflict Policy diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d393853371..9b189124d9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2312,6 +2312,7 @@ struct VTable { sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ u8 bConstraint; /* True if constraints are supported */ + u8 bAllSchemas; /* True if might use any attached schema */ u8 eVtabRisk; /* Riskiness of allowing hacker access */ int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ @@ -5340,10 +5341,7 @@ int sqlite3VtabCallDestroy(sqlite3*, int, const char *); int sqlite3VtabBegin(sqlite3 *, VTable *); FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); -#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ - && !defined(SQLITE_OMIT_VIRTUALTABLE) - void sqlite3VtabUsesAllSchemas(sqlite3_index_info*); -#endif +void sqlite3VtabUsesAllSchemas(Parse*); sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); diff --git a/src/vtab.c b/src/vtab.c index 58452dfc5d..3477d67ce3 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -1331,6 +1331,10 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ p->pVTable->eVtabRisk = SQLITE_VTABRISK_High; break; } + case SQLITE_VTAB_USES_ALL_SCHEMAS: { + p->pVTable->bAllSchemas = 1; + break; + } default: { rc = SQLITE_MISUSE_BKPT; break; diff --git a/src/where.c b/src/where.c index 509d14ff77..5beced7951 100644 --- a/src/where.c +++ b/src/where.c @@ -1487,6 +1487,9 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); } } + if( pTab->u.vtab.p->bAllSchemas ){ + sqlite3VtabUsesAllSchemas(pParse); + } sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; return rc; @@ -4118,8 +4121,6 @@ int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ return pHidden->eDistinct; } -#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ - && !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** Cause the prepared statement that is associated with a call to ** xBestIndex to potentially use all schemas. If the statement being @@ -4129,9 +4130,7 @@ int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ ** ** This is used by the (built-in) sqlite_dbpage virtual table. */ -void sqlite3VtabUsesAllSchemas(sqlite3_index_info *pIdxInfo){ - HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; - Parse *pParse = pHidden->pParse; +void sqlite3VtabUsesAllSchemas(Parse *pParse){ int nDb = pParse->db->nDb; int i; for(i=0; i Date: Mon, 3 Apr 2023 17:46:14 +0000 Subject: [PATCH 163/341] When changing a COLLATE expression node into TK_AGG_COLUMN because the nodes value is contained in an indexed expression, be sure to clear the EP_Collate property from the expression node. Fix for the assertion faults reported by [forum:/forumpost/e45108732c|forum post e45108732c] and [forum:/forumpost/44270909bb|forum post 44270909bb]. FossilOrigin-Name: cf6454ce26983b9c3ae924c44a802f3f760eaaae9547b40aee9e14e7b0c47cab --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 + test/indexexpr2.test | 24 ++++++++++++++++++++++++ 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 09464fcaf9..d430e5a075 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_VTAB_USES_ALL_SCHEMAS\soption\sto\ssqlite3_vtab_config().\s\sUpdate\nthe\ssqlite_dbpage,\ssqlite_dbdata,\sand\ssqlite_dbptr\svirtual\stables\sto\smake\nuse\sof\sthat\sinterface.\s\sThis\swas\sformerly\shandled\sby\sthe\sinternal\nsqlite3VtabUsesAllSchemas()\sroutine\sthat\swas\scalled\sdirectly\sfrom\ssqlite_dbpage.\nBut\ssince\ssqlite_dbdata\sand\ssqlite_dbptr\sare\san\sextension,\san\sexternal\ninterface\sto\sthat\sfunctionality\shad\sto\sbe\sprovided.\ndbsqlfuzz\s1a29c245175a63393b6a78c5b8cab5199939d6a8 -D 2023-04-03T15:01:37.035 +C When\schanging\sa\sCOLLATE\sexpression\snode\sinto\sTK_AGG_COLUMN\sbecause\sthe\snodes\nvalue\sis\scontained\sin\san\sindexed\sexpression,\sbe\ssure\sto\sclear\sthe\nEP_Collate\sproperty\sfrom\sthe\sexpression\snode.\s\sFix\sfor\sthe\sassertion\sfaults\nreported\sby\s[forum:/forumpost/e45108732c|forum\spost\se45108732c]\sand\n[forum:/forumpost/44270909bb|forum\spost\s44270909bb]. +D 2023-04-03T17:46:14.710 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -625,7 +625,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 2fef08c06348c9b490f047926d322b0d58e556dae658f1dac14b9e4b9410fd46 +F src/select.c 2584199f36de79d0845e1d1dedc5418b33299fa89d2b7e15df30393bcf2f3704 F src/shell.c.in 55a17c94e3e240f4d5f01eecff1f6f9bea7f8fd257e95d5ce014eef7b556a6e1 F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1196,7 +1196,7 @@ F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a91 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 F test/indexexpr1.test b2a15637dcbae7fd8d7e2fc51f74ac4feaf5510130ee2089a5ec5bd1ef7270e1 -F test/indexexpr2.test 8a4cf83e12b041334c5ef36b9a3932451808378352c24d65049b31f3737a82b9 +F test/indexexpr2.test f5732485abfe8edb5006a4e599914afd8ff7ff7b88d9b88e1731cb063495ba36 F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 4e3f0de67aac3c5be1f4aaedbcea11638f1b5cdc9a3115be14d19aa9db7623c6 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c0eff02854d469bcdb949c09531760d6f96b49c17375abd1c131d41f38b9b2f0 -R 163596c68b1c70989e5dfc4dd76d338c +P bcd51abee0b0f82bb5dbb881025a92d55baf9df6adeaf3a305e2e0da96a81d58 +R a62e4585b51771f1a5bd36f0c33ce6ea U drh -Z 23f00d31ae2c07aa6d8ef757c285add5 +Z 4c60945d3ef4bd40ca949dea1a2f0787 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6b1852d41b..0eb4abedd0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bcd51abee0b0f82bb5dbb881025a92d55baf9df6adeaf3a305e2e0da96a81d58 \ No newline at end of file +cf6454ce26983b9c3ae924c44a802f3f760eaaae9547b40aee9e14e7b0c47cab \ No newline at end of file diff --git a/src/select.c b/src/select.c index d97ecb2af6..d959b8988b 100644 --- a/src/select.c +++ b/src/select.c @@ -6491,6 +6491,7 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ pExpr->op = TK_AGG_COLUMN; pExpr->iTable = pCol->iTable; pExpr->iColumn = pCol->iColumn; + ExprClearProperty(pExpr, EP_Skip|EP_Collate); return WRC_Prune; } diff --git a/test/indexexpr2.test b/test/indexexpr2.test index 3cc030f86b..a934d0c84d 100644 --- a/test/indexexpr2.test +++ b/test/indexexpr2.test @@ -386,4 +386,28 @@ do_execsql_test 9.0 { FROM t1 WHERE a=5; } {5 -5 205 5 20 220} +# 2023-04-03 https://sqlite.org/forum/forumpost/44270909bb +# and https://sqlite.org/forum/forumpost/e45108732c which are the +# same problem, namely the failure to omit the EP_Collate property +# from an expression node when changing it from TK_COLLATE into +# TK_AGG_COLUMN because it resolves to an indexed expression. +# +reset_db +do_execsql_test 10.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT); + CREATE INDEX t1x ON t1 (b, +b COLLATE NOCASE); + INSERT INTO t1(a,b) VALUES(1,'abcde'); + SELECT * FROM t1 AS a0 + WHERE (SELECT count(a0.b=+a0.b COLLATE NOCASE IN (b)) FROM t1 GROUP BY 2.5) + ORDER BY a0.b; +} {1 abcde} +do_execsql_test 10.1 { + CREATE TABLE t2(a TEXT); + INSERT INTO t2 VALUES('alice'),('bob'),('cindy'),('david'); + CREATE INDEX t2x ON t2 (+a COLLATE NOCASE); + SELECT count(+a COLLATE NOCASE IN (SELECT 1)) AS x + FROM t2 + GROUP BY SUBSTR(0,0); +} 4 + finish_test From f61bf3dbe81bc0f535822b3802ff5fead67f09b5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 3 Apr 2023 20:11:06 +0000 Subject: [PATCH 164/341] Improved diagnostic output from PRAGMA vdbe_addoptrace. FossilOrigin-Name: 050958c1828f7446e7ee7e458123a23b735e213b76a67c3dd8356da256cbc170 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d430e5a075..9ecd2bf54b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\schanging\sa\sCOLLATE\sexpression\snode\sinto\sTK_AGG_COLUMN\sbecause\sthe\snodes\nvalue\sis\scontained\sin\san\sindexed\sexpression,\sbe\ssure\sto\sclear\sthe\nEP_Collate\sproperty\sfrom\sthe\sexpression\snode.\s\sFix\sfor\sthe\sassertion\sfaults\nreported\sby\s[forum:/forumpost/e45108732c|forum\spost\se45108732c]\sand\n[forum:/forumpost/44270909bb|forum\spost\s44270909bb]. -D 2023-04-03T17:46:14.710 +C Improved\sdiagnostic\soutput\sfrom\sPRAGMA\svdbe_addoptrace. +D 2023-04-03T20:11:06.914 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -710,7 +710,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 8c348f08017ac6a766a4990816b111c0b8a272ada8b50fd7821bd267b03e18ac +F src/where.c 16ba2073aa2139bf97576c49baa6189a33200e123cbc86374a7410a0aed5a117 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 41c17b089082558c1c5496e8a453b1f9a96485b6b355f318440fca2ee4754b2e F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bcd51abee0b0f82bb5dbb881025a92d55baf9df6adeaf3a305e2e0da96a81d58 -R a62e4585b51771f1a5bd36f0c33ce6ea +P cf6454ce26983b9c3ae924c44a802f3f760eaaae9547b40aee9e14e7b0c47cab +R a5a9cc895be767d265a98905577427bc U drh -Z 4c60945d3ef4bd40ca949dea1a2f0787 +Z 52b59f8ff34baff0c46f83506d1e601b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0eb4abedd0..f745366b3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf6454ce26983b9c3ae924c44a802f3f760eaaae9547b40aee9e14e7b0c47cab \ No newline at end of file +050958c1828f7446e7ee7e458123a23b735e213b76a67c3dd8356da256cbc170 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 5beced7951..0bed46de65 100644 --- a/src/where.c +++ b/src/where.c @@ -6698,7 +6698,8 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ k = pLevel->addrBody + 1; #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeAddopTrace ){ - printf("TRANSLATE opcodes in range %d..%d\n", k, last-1); + printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n", + pLevel->iTabCur, pLevel->iIdxCur, k, last-1); } /* Proof that the "+1" on the k value above is safe */ pOp = sqlite3VdbeGetOp(v, k - 1); From 39b07f1afce2dc222bab4dc3889eb1ce23a7cf83 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 3 Apr 2023 23:49:00 +0000 Subject: [PATCH 165/341] When translating arguments of aggregate functions into references to expression indexes, make sure to only translate them for the current aggregate when there are nested aggregates. [forum/forumpost/409ebc7368|Forum post 409ebc7368]. FossilOrigin-Name: 898bfa1afd8260eaaf2aa6db94e74d99ebf4e8a6dc02cf21d20cd981393609a5 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 9 +++++++-- src/select.c | 2 +- test/indexexpr2.test | 15 +++++++++++++++ 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 9ecd2bf54b..710c4e533f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdiagnostic\soutput\sfrom\sPRAGMA\svdbe_addoptrace. -D 2023-04-03T20:11:06.914 +C When\stranslating\sarguments\sof\saggregate\sfunctions\sinto\sreferences\sto\nexpression\sindexes,\smake\ssure\sto\sonly\stranslate\sthem\sfor\sthe\scurrent\naggregate\swhen\sthere\sare\snested\saggregates.\n[forum/forumpost/409ebc7368|Forum\spost\s409ebc7368]. +D 2023-04-03T23:49:00.977 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 5f2b5f25a90aa4ff1c84d175b1adf790be8f8719f3b38851e2b5b293e0e5e916 +F src/expr.c 9586f2dafe724b756d56657945b0d9c5177e10053d5cd83dbdd50f3f6d32c407 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -625,7 +625,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 2584199f36de79d0845e1d1dedc5418b33299fa89d2b7e15df30393bcf2f3704 +F src/select.c df007d60326a9aad94f5a4915d01e7de19d77627ed09338747a2e8e741a75add F src/shell.c.in 55a17c94e3e240f4d5f01eecff1f6f9bea7f8fd257e95d5ce014eef7b556a6e1 F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1196,7 +1196,7 @@ F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a91 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 F test/indexexpr1.test b2a15637dcbae7fd8d7e2fc51f74ac4feaf5510130ee2089a5ec5bd1ef7270e1 -F test/indexexpr2.test f5732485abfe8edb5006a4e599914afd8ff7ff7b88d9b88e1731cb063495ba36 +F test/indexexpr2.test 1c382e81ef996d8ae8b834a74f2a9013dddf59214c32201d7c8a656d739f999a F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 4e3f0de67aac3c5be1f4aaedbcea11638f1b5cdc9a3115be14d19aa9db7623c6 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cf6454ce26983b9c3ae924c44a802f3f760eaaae9547b40aee9e14e7b0c47cab -R a5a9cc895be767d265a98905577427bc +P 050958c1828f7446e7ee7e458123a23b735e213b76a67c3dd8356da256cbc170 +R 8f74c88e7968e6ca235635bc40f218cf U drh -Z 52b59f8ff34baff0c46f83506d1e601b +Z 23f5beb74f4ed7eb72f5bf65796cf745 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f745366b3b..6392ea38c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -050958c1828f7446e7ee7e458123a23b735e213b76a67c3dd8356da256cbc170 \ No newline at end of file +898bfa1afd8260eaaf2aa6db94e74d99ebf4e8a6dc02cf21d20cd981393609a5 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 4b536a8435..bcc71fad1f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6288,7 +6288,7 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ sqlite3 *db = pParse->db; assert( iAgg>=0 ); if( pExpr->op!=TK_AGG_FUNCTION ){ - if( iAggnColumn + if( ALWAYS(iAggnColumn) && pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); @@ -6441,6 +6441,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ default: { IndexedExpr *pIEpr; Expr tmp; + int i; assert( pParse->iSelfTab==0 ); if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; if( pParse->pIdxEpr==0 ) break; @@ -6451,7 +6452,11 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ } if( pIEpr==0 ) break; if( NEVER(!ExprUseYTab(pExpr)) ) break; - if( pExpr->pAggInfo!=0 ) break; /* Already resolved by outer context */ + for(i=0; inSrc; i++){ + if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break; + } + if( i>=pSrcList->nSrc ) break; + if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ if( pParse->nErr ){ return WRC_Abort; } /* If we reach this point, it means that expression pExpr can be diff --git a/src/select.c b/src/select.c index d959b8988b..9e070be0b3 100644 --- a/src/select.c +++ b/src/select.c @@ -6485,7 +6485,7 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; pAggInfo = pExpr->pAggInfo; - if( pExpr->iAgg>=pAggInfo->nColumn ) return WRC_Continue; + if( NEVER(pExpr->iAgg>=pAggInfo->nColumn) ) return WRC_Continue; assert( pExpr->iAgg>=0 ); pCol = &pAggInfo->aCol[pExpr->iAgg]; pExpr->op = TK_AGG_COLUMN; diff --git a/test/indexexpr2.test b/test/indexexpr2.test index a934d0c84d..4c21421e8e 100644 --- a/test/indexexpr2.test +++ b/test/indexexpr2.test @@ -410,4 +410,19 @@ do_execsql_test 10.1 { GROUP BY SUBSTR(0,0); } 4 +# 2023-04-03 https://sqlite.org/forum/forumpost/409ebc7368 +# When a generated column appears in both an outer and an inner loop +# (that is to say, the same table is used in both loops) and the +# generated column is indexed and it is used inside an aggregate function, +# make sure that the terms resolve to the correct aggregate. +# +do_execsql_test 11.0 { + CREATE TABLE t3 (a INT, b AS (-a)); + CREATE INDEX t3x ON t3(b, a); + INSERT INTO t3(a) VALUES(44); + SELECT * FROM t3 AS a0 + WHERE (SELECT sum(-a0.a=b) FROM t3 GROUP BY b) + GROUP BY b; +} {44 -44} + finish_test From 0d3651bd856f9ff0b5dee1ea499f2ab92eaa0748 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 4 Apr 2023 08:49:22 +0000 Subject: [PATCH 166/341] Omit shell call to sqlite3_dbdata_init() when it is omitted. FossilOrigin-Name: 0421cc03e0efa8f1a83e46e91f26837366c10eef0903d099e25eb0cc99c297df --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 710c4e533f..75f9a6883e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\stranslating\sarguments\sof\saggregate\sfunctions\sinto\sreferences\sto\nexpression\sindexes,\smake\ssure\sto\sonly\stranslate\sthem\sfor\sthe\scurrent\naggregate\swhen\sthere\sare\snested\saggregates.\n[forum/forumpost/409ebc7368|Forum\spost\s409ebc7368]. -D 2023-04-03T23:49:00.977 +C Omit\sshell\scall\sto\ssqlite3_dbdata_init()\swhen\sit\sis\somitted. +D 2023-04-04T08:49:22.005 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c df007d60326a9aad94f5a4915d01e7de19d77627ed09338747a2e8e741a75add -F src/shell.c.in 55a17c94e3e240f4d5f01eecff1f6f9bea7f8fd257e95d5ce014eef7b556a6e1 +F src/shell.c.in b3bc71c0f3259992b7279bbc5a419ab2a9678fad565ae5ec3ed07d291fc027a6 F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 050958c1828f7446e7ee7e458123a23b735e213b76a67c3dd8356da256cbc170 -R 8f74c88e7968e6ca235635bc40f218cf -U drh -Z 23f5beb74f4ed7eb72f5bf65796cf745 +P 898bfa1afd8260eaaf2aa6db94e74d99ebf4e8a6dc02cf21d20cd981393609a5 +R b20e56a69ad2de928705c998949ae0d5 +U larrybr +Z c95b11a68fcbec82a06363e2fd4c58bc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6392ea38c8..77f96ee5e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -898bfa1afd8260eaaf2aa6db94e74d99ebf4e8a6dc02cf21d20cd981393609a5 \ No newline at end of file +0421cc03e0efa8f1a83e46e91f26837366c10eef0903d099e25eb0cc99c297df \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index a4bce70582..582a7594ab 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5203,7 +5203,9 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); +#ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3_dbdata_init(p->db, 0, 0); +#endif #ifndef SQLITE_SHELL_FIDDLE sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); From 20537f0dc76137f8ca08b372690cd0b22ef18681 Mon Sep 17 00:00:00 2001 From: stephan Date: Tue, 4 Apr 2023 17:35:38 +0000 Subject: [PATCH 167/341] Expose the new SQLITE_VTAB_USES_ALL_SCHEMAS to JS. FossilOrigin-Name: b7ef09be667dd349e195842304c03cbebb1693164e1c3587a5d7c96baa713edf --- ext/wasm/api/sqlite3-wasm.c | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index a3b349e5ff..61ebe084da 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -928,6 +928,7 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_VTAB_CONSTRAINT_SUPPORT); DefInt(SQLITE_VTAB_INNOCUOUS); DefInt(SQLITE_VTAB_DIRECTONLY); + DefInt(SQLITE_VTAB_USES_ALL_SCHEMAS); DefInt(SQLITE_ROLLBACK); //DefInt(SQLITE_IGNORE); // Also used by sqlite3_authorizer() callback DefInt(SQLITE_FAIL); diff --git a/manifest b/manifest index 75f9a6883e..d2efe01134 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sshell\scall\sto\ssqlite3_dbdata_init()\swhen\sit\sis\somitted. -D 2023-04-04T08:49:22.005 +C Expose\sthe\snew\sSQLITE_VTAB_USES_ALL_SCHEMAS\sto\sJS. +D 2023-04-04T17:35:38.816 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 5cb63f847737ded723d0bdf9b3313e8f7deb0bbfd0082dedde540991edde155b F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c e04e863be7c3a58aa144a463770c1b68ce69f7cfc9e1a5d6397a67877acaf006 +F ext/wasm/api/sqlite3-wasm.c c42413ca9f3e64c424b2bbfc5decf639670ca38bc8f7afb7760d5379398c9307 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 898bfa1afd8260eaaf2aa6db94e74d99ebf4e8a6dc02cf21d20cd981393609a5 -R b20e56a69ad2de928705c998949ae0d5 -U larrybr -Z c95b11a68fcbec82a06363e2fd4c58bc +P 0421cc03e0efa8f1a83e46e91f26837366c10eef0903d099e25eb0cc99c297df +R 0e21ece7a390cae1473c17b5a84e1bea +U stephan +Z 9f2003caba47df8e36725e986bdaab35 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 77f96ee5e2..6dd838f59e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0421cc03e0efa8f1a83e46e91f26837366c10eef0903d099e25eb0cc99c297df \ No newline at end of file +b7ef09be667dd349e195842304c03cbebb1693164e1c3587a5d7c96baa713edf \ No newline at end of file From c6a54f418c3dc75ac552e03a084b4d6e29c258e3 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 4 Apr 2023 18:10:23 +0000 Subject: [PATCH 168/341] Remove an assert() statement that is no longer valid due to enhancements to query planner for improved use of indexes. [forum:/forumpost/dc16ec63d3 |Forum post dc16ec63d3]. FossilOrigin-Name: 2b23dd249d0bc254308f5539936d33ee558f1100dec616caac7317dbe70db761 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 1 - test/in.test | 13 +++++++++++++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d2efe01134..7491cf9216 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expose\sthe\snew\sSQLITE_VTAB_USES_ALL_SCHEMAS\sto\sJS. -D 2023-04-04T17:35:38.816 +C Remove\san\sassert()\sstatement\sthat\sis\sno\slonger\svalid\sdue\sto\senhancements\nto\squery\splanner\sfor\simproved\suse\sof\sindexes.\n[forum:/forumpost/dc16ec63d3\s|Forum\spost\sdc16ec63d3]. +D 2023-04-04T18:10:23.765 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 9586f2dafe724b756d56657945b0d9c5177e10053d5cd83dbdd50f3f6d32c407 +F src/expr.c 8e30ea19cd8c08659604fb6a25841f6c2878ad6af6929a7d8bf1fe2401b34db1 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -1168,7 +1168,7 @@ F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 716a6b89fbabe5cc63e0cd4c260befb08fd7b9d761f04d43669233292f0753b1 F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 -F test/in.test 7399a6562fb09de7f786901f0b24b7bbe7c0b541f29ead9eb13f091edea37da4 +F test/in.test faf3be786a263d18eee6c2a8c3a81fbf58364ed40dedabc7e4f09de61dc91292 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test fdd1d8134da8376985c2edba6035a2de1f6c731524d2ffa651419e8fe2cd1c5a @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0421cc03e0efa8f1a83e46e91f26837366c10eef0903d099e25eb0cc99c297df -R 0e21ece7a390cae1473c17b5a84e1bea -U stephan -Z 9f2003caba47df8e36725e986bdaab35 +P b7ef09be667dd349e195842304c03cbebb1693164e1c3587a5d7c96baa713edf +R 99691209e816c0f3ddcb06c83f00e948 +U drh +Z 31348ba733ecca5969721d9d50a8c15c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6dd838f59e..a2984fa465 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b7ef09be667dd349e195842304c03cbebb1693164e1c3587a5d7c96baa713edf \ No newline at end of file +2b23dd249d0bc254308f5539936d33ee558f1100dec616caac7317dbe70db761 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index bcc71fad1f..47cfba2232 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2936,7 +2936,6 @@ int sqlite3FindInIndex( CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); int j; - assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr ); for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; assert( pIdx->azColl[j] ); diff --git a/test/in.test b/test/in.test index 716c17f593..f1632b0d96 100644 --- a/test/in.test +++ b/test/in.test @@ -829,6 +829,19 @@ do_execsql_test in-22.4 { SELECT * FROM t1 WHERE x IN ((((((SELECT a FROM t2)))))); } {2 200 4 400 6 600} +# 2023-04-04 https://sqlite.org/forum/forumpost/dc16ec63d3 +# Faulty assert() statement in the IN optimization. +# +do_execsql_test in-23.0 { + DROP TABLE IF EXISTS t4; + CREATE TABLE t4(a TEXT, b INT); + INSERT INTO t4(a,b) VALUES('abc',0),('ABC',1),('def',2); + CREATE INDEX t4x ON t4(a, +a COLLATE NOCASE); + SELECT a0.a, group_concat(a1.a) AS b + FROM t4 AS a0 JOIN t4 AS a1 + GROUP BY a0.a + HAVING (SELECT sum( (a1.a == +a0.a COLLATE NOCASE) IN (SELECT b FROM t4))); +} {ABC abc,ABC,def abc abc,ABC,def def abc,ABC,def} From cd0f540761a1e54a8f2c6d5260315875b642b061 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 4 Apr 2023 18:55:31 +0000 Subject: [PATCH 169/341] Fix an incorrect entry in the array that maps sqlite3_value values into actual datatype numbers. dbsqlfuzz f660c659bcec48577a43d3bab37f46baaa22f59e FossilOrigin-Name: fa8537dc90cad962dec695418d48da2890a172801ecad0c7c804023a063d3a02 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7491cf9216..94d30f5cb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sassert()\sstatement\sthat\sis\sno\slonger\svalid\sdue\sto\senhancements\nto\squery\splanner\sfor\simproved\suse\sof\sindexes.\n[forum:/forumpost/dc16ec63d3\s|Forum\spost\sdc16ec63d3]. -D 2023-04-04T18:10:23.765 +C Fix\san\sincorrect\sentry\sin\sthe\sarray\sthat\smaps\ssqlite3_value\svalues\sinto\nactual\sdatatype\snumbers.\s\sdbsqlfuzz\sf660c659bcec48577a43d3bab37f46baaa22f59e +D 2023-04-04T18:55:31.328 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -698,7 +698,7 @@ F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb -F src/vdbeapi.c 03a21e617ce787300699cfda7ec12053e74b5194af34f4dd2593e4190d3d20de +F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 F src/vdbeaux.c 2ec444e0be19d2cf671e2f3ae331965f5d9e24625525cc947048ab61625e1a37 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c db0458d11a51f6cfad2333a41e36a3795be0b2f316d070df5d33543a9ac884ac @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b7ef09be667dd349e195842304c03cbebb1693164e1c3587a5d7c96baa713edf -R 99691209e816c0f3ddcb06c83f00e948 +P 2b23dd249d0bc254308f5539936d33ee558f1100dec616caac7317dbe70db761 +R f84b3fd47f49ba254250f609c6bf0e88 U drh -Z 31348ba733ecca5969721d9d50a8c15c +Z 531e29c60ba2bb032a54c2bb58a3e2cb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a2984fa465..281638a3df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b23dd249d0bc254308f5539936d33ee558f1100dec616caac7317dbe70db761 \ No newline at end of file +fa8537dc90cad962dec695418d48da2890a172801ecad0c7c804023a063d3a02 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 794e2289e8..d1e1232b1a 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -271,7 +271,7 @@ int sqlite3_value_type(sqlite3_value* pVal){ SQLITE_NULL, /* 0x1f (not possible) */ SQLITE_FLOAT, /* 0x20 INTREAL */ SQLITE_NULL, /* 0x21 (not possible) */ - SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ + SQLITE_FLOAT, /* 0x22 INTREAL + TEXT */ SQLITE_NULL, /* 0x23 (not possible) */ SQLITE_FLOAT, /* 0x24 (not possible) */ SQLITE_NULL, /* 0x25 (not possible) */ From c6c4bae80aa5caa68225048646cb6d5eaf561f70 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 4 Apr 2023 19:56:28 +0000 Subject: [PATCH 170/341] Shell to use SQLITE_SHELL_HAVE_RECOVER consistently (correcting check-in 0421cc03e0efa8f1) FossilOrigin-Name: 5b980d72a03fa1cfd0f1bf3ed04068b9f216b75a304deb2b7bbe8ddce0e6fb96 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 6 ++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 94d30f5cb1..6ee44ae2a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\sentry\sin\sthe\sarray\sthat\smaps\ssqlite3_value\svalues\sinto\nactual\sdatatype\snumbers.\s\sdbsqlfuzz\sf660c659bcec48577a43d3bab37f46baaa22f59e -D 2023-04-04T18:55:31.328 +C Shell\sto\suse\sSQLITE_SHELL_HAVE_RECOVER\sconsistently\s(correcting\scheck-in\s0421cc03e0efa8f1) +D 2023-04-04T19:56:28.782 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c df007d60326a9aad94f5a4915d01e7de19d77627ed09338747a2e8e741a75add -F src/shell.c.in b3bc71c0f3259992b7279bbc5a419ab2a9678fad565ae5ec3ed07d291fc027a6 +F src/shell.c.in 4532aa8748d0aa76ac1d5ffb1e58405556197c3482b033b91c536501adb78a6f F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2b23dd249d0bc254308f5539936d33ee558f1100dec616caac7317dbe70db761 -R f84b3fd47f49ba254250f609c6bf0e88 -U drh -Z 531e29c60ba2bb032a54c2bb58a3e2cb +P fa8537dc90cad962dec695418d48da2890a172801ecad0c7c804023a063d3a02 +R b56dd0792b457a030e49bb49e1966109 +U larrybr +Z 9c0bb6709148976b80e47a4a30512730 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 281638a3df..48a4070c86 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa8537dc90cad962dec695418d48da2890a172801ecad0c7c804023a063d3a02 \ No newline at end of file +5b980d72a03fa1cfd0f1bf3ed04068b9f216b75a304deb2b7bbe8ddce0e6fb96 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 582a7594ab..cf7606ddaf 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1183,10 +1183,8 @@ INCLUDE ../ext/expert/sqlite3expert.c #endif #if SQLITE_SHELL_HAVE_RECOVER INCLUDE ../ext/recover/sqlite3recover.h -# ifndef SQLITE_HAVE_SQLITE3R INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.c -# endif #endif #ifdef SQLITE_SHELL_EXTSRC # include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) @@ -5203,7 +5201,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_regexp_init(p->db, 0, 0); sqlite3_ieee_init(p->db, 0, 0); sqlite3_series_init(p->db, 0, 0); -#ifndef SQLITE_OMIT_VIRTUALTABLE +#if SQLITE_SHELL_HAVE_RECOVER sqlite3_dbdata_init(p->db, 0, 0); #endif #ifndef SQLITE_SHELL_FIDDLE @@ -6021,7 +6019,7 @@ static int db_int(sqlite3 *db, const char *zSql){ return res; } -#if defined(SQLITE_SHELL_HAVE_RECOVER) +#if SQLITE_SHELL_HAVE_RECOVER /* ** Convert a 2-byte or 4-byte big-endian integer into a native integer */ From 8907cb2f88be0528092661a9d40fc57911027e9b Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 5 Apr 2023 02:21:57 +0000 Subject: [PATCH 171/341] Fix the function that determines the collating function for an expression tree to handle new cases that arise as a result of the recently added ability to use indexed expressions in aggregate queries. [forum/forumpost/0713a16a44|Forum post 0713a16a44]. FossilOrigin-Name: cc5041f3f067cf610adffb868b4e2d1b5d248dc5a0ecc551339b670800ecb0ff --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 5 ++--- test/in.test | 13 +++++++++++-- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 6ee44ae2a5..08dc7efb09 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Shell\sto\suse\sSQLITE_SHELL_HAVE_RECOVER\sconsistently\s(correcting\scheck-in\s0421cc03e0efa8f1) -D 2023-04-04T19:56:28.782 +C Fix\sthe\sfunction\sthat\sdetermines\sthe\scollating\sfunction\sfor\san\sexpression\ntree\sto\shandle\snew\scases\sthat\sarise\sas\sa\sresult\sof\sthe\srecently\sadded\nability\sto\suse\sindexed\sexpressions\sin\saggregate\squeries.\n[forum/forumpost/0713a16a44|Forum\spost\s0713a16a44]. +D 2023-04-05T02:21:57.915 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 8e30ea19cd8c08659604fb6a25841f6c2878ad6af6929a7d8bf1fe2401b34db1 +F src/expr.c 60b2e406df2d3ebd05302659b0f1354026f1145477428436ef3ae9ce98957908 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -1168,7 +1168,7 @@ F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 716a6b89fbabe5cc63e0cd4c260befb08fd7b9d761f04d43669233292f0753b1 F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 -F test/in.test faf3be786a263d18eee6c2a8c3a81fbf58364ed40dedabc7e4f09de61dc91292 +F test/in.test 291d881deae63fc2f0f3a9d7fdc1d300db58a6812c08e7c0370c970a984ecfbf F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test fdd1d8134da8376985c2edba6035a2de1f6c731524d2ffa651419e8fe2cd1c5a @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fa8537dc90cad962dec695418d48da2890a172801ecad0c7c804023a063d3a02 -R b56dd0792b457a030e49bb49e1966109 -U larrybr -Z 9c0bb6709148976b80e47a4a30512730 +P 5b980d72a03fa1cfd0f1bf3ed04068b9f216b75a304deb2b7bbe8ddce0e6fb96 +R 7c7fc015455f5e833e4cdc87fb91e9e2 +U drh +Z ea122c35df9ab34c8929b37c7d61629c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 48a4070c86..ed4439c275 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b980d72a03fa1cfd0f1bf3ed04068b9f216b75a304deb2b7bbe8ddce0e6fb96 \ No newline at end of file +cc5041f3f067cf610adffb868b4e2d1b5d248dc5a0ecc551339b670800ecb0ff \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 47cfba2232..f1ce6ba4a2 100644 --- a/src/expr.c +++ b/src/expr.c @@ -278,9 +278,8 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ }else{ Expr *pNext = p->pRight; /* The Expr.x union is never used at the same time as Expr.pRight */ - assert( ExprUseXList(p) ); - assert( p->x.pList==0 || p->pRight==0 ); - if( p->x.pList!=0 && !db->mallocFailed ){ + assert( !ExprUseXList(p) || p->x.pList==0 || p->pRight==0 ); + if( ExprUseXList(p) && p->x.pList!=0 && !db->mallocFailed ){ int i; for(i=0; ALWAYS(ix.pList->nExpr); i++){ if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ diff --git a/test/in.test b/test/in.test index f1632b0d96..0fc7a4e2c1 100644 --- a/test/in.test +++ b/test/in.test @@ -842,7 +842,16 @@ do_execsql_test in-23.0 { GROUP BY a0.a HAVING (SELECT sum( (a1.a == +a0.a COLLATE NOCASE) IN (SELECT b FROM t4))); } {ABC abc,ABC,def abc abc,ABC,def def abc,ABC,def} - - +# +# Follow-up forum/forumpost/0713a16a44 +# +do_execsql_test in-23.1 { + CREATE VIEW t5 AS + SELECT 1 AS b + WHERE (SELECT count(0=NOT+a COLLATE NOCASE IN (SELECT 0)) + FROM t4 + GROUP BY a); + SELECT * FROM t5; +} 1 finish_test From 9d10ba8b1f16db5ee51fd661edd8c7a336fcabb1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 5 Apr 2023 02:50:00 +0000 Subject: [PATCH 172/341] Remove an ALWAYS() that might now be false due to the prior check-in. FossilOrigin-Name: fc68993501aaa7180f5457dcb3c296e5b199904a385d98e2bcad7854e34d428e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 08dc7efb09..8c4aa0ad86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sfunction\sthat\sdetermines\sthe\scollating\sfunction\sfor\san\sexpression\ntree\sto\shandle\snew\scases\sthat\sarise\sas\sa\sresult\sof\sthe\srecently\sadded\nability\sto\suse\sindexed\sexpressions\sin\saggregate\squeries.\n[forum/forumpost/0713a16a44|Forum\spost\s0713a16a44]. -D 2023-04-05T02:21:57.915 +C Remove\san\sALWAYS()\sthat\smight\snow\sbe\sfalse\sdue\sto\sthe\sprior\scheck-in. +D 2023-04-05T02:50:00.410 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 60b2e406df2d3ebd05302659b0f1354026f1145477428436ef3ae9ce98957908 +F src/expr.c 1c17d1ff2e5924178b82355d192b4927030b3c39317feff5633aecec8f77d0fe F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5b980d72a03fa1cfd0f1bf3ed04068b9f216b75a304deb2b7bbe8ddce0e6fb96 -R 7c7fc015455f5e833e4cdc87fb91e9e2 +P cc5041f3f067cf610adffb868b4e2d1b5d248dc5a0ecc551339b670800ecb0ff +R 13f76b464bbf0db7eee4f47f89c8f40d U drh -Z ea122c35df9ab34c8929b37c7d61629c +Z c4aa246b09ce02d9717280a76242d5a6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ed4439c275..e4e229337f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc5041f3f067cf610adffb868b4e2d1b5d248dc5a0ecc551339b670800ecb0ff \ No newline at end of file +fc68993501aaa7180f5457dcb3c296e5b199904a385d98e2bcad7854e34d428e \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index f1ce6ba4a2..3dccb3bacb 100644 --- a/src/expr.c +++ b/src/expr.c @@ -281,7 +281,7 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ assert( !ExprUseXList(p) || p->x.pList==0 || p->pRight==0 ); if( ExprUseXList(p) && p->x.pList!=0 && !db->mallocFailed ){ int i; - for(i=0; ALWAYS(ix.pList->nExpr); i++){ + for(i=0; ix.pList->nExpr; i++){ if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ pNext = p->x.pList->a[i].pExpr; break; From 072a02ce2c96436d4f60597896a2d6741322de1c Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 5 Apr 2023 02:55:08 +0000 Subject: [PATCH 173/341] Add a test case for the ALWAYS() macro removed by the previous check-in. FossilOrigin-Name: 68a1a837493a0bc5e0e0f2373ac76cb575078cec08990c017fdcb51a4ba363a1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/in.test | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8c4aa0ad86..0bc5c92b75 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sALWAYS()\sthat\smight\snow\sbe\sfalse\sdue\sto\sthe\sprior\scheck-in. -D 2023-04-05T02:50:00.410 +C Add\sa\stest\scase\sfor\sthe\sALWAYS()\smacro\sremoved\sby\sthe\sprevious\scheck-in. +D 2023-04-05T02:55:08.114 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1168,7 +1168,7 @@ F test/hook2.test b9ff3b8c6519fb67f33192f1afe86e7782ee4ac8 F test/icu.test 716a6b89fbabe5cc63e0cd4c260befb08fd7b9d761f04d43669233292f0753b1 F test/ieee754.test b0945d12be7d255f3dfa18e2511b17ca37e0edd2b803231c52d05b86c04ab26e F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8 -F test/in.test 291d881deae63fc2f0f3a9d7fdc1d300db58a6812c08e7c0370c970a984ecfbf +F test/in.test d1cad4ededd425568b2e39fb0c31fa9a3772311dd595801ff13ba3912b69bba6 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test fdd1d8134da8376985c2edba6035a2de1f6c731524d2ffa651419e8fe2cd1c5a @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cc5041f3f067cf610adffb868b4e2d1b5d248dc5a0ecc551339b670800ecb0ff -R 13f76b464bbf0db7eee4f47f89c8f40d +P fc68993501aaa7180f5457dcb3c296e5b199904a385d98e2bcad7854e34d428e +R 46750fc7c83a7b3de64aef61186b2ed5 U drh -Z c4aa246b09ce02d9717280a76242d5a6 +Z 63d4005b82e21c619045072b7f26d1d8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e4e229337f..4ee0ccdd66 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc68993501aaa7180f5457dcb3c296e5b199904a385d98e2bcad7854e34d428e \ No newline at end of file +68a1a837493a0bc5e0e0f2373ac76cb575078cec08990c017fdcb51a4ba363a1 \ No newline at end of file diff --git a/test/in.test b/test/in.test index 0fc7a4e2c1..601c7e3b4d 100644 --- a/test/in.test +++ b/test/in.test @@ -842,6 +842,12 @@ do_execsql_test in-23.0 { GROUP BY a0.a HAVING (SELECT sum( (a1.a == +a0.a COLLATE NOCASE) IN (SELECT b FROM t4))); } {ABC abc,ABC,def abc abc,ABC,def def abc,ABC,def} +do_execsql_test in-23.0-b { + SELECT a0.a, group_concat(a1.a) AS b + FROM t4 AS a0 JOIN t4 AS a1 + GROUP BY a0.a + HAVING (SELECT sum( (a1.a GLOB +a0.a COLLATE NOCASE) IN (SELECT b FROM t4))); +} {ABC abc,ABC,def abc abc,ABC,def def abc,ABC,def} # # Follow-up forum/forumpost/0713a16a44 # From 1ba18066376c2a7904523620ed52b4a444075f22 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 6 Apr 2023 00:18:31 +0000 Subject: [PATCH 174/341] In the new .scanstatus command in the CLI, make sure the database is opened before invoking sqlite3_db_config(). [forum:/forumpost/6e26dcf544|Forum post 6e26dcf544]. FossilOrigin-Name: 1cd993c45cd6b60e00d1426dd01d63efad13f7258636b5fa694f21499e77955a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 0bc5c92b75..40053fb961 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sfor\sthe\sALWAYS()\smacro\sremoved\sby\sthe\sprevious\scheck-in. -D 2023-04-05T02:55:08.114 +C In\sthe\snew\s.scanstatus\scommand\sin\sthe\sCLI,\smake\ssure\sthe\sdatabase\sis\sopened\nbefore\sinvoking\ssqlite3_db_config().\n[forum:/forumpost/6e26dcf544|Forum\spost\s6e26dcf544]. +D 2023-04-06T00:18:31.546 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c df007d60326a9aad94f5a4915d01e7de19d77627ed09338747a2e8e741a75add -F src/shell.c.in 4532aa8748d0aa76ac1d5ffb1e58405556197c3482b033b91c536501adb78a6f +F src/shell.c.in ca7161bd3d0c54b7b4dc02651fd2454b05e0ecdb5ba076a61215cd8bb0c767e8 F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fc68993501aaa7180f5457dcb3c296e5b199904a385d98e2bcad7854e34d428e -R 46750fc7c83a7b3de64aef61186b2ed5 +P 68a1a837493a0bc5e0e0f2373ac76cb575078cec08990c017fdcb51a4ba363a1 +R 5613993872d968e85760c42dbc57645a U drh -Z 63d4005b82e21c619045072b7f26d1d8 +Z fba861d77ab7fe200695129e0e7e4491 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4ee0ccdd66..ae1de680a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68a1a837493a0bc5e0e0f2373ac76cb575078cec08990c017fdcb51a4ba363a1 \ No newline at end of file +1cd993c45cd6b60e00d1426dd01d63efad13f7258636b5fa694f21499e77955a \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index cf7606ddaf..a0fb1ac33f 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -9649,6 +9649,7 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ p->scanstatsOn = (u8)booleanValue(azArg[1]); } + open_db(p, 0); sqlite3_db_config( p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 ); From c6349ce205e945da9f4a80f114ada6e7e30081eb Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 6 Apr 2023 00:59:41 +0000 Subject: [PATCH 175/341] In the zipfile extension, defend against corrupt ZIP files that contain a zero-length filename. [forum:/forumpost/b15f5e3ad8|Forum post b15f5e3ad8]. FossilOrigin-Name: 46db2e42a5f9b18da9661ccedca68cb70257ea5c58b33b401db2a5e030c1346a --- ext/misc/zipfile.c | 5 ++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ext/misc/zipfile.c b/ext/misc/zipfile.c index 480fbe3990..9b49fb4df6 100644 --- a/ext/misc/zipfile.c +++ b/ext/misc/zipfile.c @@ -1097,7 +1097,10 @@ static int zipfileColumn( ** it to be a directory either if the mode suggests so, or if ** the final character in the name is '/'. */ u32 mode = pCDS->iExternalAttr >> 16; - if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){ + if( !(mode & S_IFDIR) + && pCDS->nFile>=1 + && pCDS->zFile[pCDS->nFile-1]!='/' + ){ sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); } } diff --git a/manifest b/manifest index 40053fb961..c94b14ec3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\snew\s.scanstatus\scommand\sin\sthe\sCLI,\smake\ssure\sthe\sdatabase\sis\sopened\nbefore\sinvoking\ssqlite3_db_config().\n[forum:/forumpost/6e26dcf544|Forum\spost\s6e26dcf544]. -D 2023-04-06T00:18:31.546 +C In\sthe\szipfile\sextension,\sdefend\sagainst\scorrupt\sZIP\sfiles\sthat\scontain\na\szero-length\sfilename.\n[forum:/forumpost/b15f5e3ad8|Forum\spost\sb15f5e3ad8]. +D 2023-04-06T00:59:41.126 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -316,7 +316,7 @@ F ext/misc/vfsstat.c 474d08efc697b8eba300082cb1eb74a5f0f3df31ed257db1cb07e72ab0e F ext/misc/vtablog.c 5538acd0c8ddaae372331bee11608d76973436b77d6a91e8635cfc9432fba5ae F ext/misc/vtshim.c 1976e6dd68dd0d64508c91a6dfab8e75f8aaf6cd F ext/misc/wholenumber.c a838d1bea913c514ff316c69695efbb49ea3b8cb37d22afc57f73b6b010b4546 -F ext/misc/zipfile.c f98239261488397618ce4754c500626d1de20cd2d44bf2f2d571d7ddaab668a7 +F ext/misc/zipfile.c b9d615e1d9af7577833861cfaa79b253aec0f26c89239c75af8c790d287d1d39 F ext/misc/zorder.c b0ff58fa643afa1d846786d51ea8d5c4b6b35aa0254ab5a82617db92f3adda64 F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 68a1a837493a0bc5e0e0f2373ac76cb575078cec08990c017fdcb51a4ba363a1 -R 5613993872d968e85760c42dbc57645a +P 1cd993c45cd6b60e00d1426dd01d63efad13f7258636b5fa694f21499e77955a +R 83db8d662a6f7a0bae20734c688d29bc U drh -Z fba861d77ab7fe200695129e0e7e4491 +Z d955adab3dd7b2a13ea222fedf06115c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ae1de680a8..adf9971325 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1cd993c45cd6b60e00d1426dd01d63efad13f7258636b5fa694f21499e77955a \ No newline at end of file +46db2e42a5f9b18da9661ccedca68cb70257ea5c58b33b401db2a5e030c1346a \ No newline at end of file From 5482afd6cff48b3234f04f7aaa4187ad51e164c0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 6 Apr 2023 01:05:52 +0000 Subject: [PATCH 176/341] In the CLI, during error processing while looking for a word boundary, avoid being deceived by malformed input that has a very long sequence of 0x80 characters. [forum:/forumpost/ab93a23ba1|forum post ab93a23ba1]. FossilOrigin-Name: 82609d5a2d4eba741d48ea265f4e749578964961903c072c7b222ffe2aefaa3c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c94b14ec3c..f963e7612c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\szipfile\sextension,\sdefend\sagainst\scorrupt\sZIP\sfiles\sthat\scontain\na\szero-length\sfilename.\n[forum:/forumpost/b15f5e3ad8|Forum\spost\sb15f5e3ad8]. -D 2023-04-06T00:59:41.126 +C In\sthe\sCLI,\sduring\serror\sprocessing\swhile\slooking\sfor\sa\sword\sboundary,\navoid\sbeing\sdeceived\sby\smalformed\sinput\sthat\shas\sa\svery\slong\ssequence\nof\s0x80\scharacters.\n[forum:/forumpost/ab93a23ba1|forum\spost\sab93a23ba1]. +D 2023-04-06T01:05:52.467 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c df007d60326a9aad94f5a4915d01e7de19d77627ed09338747a2e8e741a75add -F src/shell.c.in ca7161bd3d0c54b7b4dc02651fd2454b05e0ecdb5ba076a61215cd8bb0c767e8 +F src/shell.c.in 2140c98b8185ce5f024d706a552dd0ee861c35621dab6599a9234019348bf9dc F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1cd993c45cd6b60e00d1426dd01d63efad13f7258636b5fa694f21499e77955a -R 83db8d662a6f7a0bae20734c688d29bc +P 46db2e42a5f9b18da9661ccedca68cb70257ea5c58b33b401db2a5e030c1346a +R 3627c9efb7548c46bc3a47ca70e854b9 U drh -Z d955adab3dd7b2a13ea222fedf06115c +Z 9baf3dd4e699627dd6a76c9e074307b5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index adf9971325..131ccf78b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46db2e42a5f9b18da9661ccedca68cb70257ea5c58b33b401db2a5e030c1346a \ No newline at end of file +82609d5a2d4eba741d48ea265f4e749578964961903c072c7b222ffe2aefaa3c \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index a0fb1ac33f..67f7ba30b0 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -2786,7 +2786,7 @@ static char *shell_error_context(const char *zSql, sqlite3 *db){ len = strlen(zSql); if( len>78 ){ len = 78; - while( (zSql[len]&0xc0)==0x80 ) len--; + while( len>0 && (zSql[len]&0xc0)==0x80 ) len--; } zCode = sqlite3_mprintf("%.*s", len, zSql); shell_check_oom(zCode); From ebec9c475e782a52e6e8e6d024a23d82a64615ac Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 6 Apr 2023 13:35:42 +0000 Subject: [PATCH 177/341] A prepared statement that aborts due to SQLITE_SCHEMA should not invalidate cursors in other prepared statements that are already running. See [forum:/forumpost/cae4367d9b|forum post cae4367d9b] for the original trouble report. FossilOrigin-Name: 857d0f5e16ba69ac9e5ee581befca2f7ed933edfe6e36396b10dae7979b44a57 --- ext/fts5/test/fts5ak.test | 26 ++++++++++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 2 ++ 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/ext/fts5/test/fts5ak.test b/ext/fts5/test/fts5ak.test index 0a3cd6a783..e248f2a328 100644 --- a/ext/fts5/test/fts5ak.test +++ b/ext/fts5/test/fts5ak.test @@ -154,4 +154,30 @@ do_execsql_test 3.2 { } +# 2023-04-06 https://sqlite.org/forum/forumpost/cae4367d9b +# +# This is not a test of FTS5, but rather a test of the of what happens to +# prepared statements that encounter SQLITE_SCHEMA while other prepared +# statements are running. The original problem POC used FTS5, and so +# is seems reasonable to put the test here. +# +# The vdbeaux24.test module in TH3 also tests this same behavior but +# without requiring FTS5 or an other extension. +# +reset_db +db null NULL +do_execsql_test 4.0 { + CREATE TABLE t5(a PRIMARY KEY); + INSERT INTO t5 VALUES(0); + CREATE VIRTUAL TABLE t6 USING fts5(0); + DELETE FROM t6; + CREATE TABLE t7(x); + WITH cte(a) AS ( + SELECT a FROM t5 + WHERE ((0,0) IN (SELECT 0, LAG(0) OVER (PARTITION BY 0) FROM t6), 0) + < (a,0) + ) + SELECT max(a) FROM cte; +} NULL + finish_test diff --git a/manifest b/manifest index f963e7612c..6876106573 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sCLI,\sduring\serror\sprocessing\swhile\slooking\sfor\sa\sword\sboundary,\navoid\sbeing\sdeceived\sby\smalformed\sinput\sthat\shas\sa\svery\slong\ssequence\nof\s0x80\scharacters.\n[forum:/forumpost/ab93a23ba1|forum\spost\sab93a23ba1]. -D 2023-04-06T01:05:52.467 +C A\sprepared\sstatement\sthat\saborts\sdue\sto\sSQLITE_SCHEMA\sshould\snot\sinvalidate\ncursors\sin\sother\sprepared\sstatements\sthat\sare\salready\srunning.\nSee\s[forum:/forumpost/cae4367d9b|forum\spost\scae4367d9b]\sfor\sthe\soriginal\ntrouble\sreport. +D 2023-04-06T13:35:42.258 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -115,7 +115,7 @@ F ext/fts5/test/fts5ag.test 7816f25a0707578f08145ab539fc0ca025f8951e788b28a6a18a F ext/fts5/test/fts5ah.test 2f047dfe89dc8611fa53e3d8bfc453b79cff037aa423c8d171e91e645745aa2c F ext/fts5/test/fts5ai.test bc97e4758cc93e06bf851d61c98fdf4e8b8f8315ee28a84fb15f916360856414 F ext/fts5/test/fts5aj.test 745020852d85f5dd49d11cb7ad11d3cc6dafc4fe6d6d24bc0875ac8f43ee4149 -F ext/fts5/test/fts5ak.test fc3595f8e6873bb86d70c9bd4b67d0413ce577bd4793c39a2b60a7b8825b60a6 +F ext/fts5/test/fts5ak.test f459a64c9d38698af72a7c657ab6349bca96150241dd69fcce752634b2742d41 F ext/fts5/test/fts5al.test 00c4c1c6a1366b73aa48ce2068c634520867c3cf7f5d1676ebbb775ee1f35734 F ext/fts5/test/fts5alter.test 5565f7e4605512b69171ac18ca84398603f9f6456dbe377beeca97e83cc242cd F ext/fts5/test/fts5auto.test 78989e6527ce69c9eddbef7392fea5c10b0010cd2b2ae68eec7bc869c471e691 @@ -699,7 +699,7 @@ F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 -F src/vdbeaux.c 2ec444e0be19d2cf671e2f3ae331965f5d9e24625525cc947048ab61625e1a37 +F src/vdbeaux.c 0379f2529aa16ab82fefddb1163c3138d6552ede80962b599ae711bbf5777608 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c db0458d11a51f6cfad2333a41e36a3795be0b2f316d070df5d33543a9ac884ac F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 46db2e42a5f9b18da9661ccedca68cb70257ea5c58b33b401db2a5e030c1346a -R 3627c9efb7548c46bc3a47ca70e854b9 +P 82609d5a2d4eba741d48ea265f4e749578964961903c072c7b222ffe2aefaa3c +R 34575d0f1bbe64bf65b9c8db61b744d0 U drh -Z 9baf3dd4e699627dd6a76c9e074307b5 +Z 0b0e55febdd72c86a8dbd43eaed96ed2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 131ccf78b3..b6ecb6a560 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -82609d5a2d4eba741d48ea265f4e749578964961903c072c7b222ffe2aefaa3c \ No newline at end of file +857d0f5e16ba69ac9e5ee581befca2f7ed933edfe6e36396b10dae7979b44a57 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e92523710f..1c3d298bd6 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3326,6 +3326,8 @@ int sqlite3VdbeHalt(Vdbe *p){ db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } + }else if( p->rc==SQLITE_SCHEMA && db->nVdbeActive>1 ){ + p->nChange = 0; }else{ sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; From 82412051dbf9edebf7bee53770660f80e7abb3a1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 6 Apr 2023 17:29:38 +0000 Subject: [PATCH 178/341] Work around a harmless assertion fault associated with sqlite3VdbeMemAboutToChange() such that the detection of stale values in registers is preserved in debugging builds, but we avoid a false-positive assertion fault in cases involving a virtual table with a LIMIT clause in an IN-operator loop. dbsqlfuzz 3fd70d4ab4950acf1deb8f610a7a7c67cd38713b FossilOrigin-Name: 56ea2c2fe6108d39833ac40957afab59ade01a216639d5bafdeeca53bbf4cd67 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6876106573..a2e471df47 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sprepared\sstatement\sthat\saborts\sdue\sto\sSQLITE_SCHEMA\sshould\snot\sinvalidate\ncursors\sin\sother\sprepared\sstatements\sthat\sare\salready\srunning.\nSee\s[forum:/forumpost/cae4367d9b|forum\spost\scae4367d9b]\sfor\sthe\soriginal\ntrouble\sreport. -D 2023-04-06T13:35:42.258 +C Work\saround\sa\sharmless\sassertion\sfault\sassociated\swith\nsqlite3VdbeMemAboutToChange()\ssuch\sthat\sthe\sdetection\sof\sstale\svalues\nin\sregisters\sis\spreserved\sin\sdebugging\sbuilds,\sbut\swe\savoid\sa\sfalse-positive\nassertion\sfault\sin\scases\sinvolving\sa\svirtual\stable\swith\sa\sLIMIT\sclause\nin\san\sIN-operator\sloop.\ndbsqlfuzz\s3fd70d4ab4950acf1deb8f610a7a7c67cd38713b +D 2023-04-06T17:29:38.736 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 1c17d1ff2e5924178b82355d192b4927030b3c39317feff5633aecec8f77d0fe +F src/expr.c 874702ffa80dec44cd4b5dd456627136750e1f585568d51febbdb1f4a7f21970 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 82609d5a2d4eba741d48ea265f4e749578964961903c072c7b222ffe2aefaa3c -R 34575d0f1bbe64bf65b9c8db61b744d0 +P 857d0f5e16ba69ac9e5ee581befca2f7ed933edfe6e36396b10dae7979b44a57 +R 2288ff5cfd8b0483680fe089706fc5b1 U drh -Z 0b0e55febdd72c86a8dbd43eaed96ed2 +Z a25cf56afdc01321117f5bc71f12f074 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b6ecb6a560..01365fb8b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -857d0f5e16ba69ac9e5ee581befca2f7ed933edfe6e36396b10dae7979b44a57 \ No newline at end of file +56ea2c2fe6108d39833ac40957afab59ade01a216639d5bafdeeca53bbf4cd67 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 3dccb3bacb..81c8124de2 100644 --- a/src/expr.c +++ b/src/expr.c @@ -5099,7 +5099,9 @@ void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); if( inReg!=target ){ u8 op; - if( ALWAYS(pExpr) && ExprHasProperty(pExpr,EP_Subquery) ){ + if( ALWAYS(pExpr) + && (ExprHasProperty(pExpr,EP_Subquery) || pExpr->op==TK_REGISTER) + ){ op = OP_Copy; }else{ op = OP_SCopy; From 50dc8d972086dfbbc3ef3c7f664d58b5c44c4ac8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 6 Apr 2023 20:14:10 +0000 Subject: [PATCH 179/341] Increase the size of the cache of free blocks inside of pageFreeArray() to reduce the number of calls to freeSpace(). FossilOrigin-Name: 27c59f1ea789f3ff245f23e79ded5cd71c48e3a51ffbb8c220b51101a4e69fd7 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/btree.c | 47 +++++++++++++++++++++++++++-------------------- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index a2e471df47..7b23595ea4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\sa\sharmless\sassertion\sfault\sassociated\swith\nsqlite3VdbeMemAboutToChange()\ssuch\sthat\sthe\sdetection\sof\sstale\svalues\nin\sregisters\sis\spreserved\sin\sdebugging\sbuilds,\sbut\swe\savoid\sa\sfalse-positive\nassertion\sfault\sin\scases\sinvolving\sa\svirtual\stable\swith\sa\sLIMIT\sclause\nin\san\sIN-operator\sloop.\ndbsqlfuzz\s3fd70d4ab4950acf1deb8f610a7a7c67cd38713b -D 2023-04-06T17:29:38.736 +C Increase\sthe\ssize\sof\sthe\scache\sof\sfree\sblocks\sinside\sof\spageFreeArray()\sto\nreduce\sthe\snumber\sof\scalls\sto\sfreeSpace(). +D 2023-04-06T20:14:10.861 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 023b41e0a4563880812288e32682538013287b7b82e0c24a57b0e26b6f0ef168 +F src/btree.c 1a5f31d548a68d2bd537919d8afa176540c3ea70787fe10d5873beb00f9db615 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -2052,8 +2052,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 857d0f5e16ba69ac9e5ee581befca2f7ed933edfe6e36396b10dae7979b44a57 -R 2288ff5cfd8b0483680fe089706fc5b1 +P 56ea2c2fe6108d39833ac40957afab59ade01a216639d5bafdeeca53bbf4cd67 +R f4ea0d3a1a9bcf702848933fabd592b4 +T *branch * btree-freespace-opt +T *sym-btree-freespace-opt * +T -sym-trunk * U drh -Z a25cf56afdc01321117f5bc71f12f074 +Z a279ebfc98f243d3f64ca427ad2184cf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 01365fb8b8..38cf4ef4af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -56ea2c2fe6108d39833ac40957afab59ade01a216639d5bafdeeca53bbf4cd67 \ No newline at end of file +27c59f1ea789f3ff245f23e79ded5cd71c48e3a51ffbb8c220b51101a4e69fd7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2f3063c3a3..86ff2b466f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7496,42 +7496,49 @@ static int pageFreeArray( u8 * const pEnd = &aData[pPg->pBt->usableSize]; u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; - int i; + int i, j; int iEnd = iFirst + nCell; - u8 *pFree = 0; /* \__ Parameters for pending call to */ - int szFree = 0; /* / freeSpace() */ + int nFree = 0; + int aOfst[10]; + int aAfter[10]; for(i=iFirst; iapCell[i]; if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ int sz; + int iAfter; + int iOfst; /* No need to use cachedCellSize() here. The sizes of all cells that ** are to be freed have already been computing while deciding which ** cells need freeing */ sz = pCArray->szCell[i]; assert( sz>0 ); - if( pFree!=(pCell + sz) ){ - if( pFree ){ - assert( pFree>aData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); + iOfst = (u16)(pCell - aData); + iAfter = iOfst+sz; + for(j=0; jpEnd ){ - return 0; + } + if( j>=nFree ){ + if( nFree>=sizeof(aOfst)/sizeof(aOfst[0]) ){ + for(j=0; jaData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); + for(j=0; j Date: Fri, 7 Apr 2023 11:18:08 +0000 Subject: [PATCH 180/341] Add further tests for the rbu_exclusive_lock=1 URI option. FossilOrigin-Name: c07b62bef96bb69e9b1ce08f0084fdce8de981f4d8ea9689c87a41f1e4451ac7 --- ext/rbu/rbuexlock.test | 80 ++++++++++++++++++++++++++++++++++++++++++ manifest | 14 ++++---- manifest.uuid | 2 +- 3 files changed, 88 insertions(+), 8 deletions(-) diff --git a/ext/rbu/rbuexlock.test b/ext/rbu/rbuexlock.test index 27fd6c4ba2..aceee45a59 100644 --- a/ext/rbu/rbuexlock.test +++ b/ext/rbu/rbuexlock.test @@ -207,5 +207,85 @@ do_test 3.4.0 { } SQLITE_OK rbu close +#------------------------------------------------------------------------- +reset_db +forcedelete rbu1.db +forcedelete rbu2.db + +sqlite3 rbu rbu1.db +do_execsql_test -db rbu 4.1 { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 'one', 0); +} +rbu close +sqlite3 rbu rbu2.db +do_execsql_test -db rbu 4.2 { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(2, 'two', 0); +} +rbu close + +do_execsql_test 4.3 { + CREATE TABLE t1(a PRIMARY KEY, b); +} +db close + +do_test 4.4 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu1.db + rbu step + rbu state +} {oal} + +sqlite3 cons test.db +do_execsql_test -db cons 4.5 { + SELECT * FROM t1 +} {} + +do_test 4.6 { rbu step ; rbu state } {oal} +do_test 4.7 { rbu step ; rbu state } {move} +do_execsql_test -db cons 4.8 { + SELECT * FROM t1 +} {} +do_test 4.9 { rbu step ; rbu state } {checkpoint} +do_test 4.10 { + catchsql { SELECT * FROM t1 } cons +} {1 {database is locked}} +do_test 4.11 { rbu step ; rbu state } {checkpoint} +do_test 4.11 { rbu step ; rbu state } {done} +rbu close + +do_test 4.12 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one}} + +do_test 4.13 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu2.db + rbu step + rbu state +} {oal} + +do_test 4.14 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one}} + +do_test 4.15 { rbu step ; rbu state } {oal} +do_test 4.16 { rbu step ; rbu state } {move} + +do_test 4.17 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one}} + +do_test 4.18 { rbu step ; rbu state } {checkpoint} +do_test 4.19 { + catchsql { SELECT * FROM t1 } cons +} {1 {database is locked}} +do_test 4.20 { rbu step ; rbu state } {checkpoint} +do_test 4.21 { rbu step ; rbu state } {done} +rbu close + +do_test 4.22 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one 2 two}} + finish_test diff --git a/manifest b/manifest index a2e471df47..07db86167a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\sa\sharmless\sassertion\sfault\sassociated\swith\nsqlite3VdbeMemAboutToChange()\ssuch\sthat\sthe\sdetection\sof\sstale\svalues\nin\sregisters\sis\spreserved\sin\sdebugging\sbuilds,\sbut\swe\savoid\sa\sfalse-positive\nassertion\sfault\sin\scases\sinvolving\sa\svirtual\stable\swith\sa\sLIMIT\sclause\nin\san\sIN-operator\sloop.\ndbsqlfuzz\s3fd70d4ab4950acf1deb8f610a7a7c67cd38713b -D 2023-04-06T17:29:38.736 +C Add\sfurther\stests\sfor\sthe\srbu_exclusive_lock=1\sURI\soption. +D 2023-04-07T11:18:08.946 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -341,7 +341,7 @@ F ext/rbu/rbucrash.test d2b5d619d9281c89cad74401b73b46172daa89906940b1d739c813dd F ext/rbu/rbucrash2.test 0a1a72223d880215ce2893a3260320c31a9358d23cb124c610e4f0d984a93285 F ext/rbu/rbudiff.test 8b8b8b569c68fc880134e0fac4bf6b4b7a907aea4cc6eacf7e1d45e1d47b6aac F ext/rbu/rbudor.test 293a192e668bb8e9c7c9704b080c1086ee17496f768e0f1823049e7d02651d1b -F ext/rbu/rbuexlock.test e3ece733cc8e0a6cb08533a41ed6f562438539ed309028c63375a5adee4a263b +F ext/rbu/rbuexlock.test 703bb26cb13d655920670974a5d6823b8f7a8b1d4451162a0c1caf44c3d44252 F ext/rbu/rbuexpr.test 2c91617509c88b6e9030f7bf6ff720df26032fcd801adc25533feae726a57382 F ext/rbu/rbufault.test c51de14067cfe867849530d3d1718ffeb28522f28d52937f95dd7bc2116eb42e F ext/rbu/rbufault2.test 8cc8f6298d2d7d20080b2c77e65b607af8b89839f9d87c0972b27e6442edc258 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 857d0f5e16ba69ac9e5ee581befca2f7ed933edfe6e36396b10dae7979b44a57 -R 2288ff5cfd8b0483680fe089706fc5b1 -U drh -Z a25cf56afdc01321117f5bc71f12f074 +P 56ea2c2fe6108d39833ac40957afab59ade01a216639d5bafdeeca53bbf4cd67 +R 9275838142062bf058148e4ea2fb7583 +U dan +Z c616b58415bad4443461c0a782c05091 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 01365fb8b8..3d6ed8c0ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -56ea2c2fe6108d39833ac40957afab59ade01a216639d5bafdeeca53bbf4cd67 \ No newline at end of file +c07b62bef96bb69e9b1ce08f0084fdce8de981f4d8ea9689c87a41f1e4451ac7 \ No newline at end of file From b39f037ca58bba4643e47552efe3c7092092d46f Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 7 Apr 2023 13:21:20 +0000 Subject: [PATCH 181/341] Small performance improvement in freeSpace(). FossilOrigin-Name: 8dc5292ee592f16451441e33ad0800ba10a21ecd63f1f9926d6915a59a1552d3 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/btree.c | 14 +++++++------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 7b23595ea4..b964136df4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\ssize\sof\sthe\scache\sof\sfree\sblocks\sinside\sof\spageFreeArray()\sto\nreduce\sthe\snumber\sof\scalls\sto\sfreeSpace(). -D 2023-04-06T20:14:10.861 +C Small\sperformance\simprovement\sin\sfreeSpace(). +D 2023-04-07T13:21:20.818 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 1a5f31d548a68d2bd537919d8afa176540c3ea70787fe10d5873beb00f9db615 +F src/btree.c ab147fedb774a7af5a2b665091aaf1b3d379c0215f00958f0d8b62d0c3d8c462 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -2052,11 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 56ea2c2fe6108d39833ac40957afab59ade01a216639d5bafdeeca53bbf4cd67 -R f4ea0d3a1a9bcf702848933fabd592b4 -T *branch * btree-freespace-opt -T *sym-btree-freespace-opt * -T -sym-trunk * +P 27c59f1ea789f3ff245f23e79ded5cd71c48e3a51ffbb8c220b51101a4e69fd7 +R d6591cb8459c4b329119f7b1d9a296cf U drh -Z a279ebfc98f243d3f64ca427ad2184cf +Z 18a83276bbdc23812267f1b7bee10433 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 38cf4ef4af..4ab902253d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27c59f1ea789f3ff245f23e79ded5cd71c48e3a51ffbb8c220b51101a4e69fd7 \ No newline at end of file +8dc5292ee592f16451441e33ad0800ba10a21ecd63f1f9926d6915a59a1552d3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 86ff2b466f..bf82203987 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1919,6 +1919,11 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ } pTmp = &data[hdr+5]; x = get2byte(pTmp); + if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[iStart], 0, iSize); + } if( iStart<=x ){ /* The new freeblock is at the beginning of the cell content area, ** so just extend the cell content area rather than create another @@ -1930,14 +1935,9 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ }else{ /* Insert the new freeblock into the freelist */ put2byte(&data[iPtr], iStart); + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); } - if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ - memset(&data[iStart], 0, iSize); - } - put2byte(&data[iStart], iFreeBlk); - put2byte(&data[iStart+2], iSize); pPage->nFree += iOrigSize; return SQLITE_OK; } From 6d47af64e4d59951c8e8dd0d064fd6c2e4fe13d8 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 7 Apr 2023 14:03:30 +0000 Subject: [PATCH 182/341] Fix a problem causing the rbu_exclusive_checkpoint=1 to be ignored with zipvfs databases. FossilOrigin-Name: d8f50b31e8d64fce5141da4a016767a15482703364692a55df346f059fc9d30c --- ext/rbu/sqlite3rbu.c | 11 ++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 2db66f67ac..f65ef83864 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3141,6 +3141,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff); } +/* +** This value is copied from the definition of ZIPVFS_CTRL_FILE_POINTER +** in zipvfs.h. +*/ +#define RBU_ZIPVFS_CTRL_FILE_POINTER 230439 /* ** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if @@ -3149,7 +3154,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ static int rbuLockDatabase(sqlite3 *db){ int rc = SQLITE_OK; sqlite3_file *fd = 0; - sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + if( fd==0 ){ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + } if( fd->pMethods ){ rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); diff --git a/manifest b/manifest index 07db86167a..657c476a37 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\sthe\srbu_exclusive_lock=1\sURI\soption. -D 2023-04-07T11:18:08.946 +C Fix\sa\sproblem\scausing\sthe\srbu_exclusive_checkpoint=1\sto\sbe\signored\swith\szipvfs\sdatabases. +D 2023-04-07T14:03:30.087 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -362,7 +362,7 @@ F ext/rbu/rbuvacuum.test 542561741ff2b262e3694bc6012b44694ee62c545845319a06f3237 F ext/rbu/rbuvacuum2.test ae097d04feb041446a74fac94b24bffeb3fdd60e32b848c5611e507ab702b81b F ext/rbu/rbuvacuum3.test 3ce42695fdf21aaa3499e857d7d4253bc499ad759bcd6c9362042c13cd37d8de F ext/rbu/rbuvacuum4.test ffccd22f67e2d0b380d2889685742159dfe0d19a3880ca3d2d1d69eefaebb205 -F ext/rbu/sqlite3rbu.c 348bb6251e6ec459de102f8b2dd50789a98643ef7a28e56e4c787ac9659c15ea +F ext/rbu/sqlite3rbu.c 71a7f0dea3a846ff7c2499dc34a2528f5ddcbe23e2c54dc3cd1fa4d933377c6d F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 F ext/recover/dbdata.c 31d580785cf14eb3c20ed6fbb421a10a66569858f837928e6b326088c38d4c72 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 56ea2c2fe6108d39833ac40957afab59ade01a216639d5bafdeeca53bbf4cd67 -R 9275838142062bf058148e4ea2fb7583 +P c07b62bef96bb69e9b1ce08f0084fdce8de981f4d8ea9689c87a41f1e4451ac7 +R 8848363a9c3a234e64f59bd74865cd9d U dan -Z c616b58415bad4443461c0a782c05091 +Z 1cd4b7dd2aa9fce599ff7d1499f29586 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3d6ed8c0ef..4cf0ab8070 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c07b62bef96bb69e9b1ce08f0084fdce8de981f4d8ea9689c87a41f1e4451ac7 \ No newline at end of file +d8f50b31e8d64fce5141da4a016767a15482703364692a55df346f059fc9d30c \ No newline at end of file From 34ceb7e62279f2bb274e70166a4e9c56e6794853 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 7 Apr 2023 14:33:33 +0000 Subject: [PATCH 183/341] Clone insertCell() into insertCellFast() for use by sqlite3BtreeInsert() for a substantial performance increase. FossilOrigin-Name: f225afd90c8e65661d8b855050f0ee1a8fe4c0f3bcec824aa5a66d906f3c7119 --- manifest | 14 ++++---- manifest.uuid | 2 +- src/btree.c | 95 +++++++++++++++++++++++++++++++++++++++++++++++-- src/sqliteInt.h | 3 ++ 4 files changed, 104 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b964136df4..6c6258fe15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sin\sfreeSpace(). -D 2023-04-07T13:21:20.818 +C Clone\sinsertCell()\sinto\sinsertCellFast()\sfor\suse\sby\ssqlite3BtreeInsert()\sfor\na\ssubstantial\sperformance\sincrease. +D 2023-04-07T14:33:33.532 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c ab147fedb774a7af5a2b665091aaf1b3d379c0215f00958f0d8b62d0c3d8c462 +F src/btree.c ba9cc15bd5c74ad9e26cd474837429c53695a86f4dd6ed1ede13102f5af1f27d F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -630,7 +630,7 @@ F src/shell.c.in 2140c98b8185ce5f024d706a552dd0ee861c35621dab6599a9234019348bf9d F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h 899781baef0d1dd0910524df6350e0ef7e2761131f6e04ec5e34f3b32e262998 +F src/sqliteInt.h b2e1fb7dc1ae6103fcfdcd6bc47bafd1b517d180473e4f560feaae7e8a8a1455 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27c59f1ea789f3ff245f23e79ded5cd71c48e3a51ffbb8c220b51101a4e69fd7 -R d6591cb8459c4b329119f7b1d9a296cf +P 8dc5292ee592f16451441e33ad0800ba10a21ecd63f1f9926d6915a59a1552d3 +R 4cd0c4e73a1dfc42c4a1aab1944d088a U drh -Z 18a83276bbdc23812267f1b7bee10433 +Z 8d2b56b02beff3e147538fb72d3dce05 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4ab902253d..6aa70bab6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dc5292ee592f16451441e33ad0800ba10a21ecd63f1f9926d6915a59a1552d3 \ No newline at end of file +f225afd90c8e65661d8b855050f0ee1a8fe4c0f3bcec824aa5a66d906f3c7119 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index bf82203987..60fd496b34 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1746,7 +1746,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** allocation is being made in order to insert a new cell, so we will ** also end up needing a new cell pointer. */ -static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ +static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ int top; /* First byte of cell content area */ @@ -7096,6 +7096,14 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. +** +** The insertCellFast() routine below works exactly the same as +** insertCell() except that it lacks the pTemp and iChild parameters +** which are assumed zero. Other than that, the two routines are the +** same. +** +** Fixes or enhancements to this routine should be reflected in +** insertCellFast()! */ static int insertCell( MemPage *pPage, /* Page into which we are copying */ @@ -7189,6 +7197,89 @@ static int insertCell( return SQLITE_OK; } +/* +** This variant of insertCell() assumes that the pTemp and iChild +** parameters are both zero. Use this variant in sqlite3BtreeInsert() +** for performance improvement, and also so that this variant is only +** called from that one place, and is thus inlined, and thus runs must +** faster. +** +** Fixes or enhancements to this routine should be reflected into +** the insertCell() routine. +*/ +static int insertCellFast( + MemPage *pPage, /* Page into which we are copying */ + int i, /* New cell becomes the i-th cell of the page */ + u8 *pCell, /* Content of the new cell */ + int sz /* Bytes of content in pCell */ +){ + int idx = 0; /* Where to write new cell content in data[] */ + int j; /* Loop counter */ + u8 *data; /* The content of the whole page */ + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ + + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + if( pPage->nOverflow || sz+2>pPage->nFree ){ + j = pPage->nOverflow++; + /* Comparison against ArraySize-1 since we hold back one extra slot + ** as a contingency. In other words, never need more than 3 overflow + ** slots but 4 are allocated, just to be safe. */ + assert( j < ArraySize(pPage->apOvfl)-1 ); + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ + }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + data = pPage->aData; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); + rc = allocateSpace(pPage, sz, &idx); + if( rc ){ return rc; } + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); + assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nFree -= (u16)(2 + sz); + memcpy(&data[idx], pCell, sz); + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pPage->pBt->autoVacuum ){ + int rc2 = SQLITE_OK; + /* The cell may contain a pointer to an overflow page. If so, write + ** the entry for the overflow page into the pointer map. + */ + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc2; + } +#endif + } + return SQLITE_OK; +} + /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -9319,7 +9410,7 @@ int sqlite3BtreeInsert( }else{ assert( pPage->leaf ); } - rc = insertCell(pPage, idx, newCell, szNew, 0, 0); + rc = insertCellFast(pPage, idx, newCell, szNew); assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9b189124d9..752cbbdb78 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -286,10 +286,13 @@ */ #if defined(__GNUC__) # define SQLITE_NOINLINE __attribute__((noinline)) +# define SQLITE_INLINE __attribute__((always_inline)) inline #elif defined(_MSC_VER) && _MSC_VER>=1310 # define SQLITE_NOINLINE __declspec(noinline) +# define SQLITE_INLINE __forceinline #else # define SQLITE_NOINLINE +# define SQLITE_INLINE #endif /* From 706c33d9bac7fd5d9e02ec0907b60226a250b719 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 7 Apr 2023 15:07:58 +0000 Subject: [PATCH 184/341] Tweaks to the new insertCellFast(). FossilOrigin-Name: 203a581a9177c1083e8d5b49e8ff026af33b5c5e3e144aeda126f07a3a2953bf --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 26 +++++++++++--------------- src/sqliteInt.h | 6 +++++- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 6c6258fe15..88efe08c72 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clone\sinsertCell()\sinto\sinsertCellFast()\sfor\suse\sby\ssqlite3BtreeInsert()\sfor\na\ssubstantial\sperformance\sincrease. -D 2023-04-07T14:33:33.532 +C Tweaks\sto\sthe\snew\sinsertCellFast(). +D 2023-04-07T15:07:58.639 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c ba9cc15bd5c74ad9e26cd474837429c53695a86f4dd6ed1ede13102f5af1f27d +F src/btree.c 5e041121901c610debad04a39043ccc59ea1ec97ed2cc917f50f0220454baef3 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -630,7 +630,7 @@ F src/shell.c.in 2140c98b8185ce5f024d706a552dd0ee861c35621dab6599a9234019348bf9d F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h b2e1fb7dc1ae6103fcfdcd6bc47bafd1b517d180473e4f560feaae7e8a8a1455 +F src/sqliteInt.h a605bcdaf9083655e848b54d55c053000ed4ab8325cca8696c41585ba8ac05fd F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8dc5292ee592f16451441e33ad0800ba10a21ecd63f1f9926d6915a59a1552d3 -R 4cd0c4e73a1dfc42c4a1aab1944d088a +P f225afd90c8e65661d8b855050f0ee1a8fe4c0f3bcec824aa5a66d906f3c7119 +R 68ed98237478b3a68b6eb72f11a471eb U drh -Z 8d2b56b02beff3e147538fb72d3dce05 +Z 69bb1d90ac950c6d096f89e22cd7c390 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6aa70bab6f..bba05ea546 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f225afd90c8e65661d8b855050f0ee1a8fe4c0f3bcec824aa5a66d906f3c7119 \ No newline at end of file +203a581a9177c1083e8d5b49e8ff026af33b5c5e3e144aeda126f07a3a2953bf \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 60fd496b34..e0eff6d87f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7126,14 +7126,13 @@ static int insertCell( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); assert( pPage->nFree>=0 ); + assert( iChild>0 ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); pCell = pTemp; } - if( iChild ){ - put4byte(pCell, iChild); - } + put4byte(pCell, iChild); j = pPage->nOverflow++; /* Comparison against ArraySize-1 since we hold back one extra slot ** as a contingency. In other words, never need more than 3 overflow @@ -7165,17 +7164,13 @@ static int insertCell( assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); pPage->nFree -= (u16)(2 + sz); - if( iChild ){ - /* In a corrupt database where an entry in the cell index section of - ** a btree page has a value of 3 or less, the pCell value might point - ** as many as 4 bytes in front of the start of the aData buffer for - ** the source page. Make sure this does not cause problems by not - ** reading the first 4 bytes */ - memcpy(&data[idx+4], pCell+4, sz-4); - put4byte(&data[idx], iChild); - }else{ - memcpy(&data[idx], pCell, sz); - } + /* In a corrupt database where an entry in the cell index section of + ** a btree page has a value of 3 or less, the pCell value might point + ** as many as 4 bytes in front of the start of the aData buffer for + ** the source page. Make sure this does not cause problems by not + ** reading the first 4 bytes */ + memcpy(&data[idx+4], pCell+4, sz-4); + put4byte(&data[idx], iChild); pIns = pPage->aCellIdx + i*2; memmove(pIns+2, pIns, 2*(pPage->nCell - i)); put2byte(pIns, idx); @@ -7226,7 +7221,8 @@ static int insertCellFast( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); assert( pPage->nFree>=0 ); - if( pPage->nOverflow || sz+2>pPage->nFree ){ + assert( pPage->nOverflow==0 ); + if( sz+2>pPage->nFree ){ j = pPage->nOverflow++; /* Comparison against ArraySize-1 since we hold back one extra slot ** as a contingency. In other words, never need more than 3 overflow diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 752cbbdb78..f33f6eb20f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -281,7 +281,7 @@ #endif /* -** A macro to hint to the compiler that a function should not be +** Macros to hint to the compiler that a function should or should not be ** inlined. */ #if defined(__GNUC__) @@ -294,6 +294,10 @@ # define SQLITE_NOINLINE # define SQLITE_INLINE #endif +#if defined(SQLITE_COVERAGE_TEST) +# undef SQLITE_INLINE +# define SQLITE_INLINE +#endif /* ** Make sure that the compiler intrinsics we desire are enabled when From 81e31c9c2b50260ba0c9ef5c9d394a4bceddd95d Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 7 Apr 2023 15:49:42 +0000 Subject: [PATCH 185/341] Fix an assert(). FossilOrigin-Name: 7eff46ba97dd8a3d6a5d01d5d61e98a2805deceafa47335eded7d784a8304525 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 88efe08c72..91384f3219 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tweaks\sto\sthe\snew\sinsertCellFast(). -D 2023-04-07T15:07:58.639 +C Fix\san\sassert(). +D 2023-04-07T15:49:42.983 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 5e041121901c610debad04a39043ccc59ea1ec97ed2cc917f50f0220454baef3 +F src/btree.c 467786f4d7eb53e5fcd7b08d71de3b8af3360e19ce1fdecbf3d270772e0284a5 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f225afd90c8e65661d8b855050f0ee1a8fe4c0f3bcec824aa5a66d906f3c7119 -R 68ed98237478b3a68b6eb72f11a471eb +P 203a581a9177c1083e8d5b49e8ff026af33b5c5e3e144aeda126f07a3a2953bf +R 54f048283a72db7525d9a6de65af4d6f U drh -Z 69bb1d90ac950c6d096f89e22cd7c390 +Z 23390f9086f977a9a4d3c1c740b228e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bba05ea546..1a9b087aac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -203a581a9177c1083e8d5b49e8ff026af33b5c5e3e144aeda126f07a3a2953bf \ No newline at end of file +7eff46ba97dd8a3d6a5d01d5d61e98a2805deceafa47335eded7d784a8304525 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e0eff6d87f..396d0cd4b8 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1862,7 +1862,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( iStart<=pPage->pBt->usableSize-4 ); + assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 ); /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. From 3da5e2a9f3c45c536d3b058b74cfae5dd67876a4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 7 Apr 2023 16:30:33 +0000 Subject: [PATCH 186/341] Add NEVER on an unreachable branch. FossilOrigin-Name: 9b3febbd988be05807ada20146d3e196ae17c966722fff049feb32292157bff2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 91384f3219..436e685b2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert(). -D 2023-04-07T15:49:42.983 +C Add\sNEVER\son\san\sunreachable\sbranch. +D 2023-04-07T16:30:33.632 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 467786f4d7eb53e5fcd7b08d71de3b8af3360e19ce1fdecbf3d270772e0284a5 +F src/btree.c 119a010f26b0ddbfa8117917bcb0a95b1286c5550e7222a72f3a0565599f6379 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 203a581a9177c1083e8d5b49e8ff026af33b5c5e3e144aeda126f07a3a2953bf -R 54f048283a72db7525d9a6de65af4d6f +P 7eff46ba97dd8a3d6a5d01d5d61e98a2805deceafa47335eded7d784a8304525 +R d683893a4f37f40e4189174d14b43d68 U drh -Z 23390f9086f977a9a4d3c1c740b228e3 +Z 012057b737c59630df75dcd34aedd019 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1a9b087aac..ac529a1ac6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7eff46ba97dd8a3d6a5d01d5d61e98a2805deceafa47335eded7d784a8304525 \ No newline at end of file +9b3febbd988be05807ada20146d3e196ae17c966722fff049feb32292157bff2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 396d0cd4b8..9e22944cf3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7150,7 +7150,7 @@ static int insertCell( assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); - if( rc!=SQLITE_OK ){ + if( NEVER(rc!=SQLITE_OK) ){ return rc; } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); From 65aae44b87551bac15ff2a58ca7d7d7f86ea1114 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 7 Apr 2023 18:27:32 +0000 Subject: [PATCH 187/341] Guard against oversized cells in the newly enhanced pageFreeArray(). FossilOrigin-Name: 2dcdbb50356edbd3a79e53fa0bee4e700c2bdea78e27173b62ddabe44b066726 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 436e685b2c..8efcf81a61 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sNEVER\son\san\sunreachable\sbranch. -D 2023-04-07T16:30:33.632 +C Guard\sagainst\soversized\scells\sin\sthe\snewly\senhanced\spageFreeArray(). +D 2023-04-07T18:27:32.091 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 119a010f26b0ddbfa8117917bcb0a95b1286c5550e7222a72f3a0565599f6379 +F src/btree.c 895a4ab3a5aeb7db9ed894bb3fd1e2eee4422e95fa6ca3d3d0e803bc0080ec55 F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7eff46ba97dd8a3d6a5d01d5d61e98a2805deceafa47335eded7d784a8304525 -R d683893a4f37f40e4189174d14b43d68 +P 9b3febbd988be05807ada20146d3e196ae17c966722fff049feb32292157bff2 +R 6d5ac3499b5cfed37f1e7ebe2e26c3c7 U drh -Z 012057b737c59630df75dcd34aedd019 +Z 765df8c6a87f2cf39432d4f3f8e36810 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ac529a1ac6..224e5e3627 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b3febbd988be05807ada20146d3e196ae17c966722fff049feb32292157bff2 \ No newline at end of file +2dcdbb50356edbd3a79e53fa0bee4e700c2bdea78e27173b62ddabe44b066726 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9e22944cf3..0a3d713174 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7619,6 +7619,7 @@ static int pageFreeArray( } aOfst[nFree] = iOfst; aAfter[nFree] = iAfter; + if( &aData[iAfter]>pEnd ) return 0; nFree++; } nRet++; From a9b20c09378753e875cfe6e4e86039b66919012c Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 8 Apr 2023 13:31:17 +0000 Subject: [PATCH 188/341] Fix a harmless compiler warning. FossilOrigin-Name: c9559ba62191fe7fa2a718233afaa841e2594d1fc833314bf5b0a6b775e87c35 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/expr.c | 1 - 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bf16d6cba4..2686add86e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimizations\sto\sbtree.c\ssave\sabout\s4.5\smillion\sCPU\scycles:\n(1)\sClone\sinsertCell()\sinto\sa\sseparate\sinsertCellFast()\sroutine\sfor\nuse\sby\ssqlite3BtreeInsert().\s\s(2)\sMark\sallocateSpace()\sas\salways-inline.\n(3)\sImproved\scoalesence\sof\sadjacent\sfree\sblocks\sin\spageFreeArray(). -D 2023-04-08T13:01:34.177 +C Fix\sa\sharmless\scompiler\swarning. +D 2023-04-08T13:31:17.304 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 874702ffa80dec44cd4b5dd456627136750e1f585568d51febbdb1f4a7f21970 +F src/expr.c 067ed64192802739d2a6d4f4ecab5773874c0602dadce89a8b941f46eaddc7ac F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -2052,9 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d8f50b31e8d64fce5141da4a016767a15482703364692a55df346f059fc9d30c 2dcdbb50356edbd3a79e53fa0bee4e700c2bdea78e27173b62ddabe44b066726 -R 88a02467116587aeb37e104f663e3a20 -T +closed 2dcdbb50356edbd3a79e53fa0bee4e700c2bdea78e27173b62ddabe44b066726 +P 5c12c400fe8eb4e86e14c69a6c34d0d78d9861e5d40a36c6a596a81c6dd65977 +R 9bee361640c8d6b170c8d56996d80384 U drh -Z f3818585026019f77b55a7228538100f +Z 6d533f3536f56e3e50b677595cc8fd42 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 691fc213b4..c39e28935a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c12c400fe8eb4e86e14c69a6c34d0d78d9861e5d40a36c6a596a81c6dd65977 \ No newline at end of file +c9559ba62191fe7fa2a718233afaa841e2594d1fc833314bf5b0a6b775e87c35 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 81c8124de2..c3afdd1cba 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6441,7 +6441,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ default: { IndexedExpr *pIEpr; Expr tmp; - int i; assert( pParse->iSelfTab==0 ); if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; if( pParse->pIdxEpr==0 ) break; From 9e0181fcf703cc6bfceca78554ab5e7b1b8b797b Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 8 Apr 2023 16:51:08 +0000 Subject: [PATCH 189/341] Faster implementation of keywordCode() - the routine that determines if an identifier is really a keyword and if so, which keyword. FossilOrigin-Name: 0ff3d3d53709b7f18bf01ded1f988e41b7f8471072cf4f2702a3a8b79964be3f --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/mkkeywordhash.c | 13 +++++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 2686add86e..263de8fa82 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning. -D 2023-04-08T13:31:17.304 +C Faster\simplementation\sof\skeywordCode()\s-\sthe\sroutine\sthat\sdetermines\sif\san\nidentifier\sis\sreally\sa\skeyword\sand\sif\sso,\swhich\skeyword. +D 2023-04-08T16:51:08.427 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1978,7 +1978,7 @@ F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a19 F tool/mkautoconfamal.sh f62353eb6c06ab264da027fd4507d09914433dbdcab9cb011cdc18016f1ab3b8 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x F tool/mkctimec.tcl 38e3db33210a200aae791635125052a643a27aa0619a0debf19aa9c55e1b2dde x -F tool/mkkeywordhash.c 35bfc41adacc4aa6ef6fca7fd0c63e0ec0534b78daf4d0cfdebe398216bbffc3 +F tool/mkkeywordhash.c 9822bd1f58a70e5f84179df3045bba4791df69180e991bec88662703f2e93761 F tool/mkmsvcmin.tcl 6ecab9fe22c2c8de4d82d4c46797bda3d2deac8e763885f5a38d0c44a895ab33 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef F tool/mkopcodeh.tcl 769d9e6a8b462323150dc13a8539d6064664b72974f7894befe2491cc73e05cd @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5c12c400fe8eb4e86e14c69a6c34d0d78d9861e5d40a36c6a596a81c6dd65977 -R 9bee361640c8d6b170c8d56996d80384 +P c9559ba62191fe7fa2a718233afaa841e2594d1fc833314bf5b0a6b775e87c35 +R a6a030871b8c18c0abdeba3bc5e7f2b5 U drh -Z 6d533f3536f56e3e50b677595cc8fd42 +Z b5330e1f9352658322b2ceaebf7861af # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c39e28935a..990aa89fee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c9559ba62191fe7fa2a718233afaa841e2594d1fc833314bf5b0a6b775e87c35 \ No newline at end of file +0ff3d3d53709b7f18bf01ded1f988e41b7f8471072cf4f2702a3a8b79964be3f \ No newline at end of file diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c index fe25b5abc0..44e1d9c27c 100644 --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@ -596,7 +596,7 @@ int main(int argc, char **argv){ printf("/* aKWNext[] forms the hash collision chain. If aKWHash[i]==0\n"); printf("** then the i-th keyword has no more hash collisions. Otherwise,\n"); printf("** the next keyword with the same hash is aKWHash[i]-1. */\n"); - printf("static const unsigned char aKWNext[%d] = {\n", nKeyword); + printf("static const unsigned char aKWNext[%d] = {0,\n", nKeyword+1); for(i=j=0; i=0; i=((int)aKWNext[i])-1){\n"); + printf(" for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){\n"); printf(" if( aKWLen[i]!=n ) continue;\n"); printf(" zKW = &zKWText[aKWOffset[i]];\n"); printf("#ifdef SQLITE_ASCII\n"); @@ -687,7 +687,7 @@ int main(int argc, char **argv){ printf(" if( j=SQLITE_N_KEYWORD ) return SQLITE_ERROR;\n"); + printf(" i++;\n"); printf(" *pzName = zKWText + aKWOffset[i];\n"); printf(" *pnName = aKWLen[i];\n"); printf(" return SQLITE_OK;\n"); From df3944ecb8685b450b68028d3f1d04588ead4f35 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 8 Apr 2023 19:23:13 +0000 Subject: [PATCH 190/341] Fix a typo in [83e84531b46814ae] that changed the value of SQLITE_DBCONFIG_STMT_SCANSTATUS. FossilOrigin-Name: b71776539183e5fd8678ce8f3272180d3dbb8214ed7b7bb24d35c0ba027a6ac5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 263de8fa82..06364b0d8f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Faster\simplementation\sof\skeywordCode()\s-\sthe\sroutine\sthat\sdetermines\sif\san\nidentifier\sis\sreally\sa\skeyword\sand\sif\sso,\swhich\skeyword. -D 2023-04-08T16:51:08.427 +C Fix\sa\stypo\sin\s[83e84531b46814ae]\sthat\schanged\sthe\svalue\sof\nSQLITE_DBCONFIG_STMT_SCANSTATUS. +D 2023-04-08T19:23:13.990 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -627,7 +627,7 @@ F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c df007d60326a9aad94f5a4915d01e7de19d77627ed09338747a2e8e741a75add F src/shell.c.in 2140c98b8185ce5f024d706a552dd0ee861c35621dab6599a9234019348bf9dc -F src/sqlite.h.in 84f0e61a07292977c31f108776e5148eb1c761e7c276de2290c1511dad7c7d3a +F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 F src/sqliteInt.h a605bcdaf9083655e848b54d55c053000ed4ab8325cca8696c41585ba8ac05fd @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c9559ba62191fe7fa2a718233afaa841e2594d1fc833314bf5b0a6b775e87c35 -R a6a030871b8c18c0abdeba3bc5e7f2b5 +P 0ff3d3d53709b7f18bf01ded1f988e41b7f8471072cf4f2702a3a8b79964be3f +R 616ff76d39a6297528a440b3518c2684 U drh -Z b5330e1f9352658322b2ceaebf7861af +Z ba1dc7eae1a8d3d3e1f2a79fb06d4e4d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 990aa89fee..653c2b51a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ff3d3d53709b7f18bf01ded1f988e41b7f8471072cf4f2702a3a8b79964be3f \ No newline at end of file +b71776539183e5fd8678ce8f3272180d3dbb8214ed7b7bb24d35c0ba027a6ac5 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index be86c1b17f..117a61f8ca 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2496,7 +2496,7 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1080 /* int int* */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ #define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ #define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ From 1a2f490fbce07c172d2f25efd2796480493f2c97 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 8 Apr 2023 19:27:03 +0000 Subject: [PATCH 191/341] Fix a couple harmless compiler warnings seen with MSVC. FossilOrigin-Name: 1b864a370fd04bba11487e3dab0388394bc71e1f3f006f7bd9d64eefe04e8b61 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c.in | 2 +- src/vdbemem.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 06364b0d8f..473001a275 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\s[83e84531b46814ae]\sthat\schanged\sthe\svalue\sof\nSQLITE_DBCONFIG_STMT_SCANSTATUS. -D 2023-04-08T19:23:13.990 +C Fix\sa\scouple\sharmless\scompiler\swarnings\sseen\swith\sMSVC. +D 2023-04-08T19:27:03.506 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c df007d60326a9aad94f5a4915d01e7de19d77627ed09338747a2e8e741a75add -F src/shell.c.in 2140c98b8185ce5f024d706a552dd0ee861c35621dab6599a9234019348bf9dc +F src/shell.c.in 80f7c4c1bdabc9dab416b59287458044c01df97035d626ea57ffeabbb289bdc6 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -701,7 +701,7 @@ F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 F src/vdbeaux.c 0379f2529aa16ab82fefddb1163c3138d6552ede80962b599ae711bbf5777608 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd -F src/vdbemem.c db0458d11a51f6cfad2333a41e36a3795be0b2f316d070df5d33543a9ac884ac +F src/vdbemem.c 6bf53dee95b4fd8585aef191ee381259b546256b3c056669c20f0fecd2a2fe1b F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0ff3d3d53709b7f18bf01ded1f988e41b7f8471072cf4f2702a3a8b79964be3f -R 616ff76d39a6297528a440b3518c2684 -U drh -Z ba1dc7eae1a8d3d3e1f2a79fb06d4e4d +P b71776539183e5fd8678ce8f3272180d3dbb8214ed7b7bb24d35c0ba027a6ac5 +R ad83a00170dacde77d1c0a151d571a0d +U mistachkin +Z 3bacef294df0e363471d64ff660e594a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 653c2b51a8..e7c70a0023 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b71776539183e5fd8678ce8f3272180d3dbb8214ed7b7bb24d35c0ba027a6ac5 \ No newline at end of file +1b864a370fd04bba11487e3dab0388394bc71e1f3f006f7bd9d64eefe04e8b61 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 67f7ba30b0..220e91fd1d 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -2774,7 +2774,7 @@ static char *shell_error_context(const char *zSql, sqlite3 *db){ if( db==0 || zSql==0 || (iOffset = sqlite3_error_offset(db))<0 - || iOffset>=strlen(zSql) + || iOffset>=(int)strlen(zSql) ){ return sqlite3_mprintf(""); } diff --git a/src/vdbemem.c b/src/vdbemem.c index 58621de89d..e81447fbe6 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -439,7 +439,7 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ vdbeMemRenderNum(nByte, pMem->z, pMem); assert( pMem->z!=0 ); - assert( pMem->n==sqlite3Strlen30NN(pMem->z) ); + assert( pMem->n==(int)sqlite3Strlen30NN(pMem->z) ); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); From d4fe3c77e6433adb6880968b3621cb09e003e46a Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 9 Apr 2023 10:09:28 +0000 Subject: [PATCH 192/341] Fix a faulty assert() in whereRangeScanEst() that should only apply if there are not prior errors. dbsqlfuzz 567ad91132879cbab8172b38c6a68ad40fa8d650. FossilOrigin-Name: 8ba9b08bd976ed2c024f5f378b4dd40a55226e950af77c476c21273fccaf7e26 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 473001a275..0b9873f44b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sharmless\scompiler\swarnings\sseen\swith\sMSVC. -D 2023-04-08T19:27:03.506 +C Fix\sa\sfaulty\sassert()\sin\swhereRangeScanEst()\sthat\sshould\sonly\sapply\sif\sthere\nare\snot\sprior\serrors.\ndbsqlfuzz\s567ad91132879cbab8172b38c6a68ad40fa8d650. +D 2023-04-09T10:09:28.921 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -710,7 +710,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 16ba2073aa2139bf97576c49baa6189a33200e123cbc86374a7410a0aed5a117 +F src/where.c ef9e644d1d76e86f68c941eb05d30a98fc0811eeef2a110906d81fedde81b6bf F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 41c17b089082558c1c5496e8a453b1f9a96485b6b355f318440fca2ee4754b2e F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b71776539183e5fd8678ce8f3272180d3dbb8214ed7b7bb24d35c0ba027a6ac5 -R ad83a00170dacde77d1c0a151d571a0d -U mistachkin -Z 3bacef294df0e363471d64ff660e594a +P 1b864a370fd04bba11487e3dab0388394bc71e1f3f006f7bd9d64eefe04e8b61 +R eb05fc199e96c886a58dbf98249166ca +U drh +Z 8ca7ee35df5ac0bab38cf17f583933ba # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e7c70a0023..48a6df54f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b864a370fd04bba11487e3dab0388394bc71e1f3f006f7bd9d64eefe04e8b61 \ No newline at end of file +8ba9b08bd976ed2c024f5f378b4dd40a55226e950af77c476c21273fccaf7e26 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0bed46de65..8200daca80 100644 --- a/src/where.c +++ b/src/where.c @@ -2020,7 +2020,7 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); assert( pLower || pUpper ); #endif - assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); From d27c18818c7399f07092f041768107e3d68e9733 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 9 Apr 2023 20:44:09 +0000 Subject: [PATCH 193/341] Fix an assertion fault that can occur when compiling with both SQLITE_ENABLE_UNKNOWN_SQL_FUNCTIONS and SQLITE_ENABLE_STAT4. FossilOrigin-Name: 9ff69e599817610d14948a3eebcc4d3c18245f78a1433c23ae659211cbbdb017 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0b9873f44b..77ff44023f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sin\swhereRangeScanEst()\sthat\sshould\sonly\sapply\sif\sthere\nare\snot\sprior\serrors.\ndbsqlfuzz\s567ad91132879cbab8172b38c6a68ad40fa8d650. -D 2023-04-09T10:09:28.921 +C Fix\san\sassertion\sfault\sthat\scan\soccur\swhen\scompiling\swith\sboth\nSQLITE_ENABLE_UNKNOWN_SQL_FUNCTIONS\sand\sSQLITE_ENABLE_STAT4. +D 2023-04-09T20:44:09.894 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -701,7 +701,7 @@ F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 F src/vdbeaux.c 0379f2529aa16ab82fefddb1163c3138d6552ede80962b599ae711bbf5777608 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd -F src/vdbemem.c 6bf53dee95b4fd8585aef191ee381259b546256b3c056669c20f0fecd2a2fe1b +F src/vdbemem.c 1d9a0f37b0097fbb53f0d7ba081f7181b83cee2c6f46364706ea0c3896bd8ec0 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1b864a370fd04bba11487e3dab0388394bc71e1f3f006f7bd9d64eefe04e8b61 -R eb05fc199e96c886a58dbf98249166ca +P 8ba9b08bd976ed2c024f5f378b4dd40a55226e950af77c476c21273fccaf7e26 +R 889add789cd2553848be57f0d2fe0e07 U drh -Z 8ca7ee35df5ac0bab38cf17f583933ba +Z a5ce80245e588ea8ba300425da779261 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 48a6df54f1..5937c70422 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ba9b08bd976ed2c024f5f378b4dd40a55226e950af77c476c21273fccaf7e26 \ No newline at end of file +9ff69e599817610d14948a3eebcc4d3c18245f78a1433c23ae659211cbbdb017 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index e81447fbe6..7b3b970f3f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1483,8 +1483,11 @@ static int valueFromFunction( if( pList ) nVal = pList->nExpr; assert( !ExprHasProperty(p, EP_IntValue) ); pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pFunc==0 ) return SQLITE_OK; +#endif assert( pFunc ); - if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) ){ return SQLITE_OK; From 8346f8daccc567bb2dd2d2bdcd28e670cba4b229 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 10 Apr 2023 13:20:51 +0000 Subject: [PATCH 194/341] Sync the vt02.c test virtual table with TH3, in order to pull in the fix for long delays when there are huge OFFSET values. FossilOrigin-Name: 49ba030080dd00b4fdf788fd3da057b333e705fa0fe37d653e2461bf96ca3785 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/vt02.c | 10 ++-------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 77ff44023f..be625fca08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassertion\sfault\sthat\scan\soccur\swhen\scompiling\swith\sboth\nSQLITE_ENABLE_UNKNOWN_SQL_FUNCTIONS\sand\sSQLITE_ENABLE_STAT4. -D 2023-04-09T20:44:09.894 +C Sync\sthe\svt02.c\stest\svirtual\stable\swith\sTH3,\sin\sorder\sto\spull\sin\sthe\sfix\nfor\slong\sdelays\swhen\sthere\sare\shuge\sOFFSET\svalues. +D 2023-04-10T13:20:51.922 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1807,7 +1807,7 @@ F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test d4c4281e1679245829db35597817282f60dc513fc39cc5439078f009bd118487 F test/view2.test db32c8138b5b556f610b35dfddd38c5a58a292f07fda5281eedb0851b2672679 F test/view3.test ad8a8290ee2b55ff6ce66c9ef1ce3f1e47926273a3814e1c425293e128a95456 -F test/vt02.c 86253b57d6bc2170dfca33f45fd099b66d0bf874e95ecd7786dcbb134f179469 +F test/vt02.c f4a357c8180d71120ca2b466a8df48d9c40fc50873694840327d9647450485f3 F test/vtab1.test 09a72330d0f31eda2ffaa828b06a6b917fb86250ee72de0301570af725774c07 F test/vtab2.test 14d4ab26cee13ba6cf5c5601b158e4f57552d3b055cdd9406cf7f711e9c84082 F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8ba9b08bd976ed2c024f5f378b4dd40a55226e950af77c476c21273fccaf7e26 -R 889add789cd2553848be57f0d2fe0e07 +P 9ff69e599817610d14948a3eebcc4d3c18245f78a1433c23ae659211cbbdb017 +R 7cdb734827060f13b761710e5ee55378 U drh -Z a5ce80245e588ea8ba300425da779261 +Z 7d4200ee23a188536db0fc075de1e41b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5937c70422..0afe48f4b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ff69e599817610d14948a3eebcc4d3c18245f78a1433c23ae659211cbbdb017 \ No newline at end of file +49ba030080dd00b4fdf788fd3da057b333e705fa0fe37d653e2461bf96ca3785 \ No newline at end of file diff --git a/test/vt02.c b/test/vt02.c index f83fc9af95..ddad136fba 100644 --- a/test/vt02.c +++ b/test/vt02.c @@ -177,8 +177,6 @@ #include #endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Forward declarations */ typedef struct vt02_vtab vt02_vtab; typedef struct vt02_cur vt02_cur; @@ -381,7 +379,7 @@ static int vt02Filter( sqlite3_int64 v; pVal = 0; if( sqlite3_vtab_in_first(0, &pVal)!=SQLITE_MISUSE - || sqlite3_vtab_in_first(argv[iArg], &pVal)!=SQLITE_MISUSE + || sqlite3_vtab_in_first(argv[iArg], &pVal)!=SQLITE_ERROR ){ vt02ErrMsg(pCursor->pVtab, "unexpected success from sqlite3_vtab_in_first()"); @@ -420,7 +418,7 @@ static int vt02Filter( } if( bUseOffset ){ int nSkip = sqlite3_value_int(argv[iArg]); - while( nSkip-- > 0 ) vt02Next(pCursor); + while( nSkip-- > 0 && pCur->iiEof ) vt02Next(pCursor); } return SQLITE_OK; @@ -1001,10 +999,6 @@ static void vt02CoreInit(sqlite3 *db){ sqlite3_create_module(db, "vt02pkabcd", &vt02Module, (void*)zPkABCDSchema); } -#else -# define vt02CoreInit(db) -#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ - #ifdef TH3_VERSION static void vt02_init(th3state *p, int iDb, char *zArg){ vt02CoreInit(th3dbPointer(p, iDb)); From ed36917835ceef4a32d6e3e5aa395acf3212495d Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 10 Apr 2023 18:44:00 +0000 Subject: [PATCH 195/341] New assert() statements to validate the parameters to sqlite3BtreeCursorHint(). Fix a problem with the construction of those parameters discovered by [forum:/forumpost/0b53708c95|forum post 0b53708c95]. FossilOrigin-Name: 4c5a3c5fb351cc1c2ce16c33314ce19c53531f09263f87456283d9d756002f9d --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/btree.c | 19 ++++++++++++++++++- src/sqliteInt.h | 1 + src/vdbe.h | 4 ++++ src/vdbeaux.c | 14 ++++++++++++++ src/wherecode.c | 18 ++++++++---------- test/cursorhint.test | 11 +++++++++++ 8 files changed, 68 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index be625fca08..3806628079 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Sync\sthe\svt02.c\stest\svirtual\stable\swith\sTH3,\sin\sorder\sto\spull\sin\sthe\sfix\nfor\slong\sdelays\swhen\sthere\sare\shuge\sOFFSET\svalues. -D 2023-04-10T13:20:51.922 +C New\sassert()\sstatements\sto\svalidate\sthe\sparameters\sto\nsqlite3BtreeCursorHint().\s\sFix\sa\sproblem\swith\sthe\sconstruction\sof\sthose\nparameters\sdiscovered\sby\n[forum:/forumpost/0b53708c95|forum\spost\s0b53708c95]. +D 2023-04-10T18:44:00.864 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -564,7 +564,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c 895a4ab3a5aeb7db9ed894bb3fd1e2eee4422e95fa6ca3d3d0e803bc0080ec55 +F src/btree.c af379d801906f21dc18b534fc250a479f56045a450d8d5859ba2f57fd9eefcef F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -630,7 +630,7 @@ F src/shell.c.in 80f7c4c1bdabc9dab416b59287458044c01df97035d626ea57ffeabbb289bdc F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h a605bcdaf9083655e848b54d55c053000ed4ab8325cca8696c41585ba8ac05fd +F src/sqliteInt.h bf15f7db635d2e64a227bbf86845bc19755dbd932021a6461d6dd15b0da2cfd3 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -696,10 +696,10 @@ F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b -F src/vdbe.h 73b904a6b3bb27f308c6cc287a5751ebc7f1f89456be0ed068a12b92844c6e8c +F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 -F src/vdbeaux.c 0379f2529aa16ab82fefddb1163c3138d6552ede80962b599ae711bbf5777608 +F src/vdbeaux.c 128de68cade63c914e97d31c2fa71fd6186927ebfbedabc68019ce99256c3ddf F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 1d9a0f37b0097fbb53f0d7ba081f7181b83cee2c6f46364706ea0c3896bd8ec0 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -712,7 +712,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c ef9e644d1d76e86f68c941eb05d30a98fc0811eeef2a110906d81fedde81b6bf F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c 41c17b089082558c1c5496e8a453b1f9a96485b6b355f318440fca2ee4754b2e +F src/wherecode.c 277c38607de247353af79d54385d602d8fc2e535113253c006b749d9fff48c45 F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 F src/window.c 76a27cff9ea2ded0c2c3527187029259440fabcc4cc4c07b11d942c78494a614 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -912,7 +912,7 @@ F test/createtab.test 85cdfdae5c3de331cd888d6c66e1aba575b47c2e3c3cc4a1d6f5414069 F test/cse.test 00b3aea44b16828833c94fbe92475fd6977583fcb064ae0bc590986812b38d0c F test/csv01.test 2ab5514005fd308995c8910bc313e47f0368b94213b9d6c27f9a2da78796a091 F test/ctime.test 340f362f41f92972bbd71f44e10569a5cc694062b692231bd08aa6fe6c1c4773 -F test/cursorhint.test 12e7d03262be8705798fe7b5719956bcda14989f10296a5e8fdc13466b7b75fc +F test/cursorhint.test a44811a341281ebb73b939cb11bdcc2f374dc70e7e0f6cd7bfcb02e6fee67831 F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee9ae9c42f F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 118e04db8c8b4efeb885542b4918c7b869a34c460a6bebbfe927dfd75706b80d @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ff69e599817610d14948a3eebcc4d3c18245f78a1433c23ae659211cbbdb017 -R 7cdb734827060f13b761710e5ee55378 +P 49ba030080dd00b4fdf788fd3da057b333e705fa0fe37d653e2461bf96ca3785 +R 52234413a35145fce6946ef5d4a92056 U drh -Z 7d4200ee23a188536db0fc075de1e41b +Z c6b409950782fc21232dcb9d111b6a54 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0afe48f4b9..b746816210 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49ba030080dd00b4fdf788fd3da057b333e705fa0fe37d653e2461bf96ca3785 \ No newline at end of file +4c5a3c5fb351cc1c2ce16c33314ce19c53531f09263f87456283d9d756002f9d \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0a3d713174..40a94a520d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -946,8 +946,25 @@ int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow){ */ void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ /* Used only by system that substitute their own storage engine */ +#ifdef SQLITE_DEBUG + if( ALWAYS(eHintType==BTREE_HINT_RANGE) ){ + va_list ap; + Expr *pExpr; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3CursorRangeHintExprCheck; + va_start(ap, eHintType); + pExpr = va_arg(ap, Expr*); + w.u.aMem = va_arg(ap, Mem*); + va_end(ap); + assert( pExpr!=0 ); + assert( w.u.aMem!=0 ); + sqlite3WalkExpr(&w, pExpr); + } +#endif /* SQLITE_DEBUG */ } -#endif +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + /* ** Provide flag hints to the cursor. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f33f6eb20f..4739951a5d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4186,6 +4186,7 @@ struct Walker { struct CoveringIndexCheck *pCovIdxCk; /* Check for covering index */ SrcItem *pSrcItem; /* A single FROM clause item */ DbFixer *pFix; /* See sqlite3FixSelect() */ + Mem *aMem; /* See sqlite3BtreeCursorHint() */ } u; }; diff --git a/src/vdbe.h b/src/vdbe.h index 3caf1f7872..d28837f944 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -403,4 +403,8 @@ void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int); void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); #endif +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr); +#endif + #endif /* SQLITE_VDBE_H */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 1c3d298bd6..1848cdea3c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -5275,6 +5275,20 @@ int sqlite3NotPureFunc(sqlite3_context *pCtx){ return 1; } +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +/* +** This Walker callback is used to help verify that calls to +** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have +** byte-code register values correctly initialized. +*/ +int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_REGISTER ){ + assert( (pWalker->u.aMem[pExpr->iTable].flags & MEM_Undefined)==0 ); + } + return WRC_Continue; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS && SQLITE_DEBUG */ + #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored diff --git a/src/wherecode.c b/src/wherecode.c index abb582fb7b..8b8e41f6d3 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1011,11 +1011,12 @@ static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ */ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ int rc = WRC_Continue; + int reg; struct CCurHint *pHint = pWalker->u.pCCurHint; if( pExpr->op==TK_COLUMN ){ if( pExpr->iTable!=pHint->iTabCur ){ - int reg = ++pWalker->pParse->nMem; /* Register for column value */ - sqlite3ExprCode(pWalker->pParse, pExpr, reg); + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); pExpr->op = TK_REGISTER; pExpr->iTable = reg; }else if( pHint->pIdx!=0 ){ @@ -1023,15 +1024,12 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn); assert( pExpr->iColumn>=0 ); } - }else if( pExpr->op==TK_AGG_FUNCTION ){ - /* An aggregate function in the WHERE clause of a query means this must - ** be a correlated sub-query, and expression pExpr is an aggregate from - ** the parent context. Do not walk the function arguments in this case. - ** - ** todo: It should be possible to replace this node with a TK_REGISTER - ** expression, as the result of the expression must be stored in a - ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + }else if( pExpr->pAggInfo ){ rc = WRC_Prune; + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; } return rc; } diff --git a/test/cursorhint.test b/test/cursorhint.test index eab2a2739e..47d9f76f39 100644 --- a/test/cursorhint.test +++ b/test/cursorhint.test @@ -180,4 +180,15 @@ do_execsql_test 5.1 { ORDER BY 1; } +# 2023-04-10 https://sqlite.org/forum/forumpost/0b53708c95 +# +do_execsql_test 6.0 { + CREATE TABLE t6(a TEXT UNIQUE, b TEXT); + INSERT INTO t6(a,b) VALUES('uvw','xyz'),('abc','def'); + WITH v1(a) AS (SELECT a COLLATE NOCASE FROM t6) + SELECT v1.a, count(*) FROM t6 LEFT JOIN v1 ON true + GROUP BY 1 + HAVING (SELECT true FROM t6 AS aa LEFT JOIN t6 AS bb ON length(v1.a)>5); +} {abc 2 uvw 2} + finish_test From 69881887b8172871dbd2b51d054667ea43bbf912 Mon Sep 17 00:00:00 2001 From: larrybr Date: Mon, 10 Apr 2023 23:21:19 +0000 Subject: [PATCH 196/341] For sha3 extension, mention NIST standard implemented. FossilOrigin-Name: 529ab138a438379f0cbd170bff5211aee1c327a1b3119a714ee15f68fd2239f3 --- ext/misc/shathree.c | 3 ++- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ext/misc/shathree.c b/ext/misc/shathree.c index 765c691811..ba3ea581f8 100644 --- a/ext/misc/shathree.c +++ b/ext/misc/shathree.c @@ -10,7 +10,8 @@ ** ****************************************************************************** ** -** This SQLite extension implements functions that compute SHA3 hashes. +** This SQLite extension implements functions that compute SHA3 hashes +** in the way described by the (U.S.) NIST FIPS 202 SHA-3 Standard. ** Two SQL functions are implemented: ** ** sha3(X,SIZE) diff --git a/manifest b/manifest index 3806628079..0e22070cfb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\sassert()\sstatements\sto\svalidate\sthe\sparameters\sto\nsqlite3BtreeCursorHint().\s\sFix\sa\sproblem\swith\sthe\sconstruction\sof\sthose\nparameters\sdiscovered\sby\n[forum:/forumpost/0b53708c95|forum\spost\s0b53708c95]. -D 2023-04-10T18:44:00.864 +C For\ssha3\sextension,\smention\sNIST\sstandard\simplemented. +D 2023-04-10T23:21:19.052 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -300,7 +300,7 @@ F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d385 F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 F ext/misc/series.c 8d79354f2c3d46b95ee21272a07cf0bcabb58d1f2b06d9e7b8a31dca1dacb3e5 F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d -F ext/misc/shathree.c 9b7af7b7d55b27c5bbd16548b67fe6aa47a819e71bc6a80a456144f86f4e834d +F ext/misc/shathree.c 543af7ce71d391cd3a9ab6924a6a1124efc63211fd0f2e240dc4b56077ba88ac F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 94df9bbfa514a563c1484f684a2df3d128a2f7209a84ca3ca100c68a0163e29f F ext/misc/sqlar.c 53e7d48f68d699a24f1a92e68e71eca8b3a9ff991fe9588c2a05bde103c6e7b7 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 49ba030080dd00b4fdf788fd3da057b333e705fa0fe37d653e2461bf96ca3785 -R 52234413a35145fce6946ef5d4a92056 -U drh -Z c6b409950782fc21232dcb9d111b6a54 +P 4c5a3c5fb351cc1c2ce16c33314ce19c53531f09263f87456283d9d756002f9d +R d7c4b934385537fc62cb31933fd99395 +U larrybr +Z 04beccf7708e38ec632db3683e5bb16e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b746816210..c1def15f23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c5a3c5fb351cc1c2ce16c33314ce19c53531f09263f87456283d9d756002f9d \ No newline at end of file +529ab138a438379f0cbd170bff5211aee1c327a1b3119a714ee15f68fd2239f3 \ No newline at end of file From e4d8e7e5df723bba47231999710e8b091c1b21a6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 11 Apr 2023 02:10:34 +0000 Subject: [PATCH 197/341] Better handling of OOM errors in the cursor-hint logic. dbsqlfuzz 60cd5fff91974af91c2c3692beb4a2d7fdafef46 FossilOrigin-Name: 68fcfb58df837b40e401dfa4e505f4f0b87554b9cfeb2c527f3558348b73c1be --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wherecode.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0e22070cfb..7c725fd748 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\ssha3\sextension,\smention\sNIST\sstandard\simplemented. -D 2023-04-10T23:21:19.052 +C Better\shandling\sof\sOOM\serrors\sin\sthe\scursor-hint\slogic.\ndbsqlfuzz\s60cd5fff91974af91c2c3692beb4a2d7fdafef46 +D 2023-04-11T02:10:34.486 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -712,7 +712,7 @@ F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c ef9e644d1d76e86f68c941eb05d30a98fc0811eeef2a110906d81fedde81b6bf F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c -F src/wherecode.c 277c38607de247353af79d54385d602d8fc2e535113253c006b749d9fff48c45 +F src/wherecode.c 85790d7e5365ac41085713331ce52e4343586ad3d37d218ffe00572357baa62b F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 F src/window.c 76a27cff9ea2ded0c2c3527187029259440fabcc4cc4c07b11d942c78494a614 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4c5a3c5fb351cc1c2ce16c33314ce19c53531f09263f87456283d9d756002f9d -R d7c4b934385537fc62cb31933fd99395 -U larrybr -Z 04beccf7708e38ec632db3683e5bb16e +P 529ab138a438379f0cbd170bff5211aee1c327a1b3119a714ee15f68fd2239f3 +R 2b073bb091f9647750fe13a7239467ad +U drh +Z 5c099af3d555e42160813c7dd8f116af # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c1def15f23..4dd2051cc5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -529ab138a438379f0cbd170bff5211aee1c327a1b3119a714ee15f68fd2239f3 \ No newline at end of file +68fcfb58df837b40e401dfa4e505f4f0b87554b9cfeb2c527f3558348b73c1be \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index 8b8e41f6d3..5441fb17d3 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1131,7 +1131,7 @@ static void codeCursorHint( } if( pExpr!=0 ){ sWalker.xExprCallback = codeCursorHintFixExpr; - sqlite3WalkExpr(&sWalker, pExpr); + if( pParse->nErr==0 ) sqlite3WalkExpr(&sWalker, pExpr); sqlite3VdbeAddOp4(v, OP_CursorHint, (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, (const char*)pExpr, P4_EXPR); From 82aacda7eb2ec7e63beae8a61c6d046e976c057d Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 11 Apr 2023 15:06:20 +0000 Subject: [PATCH 198/341] Remove an ALWAYS() that can sometimes be false. Add a test case that makes the test false. FossilOrigin-Name: c8fb143d64d8e823684cd26799080da4b42bef121ca3c6315b1803a593490926 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 2 +- test/window1.test | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7c725fd748..2be8f39913 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Better\shandling\sof\sOOM\serrors\sin\sthe\scursor-hint\slogic.\ndbsqlfuzz\s60cd5fff91974af91c2c3692beb4a2d7fdafef46 -D 2023-04-11T02:10:34.486 +C Remove\san\sALWAYS()\sthat\scan\ssometimes\sbe\sfalse.\s\sAdd\sa\stest\scase\sthat\smakes\nthe\stest\sfalse. +D 2023-04-11T15:06:20.059 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 067ed64192802739d2a6d4f4ecab5773874c0602dadce89a8b941f46eaddc7ac +F src/expr.c 86f843dba99f1d60836b0b9fa7f595ff2723d902416a4c7749217e950160f17c F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -1902,7 +1902,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 67aaef6e01e8d8fb3f89aa440e8c8045dbfd3c6f21628a6d7bad2b786c310d9b +F test/window1.test c607e30741c034a1e88c104f97f107f07489f538a65c0f56ccce97e7a956065a F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 529ab138a438379f0cbd170bff5211aee1c327a1b3119a714ee15f68fd2239f3 -R 2b073bb091f9647750fe13a7239467ad +P 68fcfb58df837b40e401dfa4e505f4f0b87554b9cfeb2c527f3558348b73c1be +R bc6a8cda8e023714aac7bd85583d4924 U drh -Z 5c099af3d555e42160813c7dd8f116af +Z 2237c0728bfff0a59ab0f03de511e2c3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4dd2051cc5..d69c4529fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68fcfb58df837b40e401dfa4e505f4f0b87554b9cfeb2c527f3558348b73c1be \ No newline at end of file +c8fb143d64d8e823684cd26799080da4b42bef121ca3c6315b1803a593490926 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c3afdd1cba..00c67a4e68 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6288,7 +6288,7 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ sqlite3 *db = pParse->db; assert( iAgg>=0 ); if( pExpr->op!=TK_AGG_FUNCTION ){ - if( ALWAYS(iAggnColumn) + if( iAggnColumn && pAggInfo->aCol[iAgg].pCExpr==pExpr ){ pExpr = sqlite3ExprDup(db, pExpr, 0); diff --git a/test/window1.test b/test/window1.test index 59a933c4f6..5678dbb89b 100644 --- a/test/window1.test +++ b/test/window1.test @@ -2316,4 +2316,18 @@ do_execsql_test 74.4 { AND a1.c1=0; } {} +# 2023-04-11 https://sqlite.org/forum/forumpost/6c5678e3da +# An ALWAYS() turns out to be sometimes false. +# +do_execsql_test 75.0 { + DROP TABLE t1; + CREATE TABLE t1(a INT, b INT); + CREATE INDEX t1x ON t1(a+b); +} +do_catchsql_test 75.1 { + SELECT count((SELECT count(a0.a+a0.b) ORDER BY sum(0) OVER (PARTITION BY 0))) + FROM t1 AS a0 JOIN t1 AS a1 + GROUP BY a1.a; +} {1 {misuse of aggregate: count()}} + finish_test From 67f3765876f1cbb11dee99fc66837ffaf22337b8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 11 Apr 2023 19:38:47 +0000 Subject: [PATCH 199/341] New #ifdef to enable building with -DSQLITE_OMIT_WINDOWFUNC. FossilOrigin-Name: e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2be8f39913..bc517744b1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sALWAYS()\sthat\scan\ssometimes\sbe\sfalse.\s\sAdd\sa\stest\scase\sthat\smakes\nthe\stest\sfalse. -D 2023-04-11T15:06:20.059 +C New\s#ifdef\sto\senable\sbuilding\swith\s-DSQLITE_OMIT_WINDOWFUNC. +D 2023-04-11T19:38:47.332 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -625,7 +625,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c df007d60326a9aad94f5a4915d01e7de19d77627ed09338747a2e8e741a75add +F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94 F src/shell.c.in 80f7c4c1bdabc9dab416b59287458044c01df97035d626ea57ffeabbb289bdc6 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 68fcfb58df837b40e401dfa4e505f4f0b87554b9cfeb2c527f3558348b73c1be -R bc6a8cda8e023714aac7bd85583d4924 +P c8fb143d64d8e823684cd26799080da4b42bef121ca3c6315b1803a593490926 +R 23a3c5b92c8f88b7b4f263831382c52f U drh -Z 2237c0728bfff0a59ab0f03de511e2c3 +Z 51e0cafc6ddabd2f402c0ebee1449b15 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d69c4529fb..497bd4caa0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8fb143d64d8e823684cd26799080da4b42bef121ca3c6315b1803a593490926 \ No newline at end of file +e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 9e070be0b3..fcf8ae78d9 100644 --- a/src/select.c +++ b/src/select.c @@ -5266,11 +5266,13 @@ static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ return 0; } +#ifndef SQLITE_OMIT_WINDOWFUNC if( pX->pWin ){ /* This optimization does not work for subqueries that use window ** functions. */ return 0; } +#endif } colUsed = pItem->colUsed; if( pSub->pOrderBy ){ From 015020cd1a1b25ccf750590cc4443d05929c906d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Apr 2023 17:40:44 +0000 Subject: [PATCH 200/341] Add the 'secure-delete' option to fts5. Used to configure fts5 to aggressively remove old full-text-index entries belonging to deleted or updated rows. FossilOrigin-Name: 4240fd09b717dbc69dffe3b88ec9149777ca4c3efa12f282af65be3af6fa5bb0 --- ext/fts5/fts5Int.h | 10 +- ext/fts5/fts5_config.c | 25 +- ext/fts5/fts5_index.c | 553 ++++++++++++++++++++++++++++----- ext/fts5/fts5_main.c | 24 ++ ext/fts5/test/fts5_common.tcl | 4 + ext/fts5/test/fts5secure.test | 278 +++++++++++++++++ ext/fts5/test/fts5secure2.test | 87 ++++++ ext/fts5/test/fts5secure3.test | 162 ++++++++++ ext/fts5/test/fts5version.test | 76 ++++- manifest | 30 +- manifest.uuid | 2 +- 11 files changed, 1150 insertions(+), 101 deletions(-) create mode 100644 ext/fts5/test/fts5secure.test create mode 100644 ext/fts5/test/fts5secure2.test create mode 100644 ext/fts5/test/fts5secure3.test diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index e7e7043c60..79a227cb44 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -199,6 +199,7 @@ struct Fts5Config { int ePattern; /* FTS_PATTERN_XXX constant */ /* Values loaded from the %_config table */ + int iVersion; /* fts5 file format 'version' */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ @@ -207,6 +208,7 @@ struct Fts5Config { int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ + int bSecureDelete; /* 'secure-delete' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; @@ -216,8 +218,11 @@ struct Fts5Config { #endif }; -/* Current expected value of %_config table 'version' field */ -#define FTS5_CURRENT_VERSION 4 +/* Current expected value of %_config table 'version' field. And +** the expected version if the 'secure-delete' option has ever been +** set on the table. */ +#define FTS5_CURRENT_VERSION 4 +#define FTS5_CURRENT_VERSION_SECUREDELETE 5 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 @@ -383,6 +388,7 @@ struct Fts5IndexIter { ** above. */ #define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 #define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPHASH 0x0040 /* ** Create/destroy an Fts5Index object. diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index ab1a846b12..df79605ca0 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -903,6 +903,18 @@ int sqlite3Fts5ConfigSetValue( rc = SQLITE_OK; *pbBadkey = 1; } + } + + else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bSecureDelete = (bVal ? 1 : 0); + } }else{ *pbBadkey = 1; } @@ -947,15 +959,20 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ rc = sqlite3_finalize(p); } - if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ + if( rc==SQLITE_OK + && iVersion!=FTS5_CURRENT_VERSION + && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE + ){ rc = SQLITE_ERROR; if( pConfig->pzErrmsg ){ assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf( - "invalid fts5 file format (found %d, expected %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION + *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE ); } + }else{ + pConfig->iVersion = iVersion; } if( rc==SQLITE_OK ){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 694cc16e45..705be21846 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -302,6 +302,8 @@ struct Fts5Index { sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ + sqlite3_stmt *pDeleteFromIdx; + sqlite3_stmt *pDataVersion; i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ @@ -394,9 +396,6 @@ struct Fts5CResult { ** iLeafOffset: ** Byte offset within the current leaf that is the first byte of the ** position list data (one byte passed the position-list size field). -** rowid field of the current entry. Usually this is the size field of the -** position list data. The exception is if the rowid for the current entry -** is the last thing on the leaf page. ** ** pLeaf: ** Buffer containing current leaf page data. Set to NULL at EOF. @@ -1443,42 +1442,25 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ pLvl->bEof = 1; }else{ u8 *a = pLvl->pData->p; - i64 iVal; - int iLimit; - int ii; - int nZero = 0; - /* Currently iOff points to the first byte of a varint. This block - ** decrements iOff until it points to the first byte of the previous - ** varint. Taking care not to read any memory locations that occur - ** before the buffer in memory. */ - iLimit = (iOff>9 ? iOff-9 : 0); - for(iOff--; iOff>iLimit; iOff--){ - if( (a[iOff-1] & 0x80)==0 ) break; - } + pLvl->iOff = 0; + fts5DlidxLvlNext(pLvl); + while( 1 ){ + int nZero = 0; + int ii = pLvl->iOff; + u64 delta = 0; - fts5GetVarint(&a[iOff], (u64*)&iVal); - pLvl->iRowid -= iVal; - pLvl->iLeafPgno--; - - /* Skip backwards past any 0x00 varints. */ - for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){ - nZero++; - } - if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){ - /* The byte immediately before the last 0x00 byte has the 0x80 bit - ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80 - ** bytes before a[ii]. */ - int bZero = 0; /* True if last 0x00 counts */ - if( (ii-8)>=pLvl->iFirstOff ){ - int j; - for(j=1; j<=8 && (a[ii-j] & 0x80); j++); - bZero = (j>8); + while( a[ii]==0 ){ + nZero++; + ii++; } - if( bZero==0 ) nZero--; + ii += sqlite3Fts5GetVarint(&a[ii], &delta); + + if( ii>=iOff ) break; + pLvl->iLeafPgno += nZero+1; + pLvl->iRowid += delta; + pLvl->iOff = ii; } - pLvl->iLeafPgno -= nZero; - pLvl->iOff = iOff - nZero; } return pLvl->bEof; @@ -1674,7 +1656,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ i64 iOff = pIter->iLeafOffset; ASSERT_SZLEAF_OK(pIter->pLeaf); - if( iOff>=pIter->pLeaf->szLeaf ){ + while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; @@ -1773,10 +1755,12 @@ static void fts5SegIterInit( fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; - fts5SegIterNextPage(p, pIter); + do { + fts5SegIterNextPage(p, pIter); + }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 ); } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && pIter->pLeaf ){ pIter->iLeafOffset = 4; assert( pIter->pLeaf!=0 ); assert_nc( pIter->pLeaf->nn>4 ); @@ -2163,7 +2147,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ Fts5Data *pLast = 0; int pgnoLast = 0; - if( pDlidx ){ + if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){ int iSegid = pIter->pSeg->iSegid; pgnoLast = fts5DlidxIterPgno(pDlidx); pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); @@ -2724,7 +2708,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ /* ** Move the seg-iter so that it points to the first rowid on page iLeafPgno. -** It is an error if leaf iLeafPgno does not exist or contains no rowids. +** It is an error if leaf iLeafPgno does not exist. Unless the db is +** a 'secure-delete' db, if it contains no rowids then this is also an error. */ static void fts5SegIterGotoPage( Fts5Index *p, /* FTS5 backend object */ @@ -2739,21 +2724,23 @@ static void fts5SegIterGotoPage( fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; pIter->iLeafPgno = iLeafPgno-1; - fts5SegIterNextPage(p, pIter); - assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); - if( p->rc==SQLITE_OK && ALWAYS(pIter->pLeaf!=0) ){ + while( p->rc==SQLITE_OK ){ int iOff; - u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->szLeaf; - + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) break; iOff = fts5LeafFirstRowidOff(pIter->pLeaf); - if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; - }else{ - iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; - fts5SegIterLoadNPos(p, pIter); + if( iOff>0 ){ + u8 *a = pIter->pLeaf->p; + int n = pIter->pLeaf->szLeaf; + if( iOff<4 || iOff>=n ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + fts5SegIterLoadNPos(p, pIter); + } + break; } } } @@ -3468,7 +3455,7 @@ static void fts5MultiIterNew( if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; - nSeg += (p->pHash ? 1 : 0); + nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH)); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } @@ -3489,7 +3476,7 @@ static void fts5MultiIterNew( if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; - if( p->pHash ){ + if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); @@ -4244,7 +4231,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); - fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]); + fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]); if( p->rc==SQLITE_OK ){ /* Set the szLeaf field */ fts5PutU16(&buf.p[2], (u16)buf.n); @@ -4565,6 +4552,368 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ return ret; } +/* +** Execute the SQL statement: +** +** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno); +** +** This is used when a secure-delete operation removes the last term +** from a segment leaf page. In that case the %_idx entry is removed +** too. This is done to ensure that if all instances of a token are +** removed from an fts5 database in secure-delete mode, no trace of +** the token itself remains in the database. +*/ +static void fts5SecureDeleteIdxEntry( + Fts5Index *p, /* FTS5 backend object */ + int iSegid, /* Id of segment to delete entry for */ + int iPgno /* Page number within segment */ +){ + if( iPgno!=1 ){ + assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE ); + if( p->pDeleteFromIdx==0 ){ + fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)", + p->pConfig->zDb, p->pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid); + sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno); + sqlite3_step(p->pDeleteFromIdx); + p->rc = sqlite3_reset(p->pDeleteFromIdx); + } + } +} + +/* +** This is called when a secure-delete operation removes a position-list +** that overflows onto segment page iPgno of segment pSeg. This function +** rewrites node iPgno, and possibly one or more of its right-hand peers, +** to remove this portion of the position list. +** +** Output variable (*pbLastInDoclist) is set to true if the position-list +** removed is followed by a new term or the end-of-segment, or false if +** it is followed by another rowid/position list. +*/ +static void fts5SecureDeleteOverflow( + Fts5Index *p, + Fts5StructureSegment *pSeg, + int iPgno, + int *pbLastInDoclist +){ + int pgno; + Fts5Data *pLeaf = 0; + assert( iPgno!=1 ); + + *pbLastInDoclist = 1; + for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){ + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + int iNext = 0; + u8 *aPg = 0; + + pLeaf = fts5DataRead(p, iRowid); + if( pLeaf==0 ) break; + aPg = pLeaf->p; + + iNext = fts5GetU16(&aPg[0]); + if( iNext!=0 ){ + *pbLastInDoclist = 0; + } + if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){ + fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext); + } + + if( iNext==0 ){ + /* The page contains no terms or rowids. Replace it with an empty + ** page and move on to the right-hand peer. */ + const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; + fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty)); + fts5DataRelease(pLeaf); + pLeaf = 0; + }else{ + int nShift = iNext - 4; + int nPg; + + int nIdx = 0; + u8 *aIdx = 0; + + /* Unless the current page footer is 0 bytes in size (in which case + ** the new page footer will be as well), allocate and populate a + ** buffer containing the new page footer. Set stack variables aIdx + ** and nIdx accordingly. */ + if( pLeaf->nn>pLeaf->szLeaf ){ + int iFirst = 0; + int i1 = pLeaf->szLeaf; + int i2 = 0; + + aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); + if( aIdx==0 ) break; + i1 += fts5GetVarint32(&aPg[i1], iFirst); + i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); + if( i1nn ){ + memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); + i2 += (pLeaf->nn-i1); + } + nIdx = i2; + } + + /* Modify the contents of buffer aPg[]. Set nPg to the new size + ** in bytes. The new page is always smaller than the old. */ + nPg = pLeaf->szLeaf - nShift; + memmove(&aPg[4], &aPg[4+nShift], nPg-4); + fts5PutU16(&aPg[2], nPg); + if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4); + if( nIdx>0 ){ + memcpy(&aPg[nPg], aIdx, nIdx); + nPg += nIdx; + } + sqlite3_free(aIdx); + + /* Write the new page to disk and exit the loop */ + assert( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, iRowid, aPg, nPg); + break; + } + } + fts5DataRelease(pLeaf); +} + + +/* +** This is called as part of flushing a delete to disk in 'secure-delete' +** mode. It edits the segments within the database described by argument +** pStruct to remove the entries for term zTerm, rowid iRowid. +*/ +static void fts5FlushSecureDelete( + Fts5Index *p, + Fts5Structure *pStruct, + const char *zTerm, + i64 iRowid +){ + const int f = FTS5INDEX_QUERY_SKIPHASH; + int nTerm = strlen(zTerm); + Fts5Iter *pIter = 0; /* Used to find term instance */ + + fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); + if( fts5MultiIterEof(p, pIter)==0 ){ + i64 iThis = fts5MultiIterRowid(pIter); + if( iThisrc==SQLITE_OK + && fts5MultiIterEof(p, pIter)==0 + && iRowid==fts5MultiIterRowid(pIter) + ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + int iSegid = pSeg->pSeg->iSegid; + u8 *aPg = pSeg->pLeaf->p; + int nPg = pSeg->pLeaf->nn; + int iPgIdx = pSeg->pLeaf->szLeaf; + + u64 iDelta = 0; + u64 iNextDelta = 0; + int iNextOff = 0; + int iOff = 0; + int nMove = 0; + int nIdx = 0; + + u8 *aIdx = 0; + + nIdx = nPg-iPgIdx; + aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + if( aIdx ){ + int bLastInDoclist = 0; + int iIdx = 0; + int iStart = 0; + int iKeyOff = 0; + int iPrevKeyOff = 0; + int nShift = 0; + int iDelKeyOff = 0; /* Offset of deleted key, if any */ + memcpy(aIdx, &aPg[iPgIdx], nIdx); + + /* At this point segment iterator pSeg points to the entry + ** this function should remove from the b-tree segment. + ** + ** More specifically, pSeg->iLeafOffset is the offset of the + ** first byte in the position-list for the entry to remove. + ** Immediately before this comes two varints that will also + ** need to be removed: + ** + ** + the rowid or delta rowid value for the entry, and + ** + the size of the position list in bytes. + */ + { + int iSOP; + int nPos = 0; + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ + iStart = pSeg->iTermLeafOffset; + }else{ + iStart = fts5GetU16(&aPg[0]); + } + + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + assert_nc( iSOP<=pSeg->iLeafOffset ); + while( iSOPiLeafOffset ){ + iStart = iSOP + (nPos/2); + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + } + assert_nc( iSOP==pSeg->iLeafOffset ); + } + + iOff = iStart; + iNextOff = pSeg->iLeafOffset + pSeg->nPos; + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + }else{ + /* Set bLastInDoclist to true if the entry being removed is the last + ** in its doclist. */ + for(iIdx=0, iKeyOff=0; iIdxiTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno + ){ + /* The entry being removed was the only position list in its + ** doclist. Therefore the term needs to be removed as well. */ + int iKey = 0; + for(iIdx=0, iKeyOff=0; iIdxiStart ) break; + iKeyOff += iVal; + } + + iDelKeyOff = iOff = iKeyOff; + if( iNextOff!=iPgIdx ){ + int nPrefix = 0; + int nSuffix = 0; + int nPrefix2 = 0; + int nSuffix2 = 0; + + iDelKeyOff = iNextOff; + iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + + if( iKey!=1 ){ + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + } + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + + nPrefix = MIN(nPrefix, nPrefix2); + nSuffix = (nPrefix2 + nSuffix2) - nPrefix; + + if( iKey!=1 ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + } + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); + if( nPrefix2>nPrefix ){ + memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix); + iOff += (nPrefix2-nPrefix); + } + memcpy(&aPg[iOff], &aPg[iNextOff], nSuffix2); + iOff += nSuffix2; + iNextOff += nSuffix2; + } + }else if( iStart==4 ){ + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + int iPgno = pSeg->iLeafPgno-1; + + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; + + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } + } + fts5DataRelease(pTerm); + } + } + + nMove = nPg - iNextOff; + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= (iNextOff - iOff); + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); + + nShift = iNextOff - iOff; + for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ + iKeyOff -= nShift; + nShift = 0; + } + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); + iPrevKeyOff = iKeyOff; + } + } + + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + } + + assert( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid, pSeg->iLeafPgno), aPg, nPg); + sqlite3_free(aIdx); + } + } + } + + fts5MultiIterFree(pIter); +} + + /* ** Flush the contents of in-memory hash table iHash to a new level-0 ** segment on disk. Also update the corresponding structure record. @@ -4587,6 +4936,7 @@ static void fts5FlushOneHash(Fts5Index *p){ if( iSegid ){ const int pgsz = p->pConfig->pgsz; int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ @@ -4609,29 +4959,53 @@ static void fts5FlushOneHash(Fts5Index *p){ } while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ const u8 *pDoclist; /* Pointer to doclist for this term */ int nDoclist; /* Size of doclist in bytes */ - /* Write the term for this entry to disk. */ + /* Get the term and doclist for this entry. */ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); - if( p->rc!=SQLITE_OK ) break; + nTerm = (int)strlen(zTerm); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } - assert( writer.bFirstRowidInPage==0 ); - if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ + int bTermWritten = !bSecureDelete; i64 iRowid = 0; - u64 iDelta = 0; + i64 iPrev = 0; int iOff = 0; /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOffrc!=SQLITE_OK ) break; + bTermWritten = 1; + assert( writer.bFirstRowidInPage==0 ); + } if( writer.bFirstRowidInPage ){ fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ @@ -4640,9 +5014,10 @@ static void fts5FlushOneHash(Fts5Index *p){ fts5WriteDlidxAppend(p, &writer, iRowid); if( p->rc!=SQLITE_OK ) break; }else{ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); } assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; if( eDetail==FTS5_DETAIL_NONE ){ if( iOffnLevel==0 ){ - fts5StructureAddLevel(&p->rc, &pStruct); + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); } - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); - if( p->rc==SQLITE_OK ){ - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; - pStruct->nSegment++; - } - fts5StructurePromote(p, 0, pStruct); } fts5IndexAutomerge(p, &pStruct, pgnoLast); @@ -5455,6 +5833,7 @@ int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); sqlite3_finalize(p->pDataVersion); + sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); @@ -5834,6 +6213,7 @@ static void fts5TestDlidxReverse( int pgno = fts5DlidxIterPgno(pDlidx); assert( pgno>iLeaf ); cksum1 += iRowid + ((i64)pgno<<32); + // printf("1: rowid=%lld pgno=%d\n", iRowid, pgno); } fts5DlidxIterFree(pDlidx); pDlidx = 0; @@ -5846,6 +6226,7 @@ static void fts5TestDlidxReverse( int pgno = fts5DlidxIterPgno(pDlidx); assert( fts5DlidxIterPgno(pDlidx)>iLeaf ); cksum2 += iRowid + ((i64)pgno<<32); + // printf("2: rowid=%lld pgno=%d\n", iRowid, pgno); } fts5DlidxIterFree(pDlidx); pDlidx = 0; @@ -6085,6 +6466,7 @@ static void fts5IndexIntegrityCheckSegment( Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5Config *pConfig = p->pConfig; + int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE); sqlite3_stmt *pStmt = 0; int rc2; int iIdxPrevLeaf = pSeg->pgnoFirst-1; @@ -6120,7 +6502,19 @@ static void fts5IndexIntegrityCheckSegment( ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ if( pLeaf->nn<=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + + if( nIdxTerm==0 + && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE + && pLeaf->nn==pLeaf->szLeaf + && pLeaf->nn==4 + ){ + /* special case - the very first page in a segment keeps its %_idx + ** entry even if all the terms are removed from it by secure-delete + ** operations. */ + }else{ + p->rc = FTS5_CORRUPT; + } + }else{ int iOff; /* Offset of first term on leaf */ int iRowidOff; /* Offset of first rowid on leaf */ @@ -6184,9 +6578,12 @@ static void fts5IndexIntegrityCheckSegment( ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; - }else{ + }else if( bSecureDelete==0 || iRowidOff>0 ){ + i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); - if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; + if( iRowidrc = FTS5_CORRUPT; + } } fts5DataRelease(pLeaf); } diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 5392b3ba0f..13921ce49e 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -1623,6 +1623,8 @@ static int fts5UpdateMethod( Fts5Config *pConfig = pTab->p.pConfig; int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ + int bUpdateOrDelete = 0; + /* A transaction must be open when this is called. */ assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); @@ -1633,6 +1635,11 @@ static int fts5UpdateMethod( || sqlite3_value_type(apVal[0])==SQLITE_NULL ); assert( pTab->p.pConfig->pzErrmsg==0 ); + if( pConfig->pgsz==0 ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) return rc; + } + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; /* Put any active cursors into REQUIRE_SEEK state. */ @@ -1685,6 +1692,7 @@ static int fts5UpdateMethod( else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); + bUpdateOrDelete = 1; } /* INSERT or UPDATE */ @@ -1700,6 +1708,7 @@ static int fts5UpdateMethod( if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + bUpdateOrDelete = 1; } fts5StorageInsert(&rc, pTab, apVal, pRowid); } @@ -1728,10 +1737,24 @@ static int fts5UpdateMethod( rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); fts5StorageInsert(&rc, pTab, apVal, pRowid); } + bUpdateOrDelete = 1; } } } + if( rc==SQLITE_OK + && bUpdateOrDelete + && pConfig->bSecureDelete + && pConfig->iVersion==FTS5_CURRENT_VERSION + ){ + rc = sqlite3Fts5StorageConfigValue( + pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE + ); + if( rc==SQLITE_OK ){ + pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; + } + } + pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -2591,6 +2614,7 @@ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); + pTab->p.pConfig->pgsz = 0; return sqlite3Fts5StorageRollback(pTab->pStorage); } diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 0f371dcfd9..9c012932da 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -594,6 +594,10 @@ proc nearset_rc {aCol args} { list } +proc dump {tname} { + execsql_pp "SELECT * FROM ${tname}_idx" + execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data" +} #------------------------------------------------------------------------- # Code for a simple Tcl tokenizer that supports synonyms at query time. diff --git a/ext/fts5/test/fts5secure.test b/ext/fts5/test/fts5secure.test new file mode 100644 index 0000000000..50d84cef79 --- /dev/null +++ b/ext/fts5/test/fts5secure.test @@ -0,0 +1,278 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure + +proc dump {tname} { + execsql_pp "SELECT * FROM ${tname}_idx" + execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data" +} + + +do_execsql_test 0.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + CREATE VIRTUAL TABLE v1 USING fts5vocab('t1', 'instance'); + INSERT INTO t1(rowid, ab) VALUES + (0,'abc'), (1,'abc'), (2,'abc'), (3,'abc'), (4,'def'); +} + +do_execsql_test 0.1 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 0.2 { + DELETE FROM t1 WHERE rowid=2; +} + +do_execsql_test 0.3 { + SELECT count(*) FROM t1_data +} 3 + +do_execsql_test 0.4 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 0.5 { + DELETE FROM t1 WHERE rowid=3; +} + +do_execsql_test 0.6 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 0.7 { + DELETE FROM t1 WHERE rowid=0; +} + +do_execsql_test 0.8 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#---------------------------------- + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5(ab); + INSERT INTO t2(rowid, ab) VALUES (5, 'key'), (6, 'value'); + INSERT INTO t2(t2, rank) VALUES('secure-delete', 1); +} + +#execsql_pp { SELECT id, quote(block) FROM t1_data } +#execsql_pp { SELECT segid, quote(term), pgno FROM t1_idx } + +do_execsql_test 1.1 { + DELETE FROM t2 WHERE rowid = 5; +} + +do_execsql_test 1.2 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +do_execsql_test 1.3 { + DELETE FROM t2 WHERE rowid = 6; +} + +do_execsql_test 1.4 { + INSERT INTO t2(t2) VALUES('integrity-check'); +} + +do_execsql_test 1.5 { + SELECT * FROM t2('value'); + SELECT * FROM t2('v*'); +} + +do_execsql_test 1.6 { + SELECT * FROM t2('value') ORDER BY rowid DESC; + SELECT * FROM t2('v*') ORDER BY rowid DESC; +} +execsql_pp { + SELECT id, quote(block) FROM t2_data; +} + +#---------------------------------- + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(ab); + CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', 'instance'); + INSERT INTO ft(rowid, ab) VALUES + (1, 'one'), + (2, 'two'), + (3, 'three'), + (4, 'four'), + (5, 'one one'), + (6, 'one two'), + (7, 'one three'), + (8, 'one four'), + (9, 'two one'), + (10, 'two two'), + (11, 'two three'), + (12, 'two four'), + (13, 'three one'), + (14, 'three two'), + (15, 'three three'), + (16, 'three four'); +} + +do_execsql_test 2.1 { + SELECT count(*) FROM ft_data; +} {3} + +do_execsql_test 2.2 { + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 2.3 { + DELETE FROM ft WHERE rowid=9; +} + +do_execsql_test 2.4 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.5 { + DELETE FROM ft WHERE ab LIKE '%two%' +} + +do_execsql_test 2.6 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +do_execsql_test 2.7 { + SELECT count(*) FROM ft_data; +} {3} + +#---------------------------------- +reset_db + +set ::vocab { + one two three four five six seven eight nine ten + eleven twelve thirteen fourteen fifteen sixteen + seventeen eighteen nineteen twenty +} +proc rnddoc {} { + set nVocab [llength $::vocab] + set ret [list] + for {set ii 0} {$ii < 8} {incr ii} { + lappend ret [lindex $::vocab [expr int(abs(rand()) * $nVocab)]] + } + set ret +} + +proc contains {list val} { + expr {[lsearch $list $val]>=0} +} + +foreach {tn pgsz} { + 2 64 + 1 1000 +} { + reset_db + db function rnddoc rnddoc + db function contains contains + + expr srand(1) + + do_execsql_test 3.$tn.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); + WITH s(i) AS ( + VALUES(1) UNION SELECT i+1 FROM s WHERE i<20 + ) + INSERT INTO t1 SELECT rnddoc() FROM s; + } + + do_execsql_test 3.$tn.1 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } + + foreach {rowid} { + 6 16 3 4 9 14 13 7 20 15 19 10 11 2 5 18 17 1 12 8 + } { + + do_execsql_test 3.$tn.2.$rowid { + DELETE FROM t1 WHERE rowid=$rowid; + } + do_execsql_test 3.$tn.2.$rowid.ic { + INSERT INTO t1(t1) VALUES('integrity-check'); + } + + foreach v $::vocab { + do_execsql_test 3.$tn.2.$rowid.q.$v { + SELECT rowid FROM t1($v) + } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v)}] + + do_execsql_test 3.$tn.2.$rowid.q.$v.DESC { + SELECT rowid FROM t1($v) ORDER BY 1 DESC + } [db eval {SELECT rowid FROM t1 WHERE contains(x, $v) ORDER BY 1 DESC}] + } + } +} + +do_execsql_test 3.3 { + INSERT INTO t1(x) VALUES('optimize'); + INSERT INTO t1(t1) VALUES('optimize'); + SELECT count(*) FROM t1_data; +} {3} + +#---------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +set L1 [string repeat abcdefghij 10] +set L2 [string repeat 1234567890 10] + +do_execsql_test 4.1 { + INSERT INTO t1 VALUES('aa' || $L1 || ' ' || $L2); +} +do_execsql_test 4.2 { + DELETE FROM t1 WHERE rowid=1 +} +do_execsql_test 4.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +#---------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +set doc "aa [string repeat {abc } 60]" + +do_execsql_test 5.1 { + BEGIN; + INSERT INTO t1 VALUES($doc); + INSERT INTO t1 VALUES('aa abc'); + COMMIT; +} + +do_execsql_test 5.2 { + DELETE FROM t1 WHERE rowid = 1; +} + +do_execsql_test 5.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2 +do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2 + + +finish_test + diff --git a/ext/fts5/test/fts5secure2.test b/ext/fts5/test/fts5secure2.test new file mode 100644 index 0000000000..04ff66219c --- /dev/null +++ b/ext/fts5/test/fts5secure2.test @@ -0,0 +1,87 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure2 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(col); + INSERT INTO ft VALUES('data for the table'); + INSERT INTO ft VALUES('more of the same'); + INSERT INTO ft VALUES('and extra data'); +} + +do_execsql_test 1.1 { + SELECT * FROM ft_config +} {version 4} + +do_execsql_test 1.2 { + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); + SELECT * FROM ft_config; +} {secure-delete 1 version 4} + +do_execsql_test 1.3 { + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); + SELECT * FROM ft_config; +} {secure-delete 1 version 4} + +do_execsql_test 1.4 { + DELETE FROM ft WHERE rowid=2; + SELECT * FROM ft_config; +} {secure-delete 1 version 5} + +do_execsql_test 1.5 { + SELECT rowid, col FROM ft('data'); +} {1 {data for the table} 3 {and extra data}} + +db close +sqlite3 db test.db + +do_execsql_test 1.6 { + SELECT rowid, col FROM ft('data'); +} {1 {data for the table} 3 {and extra data}} + +#------------------------------------------------------------------------ + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft USING fts5(col); + INSERT INTO ft VALUES('one zero one one zero'); + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 2.1 { + SELECT count(*) FROM ft_data WHERE block=X'00000004'; +} {0} + +do_execsql_test 2.2 { + UPDATE ft SET col = 'zero one zero zero one' WHERE rowid=1; +} + +do_execsql_test 2.3 { + SELECT count(*) FROM ft_data WHERE block=X'00000004'; +} {1} + +do_execsql_test 2.4 { + INSERT INTO ft VALUES('one zero zero one'); + DELETE FROM ft WHERE rowid=1; +} + +do_execsql_test 2.5 { + SELECT count(*) FROM ft_data WHERE block=X'00000004'; +} {2} + + +finish_test + + diff --git a/ext/fts5/test/fts5secure3.test b/ext/fts5/test/fts5secure3.test new file mode 100644 index 0000000000..70c2028df6 --- /dev/null +++ b/ext/fts5/test/fts5secure3.test @@ -0,0 +1,162 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure3 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(col); + INSERT INTO ft VALUES('data for the table'); + INSERT INTO ft VALUES('more of the same'); + INSERT INTO ft VALUES('and extra data'); + + INSERT INTO ft(ft, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft(rowid, col) VALUES(0, 'the next data'); + DELETE FROM ft WHERE rowid=1; + DELETE FROM ft WHERE rowid=2; + INSERT INTO ft(rowid, col) VALUES(6, 'with some more of the same data'); + COMMIT; +} + +do_execsql_test 1.2 { + INSERT INTO ft(ft) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- + +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + BEGIN; + INSERT INTO t1 VALUES('the start'); +} +do_test 2.1 { + for {set i 0} {$i < 1000} {incr i} { + execsql { INSERT INTO t1 VALUES('the ' || hex(randomblob(3))) } + } + execsql { + INSERT INTO t1 VALUES('the end'); + COMMIT; + } +} {} + +do_execsql_test 2.2 { + DELETE FROM t1 WHERE rowid BETWEEN 2 AND 1000; +} + +do_execsql_test 2.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 2.6 { + INSERT INTO t1(rowid, x) VALUES(500, 'middle'); + INSERT INTO t1(rowid, x) VALUES(501, 'value'); + SELECT * FROM t1('the middle'); +} + +do_execsql_test 2.7 { + INSERT INTO t1(t1) VALUES('optimize'); +} + +do_execsql_test 2.8 { + SELECT count(*) FROM t1_data +} 4 + +#execsql_pp { SELECT id, quote(block), fts5_decode(id, block) FROM t1_data; } + +#------------------------------------------------------------------------- +# Tests with large/small rowid values. +# + +reset_db + +expr srand(0) + +set vocab { + Popper Poppins Popsicle Porfirio Porrima Porsche + Porter Portia Portland Portsmouth Portugal Portuguese + Poseidon Post PostgreSQL Potemkin Potomac Potsdam + Pottawatomie Potter Potts Pound Poussin Powell + PowerPC PowerPoint Powers Powhatan Poznan Prada + Prado Praetorian Prague Praia Prakrit Pratchett + Pratt Pravda Praxiteles Preakness Precambrian Preminger + Premyslid Prensa Prentice Pres Presbyterian Presbyterianism +} +proc newdoc {} { + for {set i 0} {$i<8} {incr i} { + lappend ret [lindex $::vocab [expr int(abs(rand()) * [llength $::vocab])]] + } + set ret +} +db func newdoc newdoc + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE fff USING fts5(y); + INSERT INTO fff(fff, rank) VALUES('pgsz', 64); + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + WITH s(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM s WHERE x<1000 ) + INSERT INTO fff(rowid, y) SELECT random() , newdoc() FROM s; + + INSERT INTO fff(fff, rank) VALUES('secure-delete', 1); +} + +proc lshuffle {in} { + set out [list] + while {[llength $in]>0} { + set idx [expr int(abs(rand()) * [llength $in])] + lappend out [lindex $in $idx] + set in [lreplace $in $idx $idx] + } + set out +} + +#dump fff + +set iTest 1 +foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { + #if {$iTest==1} { dump fff } + #if {$iTest==1} { breakpoint } + do_execsql_test 3.1.$iTest.$ii { + DELETE FROM fff WHERE rowid=$ii; + } + #if {$iTest==1} { dump fff } + do_execsql_test 3.1.$iTest.$ii.ic { + INSERT INTO fff(fff) VALUES('integrity-check'); + } + #if {$iTest==1} { break } + incr iTest +} + +#execsql_pp { SELECT rowid FROM fff('post') ORDER BY rowid ASC } +#breakpoint +#execsql_pp { +# SELECT rowid FROM fff('post') ORDER BY rowid DESC +#} +# +#dump fff + + +finish_test + diff --git a/ext/fts5/test/fts5version.test b/ext/fts5/test/fts5version.test index 60ec81c03d..79fd94e6bc 100644 --- a/ext/fts5/test/fts5version.test +++ b/ext/fts5/test/fts5version.test @@ -38,20 +38,20 @@ do_execsql_test 1.3 { sqlite3_db_config db DEFENSIVE 0 do_execsql_test 1.4 { - UPDATE t1_config set v=5 WHERE k='version'; + UPDATE t1_config set v=6 WHERE k='version'; } do_test 1.5 { db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}} do_test 1.6 { db close sqlite3 db test.db catchsql { INSERT INTO t1 VALUES('x y z') } -} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 6, expected 4 or 5) - run 'rebuild'}} do_test 1.7 { sqlite3_db_config db DEFENSIVE 0 @@ -59,7 +59,75 @@ do_test 1.7 { db close sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' } -} {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}} +} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}} + +do_test 1.8 { + sqlite3_db_config db DEFENSIVE 0 + execsql { INSERT INTO t1_config VALUES('version', 4) } + execsql { INSERT INTO t1(t1, rank) VALUES('secure-delete', 1) } +} {} + +do_execsql_test 1.10 { + SELECT * FROM t1_config +} {secure-delete 1 version 4} + +do_execsql_test 1.11 { + INSERT INTO t1(rowid, one) VALUES(123, 'one two three'); + DELETE FROM t1 WHERE rowid=123; + SELECT * FROM t1_config +} {secure-delete 1 version 5} + +do_execsql_test 1.11 { + INSERT INTO t1(t1) VALUES('rebuild'); + SELECT * FROM t1_config +} {secure-delete 1 version 4} + +do_execsql_test 1.12 { + SELECT * FROM t1_config +} {secure-delete 1 version 4} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xyz USING fts5(x); + INSERT INTO xyz(rowid, x) VALUES + (1, 'one document'), + (2, 'two document'), + (3, 'three document'), + (4, 'four document'), + (5, 'five document'), + (6, 'six document'); + + INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1); + SELECT v FROM xyz_config WHERE k='version'; +} {4} + +do_execsql_test 2.1 { + BEGIN; + INSERT INTO xyz(rowid, x) VALUES(7, 'seven document'); + SAVEPOINT one; + DELETE FROM xyz WHERE rowid = 4; +} + +do_execsql_test 2.2 { + SELECT v FROM xyz_config WHERE k='version'; +} {5} + +do_execsql_test 2.3 { + ROLLBACK TO one; + SELECT v FROM xyz_config WHERE k='version'; +} {4} + + +do_execsql_test 2.4 { + DELETE FROM xyz WHERE rowid = 3; + COMMIT; + SELECT v FROM xyz_config WHERE k='version'; +} {5} + + finish_test + diff --git a/manifest b/manifest index bc517744b1..ebb9c13a4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\s#ifdef\sto\senable\sbuilding\swith\s-DSQLITE_OMIT_WINDOWFUNC. -D 2023-04-11T19:38:47.332 +C Add\sthe\s'secure-delete'\soption\sto\sfts5.\sUsed\sto\sconfigure\sfts5\sto\saggressively\sremove\sold\sfull-text-index\sentries\sbelonging\sto\sdeleted\sor\supdated\srows. +D 2023-04-12T17:40:44.865 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -86,14 +86,14 @@ F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6d F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a -F ext/fts5/fts5Int.h c0d46e399e345e35985b72a1c1af025973bfaa5b1e3563b0ce3bb0ce144a7ca3 +F ext/fts5/fts5Int.h f473de2bdae0977af0d6c8cce96e3666821b85efba5f6006c7732662c3aabcb3 F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 -F ext/fts5/fts5_config.c 501e7d3566bc92766b0e11c0109a7c5a6146bc41144195459af5422f6c2078aa +F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c df5b29576a409f673e54b470723d817df9d5167cff208c48ab9a3773cba6fa89 -F ext/fts5/fts5_main.c fe67b6fb2ef134d9dbfa3941c63f777d755b075449be1863cb913a7f8754cb69 +F ext/fts5/fts5_index.c 111da0b3226461111bf014a4e0c57dc61bcd7947b90584699ca1090f409fef45 +F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -104,7 +104,7 @@ F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d0988 F ext/fts5/fts5_vocab.c 12138e84616b56218532e3e8feb1d3e0e7ae845e33408dbe911df520424dc9d6 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841 +F ext/fts5/test/fts5_common.tcl a9de9c2209cc4e7ae3c753e783504e67206c6c1467d08f209cd0c5923d3e8d8b F ext/fts5/test/fts5aa.test 5bd43427b7d08ce2e19c488a26534be450538b9232d4d5305049e8de236e9aa9 F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18ca9385880bb7 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d @@ -187,6 +187,9 @@ F ext/fts5/test/fts5rebuild.test 55d6f17715cddbf825680dd6551efbc72ed916d8cf1cde4 F ext/fts5/test/fts5restart.test 835ecc8f449e3919f72509ab58056d0cedca40d1fe04108ccf8ac4c2ba41f415 F ext/fts5/test/fts5rowid.test b8790ec170a8dc1942a15aef3db926a5f3061b1ff171013003d8297203a20ad6 F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e332d654375690f8e6e43f +F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64 +F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc +F ext/fts5/test/fts5secure3.test fd73b98a6e11038960b84109fed42f9e0a098a31338c94d07c163244d4bd7254 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f @@ -204,7 +207,7 @@ F ext/fts5/test/fts5unicode3.test 590c72e18195bda2446133f9d82d04a4e89d094bba58c7 F ext/fts5/test/fts5unicode4.test 6463301d669f963c83988017aa354108be0b947d325aef58d3abddf27147b687 F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db63892db6bafabbec21af4d F ext/fts5/test/fts5update.test b8affd796e45c94a4d19ad5c26606ea06065a0f162a9562d9f005b5a80ccf0bc -F ext/fts5/test/fts5version.test c8f2cc105f0abf0224965f93e584633dee3e06c91478bc67e468f7cfdf97fd6a +F ext/fts5/test/fts5version.test d6e5a5897550afeccc2f8531d87404dc1c289ee89385dd4318dbdd75e71d7a67 F ext/fts5/test/fts5vocab.test 7ed80d9af1ddaaa1637da05e406327b5aac250848bc604c1c1cc667908b87760 F ext/fts5/test/fts5vocab2.test 681980e92e031c9f3fe8d9c149189e876c108da2fb0fb3a25bd8a9b94bff8f68 F ext/fts5/tool/fts5speed.tcl b0056f91a55b2d1a3684ec05729de92b042e2f85 @@ -2052,8 +2055,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8fb143d64d8e823684cd26799080da4b42bef121ca3c6315b1803a593490926 -R 23a3c5b92c8f88b7b4f263831382c52f -U drh -Z 51e0cafc6ddabd2f402c0ebee1449b15 +P e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c +R 77520373799013e82dff45638b253c47 +T *branch * fts5-secure-delete +T *sym-fts5-secure-delete * +T -sym-trunk * +U dan +Z 0c15d52b2e44a1574709121e2f6a7b6b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 497bd4caa0..5f639c5e40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c \ No newline at end of file +4240fd09b717dbc69dffe3b88ec9149777ca4c3efa12f282af65be3af6fa5bb0 \ No newline at end of file From ad5a7da489f3657c42fe6770309a5d0fcccf92d5 Mon Sep 17 00:00:00 2001 From: larrybr Date: Wed, 12 Apr 2023 17:54:52 +0000 Subject: [PATCH 201/341] WIP: CLI option to take control of console on Windows and make it support UTF-8 input pasting (or typing). Needs work to become robust per "ToDo:". FossilOrigin-Name: 824382393d92d9eb6df8701de7c263280150569a708759c4a539acc6d8d38821 --- manifest | 17 ++++---- manifest.uuid | 2 +- src/shell.c.in | 110 ++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 120 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bc517744b1..ef57a3a675 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\s#ifdef\sto\senable\sbuilding\swith\s-DSQLITE_OMIT_WINDOWFUNC. -D 2023-04-11T19:38:47.332 +C WIP:\sCLI\soption\sto\stake\scontrol\sof\sconsole\son\sWindows\sand\smake\sit\ssupport\sUTF-8\sinput\spasting\s(or\styping).\sNeeds\swork\sto\sbecome\srobust\sper\s"ToDo:". +D 2023-04-12T17:54:52.445 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94 -F src/shell.c.in 80f7c4c1bdabc9dab416b59287458044c01df97035d626ea57ffeabbb289bdc6 +F src/shell.c.in 156279044afe3acdd48997531617c6bd1b5a37b5761b67900cc4dbfcc8af7130 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8fb143d64d8e823684cd26799080da4b42bef121ca3c6315b1803a593490926 -R 23a3c5b92c8f88b7b4f263831382c52f -U drh -Z 51e0cafc6ddabd2f402c0ebee1449b15 +P e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c +R 42a09bb2160cadfa8aa798fa30b96ed8 +T *branch * cli-utf8 +T *sym-cli-utf8 * +T -sym-trunk * +U larrybr +Z e14b183810e19b5d66903057d8becd3e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 497bd4caa0..230ab719ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c \ No newline at end of file +824382393d92d9eb6df8701de7c263280150569a708759c4a539acc6d8d38821 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 220e91fd1d..c7c0d537ec 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -228,6 +228,7 @@ typedef unsigned char u8; #if SQLITE_OS_WINRT #include #endif +#define WIN32_LEAN_AND_MEAN #include /* string conversion routines only needed on Win32 */ @@ -442,11 +443,18 @@ static void endTimer(void){ static int bail_on_error = 0; /* -** Threat stdin as an interactive input if the following variable +** Treat stdin as an interactive input if the following variable ** is true. Otherwise, assume stdin is connected to a file or pipe. */ static int stdin_is_interactive = 1; +#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) +/* +** Setup console for UTF-8 input/output when following variable true. +*/ +static int console_utf8 = 0; +#endif + /* ** On Windows systems we have to know if standard output is a console ** in order to translate UTF-8 into MBCS. The following variable is @@ -579,6 +587,80 @@ static char *dynamicContinuePrompt(void){ } #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ +#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) +static int infsMode; +static UINT codePage; +static HANDLE hConsoleIn; +static DWORD consoleMode; +/* +** Prepare input stream, (if known to be a WIN32 console), for UTF-8 +** input (from either typing or suitable paste operations) and for +** UTF-8 rendering. This may "fail" with a message to stderr, where +** the preparation is not done and common "code page" issues occur. +*/ +static void instream_prepare(){ + if( isatty(0) ){ + if( !IsValidCodePage(CP_UTF8) ){ + fprintf(stderr, "Cannot use UTF-8 code page.\n"); + console_utf8 = 0; + return; + } + codePage = GetConsoleCP(); + SetConsoleCP(CP_UTF8); + hConsoleIn = GetStdHandle(STD_INPUT_HANDLE); + GetConsoleMode( hConsoleIn, &consoleMode); + SetConsoleMode( hConsoleIn, consoleMode | ENABLE_LINE_INPUT ); + infsMode = _setmode(_fileno(stdin), _O_U16TEXT); + console_utf8 = 1; + }else{ + console_utf8 = 0; + } +} + +/* +** Undo the effects of instream_prepare(), if any. +*/ +static void instream_restore(){ + if( console_utf8 ){ + _setmode(_fileno(stdin), infsMode); + SetConsoleCP(codePage); + SetConsoleMode( hConsoleIn, consoleMode ); + } +} + +/* +** Collect input like fgets(...) with special provisions for input +** from the Windows console to get around its strange coding issues. +** Defers to plain fgets() when input is not interactive or when the +** startup option, -utf8, has not been provided or taken effect. +** WIP: This routine needs work to consume only as many characters +** as can be fit into the provided input buffer. ToDo: That work. +** Avoiding this issue will require some refactoring of CLI input. +*/ +static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ + if( fin==stdin && stdin_is_interactive && console_utf8 ){ + wchar_t * wbuf = (wchar_t*) malloc(ncmax * sizeof(wchar_t)); + wchar_t * wz; + int nmb; + if( wbuf==0 ) return 0; + if( 0 == (wz = fgetws(wbuf, ncmax, stdin)) + || 0 == (nmb = WideCharToMultiByte(CP_UTF8, 0, wz, -1, 0, 0, 0, 0)) + || nmb > ncmax ){ + buf = 0; + goto bail; + } + WideCharToMultiByte(CP_UTF8, 0, wz, -1, buf, nmb, 0, 0); + bail: + free(wbuf); + return buf; + }else{ + return fgets(buf, ncmax, fin); + } +} + +# define fgets(b,n,f) utf8_fgets(b,n,f) +#endif /* (defined(_WIN32)||defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) */ + /* ** Render output like fprintf(). Except, if the output is going to the ** console and if this is running on a Windows machine, translate the @@ -802,7 +884,11 @@ static char *local_getline(char *zLine, FILE *in){ /* For interactive input on Windows systems, translate the ** multi-byte characterset characters into UTF-8. */ if( stdin_is_interactive && in==stdin ){ + // FILE *cil = fopen("cin.log", "wa"); + // i64 nzl = strlen(zLine); char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); + // fwrite(zLine, nzl, 1, cil); + // fwrite("\n", 1,1, cil); if( zTrans ){ i64 nTrans = strlen(zTrans)+1; if( nTrans>nLine ){ @@ -810,8 +896,11 @@ static char *local_getline(char *zLine, FILE *in){ shell_check_oom(zLine); } memcpy(zLine, zTrans, nTrans); + // fwrite(zLine, nTrans-1, 1, cil); + // fwrite("\n", 1,1, cil); sqlite3_free(zTrans); } + // fclose(cil); } #endif /* defined(_WIN32) || defined(WIN32) */ return zLine; @@ -11585,6 +11674,9 @@ static const char zOptions[] = " -stats print memory stats before each finalize\n" " -table set output mode to 'table'\n" " -tabs set output mode to 'tabs'\n" +#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) + " -utf8 setup interactive console code page for UTF-8\n" +#endif " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" #ifdef SQLITE_ENABLE_VFSTRACE @@ -12116,6 +12208,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = 1; }else if( cli_strcmp(z,"-batch")==0 ){ stdin_is_interactive = 0; + }else if( cli_strcmp(z,"-utf8")==0 ){ +#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) + console_utf8 = 1; +#endif }else if( cli_strcmp(z,"-heap")==0 ){ i++; }else if( cli_strcmp(z,"-pagecache")==0 ){ @@ -12193,6 +12289,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } data.cMode = data.mode; } +#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) + if( console_utf8 && stdin_is_interactive ){ + instream_prepare(); + }else{ + console_utf8 = 0; + } +#endif if( !readStdin ){ /* Run all arguments that do not begin with '-' as if they were separate @@ -12267,6 +12370,11 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ rc = process_input(&data); } } +#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) + if( console_utf8 ){ + instream_restore(); + } +#endif #ifndef SQLITE_SHELL_FIDDLE /* In WASM mode we have to leave the db state in place so that ** client code can "push" SQL into it after this call returns. */ From 02877e9558bc1867f5641fafb56591ade277f4e8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Apr 2023 18:06:43 +0000 Subject: [PATCH 202/341] Remove temporary debugging code accidentally left on this branch. FossilOrigin-Name: 0a0f64870feaf95d2673efc8884ad1236d9bbf9bdb94364f8d2602221bd36ff3 --- ext/fts5/fts5_index.c | 2 -- manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 705be21846..d6dc9fb9a4 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6213,7 +6213,6 @@ static void fts5TestDlidxReverse( int pgno = fts5DlidxIterPgno(pDlidx); assert( pgno>iLeaf ); cksum1 += iRowid + ((i64)pgno<<32); - // printf("1: rowid=%lld pgno=%d\n", iRowid, pgno); } fts5DlidxIterFree(pDlidx); pDlidx = 0; @@ -6226,7 +6225,6 @@ static void fts5TestDlidxReverse( int pgno = fts5DlidxIterPgno(pDlidx); assert( fts5DlidxIterPgno(pDlidx)>iLeaf ); cksum2 += iRowid + ((i64)pgno<<32); - // printf("2: rowid=%lld pgno=%d\n", iRowid, pgno); } fts5DlidxIterFree(pDlidx); pDlidx = 0; diff --git a/manifest b/manifest index ebb9c13a4c..e432d7ed9a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s'secure-delete'\soption\sto\sfts5.\sUsed\sto\sconfigure\sfts5\sto\saggressively\sremove\sold\sfull-text-index\sentries\sbelonging\sto\sdeleted\sor\supdated\srows. -D 2023-04-12T17:40:44.865 +C Remove\stemporary\sdebugging\scode\saccidentally\sleft\son\sthis\sbranch. +D 2023-04-12T18:06:43.220 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 111da0b3226461111bf014a4e0c57dc61bcd7947b90584699ca1090f409fef45 +F ext/fts5/fts5_index.c 4dc0f8b79af771a6a490e4c3476729cba201fb9081c88324da45ece57c7a3372 F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -2055,11 +2055,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c -R 77520373799013e82dff45638b253c47 -T *branch * fts5-secure-delete -T *sym-fts5-secure-delete * -T -sym-trunk * +P 4240fd09b717dbc69dffe3b88ec9149777ca4c3efa12f282af65be3af6fa5bb0 +R 94c13bd9e875c4e10b2fe9e0cf572c45 U dan -Z 0c15d52b2e44a1574709121e2f6a7b6b +Z 057142ccfc1c60d57ad54409384a9ea2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5f639c5e40..63292c41a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4240fd09b717dbc69dffe3b88ec9149777ca4c3efa12f282af65be3af6fa5bb0 \ No newline at end of file +0a0f64870feaf95d2673efc8884ad1236d9bbf9bdb94364f8d2602221bd36ff3 \ No newline at end of file From 7877dca93f56da8c541c8952eba6938ed8984877 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Apr 2023 18:18:28 +0000 Subject: [PATCH 203/341] Avoid leaking a database handle in test script rbuexlock.test. FossilOrigin-Name: 2edf98d128cabc0c9de1295470d7f87eb64e4cc1f13344fe3c198fcf9c7bf019 --- ext/rbu/rbuexlock.test | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ext/rbu/rbuexlock.test b/ext/rbu/rbuexlock.test index aceee45a59..28ab308fc0 100644 --- a/ext/rbu/rbuexlock.test +++ b/ext/rbu/rbuexlock.test @@ -287,5 +287,6 @@ do_test 4.22 { catchsql { SELECT * FROM t1 } cons } {0 {1 one 2 two}} +cons close finish_test diff --git a/manifest b/manifest index bc517744b1..2b37ea0bf8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\s#ifdef\sto\senable\sbuilding\swith\s-DSQLITE_OMIT_WINDOWFUNC. -D 2023-04-11T19:38:47.332 +C Avoid\sleaking\sa\sdatabase\shandle\sin\stest\sscript\srbuexlock.test. +D 2023-04-12T18:18:28.553 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -341,7 +341,7 @@ F ext/rbu/rbucrash.test d2b5d619d9281c89cad74401b73b46172daa89906940b1d739c813dd F ext/rbu/rbucrash2.test 0a1a72223d880215ce2893a3260320c31a9358d23cb124c610e4f0d984a93285 F ext/rbu/rbudiff.test 8b8b8b569c68fc880134e0fac4bf6b4b7a907aea4cc6eacf7e1d45e1d47b6aac F ext/rbu/rbudor.test 293a192e668bb8e9c7c9704b080c1086ee17496f768e0f1823049e7d02651d1b -F ext/rbu/rbuexlock.test 703bb26cb13d655920670974a5d6823b8f7a8b1d4451162a0c1caf44c3d44252 +F ext/rbu/rbuexlock.test e07a0875d0b72f7c007e5d5dcf424e9d48a4752a1a9bcee8ee36947e6add6d5b F ext/rbu/rbuexpr.test 2c91617509c88b6e9030f7bf6ff720df26032fcd801adc25533feae726a57382 F ext/rbu/rbufault.test c51de14067cfe867849530d3d1718ffeb28522f28d52937f95dd7bc2116eb42e F ext/rbu/rbufault2.test 8cc8f6298d2d7d20080b2c77e65b607af8b89839f9d87c0972b27e6442edc258 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8fb143d64d8e823684cd26799080da4b42bef121ca3c6315b1803a593490926 -R 23a3c5b92c8f88b7b4f263831382c52f -U drh -Z 51e0cafc6ddabd2f402c0ebee1449b15 +P e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c +R 63ae784cbfa30c9af9f17fd8269f114d +U dan +Z 80aaf53aef46b41d0dd8a763fb401b0e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 497bd4caa0..d7ece8acd9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c \ No newline at end of file +2edf98d128cabc0c9de1295470d7f87eb64e4cc1f13344fe3c198fcf9c7bf019 \ No newline at end of file From 6b507423a0a67a589a85e007da1b21dfb9e2ba9b Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 12 Apr 2023 18:57:50 +0000 Subject: [PATCH 204/341] Small performance enhancement to integer-to-text conversion. FossilOrigin-Name: cfb3dba9b015ce7a75857978bfd2540b4d5be985d9d2ec4a5842b3089ed3da60 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/util.c | 14 ++++++++------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 2b37ea0bf8..bbeeceb71f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sleaking\sa\sdatabase\shandle\sin\stest\sscript\srbuexlock.test. -D 2023-04-12T18:18:28.553 +C Small\sperformance\senhancement\sto\sinteger-to-text\sconversion. +D 2023-04-12T18:57:50.762 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -693,7 +693,7 @@ F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0 F src/update.c 3f4fb5ad7c9b48d7911974d6579192bb3a6c27f46140b6cbb9139cc8a77b8691 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c 3ff7bc2b48dd425b1448304bb86273b05da1621f136d51dbb9789f8803559a1f +F src/util.c 2106b5176a847d5b91a9345b38a098c202b7691a19070e0fa194cbd3ff139d24 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c -R 63ae784cbfa30c9af9f17fd8269f114d -U dan -Z 80aaf53aef46b41d0dd8a763fb401b0e +P 2edf98d128cabc0c9de1295470d7f87eb64e4cc1f13344fe3c198fcf9c7bf019 +R 32bf4858ec539f44d2811a04e07432be +U drh +Z 133353557a8d275c28d13edc04e62688 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d7ece8acd9..6ad87080d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2edf98d128cabc0c9de1295470d7f87eb64e4cc1f13344fe3c198fcf9c7bf019 \ No newline at end of file +cfb3dba9b015ce7a75857978bfd2540b4d5be985d9d2ec4a5842b3089ed3da60 \ No newline at end of file diff --git a/src/util.c b/src/util.c index 632d317e31..8e3c6c9dbb 100644 --- a/src/util.c +++ b/src/util.c @@ -670,13 +670,15 @@ int sqlite3Int64ToText(i64 v, char *zOut){ } i = sizeof(zTemp)-2; zTemp[sizeof(zTemp)-1] = 0; - do{ - zTemp[i--] = (x%10) + '0'; + while( 1 /*exit-by-break*/ ){ + zTemp[i] = (x%10) + '0'; x = x/10; - }while( x ); - if( v<0 ) zTemp[i--] = '-'; - memcpy(zOut, &zTemp[i+1], sizeof(zTemp)-1-i); - return sizeof(zTemp)-2-i; + if( x==0 ) break; + i--; + }; + if( v<0 ) zTemp[--i] = '-'; + memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); + return sizeof(zTemp)-1-i; } /* From be0023fabab3080d0ba17c3a0e21c969b2dac4aa Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 12 Apr 2023 19:40:00 +0000 Subject: [PATCH 205/341] Automatically set HAVE_PREAD and HAVE_PWRITE on linux, as has been done in MacOS for a long time now. FossilOrigin-Name: 2f7a36d2c374100019bd9d38aea51ea10cfcf9b1f0330ba084a75f0773df8ebf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 8 +------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index bbeeceb71f..a891d30aca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\senhancement\sto\sinteger-to-text\sconversion. -D 2023-04-12T18:57:50.762 +C Automatically\sset\sHAVE_PREAD\sand\sHAVE_PWRITE\son\slinux,\sas\shas\sbeen\sdone\sin\nMacOS\sfor\sa\slong\stime\snow. +D 2023-04-12T19:40:00.840 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -609,7 +609,7 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 -F src/os_unix.c 90c4fa0a88c8b0817c7ce4dbea0ac3aebfd7deb0797945ac34dfd25006ba393a +F src/os_unix.c 1b3ddb7814c4bf37f494c04d2ab30c1ced5b2c927267e1930ce7cd388787a96d F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 39af8ff7c73a991f61f4d1e3a6f8f98b1c8e29144723507822774cac5e6ee0b5 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2edf98d128cabc0c9de1295470d7f87eb64e4cc1f13344fe3c198fcf9c7bf019 -R 32bf4858ec539f44d2811a04e07432be +P cfb3dba9b015ce7a75857978bfd2540b4d5be985d9d2ec4a5842b3089ed3da60 +R a27797e00df587d05c58ecedf6ed69c3 U drh -Z 133353557a8d275c28d13edc04e62688 +Z 0702655698bfbb6d67bb8fe5728ec984 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6ad87080d8..e50fab7945 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cfb3dba9b015ce7a75857978bfd2540b4d5be985d9d2ec4a5842b3089ed3da60 \ No newline at end of file +2f7a36d2c374100019bd9d38aea51ea10cfcf9b1f0330ba084a75f0773df8ebf \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 6e9ee3263b..d0ebb86b95 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -72,7 +72,7 @@ #endif /* Use pread() and pwrite() if they are available */ -#if defined(__APPLE__) +#if defined(__APPLE__) || defined(__linux__) # define HAVE_PREAD 1 # define HAVE_PWRITE 1 #endif @@ -3322,12 +3322,6 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ ** Seek to the offset passed as the second argument, then read cnt ** bytes into pBuf. Return the number of bytes actually read. ** -** NB: If you define USE_PREAD or USE_PREAD64, then it might also -** be necessary to define _XOPEN_SOURCE to be 500. This varies from -** one system to another. Since SQLite does not define USE_PREAD -** in any form by default, we will not attempt to define _XOPEN_SOURCE. -** See tickets #2741 and #2681. -** ** To avoid stomping the errno value on a failed read the lastErrno value ** is set before returning. */ From 3f4a319934d59192880d3a5570b42381b985c61a Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 12 Apr 2023 20:23:03 +0000 Subject: [PATCH 206/341] Unwrap the loop in the WAL hash function. FossilOrigin-Name: eb94ae13206762a42ddad12e0d1461df83e9759e44a369f188cd3aad82f211c4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 33 +++++++++++++++++++++++++++------ 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index a891d30aca..725c664488 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Automatically\sset\sHAVE_PREAD\sand\sHAVE_PWRITE\son\slinux,\sas\shas\sbeen\sdone\sin\nMacOS\sfor\sa\slong\stime\snow. -D 2023-04-12T19:40:00.840 +C Unwrap\sthe\sloop\sin\sthe\sWAL\shash\sfunction. +D 2023-04-12T20:23:03.475 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -707,7 +707,7 @@ F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf8 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac F src/vtab.c 4a1b231b5938de0282fbb605eb068ca673a1b6a383130f54d1bcbbac951988ad F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d +F src/wal.c 3f4ac276a60bda76f9f1f6f1c2c38599bacd4987e5efcd3f7fed2647bf97280a F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b F src/where.c ef9e644d1d76e86f68c941eb05d30a98fc0811eeef2a110906d81fedde81b6bf @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cfb3dba9b015ce7a75857978bfd2540b4d5be985d9d2ec4a5842b3089ed3da60 -R a27797e00df587d05c58ecedf6ed69c3 +P 2f7a36d2c374100019bd9d38aea51ea10cfcf9b1f0330ba084a75f0773df8ebf +R 0a0fe6e09d894bc0529190e100b44f7b U drh -Z 0702655698bfbb6d67bb8fe5728ec984 +Z 3c22931f459153fffc356af79809978d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e50fab7945..6df6e5eb70 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f7a36d2c374100019bd9d38aea51ea10cfcf9b1f0330ba084a75f0773df8ebf \ No newline at end of file +eb94ae13206762a42ddad12e0d1461df83e9759e44a369f188cd3aad82f211c4 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index fdc4ac39b6..43bae13e04 100644 --- a/src/wal.c +++ b/src/wal.c @@ -746,19 +746,40 @@ static void walChecksumBytes( assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); assert( nByte<=65536 ); + assert( nByte%4==0 ); - if( nativeCksum ){ - do { - s1 += *aData++ + s2; - s2 += *aData++ + s1; - }while( aData Date: Thu, 13 Apr 2023 14:14:27 +0000 Subject: [PATCH 207/341] Get CLI utf8_fgets() to not consume more input than it returns. Get console setup restoration to happen for all non-crash exits. FossilOrigin-Name: b4fa233d3dda54fa83771844cf5156bf1275c687925340af17a7713a9400dfef --- manifest | 15 +++++------- manifest.uuid | 2 +- src/shell.c.in | 62 ++++++++++++++++++++++++++++---------------------- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index ef57a3a675..260a215b52 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C WIP:\sCLI\soption\sto\stake\scontrol\sof\sconsole\son\sWindows\sand\smake\sit\ssupport\sUTF-8\sinput\spasting\s(or\styping).\sNeeds\swork\sto\sbecome\srobust\sper\s"ToDo:". -D 2023-04-12T17:54:52.445 +C Get\sCLI\sutf8_fgets()\sto\snot\sconsume\smore\sinput\sthan\sit\sreturns.\sGet\sconsole\ssetup\srestoration\sto\shappen\sfor\sall\snon-crash\sexits. +D 2023-04-13T14:14:27.434 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94 -F src/shell.c.in 156279044afe3acdd48997531617c6bd1b5a37b5761b67900cc4dbfcc8af7130 +F src/shell.c.in d3a5bef33e4ad55f3feeaeb0deaed32dcb80252a5b2ca184d1eefa816a9f92cf F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,11 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1ff83fa2565334b28bd0d6582088c4ae0d2d9a590d973615a4a598683fe419c -R 42a09bb2160cadfa8aa798fa30b96ed8 -T *branch * cli-utf8 -T *sym-cli-utf8 * -T -sym-trunk * +P 824382393d92d9eb6df8701de7c263280150569a708759c4a539acc6d8d38821 +R 692e15a266cff0987f54e086d515e8e5 U larrybr -Z e14b183810e19b5d66903057d8becd3e +Z d0dd4db17efed3a7d98e05299e7fc21d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 230ab719ef..065235828d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -824382393d92d9eb6df8701de7c263280150569a708759c4a539acc6d8d38821 \ No newline at end of file +b4fa233d3dda54fa83771844cf5156bf1275c687925340af17a7713a9400dfef \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c7c0d537ec..12748b29a3 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11,6 +11,7 @@ ************************************************************************* ** This file contains code to implement the "sqlite" command line ** utility for accessing SQLite databases. +ToDo: The -utf8 option may not play well with line editing input libraries. */ #if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS) /* This needs to come before any includes for MSVC compiler */ @@ -20,7 +21,7 @@ typedef unsigned int u32; typedef unsigned short int u16; /* -** Optionally #include a user-defined header, whereby compilation options +** optionally #include a user-defined header, whereby compilation options ** may be set prior to where they take effect, but after platform setup. ** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include ** file. Note that this macro has a like effect on sqlite3.c compilation. @@ -589,16 +590,17 @@ static char *dynamicContinuePrompt(void){ #if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) static int infsMode; -static UINT codePage; +static UINT codePage = 0; static HANDLE hConsoleIn; static DWORD consoleMode; + /* ** Prepare input stream, (if known to be a WIN32 console), for UTF-8 ** input (from either typing or suitable paste operations) and for ** UTF-8 rendering. This may "fail" with a message to stderr, where ** the preparation is not done and common "code page" issues occur. */ -static void instream_prepare(){ +static void instream_prepare(void){ if( isatty(0) ){ if( !IsValidCodePage(CP_UTF8) ){ fprintf(stderr, "Cannot use UTF-8 code page.\n"); @@ -620,11 +622,12 @@ static void instream_prepare(){ /* ** Undo the effects of instream_prepare(), if any. */ -static void instream_restore(){ - if( console_utf8 ){ +static void SQLITE_CDECL instream_restore(void){ + if( console_utf8 && codePage!=0 ){ _setmode(_fileno(stdin), infsMode); SetConsoleCP(codePage); SetConsoleMode( hConsoleIn, consoleMode ); + console_utf8 = 0; /* Avoid multiple calls. */ } } @@ -633,25 +636,30 @@ static void instream_restore(){ ** from the Windows console to get around its strange coding issues. ** Defers to plain fgets() when input is not interactive or when the ** startup option, -utf8, has not been provided or taken effect. -** WIP: This routine needs work to consume only as many characters -** as can be fit into the provided input buffer. ToDo: That work. -** Avoiding this issue will require some refactoring of CLI input. */ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ + #define SQLITE_IA_LINE_LEN 150 if( fin==stdin && stdin_is_interactive && console_utf8 ){ - wchar_t * wbuf = (wchar_t*) malloc(ncmax * sizeof(wchar_t)); - wchar_t * wz; - int nmb; - if( wbuf==0 ) return 0; - if( 0 == (wz = fgetws(wbuf, ncmax, stdin)) - || 0 == (nmb = WideCharToMultiByte(CP_UTF8, 0, wz, -1, 0, 0, 0, 0)) - || nmb > ncmax ){ - buf = 0; - goto bail; + wchar_t wbuf[SQLITE_IA_LINE_LEN]; + int noc = 0; + if( ncmax == 0 ) return 0; + buf[0] = 0; + while( noc < ncmax-7-1 ){ + /* There is room for at least 2 more characters and the 0-terminator. */ + int na = (ncmax-1 - noc)/4; + wchar_t * wz; + if( na > SQLITE_IA_LINE_LEN ) na = SQLITE_IA_LINE_LEN; + wz = fgetws(wbuf, na, stdin); + if( wz != 0 ){ + int nmb = WideCharToMultiByte(CP_UTF8, 0, wz, -1, 0, 0, 0, 0); + if( nmb !=0 && noc+nmb <= ncmax ){ + WideCharToMultiByte(CP_UTF8, 0, wz, -1, buf+noc, nmb, 0, 0); + noc += nmb-1; /* Strip 0-terminator added by above call. */ + if( buf[noc-1] == '\n' ) break; + }else break; /* Drop apparent garbage in. (Could assert.) */ + }else break; } - WideCharToMultiByte(CP_UTF8, 0, wz, -1, buf, nmb, 0, 0); - bail: - free(wbuf); + if( noc == 0 ) return 0; return buf; }else{ return fgets(buf, ncmax, fin); @@ -11789,7 +11797,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char **argv; #endif #ifdef SQLITE_DEBUG - sqlite3_int64 mem_main_enter = sqlite3_memory_used(); + sqlite3_int64 mem_main_enter = 0; #endif char *zErrMsg = 0; #ifdef SQLITE_SHELL_FIDDLE @@ -11821,7 +11829,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = isatty(0); stdout_is_console = isatty(1); #endif - +#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) + atexit(instream_restore); /* Needs revision for CLI as library call */ +#endif +#ifdef SQLITE_DEBUG + mem_main_enter = sqlite3_memory_used(); +#endif #if !defined(_WIN32_WCE) if( getenv("SQLITE_DEBUG_BREAK") ){ if( isatty(0) && isatty(2) ){ @@ -12370,11 +12383,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ rc = process_input(&data); } } -#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) - if( console_utf8 ){ - instream_restore(); - } -#endif #ifndef SQLITE_SHELL_FIDDLE /* In WASM mode we have to leave the db state in place so that ** client code can "push" SQL into it after this call returns. */ From 017bdf7838183f6978796c9ae852895ee26e59ff Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 13 Apr 2023 14:50:50 +0000 Subject: [PATCH 208/341] Fix a code-generator issue associated with very unusual use of window functions. Both the expr.c or the window.c changes will each independently fix the problem. They are both included in this patch for defense in depth. [forum:/forumpost/0d48347967|Forum post 0d48347967]. FossilOrigin-Name: 1ba22631a7831e3562eda0eb6a5edf7f009c85c7ab4451d9eacd13ef0fb6036a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 2 +- src/window.c | 1 + test/window1.test | 17 +++++++++++++++++ 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 725c664488..a51af72cb2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Unwrap\sthe\sloop\sin\sthe\sWAL\shash\sfunction. -D 2023-04-12T20:23:03.475 +C Fix\sa\scode-generator\sissue\sassociated\swith\svery\sunusual\suse\sof\swindow\nfunctions.\s\sBoth\sthe\sexpr.c\sor\sthe\swindow.c\schanges\swill\seach\nindependently\sfix\sthe\sproblem.\s\sThey\sare\sboth\sincluded\sin\sthis\spatch\sfor\ndefense\sin\sdepth.\s\s[forum:/forumpost/0d48347967|Forum\spost\s0d48347967]. +D 2023-04-13T14:50:50.896 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 86f843dba99f1d60836b0b9fa7f595ff2723d902416a4c7749217e950160f17c +F src/expr.c 6353f4d92d9f67ec3466d8e6978cd31a45e34cb755c4d11e689077f03f7c0a15 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -714,7 +714,7 @@ F src/where.c ef9e644d1d76e86f68c941eb05d30a98fc0811eeef2a110906d81fedde81b6bf F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 85790d7e5365ac41085713331ce52e4343586ad3d37d218ffe00572357baa62b F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 -F src/window.c 76a27cff9ea2ded0c2c3527187029259440fabcc4cc4c07b11d942c78494a614 +F src/window.c e075ea85bea322e30e361fa6e69eddba74f461e99e2a564dc09973f8a1fb27d9 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627 F test/affinity3.test f094773025eddf31135c7ad4cde722b7696f8eb07b97511f98585addf2a510a9 @@ -1902,7 +1902,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test c607e30741c034a1e88c104f97f107f07489f538a65c0f56ccce97e7a956065a +F test/window1.test 57a2107dd1a869d857be560927f9424486da98e849256700ff7e90adbe164f38 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2f7a36d2c374100019bd9d38aea51ea10cfcf9b1f0330ba084a75f0773df8ebf -R 0a0fe6e09d894bc0529190e100b44f7b +P eb94ae13206762a42ddad12e0d1461df83e9759e44a369f188cd3aad82f211c4 +R 790521fc0147ba246247d1806b2d1934 U drh -Z 3c22931f459153fffc356af79809978d +Z 4c16acea7a283d44dd652b716e84bf87 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6df6e5eb70..22a4facc92 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb94ae13206762a42ddad12e0d1461df83e9759e44a369f188cd3aad82f211c4 \ No newline at end of file +1ba22631a7831e3562eda0eb6a5edf7f009c85c7ab4451d9eacd13ef0fb6036a \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 00c67a4e68..ebaf13af17 100644 --- a/src/expr.c +++ b/src/expr.c @@ -6493,7 +6493,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ } - return WRC_Prune; + return WRC_Continue; } case TK_AGG_FUNCTION: { if( (pNC->ncFlags & NC_InAggFunc)==0 diff --git a/src/window.c b/src/window.c index 56de38ba39..a8081aa244 100644 --- a/src/window.c +++ b/src/window.c @@ -785,6 +785,7 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ } /* no break */ deliberate_fall_through + case TK_IF_NULL_ROW: case TK_AGG_FUNCTION: case TK_COLUMN: { int iCol = -1; diff --git a/test/window1.test b/test/window1.test index 5678dbb89b..05ceab073c 100644 --- a/test/window1.test +++ b/test/window1.test @@ -2330,4 +2330,21 @@ do_catchsql_test 75.1 { GROUP BY a1.a; } {1 {misuse of aggregate: count()}} +# 2023-04-13 https://sqlite.org/forum/forumpost/0d48347967 +reset_db +do_execsql_test 76.0 { + CREATE TABLE t1(a INT, b INT); + INSERT INTO t1(a,b) VALUES (111,222),(111,223),(118,229); + CREATE INDEX t1a ON t1(a); + CREATE TABLE t2(x INT); + INSERT INTO t2 VALUES (333),(444),(555); +} +do_execsql_test 76.1 { + SELECT c, (SELECT c + sum(1) OVER ()) AS "res" + FROM t2 LEFT JOIN (SELECT +a AS c FROM t1) AS v1 ON true + GROUP BY c + ORDER by c; +} {111 112 118 119} +# ^^^^^^^^^^^^^^^^^-- results verified against PG 14.2 + finish_test From 8efa288f99675acd237f02bfdaca63a9542eb426 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 13 Apr 2023 14:53:19 +0000 Subject: [PATCH 209/341] Add extra test cases to window1.test. FossilOrigin-Name: ebc844fbfb046c5789efe21fd607c9301cb7b3d78edef72b2926b8c889b048a9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/window1.test | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a51af72cb2..7f12d79dd6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scode-generator\sissue\sassociated\swith\svery\sunusual\suse\sof\swindow\nfunctions.\s\sBoth\sthe\sexpr.c\sor\sthe\swindow.c\schanges\swill\seach\nindependently\sfix\sthe\sproblem.\s\sThey\sare\sboth\sincluded\sin\sthis\spatch\sfor\ndefense\sin\sdepth.\s\s[forum:/forumpost/0d48347967|Forum\spost\s0d48347967]. -D 2023-04-13T14:50:50.896 +C Add\sextra\stest\scases\sto\swindow1.test. +D 2023-04-13T14:53:19.960 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1902,7 +1902,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test e0924eb8daac02bf80e9da88930747bd44dd9b230b7759fed927b1655b467c9c F test/win32longpath.test 4baffc3acb2e5188a5e3a895b2b543ed09e62f7c72d713c1feebf76222fe9976 F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 57a2107dd1a869d857be560927f9424486da98e849256700ff7e90adbe164f38 +F test/window1.test 5ba48e9d33231e6ef16f21426bade9ccc52abf65a10587bff90a6c14fe174594 F test/window2.tcl 492c125fa550cda1dd3555768a2303b3effbeceee215293adf8871efc25f1476 F test/window2.test e466a88bd626d66edc3d352d7d7e1d5531e0079b549ba44efb029d1fbff9fd3c F test/window3.tcl acea6e86a4324a210fd608d06741010ca83ded9fde438341cb978c49928faf03 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P eb94ae13206762a42ddad12e0d1461df83e9759e44a369f188cd3aad82f211c4 -R 790521fc0147ba246247d1806b2d1934 -U drh -Z 4c16acea7a283d44dd652b716e84bf87 +P 1ba22631a7831e3562eda0eb6a5edf7f009c85c7ab4451d9eacd13ef0fb6036a +R 5689abcdd9a92d7ec887a4030b4e7bcf +U dan +Z 478a4462c289eca49f2307135f0a942e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 22a4facc92..a6372b0e36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ba22631a7831e3562eda0eb6a5edf7f009c85c7ab4451d9eacd13ef0fb6036a \ No newline at end of file +ebc844fbfb046c5789efe21fd607c9301cb7b3d78edef72b2926b8c889b048a9 \ No newline at end of file diff --git a/test/window1.test b/test/window1.test index 05ceab073c..783a739e3f 100644 --- a/test/window1.test +++ b/test/window1.test @@ -2347,4 +2347,20 @@ do_execsql_test 76.1 { } {111 112 118 119} # ^^^^^^^^^^^^^^^^^-- results verified against PG 14.2 +do_execsql_test 76.2 { + CREATE TABLE t3(x); + CREATE TABLE t4(y); + INSERT INTO t3 VALUES(100), (200), (400); + INSERT INTO t4 VALUES(100), (300), (400); +} +do_execsql_test 76.3 { + SELECT (SELECT y+sum(0) OVER ()) FROM t3 LEFT JOIN t4 ON x=y; +} {100 {} 400} +do_execsql_test 76.4 { + SELECT (SELECT y+sum(0) OVER ()) FROM t3 LEFT JOIN t4 ON x=y GROUP BY x; +} {100 {} 400} +do_execsql_test 76.5 { + SELECT (SELECT max(y)+sum(0) OVER ()) FROM t3 LEFT JOIN t4 ON x=y GROUP BY x; +} {100 {} 400} + finish_test From 731a1aaeb2de2d7b69b2b4f0d08d1f5f33ae9a26 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 13 Apr 2023 18:44:59 +0000 Subject: [PATCH 210/341] Fix an obscure issue with ALTER TABLE RENAME that comes up with triggers that have UPDATE statements that contain errors. [forum:/forumpost/ff3840145a|Forum post ff3840145a]. FossilOrigin-Name: c4845a7c5f7f219848d3ee32eef0f9c69ad6dc6e8509da84d612f41e1e05f007 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/alter.c | 23 +++++++++++++++++++++++ src/select.c | 2 +- test/altertab.test | 18 ++++++++++++++++++ 5 files changed, 52 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7f12d79dd6..adaa979df9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stest\scases\sto\swindow1.test. -D 2023-04-13T14:53:19.960 +C Fix\san\sobscure\sissue\swith\sALTER\sTABLE\sRENAME\sthat\scomes\sup\swith\striggers\nthat\shave\sUPDATE\sstatements\sthat\scontain\serrors.\n[forum:/forumpost/ff3840145a|Forum\spost\sff3840145a]. +D 2023-04-13T18:44:59.982 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -557,7 +557,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 -F src/alter.c 3ca2f449c890f8b86ec9e06f0c4fccf0648941c3308a16904cb2852227db83f7 +F src/alter.c 482c534877fbb543f8295992cde925df55443febac5db5438d5aaba6f78c4940 F src/analyze.c 01bfd40026632eaae1d93212b684f539c6674cb573535dc90199674cbf7e0cdc F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf @@ -625,7 +625,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94 +F src/select.c d0c7187b0f557337cd187e5c37ae43012a5d902f7e35d902423cb3cb81f21c76 F src/shell.c.in 80f7c4c1bdabc9dab416b59287458044c01df97035d626ea57ffeabbb289bdc6 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -739,7 +739,7 @@ F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74 F test/altermalloc2.test 17fb3724c4b004c469c27dc4ef181608aa644555fbd3f3236767584f73747c81 F test/altermalloc3.test 8531b3086f0a7889f43971a579a8c81832d5123f4703d8c86b89ba1136c63b9a F test/alterqf.test ff6c6f881485c29ed699b8ef4774864ca1b0c01a6c08f5cdd624a008e4b40fca -F test/altertab.test 7273b8506eab46342be016af78028df49f3bd99037412f997a8f1011b37a6912 +F test/altertab.test 8a2712f9076da5012a002d0b5cc0a421398a5bf61c25bab41b77c427586a7a27 F test/altertab2.test 62597b6fd08feaba1b6bfe7d31dac6117c67e06dc9ce9c478a3abe75b5926de0 F test/altertab3.test 6c432fbb9963e0bd6549bf1422f6861d744ee5a80cb3298564e81e556481df16 F test/altertrig.test fb5951d21a2c954be3b8a8cf8e10b5c0fa20687c53fd67d63cea88d08dd058d5 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1ba22631a7831e3562eda0eb6a5edf7f009c85c7ab4451d9eacd13ef0fb6036a -R 5689abcdd9a92d7ec887a4030b4e7bcf -U dan -Z 478a4462c289eca49f2307135f0a942e +P ebc844fbfb046c5789efe21fd607c9301cb7b3d78edef72b2926b8c889b048a9 +R 11b569adbba936283758346fb63ba407 +U drh +Z ffa219ef522bb31dbb98533896877183 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a6372b0e36..4ed5f2dbb6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebc844fbfb046c5789efe21fd607c9301cb7b3d78edef72b2926b8c889b048a9 \ No newline at end of file +c4845a7c5f7f219848d3ee32eef0f9c69ad6dc6e8509da84d612f41e1e05f007 \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index cebeec3855..121b617117 100644 --- a/src/alter.c +++ b/src/alter.c @@ -1279,6 +1279,19 @@ static int renameEditSql( return rc; } +/* +** Set all pEList->a[].fg.eEName fields in the expression-list to val. +*/ +static void renameSetENames(ExprList *pEList, int val){ + if( pEList ){ + int i; + for(i=0; inExpr; i++){ + assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME ); + pEList->a[i].fg.eEName = val; + } + } +} + /* ** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming ** it was read from the schema of database zDb. Return SQLITE_OK if @@ -1326,7 +1339,17 @@ static int renameResolveTrigger(Parse *pParse){ pSrc = 0; rc = SQLITE_NOMEM; }else{ + /* pStep->pExprList contains an expression-list used for an UPDATE + ** statement. So the a[].zEName values are the RHS of the + ** " = " clauses of the UPDATE statement. So, before + ** running SelectPrep(), change all the eEName values in + ** pStep->pExprList to ENAME_SPAN (from their current value of + ** ENAME_NAME). This is to prevent any ids in ON() clauses that are + ** part of pSrc from being incorrectly resolved against the + ** a[].zEName values as if they were column aliases. */ + renameSetENames(pStep->pExprList, ENAME_SPAN); sqlite3SelectPrep(pParse, pSel, 0); + renameSetENames(pStep->pExprList, ENAME_NAME); rc = pParse->nErr ? SQLITE_ERROR : SQLITE_OK; assert( pStep->pExprList==0 || pStep->pExprList==pSel->pEList ); assert( pSrc==pSel->pSrc ); diff --git a/src/select.c b/src/select.c index fcf8ae78d9..9edc04f460 100644 --- a/src/select.c +++ b/src/select.c @@ -2318,7 +2318,7 @@ void sqlite3SubqueryColumnTypes( assert( (pSelect->selFlags & SF_Resolved)!=0 ); assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); - if( db->mallocFailed ) return; + if( db->mallocFailed || IN_RENAME_OBJECT ) return; while( pSelect->pPrior ) pSelect = pSelect->pPrior; a = pSelect->pEList->a; memset(&sNC, 0, sizeof(sNC)); diff --git a/test/altertab.test b/test/altertab.test index 6d8347ec1e..9cc43e14de 100644 --- a/test/altertab.test +++ b/test/altertab.test @@ -981,4 +981,22 @@ do_catchsql_test 32.0 { ALTER TABLE t1 RENAME TO x; } {1 {error in trigger r1: no tables specified}} +# 2023-04-13 https://sqlite.org/forum/forumpost/ff3840145a +# +reset_db +do_execsql_test 33.0 { + CREATE TABLE t1(a TEXT); + INSERT INTO t1(a) VALUES('abc'),('def'),(NULL); + CREATE TABLE t2(b TEXT); + CREATE TRIGGER r3 AFTER INSERT ON t1 BEGIN + UPDATE t2 SET (b,a)=(SELECT 1) FROM t1 JOIN t2 ON (SELECT * FROM (SELECT a)); + END; +} +do_catchsql_test 33.1 { + ALTER TABLE t1 RENAME COLUMN a TO b; +} {1 {error in trigger r3 after rename: no such column: a}} +do_execsql_test 33.2 { + SELECT quote(a) FROM t1 ORDER BY +a; +} {NULL 'abc' 'def'} + finish_test From 2142b7c0a6ce3e01424b04155a168ae5cca9d21f Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 14 Apr 2023 00:20:16 +0000 Subject: [PATCH 211/341] Avoid double de-quoting of table names when processing RESTRICT actions in foreign key constraints. [https://bugs.chromium.org/p/chromium/issues/detail?id=1405220|Chromium 1405220]. FossilOrigin-Name: bb2b5ab172f0751c00343facf36fb12db10c88220caece31849f2711f12293d9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/fkey.c | 16 ++++++++-------- test/fkey1.test | 11 +++++++++++ 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index adaa979df9..bcff963d99 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sobscure\sissue\swith\sALTER\sTABLE\sRENAME\sthat\scomes\sup\swith\striggers\nthat\shave\sUPDATE\sstatements\sthat\scontain\serrors.\n[forum:/forumpost/ff3840145a|Forum\spost\sff3840145a]. -D 2023-04-13T18:44:59.982 +C Avoid\sdouble\sde-quoting\sof\stable\snames\swhen\sprocessing\sRESTRICT\sactions\nin\sforeign\skey\sconstraints.\n[https://bugs.chromium.org/p/chromium/issues/detail?id=1405220|Chromium\s1405220]. +D 2023-04-14T00:20:16.995 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -577,7 +577,7 @@ F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 F src/expr.c 6353f4d92d9f67ec3466d8e6978cd31a45e34cb755c4d11e689077f03f7c0a15 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 -F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 +F src/fkey.c 03c134cc8bffe54835f742ddea0b72ebfc8f6b32773d175c71b8afeea6cb5c83 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 F src/global.c 428d2580a1cdf5dbe1f356d1feab83710ae0cc862ece0fb57bc8259e43838c74 F src/hash.c c6af5f96a7a76d000f07c5402c48c318c2566beecdee9e78b9d9f60ce7119565 @@ -991,7 +991,7 @@ F test/filter1.test 590f8ba9a0cd0823b80d89ac75c5ce72276189cef9225d2436adaf1ee87f F test/filter2.tcl 44e525497ce07382915f01bd29ffd0fa49dab3adb87253b5e5103ba8f93393e8 F test/filter2.test 485cf95d1f6d6ceee5632201ca52a71868599836f430cdee42e5f7f14666e30a F test/filterfault.test c08fb491d698e8df6c122c98f7db1c65ffcfcad2c1ab0e07fa8a5be1b34eaa8b -F test/fkey1.test 55663090ab6735319a52647057b9f19f8ec8c6c7d7da25170b71a75e3e5bdeb7 +F test/fkey1.test e563bcb4cb108ce3f40363cda4f84009dc89a39e2973076e5057ba99fca35378 F test/fkey2.test 1063d65e5923c054cfb8f0555a92a3ae0fa8c067275a33ee1715bd856cdb304c F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ebc844fbfb046c5789efe21fd607c9301cb7b3d78edef72b2926b8c889b048a9 -R 11b569adbba936283758346fb63ba407 +P c4845a7c5f7f219848d3ee32eef0f9c69ad6dc6e8509da84d612f41e1e05f007 +R e187e9998555a73ed3722571589e480f U drh -Z ffa219ef522bb31dbb98533896877183 +Z 8502a1b30a3157b76d54d4493f6eaab0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4ed5f2dbb6..79e2afc1be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c4845a7c5f7f219848d3ee32eef0f9c69ad6dc6e8509da84d612f41e1e05f007 \ No newline at end of file +bb2b5ab172f0751c00343facf36fb12db10c88220caece31849f2711f12293d9 \ No newline at end of file diff --git a/src/fkey.c b/src/fkey.c index cae6bb19d3..29609916b6 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -1317,22 +1317,22 @@ static Trigger *fkActionTrigger( if( action==OE_Restrict ){ int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - Token tFrom; - Token tDb; + SrcList *pSrc; Expr *pRaise; - tFrom.z = zFrom; - tFrom.n = nFrom; - tDb.z = db->aDb[iDb].zDbSName; - tDb.n = sqlite3Strlen30(tDb.z); - pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); if( pRaise ){ pRaise->affExpr = OE_Abort; } + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pSrc ){ + assert( pSrc->nSrc==1 ); + pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); + pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), - sqlite3SrcListAppend(pParse, 0, &tDb, &tFrom), + pSrc, pWhere, 0, 0, 0, 0, 0 ); diff --git a/test/fkey1.test b/test/fkey1.test index db93be501d..46e7f64a19 100644 --- a/test/fkey1.test +++ b/test/fkey1.test @@ -272,4 +272,15 @@ do_catchsql_test 8.3 { REINDEX; } {1 {database disk image is malformed}} +# 2023-04-13 https://bugs.chromium.org/p/chromium/issues/detail?id=1405220 +# Avoid double-de-quoting of table names when processing foreign keys. +# +reset_db +do_execsql_test 9.1 { + PRAGMA foreign_keys = ON; + CREATE TABLE """1"("""2", """3" PRIMARY KEY); + CREATE TABLE """4"("""5" REFERENCES """1" ON DELETE RESTRICT); + DELETE FROM """1"; +} + finish_test From a73f51447b0259d226c1fe9de5a5ed34d05a5e0e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 14 Apr 2023 10:35:29 +0000 Subject: [PATCH 212/341] Fix a cosmetic indentation issue. FossilOrigin-Name: 90deb84486494b903bab065fbf0174b56483c190353992b907359661393f2c54 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index bcff963d99..00882f03cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sdouble\sde-quoting\sof\stable\snames\swhen\sprocessing\sRESTRICT\sactions\nin\sforeign\skey\sconstraints.\n[https://bugs.chromium.org/p/chromium/issues/detail?id=1405220|Chromium\s1405220]. -D 2023-04-14T00:20:16.995 +C Fix\sa\scosmetic\sindentation\sissue. +D 2023-04-14T10:35:29.876 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -625,7 +625,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c d0c7187b0f557337cd187e5c37ae43012a5d902f7e35d902423cb3cb81f21c76 +F src/select.c f803ac95411fec545e96a2667734e300428aaeb75a27d96b5b3ac62a7461b25a F src/shell.c.in 80f7c4c1bdabc9dab416b59287458044c01df97035d626ea57ffeabbb289bdc6 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c4845a7c5f7f219848d3ee32eef0f9c69ad6dc6e8509da84d612f41e1e05f007 -R e187e9998555a73ed3722571589e480f +P bb2b5ab172f0751c00343facf36fb12db10c88220caece31849f2711f12293d9 +R 98fb747f8b76030eab4ec959dcc2d1ae U drh -Z 8502a1b30a3157b76d54d4493f6eaab0 +Z 4db1767c2c2b21dfcda57aec5fef47c6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 79e2afc1be..528d4b3db8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb2b5ab172f0751c00343facf36fb12db10c88220caece31849f2711f12293d9 \ No newline at end of file +90deb84486494b903bab065fbf0174b56483c190353992b907359661393f2c54 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 9edc04f460..b0c2b90e91 100644 --- a/src/select.c +++ b/src/select.c @@ -2363,17 +2363,17 @@ void sqlite3SubqueryColumnTypes( break; } } - } - } - if( zType ){ - i64 m = sqlite3Strlen30(zType); - n = sqlite3Strlen30(pCol->zCnName); - pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); - if( pCol->zCnName ){ - memcpy(&pCol->zCnName[n+1], zType, m+1); - pCol->colFlags |= COLFLAG_HASTYPE; - }else{ - testcase( pCol->colFlags & COLFLAG_HASTYPE ); + } + } + if( zType ){ + i64 m = sqlite3Strlen30(zType); + n = sqlite3Strlen30(pCol->zCnName); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + if( pCol->zCnName ){ + memcpy(&pCol->zCnName[n+1], zType, m+1); + pCol->colFlags |= COLFLAG_HASTYPE; + }else{ + testcase( pCol->colFlags & COLFLAG_HASTYPE ); pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); } } From deabd6969b3e5127cfa73e1ebc4d601518a4f736 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 14 Apr 2023 10:40:50 +0000 Subject: [PATCH 213/341] When setting the column types on a subquery, ensure that the COLFLAG_HASCOLL flag is cleared from column names from when the collating sequence name has been removed. [forum:/forumpost/6916dacf83|Forum post 6916dacf83]. FossilOrigin-Name: 8d9dcd7cfdd53034e21cdb2cb997b6f5fe3207ca5d0d7a0b841b8a68e6ce8624 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 00882f03cf..ada9f0b5f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scosmetic\sindentation\sissue. -D 2023-04-14T10:35:29.876 +C When\ssetting\sthe\scolumn\stypes\son\sa\ssubquery,\sensure\sthat\sthe\sCOLFLAG_HASCOLL\nflag\sis\scleared\sfrom\scolumn\snames\sfrom\swhen\sthe\scollating\ssequence\sname\shas\nbeen\sremoved.\n[forum:/forumpost/6916dacf83|Forum\spost\s6916dacf83]. +D 2023-04-14T10:40:50.957 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -625,7 +625,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c f803ac95411fec545e96a2667734e300428aaeb75a27d96b5b3ac62a7461b25a +F src/select.c ad6f48e0953fb0d9dd434a7fcd8bc6bde05beb82500a59fea7a7b02dc0ecfb3b F src/shell.c.in 80f7c4c1bdabc9dab416b59287458044c01df97035d626ea57ffeabbb289bdc6 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb2b5ab172f0751c00343facf36fb12db10c88220caece31849f2711f12293d9 -R 98fb747f8b76030eab4ec959dcc2d1ae +P 90deb84486494b903bab065fbf0174b56483c190353992b907359661393f2c54 +R 74f079a7a3e0405a510d653a98c3a5fe U drh -Z 4db1767c2c2b21dfcda57aec5fef47c6 +Z 0bf573c21b8ca5b8de34fe3cc5060e37 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 528d4b3db8..fab28ff1d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90deb84486494b903bab065fbf0174b56483c190353992b907359661393f2c54 \ No newline at end of file +8d9dcd7cfdd53034e21cdb2cb997b6f5fe3207ca5d0d7a0b841b8a68e6ce8624 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b0c2b90e91..80a51d61d1 100644 --- a/src/select.c +++ b/src/select.c @@ -2369,12 +2369,12 @@ void sqlite3SubqueryColumnTypes( i64 m = sqlite3Strlen30(zType); n = sqlite3Strlen30(pCol->zCnName); pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); if( pCol->zCnName ){ memcpy(&pCol->zCnName[n+1], zType, m+1); pCol->colFlags |= COLFLAG_HASTYPE; }else{ testcase( pCol->colFlags & COLFLAG_HASTYPE ); - pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); } } pColl = sqlite3ExprCollSeq(pParse, p); From 95dc3e08bd16b1e2e55d877d16d91d47376f2c37 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Apr 2023 13:34:26 +0000 Subject: [PATCH 214/341] Update test script recoverbuild.test so that it may be run using old test harnesses that run more than one test script in a process. FossilOrigin-Name: 5135aab0c0ab39f3b9cc84efbbc57f7266b42d98134fc0c5dace06f707374195 --- ext/recover/recoverbuild.test | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/recover/recoverbuild.test b/ext/recover/recoverbuild.test index 9f33731d61..d119be7fde 100644 --- a/ext/recover/recoverbuild.test +++ b/ext/recover/recoverbuild.test @@ -28,7 +28,7 @@ do_execsql_test 1.0 { forcedelete test.db2 do_test 1.1 { set R [sqlite3_recover_init db main test.db2] -} {sqlite_recover1} +} {/sqlite_recover.*/} do_test 1.2 { $R run diff --git a/manifest b/manifest index ada9f0b5f4..66bb24ec21 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\ssetting\sthe\scolumn\stypes\son\sa\ssubquery,\sensure\sthat\sthe\sCOLFLAG_HASCOLL\nflag\sis\scleared\sfrom\scolumn\snames\sfrom\swhen\sthe\scollating\ssequence\sname\shas\nbeen\sremoved.\n[forum:/forumpost/6916dacf83|Forum\spost\s6916dacf83]. -D 2023-04-14T10:40:50.957 +C Update\stest\sscript\srecoverbuild.test\sso\sthat\sit\smay\sbe\srun\susing\sold\stest\sharnesses\sthat\srun\smore\sthan\sone\stest\sscript\sin\sa\sprocess. +D 2023-04-14T13:34:26.137 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -368,7 +368,7 @@ F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b F ext/recover/dbdata.c 31d580785cf14eb3c20ed6fbb421a10a66569858f837928e6b326088c38d4c72 F ext/recover/recover1.test 2072993624d5e32fef20ae03b17fc06c02bcb344421fe17bb329b24d2a51e647 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a -F ext/recover/recoverbuild.test a6f05273ff5fe517afd166444597c70cb97033e7f58496433a4428a1ecb5d79f +F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3 F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e F ext/recover/recovercorrupt.test 64c081ad1200ae77b447da99eb724785d6bf71715f394543dc7689642e92bf49 F ext/recover/recovercorrupt2.test 74bef7dd2d7dd4856f3da21be6e213d27da44827e0f5f0946ca0325b46d163ed @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 90deb84486494b903bab065fbf0174b56483c190353992b907359661393f2c54 -R 74f079a7a3e0405a510d653a98c3a5fe -U drh -Z 0bf573c21b8ca5b8de34fe3cc5060e37 +P 8d9dcd7cfdd53034e21cdb2cb997b6f5fe3207ca5d0d7a0b841b8a68e6ce8624 +R fc31d3b3b970efbe03486f41cc3d2232 +U dan +Z 5724f6d7abc60141a98ec759553138ed # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fab28ff1d3..bcb9d41641 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d9dcd7cfdd53034e21cdb2cb997b6f5fe3207ca5d0d7a0b841b8a68e6ce8624 \ No newline at end of file +5135aab0c0ab39f3b9cc84efbbc57f7266b42d98134fc0c5dace06f707374195 \ No newline at end of file From 41334cf960a472a31729e2914b7c3c7f9014a555 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Apr 2023 14:36:34 +0000 Subject: [PATCH 215/341] Fix SQLITE_OMIT_VIRTUALTABLE builds of testfixture. FossilOrigin-Name: 430ec1a61507d8afd8adc161f577f32856dba21506db9717fae3d14570786439 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test1.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 66bb24ec21..9fd72cd345 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stest\sscript\srecoverbuild.test\sso\sthat\sit\smay\sbe\srun\susing\sold\stest\sharnesses\sthat\srun\smore\sthan\sone\stest\sscript\sin\sa\sprocess. -D 2023-04-14T13:34:26.137 +C Fix\sSQLITE_OMIT_VIRTUALTABLE\sbuilds\sof\stestfixture. +D 2023-04-14T14:36:34.208 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -635,7 +635,7 @@ F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a3 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 F src/tclsqlite.c 8522a04fb9c84faa1d80354430ae0ee9349727a3a4b32e3cfe39b9be8324cabd -F src/test1.c 74d87bbc29954c239fdcd7c0c3dd472c0eb9761c7289b85389f322233932780a +F src/test1.c 8eab61fb2813aa212d97ab188e85fc9ca7b89d9ff5ff05d59d9aa0c491a6c721 F src/test2.c 827446e259a3b7ab949da1542953edda7b5117982576d3e6f1c24a0dd20a5cef F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644 F src/test4.c 4533b76419e7feb41b40582554663ed3cd77aaa54e135cf76b3205098cd6e664 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d9dcd7cfdd53034e21cdb2cb997b6f5fe3207ca5d0d7a0b841b8a68e6ce8624 -R fc31d3b3b970efbe03486f41cc3d2232 +P 5135aab0c0ab39f3b9cc84efbbc57f7266b42d98134fc0c5dace06f707374195 +R aa9b2e71cffb30923db6734d3f4be5fa U dan -Z 5724f6d7abc60141a98ec759553138ed +Z 7df8dce1a6232ffba1023b1f981febc5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index bcb9d41641..58dddb44dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5135aab0c0ab39f3b9cc84efbbc57f7266b42d98134fc0c5dace06f707374195 \ No newline at end of file +430ec1a61507d8afd8adc161f577f32856dba21506db9717fae3d14570786439 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index a4cfd28ab6..772b896acf 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2391,6 +2391,7 @@ static int SQLITE_TCLAPI vfsCurrentTimeInt64( return TCL_OK; } +#ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Usage: create_null_module DB NAME */ @@ -2413,6 +2414,7 @@ static int SQLITE_TCLAPI test_create_null_module( sqlite3_create_module(db, zName, 0, 0); return TCL_OK; } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ #ifdef SQLITE_ENABLE_SNAPSHOT /* @@ -9005,7 +9007,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_register_cksumvfs", test_register_cksumvfs, 0 }, { "sqlite3_unregister_cksumvfs", test_unregister_cksumvfs, 0 }, { "number_of_cores", guess_number_of_cores, 0 }, +#ifndef SQLITE_OMIT_VIRTUALTABLE { "create_null_module", test_create_null_module, 0 }, +#endif }; static int bitmask_size = sizeof(Bitmask)*8; static int longdouble_size = sizeof(LONGDOUBLE_TYPE); From d36106c954ed9a5636c5603e25ff032177f1eefa Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Apr 2023 16:11:05 +0000 Subject: [PATCH 216/341] Add tests for the new feature on this branch. FossilOrigin-Name: 0268d1a59316d169927cbd4c562725e46a9023b7d7123ade911c12203e222adf --- ext/fts5/fts5_index.c | 3 +- ext/fts5/test/fts5secure4.test | 54 +++++++++++ ext/fts5/test/fts5securefault.test | 138 +++++++++++++++++++++++++++++ manifest | 14 +-- manifest.uuid | 2 +- 5 files changed, 202 insertions(+), 9 deletions(-) create mode 100644 ext/fts5/test/fts5secure4.test create mode 100644 ext/fts5/test/fts5securefault.test diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index d6dc9fb9a4..5d16645ea3 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5002,7 +5002,6 @@ static void fts5FlushOneHash(Fts5Index *p){ if( bTermWritten==0 ){ fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); - if( p->rc!=SQLITE_OK ) break; bTermWritten = 1; assert( writer.bFirstRowidInPage==0 ); } @@ -5012,10 +5011,10 @@ static void fts5FlushOneHash(Fts5Index *p){ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); writer.bFirstRowidInPage = 0; fts5WriteDlidxAppend(p, &writer, iRowid); - if( p->rc!=SQLITE_OK ) break; }else{ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); } + if( p->rc!=SQLITE_OK ) break; assert( pBuf->n<=pBuf->nSpace ); iPrev = iRowid; diff --git a/ext/fts5/test/fts5secure4.test b/ext/fts5/test/fts5secure4.test new file mode 100644 index 0000000000..b6a3be5e5b --- /dev/null +++ b/ext/fts5/test/fts5secure4.test @@ -0,0 +1,54 @@ +# 2023 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +return_if_no_fts5 +set ::testprefix fts5secure4 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, content=x1); + + CREATE TABLE x1(rowid INTEGER PRIMARY KEY, a, b); + INSERT INTO x1 VALUES + (1, 'hello world', 'today xyz'), + (2, 'not the day', 'crunch crumble and chomp'), + (3, 'one', 'two'); + INSERT INTO t1(t1) VALUES('rebuild'); +} + +do_execsql_test 1.1 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 1.2 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 4, 'nosuchtoken', ''); +} + +do_execsql_test 1.3 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + +do_execsql_test 1.4 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 1, 'crunch', ''); +} + +do_execsql_test 1.5 { + INSERT INTO t1(t1, rowid, a, b) VALUES('delete', 3, 'crunch', ''); +} + +do_execsql_test 1.6 { + INSERT INTO t1(t1) VALUES('integrity-check'); +} + + +finish_test + diff --git a/ext/fts5/test/fts5securefault.test b/ext/fts5/test/fts5securefault.test new file mode 100644 index 0000000000..1b0cad656c --- /dev/null +++ b/ext/fts5/test/fts5securefault.test @@ -0,0 +1,138 @@ +# 2023 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS5 module. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5securefault + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +return_if_no_fts5 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(rowid, ab) VALUES + (0, 'abc'), (1, 'abc'), (2, 'abc'), (3, 'abc'), (4, 'def'); +} +faultsim_save_and_close + +do_faultsim_test 1.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid=2 } +} -test { + faultsim_test_result {0 {}} +} +do_faultsim_test 1.2 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid IN(0, 1, 2, 3, 4) } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set big [string repeat abcdefghij 5] +set big2 [string repeat klmnopqrst 5] +set doc "$big $big2" + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<4 + ) + INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s; +} +faultsim_save_and_close + +do_faultsim_test 2.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 3 } + execsql { DELETE FROM t1 WHERE rowid = 4 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set big [string repeat abcdefghij 5] +set big2 [string repeat klmnopqrst 5] +set doc "$big $big2" + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<25 + ) + INSERT INTO t1(rowid, ab) SELECT i, $doc FROM s; + + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + DELETE FROM t1 WHERE rowid BETWEEN 3 AND 23; +} +faultsim_save_and_close + +do_faultsim_test 3.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 24 } + execsql { DELETE FROM t1 WHERE rowid = 25 } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set doc [string repeat "tok " 400] + +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + INSERT INTO t1(rowid, ab) VALUES(1, $doc), (2, $doc), (3, $doc); +} +faultsim_save_and_close + +do_faultsim_test 4.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 2 } +} -test { + faultsim_test_result {0 {}} +} + + + + + +finish_test diff --git a/manifest b/manifest index e432d7ed9a..ad08000572 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\stemporary\sdebugging\scode\saccidentally\sleft\son\sthis\sbranch. -D 2023-04-12T18:06:43.220 +C Add\stests\sfor\sthe\snew\sfeature\son\sthis\sbranch. +D 2023-04-14T16:11:05.842 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 4dc0f8b79af771a6a490e4c3476729cba201fb9081c88324da45ece57c7a3372 +F ext/fts5/fts5_index.c 2e7829ffae1185961a87cec13052d9781e74eaf0d8831ea6743ee88f286f17ce F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -190,6 +190,8 @@ F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e33 F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc F ext/fts5/test/fts5secure3.test fd73b98a6e11038960b84109fed42f9e0a098a31338c94d07c163244d4bd7254 +F ext/fts5/test/fts5secure4.test 3eed9eaece133fa4921b4383b13f71db1c0e415324cd86bb9dc5ac91609d5776 +F ext/fts5/test/fts5securefault.test 707483e996f7b12077a490cd71a0ca45ccde0e7ffc737c3ab52ff7d6ad8e3a7c F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f @@ -2055,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4240fd09b717dbc69dffe3b88ec9149777ca4c3efa12f282af65be3af6fa5bb0 -R 94c13bd9e875c4e10b2fe9e0cf572c45 +P 0a0f64870feaf95d2673efc8884ad1236d9bbf9bdb94364f8d2602221bd36ff3 +R b7a845354df37e1aac7f6581514d1050 U dan -Z 057142ccfc1c60d57ad54409384a9ea2 +Z 01ce26ba699a74240fcd6e679f13a11b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 63292c41a1..9ea9c8f842 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a0f64870feaf95d2673efc8884ad1236d9bbf9bdb94364f8d2602221bd36ff3 \ No newline at end of file +0268d1a59316d169927cbd4c562725e46a9023b7d7123ade911c12203e222adf \ No newline at end of file From 3fc123b7bcc966a3b456bc4bc049de2d5e5505dd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Apr 2023 17:00:29 +0000 Subject: [PATCH 217/341] Add extra OOM test for the new code on this branch. FossilOrigin-Name: 846ae7e099ce67dab6b5df0ad5648a01d7bf7f2acf8312d817ed8505d4ea5aec --- ext/fts5/test/fts5securefault.test | 33 ++++++++++++++++++++++++++++++ manifest | 12 +++++------ manifest.uuid | 2 +- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/ext/fts5/test/fts5securefault.test b/ext/fts5/test/fts5securefault.test index 1b0cad656c..08aee2618e 100644 --- a/ext/fts5/test/fts5securefault.test +++ b/ext/fts5/test/fts5securefault.test @@ -131,8 +131,41 @@ do_faultsim_test 4.1 -faults oom* -prep { faultsim_test_result {0 {}} } +#------------------------------------------------------------------------- +# +reset_db +set doc1 [string repeat "abc " 10] +set doc2 [string repeat "def " 10] +do_test 5.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + BEGIN; + } + for {set i 0} {$i < 50} {incr i} { + execsql { + INSERT INTO t1(rowid, ab) VALUES($i, 'abcdefg'); + } + } + execsql { + INSERT INTO t1(rowid, ab) VALUES(105, 'def'); + COMMIT; + } +} {} +faultsim_save_and_close + +do_faultsim_test 5.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { DELETE FROM t1 WHERE rowid = 105 } +} -test { + faultsim_test_result {0 {}} +} finish_test diff --git a/manifest b/manifest index ad08000572..889cc1e34b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sthe\snew\sfeature\son\sthis\sbranch. -D 2023-04-14T16:11:05.842 +C Add\sextra\sOOM\stest\sfor\sthe\snew\scode\son\sthis\sbranch. +D 2023-04-14T17:00:29.247 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -191,7 +191,7 @@ F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc7 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc F ext/fts5/test/fts5secure3.test fd73b98a6e11038960b84109fed42f9e0a098a31338c94d07c163244d4bd7254 F ext/fts5/test/fts5secure4.test 3eed9eaece133fa4921b4383b13f71db1c0e415324cd86bb9dc5ac91609d5776 -F ext/fts5/test/fts5securefault.test 707483e996f7b12077a490cd71a0ca45ccde0e7ffc737c3ab52ff7d6ad8e3a7c +F ext/fts5/test/fts5securefault.test d18e3cab7a3c4a66f44812202e1b0ea0b4bd0f9ee766951749737cfbab100e7e F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a0f64870feaf95d2673efc8884ad1236d9bbf9bdb94364f8d2602221bd36ff3 -R b7a845354df37e1aac7f6581514d1050 +P 0268d1a59316d169927cbd4c562725e46a9023b7d7123ade911c12203e222adf +R d8f750d5cff7a2eb655ae379ef3df034 U dan -Z 01ce26ba699a74240fcd6e679f13a11b +Z babd8fc4a31fce6ad5175a62742eb085 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9ea9c8f842..294d07f50b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0268d1a59316d169927cbd4c562725e46a9023b7d7123ade911c12203e222adf \ No newline at end of file +846ae7e099ce67dab6b5df0ad5648a01d7bf7f2acf8312d817ed8505d4ea5aec \ No newline at end of file From 112500028ef1d14cadb8a73a7b8f8fd26366231e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Apr 2023 18:00:03 +0000 Subject: [PATCH 218/341] Further tests for fts5 secure-delete mode. FossilOrigin-Name: c5a47063b7971868ebbb3f0718434dd6bd22d01c43697f48941df9aae3885abc --- ext/fts5/test/fts5secure3.test | 8 +++- ext/fts5/test/fts5secure4.test | 82 ++++++++++++++++++++++++++++++++++ manifest | 14 +++--- manifest.uuid | 2 +- 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/ext/fts5/test/fts5secure3.test b/ext/fts5/test/fts5secure3.test index 70c2028df6..bc56e08200 100644 --- a/ext/fts5/test/fts5secure3.test +++ b/ext/fts5/test/fts5secure3.test @@ -9,6 +9,8 @@ # #************************************************************************* # +# TESTRUNNER: slow +# source [file join [file dirname [info script]] fts5_common.tcl] ifcapable !fts5 { finish_test ; return } @@ -142,8 +144,10 @@ foreach ii [lshuffle [db eval {SELECT rowid FROM fff}]] { DELETE FROM fff WHERE rowid=$ii; } #if {$iTest==1} { dump fff } - do_execsql_test 3.1.$iTest.$ii.ic { - INSERT INTO fff(fff) VALUES('integrity-check'); + if {($iTest % 20)==0} { + do_execsql_test 3.1.$iTest.$ii.ic { + INSERT INTO fff(fff) VALUES('integrity-check'); + } } #if {$iTest==1} { break } incr iTest diff --git a/ext/fts5/test/fts5secure4.test b/ext/fts5/test/fts5secure4.test index b6a3be5e5b..48d9242b6b 100644 --- a/ext/fts5/test/fts5secure4.test +++ b/ext/fts5/test/fts5secure4.test @@ -14,6 +14,10 @@ source [file join [file dirname [info script]] fts5_common.tcl] return_if_no_fts5 set ::testprefix fts5secure4 +#------------------------------------------------------------------------- +# Test using the 'delete' command to attempt to delete a token that +# is not present in the index in secure-delete mode. +# do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, content=x1); @@ -49,6 +53,84 @@ do_execsql_test 1.6 { INSERT INTO t1(t1) VALUES('integrity-check'); } +#------------------------------------------------------------------------- +# Test using secure-delete with detail=none or detail=col. +# +foreach {tn d} {1 full 2 none 3 column} { + reset_db + do_execsql_test 2.$tn.1 " + CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='1,2'); + INSERT INTO x1(x1, rank) VALUES('pgsz', 64); + " + + do_execsql_test 2.$tn.2 { + BEGIN; + INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); + COMMIT; + } + + do_execsql_test 2.$tn.3 { + DELETE FROM x1 WHERE rowid IN (2, 4, 6); + INSERT INTO x1(x1) VALUES('integrity-check'); + } + + do_execsql_test 2.$tn.4 { + DELETE FROM x1 WHERE rowid IN (1, 3, 5); + INSERT INTO x1(x1) VALUES('integrity-check'); + } + + do_execsql_test 2.$tn.5 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO x1 + SELECT 'seems to be', 'used brew to', 'everything is working' FROM s + UNION ALL + SELECT 'used brew to', 'everything is working', 'seems to be' FROM s + UNION ALL + SELECT 'everything is working', 'seems to be', 'used brew to' FROM s + UNION ALL + SELECT 'abc', 'zzz', 'a b c d' + UNION ALL + SELECT 'z', 'z', 'z' FROM s + } + + do_test 2.$tn.6 { + for {set i 300} {$i > 200} {incr i -1} { + execsql { + DELETE FROM x1 WHERE rowid=$i; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + } + } {} + + do_test 2.$tn.7 { + for {set i 1} {$i < 100} {incr i} { + execsql { + DELETE FROM x1 WHERE rowid=$i; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + } + } {} + + do_test 2.$tn.8 { + foreach i [db eval {SELECT rowid FROM x1}] { + execsql { + DELETE FROM x1 WHERE rowid=$i; + INSERT INTO x1(x1) VALUES('integrity-check'); + } + } + } {} + + do_execsql_test 2.$tn.9 { + SELECT * FROM x1 + } {} +} + finish_test diff --git a/manifest b/manifest index 889cc1e34b..44ea6b43cb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\sOOM\stest\sfor\sthe\snew\scode\son\sthis\sbranch. -D 2023-04-14T17:00:29.247 +C Further\stests\sfor\sfts5\ssecure-delete\smode. +D 2023-04-14T18:00:03.868 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -189,8 +189,8 @@ F ext/fts5/test/fts5rowid.test b8790ec170a8dc1942a15aef3db926a5f3061b1ff17101300 F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e332d654375690f8e6e43f F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc -F ext/fts5/test/fts5secure3.test fd73b98a6e11038960b84109fed42f9e0a098a31338c94d07c163244d4bd7254 -F ext/fts5/test/fts5secure4.test 3eed9eaece133fa4921b4383b13f71db1c0e415324cd86bb9dc5ac91609d5776 +F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 +F ext/fts5/test/fts5secure4.test 17ac602024db3953401c3e51ad8e03de3b972c1770554585f63812ca4083550e F ext/fts5/test/fts5securefault.test d18e3cab7a3c4a66f44812202e1b0ea0b4bd0f9ee766951749737cfbab100e7e F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0268d1a59316d169927cbd4c562725e46a9023b7d7123ade911c12203e222adf -R d8f750d5cff7a2eb655ae379ef3df034 +P 846ae7e099ce67dab6b5df0ad5648a01d7bf7f2acf8312d817ed8505d4ea5aec +R 288b188c0f389f12eebf550efb7220b6 U dan -Z babd8fc4a31fce6ad5175a62742eb085 +Z 9dffa1950ad50c2fc3482bbd0532371a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 294d07f50b..72e3e9e1e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -846ae7e099ce67dab6b5df0ad5648a01d7bf7f2acf8312d817ed8505d4ea5aec \ No newline at end of file +c5a47063b7971868ebbb3f0718434dd6bd22d01c43697f48941df9aae3885abc \ No newline at end of file From 67f678138dc7e601db9e6e260195d4234d7d7734 Mon Sep 17 00:00:00 2001 From: larrybr Date: Fri, 14 Apr 2023 19:56:32 +0000 Subject: [PATCH 219/341] Cure CLI double-prompting (by ditching gcc fgetws()), general cleanup. Work remaining is to avoid effect of -utf8 when a line editor is linked/used as part of CLI. FossilOrigin-Name: 73a5f54231e9f6ad8f013df3987ea48c516080f9193ed873b56f982ee75658c2 --- manifest | 12 +++---- manifest.uuid | 2 +- src/shell.c.in | 87 +++++++++++++++++++++++++++++++------------------- 3 files changed, 62 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index 260a215b52..6dfe5033f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sCLI\sutf8_fgets()\sto\snot\sconsume\smore\sinput\sthan\sit\sreturns.\sGet\sconsole\ssetup\srestoration\sto\shappen\sfor\sall\snon-crash\sexits. -D 2023-04-13T14:14:27.434 +C Cure\sCLI\sdouble-prompting\s(by\sditching\sgcc\sfgetws()),\sgeneral\scleanup.\sWork\sremaining\sis\sto\savoid\seffect\sof\s-utf8\swhen\sa\sline\seditor\sis\slinked/used\sas\spart\sof\sCLI. +D 2023-04-14T19:56:32.111 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94 -F src/shell.c.in d3a5bef33e4ad55f3feeaeb0deaed32dcb80252a5b2ca184d1eefa816a9f92cf +F src/shell.c.in 23c6ed8477303108f53ec73c8333aaff8d11267276f3852597304d8915bc55d7 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 824382393d92d9eb6df8701de7c263280150569a708759c4a539acc6d8d38821 -R 692e15a266cff0987f54e086d515e8e5 +P b4fa233d3dda54fa83771844cf5156bf1275c687925340af17a7713a9400dfef +R c50b2d60063b3025d3b72bba985f6bbb U larrybr -Z d0dd4db17efed3a7d98e05299e7fc21d +Z 16fa3e5dbb2e04c40e7812c85b6fd381 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 065235828d..d526131411 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4fa233d3dda54fa83771844cf5156bf1275c687925340af17a7713a9400dfef \ No newline at end of file +73a5f54231e9f6ad8f013df3987ea48c516080f9193ed873b56f982ee75658c2 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 12748b29a3..2d954bb218 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -589,10 +589,12 @@ static char *dynamicContinuePrompt(void){ #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ #if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) -static int infsMode; -static UINT codePage = 0; -static HANDLE hConsoleIn; -static DWORD consoleMode; +/* Following variables are used for -utf8 operation. */ +static int stdinEof = 0; /* EOF seen on console input */ +static int infsMode; /* Input file stream mode upon shell start */ +static UINT codePage = 0; /* Input code page upon shell start */ +static HANDLE hConsoleIn = INVALID_HANDLE_VALUE; /* Console input handle */ +static DWORD consoleMode; /* Console mode upon shell start */ /* ** Prepare input stream, (if known to be a WIN32 console), for UTF-8 @@ -601,7 +603,8 @@ static DWORD consoleMode; ** the preparation is not done and common "code page" issues occur. */ static void instream_prepare(void){ - if( isatty(0) ){ + hConsoleIn = GetStdHandle(STD_INPUT_HANDLE); + if( isatty(0) && GetFileType(hConsoleIn)==FILE_TYPE_CHAR ){ if( !IsValidCodePage(CP_UTF8) ){ fprintf(stderr, "Cannot use UTF-8 code page.\n"); console_utf8 = 0; @@ -609,7 +612,6 @@ static void instream_prepare(void){ } codePage = GetConsoleCP(); SetConsoleCP(CP_UTF8); - hConsoleIn = GetStdHandle(STD_INPUT_HANDLE); GetConsoleMode( hConsoleIn, &consoleMode); SetConsoleMode( hConsoleIn, consoleMode | ENABLE_LINE_INPUT ); infsMode = _setmode(_fileno(stdin), _O_U16TEXT); @@ -623,7 +625,7 @@ static void instream_prepare(void){ ** Undo the effects of instream_prepare(), if any. */ static void SQLITE_CDECL instream_restore(void){ - if( console_utf8 && codePage!=0 ){ + if( console_utf8 && codePage!=0 &&hConsoleIn != INVALID_HANDLE_VALUE ){ _setmode(_fileno(stdin), infsMode); SetConsoleCP(codePage); SetConsoleMode( hConsoleIn, consoleMode ); @@ -638,28 +640,55 @@ static void SQLITE_CDECL instream_restore(void){ ** startup option, -utf8, has not been provided or taken effect. */ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ - #define SQLITE_IA_LINE_LEN 150 + if( fin==0 ) fin = stdin; if( fin==stdin && stdin_is_interactive && console_utf8 ){ - wchar_t wbuf[SQLITE_IA_LINE_LEN]; +# define SQLITE_IALIM 150 + wchar_t wbuf[SQLITE_IALIM]; + int lend = 0; int noc = 0; - if( ncmax == 0 ) return 0; + if( ncmax == 0 || stdinEof ) return 0; buf[0] = 0; - while( noc < ncmax-7-1 ){ - /* There is room for at least 2 more characters and the 0-terminator. */ - int na = (ncmax-1 - noc)/4; - wchar_t * wz; - if( na > SQLITE_IA_LINE_LEN ) na = SQLITE_IA_LINE_LEN; - wz = fgetws(wbuf, na, stdin); - if( wz != 0 ){ - int nmb = WideCharToMultiByte(CP_UTF8, 0, wz, -1, 0, 0, 0, 0); + while( noc < ncmax-7-1 && !lend ){ + /* There is room for at least 2 more characters and a 0-terminator. */ + int na = (ncmax > SQLITE_IALIM*4+1 + noc) + ? SQLITE_IALIM : (ncmax-1 - noc)/4; +# undef SQLITE_IALIM + DWORD nbr = 0; + BOOL bRC = ReadConsoleW(hConsoleIn, wbuf, na, &nbr, 0); + if( !bRC || (noc==0 && nbr==0) ) return 0; + if( nbr > 0 ){ + int nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR, + wbuf,nbr, 0,0, 0, 0); if( nmb !=0 && noc+nmb <= ncmax ){ - WideCharToMultiByte(CP_UTF8, 0, wz, -1, buf+noc, nmb, 0, 0); - noc += nmb-1; /* Strip 0-terminator added by above call. */ - if( buf[noc-1] == '\n' ) break; + int iseg = noc; + nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR, + wbuf,nbr, buf+noc,nmb, 0,0); + noc += nmb; + /* Fixup line-ends as coded by Windows for CR (or "Enter".)*/ + if( noc > 0 ){ + if( buf[noc-1]=='\n' ){ + lend = 1; + if( noc > 1 && buf[noc-2]=='\r' ){ + buf[noc-2] = '\n'; + --noc; + } + } + } + /* Check for ^Z (anywhere in line) too. */ + while( iseg < noc ){ + if( buf[iseg]==0x1a ){ + stdinEof = 1; + noc = iseg; /* Chop ^Z and anything following. */ + break; + } + ++iseg; + } }else break; /* Drop apparent garbage in. (Could assert.) */ }else break; } + /* If got nothing, (after ^Z chop), must be at end-of-file. */ if( noc == 0 ) return 0; + buf[noc] = 0; return buf; }else{ return fgets(buf, ncmax, fin); @@ -889,14 +918,11 @@ static char *local_getline(char *zLine, FILE *in){ } } #if defined(_WIN32) || defined(WIN32) - /* For interactive input on Windows systems, translate the - ** multi-byte characterset characters into UTF-8. */ - if( stdin_is_interactive && in==stdin ){ - // FILE *cil = fopen("cin.log", "wa"); - // i64 nzl = strlen(zLine); + /* For interactive input on Windows systems, without -utf8, + ** translate the multi-byte characterset characters into UTF-8. + ** This is the translation that predates the -utf8 option. */ + if( stdin_is_interactive && in==stdin && !console_utf8 ){ char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); - // fwrite(zLine, nzl, 1, cil); - // fwrite("\n", 1,1, cil); if( zTrans ){ i64 nTrans = strlen(zTrans)+1; if( nTrans>nLine ){ @@ -904,11 +930,8 @@ static char *local_getline(char *zLine, FILE *in){ shell_check_oom(zLine); } memcpy(zLine, zTrans, nTrans); - // fwrite(zLine, nTrans-1, 1, cil); - // fwrite("\n", 1,1, cil); sqlite3_free(zTrans); } - // fclose(cil); } #endif /* defined(_WIN32) || defined(WIN32) */ return zLine; @@ -11819,7 +11842,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int argcToFree = 0; #endif - setBinaryMode(stdin, 0); setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ #ifdef SQLITE_SHELL_FIDDLE stdin_is_interactive = 0; @@ -12306,6 +12328,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( console_utf8 && stdin_is_interactive ){ instream_prepare(); }else{ + setBinaryMode(stdin, 0); console_utf8 = 0; } #endif From cdb8a698c1997327d4bf9c23d1c9b295db20d5d5 Mon Sep 17 00:00:00 2001 From: larrybr Date: Fri, 14 Apr 2023 21:23:08 +0000 Subject: [PATCH 220/341] Set CLI -utf8 option and build with line-editing package to be mutually exclusive. Integration of console-invasive UTF-8 handling with line-editing takeover of console may come later. FossilOrigin-Name: 047344a91583f273a55fe3659bb8020ffc947c05c4274fbda54ee0608e62adb1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 28 +++++++++++++++++++--------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 6dfe5033f4..40333232b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cure\sCLI\sdouble-prompting\s(by\sditching\sgcc\sfgetws()),\sgeneral\scleanup.\sWork\sremaining\sis\sto\savoid\seffect\sof\s-utf8\swhen\sa\sline\seditor\sis\slinked/used\sas\spart\sof\sCLI. -D 2023-04-14T19:56:32.111 +C Set\sCLI\s-utf8\soption\sand\sbuild\swith\sline-editing\spackage\sto\sbe\smutually\sexclusive.\sIntegration\sof\sconsole-invasive\sUTF-8\shandling\swith\sline-editing\stakeover\sof\sconsole\smay\scome\slater. +D 2023-04-14T21:23:08.731 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94 -F src/shell.c.in 23c6ed8477303108f53ec73c8333aaff8d11267276f3852597304d8915bc55d7 +F src/shell.c.in 48ca1d8461e3f39bc10b7a4600a4e6770e83af5c22c97f6a6cbeb72cc073fdbb F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b4fa233d3dda54fa83771844cf5156bf1275c687925340af17a7713a9400dfef -R c50b2d60063b3025d3b72bba985f6bbb +P 73a5f54231e9f6ad8f013df3987ea48c516080f9193ed873b56f982ee75658c2 +R 6cbccd65b09f7c37b3a15e7a5991ab63 U larrybr -Z 16fa3e5dbb2e04c40e7812c85b6fd381 +Z d6006bc3d1035c124212e23fa4c9f69b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d526131411..aad25c04fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -73a5f54231e9f6ad8f013df3987ea48c516080f9193ed873b56f982ee75658c2 \ No newline at end of file +047344a91583f273a55fe3659bb8020ffc947c05c4274fbda54ee0608e62adb1 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 2d954bb218..6aaf5cb33e 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -449,7 +449,13 @@ static int bail_on_error = 0; */ static int stdin_is_interactive = 1; -#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) +#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE +# define SHELL_WIN_UTF8_OPT 1 +#else +# define SHELL_WIN_UTF8_OPT 0 +#endif + +#if SHELL_WIN_UTF8_OPT /* ** Setup console for UTF-8 input/output when following variable true. */ @@ -588,7 +594,7 @@ static char *dynamicContinuePrompt(void){ } #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ -#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) +#if SHELL_WIN_UTF8_OPT /* Following variables are used for -utf8 operation. */ static int stdinEof = 0; /* EOF seen on console input */ static int infsMode; /* Input file stream mode upon shell start */ @@ -696,7 +702,7 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ } # define fgets(b,n,f) utf8_fgets(b,n,f) -#endif /* (defined(_WIN32)||defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) */ +#endif /* SHELL_WIN_UTF8_OPT */ /* ** Render output like fprintf(). Except, if the output is going to the @@ -921,7 +927,11 @@ static char *local_getline(char *zLine, FILE *in){ /* For interactive input on Windows systems, without -utf8, ** translate the multi-byte characterset characters into UTF-8. ** This is the translation that predates the -utf8 option. */ - if( stdin_is_interactive && in==stdin && !console_utf8 ){ + if( stdin_is_interactive && in==stdin +# if SHELL_WIN_UTF8_OPT + && !console_utf8 +# endif /* SHELL_WIN_UTF8_OPT */ + ){ char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); if( zTrans ){ i64 nTrans = strlen(zTrans)+1; @@ -11705,7 +11715,7 @@ static const char zOptions[] = " -stats print memory stats before each finalize\n" " -table set output mode to 'table'\n" " -tabs set output mode to 'tabs'\n" -#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) +#if SHELL_WIN_UTF8_OPT " -utf8 setup interactive console code page for UTF-8\n" #endif " -version show SQLite version\n" @@ -11851,7 +11861,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = isatty(0); stdout_is_console = isatty(1); #endif -#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) +#if SHELL_WIN_UTF8_OPT atexit(instream_restore); /* Needs revision for CLI as library call */ #endif #ifdef SQLITE_DEBUG @@ -12244,9 +12254,9 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-batch")==0 ){ stdin_is_interactive = 0; }else if( cli_strcmp(z,"-utf8")==0 ){ -#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) +#if SHELL_WIN_UTF8_OPT console_utf8 = 1; -#endif +#endif /* SHELL_WIN_UTF8_OPT */ }else if( cli_strcmp(z,"-heap")==0 ){ i++; }else if( cli_strcmp(z,"-pagecache")==0 ){ @@ -12324,7 +12334,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } data.cMode = data.mode; } -#if (defined(_WIN32) || defined(WIN32)) && !defined(SQLITE_SHELL_FIDDLE) +#if SHELL_WIN_UTF8_OPT if( console_utf8 && stdin_is_interactive ){ instream_prepare(); }else{ From 5f347a426de9e39ab315c3963a7b6594ea727888 Mon Sep 17 00:00:00 2001 From: larrybr Date: Fri, 14 Apr 2023 21:34:58 +0000 Subject: [PATCH 221/341] Fix stray edits in comments in last check-in. FossilOrigin-Name: fa3ce7f013d40147f012e26d0ede75b134174f144d314336310c9ac585e23fb7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 40333232b4..f40d839a7a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Set\sCLI\s-utf8\soption\sand\sbuild\swith\sline-editing\spackage\sto\sbe\smutually\sexclusive.\sIntegration\sof\sconsole-invasive\sUTF-8\shandling\swith\sline-editing\stakeover\sof\sconsole\smay\scome\slater. -D 2023-04-14T21:23:08.731 +C Fix\sstray\sedits\sin\scomments\sin\slast\scheck-in. +D 2023-04-14T21:34:58.853 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94 -F src/shell.c.in 48ca1d8461e3f39bc10b7a4600a4e6770e83af5c22c97f6a6cbeb72cc073fdbb +F src/shell.c.in 904c52bc168b5400838461b8c3cad93a47a667670cac537ed2eadc7ac3513a48 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 73a5f54231e9f6ad8f013df3987ea48c516080f9193ed873b56f982ee75658c2 -R 6cbccd65b09f7c37b3a15e7a5991ab63 +P 047344a91583f273a55fe3659bb8020ffc947c05c4274fbda54ee0608e62adb1 +R 871ffb5c7dac7776c2e69fb0be1e0787 U larrybr -Z d6006bc3d1035c124212e23fa4c9f69b +Z 211efb5182446fc16d9890afb7d4aec9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index aad25c04fc..d59bc1bfb8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -047344a91583f273a55fe3659bb8020ffc947c05c4274fbda54ee0608e62adb1 \ No newline at end of file +fa3ce7f013d40147f012e26d0ede75b134174f144d314336310c9ac585e23fb7 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 6aaf5cb33e..79d43ff909 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -11,7 +11,6 @@ ************************************************************************* ** This file contains code to implement the "sqlite" command line ** utility for accessing SQLite databases. -ToDo: The -utf8 option may not play well with line editing input libraries. */ #if (defined(_WIN32) || defined(WIN32)) && !defined(_CRT_SECURE_NO_WARNINGS) /* This needs to come before any includes for MSVC compiler */ @@ -21,7 +20,7 @@ typedef unsigned int u32; typedef unsigned short int u16; /* -** optionally #include a user-defined header, whereby compilation options +** Optionally #include a user-defined header, whereby compilation options ** may be set prior to where they take effect, but after platform setup. ** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include ** file. Note that this macro has a like effect on sqlite3.c compilation. From 9726db69e6bd6e800a2f8fa6b2102370dcf72df4 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 15 Apr 2023 16:12:46 +0000 Subject: [PATCH 222/341] For CLI -utf8, set output codepage too. Adjust PP vars so that the code can be entirely omitted for targets pretending to be "WIN32" but not quite doing so. FossilOrigin-Name: 543594a7277b12d1c58af8e3cba3841667cf924172995536489ce7da704a11d7 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/shell.c.in | 61 +++++++++++++++++++++++++++++--------------------- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index f40d839a7a..454e8ac7f9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sstray\sedits\sin\scomments\sin\slast\scheck-in. -D 2023-04-14T21:34:58.853 +C For\sCLI\s-utf8,\sset\soutput\scodepage\stoo.\sAdjust\sPP\svars\sso\sthat\sthe\scode\scan\sbe\sentirely\somitted\sfor\stargets\spretending\sto\sbe\s"WIN32"\sbut\snot\squite\sdoing\sso. +D 2023-04-15T16:12:46.806 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 93bb02212256b49a90589a4664031896e2b2991198153dff1a33a72f437dad94 -F src/shell.c.in 904c52bc168b5400838461b8c3cad93a47a667670cac537ed2eadc7ac3513a48 +F src/shell.c.in ec8f1e4f87882670885e27037b8c236f569932a823feecbefc25a910c851d4e8 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 047344a91583f273a55fe3659bb8020ffc947c05c4274fbda54ee0608e62adb1 -R 871ffb5c7dac7776c2e69fb0be1e0787 +P fa3ce7f013d40147f012e26d0ede75b134174f144d314336310c9ac585e23fb7 +R bf7c6e9f3d2c14f31428bf0dcd17722f U larrybr -Z 211efb5182446fc16d9890afb7d4aec9 +Z e087e76624523fc274ea2606bdd42051 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d59bc1bfb8..ccd8ec1c4a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa3ce7f013d40147f012e26d0ede75b134174f144d314336310c9ac585e23fb7 \ No newline at end of file +543594a7277b12d1c58af8e3cba3841667cf924172995536489ce7da704a11d7 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 79d43ff909..06648f494a 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -448,7 +448,8 @@ static int bail_on_error = 0; */ static int stdin_is_interactive = 1; -#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE +#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE \ + && !defined(SHELL_OMIT_WIN_UTF8) # define SHELL_WIN_UTF8_OPT 1 #else # define SHELL_WIN_UTF8_OPT 0 @@ -594,32 +595,38 @@ static char *dynamicContinuePrompt(void){ #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ #if SHELL_WIN_UTF8_OPT -/* Following variables are used for -utf8 operation. */ -static int stdinEof = 0; /* EOF seen on console input */ -static int infsMode; /* Input file stream mode upon shell start */ -static UINT codePage = 0; /* Input code page upon shell start */ -static HANDLE hConsoleIn = INVALID_HANDLE_VALUE; /* Console input handle */ -static DWORD consoleMode; /* Console mode upon shell start */ +/* Following struct is used for -utf8 operation. */ +static struct ConsoleState { + int stdinEof; /* EOF has been seen on console input */ + int infsMode; /* Input file stream mode upon shell start */ + UINT inCodePage; /* Input code page upon shell start */ + UINT outCodePage; /* Output code page upon shell start */ + HANDLE hConsoleIn; /* Console input handle */ + DWORD consoleMode; /* Console mode upon shell start */ +} conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 }; /* -** Prepare input stream, (if known to be a WIN32 console), for UTF-8 +** Prepare console, (if known to be a WIN32 console), for UTF-8 ** input (from either typing or suitable paste operations) and for ** UTF-8 rendering. This may "fail" with a message to stderr, where ** the preparation is not done and common "code page" issues occur. */ -static void instream_prepare(void){ - hConsoleIn = GetStdHandle(STD_INPUT_HANDLE); - if( isatty(0) && GetFileType(hConsoleIn)==FILE_TYPE_CHAR ){ +static void console_prepare(void){ + conState.hConsoleIn = GetStdHandle(STD_INPUT_HANDLE); + if( isatty(0) && GetFileType(conState.hConsoleIn)==FILE_TYPE_CHAR ){ if( !IsValidCodePage(CP_UTF8) ){ fprintf(stderr, "Cannot use UTF-8 code page.\n"); console_utf8 = 0; return; } - codePage = GetConsoleCP(); + conState.inCodePage = GetConsoleCP(); + conState.outCodePage = GetConsoleOutputCP(); SetConsoleCP(CP_UTF8); - GetConsoleMode( hConsoleIn, &consoleMode); - SetConsoleMode( hConsoleIn, consoleMode | ENABLE_LINE_INPUT ); - infsMode = _setmode(_fileno(stdin), _O_U16TEXT); + SetConsoleOutputCP(CP_UTF8); + GetConsoleMode( conState.hConsoleIn, &conState.consoleMode); + SetConsoleMode( conState.hConsoleIn, + conState.consoleMode | ENABLE_LINE_INPUT ); + conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT); console_utf8 = 1; }else{ console_utf8 = 0; @@ -627,13 +634,15 @@ static void instream_prepare(void){ } /* -** Undo the effects of instream_prepare(), if any. +** Undo the effects of console_prepare(), if any. */ -static void SQLITE_CDECL instream_restore(void){ - if( console_utf8 && codePage!=0 &&hConsoleIn != INVALID_HANDLE_VALUE ){ - _setmode(_fileno(stdin), infsMode); - SetConsoleCP(codePage); - SetConsoleMode( hConsoleIn, consoleMode ); +static void SQLITE_CDECL console_restore(void){ + if( console_utf8 && conState.inCodePage!=0 + && conState.hConsoleIn != INVALID_HANDLE_VALUE ){ + _setmode(_fileno(stdin), conState.infsMode); + SetConsoleCP(conState.inCodePage); + SetConsoleOutputCP(conState.outCodePage); + SetConsoleMode( conState.hConsoleIn, conState.consoleMode ); console_utf8 = 0; /* Avoid multiple calls. */ } } @@ -651,7 +660,7 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ wchar_t wbuf[SQLITE_IALIM]; int lend = 0; int noc = 0; - if( ncmax == 0 || stdinEof ) return 0; + if( ncmax == 0 || conState.stdinEof ) return 0; buf[0] = 0; while( noc < ncmax-7-1 && !lend ){ /* There is room for at least 2 more characters and a 0-terminator. */ @@ -659,7 +668,7 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ ? SQLITE_IALIM : (ncmax-1 - noc)/4; # undef SQLITE_IALIM DWORD nbr = 0; - BOOL bRC = ReadConsoleW(hConsoleIn, wbuf, na, &nbr, 0); + BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0); if( !bRC || (noc==0 && nbr==0) ) return 0; if( nbr > 0 ){ int nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR, @@ -682,7 +691,7 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ /* Check for ^Z (anywhere in line) too. */ while( iseg < noc ){ if( buf[iseg]==0x1a ){ - stdinEof = 1; + conState.stdinEof = 1; noc = iseg; /* Chop ^Z and anything following. */ break; } @@ -11861,7 +11870,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdout_is_console = isatty(1); #endif #if SHELL_WIN_UTF8_OPT - atexit(instream_restore); /* Needs revision for CLI as library call */ + atexit(console_restore); /* Needs revision for CLI as library call */ #endif #ifdef SQLITE_DEBUG mem_main_enter = sqlite3_memory_used(); @@ -12335,7 +12344,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } #if SHELL_WIN_UTF8_OPT if( console_utf8 && stdin_is_interactive ){ - instream_prepare(); + console_prepare(); }else{ setBinaryMode(stdin, 0); console_utf8 = 0; From e45317a50a0d0038c1135bc6a356805954b7e452 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 15 Apr 2023 19:13:51 +0000 Subject: [PATCH 223/341] Avoid dropping the error code following an OOM in fts5 when using 'secure-delete' mode. FossilOrigin-Name: 9ee24a499c7b4a048f269f1c03c7c0b22b2898a3c23ea3e2511c2ba0342afa30 --- ext/fts5/fts5_index.c | 2 +- ext/fts5/test/fts5securefault.test | 31 ++++++++++++++++++++++++++++++ manifest | 16 +++++++-------- manifest.uuid | 2 +- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 5d16645ea3..a8662240d2 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5000,7 +5000,7 @@ static void fts5FlushOneHash(Fts5Index *p){ } } - if( bTermWritten==0 ){ + if( p->rc==SQLITE_OK && bTermWritten==0 ){ fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); bTermWritten = 1; assert( writer.bFirstRowidInPage==0 ); diff --git a/ext/fts5/test/fts5securefault.test b/ext/fts5/test/fts5securefault.test index 08aee2618e..f6124bd3bf 100644 --- a/ext/fts5/test/fts5securefault.test +++ b/ext/fts5/test/fts5securefault.test @@ -167,5 +167,36 @@ do_faultsim_test 5.1 -faults oom* -prep { faultsim_test_result {0 {}} } +#------------------------------------------------------------------------- +# +reset_db +do_test 6.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + BEGIN; + INSERT INTO t1(rowid, ab) VALUES(1, 'abcdefg'); + INSERT INTO t1(rowid, ab) VALUES(2, 'abcdefg'); + INSERT INTO t1(rowid, ab) VALUES(3, 'abcdefg'); + COMMIT; + } +} {} +faultsim_save_and_close + +do_faultsim_test 6.1 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} -body { + execsql { + BEGIN; + UPDATE t1 SET ab='abcdefg' WHERE rowid=2; + COMMIT; + } +} -test { + faultsim_test_result {0 {}} +} + finish_test diff --git a/manifest b/manifest index 8d80405dcb..fcdf9706af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s-utf8\soption\sto\sCLI,\sfor\simproved\sconsole\sbehavior\son\sWindows. -D 2023-04-15T17:56:43.253 +C Avoid\sdropping\sthe\serror\scode\sfollowing\san\sOOM\sin\sfts5\swhen\susing\s'secure-delete'\smode. +D 2023-04-15T19:13:51.887 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 2e7829ffae1185961a87cec13052d9781e74eaf0d8831ea6743ee88f286f17ce +F ext/fts5/fts5_index.c 185d989a3c875771d6a55e20d0f982590d194d79c36d78b90b1e593dc96ea615 F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -191,7 +191,7 @@ F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc7 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 F ext/fts5/test/fts5secure4.test 17ac602024db3953401c3e51ad8e03de3b972c1770554585f63812ca4083550e -F ext/fts5/test/fts5securefault.test d18e3cab7a3c4a66f44812202e1b0ea0b4bd0f9ee766951749737cfbab100e7e +F ext/fts5/test/fts5securefault.test 66d3988ca96bef56d6143a6b40203a78eab4cf50665c93261a7b9021a8d81319 F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 394980e4fe12125460ab14da41edae9089a4da332a46f3124bf0b9778793791f 543594a7277b12d1c58af8e3cba3841667cf924172995536489ce7da704a11d7 -R 62a74a4bec1e1957233b438563cd2695 -U larrybr -Z 9c14b9b91eb32fedfce65c74265faeec +P 414010d2366477285cb238f7af1e3d511ab4cda4bd4493441bd9868387dd5b66 +R db4ac6d716f6b37f938851dbdc9c0836 +U dan +Z 1156bf072ee2940913d49643d9c3a7fc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 03cb98a352..a2a6beba1d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -414010d2366477285cb238f7af1e3d511ab4cda4bd4493441bd9868387dd5b66 \ No newline at end of file +9ee24a499c7b4a048f269f1c03c7c0b22b2898a3c23ea3e2511c2ba0342afa30 \ No newline at end of file From 0e429cf934a981ab9152703568393bb05070dc25 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 15 Apr 2023 19:58:58 +0000 Subject: [PATCH 224/341] Fix a use-after-free that could follow an OOM or IO error in fts5 when using 'secure-delete' mode. FossilOrigin-Name: 904be83d9f3dd4b21a3aa47dde337b5c045b925ffdde4bccd1f8e8fdbef68c18 --- ext/fts5/fts5_index.c | 2 +- ext/fts5/test/fts5securefault.test | 4 +--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index a8662240d2..2a53514506 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4994,7 +4994,7 @@ static void fts5FlushOneHash(Fts5Index *p){ ** using fts5FlushSecureDelete(). */ if( bSecureDelete && (pDoclist[iOff] & 0x01) ){ fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); - if( pDoclist[iOff]==0x01 ){ + if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ iOff++; continue; } diff --git a/ext/fts5/test/fts5securefault.test b/ext/fts5/test/fts5securefault.test index f6124bd3bf..09e44b0903 100644 --- a/ext/fts5/test/fts5securefault.test +++ b/ext/fts5/test/fts5securefault.test @@ -190,9 +190,7 @@ do_faultsim_test 6.1 -faults oom* -prep { } } -body { execsql { - BEGIN; - UPDATE t1 SET ab='abcdefg' WHERE rowid=2; - COMMIT; + UPDATE t1 SET ab='abcdefg' WHERE rowid=2; } } -test { faultsim_test_result {0 {}} diff --git a/manifest b/manifest index fcdf9706af..22c4bbb0d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sdropping\sthe\serror\scode\sfollowing\san\sOOM\sin\sfts5\swhen\susing\s'secure-delete'\smode. -D 2023-04-15T19:13:51.887 +C Fix\sa\suse-after-free\sthat\scould\sfollow\san\sOOM\sor\sIO\serror\sin\sfts5\swhen\susing\s'secure-delete'\smode. +D 2023-04-15T19:58:58.601 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 185d989a3c875771d6a55e20d0f982590d194d79c36d78b90b1e593dc96ea615 +F ext/fts5/fts5_index.c 50c0e72697c310011b1520428655e79602bb92536bd7851d40f03349c21d8863 F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -191,7 +191,7 @@ F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc7 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 F ext/fts5/test/fts5secure4.test 17ac602024db3953401c3e51ad8e03de3b972c1770554585f63812ca4083550e -F ext/fts5/test/fts5securefault.test 66d3988ca96bef56d6143a6b40203a78eab4cf50665c93261a7b9021a8d81319 +F ext/fts5/test/fts5securefault.test 7208daed4171de8a54a293ef800411e3cec1ffc6e9d34209b53e56a70ee554da F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 414010d2366477285cb238f7af1e3d511ab4cda4bd4493441bd9868387dd5b66 -R db4ac6d716f6b37f938851dbdc9c0836 +P 9ee24a499c7b4a048f269f1c03c7c0b22b2898a3c23ea3e2511c2ba0342afa30 +R 7d4e8ae6ce7108b997bebd6ab3f8177f U dan -Z 1156bf072ee2940913d49643d9c3a7fc +Z 0a97f4e707f53c68d15ac47e51716a58 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a2a6beba1d..9a94488d87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ee24a499c7b4a048f269f1c03c7c0b22b2898a3c23ea3e2511c2ba0342afa30 \ No newline at end of file +904be83d9f3dd4b21a3aa47dde337b5c045b925ffdde4bccd1f8e8fdbef68c18 \ No newline at end of file From ae6f6ef3b0010d7d9f612442473f64836ad58335 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 15 Apr 2023 21:00:55 +0000 Subject: [PATCH 225/341] Change a memcpy() to memmove() in new fts5 code. FossilOrigin-Name: 8de4542553ae20947787b4daf5e88bbe69f815679960e700e19f9c1148234b08 --- ext/fts5/fts5_index.c | 2 +- ext/fts5/test/fts5secure4.test | 18 ++++++++++++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 2a53514506..adcb7045bc 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4831,7 +4831,7 @@ static void fts5FlushSecureDelete( memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix); iOff += (nPrefix2-nPrefix); } - memcpy(&aPg[iOff], &aPg[iNextOff], nSuffix2); + memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); iOff += nSuffix2; iNextOff += nSuffix2; } diff --git a/ext/fts5/test/fts5secure4.test b/ext/fts5/test/fts5secure4.test index 48d9242b6b..c2fd369958 100644 --- a/ext/fts5/test/fts5secure4.test +++ b/ext/fts5/test/fts5secure4.test @@ -53,6 +53,23 @@ do_execsql_test 1.6 { INSERT INTO t1(t1) VALUES('integrity-check'); } +do_execsql_test 1.7 { +CREATE VIRTUAL TABLE y1 USING fts5(xx, prefix='1,2'); +INSERT INTO y1(y1, rank) VALUES('pgsz', 64); +INSERT INTO y1(y1, rank) VALUES('secure-delete', 1); +} +do_execsql_test 1.8 { + BEGIN; + INSERT INTO y1(rowid, xx) VALUES(1, 'abc def'); + INSERT INTO y1(rowid, xx) VALUES(2, 'reallyreallylongtoken'); + COMMIT; +} +do_execsql_test 1.9 { + DELETE FROM y1 WHERE rowid=1; + INSERT INTO y1(y1) VALUES('integrity-check'); +} + + #------------------------------------------------------------------------- # Test using secure-delete with detail=none or detail=col. # @@ -132,5 +149,6 @@ foreach {tn d} {1 full 2 none 3 column} { } + finish_test diff --git a/manifest b/manifest index 22c4bbb0d1..d6fb6fb3be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\suse-after-free\sthat\scould\sfollow\san\sOOM\sor\sIO\serror\sin\sfts5\swhen\susing\s'secure-delete'\smode. -D 2023-04-15T19:58:58.601 +C Change\sa\smemcpy()\sto\smemmove()\sin\snew\sfts5\scode. +D 2023-04-15T21:00:55.998 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 50c0e72697c310011b1520428655e79602bb92536bd7851d40f03349c21d8863 +F ext/fts5/fts5_index.c 3d50e647b85c3563ba651eb241d7d7cbf52ebd0e8ec20480eaa8526ef67f964c F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -190,7 +190,7 @@ F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e33 F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 -F ext/fts5/test/fts5secure4.test 17ac602024db3953401c3e51ad8e03de3b972c1770554585f63812ca4083550e +F ext/fts5/test/fts5secure4.test 1bc36e827093c0f5b93bd4bad3c6ec8caaa9f7ab830633b49a465f67d48f3e7c F ext/fts5/test/fts5securefault.test 7208daed4171de8a54a293ef800411e3cec1ffc6e9d34209b53e56a70ee554da F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ee24a499c7b4a048f269f1c03c7c0b22b2898a3c23ea3e2511c2ba0342afa30 -R 7d4e8ae6ce7108b997bebd6ab3f8177f +P 904be83d9f3dd4b21a3aa47dde337b5c045b925ffdde4bccd1f8e8fdbef68c18 +R 42a22772112d38795073bd5ae2e55e02 U dan -Z 0a97f4e707f53c68d15ac47e51716a58 +Z 70b2307a762c2e6a4eaceb449e3ddd17 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9a94488d87..d1cd282ed6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -904be83d9f3dd4b21a3aa47dde337b5c045b925ffdde4bccd1f8e8fdbef68c18 \ No newline at end of file +8de4542553ae20947787b4daf5e88bbe69f815679960e700e19f9c1148234b08 \ No newline at end of file From 5c45a9fbaf071975b672a8eed76cd0ac659ca331 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Apr 2023 10:39:51 +0000 Subject: [PATCH 226/341] Fix an assert() in fts5 that may be true when dealing with corrupt records. FossilOrigin-Name: e8de2774463c07b76cd6b9a44ab32a583fe973f42a21b21af46fe82ab177c1b8 --- ext/fts5/fts5_index.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index adcb7045bc..b93957bbd0 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4903,7 +4903,7 @@ static void fts5FlushSecureDelete( fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); } - assert( nPg>4 || fts5GetU16(aPg)==0 ); + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid, pSeg->iLeafPgno), aPg, nPg); sqlite3_free(aIdx); } diff --git a/manifest b/manifest index d6fb6fb3be..5c33c95e0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\smemcpy()\sto\smemmove()\sin\snew\sfts5\scode. -D 2023-04-15T21:00:55.998 +C Fix\san\sassert()\sin\sfts5\sthat\smay\sbe\strue\swhen\sdealing\swith\scorrupt\srecords. +D 2023-04-17T10:39:51.274 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 3d50e647b85c3563ba651eb241d7d7cbf52ebd0e8ec20480eaa8526ef67f964c +F ext/fts5/fts5_index.c 0fe4bf689b04c9a662bacd6936f7f07926eb61cf2024c8d4c8241d3bd3efe723 F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 904be83d9f3dd4b21a3aa47dde337b5c045b925ffdde4bccd1f8e8fdbef68c18 -R 42a22772112d38795073bd5ae2e55e02 +P 8de4542553ae20947787b4daf5e88bbe69f815679960e700e19f9c1148234b08 +R afbeea9d1c47bed909416ab95ba6cabc U dan -Z 70b2307a762c2e6a4eaceb449e3ddd17 +Z 13440c307a288857a675d3a4be022dc6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d1cd282ed6..75177acf32 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8de4542553ae20947787b4daf5e88bbe69f815679960e700e19f9c1148234b08 \ No newline at end of file +e8de2774463c07b76cd6b9a44ab32a583fe973f42a21b21af46fe82ab177c1b8 \ No newline at end of file From 48824f23cbc68b60f337127578bb2c5c7712b340 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Apr 2023 11:41:42 +0000 Subject: [PATCH 227/341] Fix problems handling corrupt records in new fts5 code. FossilOrigin-Name: 2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900 --- ext/fts5/fts5_index.c | 88 ++++++++++++++++++---------------- ext/fts5/test/fts5secure4.test | 14 ++++++ manifest | 14 +++--- manifest.uuid | 2 +- 4 files changed, 69 insertions(+), 49 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index b93957bbd0..c51d82be99 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4509,16 +4509,16 @@ static void fts5IndexCrisismerge( ){ const int nCrisis = p->pConfig->nCrisisMerge; Fts5Structure *pStruct = *ppStruct; - int iLvl = 0; - - assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); - while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ - fts5IndexMergeLevel(p, &pStruct, iLvl, 0); - assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); - fts5StructurePromote(p, iLvl+1, pStruct); - iLvl++; + if( pStruct && pStruct->nLevel>0 ){ + int iLvl = 0; + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; } - *ppStruct = pStruct; } static int fts5IndexReturn(Fts5Index *p){ @@ -4823,17 +4823,21 @@ static void fts5FlushSecureDelete( nPrefix = MIN(nPrefix, nPrefix2); nSuffix = (nPrefix2 + nSuffix2) - nPrefix; - if( iKey!=1 ){ - iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + p->rc = FTS5_CORRUPT; + }else{ + if( iKey!=1 ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + } + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); + if( nPrefix2>nPrefix ){ + memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix); + iOff += (nPrefix2-nPrefix); + } + memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); + iOff += nSuffix2; + iNextOff += nSuffix2; } - iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); - if( nPrefix2>nPrefix ){ - memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix); - iOff += (nPrefix2-nPrefix); - } - memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); - iOff += nSuffix2; - iNextOff += nSuffix2; } }else if( iStart==4 ){ assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); @@ -4878,33 +4882,35 @@ static void fts5FlushSecureDelete( } } - nMove = nPg - iNextOff; - memmove(&aPg[iOff], &aPg[iNextOff], nMove); - iPgIdx -= (iNextOff - iOff); - nPg = iPgIdx; - fts5PutU16(&aPg[2], iPgIdx); + if( p->rc==SQLITE_OK ){ + nMove = nPg - iNextOff; + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= (iNextOff - iOff); + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); - nShift = iNextOff - iOff; - for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ - iKeyOff -= nShift; - nShift = 0; + nShift = iNextOff - iOff; + for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ + iKeyOff -= nShift; + nShift = 0; + } + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); + iPrevKeyOff = iKeyOff; } - nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); - iPrevKeyOff = iKeyOff; } - } - if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); - } + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + } - assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); - fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid, pSeg->iLeafPgno), aPg, nPg); + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg); + } sqlite3_free(aIdx); } } diff --git a/ext/fts5/test/fts5secure4.test b/ext/fts5/test/fts5secure4.test index c2fd369958..7cfc5a73e9 100644 --- a/ext/fts5/test/fts5secure4.test +++ b/ext/fts5/test/fts5secure4.test @@ -69,6 +69,20 @@ do_execsql_test 1.9 { INSERT INTO y1(y1) VALUES('integrity-check'); } +do_execsql_test 1.10 { + CREATE VIRTUAL TABLE w1 USING fts5(ww, content=""); + INSERT INTO w1(rowid, ww) VALUES(123, ''); +} +do_catchsql_test 1.11 { + INSERT INTO w1(w1, rowid, ww) VALUES('delete', 123, 'xyz'); +} {1 {database disk image is malformed}} +do_catchsql_test 1.12 { + DROP TABLE w1; + CREATE VIRTUAL TABLE w1 USING fts5(ww, content=""); + INSERT INTO w1(rowid, ww) VALUES(123, ''); + DELETE FROM w1_data WHERE id>10; + INSERT INTO w1(w1, rowid, ww) VALUES('delete', 123, 'xyz'); +} {1 {database disk image is malformed}} #------------------------------------------------------------------------- # Test using secure-delete with detail=none or detail=col. diff --git a/manifest b/manifest index 5c33c95e0e..26932a3817 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sin\sfts5\sthat\smay\sbe\strue\swhen\sdealing\swith\scorrupt\srecords. -D 2023-04-17T10:39:51.274 +C Fix\sproblems\shandling\scorrupt\srecords\sin\snew\sfts5\scode. +D 2023-04-17T11:41:42.438 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 0fe4bf689b04c9a662bacd6936f7f07926eb61cf2024c8d4c8241d3bd3efe723 +F ext/fts5/fts5_index.c 13c4ea17f9e6b155d3c6d78b54ae4e5ed22e1692041958f3ec71a5712172d915 F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -190,7 +190,7 @@ F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e33 F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 -F ext/fts5/test/fts5secure4.test 1bc36e827093c0f5b93bd4bad3c6ec8caaa9f7ab830633b49a465f67d48f3e7c +F ext/fts5/test/fts5secure4.test 58638eab7e80571649d08673e7924d85ec293566377a9e7f49160857f001d529 F ext/fts5/test/fts5securefault.test 7208daed4171de8a54a293ef800411e3cec1ffc6e9d34209b53e56a70ee554da F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 @@ -2057,8 +2057,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8de4542553ae20947787b4daf5e88bbe69f815679960e700e19f9c1148234b08 -R afbeea9d1c47bed909416ab95ba6cabc +P e8de2774463c07b76cd6b9a44ab32a583fe973f42a21b21af46fe82ab177c1b8 +R e73db0b2fec49cf21caca2b40145b9f4 U dan -Z 13440c307a288857a675d3a4be022dc6 +Z 8d23b3253d2a52bc5cc6c258ff86d3c2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 75177acf32..5bab2e43f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e8de2774463c07b76cd6b9a44ab32a583fe973f42a21b21af46fe82ab177c1b8 \ No newline at end of file +2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900 \ No newline at end of file From 68c0da6f66f6468aae09f7976ecf95cdeebca4e6 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Apr 2023 18:32:22 +0000 Subject: [PATCH 228/341] Fix some problems with using fts5 options 'secure-delete' and detail=none together. FossilOrigin-Name: 4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed --- ext/fts5/fts5_index.c | 478 ++++++++++++++++++--------------- ext/fts5/test/fts5secure4.test | 14 +- ext/fts5/test/fts5secure5.test | 129 +++++++++ manifest | 15 +- manifest.uuid | 2 +- 5 files changed, 410 insertions(+), 228 deletions(-) create mode 100644 ext/fts5/test/fts5secure5.test diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index c51d82be99..9c14023889 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1954,7 +1954,7 @@ static void fts5SegIterNext_None( iOff = pIter->iLeafOffset; /* Next entry is on the next page */ - if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( p->rc || pIter->pLeaf==0 ) return; pIter->iRowid = 0; @@ -4601,6 +4601,7 @@ static void fts5SecureDeleteOverflow( int iPgno, int *pbLastInDoclist ){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); int pgno; Fts5Data *pLeaf = 0; assert( iPgno!=1 ); @@ -4627,9 +4628,12 @@ static void fts5SecureDeleteOverflow( /* The page contains no terms or rowids. Replace it with an empty ** page and move on to the right-hand peer. */ const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; - fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty)); + assert_nc( bDetailNone==0 || pLeaf->nn==4 ); + if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty)); fts5DataRelease(pLeaf); pLeaf = 0; + }else if( bDetailNone ){ + break; }else{ int nShift = iNext - 4; int nPg; @@ -4678,6 +4682,245 @@ static void fts5SecureDeleteOverflow( fts5DataRelease(pLeaf); } +/* +** Completely remove the entry that pSeg currently points to from +** the database. +*/ +static void fts5DoSecureDelete( + Fts5Index *p, + Fts5Structure *pStruct, + Fts5SegIter *pSeg +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int iSegid = pSeg->pSeg->iSegid; + u8 *aPg = pSeg->pLeaf->p; + int nPg = pSeg->pLeaf->nn; + int iPgIdx = pSeg->pLeaf->szLeaf; + + u64 iDelta = 0; + u64 iNextDelta = 0; + int iNextOff = 0; + int iOff = 0; + int nIdx = 0; + u8 *aIdx = 0; + int bLastInDoclist = 0; + int iIdx = 0; + int iStart = 0; + int iKeyOff = 0; + int iPrevKeyOff = 0; + int iDelKeyOff = 0; /* Offset of deleted key, if any */ + + nIdx = nPg-iPgIdx; + aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + if( p->rc ) return; + memcpy(aIdx, &aPg[iPgIdx], nIdx); + + /* At this point segment iterator pSeg points to the entry + ** this function should remove from the b-tree segment. + ** + ** In detail=full or detail=column mode, pSeg->iLeafOffset is the + ** offset of the first byte in the position-list for the entry to + ** remove. Immediately before this comes two varints that will also + ** need to be removed: + ** + ** + the rowid or delta rowid value for the entry, and + ** + the size of the position list in bytes. + ** + ** Or, in detail=none mode, there is a single varint prior to + ** pSeg->iLeafOffset - the rowid or delta rowid value. + ** + ** This block sets the following variables: + ** + ** iStart: + ** iDelta: + */ + { + int iSOP; + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ + iStart = pSeg->iTermLeafOffset; + }else{ + iStart = fts5GetU16(&aPg[0]); + } + + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + assert_nc( iSOP<=pSeg->iLeafOffset ); + + if( bDetailNone ){ + while( iSOPiLeafOffset ){ + if( aPg[iSOP]==0x00 ) iSOP++; + if( aPg[iSOP]==0x00 ) iSOP++; + iStart = iSOP; + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + } + + iNextOff = iSOP; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + + }else{ + int nPos = 0; + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + while( iSOPiLeafOffset ){ + iStart = iSOP + (nPos/2); + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + } + assert_nc( iSOP==pSeg->iLeafOffset ); + iNextOff = pSeg->iLeafOffset + pSeg->nPos; + } + } + + iOff = iStart; + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + }else{ + /* Set bLastInDoclist to true if the entry being removed is the last + ** in its doclist. */ + for(iIdx=0, iKeyOff=0; iIdxiTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno + ){ + /* The entry being removed was the only position list in its + ** doclist. Therefore the term needs to be removed as well. */ + int iKey = 0; + for(iIdx=0, iKeyOff=0; iIdxiStart ) break; + iKeyOff += iVal; + } + + iDelKeyOff = iOff = iKeyOff; + if( iNextOff!=iPgIdx ){ + int nPrefix = 0; + int nSuffix = 0; + int nPrefix2 = 0; + int nSuffix2 = 0; + + iDelKeyOff = iNextOff; + iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + + if( iKey!=1 ){ + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + } + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + + nPrefix = MIN(nPrefix, nPrefix2); + nSuffix = (nPrefix2 + nSuffix2) - nPrefix; + + if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + p->rc = FTS5_CORRUPT; + }else{ + if( iKey!=1 ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + } + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); + if( nPrefix2>nPrefix ){ + memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); + iOff += (nPrefix2-nPrefix); + } + memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); + iOff += nSuffix2; + iNextOff += nSuffix2; + } + } + }else if( iStart==4 ){ + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + int iPgno = pSeg->iLeafPgno-1; + + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; + + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } + } + fts5DataRelease(pTerm); + } + } + + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; + int nShift = 0; + + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= (iNextOff - iOff); + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); + + nShift = iNextOff - iOff; + for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ + iKeyOff -= nShift; + nShift = 0; + } + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); + iPrevKeyOff = iKeyOff; + } + } + + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + } + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg); + } + sqlite3_free(aIdx); +} /* ** This is called as part of flushing a delete to disk in 'secure-delete' @@ -4706,213 +4949,7 @@ static void fts5FlushSecureDelete( && iRowid==fts5MultiIterRowid(pIter) ){ Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; - int iSegid = pSeg->pSeg->iSegid; - u8 *aPg = pSeg->pLeaf->p; - int nPg = pSeg->pLeaf->nn; - int iPgIdx = pSeg->pLeaf->szLeaf; - - u64 iDelta = 0; - u64 iNextDelta = 0; - int iNextOff = 0; - int iOff = 0; - int nMove = 0; - int nIdx = 0; - - u8 *aIdx = 0; - - nIdx = nPg-iPgIdx; - aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); - if( aIdx ){ - int bLastInDoclist = 0; - int iIdx = 0; - int iStart = 0; - int iKeyOff = 0; - int iPrevKeyOff = 0; - int nShift = 0; - int iDelKeyOff = 0; /* Offset of deleted key, if any */ - memcpy(aIdx, &aPg[iPgIdx], nIdx); - - /* At this point segment iterator pSeg points to the entry - ** this function should remove from the b-tree segment. - ** - ** More specifically, pSeg->iLeafOffset is the offset of the - ** first byte in the position-list for the entry to remove. - ** Immediately before this comes two varints that will also - ** need to be removed: - ** - ** + the rowid or delta rowid value for the entry, and - ** + the size of the position list in bytes. - */ - { - int iSOP; - int nPos = 0; - if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ - iStart = pSeg->iTermLeafOffset; - }else{ - iStart = fts5GetU16(&aPg[0]); - } - - iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); - iSOP += fts5GetVarint32(&aPg[iSOP], nPos); - assert_nc( iSOP<=pSeg->iLeafOffset ); - while( iSOPiLeafOffset ){ - iStart = iSOP + (nPos/2); - iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); - iSOP += fts5GetVarint32(&aPg[iSOP], nPos); - } - assert_nc( iSOP==pSeg->iLeafOffset ); - } - - iOff = iStart; - iNextOff = pSeg->iLeafOffset + pSeg->nPos; - if( iNextOff>=iPgIdx ){ - int pgno = pSeg->iLeafPgno+1; - fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); - iNextOff = iPgIdx; - }else{ - /* Set bLastInDoclist to true if the entry being removed is the last - ** in its doclist. */ - for(iIdx=0, iKeyOff=0; iIdxiTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno - ){ - /* The entry being removed was the only position list in its - ** doclist. Therefore the term needs to be removed as well. */ - int iKey = 0; - for(iIdx=0, iKeyOff=0; iIdxiStart ) break; - iKeyOff += iVal; - } - - iDelKeyOff = iOff = iKeyOff; - if( iNextOff!=iPgIdx ){ - int nPrefix = 0; - int nSuffix = 0; - int nPrefix2 = 0; - int nSuffix2 = 0; - - iDelKeyOff = iNextOff; - iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); - iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); - - if( iKey!=1 ){ - iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); - } - iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); - - nPrefix = MIN(nPrefix, nPrefix2); - nSuffix = (nPrefix2 + nSuffix2) - nPrefix; - - if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ - p->rc = FTS5_CORRUPT; - }else{ - if( iKey!=1 ){ - iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); - } - iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); - if( nPrefix2>nPrefix ){ - memcpy(&aPg[iOff], &zTerm[nPrefix], nPrefix2-nPrefix); - iOff += (nPrefix2-nPrefix); - } - memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); - iOff += nSuffix2; - iNextOff += nSuffix2; - } - } - }else if( iStart==4 ){ - assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); - /* The entry being removed may be the only position list in - ** its doclist. */ - int iPgno = pSeg->iLeafPgno-1; - - for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ - Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); - int bEmpty = (pPg && pPg->nn==4); - fts5DataRelease(pPg); - if( bEmpty==0 ) break; - } - - if( iPgno==pSeg->iTermLeafPgno ){ - i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); - Fts5Data *pTerm = fts5DataRead(p, iId); - if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ - u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; - int nTermIdx = pTerm->nn - pTerm->szLeaf; - int iTermIdx = 0; - int iTermOff = 0; - - while( 1 ){ - u32 iVal = 0; - int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); - iTermOff += iVal; - if( (iTermIdx+nByte)>=nTermIdx ) break; - iTermIdx += nByte; - } - nTermIdx = iTermIdx; - - memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); - fts5PutU16(&pTerm->p[2], iTermOff); - - fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); - if( nTermIdx==0 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); - } - } - fts5DataRelease(pTerm); - } - } - - if( p->rc==SQLITE_OK ){ - nMove = nPg - iNextOff; - memmove(&aPg[iOff], &aPg[iNextOff], nMove); - iPgIdx -= (iNextOff - iOff); - nPg = iPgIdx; - fts5PutU16(&aPg[2], iPgIdx); - - nShift = iNextOff - iOff; - for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ - iKeyOff -= nShift; - nShift = 0; - } - nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); - iPrevKeyOff = iKeyOff; - } - } - - if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); - } - - assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); - fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg); - } - sqlite3_free(aIdx); - } + fts5DoSecureDelete(p, pStruct, pSeg); } } @@ -4998,11 +5035,24 @@ static void fts5FlushOneHash(Fts5Index *p){ /* If in secure delete mode, and if this entry in the poslist is ** in fact a delete, then edit the existing segments directly ** using fts5FlushSecureDelete(). */ - if( bSecureDelete && (pDoclist[iOff] & 0x01) ){ - fts5FlushSecureDelete(p, pStruct, zTerm, iRowid); - if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ - iOff++; - continue; + if( bSecureDelete ){ + if( eDetail==FTS5_DETAIL_NONE ){ + if( iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ + iOff++; + continue; + } } } diff --git a/ext/fts5/test/fts5secure4.test b/ext/fts5/test/fts5secure4.test index 7cfc5a73e9..7588a34683 100644 --- a/ext/fts5/test/fts5secure4.test +++ b/ext/fts5/test/fts5secure4.test @@ -90,18 +90,20 @@ do_catchsql_test 1.12 { foreach {tn d} {1 full 2 none 3 column} { reset_db do_execsql_test 2.$tn.1 " - CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='1,2'); + CREATE VIRTUAL TABLE x1 USING fts5(xx, yy, zz, detail=$d, prefix='10,20'); INSERT INTO x1(x1, rank) VALUES('pgsz', 64); + INSERT INTO x1(x1, rank) VALUES('secure-delete', 1); " do_execsql_test 2.$tn.2 { BEGIN; - INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); - INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); - INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); - INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); - INSERT INTO x1 VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); + INSERT INTO x1(xx, yy, zz) VALUES('a b c', 'd e f', 'a b c'); COMMIT; + INSERT INTO x1(x1) VALUES('integrity-check'); } do_execsql_test 2.$tn.3 { diff --git a/ext/fts5/test/fts5secure5.test b/ext/fts5/test/fts5secure5.test new file mode 100644 index 0000000000..ca8570211c --- /dev/null +++ b/ext/fts5/test/fts5secure5.test @@ -0,0 +1,129 @@ +# 2023 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +return_if_no_fts5 +set ::testprefix fts5secure5 +return_if_no_fts5 + +proc dump {} { + execsql_pp { + SELECT id, quote(block), fts5_decode_none(id, block) FROM ft1_data + } +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO ft1(rowid, a) VALUES(1, 'abcd'); + INSERT INTO ft1(rowid, a) VALUES(2, 'abcd'); + INSERT INTO ft1(rowid, a) VALUES(3, 'abcd'); + COMMIT; +} +do_execsql_test 1.2 { + DELETE FROM ft1 WHERE rowid=1; +} +do_execsql_test 1.3 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 1.4 { + DELETE FROM ft1 WHERE rowid=3; +} +do_execsql_test 1.5 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 1.6 { + DELETE FROM ft1 WHERE rowid=3; +} +do_execsql_test 1.7 { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); +} + +do_execsql_test 2.1 { + BEGIN; + INSERT INTO ft1(rowid, a) VALUES(1, 'abcd one'); + INSERT INTO ft1(rowid, a) VALUES(2, 'abcd two'); + INSERT INTO ft1(rowid, a) VALUES(3, 'abcd two'); + INSERT INTO ft1(rowid, a) VALUES(4, 'abcd two'); + INSERT INTO ft1(rowid, a) VALUES(5, 'abcd three'); + COMMIT; +} + +do_execsql_test 2.2a { + DELETE FROM ft1 WHERE rowid=3; +} +do_execsql_test 2.2b { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 2.3a { + DELETE FROM ft1 WHERE rowid=2; +} +do_execsql_test 2.3b { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} +do_execsql_test 2.4a { + DELETE FROM ft1 WHERE rowid=4; +} +do_execsql_test 2.4b { + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none, prefix=1); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); + INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); +} +do_execsql_test 3.1 { + BEGIN; + INSERT INTO ft1(a) VALUES('c'); + COMMIT; +} +do_execsql_test 3.2 { + DELETE FROM ft1 WHERE rowid IN (1); + INSERT INTO ft1(ft1) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE ft1 USING fts5(a, detail=none); + INSERT INTO ft1(ft1, rank) VALUES('secure-delete', 1); + INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500 + ) + INSERT INTO ft1 SELECT 'abcdefg' FROM s; +} + +do_test 4.1 { + for {set i 500} {$i > 0} {incr i -1} { + execsql { DELETE FROM ft1 WHERE rowid=$i } + execsql { INSERT INTO ft1(ft1) VALUES('integrity-check') } + } +} {} + +finish_test + diff --git a/manifest b/manifest index 26932a3817..010a7b6132 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\shandling\scorrupt\srecords\sin\snew\sfts5\scode. -D 2023-04-17T11:41:42.438 +C Fix\ssome\sproblems\swith\susing\sfts5\soptions\s'secure-delete'\sand\sdetail=none\stogether. +D 2023-04-17T18:32:22.329 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 13c4ea17f9e6b155d3c6d78b54ae4e5ed22e1692041958f3ec71a5712172d915 +F ext/fts5/fts5_index.c 2a59602a7ea830ae40b742277f4d89cae60a646bb2fb9c2ce895aea5e7ca89ce F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -190,7 +190,8 @@ F ext/fts5/test/fts5savepoint.test fc02929f238d02a22df4172625704e029f7c1e0e92e33 F ext/fts5/test/fts5secure.test 214a561519d1b1817f146efd1057e2a97cc896e75c2accc77157d874154bda64 F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd5e5e620bdf1ba6bbc F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 -F ext/fts5/test/fts5secure4.test 58638eab7e80571649d08673e7924d85ec293566377a9e7f49160857f001d529 +F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97 +F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a F ext/fts5/test/fts5securefault.test 7208daed4171de8a54a293ef800411e3cec1ffc6e9d34209b53e56a70ee554da F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 @@ -2057,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e8de2774463c07b76cd6b9a44ab32a583fe973f42a21b21af46fe82ab177c1b8 -R e73db0b2fec49cf21caca2b40145b9f4 +P 2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900 +R 94f7d6e08b83dc7ea2e9a235119af696 U dan -Z 8d23b3253d2a52bc5cc6c258ff86d3c2 +Z 7915ab356b23565aa915e2ae0ad516a8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5bab2e43f5..f88d210269 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f63d4a8bac6dc72197df32efd84c148adbc4794943088dff08da57b15dd7900 \ No newline at end of file +4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed \ No newline at end of file From acb4050cfc84293ead68c6f4ff7ac3a99d5582a9 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 17 Apr 2023 22:28:02 +0000 Subject: [PATCH 229/341] Minor coding style changes. FossilOrigin-Name: 481aa3ccf6f12c2c2fd11a1a20d8fb59f4f9b1cd8dcff0586b4f52222463d720 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 29 ++++++++++++++++------------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 17cad130ed..9315de8d47 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sCLI\s-utf8\sactive,\sdo\sno\stranslation\sto\sMBCS\sfor\soutput\sto\sWindows\sconsole. -D 2023-04-17T21:18:53.516 +C Minor\scoding\sstyle\schanges. +D 2023-04-17T22:28:02.543 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -632,7 +632,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ad6f48e0953fb0d9dd434a7fcd8bc6bde05beb82500a59fea7a7b02dc0ecfb3b -F src/shell.c.in 6269bf25a222099205c25d4e3917493ef076586064fff0c7b2c33395377b3740 +F src/shell.c.in 2a11419d77d8e4f7b4e3aea6922a9560630889423e3728205d77b4be2511cc89 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 543594a7277b12d1c58af8e3cba3841667cf924172995536489ce7da704a11d7 394980e4fe12125460ab14da41edae9089a4da332a46f3124bf0b9778793791f 4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed -R a35a825488de1096cbecfb4e55ef07c0 -U larrybr -Z cc0b8fd4da0e661d2adcb75f4dd54fc5 +P cc1d4296d71ee6e2ba133edc581385e201c89657911c2ed39108843c776633a4 +R fffc27f23894cd7d883f97c0e1d18fe7 +U mistachkin +Z 657aeeb381d4d1312977c18f5409a163 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7079f75b36..782d2e8ed3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc1d4296d71ee6e2ba133edc581385e201c89657911c2ed39108843c776633a4 \ No newline at end of file +481aa3ccf6f12c2c2fd11a1a20d8fb59f4f9b1cd8dcff0586b4f52222463d720 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 0afc63d6b6..c8de494fe9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -613,20 +613,22 @@ static struct ConsoleState { */ static void console_prepare(void){ HANDLE hCI = GetStdHandle(STD_INPUT_HANDLE); + DWORD consoleMode = 0; if( isatty(0) && GetFileType(hCI)==FILE_TYPE_CHAR - && GetConsoleMode( hCI, &conState.consoleMode) ){ + && GetConsoleMode( hCI, &consoleMode) ){ if( !IsValidCodePage(CP_UTF8) ){ fprintf(stderr, "Cannot use UTF-8 code page.\n"); console_utf8 = 0; return; } conState.hConsoleIn = hCI; + conState.consoleMode = consoleMode; conState.inCodePage = GetConsoleCP(); conState.outCodePage = GetConsoleOutputCP(); SetConsoleCP(CP_UTF8); SetConsoleOutputCP(CP_UTF8); - SetConsoleMode( conState.hConsoleIn, - conState.consoleMode | ENABLE_LINE_INPUT ); + SetConsoleMode(conState.hConsoleIn, + conState.consoleMode | ENABLE_LINE_INPUT); conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT); console_utf8 = 1; }else{ @@ -639,13 +641,14 @@ static void console_prepare(void){ */ static void SQLITE_CDECL console_restore(void){ if( console_utf8 && conState.inCodePage!=0 - && conState.hConsoleIn != INVALID_HANDLE_VALUE ){ + && conState.hConsoleIn!=INVALID_HANDLE_VALUE ){ _setmode(_fileno(stdin), conState.infsMode); SetConsoleCP(conState.inCodePage); SetConsoleOutputCP(conState.outCodePage); - SetConsoleMode( conState.hConsoleIn, conState.consoleMode ); + SetConsoleMode(conState.hConsoleIn, conState.consoleMode); /* Avoid multiple calls. */ conState.hConsoleIn = INVALID_HANDLE_VALUE; + conState.consoleMode = 0; console_utf8 = 0; } } @@ -663,9 +666,9 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ wchar_t wbuf[SQLITE_IALIM]; int lend = 0; int noc = 0; - if( ncmax == 0 || conState.stdinEof ) return 0; + if( ncmax==0 || conState.stdinEof ) return 0; buf[0] = 0; - while( noc < ncmax-7-1 && !lend ){ + while( noc SQLITE_IALIM*4+1 + noc) ? SQLITE_IALIM : (ncmax-1 - noc)/4; @@ -674,12 +677,12 @@ static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0); if( !bRC || (noc==0 && nbr==0) ) return 0; if( nbr > 0 ){ - int nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR, - wbuf,nbr, 0,0, 0, 0); + int nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, + wbuf,nbr,0,0,0,0); if( nmb !=0 && noc+nmb <= ncmax ){ int iseg = noc; - nmb = WideCharToMultiByte(CP_UTF8, WC_COMPOSITECHECK|WC_DEFAULTCHAR, - wbuf,nbr, buf+noc,nmb, 0,0); + nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, + wbuf,nbr,buf+noc,nmb,0,0); noc += nmb; /* Fixup line-ends as coded by Windows for CR (or "Enter".)*/ if( noc > 0 ){ @@ -730,7 +733,7 @@ void utf8_printf(FILE *out, const char *zFormat, ...){ # if SHELL_WIN_UTF8_OPT && !console_utf8 # endif - ){ + ){ char *z1 = sqlite3_vmprintf(zFormat, ap); char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); sqlite3_free(z1); @@ -948,7 +951,7 @@ static char *local_getline(char *zLine, FILE *in){ # if SHELL_WIN_UTF8_OPT && !console_utf8 # endif /* SHELL_WIN_UTF8_OPT */ - ){ + ){ char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); if( zTrans ){ i64 nTrans = strlen(zTrans)+1; From a5eab9579acafb9a1411b60c5f3f1070ba8c70f9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 18 Apr 2023 11:33:35 +0000 Subject: [PATCH 230/341] Fix an assert() in fts5 that could fail following an OOM or IO error. FossilOrigin-Name: 49595083ba58e13d404b6a11c28f53768a257c59f3f36be3591438127df7e875 --- ext/fts5/fts5_index.c | 2 +- ext/fts5/test/fts5securefault.test | 25 +++++++++++++++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 9c14023889..186c421af8 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5059,7 +5059,7 @@ static void fts5FlushOneHash(Fts5Index *p){ if( p->rc==SQLITE_OK && bTermWritten==0 ){ fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); bTermWritten = 1; - assert( writer.bFirstRowidInPage==0 ); + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); } if( writer.bFirstRowidInPage ){ diff --git a/ext/fts5/test/fts5securefault.test b/ext/fts5/test/fts5securefault.test index 09e44b0903..63874ece5d 100644 --- a/ext/fts5/test/fts5securefault.test +++ b/ext/fts5/test/fts5securefault.test @@ -196,5 +196,30 @@ do_faultsim_test 6.1 -faults oom* -prep { faultsim_test_result {0 {}} } +#------------------------------------------------------------------------- +# +reset_db +do_test 7.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(ab); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } +} {} +faultsim_save_and_close + +do_faultsim_test 7.1 -faults oom* -prep { + faultsim_restore_and_reopen + set big1 "[string repeat x 50] [string repeat y 50] [string repeat z 50]" + execsql { + BEGIN; + INSERT INTO t1 VALUES($big1); + } +} -body { + execsql { COMMIT } +} -test { + faultsim_test_result {0 {}} +} + finish_test diff --git a/manifest b/manifest index ad783c9fb2..dc94a74a86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sCLI\s-utf8\sactive,\sdo\sno\stranslation\sto\sMBCS\sfor\soutput\sto\sWindows\sconsole.\splus\scode\sstyle\simprovements -D 2023-04-17T23:59:15.626 +C Fix\san\sassert()\sin\sfts5\sthat\scould\sfail\sfollowing\san\sOOM\sor\sIO\serror. +D 2023-04-18T11:33:35.337 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 2a59602a7ea830ae40b742277f4d89cae60a646bb2fb9c2ce895aea5e7ca89ce +F ext/fts5/fts5_index.c 17dca8e874df04182bc45063dc0b761acc242b91f1264d2257b7e37bd4e4c2ad F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -192,7 +192,7 @@ F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a -F ext/fts5/test/fts5securefault.test 7208daed4171de8a54a293ef800411e3cec1ffc6e9d34209b53e56a70ee554da +F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 F ext/fts5/test/fts5simple3.test d5c74a9d3ca71bd5dd5cacb7c55b86ea12cdddfc8b1910e3de2995206898380f @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d3f27ba90aa59837e49ceaae9f36cad426af3a33707d85d1bb848fc1bda6fed 481aa3ccf6f12c2c2fd11a1a20d8fb59f4f9b1cd8dcff0586b4f52222463d720 -R fffc27f23894cd7d883f97c0e1d18fe7 -U larrybr -Z 102eab70b9a2d815367ccdeebebab9e6 +P 25edf6089724bf9f97c796e5abf838ec508ebc132895d8325620c16d63996fef +R 2bb451a8ca6c1fc68e0f9e87de9c4cb2 +U dan +Z 766d0802e43dc7bbd454875eec831ea6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0546af7fda..dc2048f98b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25edf6089724bf9f97c796e5abf838ec508ebc132895d8325620c16d63996fef \ No newline at end of file +49595083ba58e13d404b6a11c28f53768a257c59f3f36be3591438127df7e875 \ No newline at end of file From 053a29c29124334f8664d77efef3b1b27c854dab Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 18 Apr 2023 11:35:09 +0000 Subject: [PATCH 231/341] Add the --fullsync option to speedtest1. FossilOrigin-Name: 960fe54c8227a376c97bb2a3ba604d97e79e27e51a9798be9d318cf56c04fa99 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/speedtest1.c | 10 +++++++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index dc94a74a86..c6fee5bf84 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sin\sfts5\sthat\scould\sfail\sfollowing\san\sOOM\sor\sIO\serror. -D 2023-04-18T11:33:35.337 +C Add\sthe\s--fullsync\soption\sto\sspeedtest1. +D 2023-04-18T11:35:09.601 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1531,7 +1531,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 377a0c48e5a92e0b11c1c5ebb1bc9d83a7312c922bc0cb05970ef5d6a96d1f0c -F test/speedtest1.c 4a883fe277c289161f84f8c805a38f78bfbd9515eb417071f9bce817c2ca8756 +F test/speedtest1.c 9703465daf778dd90bad7adadc1d2ee205ec90a79437cababab0a4111190f0c8 F test/spellfix.test 951a6405d49d1a23d6b78027d3877b4a33eeb8221dcab5704b499755bb4f552e F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 25edf6089724bf9f97c796e5abf838ec508ebc132895d8325620c16d63996fef -R 2bb451a8ca6c1fc68e0f9e87de9c4cb2 -U dan -Z 766d0802e43dc7bbd454875eec831ea6 +P 49595083ba58e13d404b6a11c28f53768a257c59f3f36be3591438127df7e875 +R cc728f1caef69a29ba8fe0bb9cbdc7a1 +U drh +Z a2caae4ae9157cd4de2a01e274c37a8f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dc2048f98b..4bc144f09b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49595083ba58e13d404b6a11c28f53768a257c59f3f36be3591438127df7e875 \ No newline at end of file +960fe54c8227a376c97bb2a3ba604d97e79e27e51a9798be9d318cf56c04fa99 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 1dd3bd5fb3..6aff89b376 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -12,6 +12,7 @@ static const char zHelp[] = " --checkpoint Run PRAGMA wal_checkpoint after each test case\n" " --exclusive Enable locking_mode=EXCLUSIVE\n" " --explain Like --sqlonly but with added EXPLAIN keywords\n" + " --fullfsync Enable fullfsync=TRUE\n" " --heap SZ MIN Memory allocator uses SZ bytes & min allocation MIN\n" " --incrvacuum Enable incremenatal vacuum mode\n" " --journal M Set the journal_mode to M\n" @@ -2203,6 +2204,7 @@ int main(int argc, char **argv){ int doAutovac = 0; /* True for --autovacuum */ int cacheSize = 0; /* Desired cache size. 0 means default */ int doExclusive = 0; /* True for --exclusive */ + int doFullFSync = 0; /* True for --fullfsync */ int nHeap = 0, mnHeap = 0; /* Heap size from --heap */ int doIncrvac = 0; /* True for --incrvacuum */ const char *zJMode = 0; /* Journal mode */ @@ -2267,6 +2269,8 @@ int main(int argc, char **argv){ cacheSize = integerValue(argv[++i]); }else if( strcmp(z,"exclusive")==0 ){ doExclusive = 1; + }else if( strcmp(z,"fullfsync")==0 ){ + doFullFSync = 1; }else if( strcmp(z,"checkpoint")==0 ){ g.doCheckpoint = 1; }else if( strcmp(z,"explain")==0 ){ @@ -2511,7 +2515,11 @@ int main(int argc, char **argv){ if( cacheSize ){ speedtest1_exec("PRAGMA cache_size=%d", cacheSize); } - if( noSync ) speedtest1_exec("PRAGMA synchronous=OFF"); + if( noSync ){ + speedtest1_exec("PRAGMA synchronous=OFF"); + }else if( doFullFSync ){ + speedtest1_exec("PRAGMA fullfsync=ON"); + } if( doExclusive ){ speedtest1_exec("PRAGMA locking_mode=EXCLUSIVE"); } From f84aeac274a5494322475254ef5fbab0a0969616 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 18 Apr 2023 14:13:54 +0000 Subject: [PATCH 232/341] Another fix to the indexed expressions in aggregate queries with GROUP BY enhancement of ticket [99378177930f87bd] and implemented by check-in [b9190d3da70c4171] to address a problem described by [forum:/forumpost/f34e32d120|forum post f34e32d120], FossilOrigin-Name: 5acc3ef83e16a9f2803981a6e61eca62fe177cac4eed1befe2ade3a5977a1eab --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 11 ++++++----- test/indexexpr1.test | 25 ++++++++++++++++++++++++- test/tkt-99378177930f87bd.test | 2 ++ 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c6fee5bf84..789885d3b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--fullsync\soption\sto\sspeedtest1. -D 2023-04-18T11:35:09.601 +C Another\sfix\sto\sthe\sindexed\sexpressions\sin\saggregate\squeries\swith\sGROUP\sBY\nenhancement\sof\sticket\s[99378177930f87bd]\sand\simplemented\sby\ncheck-in\s[b9190d3da70c4171]\sto\saddress\sa\sproblem\sdescribed\sby\n[forum:/forumpost/f34e32d120|forum\spost\sf34e32d120], +D 2023-04-18T14:13:54.801 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -631,7 +631,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c ad6f48e0953fb0d9dd434a7fcd8bc6bde05beb82500a59fea7a7b02dc0ecfb3b +F src/select.c 4054886848b2aaf3d92d28bda9fabff3b5c5c1ffe8511b0dbafcea008d58a66b F src/shell.c.in 2a11419d77d8e4f7b4e3aea6922a9560630889423e3728205d77b4be2511cc89 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1201,7 +1201,7 @@ F test/index7.test b238344318e0b4e42126717f6554f0e7dfd0b39cecad4b736039b43e1e3b6 F test/index8.test caa097735c91dbc23d8a402f5e63a2a03c83840ba3928733ed7f9a03f8a912a3 F test/index9.test 2ac891806a4136ef3e91280477e23114e67575207dc331e6797fa0ed9379f997 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0 -F test/indexexpr1.test b2a15637dcbae7fd8d7e2fc51f74ac4feaf5510130ee2089a5ec5bd1ef7270e1 +F test/indexexpr1.test 62558b1cfd7ccbe7bc015849cc6d1a13ef124e80cbd5b3a98dc66c3c9cce0cf4 F test/indexexpr2.test 1c382e81ef996d8ae8b834a74f2a9013dddf59214c32201d7c8a656d739f999a F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -1625,7 +1625,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 -F test/tkt-99378177930f87bd.test 28530bf9903dcd7743185ce78b1c02b1f9ba09fe4fa77a70ecbd0af83fe3353c +F test/tkt-99378177930f87bd.test 9d6cff39b50d062c813ae1cb0ebbd1b7acf81ecc23ae5d5215e5bb05667dc137 F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6 F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 49595083ba58e13d404b6a11c28f53768a257c59f3f36be3591438127df7e875 -R cc728f1caef69a29ba8fe0bb9cbdc7a1 +P 960fe54c8227a376c97bb2a3ba604d97e79e27e51a9798be9d318cf56c04fa99 +R a2cf5162922bbbf19ca3c71a88c9436f U drh -Z a2caae4ae9157cd4de2a01e274c37a8f +Z 9e57bb75bc9de4a8455f559e00ff6f17 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4bc144f09b..fd881b095c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -960fe54c8227a376c97bb2a3ba604d97e79e27e51a9798be9d318cf56c04fa99 \ No newline at end of file +5acc3ef83e16a9f2803981a6e61eca62fe177cac4eed1befe2ade3a5977a1eab \ No newline at end of file diff --git a/src/select.c b/src/select.c index 80a51d61d1..da454efc33 100644 --- a/src/select.c +++ b/src/select.c @@ -6448,12 +6448,13 @@ static void optimizeAggregateUseOfIndexedExpr( assert( pSelect->pGroupBy!=0 ); pAggInfo->nColumn = pAggInfo->nAccumulator; if( ALWAYS(pAggInfo->nSortingColumn>0) ){ - if( pAggInfo->nColumn==0 ){ - pAggInfo->nSortingColumn = pSelect->pGroupBy->nExpr; - }else{ - pAggInfo->nSortingColumn = - pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; + int mx = pSelect->pGroupBy->nExpr - 1; + int j, k; + for(j=0; jnColumn; j++){ + k = pAggInfo->aCol[j].iSorterColumn; + if( k>mx ) mx = k; } + pAggInfo->nSortingColumn = mx+1; } analyzeAggFuncArgs(pAggInfo, pNC); #if TREETRACE_ENABLED diff --git a/test/indexexpr1.test b/test/indexexpr1.test index 7d94806cdd..51ef73bbf5 100644 --- a/test/indexexpr1.test +++ b/test/indexexpr1.test @@ -591,6 +591,29 @@ do_execsql_test indexexpr1-2140 { SELECT b FROM t1; } 400 - +# 2023-04-18 Forum post https://sqlite.org/forum/forumpost/f34e32d120 from +# Alexis King. +# +# This problem originates at check-in b9190d3da70c4171 (2022-11-25). +# A similar problem arose on 2023-03-04 at +# https://sqlite.org/forum/forumpost/a68313d054 and was fixed at +# check-in e06973876993926f. See the test case tkt-99378-400. +# +reset_db +do_execsql_test indexexpr1-2200 { + CREATE TABLE t1(id INTEGER PRIMARY KEY, tag INT); + INSERT INTO t1 VALUES (0, 7), (1, 8); + CREATE TABLE t2(type INT, t1_id INT, value INT); + INSERT INTO t2 VALUES (0, 0, 100), (0, 1, 101); + CREATE INDEX t1x ON t1(-tag); + SELECT u.tag, v.max_value + FROM (SELECT tag FROM t1 GROUP BY -tag) u + JOIN (SELECT t1.tag AS "tag", t2.type AS "type", + MAX(t2.value) AS "max_value" + FROM t1 + JOIN t2 ON t2.t1_id = t1.id + GROUP BY t2.type, t1.tag + ) v ON v.type = 0 AND v.tag = u.tag; +} {7 100 8 101} finish_test diff --git a/test/tkt-99378177930f87bd.test b/test/tkt-99378177930f87bd.test index ff73529bbd..ba9fdc7027 100644 --- a/test/tkt-99378177930f87bd.test +++ b/test/tkt-99378177930f87bd.test @@ -178,6 +178,8 @@ do_execsql_test tkt-99378-310 { # 2023-03-04 https://sqlite.org/forum/forumpost/a68313d054 # +# See also indexexpr1-2200 added on 2023-03-18. +# do_execsql_test tkt-99378-400 { DROP TABLE t1; CREATE TABLE t0(w); From 359c5ac1fb540a00e63138266c4b50740610b250 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 18 Apr 2023 15:21:55 +0000 Subject: [PATCH 233/341] Ensure that the VACUUM command is not confused by alternative encodings when it is the first command run on new database connection. [forum:/forumpost/09503b4d33|Forum post 09503b4d33]. FossilOrigin-Name: 8b0fe63f873661037351fc1eba6071b98b5c97a0fc8bd8a504a90d047f78bc0d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/prepare.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 789885d3b0..7c448800b8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\sfix\sto\sthe\sindexed\sexpressions\sin\saggregate\squeries\swith\sGROUP\sBY\nenhancement\sof\sticket\s[99378177930f87bd]\sand\simplemented\sby\ncheck-in\s[b9190d3da70c4171]\sto\saddress\sa\sproblem\sdescribed\sby\n[forum:/forumpost/f34e32d120|forum\spost\sf34e32d120], -D 2023-04-18T14:13:54.801 +C Ensure\sthat\sthe\sVACUUM\scommand\sis\snot\sconfused\sby\salternative\sencodings\swhen\nit\sis\sthe\sfirst\scommand\srun\son\snew\sdatabase\sconnection.\n[forum:/forumpost/09503b4d33|Forum\spost\s09503b4d33]. +D 2023-04-18T15:21:55.208 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -626,7 +626,7 @@ F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 26ed2cfdc5c12aa1c707178635709684960288cacc9cff9d491a38ff10e395f1 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 -F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3 +F src/prepare.c 06ecbb25db151a009be99471757c73272cd61306903dc3eeb7250bdcf54896dd F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 960fe54c8227a376c97bb2a3ba604d97e79e27e51a9798be9d318cf56c04fa99 -R a2cf5162922bbbf19ca3c71a88c9436f +P 5acc3ef83e16a9f2803981a6e61eca62fe177cac4eed1befe2ade3a5977a1eab +R 1c9639553f364b83ea46a14ee324415d U drh -Z 9e57bb75bc9de4a8455f559e00ff6f17 +Z 1f598f7bd83b32193e1562d0ae423411 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fd881b095c..e780b63ede 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5acc3ef83e16a9f2803981a6e61eca62fe177cac4eed1befe2ade3a5977a1eab \ No newline at end of file +8b0fe63f873661037351fc1eba6071b98b5c97a0fc8bd8a504a90d047f78bc0d \ No newline at end of file diff --git a/src/prepare.c b/src/prepare.c index b2613e2c1d..3f87a37199 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -306,7 +306,9 @@ int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){ #else encoding = SQLITE_UTF8; #endif - if( db->nVdbeActive>0 && encoding!=ENC(db) ){ + if( db->nVdbeActive>0 && encoding!=ENC(db) + && (db->mDbFlags & DBFLAG_Vacuum)==0 + ){ rc = SQLITE_LOCKED; goto initone_error_out; }else{ From 7b1a49c9934af4efcfcdc70d7cd544e5d9237d33 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 18 Apr 2023 22:28:48 +0000 Subject: [PATCH 234/341] CLI to cease dumping back to OS shel on ^C. FossilOrigin-Name: 7b1f635268bd5b093b22f832c3ac9e5d642d1aac5ebc620619be5a24930feb83 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/shell.c.in | 41 ++++++++++++++++++++++++++--------------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 7c448800b8..e232bac86c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\sVACUUM\scommand\sis\snot\sconfused\sby\salternative\sencodings\swhen\nit\sis\sthe\sfirst\scommand\srun\son\snew\sdatabase\sconnection.\n[forum:/forumpost/09503b4d33|Forum\spost\s09503b4d33]. -D 2023-04-18T15:21:55.208 +C CLI\sto\scease\sdumping\sback\sto\sOS\sshel\son\s^C. +D 2023-04-18T22:28:48.172 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -632,7 +632,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4054886848b2aaf3d92d28bda9fabff3b5c5c1ffe8511b0dbafcea008d58a66b -F src/shell.c.in 2a11419d77d8e4f7b4e3aea6922a9560630889423e3728205d77b4be2511cc89 +F src/shell.c.in eb273024a60f6d9f56865eeacd83247fd20919d8248ea0127ee54d6f0d6abe91 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2058,8 +2058,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5acc3ef83e16a9f2803981a6e61eca62fe177cac4eed1befe2ade3a5977a1eab -R 1c9639553f364b83ea46a14ee324415d -U drh -Z 1f598f7bd83b32193e1562d0ae423411 +P 8b0fe63f873661037351fc1eba6071b98b5c97a0fc8bd8a504a90d047f78bc0d +R 734cf00e9ab5f48eb416aa4234b55f52 +T *branch * cli-no-dump +T *sym-cli-no-dump * +T -sym-trunk * +U larrybr +Z 0e6d888acccf444ddfa65e448f5f1581 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e780b63ede..2e69b9565b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b0fe63f873661037351fc1eba6071b98b5c97a0fc8bd8a504a90d047f78bc0d \ No newline at end of file +7b1f635268bd5b093b22f832c3ac9e5d642d1aac5ebc620619be5a24930feb83 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c8de494fe9..b7fa11463d 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -627,8 +627,8 @@ static void console_prepare(void){ conState.outCodePage = GetConsoleOutputCP(); SetConsoleCP(CP_UTF8); SetConsoleOutputCP(CP_UTF8); - SetConsoleMode(conState.hConsoleIn, - conState.consoleMode | ENABLE_LINE_INPUT); + consoleMode |= ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT; + SetConsoleMode(conState.hConsoleIn, consoleMode); conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT); console_utf8 = 1; }else{ @@ -992,10 +992,18 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ #if SHELL_USE_LOCAL_GETLINE printf("%s", zPrompt); fflush(stdout); - zResult = local_getline(zPrior, stdin); + do{ + zResult = local_getline(zPrior, stdin); + if( zResult==0 ) sqlite3_sleep(50); + }while( zResult==0 && seenInterrupt>0 ); #else free(zPrior); zResult = shell_readline(zPrompt); + while( zResult==0 ){ + sqlite3_sleep(50); + if( seenInterrupt==0 ) break; + zResult = shell_readline(""); + } if( zResult && *zResult ) shell_add_history(zResult); #endif } @@ -2102,8 +2110,7 @@ static void output_csv(ShellState *p, const char *z, int bSep){ */ static void interrupt_handler(int NotUsed){ UNUSED_PARAMETER(NotUsed); - seenInterrupt++; - if( seenInterrupt>2 ) exit(1); + if( ++seenInterrupt>1 ) exit(1); if( globalDb ) sqlite3_interrupt(globalDb); } @@ -11830,6 +11837,10 @@ static char *cmdline_option_value(int argc, char **argv, int i){ return argv[i]; } +static void sayAbnormalExit(void){ + if( seenInterrupt ) fprintf(stderr, "Program interrupted.\n"); +} + #ifndef SQLITE_SHELL_IS_UTF8 # if (defined(_WIN32) || defined(WIN32)) \ && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__))) @@ -11871,8 +11882,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char **argvToFree = 0; int argcToFree = 0; #endif - setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ + #ifdef SQLITE_SHELL_FIDDLE stdin_is_interactive = 0; stdout_is_console = 1; @@ -11884,6 +11895,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #if SHELL_WIN_UTF8_OPT atexit(console_restore); /* Needs revision for CLI as library call */ #endif + atexit(sayAbnormalExit); #ifdef SQLITE_DEBUG mem_main_enter = sqlite3_memory_used(); #endif @@ -11907,6 +11919,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } } #endif + /* Register a valid signal handler early, before much else is done. */ +#ifdef SIGINT + signal(SIGINT, interrupt_handler); +#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) + if( !SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE) ){ + fprintf(stderr, "No ^C handler.\n"); + } +#endif #if USE_SYSTEM_SQLITE+0!=1 if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ @@ -11946,15 +11966,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ assert( argc>=1 && argv && argv[0] ); Argv0 = argv[0]; - /* Make sure we have a valid signal handler early, before anything - ** else is done. - */ -#ifdef SIGINT - signal(SIGINT, interrupt_handler); -#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) - SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); -#endif - #ifdef SQLITE_SHELL_DBNAME_PROC { /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name From 837b661f2c1532186a42ee724757757bfc9607b1 Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 18 Apr 2023 22:53:30 +0000 Subject: [PATCH 235/341] Add comment on CLI ^C fix (because it is a bit subtle.) FossilOrigin-Name: 19c062f1cd814a33bf9c10928217464cddaea7c3291ff44e605901a86b82a06d --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/shell.c.in | 2 ++ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index e232bac86c..47bc9b974a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C CLI\sto\scease\sdumping\sback\sto\sOS\sshel\son\s^C. -D 2023-04-18T22:28:48.172 +C Add\scomment\son\sCLI\s^C\sfix\s(because\sit\sis\sa\sbit\ssubtle.) +D 2023-04-18T22:53:30.926 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -632,7 +632,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c 4054886848b2aaf3d92d28bda9fabff3b5c5c1ffe8511b0dbafcea008d58a66b -F src/shell.c.in eb273024a60f6d9f56865eeacd83247fd20919d8248ea0127ee54d6f0d6abe91 +F src/shell.c.in 9d5da77007123401da702e8587558edddd520c925f8f561aa7b8d9387e71e096 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2058,11 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8b0fe63f873661037351fc1eba6071b98b5c97a0fc8bd8a504a90d047f78bc0d -R 734cf00e9ab5f48eb416aa4234b55f52 -T *branch * cli-no-dump -T *sym-cli-no-dump * -T -sym-trunk * +P 7b1f635268bd5b093b22f832c3ac9e5d642d1aac5ebc620619be5a24930feb83 +R 94128278c59c8b6776a5748252e83b85 U larrybr -Z 0e6d888acccf444ddfa65e448f5f1581 +Z e64ea4c126276298324ed24c1a272560 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2e69b9565b..bbf8851516 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b1f635268bd5b093b22f832c3ac9e5d642d1aac5ebc620619be5a24930feb83 \ No newline at end of file +19c062f1cd814a33bf9c10928217464cddaea7c3291ff44e605901a86b82a06d \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index b7fa11463d..8ce845ba4d 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -994,12 +994,14 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ fflush(stdout); do{ zResult = local_getline(zPrior, stdin); + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ if( zResult==0 ) sqlite3_sleep(50); }while( zResult==0 && seenInterrupt>0 ); #else free(zPrior); zResult = shell_readline(zPrompt); while( zResult==0 ){ + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ sqlite3_sleep(50); if( seenInterrupt==0 ) break; zResult = shell_readline(""); From e61aa230ce473404396e9d22bbbce8c8fe97b68f Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Apr 2023 12:08:46 +0000 Subject: [PATCH 236/341] Attempt to suppress harmless compiler warnings reported by [forum:/forumpost/fc98845c06|forum post fc98845c06]. FossilOrigin-Name: f6e6c5f94cef83645288568a344a0ff64e4ef17b10fab28996567a94db0a002a --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/parse.y | 2 +- src/util.c | 2 +- src/vdbeaux.c | 3 +++ 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 480b83ce09..eb01fa12ae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C CLI\sto\scease\sdumping\sback\sto\sOS\sshell\son\ssingle\s^C\swhen\sinteractive. -D 2023-04-18T23:05:34.073 +C Attempt\sto\ssuppress\sharmless\scompiler\swarnings\sreported\sby\n[forum:/forumpost/fc98845c06|forum\spost\sfc98845c06]. +D 2023-04-19T12:08:46.749 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -620,7 +620,7 @@ F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 39af8ff7c73a991f61f4d1e3a6f8f98b1c8e29144723507822774cac5e6ee0b5 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 -F src/parse.y 424e49ed8fc6c907920db9be5a13a75ed43811e1ea8dd21b0fa9ef97f083dc6b +F src/parse.y e8b5c753e3194e03d69e108753c1dbceb01fa4c158b2c6b726a048023ea0fdc1 F src/pcache.c 842410539b544e12d5fccfcf29890782f46a58f227a77bc0bd76243799662c0c F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc @@ -699,13 +699,13 @@ F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0 F src/update.c 3f4fb5ad7c9b48d7911974d6579192bb3a6c27f46140b6cbb9139cc8a77b8691 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c 2106b5176a847d5b91a9345b38a098c202b7691a19070e0fa194cbd3ff139d24 +F src/util.c b1d8d87c4c8c77e70f48c43f91444fd66d91532693573b70b837afd572010176 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 -F src/vdbeaux.c 128de68cade63c914e97d31c2fa71fd6186927ebfbedabc68019ce99256c3ddf +F src/vdbeaux.c 913801bb824d6ef0a49c80e973563063e453f41e6c382c5940de8e5d34f0cd56 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 1d9a0f37b0097fbb53f0d7ba081f7181b83cee2c6f46364706ea0c3896bd8ec0 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8b0fe63f873661037351fc1eba6071b98b5c97a0fc8bd8a504a90d047f78bc0d 19c062f1cd814a33bf9c10928217464cddaea7c3291ff44e605901a86b82a06d -R 94128278c59c8b6776a5748252e83b85 -U larrybr -Z 708f7e71cc9002b37b9149765bf28cad +P ba8e4378f21d529a248d2d60b60974fef0d6612dcaaa1f6b7f8a6946fc46b39b +R 70cfff64a8ee73f6f83838961859b992 +U drh +Z 4d312cdcd589df887faf890c715735dd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c2ce3e5176..084a1ef972 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba8e4378f21d529a248d2d60b60974fef0d6612dcaaa1f6b7f8a6946fc46b39b \ No newline at end of file +f6e6c5f94cef83645288568a344a0ff64e4ef17b10fab28996567a94db0a002a \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 73508b046b..42876feebb 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1378,7 +1378,7 @@ case_else(A) ::= ELSE expr(X). {A = X;} case_else(A) ::= . {A = 0;} %type case_operand {Expr*} %destructor case_operand {sqlite3ExprDelete(pParse->db, $$);} -case_operand(A) ::= expr(X). {A = X; /*A-overwrites-X*/} +case_operand(A) ::= expr(A). case_operand(A) ::= . {A = 0;} %type exprlist {ExprList*} diff --git a/src/util.c b/src/util.c index 8e3c6c9dbb..72e8a18b2f 100644 --- a/src/util.c +++ b/src/util.c @@ -879,7 +879,7 @@ int sqlite3GetInt32(const char *zNum, int *pValue){ u32 u = 0; zNum += 2; while( zNum[0]=='0' ) zNum++; - for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + for(i=0; i<8 && sqlite3Isxdigit(zNum[i]); i++){ u = u*16 + sqlite3HexToInt(zNum[i]); } if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 1848cdea3c..07145bb052 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -820,6 +820,8 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; + + assert( pParse->db->mallocFailed==0 ); /* tag-20230419-1 */ p->readOnly = 1; p->bIsReader = 0; pOp = &p->aOp[p->nOp-1]; @@ -879,6 +881,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); assert( ADDR(pOp->p2)<-pParse->nLabel ); + assert( aLabel!=0 ); /* True because of tag-20230419-1 */ pOp->p2 = aLabel[ADDR(pOp->p2)]; } break; From d9ae63144eebb77e278fc9ba5777e44c61b75caa Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Apr 2023 13:30:17 +0000 Subject: [PATCH 237/341] Add pager debug tracing of truncate operations. FossilOrigin-Name: 0daadf36229816abe73ff684b55ef4f8045ed314347f3a694404a2084decb355 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index eb01fa12ae..4b3ad217a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\ssuppress\sharmless\scompiler\swarnings\sreported\sby\n[forum:/forumpost/fc98845c06|forum\spost\sfc98845c06]. -D 2023-04-19T12:08:46.749 +C Add\spager\sdebug\stracing\sof\struncate\soperations. +D 2023-04-19T13:30:17.418 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -618,7 +618,7 @@ F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d87210 F src/os_unix.c 1b3ddb7814c4bf37f494c04d2ab30c1ced5b2c927267e1930ce7cd388787a96d F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 39af8ff7c73a991f61f4d1e3a6f8f98b1c8e29144723507822774cac5e6ee0b5 +F src/pager.c 575133bda34ec8a8b11325152063e6981d8a357e553ba1cdfc8b586ea3e859ac F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y e8b5c753e3194e03d69e108753c1dbceb01fa4c158b2c6b726a048023ea0fdc1 F src/pcache.c 842410539b544e12d5fccfcf29890782f46a58f227a77bc0bd76243799662c0c @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ba8e4378f21d529a248d2d60b60974fef0d6612dcaaa1f6b7f8a6946fc46b39b -R 70cfff64a8ee73f6f83838961859b992 +P f6e6c5f94cef83645288568a344a0ff64e4ef17b10fab28996567a94db0a002a +R 645b34b49128ce0ee2d93bc1fed8bf95 U drh -Z 4d312cdcd589df887faf890c715735dd +Z 43ce893918c01cfc76bddfc773b7f230 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 084a1ef972..424c817060 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f6e6c5f94cef83645288568a344a0ff64e4ef17b10fab28996567a94db0a002a \ No newline at end of file +0daadf36229816abe73ff684b55ef4f8045ed314347f3a694404a2084decb355 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 89073b8c78..3c1925bd38 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2605,6 +2605,8 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; assert( pPager->eState!=PAGER_ERROR ); assert( pPager->eState!=PAGER_READER ); + PAGERTRACE(("Truncate %d npage %u\n", PAGERID(pPager), nPage)); + if( isOpen(pPager->fd) && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) From b669956637b72b20131928398228864fc4304b29 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 19 Apr 2023 13:39:13 +0000 Subject: [PATCH 238/341] Correct use of an undefined symbol in one of the OPFS VFS loading failure error reporting cases (copy/paste error). FossilOrigin-Name: c81c968b458977b2d6dd2fdd3c3d667f0e7cee069f3715363d0e8aed4e6e8377 --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 24 +++++++++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index e9dfcf4171..2e09aab21a 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -1279,14 +1279,15 @@ const installOpfsVfs = function callee(options){ promiseReject(new Error(data.payload.join(' '))); break; case 'opfs-async-loaded': - /*Arrives as soon as the asyc proxy finishes loading. - Pass our config and shared state on to the async worker.*/ + /* Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async + worker. */ W.postMessage({type: 'opfs-async-init',args: state}); break; - case 'opfs-async-inited':{ - /*Indicates that the async partner has received the 'init' - and has finished initializing, so the real work can - begin...*/ + case 'opfs-async-inited': { + /* Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin... */ if(true===promiseWasRejected){ break /* promise was already rejected via timer */; } @@ -1321,10 +1322,15 @@ const installOpfsVfs = function callee(options){ } break; } - default: - promiseReject(e); - error("Unexpected message from the async worker:",data); + default: { + const errMsg = ( + "Unexpected message from the OPFS async worker: " + + JSON.stringify(data) + ); + error(errMsg); + promiseReject(new Error(errMsg)); break; + } }/*switch(data.type)*/ }/*W.onmessage()*/; })/*thePromise*/; diff --git a/manifest b/manifest index 4b3ad217a5..f5e5e9f713 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\spager\sdebug\stracing\sof\struncate\soperations. -D 2023-04-19T13:30:17.418 +C Correct\suse\sof\san\sundefined\ssymbol\sin\sone\sof\sthe\sOPFS\sVFS\sloading\sfailure\serror\sreporting\scases\s(copy/paste\serror). +D 2023-04-19T13:39:13.019 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -495,7 +495,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 5cb63f847737ded723d0bdf9b3313e8f7deb0bbfd0082dedde540991edde155b +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 69987029bead5a35f1b854e50c3453b527a497a2be35910b2cf07a4d36ec2299 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c c42413ca9f3e64c424b2bbfc5decf639670ca38bc8f7afb7760d5379398c9307 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f6e6c5f94cef83645288568a344a0ff64e4ef17b10fab28996567a94db0a002a -R 645b34b49128ce0ee2d93bc1fed8bf95 -U drh -Z 43ce893918c01cfc76bddfc773b7f230 +P 0daadf36229816abe73ff684b55ef4f8045ed314347f3a694404a2084decb355 +R e0a847c087c5738b23e88e728c13fa85 +U stephan +Z d6d1111331e9982a922069d7625156a9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 424c817060..2c3b25146b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0daadf36229816abe73ff684b55ef4f8045ed314347f3a694404a2084decb355 \ No newline at end of file +c81c968b458977b2d6dd2fdd3c3d667f0e7cee069f3715363d0e8aed4e6e8377 \ No newline at end of file From 4c0fca00d32946e15438dc949ea2102af4b5ca96 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Apr 2023 13:56:47 +0000 Subject: [PATCH 239/341] Fix a debugging printf() in pcache.c. This only comes up on custom builds. FossilOrigin-Name: f2d6756731caadf9a30214ab7d19717a2ca1dae2652bcc6b211f6b0830362240 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pcache.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f5e5e9f713..3f02f2e5a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\suse\sof\san\sundefined\ssymbol\sin\sone\sof\sthe\sOPFS\sVFS\sloading\sfailure\serror\sreporting\scases\s(copy/paste\serror). -D 2023-04-19T13:39:13.019 +C Fix\sa\sdebugging\sprintf()\sin\spcache.c.\s\sThis\sonly\scomes\sup\son\scustom\sbuilds. +D 2023-04-19T13:56:47.024 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 575133bda34ec8a8b11325152063e6981d8a357e553ba1cdfc8b586ea3e859ac F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y e8b5c753e3194e03d69e108753c1dbceb01fa4c158b2c6b726a048023ea0fdc1 -F src/pcache.c 842410539b544e12d5fccfcf29890782f46a58f227a77bc0bd76243799662c0c +F src/pcache.c 589326520a41211b50bb946973ca7f1e30f824641af546ea8fa6224bf0e4ea5c F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 26ed2cfdc5c12aa1c707178635709684960288cacc9cff9d491a38ff10e395f1 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0daadf36229816abe73ff684b55ef4f8045ed314347f3a694404a2084decb355 -R e0a847c087c5738b23e88e728c13fa85 -U stephan -Z d6d1111331e9982a922069d7625156a9 +P c81c968b458977b2d6dd2fdd3c3d667f0e7cee069f3715363d0e8aed4e6e8377 +R 1e45a38de02e3f841d848ec85fdde14f +U drh +Z dfce4f4da24d9b79dc856d23ff7bd173 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c3b25146b..3e22473935 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c81c968b458977b2d6dd2fdd3c3d667f0e7cee069f3715363d0e8aed4e6e8377 \ No newline at end of file +f2d6756731caadf9a30214ab7d19717a2ca1dae2652bcc6b211f6b0830362240 \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index 153628dc9a..eb6b87cd99 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -71,7 +71,7 @@ struct PCache { unsigned char *a; int j; pPg = (PgHdr*)pLower->pExtra; - printf("%3lld: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); + printf("%3d: nRef %2lld flgs %02x data ", i, pPg->nRef, pPg->flags); a = (unsigned char *)pLower->pBuf; for(j=0; j<12; j++) printf("%02x", a[j]); printf(" ptr %p\n", pPg); From ec0ed68846f58047f3e563f315e593f9ad3fc907 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Apr 2023 14:26:43 +0000 Subject: [PATCH 240/341] Fix ambiguity in the header comment to the sqlite3PagerUnrefNotNull() routine. No code changes. FossilOrigin-Name: d419e65eef2954ed759121254024028854b55e8c4f52ff7a8896c8cadfc57f53 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 3f02f2e5a0..05094c8de0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sdebugging\sprintf()\sin\spcache.c.\s\sThis\sonly\scomes\sup\son\scustom\sbuilds. -D 2023-04-19T13:56:47.024 +C Fix\sambiguity\sin\sthe\sheader\scomment\sto\sthe\ssqlite3PagerUnrefNotNull()\sroutine.\nNo\scode\schanges. +D 2023-04-19T14:26:43.117 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -618,7 +618,7 @@ F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d87210 F src/os_unix.c 1b3ddb7814c4bf37f494c04d2ab30c1ced5b2c927267e1930ce7cd388787a96d F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 575133bda34ec8a8b11325152063e6981d8a357e553ba1cdfc8b586ea3e859ac +F src/pager.c 725385bdb872e1764f8698b1c2fbb87635d8720f8589ba5a27ed4b6183cbbab0 F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y e8b5c753e3194e03d69e108753c1dbceb01fa4c158b2c6b726a048023ea0fdc1 F src/pcache.c 589326520a41211b50bb946973ca7f1e30f824641af546ea8fa6224bf0e4ea5c @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c81c968b458977b2d6dd2fdd3c3d667f0e7cee069f3715363d0e8aed4e6e8377 -R 1e45a38de02e3f841d848ec85fdde14f +P f2d6756731caadf9a30214ab7d19717a2ca1dae2652bcc6b211f6b0830362240 +R d9a8cdd9e9c2d3f863d060c3de9c370c U drh -Z dfce4f4da24d9b79dc856d23ff7bd173 +Z 55eddf4ca402ac276f5fb0dc48de52ae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3e22473935..0386f2aa85 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2d6756731caadf9a30214ab7d19717a2ca1dae2652bcc6b211f6b0830362240 \ No newline at end of file +d419e65eef2954ed759121254024028854b55e8c4f52ff7a8896c8cadfc57f53 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 3c1925bd38..d816a8c322 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5688,10 +5688,12 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ /* ** Release a page reference. ** -** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be -** used if we know that the page being released is not the last page. +** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be used +** if we know that the page being released is not the last reference to page1. ** The btree layer always holds page1 open until the end, so these first -** to routines can be used to release any page other than BtShared.pPage1. +** two routines can be used to release any page other than BtShared.pPage1. +** The assert() at tag-20230419-2 proves that this constraint is always +** honored. ** ** Use sqlite3PagerUnrefPageOne() to release page1. This latter routine ** checks the total number of outstanding pages and if the number of @@ -5707,7 +5709,7 @@ void sqlite3PagerUnrefNotNull(DbPage *pPg){ sqlite3PcacheRelease(pPg); } /* Do not use this routine to release the last reference to page1 */ - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); /* tag-20230419-2 */ } void sqlite3PagerUnref(DbPage *pPg){ if( pPg ) sqlite3PagerUnrefNotNull(pPg); From 2c5c124064103fe92548c4854be5c18a4bdbe162 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Apr 2023 15:35:45 +0000 Subject: [PATCH 241/341] Do not remove pages from the cache of an in-memory database due to a failure to acquire the page due to it being larger than the maximum page size. Fix for [forum:/forumpost/a19bb49140|forum post a19bb49140]. FossilOrigin-Name: 982b35563da685dfdf50cbe4a7ae829d3b428e03587028df7efe520f819b1dc2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 05094c8de0..c08119f855 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sambiguity\sin\sthe\sheader\scomment\sto\sthe\ssqlite3PagerUnrefNotNull()\sroutine.\nNo\scode\schanges. -D 2023-04-19T14:26:43.117 +C Do\snot\sremove\spages\sfrom\sthe\scache\sof\san\sin-memory\sdatabase\sdue\sto\sa\nfailure\sto\sacquire\sthe\spage\sdue\sto\sit\sbeing\slarger\sthan\sthe\smaximum\spage\nsize.\sFix\sfor\s[forum:/forumpost/a19bb49140|forum\spost\sa19bb49140]. +D 2023-04-19T15:35:45.024 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -618,7 +618,7 @@ F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d87210 F src/os_unix.c 1b3ddb7814c4bf37f494c04d2ab30c1ced5b2c927267e1930ce7cd388787a96d F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 725385bdb872e1764f8698b1c2fbb87635d8720f8589ba5a27ed4b6183cbbab0 +F src/pager.c 9d36ddedc842e993c88c222ed914822dbd6f8ece3c648fde04468637012a034a F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y e8b5c753e3194e03d69e108753c1dbceb01fa4c158b2c6b726a048023ea0fdc1 F src/pcache.c 589326520a41211b50bb946973ca7f1e30f824641af546ea8fa6224bf0e4ea5c @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f2d6756731caadf9a30214ab7d19717a2ca1dae2652bcc6b211f6b0830362240 -R d9a8cdd9e9c2d3f863d060c3de9c370c +P d419e65eef2954ed759121254024028854b55e8c4f52ff7a8896c8cadfc57f53 +R 2ff39e988928f271722070323e9929b5 U drh -Z 55eddf4ca402ac276f5fb0dc48de52ae +Z 57826475051b9681a19f44a30db70018 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0386f2aa85..91acfb0146 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d419e65eef2954ed759121254024028854b55e8c4f52ff7a8896c8cadfc57f53 \ No newline at end of file +982b35563da685dfdf50cbe4a7ae829d3b428e03587028df7efe520f819b1dc2 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d816a8c322..063b793f7f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5524,6 +5524,10 @@ static int getPageNormal( if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; + if( pgno<=pPager->dbSize ){ + sqlite3PcacheRelease(pPg); + pPg = 0; + } goto pager_acquire_err; } if( noContent ){ From 3115b36d67ff3b15ca87c11c1b2291371a52d39c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Apr 2023 17:07:35 +0000 Subject: [PATCH 242/341] Fix a segfault that could occur if a non-empty in-memory database was the destination of a backup operation from a database with a smaller page size. FossilOrigin-Name: 020968f857d7b90bab3525fc9d6b859a019f6a80422c3d5ffb88f5bdb8e02a11 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/backup.c | 13 +++++-------- test/backup.test | 23 +++++++++++++++++++++++ 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c08119f855..32216b9c9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sremove\spages\sfrom\sthe\scache\sof\san\sin-memory\sdatabase\sdue\sto\sa\nfailure\sto\sacquire\sthe\spage\sdue\sto\sit\sbeing\slarger\sthan\sthe\smaximum\spage\nsize.\sFix\sfor\s[forum:/forumpost/a19bb49140|forum\spost\sa19bb49140]. -D 2023-04-19T15:35:45.024 +C Fix\sa\ssegfault\sthat\scould\soccur\sif\sa\snon-empty\sin-memory\sdatabase\swas\sthe\sdestination\sof\sa\sbackup\soperation\sfrom\sa\sdatabase\swith\sa\ssmaller\spage\ssize. +D 2023-04-19T17:07:35.099 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -567,7 +567,7 @@ F src/alter.c 482c534877fbb543f8295992cde925df55443febac5db5438d5aaba6f78c4940 F src/analyze.c 01bfd40026632eaae1d93212b684f539c6674cb573535dc90199674cbf7e0cdc F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf -F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7 +F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca F src/btree.c af379d801906f21dc18b534fc250a479f56045a450d8d5859ba2f57fd9eefcef @@ -794,7 +794,7 @@ F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avfs.test 0c3a38e03cccb0fc3127838462dc05dc3f4c1480d770c084b388304c25de3652 F test/avtrans.test b7dc25459ecbd86c6fa9c606ee3068f59d81e225118617dcf2bbb6ded2ade89e F test/backcompat.test 3e64cedda754c778ef6bbe417b6e7a295e662a4d -F test/backup.test fc1ecefce723fad5199b55cec7a5a992ec8c3ad6873419e5e8919066dec457f3 +F test/backup.test 3b08fd4af69f0fa786931103a31f4542b184aba16e239e5f22b18c3c2476697f F test/backup2.test 8facb54df1388419d34b362ab1f7e233310ff3a3af64e8ad5ec47ba3c2bbe5cf F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d419e65eef2954ed759121254024028854b55e8c4f52ff7a8896c8cadfc57f53 -R 2ff39e988928f271722070323e9929b5 -U drh -Z 57826475051b9681a19f44a30db70018 +P 982b35563da685dfdf50cbe4a7ae829d3b428e03587028df7efe520f819b1dc2 +R cb5ceacd1fcb276c3a493137d395e5ff +U dan +Z 770c9a899bf77db795bc4c69f9feca6f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 91acfb0146..1d22330bc8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -982b35563da685dfdf50cbe4a7ae829d3b428e03587028df7efe520f819b1dc2 \ No newline at end of file +020968f857d7b90bab3525fc9d6b859a019f6a80422c3d5ffb88f5bdb8e02a11 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index d9671750f4..22615d1499 100644 --- a/src/backup.c +++ b/src/backup.c @@ -242,13 +242,7 @@ static int backupOnePage( assert( !isFatalError(p->rc) ); assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); assert( zSrcData ); - - /* Catch the case where the destination is an in-memory database and the - ** page sizes of the source and destination differ. - */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ - rc = SQLITE_READONLY; - } + assert( nSrcPgsz==nDestPgsz || sqlite3PagerIsMemdb(pDestPager)==0 ); /* This loop runs once for each destination page spanned by the source ** page. For each iteration, variable iOff is set to the byte offset @@ -381,7 +375,10 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); pgszDest = sqlite3BtreeGetPageSize(p->pDest); destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); - if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ + if( SQLITE_OK==rc + && (destMode==PAGER_JOURNALMODE_WAL || sqlite3PagerIsMemdb(pDestPager)) + && pgszSrc!=pgszDest + ){ rc = SQLITE_READONLY; } diff --git a/test/backup.test b/test/backup.test index 1572ded554..ad1a383a08 100644 --- a/test/backup.test +++ b/test/backup.test @@ -977,5 +977,28 @@ do_test backup-11.1 { sqlite3_backup B db1 main db2 temp B finish } {SQLITE_OK} +db1 close +db2 close + +#------------------------------------------------------------------------- +do_test backup-12.1 { + sqlite3 db1 :memory: + sqlite3 db2 :memory: + db1 eval { + PRAGMA page_size = 8192; + CREATE TABLE t1(x); + } + db2 eval { + PRAGMA page_size = 1024; + CREATE TABLE t2(x); + } + + sqlite3_backup B db1 main db2 temp + B step 100 + B finish +} {SQLITE_READONLY} + + + finish_test From 2cc723a2da2350851313918fd37533954bd2efaa Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Apr 2023 18:32:32 +0000 Subject: [PATCH 243/341] Remove unreachable legacy code. FossilOrigin-Name: e3e7fb87d90690c29b70da261887c8f7d87b62188d15577e48db91232e13608c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 2 -- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 32216b9c9b..326d8dfa0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\ssegfault\sthat\scould\soccur\sif\sa\snon-empty\sin-memory\sdatabase\swas\sthe\sdestination\sof\sa\sbackup\soperation\sfrom\sa\sdatabase\swith\sa\ssmaller\spage\ssize. -D 2023-04-19T17:07:35.099 +C Remove\sunreachable\slegacy\scode. +D 2023-04-19T18:32:32.235 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -631,7 +631,7 @@ F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c 4054886848b2aaf3d92d28bda9fabff3b5c5c1ffe8511b0dbafcea008d58a66b +F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 F src/shell.c.in 9d5da77007123401da702e8587558edddd520c925f8f561aa7b8d9387e71e096 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 982b35563da685dfdf50cbe4a7ae829d3b428e03587028df7efe520f819b1dc2 -R cb5ceacd1fcb276c3a493137d395e5ff -U dan -Z 770c9a899bf77db795bc4c69f9feca6f +P 020968f857d7b90bab3525fc9d6b859a019f6a80422c3d5ffb88f5bdb8e02a11 +R 830602ab1372d787b9670b489818f9ab +U drh +Z 0011d2490fa3ab04ee8b4d3dc4a07ff1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1d22330bc8..770bd4ecc3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -020968f857d7b90bab3525fc9d6b859a019f6a80422c3d5ffb88f5bdb8e02a11 \ No newline at end of file +e3e7fb87d90690c29b70da261887c8f7d87b62188d15577e48db91232e13608c \ No newline at end of file diff --git a/src/select.c b/src/select.c index da454efc33..1f4b879bd6 100644 --- a/src/select.c +++ b/src/select.c @@ -2373,8 +2373,6 @@ void sqlite3SubqueryColumnTypes( if( pCol->zCnName ){ memcpy(&pCol->zCnName[n+1], zType, m+1); pCol->colFlags |= COLFLAG_HASTYPE; - }else{ - testcase( pCol->colFlags & COLFLAG_HASTYPE ); } } pColl = sqlite3ExprCollSeq(pParse, p); From f23adcbaa4720babf778527cc280347a92270eb4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 19 Apr 2023 18:36:49 +0000 Subject: [PATCH 244/341] Improved rebustness of the pcache tracing logic. FossilOrigin-Name: 0a43235b8368a0996d1856c4e956786a5321275458cb9179ce818aa47bb824ed --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache.c | 17 ++++++++++------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 326d8dfa0e..974719deca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunreachable\slegacy\scode. -D 2023-04-19T18:32:32.235 +C Improved\srebustness\sof\sthe\spcache\stracing\slogic. +D 2023-04-19T18:36:49.496 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,7 +621,7 @@ F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 9d36ddedc842e993c88c222ed914822dbd6f8ece3c648fde04468637012a034a F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 F src/parse.y e8b5c753e3194e03d69e108753c1dbceb01fa4c158b2c6b726a048023ea0fdc1 -F src/pcache.c 589326520a41211b50bb946973ca7f1e30f824641af546ea8fa6224bf0e4ea5c +F src/pcache.c 8ee13acccfd9accbf0af94910b7323dd7f7d55300d92ddafcf40e34fcc8e21be F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 26ed2cfdc5c12aa1c707178635709684960288cacc9cff9d491a38ff10e395f1 @@ -2058,8 +2058,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 020968f857d7b90bab3525fc9d6b859a019f6a80422c3d5ffb88f5bdb8e02a11 -R 830602ab1372d787b9670b489818f9ab +P e3e7fb87d90690c29b70da261887c8f7d87b62188d15577e48db91232e13608c +R 3964c30fad91750ec1a096a4b6637fa8 U drh -Z 0011d2490fa3ab04ee8b4d3dc4a07ff1 +Z addba7fae3e9b75b9a30be88346b1406 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 770bd4ecc3..10c439264a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3e7fb87d90690c29b70da261887c8f7d87b62188d15577e48db91232e13608c \ No newline at end of file +0a43235b8368a0996d1856c4e956786a5321275458cb9179ce818aa47bb824ed \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index eb6b87cd99..8a96cbeaee 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -70,11 +70,15 @@ struct PCache { PgHdr *pPg; unsigned char *a; int j; - pPg = (PgHdr*)pLower->pExtra; - printf("%3d: nRef %2lld flgs %02x data ", i, pPg->nRef, pPg->flags); - a = (unsigned char *)pLower->pBuf; - for(j=0; j<12; j++) printf("%02x", a[j]); - printf(" ptr %p\n", pPg); + if( pLower==0 ){ + printf("%3d: NULL\n", i); + }else{ + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2lld flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf(" ptr %p\n", pPg); + } } static void pcacheDump(PCache *pCache){ int N; @@ -87,9 +91,8 @@ struct PCache { if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; for(i=1; i<=N; i++){ pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); - if( pLower==0 ) continue; pcachePageTrace(i, pLower); - if( ((PgHdr*)pLower)->pPage==0 ){ + if( pLower && ((PgHdr*)pLower)->pPage==0 ){ sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); } } From 8cb4f0b1b2692325780bb4dabfdaa10a83b312af Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Apr 2023 20:29:26 +0000 Subject: [PATCH 245/341] Add tests for the progress-handler callback with the fts5 'secure-delete' feature. FossilOrigin-Name: 48505ad950bc0902d58210be066d4672e6085eb27c525ba2bc663fde7e932ff7 --- ext/fts5/test/fts5secure6.test | 55 ++++++++++++++++++++++++++++++++++ manifest | 13 ++++---- manifest.uuid | 2 +- 3 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 ext/fts5/test/fts5secure6.test diff --git a/ext/fts5/test/fts5secure6.test b/ext/fts5/test/fts5secure6.test new file mode 100644 index 0000000000..e561a43f7d --- /dev/null +++ b/ext/fts5/test/fts5secure6.test @@ -0,0 +1,55 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure6 + +db progress 1 progress_handler +set ::PHC 0 +proc progress_handler {args} { + incr ::PHC + if {($::PHC % 100000)==0} breakpoint + return 0 +} + +proc setup {} { + db eval { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING fts5(x); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1000 + ) + INSERT INTO t1 SELECT 'a b c d e f g h i j k' FROM s; + } +} + +foreach {tn sd} { + 1 0 + 2 1 +} { + setup + do_execsql_test 1.$tn.0 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) + } + set PHC 0 + do_execsql_test 1.$tn.1 { DELETE FROM t1; } + set phc($tn) $PHC +} + +do_test 1.3 { + expr $phc(1)*5 < $phc(2) +} {1} + + +finish_test + diff --git a/manifest b/manifest index 974719deca..4d4520ebc4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\srebustness\sof\sthe\spcache\stracing\slogic. -D 2023-04-19T18:36:49.496 +C Add\stests\sfor\sthe\sprogress-handler\scallback\swith\sthe\sfts5\s'secure-delete'\sfeature. +D 2023-04-19T20:29:26.231 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -192,6 +192,7 @@ F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a +F ext/fts5/test/fts5secure6.test 7a959d834be6725c641b3c3b38ef86570ea671216ad803e054e4fdff33a72ce2 F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 @@ -2058,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e3e7fb87d90690c29b70da261887c8f7d87b62188d15577e48db91232e13608c -R 3964c30fad91750ec1a096a4b6637fa8 -U drh -Z addba7fae3e9b75b9a30be88346b1406 +P 0a43235b8368a0996d1856c4e956786a5321275458cb9179ce818aa47bb824ed +R fec6a45b9c73831f007d768e2aab8221 +U dan +Z e1912bc11e1db99b09a714bb11e7ca84 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 10c439264a..f75858ed9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a43235b8368a0996d1856c4e956786a5321275458cb9179ce818aa47bb824ed \ No newline at end of file +48505ad950bc0902d58210be066d4672e6085eb27c525ba2bc663fde7e932ff7 \ No newline at end of file From 95492455094518ba68038b8553e6097a921fca86 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Apr 2023 10:26:56 +0000 Subject: [PATCH 246/341] Fix a broken assert() in the recovery extension. [forum:/forumpost/82a88f5812|Forum post 82a88f5812]. FossilOrigin-Name: 8a1ad137606ce5bddbed62f89ff9a5cd922c6530db84c12b8b27c08df3c66bc6 --- ext/recover/recover1.test | 13 +++++++++++++ ext/recover/sqlite3recover.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index fef1bf90f4..26894eaca2 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -316,5 +316,18 @@ do_execsql_test 16.9 { COMMIT; } {1 2 3 4} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 17.1 { + CREATE TABLE t(a, PRIMARY KEY(a, a COLLATE NOCASE)) WITHOUT ROWID; + INSERT INTO t VALUES('abc'); + INSERT INTO t VALUES('def'); +} +do_test 17.2 { + set R [sqlite3_recover_init db main test.db2] + $R run + list [catch { $R finish } msg] $msg +} {0 {}} + finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 8306e8ed8e..29fff0e7e2 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -1106,7 +1106,7 @@ static void recoverAddTable( int iField = sqlite3_column_int(pStmt, 0); int iCol = sqlite3_column_int(pStmt, 1); - assert( iFieldnCol && iColnCol ); + assert( iColnCol ); pNew->aCol[iCol].iField = iField; pNew->bIntkey = 0; diff --git a/manifest b/manifest index 4d4520ebc4..3e7412162d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sthe\sprogress-handler\scallback\swith\sthe\sfts5\s'secure-delete'\sfeature. -D 2023-04-19T20:29:26.231 +C Fix\sa\sbroken\sassert()\sin\sthe\srecovery\sextension.\s[forum:/forumpost/82a88f5812|Forum\spost\s82a88f5812]. +D 2023-04-20T10:26:56.069 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -373,7 +373,7 @@ F ext/rbu/sqlite3rbu.c 71a7f0dea3a846ff7c2499dc34a2528f5ddcbe23e2c54dc3cd1fa4d93 F ext/rbu/sqlite3rbu.h 9d923eb135c5d04aa6afd7c39ca47b0d1d0707c100e02f19fdde6a494e414304 F ext/rbu/test_rbu.c ee6ede75147bc081fe9bc3931e6b206277418d14d3fbceea6fdc6216d9b47055 F ext/recover/dbdata.c 31d580785cf14eb3c20ed6fbb421a10a66569858f837928e6b326088c38d4c72 -F ext/recover/recover1.test 2072993624d5e32fef20ae03b17fc06c02bcb344421fe17bb329b24d2a51e647 +F ext/recover/recover1.test 36a4fd33134809439e677c0e7045b1d31a0ae4f70f464830bce158e2fb1e82f8 F ext/recover/recover_common.tcl a61306c1eb45c0c3fc45652c35b2d4ec19729e340bdf65a272ce4c229cefd85a F ext/recover/recoverbuild.test c74170e0f7b02456af41838afeb5353fdb985a48cc2331d661bbabbca7c6b8e3 F ext/recover/recoverclobber.test 3ba6c0c373c5c63d17e82eced64c05c57ccaf26c1abe1ca7141334022a79f32e @@ -386,7 +386,7 @@ F ext/recover/recoverpgsz.test 3658ab8e68475b1bb87d6af88baa04551c84b73280a566a1b F ext/recover/recoverrowid.test f948bf4024a5f41b0e21b8af80c60564c5b5d78c05a8d64fc00787715ff9f45f F ext/recover/recoverslowidx.test 5205a9742dd9490ee99950dabb622307355ef1662dea6a3a21030057bfd81411 F ext/recover/recoversql.test e66d01f95302a223bcd3fd42b5ee58dc2b53d70afa90b0d00e41e4b8eab20486 -F ext/recover/sqlite3recover.c 3346e73e8570cb803618f0d4b65a0d07a3e246c6591347bdf04c7d0bb6d3bbde +F ext/recover/sqlite3recover.c b3813090c97df34858bab8ece4e8c49d9e2f56ba2b9bb019f57f44bc234b3c6d F ext/recover/sqlite3recover.h 011c799f02deb70ab685916f6f538e6bb32c4e0025e79bfd0e24ff9c74820959 F ext/recover/test_recover.c 1a34e2d04533d919a30ae4d5caeb1643f6684e9ccd7597ca27721d8af81f4ade F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a43235b8368a0996d1856c4e956786a5321275458cb9179ce818aa47bb824ed -R fec6a45b9c73831f007d768e2aab8221 +P 48505ad950bc0902d58210be066d4672e6085eb27c525ba2bc663fde7e932ff7 +R b509297d16c0cc9afe0aac10a864ef81 U dan -Z e1912bc11e1db99b09a714bb11e7ca84 +Z 36fc4028e4d720bb78b5659604227df8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f75858ed9c..6554c80932 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -48505ad950bc0902d58210be066d4672e6085eb27c525ba2bc663fde7e932ff7 \ No newline at end of file +8a1ad137606ce5bddbed62f89ff9a5cd922c6530db84c12b8b27c08df3c66bc6 \ No newline at end of file From 5eca6ffe5c6bf323574596c919780022fe15fe26 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 20 Apr 2023 10:40:37 +0000 Subject: [PATCH 247/341] Fix a function declaration so that it correctly appears as private in the amalgamation. [forum:/forumpost/ec8c77516c|Forum post ec8c77516c]. FossilOrigin-Name: 1864569bb9fe28a5e77f34b199f26ac272dbbb1a6642dacd7e50ddd29aa567b6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3e7412162d..dc6c9a535d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbroken\sassert()\sin\sthe\srecovery\sextension.\s[forum:/forumpost/82a88f5812|Forum\spost\s82a88f5812]. -D 2023-04-20T10:26:56.069 +C Fix\sa\sfunction\sdeclaration\sso\sthat\sit\scorrectly\sappears\sas\sprivate\sin\nthe\samalgamation.\s\s[forum:/forumpost/ec8c77516c|Forum\spost\sec8c77516c]. +D 2023-04-20T10:40:37.868 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 -F src/vdbeaux.c 913801bb824d6ef0a49c80e973563063e453f41e6c382c5940de8e5d34f0cd56 +F src/vdbeaux.c 24637a004cda26aca1970a78ebfb9f7895aec820525bdb784a76933bb4af83f7 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 1d9a0f37b0097fbb53f0d7ba081f7181b83cee2c6f46364706ea0c3896bd8ec0 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 48505ad950bc0902d58210be066d4672e6085eb27c525ba2bc663fde7e932ff7 -R b509297d16c0cc9afe0aac10a864ef81 -U dan -Z 36fc4028e4d720bb78b5659604227df8 +P 8a1ad137606ce5bddbed62f89ff9a5cd922c6530db84c12b8b27c08df3c66bc6 +R 5650bdd0fe3fa605ffb40959b74f4f18 +U drh +Z 9167a0af56191794ab500594595d0dd0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6554c80932..ffb8cf29a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a1ad137606ce5bddbed62f89ff9a5cd922c6530db84c12b8b27c08df3c66bc6 \ No newline at end of file +1864569bb9fe28a5e77f34b199f26ac272dbbb1a6642dacd7e50ddd29aa567b6 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 07145bb052..beb8a2f40e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1625,7 +1625,7 @@ VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ /* Return the most recently added opcode */ -VdbeOp * sqlite3VdbeGetLastOp(Vdbe *p){ +VdbeOp *sqlite3VdbeGetLastOp(Vdbe *p){ return sqlite3VdbeGetOp(p, p->nOp - 1); } From 1829f34f46a68f82384bf709f025a88eb620771a Mon Sep 17 00:00:00 2001 From: larrybr Date: Thu, 20 Apr 2023 16:15:20 +0000 Subject: [PATCH 248/341] CLI .ar made to enforce getting a proper subcommand. FossilOrigin-Name: 104ab105ac0e96c0fc61c6d69179bf984fcfe5e0efb2c82f3742456c1b5fe5d1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 5 ++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index dc6c9a535d..7cfd887db4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfunction\sdeclaration\sso\sthat\sit\scorrectly\sappears\sas\sprivate\sin\nthe\samalgamation.\s\s[forum:/forumpost/ec8c77516c|Forum\spost\sec8c77516c]. -D 2023-04-20T10:40:37.868 +C CLI\s.ar\smade\sto\senforce\sgetting\sa\sproper\ssubcommand. +D 2023-04-20T16:15:20.263 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in 9d5da77007123401da702e8587558edddd520c925f8f561aa7b8d9387e71e096 +F src/shell.c.in cea07e8be63c9dd6074ace3840c6d2e10bf3d41e52b0dc784cda6d0cc1c418a9 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8a1ad137606ce5bddbed62f89ff9a5cd922c6530db84c12b8b27c08df3c66bc6 -R 5650bdd0fe3fa605ffb40959b74f4f18 -U drh -Z 9167a0af56191794ab500594595d0dd0 +P 1864569bb9fe28a5e77f34b199f26ac272dbbb1a6642dacd7e50ddd29aa567b6 +R fb998b3563ae8890a9efea4cceb34c23 +U larrybr +Z 658a3fb9fe729e826e83c171d4af40ae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ffb8cf29a5..9a2110089f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1864569bb9fe28a5e77f34b199f26ac272dbbb1a6642dacd7e50ddd29aa567b6 \ No newline at end of file +104ab105ac0e96c0fc61c6d69179bf984fcfe5e0efb2c82f3742456c1b5fe5d1 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 8ce845ba4d..4d4b172c27 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -7058,7 +7058,10 @@ static int arParseCommand( } } } - + if( pAr->eCmd==0 ){ + utf8_printf(stderr, "Required argument missing. Usage:\n"); + return arUsage(stderr); + } return SQLITE_OK; } From 88ed72349030f5e0f79b1dcaab5ff1ae2223a25f Mon Sep 17 00:00:00 2001 From: larrybr Date: Thu, 20 Apr 2023 20:07:14 +0000 Subject: [PATCH 249/341] Increase CLI tolerance of arifically low memory. FossilOrigin-Name: 13f2638622871db9b91db547dcfd38f61c03e7696e4df5144adefcd86140aeba --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 43 +++++++++++++++++++++++++------------------ 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 7cfd887db4..d0a4959f5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C CLI\s.ar\smade\sto\senforce\sgetting\sa\sproper\ssubcommand. -D 2023-04-20T16:15:20.263 +C Increase\sCLI\stolerance\sof\sarifically\slow\smemory. +D 2023-04-20T20:07:14.732 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in cea07e8be63c9dd6074ace3840c6d2e10bf3d41e52b0dc784cda6d0cc1c418a9 +F src/shell.c.in efd4b8def6be9cdd5d8d3eb17940bc23cfce1ee5bdd419fdda933dd601de6e34 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1864569bb9fe28a5e77f34b199f26ac272dbbb1a6642dacd7e50ddd29aa567b6 -R fb998b3563ae8890a9efea4cceb34c23 +P 104ab105ac0e96c0fc61c6d69179bf984fcfe5e0efb2c82f3742456c1b5fe5d1 +R aa97778045cfce91a857103f91418d38 U larrybr -Z 658a3fb9fe729e826e83c171d4af40ae +Z aaec46b2f4c843c0506897156fa41de9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9a2110089f..3151763d58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -104ab105ac0e96c0fc61c6d69179bf984fcfe5e0efb2c82f3742456c1b5fe5d1 \ No newline at end of file +13f2638622871db9b91db547dcfd38f61c03e7696e4df5144adefcd86140aeba \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 4d4b172c27..2ef7045b96 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10464,28 +10464,35 @@ static int do_meta_command(char *zLine, ShellState *p){ shell_check_oom(zRevText); if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); - assert(lrc==SQLITE_OK); - if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); - lrc = SQLITE_ROW==sqlite3_step(pStmt); - if( lrc ){ - const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); - sqlite3_stmt *pCheckStmt; - lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); - if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); - if( SQLITE_OK==lrc ){ - if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){ - double countIrreversible = sqlite3_column_double(pCheckStmt, 0); - if( countIrreversible>0 ){ - int sz = (int)(countIrreversible + 0.5); - utf8_printf(stderr, - "Digest includes %d invalidly encoded text field%s.\n", - sz, (sz>1)? "s": ""); + if( lrc!=SQLITE_OK ){ + assert(lrc==SQLITE_NOMEM); + rc = 1; + }else{ + if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); + lrc = SQLITE_ROW==sqlite3_step(pStmt); + if( lrc ){ + const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); + sqlite3_stmt *pCheckStmt; + lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); + if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); + if( lrc!=SQLITE_OK ){ + rc = 1; + }else{ + if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){ + double countIrreversible = sqlite3_column_double(pCheckStmt, 0); + if( countIrreversible>0 ){ + int sz = (int)(countIrreversible + 0.5); + utf8_printf(stderr, + "Digest includes %d invalidly encoded text field%s.\n", + sz, (sz>1)? "s": ""); + } } + sqlite3_finalize(pCheckStmt); } - sqlite3_finalize(pCheckStmt); + sqlite3_finalize(pStmt); } - sqlite3_finalize(pStmt); } + if( rc ) utf8_printf(stderr, ".sha3sum failed.\n"); sqlite3_free(zRevText); } #endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */ From 0e38c6e696620bb3d519c7ca06df7dae4090c1b6 Mon Sep 17 00:00:00 2001 From: stephan Date: Fri, 21 Apr 2023 15:37:33 +0000 Subject: [PATCH 250/341] Change the return type of the ts_read/write() family of functions from int to ssize_t, per report in [forum post 947169d5e7](forum:947169d5e7). FossilOrigin-Name: 7809e7ce6a70657b8ea239eb4778698f7986a658e9177a57b2fb7814c069c936 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_syscall.c | 26 +++++++++++++------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index d0a4959f5a..1b419b5394 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sCLI\stolerance\sof\sarifically\slow\smemory. -D 2023-04-20T20:07:14.732 +C Change\sthe\sreturn\stype\sof\sthe\sts_read/write()\sfamily\sof\sfunctions\sfrom\sint\sto\sssize_t,\sper\sreport\sin\s[forum\spost\s947169d5e7](forum:947169d5e7). +D 2023-04-21T15:37:33.313 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -682,7 +682,7 @@ F src/test_rtree.c 671f3fae50ff116ef2e32a3bf1fe21b5615b4b7b F src/test_schema.c f5d6067dfc2f2845c4dd56df63e66ee826fb23877855c785f75cc2ca83fd0c1b F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e -F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 +F src/test_syscall.c 9fdb13b1df05e639808d44fcb8f6064aaded32b6565c00b215cfd05a060d1aca F src/test_tclsh.c 3ff5d188a72f00807425954ea3b493dfd3a4b890ecc6700ea83bad2fd1332ecf F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 7ddcf0c8b79fa3c1d172f82f322302c963d923cdb503c6171f3c8081586d0b01 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 104ab105ac0e96c0fc61c6d69179bf984fcfe5e0efb2c82f3742456c1b5fe5d1 -R aa97778045cfce91a857103f91418d38 -U larrybr -Z aaec46b2f4c843c0506897156fa41de9 +P 13f2638622871db9b91db547dcfd38f61c03e7696e4df5144adefcd86140aeba +R e9a0afe7218502491faa04d7c99873d5 +U stephan +Z 5f0988cb68325b7fd3f7bb2bc00ecc24 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3151763d58..2640ecdc68 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13f2638622871db9b91db547dcfd38f61c03e7696e4df5144adefcd86140aeba \ No newline at end of file +7809e7ce6a70657b8ea239eb4778698f7986a658e9177a57b2fb7814c069c936 \ No newline at end of file diff --git a/src/test_syscall.c b/src/test_syscall.c index 947f9a9d9d..3cd1034d3f 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -110,15 +110,15 @@ static int ts_stat(const char *zPath, struct stat *p); static int ts_fstat(int fd, struct stat *p); static int ts_ftruncate(int fd, off_t n); static int ts_fcntl(int fd, int cmd, ... ); -static int ts_read(int fd, void *aBuf, size_t nBuf); -static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off); +static ssize_t ts_read(int fd, void *aBuf, size_t nBuf); +static ssize_t ts_pread(int fd, void *aBuf, size_t nBuf, off_t off); /* Note: pread64() and pwrite64() actually use off64_t as the type on their ** last parameter. But that datatype is not defined on many systems ** (ex: Mac, OpenBSD). So substitute a likely equivalent: sqlite3_uint64 */ -static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off); -static int ts_write(int fd, const void *aBuf, size_t nBuf); -static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); -static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off); +static ssize_t ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off); +static ssize_t ts_write(int fd, const void *aBuf, size_t nBuf); +static ssize_t ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); +static ssize_t ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); static void *ts_mmap(void *, size_t, int, int, int, off_t); @@ -211,7 +211,7 @@ static int tsErrno(const char *zFunc){ /* ** A wrapper around tsIsFail(). If tsIsFail() returns non-zero, set the ** value of errno before returning. -*/ +*/ static int tsIsFailErrno(const char *zFunc){ if( tsIsFail() ){ errno = tsErrno(zFunc); @@ -313,7 +313,7 @@ static int ts_fcntl(int fd, int cmd, ... ){ /* ** A wrapper around read(). */ -static int ts_read(int fd, void *aBuf, size_t nBuf){ +static ssize_t ts_read(int fd, void *aBuf, size_t nBuf){ if( tsIsFailErrno("read") ){ return -1; } @@ -323,7 +323,7 @@ static int ts_read(int fd, void *aBuf, size_t nBuf){ /* ** A wrapper around pread(). */ -static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){ +static ssize_t ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){ if( tsIsFailErrno("pread") ){ return -1; } @@ -333,7 +333,7 @@ static int ts_pread(int fd, void *aBuf, size_t nBuf, off_t off){ /* ** A wrapper around pread64(). */ -static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){ +static ssize_t ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){ if( tsIsFailErrno("pread64") ){ return -1; } @@ -343,7 +343,7 @@ static int ts_pread64(int fd, void *aBuf, size_t nBuf, sqlite3_uint64 off){ /* ** A wrapper around write(). */ -static int ts_write(int fd, const void *aBuf, size_t nBuf){ +static ssize_t ts_write(int fd, const void *aBuf, size_t nBuf){ if( tsIsFailErrno("write") ){ if( tsErrno("write")==EINTR ) orig_write(fd, aBuf, nBuf/2); return -1; @@ -354,7 +354,7 @@ static int ts_write(int fd, const void *aBuf, size_t nBuf){ /* ** A wrapper around pwrite(). */ -static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){ +static ssize_t ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){ if( tsIsFailErrno("pwrite") ){ return -1; } @@ -364,7 +364,7 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off){ /* ** A wrapper around pwrite64(). */ -static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off){ +static ssize_t ts_pwrite64(int fd, const void *aBuf, size_t nBuf, sqlite3_uint64 off){ if( tsIsFailErrno("pwrite64") ){ return -1; } From 5870e942c2523fadb7a09f784c7218e6e4c6fe24 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 22 Apr 2023 09:14:22 +0000 Subject: [PATCH 251/341] Fix unchecked return in CLI .dbi command. ([forum 79e2cd1bc](forum:)79e2cd1bc) FossilOrigin-Name: 81a74ee9d66ba23dfadd875e372ce96491dd6bf146e03e7fec8c3fd98e6be825 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1b419b5394..ed423c62d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sreturn\stype\sof\sthe\sts_read/write()\sfamily\sof\sfunctions\sfrom\sint\sto\sssize_t,\sper\sreport\sin\s[forum\spost\s947169d5e7](forum:947169d5e7). -D 2023-04-21T15:37:33.313 +C Fix\sunchecked\sreturn\sin\sCLI\s.dbi\scommand.\s([forum\s79e2cd1bc](forum:)79e2cd1bc) +D 2023-04-22T09:14:22.636 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in efd4b8def6be9cdd5d8d3eb17940bc23cfce1ee5bdd419fdda933dd601de6e34 +F src/shell.c.in bc9c19ce1ed333a588efe9938f95737134ae2cfee8b9fa068f02eb0911667369 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 13f2638622871db9b91db547dcfd38f61c03e7696e4df5144adefcd86140aeba -R e9a0afe7218502491faa04d7c99873d5 -U stephan -Z 5f0988cb68325b7fd3f7bb2bc00ecc24 +P 7809e7ce6a70657b8ea239eb4778698f7986a658e9177a57b2fb7814c069c936 +R 971884221506697aa42bffd1340dd9c8 +U larrybr +Z 474a941f83c00eb962a4baaa47422704 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2640ecdc68..92d90b16e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7809e7ce6a70657b8ea239eb4778698f7986a658e9177a57b2fb7814c069c936 \ No newline at end of file +81a74ee9d66ba23dfadd875e372ce96491dd6bf146e03e7fec8c3fd98e6be825 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 2ef7045b96..8d65071e4e 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6241,7 +6241,9 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ if( sqlite3_step(pStmt)==SQLITE_ROW && sqlite3_column_bytes(pStmt,0)>100 ){ - memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100); + const u8 *pb = sqlite3_column_blob(pStmt,0); + shell_check_oom((void*)pb); + memcpy(aHdr, pb, 100); sqlite3_finalize(pStmt); }else{ raw_printf(stderr, "unable to read database header\n"); From c40a7c82fac9395018056cac27689852232a3b44 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 22 Apr 2023 09:26:36 +0000 Subject: [PATCH 252/341] Avoid need for cast in CLI shell_check_oom() calls. FossilOrigin-Name: c88550ef6711b03a81e2e6d121e2eb99ccff6e925c025dd6e85dd81088508400 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ed423c62d2..d6cd4f55c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sunchecked\sreturn\sin\sCLI\s.dbi\scommand.\s([forum\s79e2cd1bc](forum:)79e2cd1bc) -D 2023-04-22T09:14:22.636 +C Avoid\sneed\sfor\scast\sin\sCLI\sshell_check_oom()\scalls. +D 2023-04-22T09:26:36.892 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in bc9c19ce1ed333a588efe9938f95737134ae2cfee8b9fa068f02eb0911667369 +F src/shell.c.in ac2fd1e1c0c0c5b1b3fcb3eaf0979599f8741a7ba3dfd9746b0c58fd67aa4091 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7809e7ce6a70657b8ea239eb4778698f7986a658e9177a57b2fb7814c069c936 -R 971884221506697aa42bffd1340dd9c8 +P 81a74ee9d66ba23dfadd875e372ce96491dd6bf146e03e7fec8c3fd98e6be825 +R 888832a9c27abfed9d01271116b52d4d U larrybr -Z 474a941f83c00eb962a4baaa47422704 +Z 2e700a9c3ffdbd06ea1f995ee54b1148 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 92d90b16e7..43336de394 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -81a74ee9d66ba23dfadd875e372ce96491dd6bf146e03e7fec8c3fd98e6be825 \ No newline at end of file +c88550ef6711b03a81e2e6d121e2eb99ccff6e925c025dd6e85dd81088508400 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 8d65071e4e..49cdc98450 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -765,7 +765,7 @@ static void shell_out_of_memory(void){ /* Check a pointer to see if it is NULL. If it is NULL, exit with an ** out-of-memory error. */ -static void shell_check_oom(void *p){ +static void shell_check_oom(const void *p){ if( p==0 ) shell_out_of_memory(); } @@ -3810,7 +3810,7 @@ static void exec_prepared_stmt_columnar( azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); shell_check_oom(azData); azNextLine = sqlite3_malloc64( nColumn*sizeof(char*) ); - shell_check_oom((void*)azNextLine); + shell_check_oom(azNextLine); memset((void*)azNextLine, 0, nColumn*sizeof(char*) ); if( p->cmOpts.bQuote ){ azQuoted = sqlite3_malloc64( nColumn*sizeof(char*) ); @@ -6242,7 +6242,7 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ && sqlite3_column_bytes(pStmt,0)>100 ){ const u8 *pb = sqlite3_column_blob(pStmt,0); - shell_check_oom((void*)pb); + shell_check_oom(pb); memcpy(aHdr, pb, 100); sqlite3_finalize(pStmt); }else{ From 8bfa190e9a0c962ad95245b7cf8568f2ef628f8b Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 22 Apr 2023 11:24:52 +0000 Subject: [PATCH 253/341] CLI to handle absurd string length limit better. [forum:5180af725f1cc375|forum post 5180af725f1cc375] FossilOrigin-Name: 1489e7f53a4863b316af6dee10d9c66491d1bed6cf52a3538ac118c03b1d7ef9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d6cd4f55c8..e2a4842d8c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sneed\sfor\scast\sin\sCLI\sshell_check_oom()\scalls. -D 2023-04-22T09:26:36.892 +C CLI\sto\shandle\sabsurd\sstring\slength\slimit\sbetter.\s[forum:5180af725f1cc375|forum\spost\s5180af725f1cc375] +D 2023-04-22T11:24:52.124 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in ac2fd1e1c0c0c5b1b3fcb3eaf0979599f8741a7ba3dfd9746b0c58fd67aa4091 +F src/shell.c.in b39dd30addf6ac8d56087ea920aa303e257b0fb8a3f800110a4d5b4be1a8d12f F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 81a74ee9d66ba23dfadd875e372ce96491dd6bf146e03e7fec8c3fd98e6be825 -R 888832a9c27abfed9d01271116b52d4d +P c88550ef6711b03a81e2e6d121e2eb99ccff6e925c025dd6e85dd81088508400 +R c1d466045065a44b94ee36b4a1ca6613 U larrybr -Z 2e700a9c3ffdbd06ea1f995ee54b1148 +Z f005e7b2e85ae20e6a18880efb30cee5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 43336de394..7b1e17a7e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c88550ef6711b03a81e2e6d121e2eb99ccff6e925c025dd6e85dd81088508400 \ No newline at end of file +1489e7f53a4863b316af6dee10d9c66491d1bed6cf52a3538ac118c03b1d7ef9 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 49cdc98450..f9669dc9c3 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3840,6 +3840,7 @@ static void exec_prepared_stmt_columnar( } if( wx<0 ) wx = -wx; uz = (const unsigned char*)sqlite3_column_name(pStmt,i); + if( uz==0 ) uz = ""; azData[i] = translateForDisplayAndDup(uz, &zNotUsed, wx, bw); } do{ From d9eb39e6257aba89b84de00069baeb0bd85afca1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Apr 2023 11:29:36 +0000 Subject: [PATCH 254/341] Early out from sqlite3Prepare() following an OOM to avoid possible problems further along in the parse. Fix for the NULL pointer dereference reported by [forum:/info/2e5131839365682a|forum post 2e5131839365682a]. FossilOrigin-Name: f35ce7c122017009496b777f33e641d0be086abee6ea8e25aab4c3e0fb45731a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/parse.y | 2 +- src/prepare.c | 6 +++++- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e2a4842d8c..3f95a28d87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C CLI\sto\shandle\sabsurd\sstring\slength\slimit\sbetter.\s[forum:5180af725f1cc375|forum\spost\s5180af725f1cc375] -D 2023-04-22T11:24:52.124 +C Early\sout\sfrom\ssqlite3Prepare()\sfollowing\san\sOOM\sto\savoid\spossible\nproblems\sfurther\salong\sin\sthe\sparse.\s\sFix\sfor\sthe\sNULL\spointer\sdereference\nreported\sby\s[forum:/info/2e5131839365682a|forum\spost\s2e5131839365682a]. +D 2023-04-22T11:29:36.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -621,13 +621,13 @@ F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 9d36ddedc842e993c88c222ed914822dbd6f8ece3c648fde04468637012a034a F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 -F src/parse.y e8b5c753e3194e03d69e108753c1dbceb01fa4c158b2c6b726a048023ea0fdc1 +F src/parse.y 146f9a1db7db5ef4299c6897d335e5abed348c2626190d2877d45ffa210fd4ca F src/pcache.c 8ee13acccfd9accbf0af94910b7323dd7f7d55300d92ddafcf40e34fcc8e21be F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc F src/pragma.c 26ed2cfdc5c12aa1c707178635709684960288cacc9cff9d491a38ff10e395f1 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7 -F src/prepare.c 06ecbb25db151a009be99471757c73272cd61306903dc3eeb7250bdcf54896dd +F src/prepare.c 6350675966bd0e7ac3a464af9dbfe26db6f0d4237f4e1f1acdb17b12ad371e6e F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c88550ef6711b03a81e2e6d121e2eb99ccff6e925c025dd6e85dd81088508400 -R c1d466045065a44b94ee36b4a1ca6613 -U larrybr -Z f005e7b2e85ae20e6a18880efb30cee5 +P 1489e7f53a4863b316af6dee10d9c66491d1bed6cf52a3538ac118c03b1d7ef9 +R 462d4d9e4d63b37d965c734364dc27f6 +U drh +Z 62d7b2c15834aaaa388f530f49fabdda # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7b1e17a7e8..96b92f2556 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1489e7f53a4863b316af6dee10d9c66491d1bed6cf52a3538ac118c03b1d7ef9 \ No newline at end of file +f35ce7c122017009496b777f33e641d0be086abee6ea8e25aab4c3e0fb45731a \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 42876feebb..541b9ffe58 100644 --- a/src/parse.y +++ b/src/parse.y @@ -715,7 +715,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N seltablist(A) ::= stl_prefix(A) LP seltablist(F) RP as(Z) on_using(N). { if( A==0 && Z.n==0 && N.pOn==0 && N.pUsing==0 ){ A = F; - }else if( F->nSrc==1 ){ + }else if( ALWAYS(F!=0) && F->nSrc==1 ){ A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,&N); if( A ){ SrcItem *pNew = &A->a[A->nSrc-1]; diff --git a/src/prepare.c b/src/prepare.c index 3f87a37199..ff48265023 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -702,7 +702,11 @@ static int sqlite3Prepare( sParse.db = db; sParse.pReprepare = pReprepare; assert( ppStmt && *ppStmt==0 ); - if( db->mallocFailed ) sqlite3ErrorMsg(&sParse, "out of memory"); + if( db->mallocFailed ){ + sqlite3ErrorMsg(&sParse, "out of memory"); + db->errCode = rc = SQLITE_NOMEM; + goto end_prepare; + } assert( sqlite3_mutex_held(db->mutex) ); /* For a long-term use prepared statement avoid the use of From 1a2c7f9029b94a08bcec2511e08b4a0faf7973d5 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 22 Apr 2023 11:33:27 +0000 Subject: [PATCH 255/341] Also fix no-length string intolerance for CLI json mode. FossilOrigin-Name: 3ac1984039a50445ff65db97468a5313d7f09db7b604755e444b77ac9346ea76 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3f95a28d87..d27d12b6cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Early\sout\sfrom\ssqlite3Prepare()\sfollowing\san\sOOM\sto\savoid\spossible\nproblems\sfurther\salong\sin\sthe\sparse.\s\sFix\sfor\sthe\sNULL\spointer\sdereference\nreported\sby\s[forum:/info/2e5131839365682a|forum\spost\s2e5131839365682a]. -D 2023-04-22T11:29:36.514 +C Also\sfix\sno-length\sstring\sintolerance\sfor\sCLI\sjson\smode. +D 2023-04-22T11:33:27.966 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in b39dd30addf6ac8d56087ea920aa303e257b0fb8a3f800110a4d5b4be1a8d12f +F src/shell.c.in c9c032ea6a8f75fc844b1eac0a5b97baab77442796953de7e61e7931a9986245 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1489e7f53a4863b316af6dee10d9c66491d1bed6cf52a3538ac118c03b1d7ef9 -R 462d4d9e4d63b37d965c734364dc27f6 -U drh -Z 62d7b2c15834aaaa388f530f49fabdda +P f35ce7c122017009496b777f33e641d0be086abee6ea8e25aab4c3e0fb45731a +R 5863e993e7a680782da9f66708ce9aeb +U larrybr +Z 96dbf988c0ae45ea8aa1e82d19e17cea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 96b92f2556..ff33bdb930 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f35ce7c122017009496b777f33e641d0be086abee6ea8e25aab4c3e0fb45731a \ No newline at end of file +3ac1984039a50445ff65db97468a5313d7f09db7b604755e444b77ac9346ea76 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index f9669dc9c3..1eb8c340d1 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1988,6 +1988,7 @@ static void output_c_string(FILE *out, const char *z){ */ static void output_json_string(FILE *out, const char *z, i64 n){ unsigned int c; + if( z==0 ) z = ""; if( n<0 ) n = strlen(z); fputc('"', out); while( n-- ){ From bbe198676185a549eb5a05f49dd1d75df412a6ad Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Apr 2023 12:11:31 +0000 Subject: [PATCH 256/341] Fix harmless compiler warning in [1489e7f53a4863b3]. FossilOrigin-Name: da907dbc60b4f754845ae0bedc05cbda4cbfed9f468be3ffdc18860c763d9324 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d27d12b6cd..ea92fcbd2d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Also\sfix\sno-length\sstring\sintolerance\sfor\sCLI\sjson\smode. -D 2023-04-22T11:33:27.966 +C Fix\sharmless\scompiler\swarning\sin\s[1489e7f53a4863b3]. +D 2023-04-22T12:11:31.666 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in c9c032ea6a8f75fc844b1eac0a5b97baab77442796953de7e61e7931a9986245 +F src/shell.c.in dffb9fd88cd10b64267077edb98cb6792a014dc89dd34891b8f3a45906b6a16d F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f35ce7c122017009496b777f33e641d0be086abee6ea8e25aab4c3e0fb45731a -R 5863e993e7a680782da9f66708ce9aeb -U larrybr -Z 96dbf988c0ae45ea8aa1e82d19e17cea +P 3ac1984039a50445ff65db97468a5313d7f09db7b604755e444b77ac9346ea76 +R a7018966659aa9504cb3419068cca920 +U drh +Z 03ef28817b511e683f2c329e4cadf0e1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ff33bdb930..95c266f4c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ac1984039a50445ff65db97468a5313d7f09db7b604755e444b77ac9346ea76 \ No newline at end of file +da907dbc60b4f754845ae0bedc05cbda4cbfed9f468be3ffdc18860c763d9324 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1eb8c340d1..343e03c3a9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3841,7 +3841,7 @@ static void exec_prepared_stmt_columnar( } if( wx<0 ) wx = -wx; uz = (const unsigned char*)sqlite3_column_name(pStmt,i); - if( uz==0 ) uz = ""; + if( uz==0 ) uz = (u8*)""; azData[i] = translateForDisplayAndDup(uz, &zNotUsed, wx, bw); } do{ From fb50f96121a7b6b6a3304ac3ea6e40b260cfd124 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Apr 2023 12:37:41 +0000 Subject: [PATCH 257/341] Fix a incorrect assert() statement in the pre-update hook logic. No changes to production code. Fix for the problem reported by [forum:/info/19b217bfe709a072|forum post 19b217bfe709a072]. FossilOrigin-Name: 53a61f7423a7f057568a1d8a1e63be01a2328997850dc32e3d8c6d51f1c5ab0b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ea92fcbd2d..b856026588 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\s[1489e7f53a4863b3]. -D 2023-04-22T12:11:31.666 +C Fix\sa\sincorrect\sassert()\sstatement\sin\sthe\spre-update\shook\slogic.\s\sNo\schanges\nto\sproduction\scode.\s\sFix\sfor\sthe\sproblem\sreported\sby\n[forum:/info/19b217bfe709a072|forum\spost\s19b217bfe709a072]. +D 2023-04-22T12:37:41.683 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 -F src/vdbeaux.c 24637a004cda26aca1970a78ebfb9f7895aec820525bdb784a76933bb4af83f7 +F src/vdbeaux.c 958bb807a3756c34fc6e434065e134fa3ae5f7e46da615224e216b33d5dd4c8f F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 1d9a0f37b0097fbb53f0d7ba081f7181b83cee2c6f46364706ea0c3896bd8ec0 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3ac1984039a50445ff65db97468a5313d7f09db7b604755e444b77ac9346ea76 -R a7018966659aa9504cb3419068cca920 +P da907dbc60b4f754845ae0bedc05cbda4cbfed9f468be3ffdc18860c763d9324 +R 36e71f1e432856694c39827bc9e4c8a4 U drh -Z 03ef28817b511e683f2c329e4cadf0e1 +Z 656f30b58cde31ad921d5c9abfca4d0e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 95c266f4c7..5043c78c87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da907dbc60b4f754845ae0bedc05cbda4cbfed9f468be3ffdc18860c763d9324 \ No newline at end of file +53a61f7423a7f057568a1d8a1e63be01a2328997850dc32e3d8c6d51f1c5ab0b \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index beb8a2f40e..71813baa5b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -5354,6 +5354,9 @@ void sqlite3VdbePreUpdateHook( PreUpdate preupdate; const char *zTbl = pTab->zName; static const u8 fakeSortOrder = 0; +#ifdef SQLITE_DEBUG + int nRealCol = (pTab->tabFlags & TF_HasVirtual) ? pTab->nNVCol : pTab->nCol; +#endif assert( db->pPreUpdate==0 ); memset(&preupdate, 0, sizeof(PreUpdate)); @@ -5370,8 +5373,8 @@ void sqlite3VdbePreUpdateHook( assert( pCsr!=0 ); assert( pCsr->eCurType==CURTYPE_BTREE ); - assert( pCsr->nField==pTab->nCol - || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1) + assert( pCsr->nField==nRealCol + || (pCsr->nField==nRealCol+1 && op==SQLITE_DELETE && iReg==-1) ); preupdate.v = v; From a9c4c82ef1197681bb78a4ea1c996fc678fa3e3e Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Apr 2023 12:47:16 +0000 Subject: [PATCH 258/341] Fix an incorrect assert() statement in btree.c as found by [forum:/info/d03345d572713fe6|forum post d03345d572713fe6]. FossilOrigin-Name: cd485b302c54aef066d751a153df34eea0cd23e54a414e291f636ad9929fe78a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b856026588..eb3909da76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sincorrect\sassert()\sstatement\sin\sthe\spre-update\shook\slogic.\s\sNo\schanges\nto\sproduction\scode.\s\sFix\sfor\sthe\sproblem\sreported\sby\n[forum:/info/19b217bfe709a072|forum\spost\s19b217bfe709a072]. -D 2023-04-22T12:37:41.683 +C Fix\san\sincorrect\sassert()\sstatement\sin\sbtree.c\sas\sfound\sby\n[forum:/info/d03345d572713fe6|forum\spost\sd03345d572713fe6]. +D 2023-04-22T12:47:16.752 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -571,7 +571,7 @@ F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 F src/bitvec.c 7c849aac407230278445cb069bebc5f89bf2ddd87c5ed9459b070a9175707b3d F src/btmutex.c 6ffb0a22c19e2f9110be0964d0731d2ef1c67b5f7fabfbaeb7b9dabc4b7740ca -F src/btree.c af379d801906f21dc18b534fc250a479f56045a450d8d5859ba2f57fd9eefcef +F src/btree.c 37292454e4f5cd148b3c661770df44e28060f2cefd227ad979bf9b5f155db11b F src/btree.h aa354b9bad4120af71e214666b35132712b8f2ec11869cb2315c52c81fad45cc F src/btreeInt.h a3268a60cbc91f578001f44ba40aae9c1b8aecbb0d2c095dd7fc54b0872ea4b8 F src/build.c 8357d6ca9a8c9afc297c431df28bc2af407b47f3ef2311875276c944b30c4d54 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P da907dbc60b4f754845ae0bedc05cbda4cbfed9f468be3ffdc18860c763d9324 -R 36e71f1e432856694c39827bc9e4c8a4 +P 53a61f7423a7f057568a1d8a1e63be01a2328997850dc32e3d8c6d51f1c5ab0b +R 5677d2da177f240210053dffbed7c16a U drh -Z 656f30b58cde31ad921d5c9abfca4d0e +Z c7873e11c59de450a8f61d4742b32434 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5043c78c87..11423ccd2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53a61f7423a7f057568a1d8a1e63be01a2328997850dc32e3d8c6d51f1c5ab0b \ No newline at end of file +cd485b302c54aef066d751a153df34eea0cd23e54a414e291f636ad9929fe78a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 40a94a520d..e6a694566f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9333,7 +9333,7 @@ int sqlite3BtreeInsert( } } assert( pCur->eState==CURSOR_VALID - || (pCur->eState==CURSOR_INVALID && loc) ); + || (pCur->eState==CURSOR_INVALID && loc) || CORRUPT_DB ); pPage = pCur->pPage; assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) ); From e945b293596fe09390c89c0749979979514d68de Mon Sep 17 00:00:00 2001 From: larrybr Date: Sat, 22 Apr 2023 16:46:38 +0000 Subject: [PATCH 259/341] Add --unsafe-testing invocation option to CLI. Needs some tests added and changed. FossilOrigin-Name: b3d9ac052d5c2dd1afeeeb5c9cfac9dd91a1b8d6a74a2ef10aa2037ca505abce --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/shell.c.in | 25 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index eb3909da76..b7d8dc7b85 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\sassert()\sstatement\sin\sbtree.c\sas\sfound\sby\n[forum:/info/d03345d572713fe6|forum\spost\sd03345d572713fe6]. -D 2023-04-22T12:47:16.752 +C Add\s--unsafe-testing\sinvocation\soption\sto\sCLI.\sNeeds\ssome\stests\sadded\sand\schanged. +D 2023-04-22T16:46:38.465 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in dffb9fd88cd10b64267077edb98cb6792a014dc89dd34891b8f3a45906b6a16d +F src/shell.c.in f85b07dc6d75d17217f6a842fa9c43c6ef6db6551b51be76055b814fe7a24376 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 53a61f7423a7f057568a1d8a1e63be01a2328997850dc32e3d8c6d51f1c5ab0b -R 5677d2da177f240210053dffbed7c16a -U drh -Z c7873e11c59de450a8f61d4742b32434 +P cd485b302c54aef066d751a153df34eea0cd23e54a414e291f636ad9929fe78a +R be58d94e52d56c14dee82adb68874a9b +T *branch * shell-for-test +T *sym-shell-for-test * +T -sym-trunk * +U larrybr +Z a5e6fd5b494a63d53c430515997654b1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 11423ccd2a..333cac216c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd485b302c54aef066d751a153df34eea0cd23e54a414e291f636ad9929fe78a \ No newline at end of file +b3d9ac052d5c2dd1afeeeb5c9cfac9dd91a1b8d6a74a2ef10aa2037ca505abce \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 343e03c3a9..c6007c5e7e 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1526,6 +1526,7 @@ static ShellState shellState; #define SHFLG_HeaderSet 0x00000080 /* showHeader has been specified */ #define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ #define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ +#define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ /* ** Macros for testing and setting shellFlgs @@ -5351,6 +5352,13 @@ static void open_db(ShellState *p, int openFlags){ } sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0); + /* Reflect the use or absence of --unsafe-testing invocation. */ + { + int testmode_on = ShellHasFlag(p,SHFLG_TestingMode); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_TRUSTED_SCHEMA, testmode_on,0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, !testmode_on,0); + } + #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif @@ -8960,6 +8968,12 @@ static int do_meta_command(char *zLine, ShellState *p){ int isWO = 0; /* True if making an imposter of a WITHOUT ROWID table */ int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */ int i; + if( !ShellHasFlag(p,SHFLG_TestingMode) ){ + utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n", + "imposter"); + rc = 1; + goto meta_command_exit; + } if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){ utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n" " .imposter off\n"); @@ -10759,6 +10773,12 @@ static int do_meta_command(char *zLine, ShellState *p){ int i, n2; const char *zCmd = 0; + if( !ShellHasFlag(p,SHFLG_TestingMode) ){ + utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n", + "testctrl"); + rc = 1; + goto meta_command_exit; + } open_db(p, 0); zCmd = nArg>=2 ? azArg[1] : "help"; @@ -11758,6 +11778,7 @@ static const char zOptions[] = " -stats print memory stats before each finalize\n" " -table set output mode to 'table'\n" " -tabs set output mode to 'tabs'\n" + " -unsafe-testing allow unsafe commands and modes for testing\n" #if SHELL_WIN_UTF8_OPT " -utf8 setup interactive console code page for UTF-8\n" #endif @@ -12138,6 +12159,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-nonce")==0 ){ free(data.zNonce); data.zNonce = strdup(argv[++i]); + }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ + ShellSetFlag(&data,SHFLG_TestingMode); }else if( cli_strcmp(z,"-safe")==0 ){ /* no-op - catch this on the second pass */ } @@ -12374,6 +12397,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif }else if( cli_strcmp(z,"-safe")==0 ){ data.bSafeMode = data.bSafeModePersist = 1; + }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ + /* Acted upon in first pass. */ }else{ utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); raw_printf(stderr,"Use -help for a list of options.\n"); From f661db305078d6109ff6adbba73e56bd45877347 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Apr 2023 20:07:48 +0000 Subject: [PATCH 260/341] The assertion-fault fix in [53a61f7423a7f057] was not quite complete. This additional change enhances it to deal with WITHOUT ROWID tables. No changes to deliverable code. FossilOrigin-Name: 4d5af42c651bc1655a4f09b743f2a1424f19c1e4248e041b3fd4a042fd1f9799 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 9 ++++++++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ecf6a80634..d038d58235 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--unsafe-testing\scommand-line\soption\sto\sthe\sCLI.\s\sWithout\sthis\soption,\nfeatures\sof\sthe\sCLI\sthat\sare\sintended\sfor\stesting\sand\sthat\smight\sresult\sin\nincorrect\sanswers,\sassertion\sfaults,\sand/or\scorrupt\sdatabase\sfiles\sif\smisused\nare\sdisabled. -D 2023-04-22T17:41:53.196 +C The\sassertion-fault\sfix\sin\s[53a61f7423a7f057]\swas\snot\squite\scomplete.\s\sThis\nadditional\schange\senhances\sit\sto\sdeal\swith\sWITHOUT\sROWID\stables.\s\sNo\schanges\nto\sdeliverable\scode. +D 2023-04-22T20:07:48.390 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 -F src/vdbeaux.c 958bb807a3756c34fc6e434065e134fa3ae5f7e46da615224e216b33d5dd4c8f +F src/vdbeaux.c 6ee48db408d4c297a363f1e31145c09793a580e7c508bb36063dd017d67117a2 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 1d9a0f37b0097fbb53f0d7ba081f7181b83cee2c6f46364706ea0c3896bd8ec0 F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 @@ -2059,9 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cd485b302c54aef066d751a153df34eea0cd23e54a414e291f636ad9929fe78a b3d9ac052d5c2dd1afeeeb5c9cfac9dd91a1b8d6a74a2ef10aa2037ca505abce -R be58d94e52d56c14dee82adb68874a9b -T +closed b3d9ac052d5c2dd1afeeeb5c9cfac9dd91a1b8d6a74a2ef10aa2037ca505abce +P f28256a96af4263c0caded79b7481b88e9c7e7f1af7dbb3fd5e22ff07fbedaf3 +R 88e863890132282405f3c5aef9851c53 U drh -Z 292d5d1b263a4abd453b30cd2562934c +Z 3ff454ccd561bde5a26c0c888ec2349a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 73fbbb4613..520f13395e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f28256a96af4263c0caded79b7481b88e9c7e7f1af7dbb3fd5e22ff07fbedaf3 \ No newline at end of file +4d5af42c651bc1655a4f09b743f2a1424f19c1e4248e041b3fd4a042fd1f9799 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 71813baa5b..ecbf2d892e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -5355,7 +5355,14 @@ void sqlite3VdbePreUpdateHook( const char *zTbl = pTab->zName; static const u8 fakeSortOrder = 0; #ifdef SQLITE_DEBUG - int nRealCol = (pTab->tabFlags & TF_HasVirtual) ? pTab->nNVCol : pTab->nCol; + int nRealCol; + if( pTab->tabFlags & TF_WithoutRowid ){ + nRealCol = sqlite3PrimaryKeyIndex(pTab)->nColumn; + }else if( pTab->tabFlags & TF_HasVirtual ){ + nRealCol = pTab->nNVCol; + }else{ + nRealCol = pTab->nCol; + } #endif assert( db->pPreUpdate==0 ); From 2f251f4929f3e0dc53fca39d6f9bd4a4a0ef1c24 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Apr 2023 20:22:17 +0000 Subject: [PATCH 261/341] Enhance the ".tables" command in the CLI so that it is able to deal gracefully with an OOM situation. FossilOrigin-Name: 5157fd1cf753b1c01c6c00e9ac2712da2631a673837ed15ec3420ec869ee66aa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d038d58235..e65c6e2989 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sassertion-fault\sfix\sin\s[53a61f7423a7f057]\swas\snot\squite\scomplete.\s\sThis\nadditional\schange\senhances\sit\sto\sdeal\swith\sWITHOUT\sROWID\stables.\s\sNo\schanges\nto\sdeliverable\scode. -D 2023-04-22T20:07:48.390 +C Enhance\sthe\s".tables"\scommand\sin\sthe\sCLI\sso\sthat\sit\sis\sable\sto\sdeal\ngracefully\swith\san\sOOM\ssituation. +D 2023-04-22T20:22:17.341 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in f85b07dc6d75d17217f6a842fa9c43c6ef6db6551b51be76055b814fe7a24376 +F src/shell.c.in 0bf660f2ed0cedb63ed2b029935fd758b3abe68369a08bdc8ad26dd4b402e3a7 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f28256a96af4263c0caded79b7481b88e9c7e7f1af7dbb3fd5e22ff07fbedaf3 -R 88e863890132282405f3c5aef9851c53 +P 4d5af42c651bc1655a4f09b743f2a1424f19c1e4248e041b3fd4a042fd1f9799 +R 6b238a70ef90eb1cf155f65ac9584665 U drh -Z 3ff454ccd561bde5a26c0c888ec2349a +Z 4687c7fa857b1ab28d6d3e205232e7ae # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 520f13395e..6b4ed07253 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d5af42c651bc1655a4f09b743f2a1424f19c1e4248e041b3fd4a042fd1f9799 \ No newline at end of file +5157fd1cf753b1c01c6c00e9ac2712da2631a673837ed15ec3420ec869ee66aa \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c6007c5e7e..c642fbf1ba 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5691,7 +5691,7 @@ static int sql_trace_callback( utf8_printf(p->traceOut, "-- closing database connection\n"); return 0; } - if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + if( mType!=SQLITE_TRACE_ROW && pX!=0 && ((const char*)pX)[0]=='-' ){ zSql = (const char*)pX; }else{ pStmt = (sqlite3_stmt*)pP; From 4d24eff8dc47b6e351c775ebc08df5a866534036 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 22 Apr 2023 20:47:08 +0000 Subject: [PATCH 262/341] Avoid a crash in fts5 when combining an empty expression with another. FossilOrigin-Name: 33681ff3d29c9f096dbd8c5d885993f780e93974a7653adc00fa41a158356d7c --- ext/fts5/fts5_expr.c | 4 ++-- ext/fts5/test/fts5misc.test | 28 ++++++++++++++++++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index e4072db7aa..e87650d47d 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -407,7 +407,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ Fts5Parse sParse; memset(&sParse, 0, sizeof(sParse)); - if( *pp1 ){ + if( *pp1 && p2 ){ Fts5Expr *p1 = *pp1; int nPhrase = p1->nPhrase + p2->nPhrase; @@ -432,7 +432,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ } sqlite3_free(p2->apExprPhrase); sqlite3_free(p2); - }else{ + }else if( p2 ){ *pp1 = p2; } diff --git a/ext/fts5/test/fts5misc.test b/ext/fts5/test/fts5misc.test index 416b4c8085..fb609e2c86 100644 --- a/ext/fts5/test/fts5misc.test +++ b/ext/fts5/test/fts5misc.test @@ -443,5 +443,33 @@ do_execsql_test -db db2 16.6 { SELECT * FROM x1 } {abc def} +#------------------------------------------------------------------------- +reset_db +do_execsql_test 17.1 { + CREATE VIRTUAL TABLE ft USING fts5(x, tokenize="unicode61 separators 'X'"); +} +do_execsql_test 17.2 { + SELECT 0 FROM ft WHERE ft MATCH 'X' AND ft MATCH 'X' +} +do_execsql_test 17.3 { + SELECT 0 FROM ft('X') +} + +do_execsql_test 17.4 { + CREATE VIRTUAL TABLE t0 USING fts5(c0, t="trigram"); + INSERT INTO t0 VALUES('assertionfaultproblem'); +} +do_execsql_test 17.5 { + SELECT 0 FROM t0(0) WHERE c0 GLOB 0; +} {} + +do_execsql_test 17.5 { + SELECT c0 FROM t0 WHERE c0 GLOB '*f*'; +} {assertionfaultproblem} +do_execsql_test 17.5 { + SELECT c0 FROM t0 WHERE c0 GLOB '*faul*'; +} {assertionfaultproblem} + + finish_test diff --git a/manifest b/manifest index e65c6e2989..1c2283c761 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\s".tables"\scommand\sin\sthe\sCLI\sso\sthat\sit\sis\sable\sto\sdeal\ngracefully\swith\san\sOOM\ssituation. -D 2023-04-22T20:22:17.341 +C Avoid\sa\scrash\sin\sfts5\swhen\scombining\san\sempty\sexpression\swith\sanother. +D 2023-04-22T20:47:08.297 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -90,7 +90,7 @@ F ext/fts5/fts5Int.h f473de2bdae0977af0d6c8cce96e3666821b85efba5f6006c7732662c3a F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd -F ext/fts5/fts5_expr.c 48e8e45261c6030cf5c77f606217a22722b1a4d0b34e2ba6cbfc386581627989 +F ext/fts5/fts5_expr.c 7d298d76ea010c339b26ca47f6f69e9aef520ea46c083deaa4e83e87cf0e94b1 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 F ext/fts5/fts5_index.c 17dca8e874df04182bc45063dc0b761acc242b91f1264d2257b7e37bd4e4c2ad F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d @@ -168,7 +168,7 @@ F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc27826807405 F ext/fts5/test/fts5matchinfo.test 10c9a6f7fe61fb132299c4183c012770b10c4d5c2f2edb6df0b6607f683d737a F ext/fts5/test/fts5merge.test e92a8db28b45931e7a9c7b1bbd36101692759d00274df74d83fd29d25d53b3a6 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2 -F ext/fts5/test/fts5misc.test d6d4fdd7ec164e69e50af539137c0565362a4124547bf841ba474f092298637b +F ext/fts5/test/fts5misc.test c02f3e78aa7e62891b5f711c498a877f03252f10a6974e23bc722533233f2603 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45 F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c2765c3ddd2ef58bd @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d5af42c651bc1655a4f09b743f2a1424f19c1e4248e041b3fd4a042fd1f9799 -R 6b238a70ef90eb1cf155f65ac9584665 -U drh -Z 4687c7fa857b1ab28d6d3e205232e7ae +P 5157fd1cf753b1c01c6c00e9ac2712da2631a673837ed15ec3420ec869ee66aa +R 7c91bd0d3c9bcec44ce2ed126717540d +U dan +Z a127cac0e89bcf30ec7e8998adcf95b2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6b4ed07253..4540a49b8a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5157fd1cf753b1c01c6c00e9ac2712da2631a673837ed15ec3420ec869ee66aa \ No newline at end of file +33681ff3d29c9f096dbd8c5d885993f780e93974a7653adc00fa41a158356d7c \ No newline at end of file From 223c6b48a9d68eb4fa6aa884fa78b60fce892495 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 22 Apr 2023 22:32:19 +0000 Subject: [PATCH 263/341] The sqlite_stat4.idx field is case insensitive. It should work even if some entries use a different case than others. Fix for [forum:/info/6c118daad0f1f5ef|forum post 6c118daad0f1f5ef]. FossilOrigin-Name: f097ca70b5b967d1aadebd74ac5020813e00b7c30cc543814dbf5f359d1328f1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 2 +- test/analyze3.test | 13 +++++++++++++ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1c2283c761..dbb82bd7bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\scrash\sin\sfts5\swhen\scombining\san\sempty\sexpression\swith\sanother. -D 2023-04-22T20:47:08.297 +C The\ssqlite_stat4.idx\sfield\sis\scase\sinsensitive.\s\sIt\sshould\swork\seven\sif\nsome\sentries\suse\sa\sdifferent\scase\sthan\sothers.\s\sFix\sfor\n[forum:/info/6c118daad0f1f5ef|forum\spost\s6c118daad0f1f5ef]. +D 2023-04-22T22:32:19.521 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -565,7 +565,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 482c534877fbb543f8295992cde925df55443febac5db5438d5aaba6f78c4940 -F src/analyze.c 01bfd40026632eaae1d93212b684f539c6674cb573535dc90199674cbf7e0cdc +F src/analyze.c 6f4d31f7b899528fa06cbc6263d019343461ebe1ce182335daec1e8c2b66a5a9 F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf F src/backup.c 5c97e8023aab1ce14a42387eb3ae00ba5a0644569e3476f38661fa6f824c3523 @@ -752,7 +752,7 @@ F test/altertab3.test 6c432fbb9963e0bd6549bf1422f6861d744ee5a80cb3298564e81e5564 F test/altertrig.test fb5951d21a2c954be3b8a8cf8e10b5c0fa20687c53fd67d63cea88d08dd058d5 F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 547bb700f903107b38611b014ca645d6b5bb819f5210d7bf39c40802aafeb7d7 -F test/analyze3.test 4440c4932247adb2b4e0c838f657c19dc7af4f56859255436dc4e855f39b9324 +F test/analyze3.test d4e09dc556c9361a699fad816051576d29aa66caf347800847354fc1071e18c3 F test/analyze4.test 68bd069f3ac7ac1e652ddd9f04f57d5606ddb4208450f5297005db7aa0dd707d F test/analyze5.test fa5131952303ac4146aba101b116b9c8cb89e2637531c334a6df7f7d19dddc0d F test/analyze6.test 028f5bdfc9e5b5294768fa9a7185b8cd1d019aa7aab5b2f8ee42d7271d9a3b28 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5157fd1cf753b1c01c6c00e9ac2712da2631a673837ed15ec3420ec869ee66aa -R 7c91bd0d3c9bcec44ce2ed126717540d -U dan -Z a127cac0e89bcf30ec7e8998adcf95b2 +P 33681ff3d29c9f096dbd8c5d885993f780e93974a7653adc00fa41a158356d7c +R 01b63f06161c14df16351e4095f8f08a +U drh +Z c0a9a444e995bf31a24cacb99b3324ee # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4540a49b8a..6ec26dc9f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -33681ff3d29c9f096dbd8c5d885993f780e93974a7653adc00fa41a158356d7c \ No newline at end of file +f097ca70b5b967d1aadebd74ac5020813e00b7c30cc543814dbf5f359d1328f1 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 6e9d72464c..bc1c5259c4 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1874,7 +1874,7 @@ static int loadStat4(sqlite3 *db, const char *zDb){ && IsOrdinaryTable(pStat4) ){ rc = loadStatTbl(db, - "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx COLLATE nocase", "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb ); diff --git a/test/analyze3.test b/test/analyze3.test index 7469c537c0..322d6fb775 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -736,4 +736,17 @@ do_execsql_test 7.2 { ANALYZE sqlite_master; } +# 2023-04-22 https://sqlite.org/forum/info/6c118daad0f1f5ef +# Case differences in the sqlite_stat4.idx field should not matter. +# +reset_db +do_execsql_test 8.0 { + CREATE TABLE t1(a PRIMARY KEY, v) WITHOUT ROWID; + ANALYZE sqlite_schema; + INSERT INTO sqlite_stat1 VALUES('t1','t1','1 1'); + INSERT INTO sqlite_stat4 VALUES('t1','t1','1','0','0',X'021b76657273696f6e'); + INSERT INTO sqlite_stat4 VALUES('T1','T1','1','0','0',X'021b76657273696f6e'); + ANALYZE sqlite_schema; +} {} + finish_test From 022a0428970c8f25b84d465592a2cc243a610da4 Mon Sep 17 00:00:00 2001 From: larrybr Date: Sun, 23 Apr 2023 00:12:23 +0000 Subject: [PATCH 264/341] CLI to have "undocumented" dot-commands, not usually shown by .help FossilOrigin-Name: 17f5dd2d2ae02a95180b9208b7de805922ba20271d3263e3193f0d46f4ec324c --- manifest | 14 +++++------ manifest.uuid | 2 +- src/shell.c.in | 63 ++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index dbb82bd7bc..f13f13de62 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\ssqlite_stat4.idx\sfield\sis\scase\sinsensitive.\s\sIt\sshould\swork\seven\sif\nsome\sentries\suse\sa\sdifferent\scase\sthan\sothers.\s\sFix\sfor\n[forum:/info/6c118daad0f1f5ef|forum\spost\s6c118daad0f1f5ef]. -D 2023-04-22T22:32:19.521 +C CLI\sto\shave\s"undocumented"\sdot-commands,\snot\susually\sshown\sby\s.help +D 2023-04-23T00:12:23.282 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -633,7 +633,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in 0bf660f2ed0cedb63ed2b029935fd758b3abe68369a08bdc8ad26dd4b402e3a7 +F src/shell.c.in e4762c17316110d4b65afac3572a4d134539a4178ac2afe4821de23f509a728f F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 33681ff3d29c9f096dbd8c5d885993f780e93974a7653adc00fa41a158356d7c -R 01b63f06161c14df16351e4095f8f08a -U drh -Z c0a9a444e995bf31a24cacb99b3324ee +P f097ca70b5b967d1aadebd74ac5020813e00b7c30cc543814dbf5f359d1328f1 +R 62624db0a9c4c2fd6a3100f573d3f96a +U larrybr +Z 113b1c901dbc06332b9973dc3c2cd6a4 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6ec26dc9f9..f93dca7933 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f097ca70b5b967d1aadebd74ac5020813e00b7c30cc543814dbf5f359d1328f1 \ No newline at end of file +17f5dd2d2ae02a95180b9208b7de805922ba20271d3263e3193f0d46f4ec324c \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c642fbf1ba..5a7a7e0091 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4776,13 +4776,13 @@ static const char *(azHelp[]) = { " input text.", #endif #ifndef SQLITE_OMIT_TEST_CONTROL - ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", + ",imposter INDEX TABLE Create imposter table TABLE on index INDEX", #endif ".indexes ?TABLE? Show names of indexes", " If TABLE is specified, only show indexes for", " tables matching TABLE using the LIKE operator.", #ifdef SQLITE_ENABLE_IOTRACE - ".iotrace FILE Enable I/O diagnostic logging to FILE", + ",iotrace FILE Enable I/O diagnostic logging to FILE", #endif ".limit ?LIMIT? ?VAL? Display or change the value of an SQLITE_LIMIT", ".lint OPTIONS Report potential schema issues.", @@ -4891,7 +4891,7 @@ static const char *(azHelp[]) = { " Options:", " --indent Try to pretty-print the schema", " --nosys Omit objects whose names start with \"sqlite_\"", - ".selftest ?OPTIONS? Run tests defined in the SELFTEST table", + ",selftest ?OPTIONS? Run tests defined in the SELFTEST table", " Options:", " --init Create a new SELFTEST table", " -v Verbose output", @@ -4933,9 +4933,9 @@ static const char *(azHelp[]) = { #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", #ifndef SQLITE_SHELL_FIDDLE - ".testcase NAME Begin redirecting output to 'testcase-out.txt'", + ",testcase NAME Begin redirecting output to 'testcase-out.txt'", #endif - ".testctrl CMD ... Run various sqlite3_test_control() operations", + ",testctrl CMD ... Run various sqlite3_test_control() operations", " Run \".testctrl\" with no arguments for details", ".timeout MS Try opening locked tables for MS milliseconds", ".timer on|off Turn SQL timer on or off", @@ -4987,16 +4987,41 @@ static int showHelp(FILE *out, const char *zPattern){ || cli_strcmp(zPattern,"-all")==0 || cli_strcmp(zPattern,"--all")==0 ){ - /* Show all commands, but only one line per command */ - if( zPattern==0 ) zPattern = ""; + enum HelpWanted { HW_NoCull = 0, HW_SummaryOnly = 1, HW_Undoc = 2 }; + enum HelpHave { HH_Undoc = 2, HH_Summary = 1, HH_More = 0 }; + /* Show all or most commands + ** *zPattern==0 => summary of documented commands only + ** *zPattern=='0' => whole help for undocumented commands + ** Otherwise => whole help for documented commands + */ + enum HelpWanted hw = HW_SummaryOnly; + if( zPattern!=0 ){ + hw = (*zPattern=='0')? HW_NoCull|HW_Undoc : HW_NoCull; + } + enum HelpHave hh = HH_More; for(i=0; i Date: Mon, 24 Apr 2023 04:25:42 +0000 Subject: [PATCH 265/341] Add a note about the journaling mode in the OPFS VFS. No code changes. FossilOrigin-Name: e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7 --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 10 +++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 2e09aab21a..13747ffecf 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -1189,7 +1189,15 @@ const installOpfsVfs = function callee(options){ /* Truncate journal mode is faster than delete for this vfs, per speedtest1. That gap seems to have closed with Chrome version 108 or 109, but "persist" is very roughly 5-6% - faster than truncate in initial tests. */ + faster than truncate in initial tests. + + For later analysis: Roy Hashimoto notes that TRUNCATE + and PERSIST modes may decrease OPFS concurrency because + multiple connections can open the journal file in those + modes: + + https://github.com/rhashimoto/wa-sqlite/issues/68 + */ "pragma journal_mode=persist;", /* This vfs benefits hugely from cache on moderate/large diff --git a/manifest b/manifest index f13f13de62..a40d72211a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C CLI\sto\shave\s"undocumented"\sdot-commands,\snot\susually\sshown\sby\s.help -D 2023-04-23T00:12:23.282 +C Add\sa\snote\sabout\sthe\sjournaling\smode\sin\sthe\sOPFS\sVFS.\sNo\scode\schanges. +D 2023-04-24T04:25:42.837 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -496,7 +496,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 69987029bead5a35f1b854e50c3453b527a497a2be35910b2cf07a4d36ec2299 +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 14130dc3f0830cd994162684eee3c42615aa3bca15950d2a7b48fe5c4b6e657f F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c c42413ca9f3e64c424b2bbfc5decf639670ca38bc8f7afb7760d5379398c9307 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f097ca70b5b967d1aadebd74ac5020813e00b7c30cc543814dbf5f359d1328f1 -R 62624db0a9c4c2fd6a3100f573d3f96a -U larrybr -Z 113b1c901dbc06332b9973dc3c2cd6a4 +P 17f5dd2d2ae02a95180b9208b7de805922ba20271d3263e3193f0d46f4ec324c +R 7c6f5a98ea12f09a60bc441886d297ed +U stephan +Z c684703beca8cd9cb5def7974649e125 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f93dca7933..20ed04ba3a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17f5dd2d2ae02a95180b9208b7de805922ba20271d3263e3193f0d46f4ec324c \ No newline at end of file +e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7 \ No newline at end of file From 655a1b7b58e614f67108f8f9a6e0995dc87169a2 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Apr 2023 19:22:21 +0000 Subject: [PATCH 266/341] Allow the sessions module to be configured to capture changes from tables with no explicit PRIMARY KEY. FossilOrigin-Name: 8a612f0860126c0c8473b1e65fcabb9a8821d8bf09fdf3f6018acfc99df9af71 --- ext/session/sessionD.test | 24 --- ext/session/session_common.tcl | 52 ++++++ ext/session/sessionrebase.test | 76 +++++++++ ext/session/sessionrowid.test | 281 +++++++++++++++++++++++++++++++++ ext/session/sessionsize.test | 14 +- ext/session/sqlite3session.c | 253 ++++++++++++++++++++--------- ext/session/sqlite3session.h | 15 +- ext/session/test_session.c | 26 ++- manifest | 30 ++-- manifest.uuid | 2 +- 10 files changed, 649 insertions(+), 124 deletions(-) create mode 100644 ext/session/sessionrowid.test diff --git a/ext/session/sessionD.test b/ext/session/sessionD.test index ec652e2d6b..9fccbfa96f 100644 --- a/ext/session/sessionD.test +++ b/ext/session/sessionD.test @@ -41,30 +41,6 @@ proc scksum {db dbname} { return [md5 $txt] } -proc do_diff_test {tn setup} { - reset_db - forcedelete test.db2 - execsql { ATTACH 'test.db2' AS aux } - execsql $setup - - sqlite3session S db main - foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { - S attach $tbl - S diff aux $tbl - } - - set C [S changeset] - S delete - - sqlite3 db2 test.db2 - sqlite3changeset_apply db2 $C "" - uplevel do_test $tn.1 [list {execsql { PRAGMA integrity_check } db2}] ok - db2 close - - set cksum [scksum db main] - uplevel do_test $tn.2 [list {scksum db aux}] [list $cksum] -} - # Ensure that the diff produced by comparing the current contents of [db] # with itself is empty. proc do_empty_diff_test {tn} { diff --git a/ext/session/session_common.tcl b/ext/session/session_common.tcl index 22e427f2be..3ff84f1c5e 100644 --- a/ext/session/session_common.tcl +++ b/ext/session/session_common.tcl @@ -52,6 +52,7 @@ proc do_conflict_test {tn args} { proc bgerror {args} { set ::background_error $args } sqlite3session S db main + S object_config rowid 1 foreach t $O(-tables) { S attach $t } execsql $O(-sql) @@ -81,6 +82,7 @@ proc changeset_from_sql {sql {dbname main}} { } set rc [catch { sqlite3session S db $dbname + S object_config rowid 1 db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { S attach $name } @@ -138,6 +140,7 @@ proc do_then_apply_sql {args} { proc xConflict args { incr ::n_conflict ; return "OMIT" } set rc [catch { sqlite3session S db $dbname + S object_config rowid 1 db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" { S attach $name } @@ -162,6 +165,8 @@ proc do_then_apply_sql {args} { proc do_iterator_test {tn tbl_list sql res} { sqlite3session S db main + S object_config rowid 1 + if {[llength $tbl_list]==0} { S attach * } foreach t $tbl_list {S attach $t} @@ -171,6 +176,7 @@ proc do_iterator_test {tn tbl_list sql res} { foreach v $res { lappend r $v } set x [list] +# set ::c [S changeset] ; execsql_pp { SELECT quote($::c) } sqlite3session_foreach c [S changeset] { lappend x $c } uplevel do_test $tn [list [list set {} $x]] [list $r] @@ -245,3 +251,49 @@ proc number_name {n} { if {$txt==""} {set txt zero} return $txt } + +proc scksum {db dbname} { + + if {$dbname=="temp"} { + set master sqlite_temp_master + } else { + set master $dbname.sqlite_master + } + + set alltab [$db eval "SELECT name FROM $master WHERE type='table'"] + set txt [$db eval "SELECT * FROM $master ORDER BY type,name,sql"] + foreach tab $alltab { + set cols [list] + db eval "PRAGMA $dbname.table_info = $tab" x { + lappend cols "quote($x(name))" + } + set cols [join $cols ,] + append txt [db eval "SELECT $cols FROM $dbname.$tab ORDER BY $cols"] + } + return [md5 $txt] +} + +proc do_diff_test {tn setup} { + reset_db + forcedelete test.db2 + execsql { ATTACH 'test.db2' AS aux } + execsql $setup + + sqlite3session S db main + S object_config rowid 1 + foreach tbl [db eval {SELECT name FROM sqlite_master WHERE type='table'}] { + S attach $tbl + S diff aux $tbl + } + + set C [S changeset] + S delete + + sqlite3 db2 test.db2 + sqlite3changeset_apply db2 $C "" + uplevel do_test $tn.1 [list {execsql { PRAGMA integrity_check } db2}] ok + db2 close + + set cksum [scksum db main] + uplevel do_test $tn.2 [list {scksum db aux}] [list $cksum] +} diff --git a/ext/session/sessionrebase.test b/ext/session/sessionrebase.test index cdf3322838..033348f9ce 100644 --- a/ext/session/sessionrebase.test +++ b/ext/session/sessionrebase.test @@ -84,6 +84,7 @@ proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} { db eval BEGIN sqlite3session S1 db main + S1 object_config rowid 1 S1 attach * execsql $sql1 db set c1 [S1 changeset] @@ -91,6 +92,7 @@ proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} { if {$i==1} { sqlite3session S2 db2 main + S2 object_config rowid 1 S2 attach * execsql $sql2 db2 set c2 [S2 changeset] @@ -100,6 +102,7 @@ proc do_rebase_test {tn sql1 sql2 conflict_handler {testsql ""} {testres ""}} { foreach sql [split $sql2 ";"] { if {[string is space $sql]} continue sqlite3session S2 db2 main + S2 object_config rowid 1 S2 attach * execsql $sql db2 lappend c2 [S2 changeset] @@ -341,6 +344,79 @@ do_rebase_test 2.2.3 { OMIT } { SELECT * FROM t2 WHERE z='B' } { 1 one B } + +reset_db +do_execsql_test 2.3.0 { + CREATE TABLE t1 (b TEXT); + INSERT INTO t1(rowid, b) VALUES(1, 'one'); + INSERT INTO t1(rowid, b) VALUES(2, 'two'); + INSERT INTO t1(rowid, b) VALUES(3, 'three'); +} +do_rebase_test 2.3.1 { + UPDATE t1 SET b = 'two.1' WHERE rowid=2 +} { + UPDATE t1 SET b = 'two.2' WHERE rowid=2; +} { + OMIT +} { SELECT rowid, * FROM t1 } {1 one 2 two.1 3 three} + +do_rebase_test 2.3.2 { + UPDATE t1 SET b = 'two.1' WHERE rowid=2 +} { + UPDATE t1 SET b = 'two.2' WHERE rowid=2; +} { + REPLACE +} { SELECT rowid, * FROM t1 } {1 one 2 two.2 3 three} + +do_rebase_test 2.3.3 { + DELETE FROM t1 WHERE rowid=3 +} { + DELETE FROM t1 WHERE rowid=3; +} { + OMIT +} { SELECT rowid, * FROM t1 } {1 one 2 two} + +do_rebase_test 2.3.4 { + DELETE FROM t1 WHERE rowid=1 +} { + UPDATE t1 SET b='one.2' WHERE rowid=1 +} { + OMIT +} { SELECT rowid, * FROM t1 } {2 two 3 three} + +do_rebase_test 2.3.6 { + UPDATE t1 SET b='three.1' WHERE rowid=3 +} { + DELETE FROM t1 WHERE rowid=3; +} { + OMIT +} { SELECT rowid, * FROM t1 } {1 one 2 two 3 three.1} + +do_rebase_test 2.3.7 { + UPDATE t1 SET b='three.1' WHERE rowid=3 +} { + DELETE FROM t1 WHERE rowid=3; +} { + REPLACE +} { SELECT rowid, * FROM t1 } {1 one 2 two} + +do_rebase_test 2.3.8 { + INSERT INTO t1(rowid, b) VALUES(4, 'four.1') +} { + INSERT INTO t1(rowid, b) VALUES(4, 'four.2'); +} { + REPLACE +} { SELECT rowid, * FROM t1 } {1 one 2 two 3 three 4 four.2} + +do_rebase_test 2.3.9 { + INSERT INTO t1(rowid, b) VALUES(4, 'four.1') +} { + INSERT INTO t1(rowid, b) VALUES(4, 'four.2'); +} { + OMIT +} { SELECT rowid, * FROM t1 } {1 one 2 two 3 three 4 four.1} + + #------------------------------------------------------------------------- reset_db do_execsql_test 3.0 { diff --git a/ext/session/sessionrowid.test b/ext/session/sessionrowid.test new file mode 100644 index 0000000000..14af90acef --- /dev/null +++ b/ext/session/sessionrowid.test @@ -0,0 +1,281 @@ +# 2011 Mar 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the session module. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix sessionrowid + +do_execsql_test 0.0 { + CREATE TABLE t1(a, b); +} + +foreach {tn rowid bEmpty} { + 1 0 1 + 2 1 0 + 3 -1 1 +} { + do_test 0.$tn { + sqlite3session S db main + if {$rowid>=0} { S object_config rowid $rowid } + S attach t1 + execsql { INSERT INTO t1 VALUES(1, 2); } + expr [string length [S changeset]]==0 + } $bEmpty + S delete +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); +} + +do_iterator_test 1.1 t1 { + INSERT INTO t1 VALUES('i', 'one'); +} { + {INSERT t1 0 X.. {} {i 1 t i t one}} +} + +do_execsql_test 1.2 { + SELECT rowid, * FROM t1 +} {1 i one} + +do_iterator_test 1.3 t1 { + UPDATE t1 SET b='two' +} { + {UPDATE t1 0 X.. {i 1 {} {} t one} {{} {} {} {} t two}} +} + +do_iterator_test 1.4 t1 { + DELETE FROM t1; +} { + {DELETE t1 0 X.. {i 1 t i t two} {}} +} + +do_iterator_test 1.5 t1 { + INSERT INTO t1(rowid, a, b) VALUES(14, 'hello', 'world'); + INSERT INTO t1(rowid, a, b) VALUES(NULL, 'yes', 'no'); + INSERT INTO t1(rowid, a, b) VALUES(-123, 'ii', 'iii'); +} { + {INSERT t1 0 X.. {} {i -123 t ii t iii}} + {INSERT t1 0 X.. {} {i 15 t yes t no}} + {INSERT t1 0 X.. {} {i 14 t hello t world}} +} + +do_iterator_test 1.6 t1 { + UPDATE t1 SET a='deluxe' WHERE rowid=14; + DELETE FROM t1 WHERE rowid=-123; + INSERT INTO t1 VALUES('x', 'xi'); +} { + {DELETE t1 0 X.. {i -123 t ii t iii} {}} + {UPDATE t1 0 X.. {i 14 t hello {} {}} {{} {} t deluxe {} {}}} + {INSERT t1 0 X.. {} {i 16 t x t xi}} +} + +#------------------------------------------------------------------------- +reset_db +forcedelete test.db2 +sqlite3 db2 test.db2 + +do_execsql_test 2.0 { + CREATE TABLE t1(a, b); +} +do_execsql_test -db db2 2.0.1 { + CREATE TABLE t1(a, b); +} + +proc xConflict {args} { + puts "CONFLICT!" + return "OMIT" +} + +do_test 2.1 { + set C [changeset_from_sql { + INSERT INTO t1 VALUES('abc', 'def'); + }] + sqlite3changeset_apply db2 $C xConflict + execsql { SELECT * FROM t1 } db2 +} {abc def} +do_test 2.2 { + set C [changeset_from_sql { + UPDATE t1 SET b='hello' + }] + sqlite3changeset_apply db2 $C xConflict + execsql { SELECT * FROM t1 } db2 +} {abc hello} +do_test 2.3 { + set C [changeset_from_sql { + DELETE FROM t1 WHERE b='hello' + }] + sqlite3changeset_apply db2 $C xConflict + execsql { SELECT * FROM t1 } db2 +} {} + +do_test 2.4 { + do_then_apply_sql { + INSERT INTO t1 VALUES('i', 'one'); + INSERT INTO t1 VALUES('ii', 'two'); + INSERT INTO t1 VALUES('iii', 'three'); + INSERT INTO t1 VALUES('iv', 'four'); + } + compare_db db db2 +} {} + +do_test 2.5 { + do_then_apply_sql { + DELETE FROM t1 WHERE a='ii'; + UPDATE t1 SET b='THREE' WHERE a='iii'; + UPDATE t1 SET a='III' WHERE a='iii'; + INSERT INTO t1 VALUES('v', 'five'); + } + compare_db db db2 +} {} + +do_execsql_test 2.6 {SELECT * FROM t1} {i one III THREE iv four v five} +do_execsql_test -db db2 2.7 {SELECT * FROM t1} {i one III THREE iv four v five} + +#------------------------------------------------------------------------- +db2 close +reset_db +forcedelete test.db2 +sqlite3 db2 test.db2 + +set init_sql { + CREATE TABlE t4(a, b); + CREATE INDEX t4a ON t4(a); + CREATE UNIQUE INDEX t4b ON t4(b); +} + +do_execsql_test 3.0 $init_sql +do_execsql_test -db db2 3.0a $init_sql + +do_execsql_test -db db2 3.1 { + INSERT INTO t4(rowid, a, b) VALUES(43, 'hello', 'world'); +} +do_conflict_test 3.2 -sql { + INSERT INTO t4(rowid, a, b) VALUES(43, 'abc', 'def'); +} -tables t4 -conflicts { + {INSERT t4 CONFLICT {i 43 t abc t def} {i 43 t hello t world}} +} +do_execsql_test -db db2 3.3 { + SELECT * FROM t4 +} {hello world} + +do_execsql_test 3.4 { DELETE FROM t4 } +do_conflict_test 3.5 -sql { + INSERT INTO t4(rowid, a, b) VALUES(43, 'abc', 'def'); +} -tables t4 -conflicts { + {INSERT t4 CONFLICT {i 43 t abc t def} {i 43 t hello t world}} +} -policy REPLACE +do_execsql_test -db db2 3.6 { + SELECT * FROM t4 +} {abc def} + +do_execsql_test 3.7 { DELETE FROM t4 } +do_conflict_test 3.8 -sql { + INSERT INTO t4(rowid, a, b) VALUES(45, 'xyz', 'def'); +} -tables t4 -conflicts { + {INSERT t4 CONSTRAINT {i 45 t xyz t def}} +} +do_execsql_test -db db2 3.9 { + SELECT * FROM t4 +} {abc def} + + +do_execsql_test -db db 3.10a { DELETE FROM t4 } +do_execsql_test -db db2 3.10b { DELETE FROM t4 } + +do_execsql_test -db db 3.11a { + INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'one'); + INSERT INTO t4(rowid, a, b) VALUES(222, 'two', 'two'); +} +do_execsql_test -db db2 3.11b { + INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'blip'); +} + +do_conflict_test 3.12 -sql { + DELETE FROM t4 WHERE a='one'; +} -tables t4 -conflicts { + {DELETE t4 DATA {i 111 t one t one} {i 111 t one t blip}} +} +do_execsql_test -db db2 3.13 { + SELECT * FROM t4 +} {one blip} + +do_conflict_test 3.14 -sql { + DELETE FROM t4 WHERE a='two'; +} -tables t4 -conflicts { + {DELETE t4 NOTFOUND {i 222 t two t two}} +} +do_execsql_test -db db2 3.15 { + SELECT * FROM t4 +} {one blip} + +do_execsql_test -db db 3.16a { DELETE FROM t4 } +do_execsql_test -db db2 3.16b { DELETE FROM t4 } + +do_execsql_test -db db 3.17a { + INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'one'); + INSERT INTO t4(rowid, a, b) VALUES(222, 'two', 'two'); +} +do_execsql_test -db db2 3.17b { + INSERT INTO t4(rowid, a, b) VALUES(111, 'one', 'blip'); +} + +do_conflict_test 3.18 -sql { + UPDATE t4 SET b='xyz' WHERE a='one' +} -tables t4 -conflicts { + {UPDATE t4 DATA {i 111 {} {} t one} {{} {} {} {} t xyz} {i 111 t one t blip}} +} +do_execsql_test -db db2 3.19 { + SELECT * FROM t4 +} {one blip} + +do_conflict_test 3.20 -sql { + UPDATE t4 SET b='123' WHERE a='two' +} -tables t4 -conflicts { + {UPDATE t4 NOTFOUND {i 222 {} {} t two} {{} {} {} {} t 123}} +} +do_execsql_test -db db2 3.21 { + SELECT * FROM t4 +} {one blip} + +#-------------------------------------------------------------------------- +breakpoint +do_diff_test 4.0 { + CREATE TABLE t1(x, y); + CREATE TABLE aux.t1(x, y); + INSERT INTO t1 VALUES(1, 2); +} + +do_diff_test 4.1 { + CREATE TABLE t1(x, y); + CREATE TABLE aux.t1(x, y); + INSERT INTO aux.t1 VALUES(1, 2); +} + +do_diff_test 4.2 { + CREATE TABLE t1(x, y); + CREATE TABLE aux.t1(x, y); + INSERT INTO t1(rowid, x, y) VALUES(413, 'hello', 'there'); + INSERT INTO aux.t1(rowid, x, y) VALUES(413, 'hello', 'world'); +} + +finish_test + diff --git a/ext/session/sessionsize.test b/ext/session/sessionsize.test index 04d05514db..01638c6677 100644 --- a/ext/session/sessionsize.test +++ b/ext/session/sessionsize.test @@ -113,17 +113,17 @@ do_execsql_test 3.0 { do_test 3.1 { sqlite3session S db main - S object_config_size -1 + S object_config size -1 } 1 -do_test 3.2.1 { S object_config_size 0 } 0 -do_test 3.2.2 { S object_config_size -1 } 0 -do_test 3.2.3 { S object_config_size 1 } 1 -do_test 3.2.4 { S object_config_size -1 } 1 +do_test 3.2.1 { S object_config size 0 } 0 +do_test 3.2.2 { S object_config size -1 } 0 +do_test 3.2.3 { S object_config size 1 } 1 +do_test 3.2.4 { S object_config size -1 } 1 do_test 3.3 { S attach t1 } {} -do_test 3.4 { S object_config_size 1 } {SQLITE_MISUSE} -do_test 3.4 { S object_config_size -1 } {1} +do_test 3.4 { S object_config size 1 } {SQLITE_MISUSE} +do_test 3.4 { S object_config size -1 } {1} S delete diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 225d0316a8..e601a4ae7a 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -25,6 +25,8 @@ typedef struct SessionInput SessionInput; # endif #endif +#define SESSIONS_ROWID "_rowid_" + static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE; typedef struct SessionHook SessionHook; @@ -46,6 +48,7 @@ struct sqlite3_session { int bEnable; /* True if currently recording */ int bIndirect; /* True if all changes are indirect */ int bAutoAttach; /* True to auto-attach tables */ + int bImplicitPK; /* True to handle tables with implicit PK */ int rc; /* Non-zero if an error has occurred */ void *pFilterCtx; /* First argument to pass to xTableFilter */ int (*xTableFilter)(void *pCtx, const char *zTab); @@ -122,6 +125,7 @@ struct SessionTable { char *zName; /* Local name of table */ int nCol; /* Number of columns in table zName */ int bStat1; /* True if this is sqlite_stat1 */ + int bRowid; /* True if this table uses rowid for PK */ const char **azCol; /* Column names */ u8 *abPK; /* Array of primary key flags */ int nEntry; /* Total number of entries in hash table */ @@ -514,6 +518,7 @@ static unsigned int sessionHashAppendType(unsigned int h, int eType){ */ static int sessionPreupdateHash( sqlite3_session *pSession, /* Session object that owns pTab */ + i64 iRowid, SessionTable *pTab, /* Session table handle */ int bNew, /* True to hash the new.* PK */ int *piHash, /* OUT: Hash value */ @@ -522,48 +527,53 @@ static int sessionPreupdateHash( unsigned int h = 0; /* Hash value to return */ int i; /* Used to iterate through columns */ - assert( *pbNullPK==0 ); - assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); - for(i=0; inCol; i++){ - if( pTab->abPK[i] ){ - int rc; - int eType; - sqlite3_value *pVal; + if( pTab->bRowid ){ + assert( pTab->nCol-1==pSession->hook.xCount(pSession->hook.pCtx) ); + h = sessionHashAppendI64(h, iRowid); + }else{ + assert( *pbNullPK==0 ); + assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); + for(i=0; inCol; i++){ + if( pTab->abPK[i] ){ + int rc; + int eType; + sqlite3_value *pVal; - if( bNew ){ - rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); - }else{ - rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); - } - if( rc!=SQLITE_OK ) return rc; + if( bNew ){ + rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); + }else{ + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); + } + if( rc!=SQLITE_OK ) return rc; - eType = sqlite3_value_type(pVal); - h = sessionHashAppendType(h, eType); - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - i64 iVal; - if( eType==SQLITE_INTEGER ){ - iVal = sqlite3_value_int64(pVal); + eType = sqlite3_value_type(pVal); + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_value_int64(pVal); + }else{ + double rVal = sqlite3_value_double(pVal); + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&iVal, &rVal, 8); + } + h = sessionHashAppendI64(h, iVal); + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + const u8 *z; + int n; + if( eType==SQLITE_TEXT ){ + z = (const u8 *)sqlite3_value_text(pVal); + }else{ + z = (const u8 *)sqlite3_value_blob(pVal); + } + n = sqlite3_value_bytes(pVal); + if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + h = sessionHashAppendBlob(h, n, z); }else{ - double rVal = sqlite3_value_double(pVal); - assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); - memcpy(&iVal, &rVal, 8); + assert( eType==SQLITE_NULL ); + assert( pTab->bStat1==0 || i!=1 ); + *pbNullPK = 1; } - h = sessionHashAppendI64(h, iVal); - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ - const u8 *z; - int n; - if( eType==SQLITE_TEXT ){ - z = (const u8 *)sqlite3_value_text(pVal); - }else{ - z = (const u8 *)sqlite3_value_blob(pVal); - } - n = sqlite3_value_bytes(pVal); - if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; - h = sessionHashAppendBlob(h, n, z); - }else{ - assert( eType==SQLITE_NULL ); - assert( pTab->bStat1==0 || i!=1 ); - *pbNullPK = 1; } } } @@ -846,6 +856,7 @@ static int sessionMergeUpdate( */ static int sessionPreupdateEqual( sqlite3_session *pSession, /* Session object that owns SessionTable */ + i64 iRowid, /* Rowid value if pTab->bRowid */ SessionTable *pTab, /* Table associated with change */ SessionChange *pChange, /* Change to compare to */ int op /* Current pre-update operation */ @@ -853,6 +864,11 @@ static int sessionPreupdateEqual( int iCol; /* Used to iterate through columns */ u8 *a = pChange->aRecord; /* Cursor used to scan change record */ + if( pTab->bRowid ){ + if( a[0]!=SQLITE_INTEGER ) return 0; + return sessionGetI64(&a[1])==iRowid; + } + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); for(iCol=0; iColnCol; iCol++){ if( !pTab->abPK[iCol] ){ @@ -997,7 +1013,8 @@ static int sessionTableInfo( int *pnCol, /* OUT: number of columns */ const char **pzTab, /* OUT: Copy of zThis */ const char ***pazCol, /* OUT: Array of column names for table */ - u8 **pabPK /* OUT: Array of booleans - true for PK col */ + u8 **pabPK, /* OUT: Array of booleans - true for PK col */ + int *pbRowid /* OUT: True if only PK is a rowid */ ){ char *zPragma; sqlite3_stmt *pStmt; @@ -1009,6 +1026,7 @@ static int sessionTableInfo( u8 *pAlloc = 0; char **azCol = 0; u8 *abPK = 0; + int bRowid = 0; /* Set to true to use rowid as PK */ assert( pazCol && pabPK ); @@ -1053,10 +1071,15 @@ static int sessionTableInfo( } nByte = nThis + 1; + bRowid = (pbRowid!=0); while( SQLITE_ROW==sqlite3_step(pStmt) ){ nByte += sqlite3_column_bytes(pStmt, 1); nDbCol++; + if( sqlite3_column_int(pStmt, 5) ) bRowid = 0; } + if( nDbCol==0 ) bRowid = 0; + nDbCol += bRowid; + nByte += strlen(SESSIONS_ROWID); rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ){ @@ -1078,6 +1101,14 @@ static int sessionTableInfo( } i = 0; + if( bRowid ){ + int nName = strlen(SESSIONS_ROWID); + memcpy(pAlloc, SESSIONS_ROWID, nName+1); + azCol[i] = (char*)pAlloc; + pAlloc += nName+1; + abPK[i] = 1; + i++; + } while( SQLITE_ROW==sqlite3_step(pStmt) ){ int nName = sqlite3_column_bytes(pStmt, 1); const unsigned char *zName = sqlite3_column_text(pStmt, 1); @@ -1089,7 +1120,6 @@ static int sessionTableInfo( i++; } rc = sqlite3_reset(pStmt); - } /* If successful, populate the output variables. Otherwise, zero them and @@ -1106,6 +1136,7 @@ static int sessionTableInfo( if( pzTab ) *pzTab = 0; sessionFree(pSession, azCol); } + if( pbRowid ) *pbRowid = bRowid; sqlite3_finalize(pStmt); return rc; } @@ -1127,7 +1158,8 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ u8 *abPK; assert( pTab->azCol==0 || pTab->abPK==0 ); pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, - pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK + pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK, + (pSession->bImplicitPK ? &pTab->bRowid : 0) ); if( pSession->rc==SQLITE_OK ){ int i; @@ -1199,6 +1231,7 @@ static int sessionUpdateMaxSize( ){ i64 nNew = 2; if( pC->op==SQLITE_INSERT ){ + if( pTab->bRowid ) nNew += 9; if( op!=SQLITE_DELETE ){ int ii; for(ii=0; iinCol; ii++){ @@ -1215,7 +1248,11 @@ static int sessionUpdateMaxSize( }else{ int ii; u8 *pCsr = pC->aRecord; - for(ii=0; iinCol; ii++){ + if( pTab->bRowid ){ + nNew += 9; + pCsr += 9; + } + for(ii=0; ii<(pTab->nCol-pTab->bRowid); ii++){ int bChanged = 1; int nOld = 0; int eType; @@ -1299,6 +1336,7 @@ static int sessionUpdateMaxSize( */ static void sessionPreupdateOneChange( int op, /* One of SQLITE_UPDATE, INSERT, DELETE */ + i64 iRowid, sqlite3_session *pSession, /* Session object pTab is attached to */ SessionTable *pTab /* Table that change applies to */ ){ @@ -1314,7 +1352,7 @@ static void sessionPreupdateOneChange( /* Check the number of columns in this xPreUpdate call matches the ** number of columns in the table. */ - if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){ + if( (pTab->nCol-pTab->bRowid)!=pSession->hook.xCount(pSession->hook.pCtx) ){ pSession->rc = SQLITE_SCHEMA; return; } @@ -1347,14 +1385,16 @@ static void sessionPreupdateOneChange( /* Calculate the hash-key for this change. If the primary key of the row ** includes a NULL value, exit early. Such changes are ignored by the ** session module. */ - rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull); + rc = sessionPreupdateHash( + pSession, iRowid, pTab, op==SQLITE_INSERT, &iHash, &bNull + ); if( rc!=SQLITE_OK ) goto error_out; if( bNull==0 ){ /* Search the hash table for an existing record for this row. */ SessionChange *pC; for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){ - if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break; + if( sessionPreupdateEqual(pSession, iRowid, pTab, pC, op) ) break; } if( pC==0 ){ @@ -1369,7 +1409,7 @@ static void sessionPreupdateOneChange( /* Figure out how large an allocation is required */ nByte = sizeof(SessionChange); - for(i=0; inCol; i++){ + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -1384,6 +1424,9 @@ static void sessionPreupdateOneChange( rc = sessionSerializeValue(0, p, &nByte); if( rc!=SQLITE_OK ) goto error_out; } + if( pTab->bRowid ){ + nByte += 9; /* Size of rowid field - an integer */ + } /* Allocate the change object */ pC = (SessionChange *)sessionMalloc64(pSession, nByte); @@ -1400,7 +1443,12 @@ static void sessionPreupdateOneChange( ** required values and encodings have already been cached in memory. ** It is not possible for an OOM to occur in this block. */ nByte = 0; - for(i=0; inCol; i++){ + if( pTab->bRowid ){ + pC->aRecord[0] = SQLITE_INTEGER; + sessionPutI64(&pC->aRecord[1], iRowid); + nByte = 9; + } + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -1515,9 +1563,10 @@ static void xPreUpdate( pSession->rc = sessionFindTable(pSession, zName, &pTab); if( pTab ){ assert( pSession->rc==SQLITE_OK ); - sessionPreupdateOneChange(op, pSession, pTab); + assert( op==SQLITE_UPDATE || iKey1==iKey2 ); + sessionPreupdateOneChange(op, iKey1, pSession, pTab); if( op==SQLITE_UPDATE ){ - sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab); + sessionPreupdateOneChange(SQLITE_INSERT, iKey2, pSession, pTab); } } } @@ -1556,6 +1605,7 @@ static void sessionPreupdateHooks( typedef struct SessionDiffCtx SessionDiffCtx; struct SessionDiffCtx { sqlite3_stmt *pStmt; + int bRowid; int nOldOff; }; @@ -1564,17 +1614,17 @@ struct SessionDiffCtx { */ static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff+p->bRowid); return SQLITE_OK; } static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->bRowid); return SQLITE_OK; } static int sessionDiffCount(void *pCtx){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt); + return (p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt)) - p->bRowid; } static int sessionDiffDepth(void *pCtx){ (void)pCtx; @@ -1653,14 +1703,16 @@ static char *sessionExprCompareOther( static char *sessionSelectFindNew( const char *zDb1, /* Pick rows in this db only */ const char *zDb2, /* But not in this one */ + int bRowid, const char *zTbl, /* Table name */ const char *zExpr ){ + const char *zSel = (bRowid ? SESSIONS_ROWID ", *" : "*"); char *zRet = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\" WHERE NOT EXISTS (" + "SELECT %s FROM \"%w\".\"%w\" WHERE NOT EXISTS (" " SELECT 1 FROM \"%w\".\"%w\" WHERE %s" ")", - zDb1, zTbl, zDb2, zTbl, zExpr + zSel, zDb1, zTbl, zDb2, zTbl, zExpr ); return zRet; } @@ -1674,7 +1726,9 @@ static int sessionDiffFindNew( char *zExpr ){ int rc = SQLITE_OK; - char *zStmt = sessionSelectFindNew(zDb1, zDb2, pTab->zName,zExpr); + char *zStmt = sessionSelectFindNew( + zDb1, zDb2, pTab->bRowid, pTab->zName, zExpr + ); if( zStmt==0 ){ rc = SQLITE_NOMEM; @@ -1685,8 +1739,10 @@ static int sessionDiffFindNew( SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = 0; + pDiffCtx->bRowid = pTab->bRowid; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(op, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(op, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } @@ -1696,6 +1752,27 @@ static int sessionDiffFindNew( return rc; } +/* +** Return a comma-separated list of the fully-qualified (with both database +** and table name) column names from table pTab. e.g. +** +** "main"."t1"."a", "main"."t1"."b", "main"."t1"."c" +*/ +static char *sessionAllCols( + const char *zDb, + SessionTable *pTab +){ + int ii; + char *zRet = 0; + for(ii=0; iinCol; ii++){ + zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"%s", + zRet, (zRet ? ", " : ""), zDb, pTab->zName, pTab->azCol[ii] + ); + if( !zRet ) break; + } + return zRet; +} + static int sessionDiffFindModified( sqlite3_session *pSession, SessionTable *pTab, @@ -1710,11 +1787,13 @@ static int sessionDiffFindModified( if( zExpr2==0 ){ rc = SQLITE_NOMEM; }else{ + char *z1 = sessionAllCols(pSession->zDb, pTab); + char *z2 = sessionAllCols(zFrom, pTab); char *zStmt = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", - pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 + "SELECT %s,%s FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", + z1, z2, pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 ); - if( zStmt==0 ){ + if( zStmt==0 || z1==0 || z2==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_stmt *pStmt; @@ -1725,12 +1804,15 @@ static int sessionDiffFindModified( pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = pTab->nCol; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(SQLITE_UPDATE, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } - sqlite3_free(zStmt); } + sqlite3_free(zStmt); + sqlite3_free(z1); + sqlite3_free(z2); } return rc; @@ -1769,9 +1851,12 @@ int sqlite3session_diff( int bHasPk = 0; int bMismatch = 0; int nCol; /* Columns in zFrom.zTbl */ + int bRowid = 0; u8 *abPK; const char **azCol = 0; - rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK); + rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK, + pSession->bImplicitPK ? &bRowid : 0 + ); if( rc==SQLITE_OK ){ if( pTo->nCol!=nCol ){ bMismatch = 1; @@ -2422,6 +2507,7 @@ static int sessionSelectStmt( int bIgnoreNoop, const char *zDb, /* Database name */ const char *zTab, /* Table name */ + int bRowid, int nCol, /* Number of columns in table */ const char **azCol, /* Names of table columns */ u8 *abPK, /* PRIMARY KEY array */ @@ -2430,7 +2516,7 @@ static int sessionSelectStmt( int rc = SQLITE_OK; char *zSql = 0; const char *zSep = ""; - const char *zCols = "*"; + const char *zCols = bRowid ? SESSIONS_ROWID ", *" : "*"; int nSql = -1; int i; @@ -2449,7 +2535,6 @@ static int sessionSelectStmt( zCols = "tbl, ?2, stat"; }else{ for(i=0; izDb, zName, &nCol, 0,&azCol,&abPK); - if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){ + rc = sessionTableInfo( + 0, db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK, + (pSession->bImplicitPK ? &bRowid : 0) + ); + if( rc==SQLITE_OK && ( + pTab->nCol!=nCol + || pTab->bRowid!=bRowid + || memcmp(abPK, pTab->abPK, nCol) + )){ rc = SQLITE_SCHEMA; } @@ -2669,7 +2762,7 @@ static int sessionGenerateChangeset( /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ rc = sessionSelectStmt( - db, 0, pSession->zDb, zName, nCol, azCol, abPK, &pSel + db, 0, pSession->zDb, zName, bRowid, nCol, azCol, abPK, &pSel ); } @@ -2753,8 +2846,8 @@ int sqlite3session_changeset( int rc; if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE; - rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset,ppChangeset); - assert( rc || pnChangeset==0 + rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); + assert( 1 || rc || pnChangeset==0 || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize ); return rc; @@ -2871,6 +2964,19 @@ int sqlite3session_object_config(sqlite3_session *pSession, int op, void *pArg){ break; } + case SQLITE_SESSION_OBJCONFIG_ROWID: { + int iArg = *(int*)pArg; + if( iArg>=0 ){ + if( pSession->pTable ){ + rc = SQLITE_MISUSE; + }else{ + pSession->bImplicitPK = (iArg!=0); + } + } + *(int*)pArg = pSession->bImplicitPK; + break; + } + default: rc = SQLITE_MISUSE; } @@ -3860,6 +3966,7 @@ struct SessionApplyCtx { u8 bRebaseStarted; /* If table header is already in rebase */ u8 bRebase; /* True to collect rebase information */ u8 bIgnoreNoop; /* True to ignore no-op conflicts */ + int bRowid; }; /* Number of prepared UPDATE statements to cache. */ @@ -4110,8 +4217,9 @@ static int sessionSelectRow( const char *zTab, /* Table name */ SessionApplyCtx *p /* Session changeset-apply context */ ){ + /* TODO */ return sessionSelectStmt(db, p->bIgnoreNoop, - "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect + "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect ); } @@ -4807,6 +4915,7 @@ static int sessionChangesetApply( sApply.bStat1 = 0; sApply.bDeferConstraints = 1; sApply.bRebaseStarted = 0; + sApply.bRowid = 0; memset(&sApply.constraints, 0, sizeof(SessionBuffer)); /* If an xFilter() callback was specified, invoke it now. If the @@ -4826,8 +4935,8 @@ static int sessionChangesetApply( int i; sqlite3changeset_pk(pIter, &abPK, 0); - rc = sessionTableInfo(0, - db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK + rc = sessionTableInfo(0, db, "main", zNew, + &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK, &sApply.bRowid ); if( rc!=SQLITE_OK ) break; for(i=0; iSQLITE_SESSION_OBJCONFIG_ROWID
+** This option is used to set, clear or query the flag that enables +** collection of data for tables with no explicit PRIMARY KEY. +** +** Normally, tables with no explicit PRIMARY KEY are simply ignored +** by the sessions module. However, if this flag is set, it behaves +** as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted +** as their leftmost columns. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. */ int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); /* */ -#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_ROWID 2 /* ** CAPI3REF: Enable Or Disable A Session Object diff --git a/ext/session/test_session.c b/ext/session/test_session.c index f10afe03f6..0836238b5d 100644 --- a/ext/session/test_session.c +++ b/ext/session/test_session.c @@ -76,9 +76,11 @@ int sql_exec_changeset( ){ sqlite3_session *pSession = 0; int rc; + int val = 1; /* Create a new session object */ rc = sqlite3session_create(db, "main", &pSession); + sqlite3session_object_config(pSession, SQLITE_SESSION_OBJCONFIG_ROWID, &val); /* Configure the session object to record changes to all tables */ if( rc==SQLITE_OK ) rc = sqlite3session_attach(pSession, NULL); @@ -260,7 +262,7 @@ static int SQLITE_TCLAPI test_session_cmd( { "diff", 2, "FROMDB TBL", }, /* 8 */ { "memory_used", 0, "", }, /* 9 */ { "changeset_size", 0, "", }, /* 10 */ - { "object_config_size", 1, "INTEGER", }, /* 11 */ + { "object_config", 2, "OPTION INTEGER", }, /* 11 */ { 0 } }; int iSub; @@ -379,15 +381,27 @@ static int SQLITE_TCLAPI test_session_cmd( Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize)); break; } - case 11: { + case 11: { /* object_config */ + struct ObjConfOpt { + const char *zName; + int opt; + } aOpt[] = { + { "size", SQLITE_SESSION_OBJCONFIG_SIZE }, + { "rowid", SQLITE_SESSION_OBJCONFIG_ROWID }, + { 0, 0 } + }; + size_t sz = sizeof(aOpt[0]); + int rc; int iArg; - if( Tcl_GetIntFromObj(interp, objv[2], &iArg) ){ + int iOpt; + if( Tcl_GetIndexFromObjStruct(interp,objv[2],aOpt,sz,"option",0,&iOpt) ){ return TCL_ERROR; } - rc = sqlite3session_object_config( - pSession, SQLITE_SESSION_OBJCONFIG_SIZE, &iArg - ); + if( Tcl_GetIntFromObj(interp, objv[3], &iArg) ){ + return TCL_ERROR; + } + rc = sqlite3session_object_config(pSession, aOpt[iOpt].opt, &iArg); if( rc!=SQLITE_OK ){ extern const char *sqlite3ErrName(int); Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); diff --git a/manifest b/manifest index a40d72211a..7b5292db51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\snote\sabout\sthe\sjournaling\smode\sin\sthe\sOPFS\sVFS.\sNo\scode\schanges. -D 2023-04-24T04:25:42.837 +C Allow\sthe\ssessions\smodule\sto\sbe\sconfigured\sto\scapture\schanges\sfrom\stables\swith\sno\sexplicit\sPRIMARY\sKEY. +D 2023-04-24T19:22:21.668 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -449,12 +449,12 @@ F ext/session/session9.test 5409d90d8141881d08285ed1c2c0d8d10fb92069 F ext/session/sessionA.test 1feeab0b8e03527f08f2f1defb442da25480138f F ext/session/sessionB.test c4fb7f8a688787111606e123a555f18ee04f65bb9f2a4bb2aa71d55ce4e6d02c F ext/session/sessionC.test f8a5508bc059ae646e5ec9bdbca66ad24bc92fe99fda5790ac57e1f59fce2fdf -F ext/session/sessionD.test 4f91d0ca8afc4c3969c72c9f0b5ea9527e21de29039937d0d973f821e8470724 +F ext/session/sessionD.test f5c6a762d00bc6ca9d561695c322ba8ecca2bed370486707ef37cf565d2f6c73 F ext/session/sessionE.test b2010949c9d7415306f64e3c2072ddabc4b8250c98478d3c0c4d064bce83111d F ext/session/sessionF.test d37ed800881e742c208df443537bf29aa49fd56eac520d0f0c6df3e6320f3401 F ext/session/sessionG.test 3efe388282d641b65485b5462e67851002cd91a282dc95b685d085eb8efdad0a F ext/session/sessionH.test 71bbff6b1abb2c4ac62b84dee53273c37e0b21e5fde3aed80929403e091ef859 -F ext/session/session_common.tcl db0dda567c75950604072251744e9a6ad5795a3009963c44eb8510f23a8cda64 +F ext/session/session_common.tcl e5598096425486b363718e2cda48ee85d660c96b4f8ea9d9d7a4c3ef514769da F ext/session/session_speed_test.c dcf0ef58d76b70c8fbd9eab3be77cf9deb8bc1638fed8be518b62d6cbdef88b3 F ext/session/sessionat.test 00c8badb35e43a2f12a716d2734a44d614ff62361979b6b85419035bc04b45ee F ext/session/sessionbig.test 47c381e7acfabeef17d98519a3080d69151723354d220afa2053852182ca7adf @@ -465,13 +465,14 @@ F ext/session/sessioninvert.test 04075517a9497a80d39c495ba6b44f3982c7371129b89e2 F ext/session/sessionmem.test f2a735db84a3e9e19f571033b725b0b2daf847f3f28b1da55a0c1a4e74f1de09 F ext/session/sessionnoop.test a9366a36a95ef85f8a3687856ebef46983df399541174cb1ede2ee53b8011bc7 F ext/session/sessionnoop2.test 5c9a882219e54711c98dccd2fd81392f189a59325e4fb5d8ed25e33a0c2f0ba2 -F ext/session/sessionrebase.test ccfa716b23bd1d3b03217ee58cfd90c78d4b99f53e6a9a2f05e82363b9142810 -F ext/session/sessionsize.test 6f644aff31c7f1e4871e9ff3542766e18da68fc7e587b83a347ea9820a002dd8 +F ext/session/sessionrebase.test 702378bdcb5062f1106e74457beca8797d09c113a81768734a58b197b5b334e2 +F ext/session/sessionrowid.test 6323ba831721205fd729929745038fd54e9d128c66c654b8d0b26853095a321c +F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c 1795263b72c1a17e48e95a131a69543af3fa31aa8e81271c7c5cb0911f063604 -F ext/session/sqlite3session.h c367c3043dbb57f69cca35258ebbeadb24e8738980b1a1ae1e281c1b0fac3989 -F ext/session/test_session.c b55a669a2150eb7c491b8b42c69a3eed9bc895cf5fea371a2c813b9618f72163 +F ext/session/sqlite3session.c a0d7d71f9b111e443b53afc49b6cb5ccf48734af216c91dbe6640b7e4cd8c740 +F ext/session/sqlite3session.h 24299a3b64f11afc4422ce92d030ffdb2d3181851a1763b4a0432e195b2a8a16 +F ext/session/test_session.c 5285482f83cd92b4c1fe12fcf88210566a18312f4f2aa110f6399dae46aeccbb F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb @@ -2059,8 +2060,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17f5dd2d2ae02a95180b9208b7de805922ba20271d3263e3193f0d46f4ec324c -R 7c6f5a98ea12f09a60bc441886d297ed -U stephan -Z c684703beca8cd9cb5def7974649e125 +P e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7 +R 1829aad474dae92cc041c644553e0142 +T *branch * sessions-rowid-tables +T *sym-sessions-rowid-tables * +T -sym-trunk * +U dan +Z 7494f98659d6e6e0ecd97f2ba99a8ae6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 20ed04ba3a..87be73f74f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7 \ No newline at end of file +8a612f0860126c0c8473b1e65fcabb9a8821d8bf09fdf3f6018acfc99df9af71 \ No newline at end of file From 7eff609d8a87c61eed915c6a43ef58dc023b000a Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 24 Apr 2023 19:23:42 +0000 Subject: [PATCH 267/341] Update the compile-time detection of architecture byte-order in the RTREE extension so that it is aligned with the latest enhancements in the core. FossilOrigin-Name: 122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a --- ext/rtree/rtree.c | 21 +++++++++++---------- manifest | 15 ++++++++------- manifest.uuid | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 38d09d2497..da5e2a97a8 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -471,16 +471,17 @@ struct RtreeMatchArg { ** at run-time. */ #ifndef SQLITE_BYTEORDER -#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) -# define SQLITE_BYTEORDER 1234 -#elif defined(sparc) || defined(__ppc__) -# define SQLITE_BYTEORDER 4321 -#else -# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ -#endif +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) || \ + defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif #endif diff --git a/manifest b/manifest index a40d72211a..0509728ae1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\snote\sabout\sthe\sjournaling\smode\sin\sthe\sOPFS\sVFS.\sNo\scode\schanges. -D 2023-04-24T04:25:42.837 +C Update\sthe\scompile-time\sdetection\sof\sarchitecture\sbyte-order\sin\sthe\sRTREE\nextension\sso\sthat\sit\sis\saligned\swith\sthe\slatest\senhancements\sin\sthe\score. +D 2023-04-24T19:23:42.196 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -400,7 +400,7 @@ F ext/repair/test/checkindex01.test b530f141413b587c9eb78ff734de6bb79bc3515c3350 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/geopoly.c 971e0b5bd9adaf0811feb8c0842a310811159da10319eb0e74fdb42bf26b99ca -F ext/rtree/rtree.c e05929e78d127613a9eea5dc372c77a049484892d8e9fac1fe0cce85ce4fba81 +F ext/rtree/rtree.c 925888f7672b326fc2a29b3a212ec3ae4aa2331507ecccfaf7f0ac0335020330 F ext/rtree/rtree.h 4a690463901cb5e6127cf05eb8e642f127012fd5003830dbc974eca5802d9412 F ext/rtree/rtree1.test d47f58832145fcfed9067bc457ca8664962196c4566c17a1ebd679367db55d11 F ext/rtree/rtree2.test 9d9deddbb16fd0c30c36e6b4fdc3ee3132d765567f0f9432ee71e1303d32603d @@ -2059,8 +2059,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17f5dd2d2ae02a95180b9208b7de805922ba20271d3263e3193f0d46f4ec324c -R 7c6f5a98ea12f09a60bc441886d297ed -U stephan -Z c684703beca8cd9cb5def7974649e125 +P e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7 +Q +491bd51da5e2069078d7295396d80d2ccdc3a5871714fef948076939174e6acd +R 0ddcb2c385e38b7edc9b293e66b74615 +U drh +Z 660ef923c98b6f83ec3ee30adf820b34 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 20ed04ba3a..07be9e66cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7 \ No newline at end of file +122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a \ No newline at end of file From be5bada4d039adbd7608cb087dbdad948374548c Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 24 Apr 2023 23:14:34 +0000 Subject: [PATCH 268/341] Allow trailing commas in objects and arrays of JSON. FossilOrigin-Name: 4031b231c223db598b45dbd192b027a99a9b82a981d43c75f723a3fb06720b82 --- manifest | 18 ++++++++++-------- manifest.uuid | 2 +- src/json.c | 4 ++-- test/json101.test | 12 ++++++++++++ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 0509728ae1..d92e1a5b2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\scompile-time\sdetection\sof\sarchitecture\sbyte-order\sin\sthe\sRTREE\nextension\sso\sthat\sit\sis\saligned\swith\sthe\slatest\senhancements\sin\sthe\score. -D 2023-04-24T19:23:42.196 +C Allow\strailing\scommas\sin\sobjects\sand\sarrays\sof\sJSON. +D 2023-04-24T23:14:34.394 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c edae65fe1f66ce8b1e7fa6eb036a3d8cf525dacd91d58f12068e80a81ac34f61 +F src/json.c ebb5827e6ab6e527c1cafdc857858cc38d822480594ba054a7b26d1493293b4b F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -1249,7 +1249,7 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test c7707ee623c57a24845ef260d4388a6fade1eb294e450caadd7f1d9ced19dea7 +F test/json101.test 7241ab038c75ba780b009c6f37e4b660e5e63d1d74c110336a4779a2db8653f7 F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6a0ac F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 @@ -2059,9 +2059,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7 -Q +491bd51da5e2069078d7295396d80d2ccdc3a5871714fef948076939174e6acd -R 0ddcb2c385e38b7edc9b293e66b74615 +P 122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a +R 9daab91f5de799face026ab747c90d1d +T *branch * json5 +T *sym-json5 * +T -sym-trunk * U drh -Z 660ef923c98b6f83ec3ee30adf820b34 +Z e1af4267df4cb005f405e96353c52d96 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 07be9e66cc..ef3e218689 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a \ No newline at end of file +4031b231c223db598b45dbd192b027a99a9b82a981d43c75f723a3fb06720b82 \ No newline at end of file diff --git a/src/json.c b/src/json.c index aa8bf64c86..4c68608deb 100644 --- a/src/json.c +++ b/src/json.c @@ -804,7 +804,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ x = jsonParseValue(pParse, j); if( x<0 ){ pParse->iDepth--; - if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; + if( x==(-2) ) break; return -1; } if( pParse->oom ) return -1; @@ -838,7 +838,7 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ x = jsonParseValue(pParse, j); pParse->iDepth--; if( x<0 ){ - if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; + if( x==(-3) ) break; return -1; } j = x; diff --git a/test/json101.test b/test/json101.test index 596a084968..58559aafe5 100644 --- a/test/json101.test +++ b/test/json101.test @@ -308,12 +308,24 @@ do_execsql_test json-5.8 { do_execsql_test json-6.1 { SELECT json_valid('{"a":55,"b":72,}'); +} {1} +do_execsql_test json-6.1b { + SELECT json_valid('{"a":55,"b":72 , }'); +} {1} +do_execsql_test json-6.1c { + SELECT json_valid('{"a":55,"b":72,,}'); } {0} do_execsql_test json-6.2 { SELECT json_valid('{"a":55,"b":72}'); } {1} do_execsql_test json-6.3 { SELECT json_valid('["a",55,"b",72,]'); +} {1} +do_execsql_test json-6.3b { + SELECT json_valid('["a",55,"b",72 , ]'); +} {1} +do_execsql_test json-6.3c { + SELECT json_valid('["a",55,"b",72,,]'); } {0} do_execsql_test json-6.4 { SELECT json_valid('["a",55,"b",72]'); From 21461407f64ef6b81b3d76ab4a23f07ececd0c1d Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 25 Apr 2023 02:44:53 +0000 Subject: [PATCH 269/341] Check for OOM sqlite_value_x() returns in base64, base85 extensions. FossilOrigin-Name: e6f9c0b1f963033a8e17d13935c5c6b12d263fe10c585035a3d1f1154c6ba5d6 --- ext/misc/base64.c | 18 ++++++++++++++++-- ext/misc/base85.c | 18 ++++++++++++++++-- manifest | 17 ++++++++--------- manifest.uuid | 2 +- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 4da3f43eb7..25e76a5e99 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -219,9 +219,16 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "blob expanded to base64 too big", -1); return; } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } cBuf = sqlite3_malloc(nc); if( !cBuf ) goto memFail; - bBuf = (u8*)sqlite3_value_blob(av[0]); nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); break; @@ -234,9 +241,16 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ }else if( nb<1 ){ nb = 1; } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } bBuf = sqlite3_malloc(nb); if( !bBuf ) goto memFail; - cBuf = (char *)sqlite3_value_text(av[0]); nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); break; diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 5ec136dbc6..8e7ea76311 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -297,9 +297,16 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "blob expanded to base85 too big", -1); return; } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } cBuf = sqlite3_malloc(nc); if( !cBuf ) goto memFail; - bBuf = (u8*)sqlite3_value_blob(av[0]); nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); break; @@ -312,9 +319,16 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ }else if( nb<1 ){ nb = 1; } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } bBuf = sqlite3_malloc(nb); if( !bBuf ) goto memFail; - cBuf = (char *)sqlite3_value_text(av[0]); nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); break; diff --git a/manifest b/manifest index 0509728ae1..965038b25b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\scompile-time\sdetection\sof\sarchitecture\sbyte-order\sin\sthe\sRTREE\nextension\sso\sthat\sit\sis\saligned\swith\sthe\slatest\senhancements\sin\sthe\score. -D 2023-04-24T19:23:42.196 +C Check\sfor\sOOM\ssqlite_value_x()\sreturns\sin\sbase64,\sbase85\sextensions. +D 2023-04-25T02:44:53.418 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -271,8 +271,8 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c e83a915fcb94c9332e9a92aa4c3beafe2552bd3da2813fc5fff31918cca0b834 -F ext/misc/base85.c 77dfd5813d23ea561d0348f922583888e78f8eaeb2b9a4a28226d092389890b8 +F ext/misc/base64.c 46e2a52e4c39b60c2b3be84001b603f9817dc90b2aba545942a1cda2b127f0be +F ext/misc/base85.c fed8e065a5eb01f70de57462dd488705cfe86120df28c39df1dad7c3470d8d89 F ext/misc/basexx.c 5e859e1820620aa8080fb9145eb47089de426ae808f6abb01a8e12921c3a8e67 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 @@ -2059,9 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e79c95fc130fc302719690eb6391d96070aff825b2b51ef6c4ad459d9a8918d7 -Q +491bd51da5e2069078d7295396d80d2ccdc3a5871714fef948076939174e6acd -R 0ddcb2c385e38b7edc9b293e66b74615 -U drh -Z 660ef923c98b6f83ec3ee30adf820b34 +P 122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a +R 3428184709b95547abf99ded23dd44a2 +U larrybr +Z e09ba3c72f888e22a41c09e468a29a9d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 07be9e66cc..6af61fd070 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a \ No newline at end of file +e6f9c0b1f963033a8e17d13935c5c6b12d263fe10c585035a3d1f1154c6ba5d6 \ No newline at end of file From 38c2052767df63ced5b3d96c46844f6a8d28a86f Mon Sep 17 00:00:00 2001 From: larrybr Date: Tue, 25 Apr 2023 04:28:39 +0000 Subject: [PATCH 270/341] Handle newline-trimmed input TEXT correctly in base64, base85 UDFs. FossilOrigin-Name: 8f637aae23e6638c064a34262dcf16a3cdfd000fb1fa1b2a834b292fe6659408 --- ext/misc/base64.c | 6 +++--- ext/misc/base85.c | 6 +++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ext/misc/base64.c b/ext/misc/base64.c index 25e76a5e99..bc7a976abc 100644 --- a/ext/misc/base64.c +++ b/ext/misc/base64.c @@ -148,9 +148,9 @@ static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ } /* Skip over text which is not base64 numeral(s). */ -static char * skipNonB64( char *s ){ +static char * skipNonB64( char *s, int nc ){ char c; - while( (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; return s; } @@ -159,7 +159,7 @@ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 && *pIn!=PAD_CHAR ){ static signed char nboi[] = { 0, 0, 1, 2, 3 }; - char *pUse = skipNonB64(pIn); + char *pUse = skipNonB64(pIn, ncIn); unsigned long qv = 0L; int nti, nbo, nac; ncIn -= (pUse - pIn); diff --git a/ext/misc/base85.c b/ext/misc/base85.c index 8e7ea76311..e7ef0a04c9 100644 --- a/ext/misc/base85.c +++ b/ext/misc/base85.c @@ -140,9 +140,9 @@ static u8 base85DigitValue( char c ){ #define B85_DARK_MAX 80 -static char * skipNonB85( char *s ){ +static char * skipNonB85( char *s, int nc ){ char c; - while( (c = *s) && !IS_B85(c) ) ++s; + while( nc-- > 0 && (c = *s) && !IS_B85(c) ) ++s; return s; } @@ -212,7 +212,7 @@ static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 ){ static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; - char *pUse = skipNonB85(pIn); + char *pUse = skipNonB85(pIn, ncIn); unsigned long qv = 0L; int nti, nbo; ncIn -= (pUse - pIn); diff --git a/manifest b/manifest index 965038b25b..3fc806927c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Check\sfor\sOOM\ssqlite_value_x()\sreturns\sin\sbase64,\sbase85\sextensions. -D 2023-04-25T02:44:53.418 +C Handle\snewline-trimmed\sinput\sTEXT\scorrectly\sin\sbase64,\sbase85\sUDFs. +D 2023-04-25T04:28:39.874 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -271,8 +271,8 @@ F ext/misc/README.md d6dd0fe1d8af77040216798a6a2b0c46c73054d2f0ea544fbbcdccf6f23 F ext/misc/amatch.c e3ad5532799cee9a97647f483f67f43b38796b84b5a8c60594fe782a4338f358 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb F ext/misc/appendvfs.c 9642c7a194a2a25dca7ad3e36af24a0a46d7702168c4ad7e59c9f9b0e16a3824 -F ext/misc/base64.c 46e2a52e4c39b60c2b3be84001b603f9817dc90b2aba545942a1cda2b127f0be -F ext/misc/base85.c fed8e065a5eb01f70de57462dd488705cfe86120df28c39df1dad7c3470d8d89 +F ext/misc/base64.c a71b131e50300c654a66c469a25b62874481f3d1cb3beb56aca9a68edd812e0d +F ext/misc/base85.c 073054111988db593ef5fdb87ab8c459df1ea0c3aaaddf0f5bfa3d72b7e6280a F ext/misc/basexx.c 5e859e1820620aa8080fb9145eb47089de426ae808f6abb01a8e12921c3a8e67 F ext/misc/blobio.c a867c4c4617f6ec223a307ebfe0eabb45e0992f74dd47722b96f3e631c0edb2a F ext/misc/btreeinfo.c d28ce349b40054eaa9473e835837bad7a71deec33ba13e39f963d50933bfa0f9 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a -R 3428184709b95547abf99ded23dd44a2 +P e6f9c0b1f963033a8e17d13935c5c6b12d263fe10c585035a3d1f1154c6ba5d6 +R 7500f54b9a757a6bdf9e41121051af70 U larrybr -Z e09ba3c72f888e22a41c09e468a29a9d +Z e45f0a7f30148e18d14433c2a4564a3c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6af61fd070..a40e09b2de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6f9c0b1f963033a8e17d13935c5c6b12d263fe10c585035a3d1f1154c6ba5d6 \ No newline at end of file +8f637aae23e6638c064a34262dcf16a3cdfd000fb1fa1b2a834b292fe6659408 \ No newline at end of file From 0941ea8a62991ab5725ff729fd4d8cacce1434e1 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Apr 2023 14:37:12 +0000 Subject: [PATCH 271/341] Avoid assuming that an expression in an ON() clause that evaluates to zero implies that the query will return zero rows when the query contains a RIGHT JOIN. [forum:95849acbe1|Forum Post 95849acbe1]. FossilOrigin-Name: 1783655ea422185e75593b89e4ef452a6f5496aefd389f88ce7fe4b7d41d6a98 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 46 +++++++++++++++++++++++++++++++++++----------- test/joinH.test | 30 ++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 3fc806927c..39088ebefe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\snewline-trimmed\sinput\sTEXT\scorrectly\sin\sbase64,\sbase85\sUDFs. -D 2023-04-25T04:28:39.874 +C Avoid\sassuming\sthat\san\sexpression\sin\san\sON()\sclause\sthat\sevaluates\sto\szero\simplies\sthat\sthe\squery\swill\sreturn\szero\srows\swhen\sthe\squery\scontains\sa\sRIGHT\sJOIN.\s[forum:95849acbe1|Forum\sPost\s95849acbe1]. +D 2023-04-25T14:37:12.443 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -717,7 +717,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 3f4ac276a60bda76f9f1f6f1c2c38599bacd4987e5efcd3f7fed2647bf97280a F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c ef9e644d1d76e86f68c941eb05d30a98fc0811eeef2a110906d81fedde81b6bf +F src/where.c 8899e22a397c839f551c8e1741c100ae2675feb0fe0a383d55816f2ca3d2b583 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 85790d7e5365ac41085713331ce52e4343586ad3d37d218ffe00572357baa62b F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -1242,7 +1242,7 @@ F test/joinC.test 1f1a602c2127f55f136e2cbd3bf2d26546614bf8cffe5902ec1ac9c07f87f2 F test/joinD.test 2ce62e7353a0702ca5e70008faf319c1d4686aa19fba34275c6d1da0e960be28 F test/joinE.test d5d182f3812771e2c0d97c9dcf5dbe4c41c8e21c82560e59358731c4a3981d6b F test/joinF.test 53dd66158806823ea680dd7543b5406af151b5aafa5cd06a7f3231cd94938127 -F test/joinH.test 15f501b33d848521964afde9865a92aeca79c8c41fa84dc4dc3f865c9ed8c868 +F test/joinH.test 705157cf9b9b7c207caf960812a7d0e4dc1dd45aa5fb2b563f12df59088645f3 F test/journal1.test c7b768041b7f494471531e17abc2f4f5ebf9e5096984f43ed17c4eb80ba34497 F test/journal2.test 9dac6b4ba0ca79c3b21446bbae993a462c2397c4 F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ffa140e @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e6f9c0b1f963033a8e17d13935c5c6b12d263fe10c585035a3d1f1154c6ba5d6 -R 7500f54b9a757a6bdf9e41121051af70 -U larrybr -Z e45f0a7f30148e18d14433c2a4564a3c +P 8f637aae23e6638c064a34262dcf16a3cdfd000fb1fa1b2a834b292fe6659408 +R 162170fabcf6c22e465922e65b3136a8 +U dan +Z 748c21dff95a1a525519cd6fda798ed3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a40e09b2de..b242eaaebc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f637aae23e6638c064a34262dcf16a3cdfd000fb1fa1b2a834b292fe6659408 \ No newline at end of file +1783655ea422185e75593b89e4ef452a6f5496aefd389f88ce7fe4b7d41d6a98 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8200daca80..bc63c53b58 100644 --- a/src/where.c +++ b/src/where.c @@ -5982,22 +5982,46 @@ WhereInfo *sqlite3WhereBegin( } if( pParse->nErr ) goto whereBeginError; - /* Special case: WHERE terms that do not refer to any tables in the join - ** (constant expressions). Evaluate each such term, and jump over all the - ** generated code if the result is not true. + /* The False-WHERE-Term-Bypass optimization: ** - ** Do not do this if the expression contains non-deterministic functions - ** that are not within a sub-select. This is not strictly required, but - ** preserves SQLite's legacy behaviour in the following two cases: + ** If there are WHERE terms that are false, then no rows will be output, + ** so skip over all of the code generated here. ** - ** FROM ... WHERE random()>0; -- eval random() once per row - ** FROM ... WHERE (SELECT random())>0; -- eval random() once overall + ** Conditions: + ** + ** (1) The WHERE term must not refer to any tables in the join. + ** (2) The term must not come from an ON clause on the + ** right-hand side of a LEFT or FULL JOIN. + ** (3) The term must not come from an ON clause, or there must be + ** no RIGHT or FULL OUTER joins in pTabList. + ** (4) If the expression contains non-deterministic functions + ** that are not within a sub-select. This is not required + ** for correctness but rather to preserves SQLite's legacy + ** behaviour in the following two cases: + ** + ** WHERE random()>0; -- eval random() once per row + ** WHERE (SELECT random())>0; -- eval random() just once overall + ** + ** Note that the Where term need not be a constant in order for this + ** optimization to apply, though it does need to be constant relative to + ** the current subquery (condition 1). The term might include variables + ** from outer queries so that the value of the term changes from one + ** invocation of the current subquery to the next. */ for(ii=0; iinBase; ii++){ - WhereTerm *pT = &sWLB.pWC->a[ii]; + WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */ + Expr *pX; /* The expression of pT */ + int ltoj; /* Left table of the join */ if( pT->wtFlags & TERM_VIRTUAL ) continue; - if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){ - sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL); + pX = pT->pExpr; + assert( pX!=0 ); + assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) ); + if( pT->prereqAll==0 /* Conditions (1) and (2) */ + && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */ + && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */ + && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 ) + ){ + sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL); pT->wtFlags |= TERM_CODED; } } diff --git a/test/joinH.test b/test/joinH.test index 1d5f66afa2..78d1556293 100644 --- a/test/joinH.test +++ b/test/joinH.test @@ -89,5 +89,35 @@ do_execsql_test 4.4 { SELECT (d IS NULL) FROM t1 RIGHT JOIN t2 ON (j=33); } {1} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.0 { + CREATE TABLE t0(w); + CREATE TABLE t1(x); + CREATE TABLE t2(y); + CREATE TABLE t3(z); + INSERT INTO t3 VALUES('t3val'); +} + +do_execsql_test 5.1 { + SELECT * FROM t1 INNER JOIN t2 ON (0) RIGHT OUTER JOIN t3; +} {{} {} t3val} + +do_execsql_test 5.2 { + SELECT * FROM t1 INNER JOIN t2 ON (0) FULL OUTER JOIN t3; +} {{} {} t3val} + +do_execsql_test 5.3 { + SELECT * FROM t3 LEFT JOIN t2 ON (0); +} {t3val {}} + +do_execsql_test 5.4 { + SELECT * FROM t0 RIGHT JOIN t1 INNER JOIN t2 ON (0) RIGHT JOIN t3 +} {{} {} {} t3val} + +do_execsql_test 5.5 { + SELECT * FROM t0 RIGHT JOIN t1 INNER JOIN t2 ON (0) +} {} finish_test From 77a159212b59a578fd3e2bb82c4a826616148ad0 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Apr 2023 14:54:18 +0000 Subject: [PATCH 272/341] Remove unused variable ltoj accidentally added by [1783655e]. FossilOrigin-Name: faa1575ce0a39a2335a77057e8769a4e67aedd6326b4627988d03f0cb21fc39a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 39088ebefe..414190236b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sassuming\sthat\san\sexpression\sin\san\sON()\sclause\sthat\sevaluates\sto\szero\simplies\sthat\sthe\squery\swill\sreturn\szero\srows\swhen\sthe\squery\scontains\sa\sRIGHT\sJOIN.\s[forum:95849acbe1|Forum\sPost\s95849acbe1]. -D 2023-04-25T14:37:12.443 +C Remove\sunused\svariable\sltoj\saccidentally\sadded\sby\s[1783655e]. +D 2023-04-25T14:54:18.391 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -717,7 +717,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 3f4ac276a60bda76f9f1f6f1c2c38599bacd4987e5efcd3f7fed2647bf97280a F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 8899e22a397c839f551c8e1741c100ae2675feb0fe0a383d55816f2ca3d2b583 +F src/where.c f69d94f34e1c523cd9b66041e4afe015cad29888617f3c09a2a5bc36018917d0 F src/whereInt.h e25203e5bfee149f5f1225ae0166cfb4f1e65490c998a024249e98bb0647377c F src/wherecode.c 85790d7e5365ac41085713331ce52e4343586ad3d37d218ffe00572357baa62b F src/whereexpr.c 1dfda1695e4480c24248157df55bb4d66c732dc8d14ac16b4f076bb15de93d63 @@ -2059,8 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8f637aae23e6638c064a34262dcf16a3cdfd000fb1fa1b2a834b292fe6659408 -R 162170fabcf6c22e465922e65b3136a8 +P 1783655ea422185e75593b89e4ef452a6f5496aefd389f88ce7fe4b7d41d6a98 +R af6a6a477260f7ae036c05114ac0158b U dan -Z 748c21dff95a1a525519cd6fda798ed3 +Z 782079e9b761f4a6226411428cf5075b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b242eaaebc..07eaee3eb2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1783655ea422185e75593b89e4ef452a6f5496aefd389f88ce7fe4b7d41d6a98 \ No newline at end of file +faa1575ce0a39a2335a77057e8769a4e67aedd6326b4627988d03f0cb21fc39a \ No newline at end of file diff --git a/src/where.c b/src/where.c index bc63c53b58..f517150e12 100644 --- a/src/where.c +++ b/src/where.c @@ -6011,7 +6011,6 @@ WhereInfo *sqlite3WhereBegin( for(ii=0; iinBase; ii++){ WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */ Expr *pX; /* The expression of pT */ - int ltoj; /* Left table of the join */ if( pT->wtFlags & TERM_VIRTUAL ) continue; pX = pT->pExpr; assert( pX!=0 ); From 058f3dbb270e9784f72cf9deb7f176fa194e56fe Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 25 Apr 2023 21:24:20 +0000 Subject: [PATCH 273/341] The json_valid() function only returns true for pure JSON. JSON5 (or at least that subset of JSON5 that has been so far implemented) is accepted by all routines, but json_valid() still returns false for JSON5 inputs. The new json_valid5(X) routine returns true or false if X is or is not valid JSON5. All of this is experimental and subject to change. FossilOrigin-Name: 5d33ab77800765c8b3a13ffcc02ba8a348d71b2b425924560418b517d723494d --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/json.c | 26 ++++++++++++++++++++++---- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d92e1a5b2b..b5b8818efe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\strailing\scommas\sin\sobjects\sand\sarrays\sof\sJSON. -D 2023-04-24T23:14:34.394 +C The\sjson_valid()\sfunction\sonly\sreturns\strue\sfor\spure\sJSON.\s\sJSON5\s(or\nat\sleast\sthat\ssubset\sof\sJSON5\sthat\shas\sbeen\sso\sfar\simplemented)\sis\saccepted\nby\sall\sroutines,\sbut\sjson_valid()\sstill\sreturns\sfalse\sfor\sJSON5\sinputs.\nThe\snew\sjson_valid5(X)\sroutine\sreturns\strue\sor\sfalse\sif\sX\sis\sor\sis\snot\svalid\nJSON5.\s\sAll\sof\sthis\sis\sexperimental\sand\ssubject\sto\schange. +D 2023-04-25T21:24:20.117 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c ebb5827e6ab6e527c1cafdc857858cc38d822480594ba054a7b26d1493293b4b +F src/json.c 1d049d073c0197d55e0046877cb87ad7b462419748cf08663d3baf04db676efb F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2059,11 +2059,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 122431d3a7267ec83768316ab146c0557fb6c0577a4a47ac6ed3d7aa6811ca9a -R 9daab91f5de799face026ab747c90d1d -T *branch * json5 -T *sym-json5 * -T -sym-trunk * +P 4031b231c223db598b45dbd192b027a99a9b82a981d43c75f723a3fb06720b82 +R 4aa06ba3d655f30dc5fe96279fd3a8e6 U drh -Z e1af4267df4cb005f405e96353c52d96 +Z 9b40834215aad90a070495da8b812bef # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ef3e218689..b9efb7f40f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4031b231c223db598b45dbd192b027a99a9b82a981d43c75f723a3fb06720b82 \ No newline at end of file +5d33ab77800765c8b3a13ffcc02ba8a348d71b2b425924560418b517d723494d \ No newline at end of file diff --git a/src/json.c b/src/json.c index 4c68608deb..b201fdde5c 100644 --- a/src/json.c +++ b/src/json.c @@ -130,9 +130,10 @@ struct JsonParse { JsonNode *aNode; /* Array of nodes containing the parse */ const char *zJson; /* Original JSON string */ u32 *aUp; /* Index of parent of each node */ - u8 oom; /* Set to true if out of memory */ - u8 nErr; /* Number of errors seen */ u16 iDepth; /* Nesting depth */ + u8 nErr; /* Number of errors seen */ + u8 oom; /* Set to true if out of memory */ + u8 has5; /* True if input has JSON5 features */ int nJson; /* Length of the zJson string in bytes */ u32 iHold; /* Replace cache line with the lowest iHold value */ }; @@ -804,7 +805,10 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ x = jsonParseValue(pParse, j); if( x<0 ){ pParse->iDepth--; - if( x==(-2) ) break; + if( x==(-2) ){ + if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; + break; + } return -1; } if( pParse->oom ) return -1; @@ -838,7 +842,10 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ x = jsonParseValue(pParse, j); pParse->iDepth--; if( x<0 ){ - if( x==(-3) ) break; + if( x==(-3) ){ + if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; + break; + } return -1; } j = x; @@ -1997,6 +2004,16 @@ static void jsonValidFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + UNUSED_PARAMETER(argc); + p = jsonParseCached(ctx, argv, 0); + sqlite3_result_int(ctx, p!=0 && p->has5==0); +} +static void jsonValid5Func( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv ){ JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); @@ -2725,6 +2742,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_type, 1, 0, jsonTypeFunc), JFUNCTION(json_type, 2, 0, jsonTypeFunc), JFUNCTION(json_valid, 1, 0, jsonValidFunc), + JFUNCTION(json_valid5, 1, 0, jsonValid5Func), #if SQLITE_DEBUG JFUNCTION(json_parse, 1, 0, jsonParseFunc), JFUNCTION(json_test1, 1, 0, jsonTest1Func), From 02efa7be9610963288319eb0f79a1a128d24bd5f Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Apr 2023 13:25:30 +0000 Subject: [PATCH 274/341] Add scripts for JSON performance testing. FossilOrigin-Name: 3051d6a7c12cbf86634d8fab64f3f3bc9a1eb184260e56a82eb543cb634ba7ef --- manifest | 14 +- manifest.uuid | 2 +- test/json/README.md | 26 +++ test/json/json-generator.tcl | 401 ++++++++++++++++++++++++++++++++++ test/json/json-q1.txt | 4 + test/json/json-speed-check.sh | 79 +++++++ 6 files changed, 520 insertions(+), 6 deletions(-) create mode 100644 test/json/README.md create mode 100644 test/json/json-generator.tcl create mode 100644 test/json/json-q1.txt create mode 100755 test/json/json-speed-check.sh diff --git a/manifest b/manifest index b5b8818efe..cc417d08a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sjson_valid()\sfunction\sonly\sreturns\strue\sfor\spure\sJSON.\s\sJSON5\s(or\nat\sleast\sthat\ssubset\sof\sJSON5\sthat\shas\sbeen\sso\sfar\simplemented)\sis\saccepted\nby\sall\sroutines,\sbut\sjson_valid()\sstill\sreturns\sfalse\sfor\sJSON5\sinputs.\nThe\snew\sjson_valid5(X)\sroutine\sreturns\strue\sor\sfalse\sif\sX\sis\sor\sis\snot\svalid\nJSON5.\s\sAll\sof\sthis\sis\sexperimental\sand\ssubject\sto\schange. -D 2023-04-25T21:24:20.117 +C Add\sscripts\sfor\sJSON\sperformance\stesting. +D 2023-04-26T13:25:30.498 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1249,6 +1249,10 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa +F test/json/README.md 9d117e0d6da9eee96de2fc8e32b603208b0e5b460ff99e5de3407bb713f0bb5c +F test/json/json-generator.tcl 229bd293f1865f787c160886cadd282631721925cca2947aaa54bbcd7f65cef7 +F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d +F test/json/json-speed-check.sh b1544c21e6f04e9c3ecc3caa858e0607703a4ed29d4d8b631a736d28e298ed9d x F test/json101.test 7241ab038c75ba780b009c6f37e4b660e5e63d1d74c110336a4779a2db8653f7 F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6a0ac F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe @@ -2059,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4031b231c223db598b45dbd192b027a99a9b82a981d43c75f723a3fb06720b82 -R 4aa06ba3d655f30dc5fe96279fd3a8e6 +P 5d33ab77800765c8b3a13ffcc02ba8a348d71b2b425924560418b517d723494d +R 370adb66bc2173cabe35b75b43054560 U drh -Z 9b40834215aad90a070495da8b812bef +Z 49f1c31b7a1708965e204247b5ad4fd6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b9efb7f40f..96d71859fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d33ab77800765c8b3a13ffcc02ba8a348d71b2b425924560418b517d723494d \ No newline at end of file +3051d6a7c12cbf86634d8fab64f3f3bc9a1eb184260e56a82eb543cb634ba7ef \ No newline at end of file diff --git a/test/json/README.md b/test/json/README.md new file mode 100644 index 0000000000..94f5516c06 --- /dev/null +++ b/test/json/README.md @@ -0,0 +1,26 @@ +The files in this subdirectory are used to help measure the performance +of the SQLite JSON parser. + +# 1.0 Prerequisites + + 1. Valgrind + + 2. Fossil + +# 2.0 Setup + + 1. Run: "`tclsh json-generator.tcl | sqlite3 json100mb.db`" to create + the 100 megabyte test database. Do this so that the "json100mb.db" + file lands in the same directory as the json-generator.tcl script. + + 2. Build the baseline sqlite3.c file. ("`make sqlite3.c`") + + 3. Run "`sh json-speed-check-1.sh trunk`". This creates the baseline + profile in "jout-trunk.txt". + +# 3.0 Testing + + 1. Build the sqlite3.c to be tested. + + 2. Run "`sh json-speed-check-1.sh x1`". The profile output will appear + in jout-x1.txt. Substitute any label you want in place of "x1". diff --git a/test/json/json-generator.tcl b/test/json/json-generator.tcl new file mode 100644 index 0000000000..e99d6bccb0 --- /dev/null +++ b/test/json/json-generator.tcl @@ -0,0 +1,401 @@ +#!/usr/bin/tclsh +# +# Generate SQL that will populate an SQLite database with about 100 megabytes +# of pseudo-random JSON text. +# +# tclsh json-generator.tcl | sqlite3 json110mb.db +# +# srand() is used to initialize the random seed so that the same JSON +# is generated for every run. +# +expr srand(12345678) +set wordlist { +ability able abroad access account act + action active actor add address adept + adroit advance advice affect age ageless + agency agent agile agree air airfare + airline airport alert almond alpha always + amend amount amplify analyst anchor angel + angelic angle ankle annual answer antique + anybody anyhow appeal apple apricot apt + area argon arm army arrival arsenic + art artful article arugula aside ask + aspect assist assume atom atone attempt + author autumn average avocado award awl + azure back bacon bag bagel bake + baker balance ball balloon bamboo banana + band banjo bank barium base basil + basin basis basket bass bat bath + battery beach beak bean bear bearcub + beauty beef beet beige being bell + belly belt bench bend benefit best + beta better beyond bicycle bid big + bike bill bird biscuit bismuth bisque + bit black blank blest blind bliss + block bloom blue board boat body + bokchoy bone bonus book bookish boot + border boron boss bossy bottle bottom + bow bowl bowtie box brain brainy + branch brave bravely bread break breath + breezy brick bridge brie brief briefly + bright broad broil bromine bronze brother + brow brown brush buddy budget buffalo + bug bugle bull bunch burger burly + burrito bus busy butter button buy + buyer byte cab cabbage cabinet cable + cadet cadmium caesium cake calcium caliper + call caller calm calmly camera camp + can canary cancel candle candy cap + capable caper capital captain car carbon + card care career careful carp carpet + carrot carry case cash cassava casual + cat catch catfish catsear catsup cause + cave celery cell century chain chair + chalk chance change channel chapter chard + charge charity chart check cheddar cheery + cheese chicken chicory chiffon child chin + chip chives choice chowder chum church + circle city claim clam class classic + classy clay clean cleaner clear clearly + clerk click client climate clock clorine + closet clothes cloud clown club clue + cluster coach coast coat cobbler cobolt + cod code coffee colby cold collar + college comb combine comet comfort command + comment common company complex concept concern + concert conduit consist contact contest context + control convert cook cookie copilot copper + copy coral cordial corn corner corny + correct cost count counter country county + couple courage course court cover cow + cowbird crab crack craft crash crazy + cream credit creek cress crevice crew + crimson croaker crop cross crowd cube + cuckoo cuisine culture cup current curve + cut cyan cycle dagger daily dance + dare darter data date day daylily + deal dear dearly debate debit decade + decimal deep deft deftly degree delay + deluxe deposit depth design desk detail + device dew diamond diet dig dill + dinner dip direct dirt dish disk + display diver divide divine doctor dodger + donut door dot double dough draft + drag dragon drama draw drawer drawing + dream drill drink drive driver drop + drum dry dryer drywall duck due + dump dusk dust duty dye eagle + ear earring earth ease east easy + eat economy edge editor eel effect + effort egg eight elbow elegant element + elf elk email emerald employ end + endive endless energy engine enjoy enter + entry equal equip error escape essay + eternal evening event exam example excuse + exit expert extent extreme eye face + fact factor factual fail failure fair + fajita fall family fan fang farm + farmer fat fault feature feed feel + feeling fench fennel festive few fiber + field fig figure file fill film + filter final finance finding finger finish + fire fish fishing fit fitting five + fix flier flight floor floral florine + flour flow flower fly flying focus + fold folding food foot force forest + forever forgive form formal format fortune + forum frame free freedom freely fresh + friend frog front fruit fuchsia fuel + fun funny future gain galaxy gallium + game gamma gap garage garden garlic + gas gate gather gauge gear gem + gene general gentle gently gherkin ghost + gift give glad glass gleeful glossy + glove glue goal goat goby gold + goldeye golf good gouda goulash gourd + grab grace grade gram grand grape + grapes grass gravy gray great green + grits grocery ground group grouper grout + growth guard guave guess guest guide + guitar gumbo guppy habit hacksaw haddock + hafnium hagfish hair half halibut hall + hammer hand handle handy hanger happy + hat havarti hay haybale head health + healthy hearing heart hearty heat heavy + heel height helium hello help helpful + herald herring hide high highly highway + hill hip hipster hire history hit + hoki hold hole holiday holly home + honest honey hook hope hopeful horizon + horn horse host hotel hour house + housing human humane humor hunt hurry + ice icecube icefish icy idea ideal + image impact impress inch income indigo + initial inkpen insect inside intense invite + iodine iridium iron island issue item + ivory jacket jargon javelin jello jelly + jewel job jocund join joint joke + jovial joy joyful joyous judge juice + jump junior jury just justice kale + keel keep kelp ketchup key keyhole + keyway khaki kick kid kidney kiloohm + kind kindly king kitchen kite kiwi + knee knife krill krypton kumquat lab + lace lack ladder lake lamp lamprey + land laser laugh law lawn lawyer + layer lead leader leading leaf leafy + league leather leave lecture leek leg + lemon length lentil lesson let letter + lettuce level library life lift light + lily lime limit line linen link + lip list listen lithium lively living + lizard load loan lobster local lock + log long longfin look lotus love + lovely loving low lucid luck luffa + lunch lung machine magenta magnet mail + main major make mall manager mango + manner many map march market maroon + martian master match math matter maximum + maybe meal meaning meat media medium + meet meeting melody melon member memory + mention menu mercury merry mess message + messy metal meter method micron middle + might mile milk mind mine minimum + minnow minor mint minute mirror miss + mission misty mix mixer mixture mobile + mode model moment monitor monk month + moon moray morning most motor mouse + mouth move mover movie much mud + mudfish muffin mullet munster muon muscle + music mustard nail name nation native + natural nature navy neat neatly nebula + neck needle neon nerve net network + neutron news nibble nice nickel night + niobium nobody noise noodle normal north + nose note nothing notice nova novel + number nurse nursery oar object offer + office officer oil okay okra old + olive one onion open opening opinion + option orange orbit orchid order oregano + other ounce outcome outside oven owner + oxygen oyster pace pack package page + pager paint pair pale pan pancake + papaya paper pardon parent park parking + parsley parsnip part partner party pass + passage past pasta path patient pattern + pause pay pea peace peach peacock + peahen peak peanut pear pearl pen + penalty pencil pension people pepper perch + perfect period permit person phase phone + photo phrase physics piano pick picture + pie piece pigeon pike pilot pin + pink pinkie pious pipe pitch pizza + place plan plane planet plant planter + plastic plate play player playful plenty + pliers plum pod poem poet poetry + point police policy pollock pony pool + pop popover poptart pork port portal + post pot potato pound powder power + present press price pride primary print + prior private prize problem process produce + product profile profit program project promise + prompt proof proper protein proton public + puff puffer pull pumpkin pup pupfish + pure purple purpose push put quality + quark quarter quiet quill quit quote + rabbit raccoon race radiant radio radish + radium radon rain rainbow raise ramp + ranch range rasp rate ratio ray + razor reach read reading real reality + reason recipe record recover red redeem + reed reef refuse region regret regular + relaxed release relief relish remote remove + rent repair repeat reply report request + reserve resist resolve resort rest result + return reveal review reward ribbon rice + rich ride ridge right ring rise + risk river rivet road roast rock + rocket role roll roof room rope + rose rough roughy round row royal + rub ruby rudder ruin rule run + runner rush rust sacred saddle safe + safety sail salad salami sale salmon + salt sample sand sander sandy sauce + save saving saw scale scampi scene + scheme school score screen script sea + search season seat second secret sector + seemly self sell senate senior sense + series serve set shake shape share + shark shell shift shine shiny ship + shock shoe shoot shop shovel show + side sign signal silk silly silver + simple sing singer single sink site + size skill skin sky slate sleep + sleepy slice slide slip smart smell + smelt smile smoke smooth snap snipe + snow snowy sock socket sodium soft + softly soil sole solid song sorrel + sort soul sound soup source south + space spare speech speed spell spend + sphere spice spider spirit spite split + spoon sport spot spray spread spring + squab square squash stable staff stage + stand staple star start state status + stay steak steel step stern stew + stick still stock stone stop store + storm story strain street stress strike + string stroke strong studio study stuff + style sugar suit sulfur summer sun + sunny sunset super superb surf survey + sweet swim swing switch symbol system + table tackle tail tale talk tan + tank tap tape target task taste + tau tea teach teal team tear + tell ten tender tennis tent term + test tetra text thanks theme theory + thing think thread throat thumb ticket + tidy tie tiger till time timely + tin tip title toast today toe + tomato tone tongue tool tooth top + topic total touch tough tour towel + tower town track trade train trash + travel tray treat tree trick trip + trout trowel truck trupet trust truth + try tube tuna tune turf turkey + turn turnip tutor tux tweet twist + two type union unique unit upbeat + upper use useful user usual valley + value van vase vast veil vein + velvet verse very vessel vest video + view violet visit visual vivid voice + volume vowel voyage waffle wait wake + walk wall warm warmth wasabi wash + watch water wave wax way wealth + wear web wedge week weekly weight + west whale what wheat wheel when + where while who whole why will + win wind window wing winner winter + wire wish witty wolf wonder wood + wool woolly word work worker world + worry worth worthy wrap wrench wrist + writer xenon yak yam yard yarrow + year yearly yellow yew yogurt young + youth zebra zephyr zinc zone zoo +} +set nwordlist [llength $wordlist] + +proc random_char {} { + return [string index \ + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" \ + [expr {int(rand()*52)}]] +} +proc random_label {} { + set label [random_char] + while {rand()>0.8} { + append label [random_char] + } + if {rand()>0.9} {append label -} + append label [format %d [expr {int(rand()*100)}]] + return $label +} +proc random_numeric {} { + set n [expr {(rand()*2-1.0)*1e6}] + switch [expr {int(rand()*6)}] { + 0 {set format %.3f} + 1 {set format %.6E} + 2 {set format %.4e} + default {set format %g} + } + return [format $format $n] +} + + +proc random_json {limit indent} { + global nwordlist wordlist + set res {} + if {$indent==0 || ($limit>0 && rand()>0.5)} { + incr limit -1 + incr indent 2 + set n [expr {int(rand()*5)+1}] + if {$n==5} {incr n [expr {int(rand()*10)}]} + if {rand()>0.5} { + set res \173\n + for {set i 0} {$i<$n} {incr i} { + append res [string repeat { } $indent] + if {rand()>0.8} { + if {rand()>0.5} { + set sep ":\n [string repeat { } $indent]" + } else { + set sep " : " + } + } else { + set sep : + } + append res \"[random_label]\"$sep[random_json $limit $indent] + if {$i<$n-1} {append res ,} + append res \n + } + incr indent -2 + append res [string repeat { } $indent] + append res \175 + return $res + } else { + set res \[\n + for {set i 0} {$i<$n} {incr i} { + append res [string repeat { } $indent] + append res [random_json $limit $indent] + if {$i<$n-1} {append res ,} + append res \n + } + incr indent -2 + append res [string repeat { } $indent] + append res \] + return $res + } + } elseif {rand()>0.9} { + if {rand()>0.7} {return "true"} + if {rand()>0.5} {return "false"} + return "null" + } elseif {rand()>0.5} { + return [random_numeric] + } else { + set res \" + set n [expr {int(rand()*4)+1}] + if {$n>=4} {set n [expr {$n+int(rand()*6)}]} + for {set i 0} {$i<$n} {incr i} { + if {rand()<0.05} { + set w [random_numeric] + } else { + set k [expr {int(rand()*$nwordlist)}] + set w [lindex $wordlist $k] + } + if {rand()<0.07} { + set w \\\"$w\\\" + } + if {$i<$n-1} { + switch [expr {int(rand()*9)}] { + 0 {set sp {, }} + 1 {set sp "\\n "} + 2 {set sp "-"} + default {set sp { }} + } + append res $w$sp + } else { + append res $w + if {rand()<0.2} {append res .} + } + } + return $res\" + } +} + +puts "CREATE TABLE IF NOT EXISTS data1(x JSON);" +puts "BEGIN;" +set sz 0 +for {set i 0} {$sz<100000000} {incr i} { + set j [random_json 7 0] + incr sz [string length $j] + puts "INSERT INTO data1(x) VALUES('$j');" +} +puts "COMMIT;" +puts "SELECT sum(length(x)) FROM data1;" diff --git a/test/json/json-q1.txt b/test/json/json-q1.txt new file mode 100644 index 0000000000..0395f0c061 --- /dev/null +++ b/test/json/json-q1.txt @@ -0,0 +1,4 @@ +.mode qbox +.timer on +.param set $label 'q87' +SELECT rowid, x->>$label FROM data1 WHERE x->>$label IS NOT NULL; diff --git a/test/json/json-speed-check.sh b/test/json/json-speed-check.sh new file mode 100755 index 0000000000..01147c18be --- /dev/null +++ b/test/json/json-speed-check.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# This is a template for a script used for day-to-day size and +# performance monitoring of SQLite. Typical usage: +# +# sh speed-check.sh trunk # Baseline measurement of trunk +# sh speed-check.sh x1 # Measure some experimental change +# fossil xdiff --tk jout-trunk.txt jout-x1.txt # View chanages +# +# There are multiple output files, all with a base name given by +# the first argument: +# +# summary-$BASE.txt # Copy of standard output +# jout-$BASE.txt # cachegrind output +# explain-$BASE.txt # EXPLAIN listings (only with --explain) +# +if test "$1" = "" +then + echo "Usage: $0 OUTPUTFILE [OPTIONS]" + exit +fi +NAME=$1 +shift +#CC_OPTS="-DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_MEMSYS5" +CC_OPTS="-DSQLITE_ENABLE_MEMSYS5" +CC=gcc +LEAN_OPTS="-DSQLITE_THREADSAFE=0" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_MEMSTATUS=0" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_LIKE_DOESNT_MATCH_BLOBS" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_MAX_EXPR_DEPTH=0" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DECLTYPE" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_DEPRECATED" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_PROGRESS_CALLBACK" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_OMIT_SHARED_CACHE" +LEAN_OPTS="$LEAN_OPTS -DSQLITE_USE_ALLOCA" +BASELINE="trunk" +doExplain=0 +doCachegrind=1 +doVdbeProfile=0 +doWal=1 +doDiff=1 +while test "$1" != ""; do + case $1 in + --nodiff) + doDiff=0 + ;; + --lean) + CC_OPTS="$CC_OPTS $LEAN_OPTS" + ;; + --clang) + CC=clang + ;; + --gcc7) + CC=gcc-7 + ;; + -*) + CC_OPTS="$CC_OPTS $1" + ;; + *) + BASELINE=$1 + ;; + esac + shift +done +echo "NAME = $NAME" | tee summary-$NAME.txt +echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt +rm -f cachegrind.out.* jsonshell +$CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -ldl -lpthread +ls -l jsonshell | tee -a summary-$NAME.txt +home=/home/drh/sqlite/json-perf +valgrind --tool=cachegrind ./jsonshell $home/json100mb.db <$home/json-q1.txt \ + 2>&1 | tee -a summary-$NAME.txt +cg_anno.tcl cachegrind.out.* >jout-$NAME.txt +echo '*****************************************************' >>jout-$NAME.txt +sed 's/^[0-9=-]\{9\}/==00000==/' summary-$NAME.txt >>jout-$NAME.txt +if test "$NAME" != "$BASELINE"; then + fossil xdiff --tk -c 20 jout-$BASELINE.txt jout-$NAME.txt +fi From ddc8b9ee0e3bccd6e488503750b31de5c672250d Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Apr 2023 13:52:20 +0000 Subject: [PATCH 275/341] Fix to the json-speed-check.sh script. FossilOrigin-Name: d839c9544d7f28c1abc779eb2d40f95c1a9386984656fbd29d19b1e7830171bc --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/json/json-speed-check.sh | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index cc417d08a7..4f17dbdff9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sscripts\sfor\sJSON\sperformance\stesting. -D 2023-04-26T13:25:30.498 +C Fix\sto\sthe\sjson-speed-check.sh\sscript. +D 2023-04-26T13:52:20.152 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1252,7 +1252,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json/README.md 9d117e0d6da9eee96de2fc8e32b603208b0e5b460ff99e5de3407bb713f0bb5c F test/json/json-generator.tcl 229bd293f1865f787c160886cadd282631721925cca2947aaa54bbcd7f65cef7 F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d -F test/json/json-speed-check.sh b1544c21e6f04e9c3ecc3caa858e0607703a4ed29d4d8b631a736d28e298ed9d x +F test/json/json-speed-check.sh 362f9c5c4a69c6f2c6fa98f538fb2e07e084dc0bef74f8bbd92cc1666e8415e5 x F test/json101.test 7241ab038c75ba780b009c6f37e4b660e5e63d1d74c110336a4779a2db8653f7 F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6a0ac F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d33ab77800765c8b3a13ffcc02ba8a348d71b2b425924560418b517d723494d -R 370adb66bc2173cabe35b75b43054560 +P 3051d6a7c12cbf86634d8fab64f3f3bc9a1eb184260e56a82eb543cb634ba7ef +R 76520c4f968a15591ad7c027fe9539e0 U drh -Z 49f1c31b7a1708965e204247b5ad4fd6 +Z 781a4e98041d9791d0ec4084972b315b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 96d71859fb..d8bd0e1a4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3051d6a7c12cbf86634d8fab64f3f3bc9a1eb184260e56a82eb543cb634ba7ef \ No newline at end of file +d839c9544d7f28c1abc779eb2d40f95c1a9386984656fbd29d19b1e7830171bc \ No newline at end of file diff --git a/test/json/json-speed-check.sh b/test/json/json-speed-check.sh index 01147c18be..343f830bda 100755 --- a/test/json/json-speed-check.sh +++ b/test/json/json-speed-check.sh @@ -68,7 +68,8 @@ echo "CC_OPTS = $CC_OPTS" | tee -a summary-$NAME.txt rm -f cachegrind.out.* jsonshell $CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -ldl -lpthread ls -l jsonshell | tee -a summary-$NAME.txt -home=/home/drh/sqlite/json-perf +home=`echo $0 | sed -e 's,/[^/]*$,,'` +echo ./jsonshell $home/json100mb.db "<$home/json-q1.txt" valgrind --tool=cachegrind ./jsonshell $home/json100mb.db <$home/json-q1.txt \ 2>&1 | tee -a summary-$NAME.txt cg_anno.tcl cachegrind.out.* >jout-$NAME.txt From f62518f65d0896d77fece571bfe63038267a9b15 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Apr 2023 15:19:19 +0000 Subject: [PATCH 276/341] Work toward implementing JSON5 whitespace. Untested and incomplete. FossilOrigin-Name: d262c059455ebe0650a45a6c1c04d1baf9609c635df352732dd192426e1bdc39 --- manifest | 14 +++--- manifest.uuid | 2 +- src/json.c | 126 +++++++++++++++++++++++++++++++++++++++++++++- test/json101.test | 37 ++++++++------ 4 files changed, 156 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 4f17dbdff9..a607ef397a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sto\sthe\sjson-speed-check.sh\sscript. -D 2023-04-26T13:52:20.152 +C Work\stoward\simplementing\sJSON5\swhitespace.\s\sUntested\sand\sincomplete. +D 2023-04-26T15:19:19.822 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 1d049d073c0197d55e0046877cb87ad7b462419748cf08663d3baf04db676efb +F src/json.c d0f9a0c630bdea3721e28b5ba2fce119c0e732ad48f6674a8f1644f2ec45ffb1 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -1253,7 +1253,7 @@ F test/json/README.md 9d117e0d6da9eee96de2fc8e32b603208b0e5b460ff99e5de3407bb713 F test/json/json-generator.tcl 229bd293f1865f787c160886cadd282631721925cca2947aaa54bbcd7f65cef7 F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d F test/json/json-speed-check.sh 362f9c5c4a69c6f2c6fa98f538fb2e07e084dc0bef74f8bbd92cc1666e8415e5 x -F test/json101.test 7241ab038c75ba780b009c6f37e4b660e5e63d1d74c110336a4779a2db8653f7 +F test/json101.test de9c93169b84ac96fd5836c638a2ae1f00e4afbd4003c6b596692d7f05e1cd69 F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6a0ac F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3051d6a7c12cbf86634d8fab64f3f3bc9a1eb184260e56a82eb543cb634ba7ef -R 76520c4f968a15591ad7c027fe9539e0 +P d839c9544d7f28c1abc779eb2d40f95c1a9386984656fbd29d19b1e7830171bc +R 9b88e23f3662645c8ffe23999211d809 U drh -Z 781a4e98041d9791d0ec4084972b315b +Z 75164b4fdafac0aaf63d4b3999912d9c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d8bd0e1a4d..0de6b4f785 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d839c9544d7f28c1abc779eb2d40f95c1a9386984656fbd29d19b1e7830171bc \ No newline at end of file +d262c059455ebe0650a45a6c1c04d1baf9609c635df352732dd192426e1bdc39 \ No newline at end of file diff --git a/src/json.c b/src/json.c index b201fdde5c..58f89b1aaa 100644 --- a/src/json.c +++ b/src/json.c @@ -758,6 +758,124 @@ static int jsonIs4Hex(const char *z){ return 1; } +/* +** Return the number of bytes of JSON5 whitespace at the beginning of +** the input string z[]. +** +** JSON5 whitespace consists of any of the following characters: +** +** Unicode UTF-8 Name +** U+0009 09 horizontal tab +** U+000a 0a line feed +** U+000b 0b vertical tab +** U+000c 0c form feed +** U+000d 0d carriage return +** U+0020 20 space +** U+00a0 c2 a0 non-breaking space +** U+1680 e1 9a 80 ogham space mark +** U+2000 e2 80 80 en quad +** U+2001 e2 80 81 em quad +** U+2002 e2 80 82 en space +** U+2003 e2 80 83 em space +** U+2004 e2 80 84 three-per-em space +** U+2005 e2 80 85 four-per-em space +** U+2006 e2 80 86 six-per-em space +** U+2007 e2 80 87 figure space +** U+2008 e2 80 88 punctuation space +** U+2009 e2 80 89 thin space +** U+200a e2 80 8a hair space +** U+2028 e2 80 a8 line separator +** U+2029 e2 80 a9 paragraph separator +** U+202f e2 80 af narrow no-break space (NNBSP) +** U+205f e2 81 9f medium mathematical space (MMSP) +** U+3000 e3 80 80 ideographical space +** U+FEFF ef bb bf byte order mark +** +** In addition, comments between '/', '*' and '*', '/' and +** from '/', '/' to end-of-line are also considered to be whitespace. +*/ +static int json5Whitespace(const char *zIn){ + int n = 0; + const u8 *z = (u8*)zIn; + while( 1 /*exit by "goto whitespace_done"*/ ){ + switch( z[n] ){ + case 0x09: + case 0x0a: + case 0x0b: + case 0x0c: + case 0x0d: + case 0x20: { + n++; + break; + } + case '/': { + if( z[n+1]=='*' && z[n+2]!=0 ){ + int j; + for(j=n+3; z[j]!='/' || z[j-1]!='*'; j++){ + if( z[j]==0 ) goto whitespace_done; + } + n += j; + break; + }else if( z[n+1]=='/' ){ + int j; + for(j=n+2; z[j] && z[j]!='\n'; j++){} + n += j; + break; + } + goto whitespace_done; + } + case 0xc2: { + if( z[n+1]==0xa0 ){ + n += 2; + break; + } + goto whitespace_done; + } + case 0xe1: { + if( z[n+1]==0x9a && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xe2: { + if( z[n+1]==0x80 ){ + u8 c = z[n+2]; + if( c<0x80 ) goto whitespace_done; + if( c<=0x8a || c==0xa8 || c==0xa9 || c==0xaf ){ + n += 3; + break; + } + }else if( z[n+1]==0x81 && z[n+2]==0x9f ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xe3: { + if( z[n+1]==0x80 && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xef: { + if( z[n+1]==0xbb && z[n+2]==0xbf ){ + n += 3; + break; + } + goto whitespace_done; + } + default: { + goto whitespace_done; + } + } + } + whitespace_done: + return n; +} + + #ifdef SQLITE_ENABLE_JSON_NAN_INF /* ** Extra floating-point literals to allow in JSON. @@ -795,7 +913,9 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ JsonNode *pNode; const char *z = pParse->zJson; while( fast_isspace(z[i]) ){ i++; } - if( (c = z[i])=='{' ){ +json_parse_restart: + c = z[i]; + if( c=='{' ){ /* Parse object */ iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); if( iThis<0 ) return -1; @@ -960,6 +1080,10 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ return -3; /* End of [...] */ }else if( c==0 ){ return 0; /* End of file */ + }else if( (j = json5Whitespace(&z[i]))>0 ){ + i += j; + pParse->has5 = 1; + goto json_parse_restart; }else{ #ifdef SQLITE_ENABLE_JSON_NAN_INF int k, nn; diff --git a/test/json101.test b/test/json101.test index 58559aafe5..3fc3923da3 100644 --- a/test/json101.test +++ b/test/json101.test @@ -308,26 +308,35 @@ do_execsql_test json-5.8 { do_execsql_test json-6.1 { SELECT json_valid('{"a":55,"b":72,}'); -} {1} -do_execsql_test json-6.1b { - SELECT json_valid('{"a":55,"b":72 , }'); -} {1} -do_execsql_test json-6.1c { - SELECT json_valid('{"a":55,"b":72,,}'); } {0} do_execsql_test json-6.2 { - SELECT json_valid('{"a":55,"b":72}'); + SELECT json_valid5('{"a":55,"b":72,}'); } {1} do_execsql_test json-6.3 { - SELECT json_valid('["a",55,"b",72,]'); + SELECT json_valid(json('{"a":55,"b":72,}')); } {1} -do_execsql_test json-6.3b { - SELECT json_valid('["a",55,"b",72 , ]'); -} {1} -do_execsql_test json-6.3c { - SELECT json_valid('["a",55,"b",72,,]'); -} {0} do_execsql_test json-6.4 { + SELECT json_valid('{"a":55,"b":72 , }'); +} {0} +do_execsql_test json-6.5 { + SELECT json_valid5('{"a":55,"b":72 , }'); +} {1} +do_execsql_test json-6.6 { + SELECT json_valid5('{"a":55,"b":72,,}'); +} {0} +do_execsql_test json-6.7 { + SELECT json_valid('{"a":55,"b":72}'); +} {1} +do_execsql_test json-6.8 { + SELECT json_valid5('["a",55,"b",72,]'); +} {1} +do_execsql_test json-6.9 { + SELECT json_valid5('["a",55,"b",72 , ]'); +} {1} +do_execsql_test json-6.10 { + SELECT json_valid5('["a",55,"b",72,,]'); +} {0} +do_execsql_test json-6.11 { SELECT json_valid('["a",55,"b",72]'); } {1} From 5255102e04d0872280dc6c776a4318e985c54abf Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Apr 2023 15:58:08 +0000 Subject: [PATCH 277/341] Improvement to the way the JSON performance measure scripts work → keep the test database in the directory of the test, not in the source tree. FossilOrigin-Name: ac411dbdcbf0b9040fb5b7de173271f383a6aa303d57f22ebd200809a5b4a6d3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/json/README.md | 3 ++- test/json/json-speed-check.sh | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a607ef397a..901405d91b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\stoward\simplementing\sJSON5\swhitespace.\s\sUntested\sand\sincomplete. -D 2023-04-26T15:19:19.822 +C Improvement\sto\sthe\sway\sthe\sJSON\sperformance\smeasure\sscripts\swork\s→\skeep\sthe\ntest\sdatabase\sin\sthe\sdirectory\sof\sthe\stest,\snot\sin\sthe\ssource\stree. +D 2023-04-26T15:58:08.228 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1249,10 +1249,10 @@ F test/journal3.test 7c3cf23ffc77db06601c1fcfc9743de8441cb77db9d1aa931863d94f5ff F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9e1946 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json/README.md 9d117e0d6da9eee96de2fc8e32b603208b0e5b460ff99e5de3407bb713f0bb5c +F test/json/README.md 506af1f54574b524106acb50d1a341ab5ddfa6d83fe25095007892b07e663e85 F test/json/json-generator.tcl 229bd293f1865f787c160886cadd282631721925cca2947aaa54bbcd7f65cef7 F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d -F test/json/json-speed-check.sh 362f9c5c4a69c6f2c6fa98f538fb2e07e084dc0bef74f8bbd92cc1666e8415e5 x +F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x F test/json101.test de9c93169b84ac96fd5836c638a2ae1f00e4afbd4003c6b596692d7f05e1cd69 F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6a0ac F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d839c9544d7f28c1abc779eb2d40f95c1a9386984656fbd29d19b1e7830171bc -R 9b88e23f3662645c8ffe23999211d809 +P d262c059455ebe0650a45a6c1c04d1baf9609c635df352732dd192426e1bdc39 +R 3eb29750520f57f10b6b50d11e64a15f U drh -Z 75164b4fdafac0aaf63d4b3999912d9c +Z f254fb62ebf6c065b09ee32328c8e984 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0de6b4f785..288ecccb2f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d262c059455ebe0650a45a6c1c04d1baf9609c635df352732dd192426e1bdc39 \ No newline at end of file +ac411dbdcbf0b9040fb5b7de173271f383a6aa303d57f22ebd200809a5b4a6d3 \ No newline at end of file diff --git a/test/json/README.md b/test/json/README.md index 94f5516c06..6a16114925 100644 --- a/test/json/README.md +++ b/test/json/README.md @@ -11,7 +11,8 @@ of the SQLite JSON parser. 1. Run: "`tclsh json-generator.tcl | sqlite3 json100mb.db`" to create the 100 megabyte test database. Do this so that the "json100mb.db" - file lands in the same directory as the json-generator.tcl script. + file lands in the directory from which you will run tests, not in + the test/json subdirectory of the source tree. 2. Build the baseline sqlite3.c file. ("`make sqlite3.c`") diff --git a/test/json/json-speed-check.sh b/test/json/json-speed-check.sh index 343f830bda..f7e7f1be58 100755 --- a/test/json/json-speed-check.sh +++ b/test/json/json-speed-check.sh @@ -69,8 +69,8 @@ rm -f cachegrind.out.* jsonshell $CC -g -Os -Wall -I. $CC_OPTS ./shell.c ./sqlite3.c -o jsonshell -ldl -lpthread ls -l jsonshell | tee -a summary-$NAME.txt home=`echo $0 | sed -e 's,/[^/]*$,,'` -echo ./jsonshell $home/json100mb.db "<$home/json-q1.txt" -valgrind --tool=cachegrind ./jsonshell $home/json100mb.db <$home/json-q1.txt \ +echo ./jsonshell json100mb.db "<$home/json-q1.txt" +valgrind --tool=cachegrind ./jsonshell json100mb.db <$home/json-q1.txt \ 2>&1 | tee -a summary-$NAME.txt cg_anno.tcl cachegrind.out.* >jout-$NAME.txt echo '*****************************************************' >>jout-$NAME.txt From f14b2e35fa2bcacdf14a3be31e7370670ebd1e34 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Apr 2023 17:30:28 +0000 Subject: [PATCH 278/341] Partial implementation of JSON5 numeric literal extensions. Use a switch() statement in the parser for better performance. FossilOrigin-Name: 78404dc37024cad5fe7eacf78ea85b56f08b129a1b9a046c3e1b11275068a485 --- manifest | 14 +++--- manifest.uuid | 2 +- src/json.c | 120 +++++++++++++++++++++++++++++++++------------- test/json102.test | 33 ++++++++----- 4 files changed, 117 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index 901405d91b..aefd7ea121 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvement\sto\sthe\sway\sthe\sJSON\sperformance\smeasure\sscripts\swork\s→\skeep\sthe\ntest\sdatabase\sin\sthe\sdirectory\sof\sthe\stest,\snot\sin\sthe\ssource\stree. -D 2023-04-26T15:58:08.228 +C Partial\simplementation\sof\sJSON5\snumeric\sliteral\sextensions.\s\sUse\sa\sswitch()\nstatement\sin\sthe\sparser\sfor\sbetter\sperformance. +D 2023-04-26T17:30:28.870 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c d0f9a0c630bdea3721e28b5ba2fce119c0e732ad48f6674a8f1644f2ec45ffb1 +F src/json.c a346dbe63628f8cb9c6bf6bbccb8fc81a49730db2311ec9b483d7fc3582da223 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -1254,7 +1254,7 @@ F test/json/json-generator.tcl 229bd293f1865f787c160886cadd282631721925cca2947aa F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x F test/json101.test de9c93169b84ac96fd5836c638a2ae1f00e4afbd4003c6b596692d7f05e1cd69 -F test/json102.test 327e77275f338c028faefa2da5164daf6b142a165e3015ff2a6e4251ddc6a0ac +F test/json102.test 1f61f469d763ff26430dbee76bc75e0aa73084ca84f10e58744fdb899e56de3e F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d262c059455ebe0650a45a6c1c04d1baf9609c635df352732dd192426e1bdc39 -R 3eb29750520f57f10b6b50d11e64a15f +P ac411dbdcbf0b9040fb5b7de173271f383a6aa303d57f22ebd200809a5b4a6d3 +R 0e63126100fc0a5fcfa419eb4dbeae91 U drh -Z f254fb62ebf6c065b09ee32328c8e984 +Z 7c56c83a7327e89fcacb5797373de3a9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 288ecccb2f..76ab21e90f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac411dbdcbf0b9040fb5b7de173271f383a6aa303d57f22ebd200809a5b4a6d3 \ No newline at end of file +78404dc37024cad5fe7eacf78ea85b56f08b129a1b9a046c3e1b11275068a485 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 58f89b1aaa..63feff9488 100644 --- a/src/json.c +++ b/src/json.c @@ -912,10 +912,9 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ int x; JsonNode *pNode; const char *z = pParse->zJson; - while( fast_isspace(z[i]) ){ i++; } json_parse_restart: - c = z[i]; - if( c=='{' ){ + switch( (u8)z[i] ){ + case '{': { /* Parse object */ iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); if( iThis<0 ) return -1; @@ -951,7 +950,8 @@ json_parse_restart: } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; return j+1; - }else if( c=='[' ){ + } + case '[': { /* Parse array */ iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); if( iThis<0 ) return -1; @@ -977,7 +977,8 @@ json_parse_restart: } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; return j+1; - }else if( c=='"' ){ + } + case '"': { /* Parse string */ u8 jnFlags = 0; j = i+1; @@ -1004,32 +1005,61 @@ json_parse_restart: jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; return j+1; - }else if( c=='n' - && strncmp(z+i,"null",4)==0 - && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return i+4; - }else if( c=='t' - && strncmp(z+i,"true",4)==0 - && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - return i+4; - }else if( c=='f' - && strncmp(z+i,"false",5)==0 - && !sqlite3Isalnum(z[i+5]) ){ - jsonParseAddNode(pParse, JSON_FALSE, 0, 0); - return i+5; - }else if( c=='-' || (c>='0' && c<='9') ){ + } + case 'n': { + if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + } + return -1; + } + case 't': { + if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + } + return -1; + } + case 'f': { + if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + } + return -1; + } + case '+': + pParse->has5 = 1; + /* fall through */ + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { /* Parse number */ u8 seenDP = 0; u8 seenE = 0; assert( '-' < '0' ); + assert( '+' < '0' ); + assert( '.' < '0' ); + c = z[i]; + if( c<='0' ){ - j = c=='-' ? i+1 : i; - if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1; + if( c=='0' ){ + if( sqlite3Isdigit(z[i+1]) ){ + pParse->has5 = 1; + } + }else{ + if( !sqlite3Isdigit(z[i+1]) ) return -1; + if( z[i+1]=='0' && sqlite3Isdigit(z[i+2]) ) pParse->has5 = 1; + } } - j = i+1; - for(;; j++){ + for(j=i+1;; j++){ c = z[j]; if( c>='0' && c<='9' ) continue; if( c=='.' ){ @@ -1074,19 +1104,44 @@ json_parse_restart: jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, j - i, &z[i]); return j; - }else if( c=='}' ){ + } + case '}': { return -2; /* End of {...} */ - }else if( c==']' ){ + } + case ']': { return -3; /* End of [...] */ - }else if( c==0 ){ + } + case 0: { return 0; /* End of file */ - }else if( (j = json5Whitespace(&z[i]))>0 ){ - i += j; - pParse->has5 = 1; + } + case 0x09: + case 0x0a: + case 0x0d: + case 0x20: { + do{ + i++; + }while( fast_isspace(z[i]) ); goto json_parse_restart; - }else{ + } + case 0x0b: + case 0x0c: + case '/': + case 0xc2: + case 0xe1: + case 0xe2: + case 0xe3: { + j = json5Whitespace(&z[i]); + if( j>0 ){ + i += j; + pParse->has5 = 1; + goto json_parse_restart; + } + return -1; + } + default: { #ifdef SQLITE_ENABLE_JSON_NAN_INF int k, nn; + c = z[i]; for(k=0; k Date: Wed, 26 Apr 2023 18:23:47 +0000 Subject: [PATCH 279/341] When changing a large integer into a floating point value, cancel the string representation. Fix for [forum:/forumpost/5c74a3bc4a|forum post 5c74a3bc4a]. FossilOrigin-Name: 3e2da8a7e35c839128d26aac575605e1e34889e8ab3484440bdd65c4d752c6bb --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d3df300e36..8a7e6b0ec7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\ssessions\smodule\sto\sbe\sconfigured\sto\scapture\schanges\sfrom\stables\swith\sno\sexplicit\sPRIMARY\sKEY. -D 2023-04-25T20:41:19.652 +C When\schanging\sa\slarge\sinteger\sinto\sa\sfloating\spoint\svalue,\scancel\sthe\nstring\srepresentation.\s\sFix\sfor\n[forum:/forumpost/5c74a3bc4a|forum\spost\s5c74a3bc4a]. +D 2023-04-26T18:23:47.136 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -703,7 +703,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c b1d8d87c4c8c77e70f48c43f91444fd66d91532693573b70b837afd572010176 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b +F src/vdbe.c 1d3e3b10ce83d1470bc349e9fe842ddded303591bae4c49e3292975f3c4d9beb F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 @@ -2060,9 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P faa1575ce0a39a2335a77057e8769a4e67aedd6326b4627988d03f0cb21fc39a 8a612f0860126c0c8473b1e65fcabb9a8821d8bf09fdf3f6018acfc99df9af71 -R ad733d62a9c199f3e70fa032fba3e6af -T +closed 8a612f0860126c0c8473b1e65fcabb9a8821d8bf09fdf3f6018acfc99df9af71 -U dan -Z 8fa09f18d44011739b0f388938c890f1 +P 5d4431bc44e206f48f3bcc1f51399b38bd4cb418e7e50a37495475fb8da04646 +R 890da1742f527a634e898a7eb7549799 +U drh +Z a31dc7374b94808e2db1fdf0feced2a2 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 67fdd8c5ed..b14d7af345 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d4431bc44e206f48f3bcc1f51399b38bd4cb418e7e50a37495475fb8da04646 \ No newline at end of file +3e2da8a7e35c839128d26aac575605e1e34889e8ab3484440bdd65c4d752c6bb \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index ba726a7954..738596daea 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3240,7 +3240,7 @@ case OP_Affinity: { }else{ pIn1->u.r = (double)pIn1->u.i; pIn1->flags |= MEM_Real; - pIn1->flags &= ~MEM_Int; + pIn1->flags &= ~(MEM_Int|MEM_Str); } } REGISTER_TRACE((int)(pIn1-aMem), pIn1); From 6a9ba9a8c2fae4c999fd152fb510537b93345df6 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 26 Apr 2023 18:50:49 +0000 Subject: [PATCH 280/341] Fix a typo in a printf() format string the sessions module. FossilOrigin-Name: 14d1c4a828660bc137dcea1bb4cc439e1ed3872867a0dab486f6a639ef3bcc45 --- ext/session/sqlite3session.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index e601a4ae7a..89cffac199 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -1765,7 +1765,7 @@ static char *sessionAllCols( int ii; char *zRet = 0; for(ii=0; iinCol; ii++){ - zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"%s", + zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"", zRet, (zRet ? ", " : ""), zDb, pTab->zName, pTab->azCol[ii] ); if( !zRet ) break; diff --git a/manifest b/manifest index 8a7e6b0ec7..c941ae48fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\schanging\sa\slarge\sinteger\sinto\sa\sfloating\spoint\svalue,\scancel\sthe\nstring\srepresentation.\s\sFix\sfor\n[forum:/forumpost/5c74a3bc4a|forum\spost\s5c74a3bc4a]. -D 2023-04-26T18:23:47.136 +C Fix\sa\stypo\sin\sa\sprintf()\sformat\sstring\sthe\ssessions\smodule. +D 2023-04-26T18:50:49.563 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -470,7 +470,7 @@ F ext/session/sessionrowid.test 6323ba831721205fd729929745038fd54e9d128c66c654b8 F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c a0d7d71f9b111e443b53afc49b6cb5ccf48734af216c91dbe6640b7e4cd8c740 +F ext/session/sqlite3session.c c30768077832f04f6ae98ab5a5347eae78b4f840bc958e16ae8aa970261a315e F ext/session/sqlite3session.h 24299a3b64f11afc4422ce92d030ffdb2d3181851a1763b4a0432e195b2a8a16 F ext/session/test_session.c 5285482f83cd92b4c1fe12fcf88210566a18312f4f2aa110f6399dae46aeccbb F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d4431bc44e206f48f3bcc1f51399b38bd4cb418e7e50a37495475fb8da04646 -R 890da1742f527a634e898a7eb7549799 -U drh -Z a31dc7374b94808e2db1fdf0feced2a2 +P 3e2da8a7e35c839128d26aac575605e1e34889e8ab3484440bdd65c4d752c6bb +R 4e23a1ec1e088b34c19a8052565eac29 +U dan +Z 09bae246348989754f2af6790026c07b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b14d7af345..6ee1d6b0d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e2da8a7e35c839128d26aac575605e1e34889e8ab3484440bdd65c4d752c6bb \ No newline at end of file +14d1c4a828660bc137dcea1bb4cc439e1ed3872867a0dab486f6a639ef3bcc45 \ No newline at end of file From c5ee2d8e8076b03a1d5d50f89c62a9d1ddbb2a5f Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Apr 2023 20:26:14 +0000 Subject: [PATCH 281/341] Implement some of the JSON5 enhancements to string and numeric literals. This is an incremental check-in of work in progress. FossilOrigin-Name: 9508efa9d61c0ff0eb73100dd52889dadc5fa2a2091b944a9b8a74e8a5c50e82 --- manifest | 12 ++-- manifest.uuid | 2 +- src/json.c | 160 +++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 147 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index aefd7ea121..4c3f3616dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Partial\simplementation\sof\sJSON5\snumeric\sliteral\sextensions.\s\sUse\sa\sswitch()\nstatement\sin\sthe\sparser\sfor\sbetter\sperformance. -D 2023-04-26T17:30:28.870 +C Implement\ssome\sof\sthe\sJSON5\senhancements\sto\sstring\sand\snumeric\sliterals.\nThis\sis\san\sincremental\scheck-in\sof\swork\sin\sprogress. +D 2023-04-26T20:26:14.388 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c a346dbe63628f8cb9c6bf6bbccb8fc81a49730db2311ec9b483d7fc3582da223 +F src/json.c f82b86599b220a2c60731eba883bbea92ee4d6e61aaa56740179e8aa5e6ec596 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac411dbdcbf0b9040fb5b7de173271f383a6aa303d57f22ebd200809a5b4a6d3 -R 0e63126100fc0a5fcfa419eb4dbeae91 +P 78404dc37024cad5fe7eacf78ea85b56f08b129a1b9a046c3e1b11275068a485 +R e7aefca3f1ce2ae26f999bdfb414a2b2 U drh -Z 7c56c83a7327e89fcacb5797373de3a9 +Z b316d61c8d04f67716fa9ffd06cd770b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 76ab21e90f..2e71d5072e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -78404dc37024cad5fe7eacf78ea85b56f08b129a1b9a046c3e1b11275068a485 \ No newline at end of file +9508efa9d61c0ff0eb73100dd52889dadc5fa2a2091b944a9b8a74e8a5c50e82 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 63feff9488..3e2d6646d8 100644 --- a/src/json.c +++ b/src/json.c @@ -104,6 +104,7 @@ static const char * const jsonType[] = { #define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ #define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ #define JNODE_LABEL 0x40 /* Is a label of an object */ +#define JNODE_JSON5 0x80 /* Node contains JSON5 enhancements */ /* A single node of parsed JSON @@ -295,6 +296,72 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ assert( p->nUsednAlloc ); } +/* +** The zIn[0..N] string is a JSON5 string literal. Append to p a translation +** of the string literal that standard JSON and that omits all JSON5 +** features. +*/ +static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ + jsonAppendChar(p, '"'); + jsonAppendRaw(p, &zIn[1], N-2); /* TODO: translate JSON5 escapes */ + jsonAppendChar(p, '"'); +} + +/* +** The zIn[0..N] string is a JSON5 integer literal. Append to p a translation +** of the string literal that standard JSON and that omits all JSON5 +** features. +*/ +static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ + if( zIn[0]=='+' ){ + zIn++; + N--; + }else if( zIn[0]=='-' ){ + jsonAppendChar(p, '-'); + zIn++; + N--; + } + while( zIn[0]=='0' && N>1 ){ + zIn++; + N--; + } + jsonAppendRaw(p, zIn, N); +} + +/* +** The zIn[0..N] string is a JSON5 real literal. Append to p a translation +** of the string literal that standard JSON and that omits all JSON5 +** features. +*/ +static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ + int i; + if( zIn[0]=='+' ){ + zIn++; + N--; + }else if( zIn[0]=='-' ){ + jsonAppendChar(p, '-'); + zIn++; + N--; + } + if( zIn[0]=='.' ){ + jsonAppendChar(p, '0'); + } + for(i=0; i0 ){ + jsonAppendRaw(p, zIn, N); + } +} + + + /* ** Append a function parameter value to the JSON string under ** construction. @@ -425,17 +492,32 @@ static void jsonRenderNode( break; } case JSON_STRING: { + assert( pNode->eU==1 ); if( pNode->jnFlags & JNODE_RAW ){ - assert( pNode->eU==1 ); jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - break; + }else if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); + }else{ + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); } - /* no break */ deliberate_fall_through + break; + } + case JSON_REAL: { + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); + }else{ + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + } + break; } - case JSON_REAL: case JSON_INT: { assert( pNode->eU==1 ); - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); + }else{ + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + } break; } case JSON_ARRAY: { @@ -615,9 +697,10 @@ static void jsonReturn( const char *z; char *zOut; u32 j; + u32 nOut = n; assert( pNode->eU==1 ); z = pNode->u.zJContent; - zOut = sqlite3_malloc( n+1 ); + zOut = sqlite3_malloc( nOut+1 ); if( zOut==0 ){ sqlite3_result_error_nomem(pCtx); break; @@ -669,6 +752,15 @@ static void jsonReturn( c = '\r'; }else if( c=='t' ){ c = '\t'; + }else if( c=='v' ){ + c = '\v'; + }else if( c=='\'' ){ + c = '\''; + }else if( c=='0' ){ + c = 0; + }else if( c=='x' ){ + c = (jsonHexToInt(z[i+1])<<4) | jsonHexToInt(z[i+2]); + i += 2; } zOut[j++] = c; } @@ -749,13 +841,18 @@ static int jsonParseAddNode( return pParse->nNode++; } +/* +** Return true if z[] begins with 2 (or more) hexadecimal digits +*/ +static int jsonIs2Hex(const char *z){ + return sqlite3Isxdigit(z[0]) && sqlite3Isxdigit(z[1]); +} + /* ** Return true if z[] begins with 4 (or more) hexadecimal digits */ static int jsonIs4Hex(const char *z){ - int i; - for(i=0; i<4; i++) if( !sqlite3Isxdigit(z[i]) ) return 0; - return 1; + return jsonIs2Hex(z) && jsonIs2Hex(&z[2]); } /* @@ -978,9 +1075,17 @@ json_parse_restart: pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; return j+1; } - case '"': { + case '\'': { + u8 jnFlags; + char cDelim; + pParse->has5 = 1; + jnFlags = JNODE_JSON5; + goto parse_string; + case '"': /* Parse string */ - u8 jnFlags = 0; + jnFlags = 0; + parse_string: + cDelim = z[i]; j = i+1; for(;;){ c = z[j]; @@ -992,12 +1097,16 @@ json_parse_restart: c = z[++j]; if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' || c=='n' || c=='r' || c=='t' - || (c=='u' && jsonIs4Hex(z+j+1)) ){ - jnFlags = JNODE_ESCAPE; + || (c=='u' && jsonIs4Hex(&z[j+1])) ){ + jnFlags |= JNODE_ESCAPE; + }else if( c=='\'' || c=='0' || c=='v' + || (c=='x' && jsonIs2Hex(&z[j+1])) ){ + jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); + pParse->has5 = 1; }else{ return -1; } - }else if( c=='"' ){ + }else if( c==cDelim ){ break; } j++; @@ -1027,9 +1136,11 @@ json_parse_restart: } return -1; } - case '+': + case '+': { + u8 seenDP, seenE, jnFlags; pParse->has5 = 1; - /* fall through */ + jnFlags = JNODE_JSON5; + goto parse_number; case '-': case '0': case '1': @@ -1040,10 +1151,12 @@ json_parse_restart: case '6': case '7': case '8': - case '9': { + case '9': /* Parse number */ - u8 seenDP = 0; - u8 seenE = 0; + jnFlags = 0; + parse_number: + seenDP = 0; + seenE = 0; assert( '-' < '0' ); assert( '+' < '0' ); assert( '.' < '0' ); @@ -1053,10 +1166,14 @@ json_parse_restart: if( c=='0' ){ if( sqlite3Isdigit(z[i+1]) ){ pParse->has5 = 1; + jnFlags = JNODE_JSON5; } }else{ if( !sqlite3Isdigit(z[i+1]) ) return -1; - if( z[i+1]=='0' && sqlite3Isdigit(z[i+2]) ) pParse->has5 = 1; + if( z[i+1]=='0' && sqlite3Isdigit(z[i+2]) ){ + pParse->has5 = 1; + jnFlags = JNODE_JSON5; + } } } for(j=i+1;; j++){ @@ -1103,6 +1220,9 @@ json_parse_restart: if( z[j-1]<'0' ) return -1; jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, j - i, &z[i]); + if( jnFlags && !pParse->oom ){ + pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + } return j; } case '}': { From f59c01e24cb3146f5e5ec6ccb73731a8a365b6c5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 12:24:00 +0000 Subject: [PATCH 282/341] Translate JSON5-only string literal escape sequences into the JSON equivalents. FossilOrigin-Name: 14e82f36eed31af1237898728bf353b968523c62b1f8d1d90dbbabd92d0c2834 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4c3f3616dd..e797d70238 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\ssome\sof\sthe\sJSON5\senhancements\sto\sstring\sand\snumeric\sliterals.\nThis\sis\san\sincremental\scheck-in\sof\swork\sin\sprogress. -D 2023-04-26T20:26:14.388 +C Translate\sJSON5-only\sstring\sliteral\sescape\ssequences\sinto\sthe\sJSON\sequivalents. +D 2023-04-27T12:24:00.195 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c f82b86599b220a2c60731eba883bbea92ee4d6e61aaa56740179e8aa5e6ec596 +F src/json.c e872bb2ba63296c21dd62f83557d504bfb94f299a6591ec735cd4a9aed4e321b F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 78404dc37024cad5fe7eacf78ea85b56f08b129a1b9a046c3e1b11275068a485 -R e7aefca3f1ce2ae26f999bdfb414a2b2 +P 9508efa9d61c0ff0eb73100dd52889dadc5fa2a2091b944a9b8a74e8a5c50e82 +R 672753aa30f1f89635383bf8b5e62624 U drh -Z b316d61c8d04f67716fa9ffd06cd770b +Z aa11f958e9e420359b5624437ca5455e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2e71d5072e..4badfeac1a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9508efa9d61c0ff0eb73100dd52889dadc5fa2a2091b944a9b8a74e8a5c50e82 \ No newline at end of file +14e82f36eed31af1237898728bf353b968523c62b1f8d1d90dbbabd92d0c2834 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3e2d6646d8..0cbcac3c4e 100644 --- a/src/json.c +++ b/src/json.c @@ -302,8 +302,42 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ ** features. */ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ + int i; jsonAppendChar(p, '"'); - jsonAppendRaw(p, &zIn[1], N-2); /* TODO: translate JSON5 escapes */ + zIn++; + N -= 2; + while( N>0 ){ + for(i=0; i0 ){ + jsonAppendRaw(p, zIn, i); + zIn += i; + N -= i; + if( N==0 ) break; + } + assert( zIn[0]=='\\' ); + switch( zIn[1] ){ + case '\'': + jsonAppendChar(p, '\''); + break; + case 'v': + jsonAppendRaw(p, "\\u0009", 6); + break; + case 'x': + jsonAppendRaw(p, "\\u00", 4); + jsonAppendRaw(p, &zIn[2], 2); + zIn += 2; + N -= 2; + break; + case '0': + jsonAppendRaw(p, "\\u0000", 6); + break; + default: + jsonAppendRaw(p, zIn, 2); + break; + } + zIn += 2; + N -= 2; + } jsonAppendChar(p, '"'); } From a549e7a527902556f8bd55e506674f8a3a983952 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 13:44:26 +0000 Subject: [PATCH 283/341] Permit JSON5 whitespace in all contexts of objects and arrays. FossilOrigin-Name: 93f3ab26b57c0469862a56e97d4b3c796b27f9f582046fcff1f2aa8d8910c550 --- manifest | 12 +++---- manifest.uuid | 2 +- src/json.c | 88 +++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 74 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index e797d70238..a76a4753ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Translate\sJSON5-only\sstring\sliteral\sescape\ssequences\sinto\sthe\sJSON\sequivalents. -D 2023-04-27T12:24:00.195 +C Permit\sJSON5\swhitespace\sin\sall\scontexts\sof\sobjects\sand\sarrays. +D 2023-04-27T13:44:26.660 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c e872bb2ba63296c21dd62f83557d504bfb94f299a6591ec735cd4a9aed4e321b +F src/json.c 28ca7f26c211e0fd705d06d9036de59304f9a3dd5e69e3745a688aafd8c618ae F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9508efa9d61c0ff0eb73100dd52889dadc5fa2a2091b944a9b8a74e8a5c50e82 -R 672753aa30f1f89635383bf8b5e62624 +P 14e82f36eed31af1237898728bf353b968523c62b1f8d1d90dbbabd92d0c2834 +R 37b7a9f174ccda66654bba8268903617 U drh -Z aa11f958e9e420359b5624437ca5455e +Z 18c492aceae37854acd7e6823a29d2b5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4badfeac1a..2f3e3c7fd1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14e82f36eed31af1237898728bf353b968523c62b1f8d1d90dbbabd92d0c2834 \ No newline at end of file +93f3ab26b57c0469862a56e97d4b3c796b27f9f582046fcff1f2aa8d8910c550 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 0cbcac3c4e..dbc1c6b833 100644 --- a/src/json.c +++ b/src/json.c @@ -136,6 +136,7 @@ struct JsonParse { u8 oom; /* Set to true if out of memory */ u8 has5; /* True if input has JSON5 features */ int nJson; /* Length of the zJson string in bytes */ + u32 iErr; /* Error location in zJson[] */ u32 iHold; /* Replace cache line with the lowest iHold value */ }; @@ -945,7 +946,7 @@ static int json5Whitespace(const char *zIn){ for(j=n+3; z[j]!='/' || z[j-1]!='*'; j++){ if( z[j]==0 ) goto whitespace_done; } - n += j; + n += j+1; break; }else if( z[n+1]=='/' ){ int j; @@ -1032,9 +1033,14 @@ static const struct NanInfName { ** Parse a single JSON value which begins at pParse->zJson[i]. Return the ** index of the first character past the end of the value parsed. ** -** Return negative for a syntax error. Special cases: return -2 if the -** first non-whitespace character is '}' and return -3 if the first -** non-whitespace character is ']'. +** Special return values: +** +** 0 End if input +** -1 Syntax error +** -2 '}' seen +** -3 ']' seen +** -4 ',' seen +** -5 ':' seen */ static int jsonParseValue(JsonParse *pParse, u32 i){ char c; @@ -1050,12 +1056,12 @@ json_parse_restart: iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); if( iThis<0 ) return -1; for(j=i+1;;j++){ - while( fast_isspace(z[j]) ){ j++; } if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; x = jsonParseValue(pParse, j); - if( x<0 ){ + if( x<=0 ){ pParse->iDepth--; if( x==(-2) ){ + j = pParse->iErr; if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; break; } @@ -1066,18 +1072,33 @@ json_parse_restart: if( pNode->eType!=JSON_STRING ) return -1; pNode->jnFlags |= JNODE_LABEL; j = x; - while( fast_isspace(z[j]) ){ j++; } - if( z[j]!=':' ) return -1; - j++; + if( z[j]==':' ){ + j++; + }else{ + x = jsonParseValue(pParse, j); + if( x!=(-5) ) return -1; + j = pParse->iErr+1; + } x = jsonParseValue(pParse, j); pParse->iDepth--; if( x<0 ) return -1; j = x; - while( fast_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!='}' ) return -1; - break; + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + }else{ + x = jsonParseValue(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-2) ){ + j = pParse->iErr; + break; + } + } + return -1; } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; return j+1; @@ -1088,23 +1109,34 @@ json_parse_restart: if( iThis<0 ) return -1; memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); for(j=i+1;;j++){ - while( fast_isspace(z[j]) ){ j++; } if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; x = jsonParseValue(pParse, j); pParse->iDepth--; if( x<0 ){ if( x==(-3) ){ + j = pParse->iErr; if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; break; } return -1; } j = x; - while( fast_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!=']' ) return -1; - break; + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + }else{ + x = jsonParseValue(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-3) ){ + j = pParse->iErr; + break; + } + } + return -1; } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; return j+1; @@ -1260,11 +1292,21 @@ json_parse_restart: return j; } case '}': { + pParse->iErr = i; return -2; /* End of {...} */ } case ']': { + pParse->iErr = i; return -3; /* End of [...] */ } + case ',': { + pParse->iErr = i; + return -4; /* List separator */ + } + case ':': { + pParse->iErr = i; + return -5; /* Object label/value separator */ + } case 0: { return 0; /* End of file */ } @@ -1334,7 +1376,11 @@ static int jsonParse( if( i>0 ){ assert( pParse->iDepth==0 ); while( fast_isspace(zJson[i]) ) i++; - if( zJson[i] ) i = -1; + if( zJson[i] ){ + i += json5Whitespace(&zJson[i]); + if( zJson[i] ) return -1; + pParse->has5 = 1; + } } if( i<=0 ){ if( pCtx!=0 ){ From 2e225a35b4242c36626a88973ee31704992b234e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Apr 2023 13:55:29 +0000 Subject: [PATCH 284/341] Fix a problem on windows sometimes causing SQLITE_NOMEM to be returned if an error occurred while opening a temp file. FossilOrigin-Name: c22a39a6b28e3039fe453c0c0db98744b8a63ef9063cc3f6e0ed67d2506d34be --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 11 ++++++++--- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c941ae48fb..38d2562688 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\sprintf()\sformat\sstring\sthe\ssessions\smodule. -D 2023-04-26T18:50:49.563 +C Fix\sa\sproblem\son\swindows\ssometimes\scausing\sSQLITE_NOMEM\sto\sbe\sreturned\sif\san\serror\soccurred\swhile\sopening\sa\stemp\sfile. +D 2023-04-27T13:55:29.757 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -618,7 +618,7 @@ F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 F src/os_unix.c 1b3ddb7814c4bf37f494c04d2ab30c1ced5b2c927267e1930ce7cd388787a96d -F src/os_win.c 295fe45f18bd86f2477f4cd79f3377c6f883ceb941b1f46808665c73747f2345 +F src/os_win.c 20c2f020b54fe6ae210c1f200efc097ac50e7e7b24f64c632c1b751ac1f96899 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 9d36ddedc842e993c88c222ed914822dbd6f8ece3c648fde04468637012a034a F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3e2da8a7e35c839128d26aac575605e1e34889e8ab3484440bdd65c4d752c6bb -R 4e23a1ec1e088b34c19a8052565eac29 +P 14d1c4a828660bc137dcea1bb4cc439e1ed3872867a0dab486f6a639ef3bcc45 +R 1e3870610983faacb84c1d55e8f701dd U dan -Z 09bae246348989754f2af6790026c07b +Z ef68dfd966e00e89b716183914f3d8f9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6ee1d6b0d8..ff4ed9d11e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14d1c4a828660bc137dcea1bb4cc439e1ed3872867a0dab486f6a639ef3bcc45 \ No newline at end of file +c22a39a6b28e3039fe453c0c0db98744b8a63ef9063cc3f6e0ed67d2506d34be \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index abecf1b936..33275113ad 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -5197,7 +5197,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -5214,7 +5214,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -5234,7 +5234,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -5453,6 +5453,11 @@ static int winAccess( void *zConverted; UNUSED_PARAMETER(pVfs); + if( zFilename==0 ){ + *pResOut = 0; + return SQLITE_OK; + } + SimulateIOError( return SQLITE_IOERR_ACCESS; ); OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", zFilename, flags, pResOut)); From 0d8579120a2d46b75d1564e7d928690fb33ef572 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 14:38:51 +0000 Subject: [PATCH 285/341] Performance optimization in the JSON parser. FossilOrigin-Name: 5a88ba743f55d45b1c0ce0090bb3b396bcf7fcf7b3bcb989aaf30b8bb772599e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 33 ++++++++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a76a4753ef..25d0e8f823 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Permit\sJSON5\swhitespace\sin\sall\scontexts\sof\sobjects\sand\sarrays. -D 2023-04-27T13:44:26.660 +C Performance\soptimization\sin\sthe\sJSON\sparser. +D 2023-04-27T14:38:51.628 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 28ca7f26c211e0fd705d06d9036de59304f9a3dd5e69e3745a688aafd8c618ae +F src/json.c d65da6a334ff2f18cea8fa618bf64e085cf3517bdb9a5e570cb1e8531040d976 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14e82f36eed31af1237898728bf353b968523c62b1f8d1d90dbbabd92d0c2834 -R 37b7a9f174ccda66654bba8268903617 +P 93f3ab26b57c0469862a56e97d4b3c796b27f9f582046fcff1f2aa8d8910c550 +R 66ad085cb2272ce979fd2644e3aac860 U drh -Z 18c492aceae37854acd7e6823a29d2b5 +Z 7c28704af9aa014ef0ec768389ba7618 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2f3e3c7fd1..6d73e72cdd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -93f3ab26b57c0469862a56e97d4b3c796b27f9f582046fcff1f2aa8d8910c550 \ No newline at end of file +5a88ba743f55d45b1c0ce0090bb3b396bcf7fcf7b3bcb989aaf30b8bb772599e \ No newline at end of file diff --git a/src/json.c b/src/json.c index dbc1c6b833..3828ca6bc3 100644 --- a/src/json.c +++ b/src/json.c @@ -1075,19 +1075,35 @@ json_parse_restart: if( z[j]==':' ){ j++; }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==':' ){ + j++; + goto parse_object_value; + } + } x = jsonParseValue(pParse, j); if( x!=(-5) ) return -1; j = pParse->iErr+1; } + parse_object_value: x = jsonParseValue(pParse, j); pParse->iDepth--; - if( x<0 ) return -1; + if( x<=0 ) return -1; j = x; if( z[j]==',' ){ continue; }else if( z[j]=='}' ){ break; }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + } + } x = jsonParseValue(pParse, j); if( x==(-4) ){ j = pParse->iErr; @@ -1112,7 +1128,7 @@ json_parse_restart: if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; x = jsonParseValue(pParse, j); pParse->iDepth--; - if( x<0 ){ + if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; @@ -1126,6 +1142,14 @@ json_parse_restart: }else if( z[j]==']' ){ break; }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + } + } x = jsonParseValue(pParse, j); if( x==(-4) ){ j = pParse->iErr; @@ -1378,7 +1402,10 @@ static int jsonParse( while( fast_isspace(zJson[i]) ) i++; if( zJson[i] ){ i += json5Whitespace(&zJson[i]); - if( zJson[i] ) return -1; + if( zJson[i] ){ + jsonParseReset(pParse); + return 1; + } pParse->has5 = 1; } } From ef3a162be6508eae2d09315f6e82d30543367e6d Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 15:11:26 +0000 Subject: [PATCH 286/341] Allow the labels on JSON objects to be unquoted identifier names. FossilOrigin-Name: fb428db3f64f148ab9a3478fdcc8b3733e58102b3c8895a482e2551d974d5661 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 16 ++++++++++++++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 25d0e8f823..ff21e8d8d7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\sthe\sJSON\sparser. -D 2023-04-27T14:38:51.628 +C Allow\sthe\slabels\son\sJSON\sobjects\sto\sbe\sunquoted\sidentifier\snames. +D 2023-04-27T15:11:26.153 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c d65da6a334ff2f18cea8fa618bf64e085cf3517bdb9a5e570cb1e8531040d976 +F src/json.c 22e67e7ae7d1bd46df5616452a7b14587799df6dfff0c47bd09c3926c89ead99 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 93f3ab26b57c0469862a56e97d4b3c796b27f9f582046fcff1f2aa8d8910c550 -R 66ad085cb2272ce979fd2644e3aac860 +P 5a88ba743f55d45b1c0ce0090bb3b396bcf7fcf7b3bcb989aaf30b8bb772599e +R 7efb5a3476c2275fadcb3d73f88a7a54 U drh -Z 7c28704af9aa014ef0ec768389ba7618 +Z 22784998a1ee89abb38fb8b4e89c3f5d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6d73e72cdd..ec0911a184 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a88ba743f55d45b1c0ce0090bb3b396bcf7fcf7b3bcb989aaf30b8bb772599e \ No newline at end of file +fb428db3f64f148ab9a3478fdcc8b3733e58102b3c8895a482e2551d974d5661 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 3828ca6bc3..178267bdd1 100644 --- a/src/json.c +++ b/src/json.c @@ -1059,13 +1059,25 @@ json_parse_restart: if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; x = jsonParseValue(pParse, j); if( x<=0 ){ - pParse->iDepth--; if( x==(-2) ){ j = pParse->iErr; if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; + pParse->iDepth--; break; } - return -1; + j += json5Whitespace(&z[j]); + if( sqlite3Isalpha(z[j]) || z[j]=='_' || z[j]=='$' ){ + int k; + for(k=j+1; sqlite3Isalnum(z[k]) || z[k]=='_' || z[k]=='$'; k++){} + jsonParseAddNode(pParse, JSON_STRING, k-j, &z[j]); + if( !pParse->oom ){ + pParse->aNode[pParse->nNode-1].jnFlags = JNODE_RAW; + } + pParse->has5 = 1; + x = k; + }else{ + return -1; + } } if( pParse->oom ) return -1; pNode = &pParse->aNode[pParse->nNode-1]; From 13f8a80d944a26a35d4cb9dbbfda50eff963502d Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 15:48:58 +0000 Subject: [PATCH 287/341] JSON string literals may span multiple lines by escaping new line characters. FossilOrigin-Name: 66da4bd4a30c390fa1a7960ce2edaef82e63971ecf33ffb6b906db9f278041c5 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/json.c | 81 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 61 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index ff21e8d8d7..fff4f23d06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\slabels\son\sJSON\sobjects\sto\sbe\sunquoted\sidentifier\snames. -D 2023-04-27T15:11:26.153 +C JSON\sstring\sliterals\smay\sspan\smultiple\slines\sby\sescaping\snew\sline\scharacters. +D 2023-04-27T15:48:58.281 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 22e67e7ae7d1bd46df5616452a7b14587799df6dfff0c47bd09c3926c89ead99 +F src/json.c d0a088f85237afcf78aa9607261a4912ce1a42db3fca907305f78da8be0c51d9 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5a88ba743f55d45b1c0ce0090bb3b396bcf7fcf7b3bcb989aaf30b8bb772599e -R 7efb5a3476c2275fadcb3d73f88a7a54 +P fb428db3f64f148ab9a3478fdcc8b3733e58102b3c8895a482e2551d974d5661 +R 187d7b6dd1b32f74babea313dab398fc U drh -Z 22784998a1ee89abb38fb8b4e89c3f5d +Z 182d7f6abcc455723bd9a825ca2f2b58 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ec0911a184..6a4b1b10bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb428db3f64f148ab9a3478fdcc8b3733e58102b3c8895a482e2551d974d5661 \ No newline at end of file +66da4bd4a30c390fa1a7960ce2edaef82e63971ecf33ffb6b906db9f278041c5 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 178267bdd1..e446830db9 100644 --- a/src/json.c +++ b/src/json.c @@ -316,7 +316,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ if( N==0 ) break; } assert( zIn[0]=='\\' ); - switch( zIn[1] ){ + switch( (u8)zIn[1] ){ case '\'': jsonAppendChar(p, '\''); break; @@ -332,6 +332,21 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ case '0': jsonAppendRaw(p, "\\u0000", 6); break; + case '\r': + if( N>=3 && zIn[2]=='\n' ){ + zIn++; + N--; + } + break; + case '\n': + break; + case 0xe2: + assert( N>=4 ); + assert( 0x80==(u8)zIn[2] ); + assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); + zIn += 2; + N -= 2; + break; default: jsonAppendRaw(p, zIn, 2); break; @@ -742,9 +757,7 @@ static void jsonReturn( } for(i=1, j=0; ihas5 = 1; From dbcea23976955b2aa36ca9eb92b0fe5315ef7665 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 16:24:12 +0000 Subject: [PATCH 288/341] Fix the handling of escape solidus in the JSON routines. FossilOrigin-Name: 676877aca235e620ee12d10235dd6ad009d4968455ec170daeb1998b94a7e0a2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index fff4f23d06..40fa719367 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C JSON\sstring\sliterals\smay\sspan\smultiple\slines\sby\sescaping\snew\sline\scharacters. -D 2023-04-27T15:48:58.281 +C Fix\sthe\shandling\sof\sescape\ssolidus\sin\sthe\sJSON\sroutines. +D 2023-04-27T16:24:12.129 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c d0a088f85237afcf78aa9607261a4912ce1a42db3fca907305f78da8be0c51d9 +F src/json.c cdf05600af31e37ad50924f39d7fa6227a7c9b94ea3d3690312cc1b7e15b36ef F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb428db3f64f148ab9a3478fdcc8b3733e58102b3c8895a482e2551d974d5661 -R 187d7b6dd1b32f74babea313dab398fc +P 66da4bd4a30c390fa1a7960ce2edaef82e63971ecf33ffb6b906db9f278041c5 +R 9748a62c2b5ef29db66fe80859611ec7 U drh -Z 182d7f6abcc455723bd9a825ca2f2b58 +Z 8f82433213e0588347a902f584c3080a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6a4b1b10bd..9374e8e5aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -66da4bd4a30c390fa1a7960ce2edaef82e63971ecf33ffb6b906db9f278041c5 \ No newline at end of file +676877aca235e620ee12d10235dd6ad009d4968455ec170daeb1998b94a7e0a2 \ No newline at end of file diff --git a/src/json.c b/src/json.c index e446830db9..e508209fc9 100644 --- a/src/json.c +++ b/src/json.c @@ -802,10 +802,8 @@ static void jsonReturn( c = '\t'; }else if( c=='v' ){ c = '\v'; - }else if( c=='\'' ){ - c = '\''; - }else if( c=='"' ){ - c = '"'; + }else if( c=='\'' || c=='"' || c=='/' ){ + /* pass through unchanged */ }else if( c=='0' ){ c = 0; }else if( c=='x' ){ From 8ded1642009400732782d4030d8d4359064531d0 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 16:57:14 +0000 Subject: [PATCH 289/341] Fix handling of reverse solidus in string literals. Allow decimal points in floating point literals to occurs and the beginning or end of the mantissa. FossilOrigin-Name: d92a6ab2871095ac66c60cfa15dbafa7b762f83d287d452f61792eb30cf5b26b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 31 ++++++++++++++++++++++++++++--- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 40fa719367..8880fe850e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\shandling\sof\sescape\ssolidus\sin\sthe\sJSON\sroutines. -D 2023-04-27T16:24:12.129 +C Fix\shandling\sof\sreverse\ssolidus\sin\sstring\sliterals.\s\sAllow\sdecimal\spoints\nin\sfloating\spoint\sliterals\sto\soccurs\sand\sthe\sbeginning\sor\send\sof\sthe\smantissa. +D 2023-04-27T16:57:14.640 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c cdf05600af31e37ad50924f39d7fa6227a7c9b94ea3d3690312cc1b7e15b36ef +F src/json.c d1a73af4f4185cfc355fa4b7be0cdc5cddd80b06da3d8ad3f966b8758e9027a1 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 66da4bd4a30c390fa1a7960ce2edaef82e63971ecf33ffb6b906db9f278041c5 -R 9748a62c2b5ef29db66fe80859611ec7 +P 676877aca235e620ee12d10235dd6ad009d4968455ec170daeb1998b94a7e0a2 +R 55fe1e7f7cae4da350b14dce9970e986 U drh -Z 8f82433213e0588347a902f584c3080a +Z eab22ac4584a027288ad76a2849798e8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9374e8e5aa..80675cd170 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -676877aca235e620ee12d10235dd6ad009d4968455ec170daeb1998b94a7e0a2 \ No newline at end of file +d92a6ab2871095ac66c60cfa15dbafa7b762f83d287d452f61792eb30cf5b26b \ No newline at end of file diff --git a/src/json.c b/src/json.c index e508209fc9..4ef53cd8ae 100644 --- a/src/json.c +++ b/src/json.c @@ -398,6 +398,7 @@ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ } for(i=0; ihas5 = 1; jnFlags = JNODE_JSON5; goto parse_number; + case '.': + if( sqlite3Isdigit(z[i+1]) ){ + pParse->has5 = 1; + jnFlags = JNODE_JSON5; + seenE = 0; + seenDP = 1; + goto parse_number_2; + } + return -1; case '-': case '0': case '1': @@ -1303,6 +1313,7 @@ json_parse_restart: } } } + parse_number_2: for(j=i+1;; j++){ c = z[j]; if( c>='0' && c<='9' ) continue; @@ -1313,7 +1324,14 @@ json_parse_restart: continue; } if( c=='e' || c=='E' ){ - if( z[j-1]<'0' ) return -1; + if( z[j-1]<'0' ){ + if( z[j-1]=='.' && j-2>=i && sqlite3Isdigit(z[j-2]) ){ + pParse->has5 = 1; + jnFlags |= JNODE_JSON5; + }else{ + return -1; + } + } if( seenE ) return -1; seenDP = seenE = 1; c = z[j+1]; @@ -1344,7 +1362,14 @@ json_parse_restart: #endif break; } - if( z[j-1]<'0' ) return -1; + if( z[j-1]<'0' ){ + if( z[j-1]=='.' && j-2>=i && sqlite3Isdigit(z[j-2]) ){ + pParse->has5 = 1; + jnFlags |= JNODE_JSON5; + }else{ + return -1; + } + } jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, j - i, &z[i]); if( jnFlags && !pParse->oom ){ From 47dd0e735de7a9bf133fc5855d1e8c021beca351 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 17:32:29 +0000 Subject: [PATCH 290/341] All floating point literals "NaN" and "Infinity". Additional variants of these literals are available if compiled with SQLITE_EXTENDED_NAN_INF. FossilOrigin-Name: c13346afbecb92275e741252897d00478dab4be2d158889bc735e80efd9444f5 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/json.c | 70 +++++++++++++++++++++++++++++++++++---------------- 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 8880fe850e..e9ac4de4a3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\shandling\sof\sreverse\ssolidus\sin\sstring\sliterals.\s\sAllow\sdecimal\spoints\nin\sfloating\spoint\sliterals\sto\soccurs\sand\sthe\sbeginning\sor\send\sof\sthe\smantissa. -D 2023-04-27T16:57:14.640 +C All\sfloating\spoint\sliterals\s"NaN"\sand\s"Infinity".\s\sAdditional\svariants\nof\sthese\sliterals\sare\savailable\sif\scompiled\swith\sSQLITE_EXTENDED_NAN_INF. +D 2023-04-27T17:32:29.533 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c d1a73af4f4185cfc355fa4b7be0cdc5cddd80b06da3d8ad3f966b8758e9027a1 +F src/json.c 530e1f4f4899a3c368bfaac63028e4c15ebd279a697780374d0cb7817103350a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 676877aca235e620ee12d10235dd6ad009d4968455ec170daeb1998b94a7e0a2 -R 55fe1e7f7cae4da350b14dce9970e986 +P d92a6ab2871095ac66c60cfa15dbafa7b762f83d287d452f61792eb30cf5b26b +R a7e436b837797408dc673da04c49a8e7 U drh -Z eab22ac4584a027288ad76a2849798e8 +Z 6274ce9cc076310c4a10615de56897ee # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 80675cd170..799d6b004d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d92a6ab2871095ac66c60cfa15dbafa7b762f83d287d452f61792eb30cf5b26b \ No newline at end of file +c13346afbecb92275e741252897d00478dab4be2d158889bc735e80efd9444f5 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 4ef53cd8ae..9029ba9f13 100644 --- a/src/json.c +++ b/src/json.c @@ -1031,7 +1031,7 @@ static int json5Whitespace(const char *zIn){ } -#ifdef SQLITE_ENABLE_JSON_NAN_INF +#ifdef SQLITE_EXTENDED_NAN_INF /* ** Extra floating-point literals to allow in JSON. */ @@ -1050,7 +1050,7 @@ static const struct NanInfName { { 'q', 'Q', 4, JSON_NULL, 4, "QNaN", "null" }, { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" }, }; -#endif /* SQLITE_ENABLE_JSON_NAN_INF */ +#endif /* SQLITE_EXTENDED_NAN_INF */ /* ** Parse a single JSON value which begins at pParse->zJson[i]. Return the @@ -1306,7 +1306,35 @@ json_parse_restart: jnFlags = JNODE_JSON5; } }else{ - if( !sqlite3Isdigit(z[i+1]) ) return -1; + if( !sqlite3Isdigit(z[i+1]) ){ + if( z[i+1]=='I' + && (c=='-' || c=='+') + && strncmp(&z[i+1], "Infinity",8)==0 + ){ + if( z[i]=='-' ){ + jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); + }else{ + jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); + } + return i+9; + } +#ifdef SQLITE_EXTENDED_NAN_INF + /* Non-standard JSON and JSON5: Allow "Inf" as an alternative + ** spelling for "Infinity" and allow it to be in any case. */ + if( (z[i+1]=='I' || z[i+1]=='i') + && (c=='-' || c=='+') + && sqlite3StrNICmp(&z[i+1], "inf",3)==0 + ){ + if( z[i]=='-' ){ + jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); + }else{ + jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); + } + return i+4; + } +#endif + return -1; + } if( z[i+1]=='0' && sqlite3Isdigit(z[i+2]) ){ pParse->has5 = 1; jnFlags = JNODE_JSON5; @@ -1342,24 +1370,6 @@ json_parse_restart: if( c<'0' || c>'9' ) return -1; continue; } -#ifdef SQLITE_ENABLE_JSON_NAN_INF - /* Non-standard JSON: Allow "-Inf" (in any case) - ** to be understood as floating point literals. */ - if( (c=='i' || c=='I') - && j==i+1 - && z[i]=='-' - && sqlite3StrNICmp(&z[j], "inf",3)==0 - ){ - if( !sqlite3Isalnum(z[j+3]) ){ - jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); - return i+4; - }else if( (sqlite3StrNICmp(&z[j],"infinity",8)==0 && - !sqlite3Isalnum(z[j+8])) ){ - jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); - return i+9; - } - } -#endif break; } if( z[j-1]<'0' ){ @@ -1377,6 +1387,22 @@ json_parse_restart: } return j; } + case 'N': { + if( strncmp(&z[i],"NaN",3)==0 ){ + jsonParseAddNode(pParse, JSON_NULL, 4, "null"); + pParse->has5 = 1; + return i+3; + } + return -1; + } + case 'I': { + if( strncmp(&z[i],"Infinity",8)==0 ){ + jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); + pParse->has5 = 1; + return i+8; + } + return -1; + } case '}': { pParse->iErr = i; return -2; /* End of {...} */ @@ -1421,7 +1447,7 @@ json_parse_restart: return -1; } default: { -#ifdef SQLITE_ENABLE_JSON_NAN_INF +#ifdef SQLITE_EXTENDED_NAN_INF int k, nn; c = z[i]; for(k=0; k Date: Thu, 27 Apr 2023 18:28:10 +0000 Subject: [PATCH 291/341] Add support for hexadecimal integer literals in JSON. FossilOrigin-Name: 85e00c9e68d0695592e8f72555ee133c096bfca5a860a8e21d1e0ef756705aaf --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/json.c | 62 ++++++++++++++++++++++++++++++--------------------- src/util.c | 4 +++- 4 files changed, 47 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index e9ac4de4a3..1118208d40 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C All\sfloating\spoint\sliterals\s"NaN"\sand\s"Infinity".\s\sAdditional\svariants\nof\sthese\sliterals\sare\savailable\sif\scompiled\swith\sSQLITE_EXTENDED_NAN_INF. -D 2023-04-27T17:32:29.533 +C Add\ssupport\sfor\shexadecimal\sinteger\sliterals\sin\sJSON. +D 2023-04-27T18:28:10.882 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 530e1f4f4899a3c368bfaac63028e4c15ebd279a697780374d0cb7817103350a +F src/json.c e2830f713ac361dbdabcd9017acfb8e8e4a783ff5ca0f3925626edd8c7b11909 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -700,7 +700,7 @@ F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0 F src/update.c 3f4fb5ad7c9b48d7911974d6579192bb3a6c27f46140b6cbb9139cc8a77b8691 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c b1d8d87c4c8c77e70f48c43f91444fd66d91532693573b70b837afd572010176 +F src/util.c d4bcb560471cd94e6e17d448311f8d5bf81a7e5276295a53501058ef1b95dd1a F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c a6c52ba65e8ceb574fe0eda62af84e6c50c176ffc5f310c613425f7ab2b1484b F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d92a6ab2871095ac66c60cfa15dbafa7b762f83d287d452f61792eb30cf5b26b -R a7e436b837797408dc673da04c49a8e7 +P c13346afbecb92275e741252897d00478dab4be2d158889bc735e80efd9444f5 +R 6e312c065e46414b9040d6fd248d4cee U drh -Z 6274ce9cc076310c4a10615de56897ee +Z e7a3b5e2b439e9e4434717ea6eca66ea # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 799d6b004d..30c4114d5d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c13346afbecb92275e741252897d00478dab4be2d158889bc735e80efd9444f5 \ No newline at end of file +85e00c9e68d0695592e8f72555ee133c096bfca5a860a8e21d1e0ef756705aaf \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9029ba9f13..10f6b9839d 100644 --- a/src/json.c +++ b/src/json.c @@ -372,6 +372,18 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ N--; } while( zIn[0]=='0' && N>1 ){ + if( zIn[1]=='x' || zIn[1]=='X' ){ + sqlite3_int64 i = 0; + int rc = sqlite3DecOrHexToI64(zIn, &i); + if( rc<=1 ){ + jsonPrintf(100,p,"%lld",i); + }else if( rc==2 ){ + jsonAppendRaw(p, "9.0e999", 7); + }else{ + jsonAppendRaw(p, "9223372036854775808", 18); + } + return; + } zIn++; N--; } @@ -684,44 +696,32 @@ static void jsonReturn( } case JSON_INT: { sqlite3_int64 i = 0; + int rc; + int bNeg = 0; const char *z; + + assert( pNode->eU==1 ); z = pNode->u.zJContent; - if( z[0]=='-' ){ z++; } - while( z[0]>='0' && z[0]<='9' ){ - unsigned v = *(z++) - '0'; - if( i>=LARGEST_INT64/10 ){ - if( i>LARGEST_INT64/10 ) goto int_as_real; - if( z[0]>='0' && z[0]<='9' ) goto int_as_real; - if( v==9 ) goto int_as_real; - if( v==8 ){ - if( pNode->u.zJContent[0]=='-' ){ - sqlite3_result_int64(pCtx, SMALLEST_INT64); - goto int_done; - }else{ - goto int_as_real; - } - } - } - i = i*10 + v; + if( z[0]=='-' ){ z++; bNeg = 1; } + else if( z[0]=='+' ){ z++; } + rc = sqlite3DecOrHexToI64(z, &i); + if( rc<=1 ){ + sqlite3_result_int64(pCtx, bNeg ? -i : i); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else{ + goto to_double; } - if( pNode->u.zJContent[0]=='-' ){ i = -i; } - sqlite3_result_int64(pCtx, i); - int_done: break; - int_as_real: ; /* no break */ deliberate_fall_through } case JSON_REAL: { double r; -#ifdef SQLITE_AMALGAMATION const char *z; assert( pNode->eU==1 ); + to_double: z = pNode->u.zJContent; sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); -#else - assert( pNode->eU==1 ); - r = strtod(pNode->u.zJContent, 0); -#endif sqlite3_result_double(pCtx, r); break; } @@ -1370,6 +1370,16 @@ json_parse_restart: if( c<'0' || c>'9' ) return -1; continue; } + if( (c=='x' || c=='X') + && (j==i+1 || (j==i+2 && (z[i]=='-' || z[i]=='+'))) + && z[j-1]=='0' + && sqlite3Isxdigit(z[j+1]) + ){ + assert( seenDP==0 ); + pParse->has5 = 1; + jnFlags |= JNODE_JSON5; + for(j=j+2; sqlite3Isxdigit(z[j]); j++){} + } break; } if( z[j-1]<'0' ){ diff --git a/src/util.c b/src/util.c index 72e8a18b2f..3d4e014382 100644 --- a/src/util.c +++ b/src/util.c @@ -843,7 +843,9 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ u = u*16 + sqlite3HexToInt(z[k]); } memcpy(pOut, &u, 8); - return (z[k]==0 && k-i<=16) ? 0 : 2; + if( k-i>16 ) return 2; + if( z[k]!=0 ) return 1; + return 0; }else #endif /* SQLITE_OMIT_HEX_INTEGER */ { From 1371df21178ff21728a2d06582ac849aefdb0159 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 19:13:20 +0000 Subject: [PATCH 292/341] Faster implementation of numberic literal parsing in JSON. FossilOrigin-Name: 58398292e8473b0b9e4b77dd5ed27334bc24a85d0c399e8d0b86de6fb59dadce --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 44 ++++++++++++++++++++++++-------------------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 1118208d40..ecb7db6751 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\shexadecimal\sinteger\sliterals\sin\sJSON. -D 2023-04-27T18:28:10.882 +C Faster\simplementation\sof\snumberic\sliteral\sparsing\sin\sJSON. +D 2023-04-27T19:13:20.680 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c e2830f713ac361dbdabcd9017acfb8e8e4a783ff5ca0f3925626edd8c7b11909 +F src/json.c dc5befb58a71eca4c3703b2b62f4ef7bfe2087f261a21f84c0aa0f0d930e04f3 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c13346afbecb92275e741252897d00478dab4be2d158889bc735e80efd9444f5 -R 6e312c065e46414b9040d6fd248d4cee +P 85e00c9e68d0695592e8f72555ee133c096bfca5a860a8e21d1e0ef756705aaf +R 2024c2bf9c60e37e973abadb1fa7e9e0 U drh -Z e7a3b5e2b439e9e4434717ea6eca66ea +Z 1a9a05d48349450ebaef68906cd35ee3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 30c4114d5d..b162c93512 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -85e00c9e68d0695592e8f72555ee133c096bfca5a860a8e21d1e0ef756705aaf \ No newline at end of file +58398292e8473b0b9e4b77dd5ed27334bc24a85d0c399e8d0b86de6fb59dadce \ No newline at end of file diff --git a/src/json.c b/src/json.c index 10f6b9839d..ce33f83d54 100644 --- a/src/json.c +++ b/src/json.c @@ -1292,7 +1292,7 @@ json_parse_restart: /* Parse number */ jnFlags = 0; parse_number: - seenDP = 0; + seenDP = JSON_INT; seenE = 0; assert( '-' < '0' ); assert( '+' < '0' ); @@ -1304,6 +1304,12 @@ json_parse_restart: if( sqlite3Isdigit(z[i+1]) ){ pParse->has5 = 1; jnFlags = JNODE_JSON5; + }else if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ + assert( seenDP==JSON_INT ); + pParse->has5 = 1; + jnFlags |= JNODE_JSON5; + for(j=i+3; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; } }else{ if( !sqlite3Isdigit(z[i+1]) ){ @@ -1335,20 +1341,27 @@ json_parse_restart: #endif return -1; } - if( z[i+1]=='0' && sqlite3Isdigit(z[i+2]) ){ - pParse->has5 = 1; - jnFlags = JNODE_JSON5; + if( z[i+1]=='0' ){ + if( sqlite3Isdigit(z[i+2]) ){ + pParse->has5 = 1; + jnFlags = JNODE_JSON5; + }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ + pParse->has5 = 1; + jnFlags |= JNODE_JSON5; + for(j=i+4; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + } } } } parse_number_2: for(j=i+1;; j++){ c = z[j]; - if( c>='0' && c<='9' ) continue; + if( sqlite3Isdigit(c) ) continue; if( c=='.' ){ if( z[j-1]=='-' ) return -1; - if( seenDP ) return -1; - seenDP = 1; + if( seenDP==JSON_REAL ) return -1; + seenDP = JSON_REAL; continue; } if( c=='e' || c=='E' ){ @@ -1361,7 +1374,8 @@ json_parse_restart: } } if( seenE ) return -1; - seenDP = seenE = 1; + seenDP = JSON_REAL; + seenE = 1; c = z[j+1]; if( c=='+' || c=='-' ){ j++; @@ -1370,16 +1384,6 @@ json_parse_restart: if( c<'0' || c>'9' ) return -1; continue; } - if( (c=='x' || c=='X') - && (j==i+1 || (j==i+2 && (z[i]=='-' || z[i]=='+'))) - && z[j-1]=='0' - && sqlite3Isxdigit(z[j+1]) - ){ - assert( seenDP==0 ); - pParse->has5 = 1; - jnFlags |= JNODE_JSON5; - for(j=j+2; sqlite3Isxdigit(z[j]); j++){} - } break; } if( z[j-1]<'0' ){ @@ -1390,8 +1394,8 @@ json_parse_restart: return -1; } } - jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, - j - i, &z[i]); + parse_number_finish: + jsonParseAddNode(pParse, seenDP, j - i, &z[i]); if( jnFlags && !pParse->oom ){ pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; } From ae31f48826a250096945ab1494dad4c1c30852ee Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 19:30:53 +0000 Subject: [PATCH 293/341] Further optimizations to the JSON parser. FossilOrigin-Name: bb8f1c16f244f893170f3d03bc445bd15fc337804c7c3e76c548397f5b95b39a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 17 +++++------------ 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index ecb7db6751..a630d6827f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Faster\simplementation\sof\snumberic\sliteral\sparsing\sin\sJSON. -D 2023-04-27T19:13:20.680 +C Further\soptimizations\sto\sthe\sJSON\sparser. +D 2023-04-27T19:30:53.107 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c dc5befb58a71eca4c3703b2b62f4ef7bfe2087f261a21f84c0aa0f0d930e04f3 +F src/json.c 1c1cc509dd866dc05973eea6bd390c318c57fdad63c239ed3e2d214f78560dd5 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2063,8 +2063,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 85e00c9e68d0695592e8f72555ee133c096bfca5a860a8e21d1e0ef756705aaf -R 2024c2bf9c60e37e973abadb1fa7e9e0 +P 58398292e8473b0b9e4b77dd5ed27334bc24a85d0c399e8d0b86de6fb59dadce +R 89caa52021148a7b8eb691f84233609e U drh -Z 1a9a05d48349450ebaef68906cd35ee3 +Z e1f8b732b22a9ab7f249168903fe39b6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b162c93512..29f502947f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58398292e8473b0b9e4b77dd5ed27334bc24a85d0c399e8d0b86de6fb59dadce \ No newline at end of file +bb8f1c16f244f893170f3d03bc445bd15fc337804c7c3e76c548397f5b95b39a \ No newline at end of file diff --git a/src/json.c b/src/json.c index ce33f83d54..72ec1af10e 100644 --- a/src/json.c +++ b/src/json.c @@ -891,8 +891,8 @@ static int jsonParseAddNode( return jsonParseAddNodeExpand(pParse, eType, n, zContent); } p = &pParse->aNode[pParse->nNode]; - p->eType = (u8)eType; - p->jnFlags = 0; + p->eType = (u8)(eType & 0xff); + p->jnFlags = (u8)(eType >> 8); VVA( p->eU = zContent ? 1 : 0 ); p->n = n; p->u.zJContent = zContent; @@ -1092,10 +1092,7 @@ json_parse_restart: if( sqlite3Isalpha(z[j]) || z[j]=='_' || z[j]=='$' ){ int k; for(k=j+1; sqlite3Isalnum(z[k]) || z[k]=='_' || z[k]=='$'; k++){} - jsonParseAddNode(pParse, JSON_STRING, k-j, &z[j]); - if( !pParse->oom ){ - pParse->aNode[pParse->nNode-1].jnFlags = JNODE_RAW; - } + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), k-j, &z[j]); pParse->has5 = 1; x = k; }else{ @@ -1239,8 +1236,7 @@ json_parse_restart: } j++; } - jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); - if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]); return j+1; } case 'n': { @@ -1395,10 +1391,7 @@ json_parse_restart: } } parse_number_finish: - jsonParseAddNode(pParse, seenDP, j - i, &z[i]); - if( jnFlags && !pParse->oom ){ - pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; - } + jsonParseAddNode(pParse, seenDP | (jnFlags<<8), j - i, &z[i]); return j; } case 'N': { From cf5dce045f5d34df466993b40d1a9feae0915abb Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 27 Apr 2023 21:31:21 +0000 Subject: [PATCH 294/341] Adjust OSTRACE usage so that it works with the changes in the previous check-in. FossilOrigin-Name: 1eb4573e9092d350d00d05ba716cb0272e2dbb1ecbaadc2a7cc22de0b8cc2bda --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 12 +++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 38d2562688..eb73222d1b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\son\swindows\ssometimes\scausing\sSQLITE_NOMEM\sto\sbe\sreturned\sif\san\serror\soccurred\swhile\sopening\sa\stemp\sfile. -D 2023-04-27T13:55:29.757 +C Adjust\sOSTRACE\susage\sso\sthat\sit\sworks\swith\sthe\schanges\sin\sthe\sprevious\scheck-in. +D 2023-04-27T21:31:21.305 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -618,7 +618,7 @@ F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107 F src/os_unix.c 1b3ddb7814c4bf37f494c04d2ab30c1ced5b2c927267e1930ce7cd388787a96d -F src/os_win.c 20c2f020b54fe6ae210c1f200efc097ac50e7e7b24f64c632c1b751ac1f96899 +F src/os_win.c 2b2411279f7b24f927591561303fc5871845732df42641cbf695c23640b16975 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 9d36ddedc842e993c88c222ed914822dbd6f8ece3c648fde04468637012a034a F src/pager.h f82e9844166e1585f5786837ddc7709966138ced17f568c16af7ccf946c2baa3 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14d1c4a828660bc137dcea1bb4cc439e1ed3872867a0dab486f6a639ef3bcc45 -R 1e3870610983faacb84c1d55e8f701dd -U dan -Z ef68dfd966e00e89b716183914f3d8f9 +P c22a39a6b28e3039fe453c0c0db98744b8a63ef9063cc3f6e0ed67d2506d34be +R cc0091be8db1b63c66828854cc8ff562 +U mistachkin +Z ac07319bd47da651541c0acc4d52b4bc # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ff4ed9d11e..d3c1b0063d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c22a39a6b28e3039fe453c0c0db98744b8a63ef9063cc3f6e0ed67d2506d34be \ No newline at end of file +1eb4573e9092d350d00d05ba716cb0272e2dbb1ecbaadc2a7cc22de0b8cc2bda \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 33275113ad..b7b6897452 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -5453,15 +5453,17 @@ static int winAccess( void *zConverted; UNUSED_PARAMETER(pVfs); - if( zFilename==0 ){ - *pResOut = 0; - return SQLITE_OK; - } - SimulateIOError( return SQLITE_IOERR_ACCESS; ); OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", zFilename, flags, pResOut)); + if( zFilename==0 ){ + *pResOut = 0; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); + return SQLITE_OK; + } + zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); From 263b86c54c9379a7a667956d15953bd1f255ff09 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 27 Apr 2023 22:17:12 +0000 Subject: [PATCH 295/341] Fix compilation issues seen with older versions of MSVC. FossilOrigin-Name: 4ba2a651120391d779a3db9881c0fb17e1322cc9d8738f915a46e31a2f313578 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/msvc.h | 4 ++++ src/shell.c.in | 6 +++++- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index eb73222d1b..1307580539 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sOSTRACE\susage\sso\sthat\sit\sworks\swith\sthe\schanges\sin\sthe\sprevious\scheck-in. -D 2023-04-27T21:31:21.305 +C Fix\scompilation\sissues\sseen\swith\solder\sversions\sof\sMSVC. +D 2023-04-27T22:17:12.102 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -605,7 +605,7 @@ F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff F src/memdb.c 559c42e61eb70cd6d4bc692b042497133c6d96c09a3d514d92f3dac72268e223 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0 -F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8 +F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25 F src/mutex.h a7b2293c48db5f27007c3bdb21d438873637d12658f5a0bf8ad025bb96803c4a F src/mutex_noop.c 9d4309c075ba9cc7249e19412d3d62f7f94839c4 @@ -634,7 +634,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in e4762c17316110d4b65afac3572a4d134539a4178ac2afe4821de23f509a728f +F src/shell.c.in 5dcbcbcd8d984d05737172bcc23678ec34367905871ca1fdec4ca831bfd806fa F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c22a39a6b28e3039fe453c0c0db98744b8a63ef9063cc3f6e0ed67d2506d34be -R cc0091be8db1b63c66828854cc8ff562 +P 1eb4573e9092d350d00d05ba716cb0272e2dbb1ecbaadc2a7cc22de0b8cc2bda +R 1210fb0fa6a88a694d8f60efc7074476 U mistachkin -Z ac07319bd47da651541c0acc4d52b4bc +Z e10e84fb88698cc9750bcbdb28b62bd0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d3c1b0063d..fc0f04b096 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1eb4573e9092d350d00d05ba716cb0272e2dbb1ecbaadc2a7cc22de0b8cc2bda \ No newline at end of file +4ba2a651120391d779a3db9881c0fb17e1322cc9d8738f915a46e31a2f313578 \ No newline at end of file diff --git a/src/msvc.h b/src/msvc.h index f174227330..10675e9a71 100644 --- a/src/msvc.h +++ b/src/msvc.h @@ -38,4 +38,8 @@ #define SQLITE_4_BYTE_ALIGNED_MALLOC #endif /* defined(_MSC_VER) && !defined(_WIN64) */ +#if !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 +#define HAVE_LOG2 0 +#endif /* !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 */ + #endif /* SQLITE_MSVC_H */ diff --git a/src/shell.c.in b/src/shell.c.in index 5a7a7e0091..a6d67f4095 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3565,10 +3565,14 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ sqlite3_bind_text(pQ, 1, zVar, -1, SQLITE_STATIC); if( rc==SQLITE_OK && pQ && sqlite3_step(pQ)==SQLITE_ROW ){ sqlite3_bind_value(pStmt, i, sqlite3_column_value(pQ, 0)); +#ifdef NAN }else if( sqlite3_strlike("_NAN", zVar, 0)==0 ){ sqlite3_bind_double(pStmt, i, NAN); +#endif +#ifdef INFINITY }else if( sqlite3_strlike("_INF", zVar, 0)==0 ){ sqlite3_bind_double(pStmt, i, INFINITY); +#endif }else{ sqlite3_bind_null(pStmt, i); } @@ -4995,10 +4999,10 @@ static int showHelp(FILE *out, const char *zPattern){ ** Otherwise => whole help for documented commands */ enum HelpWanted hw = HW_SummaryOnly; + enum HelpHave hh = HH_More; if( zPattern!=0 ){ hw = (*zPattern=='0')? HW_NoCull|HW_Undoc : HW_NoCull; } - enum HelpHave hh = HH_More; for(i=0; i Date: Thu, 27 Apr 2023 23:29:09 +0000 Subject: [PATCH 296/341] Test cases added, and some bugs fixed. FossilOrigin-Name: bc84a82e4ddc1b71025c56c49e62a44f0b12fa87a6417ad61967d9d3121a0d4e --- manifest | 13 +- manifest.uuid | 2 +- src/json.c | 18 ++- test/json501.test | 307 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 328 insertions(+), 12 deletions(-) create mode 100644 test/json501.test diff --git a/manifest b/manifest index a630d6827f..dcbc60c91c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\soptimizations\sto\sthe\sJSON\sparser. -D 2023-04-27T19:30:53.107 +C Test\scases\sadded,\sand\ssome\sbugs\sfixed. +D 2023-04-27T23:29:09.447 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -592,7 +592,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 1c1cc509dd866dc05973eea6bd390c318c57fdad63c239ed3e2d214f78560dd5 +F src/json.c 76c7d11f7a5f29eaf3180c6ffcdf7fa808e931f519295d0725a736be8956a5cd F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -1258,6 +1258,7 @@ F test/json102.test 1f61f469d763ff26430dbee76bc75e0aa73084ca84f10e58744fdb899e56 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d +F test/json501.test 43614faefc9ac708e29d55cdc290b5cea336c151030196885708ea3d132fce63 F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c feb4358fb022da8ebd098c45811f2f6507688bb6c43aa72b3e840df19026317b F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -2063,8 +2064,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 58398292e8473b0b9e4b77dd5ed27334bc24a85d0c399e8d0b86de6fb59dadce -R 89caa52021148a7b8eb691f84233609e +P bb8f1c16f244f893170f3d03bc445bd15fc337804c7c3e76c548397f5b95b39a +R 6a60f8ad3d57ac84f5b8c0860a0c4b76 U drh -Z e1f8b732b22a9ab7f249168903fe39b6 +Z d1a0a8388ccda2f713b3fe179f34f0f5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 29f502947f..89e8dac194 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bb8f1c16f244f893170f3d03bc445bd15fc337804c7c3e76c548397f5b95b39a \ No newline at end of file +bc84a82e4ddc1b71025c56c49e62a44f0b12fa87a6417ad61967d9d3121a0d4e \ No newline at end of file diff --git a/src/json.c b/src/json.c index 72ec1af10e..6bac63ec51 100644 --- a/src/json.c +++ b/src/json.c @@ -816,7 +816,7 @@ static void jsonReturn( }else if( 0xe2==(u8)c ){ assert( 0x80==(u8)z[i+1] ); assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] ); - i+= 2; + i += 2; continue; }else{ continue; @@ -1221,13 +1221,17 @@ json_parse_restart: || c=='n' || c=='r' || c=='t' || (c=='u' && jsonIs4Hex(&z[j+1])) ){ jnFlags |= JNODE_ESCAPE; - }else if( c=='\'' || c=='0' || c=='v' - || c=='\r' || c=='\n' + }else if( c=='\'' || c=='0' || c=='v' || c=='\n' || (0xe2==(u8)c && 0x80==(u8)z[j+1] && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) || (c=='x' && jsonIs2Hex(&z[j+1])) ){ jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); pParse->has5 = 1; + }else if( c=='\r' ){ + j++; + if( z[j+1]=='\n' ) j++; + jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); + pParse->has5 = 1; }else{ return -1; } @@ -1270,7 +1274,7 @@ json_parse_restart: pParse->has5 = 1; jnFlags = JNODE_JSON5; seenE = 0; - seenDP = 1; + seenDP = JSON_REAL; goto parse_number_2; } return -1; @@ -1335,6 +1339,11 @@ json_parse_restart: return i+4; } #endif + if( z[i+1]=='.' ){ + pParse->has5 = 1; + jnFlags |= JNODE_JSON5; + goto parse_number_2; + } return -1; } if( z[i+1]=='0' ){ @@ -1355,7 +1364,6 @@ json_parse_restart: c = z[j]; if( sqlite3Isdigit(c) ) continue; if( c=='.' ){ - if( z[j-1]=='-' ) return -1; if( seenDP==JSON_REAL ) return -1; seenDP = JSON_REAL; continue; diff --git a/test/json501.test b/test/json501.test new file mode 100644 index 0000000000..fbf7bfe9db --- /dev/null +++ b/test/json501.test @@ -0,0 +1,307 @@ +# 2023-04-27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements tests for the JSON5 enhancements to the +# JSON SQL functions extension to the SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json501 + +# From https://spec.json5.org/#introduction +# +#----------------------------------------------------------------------------- +# Summary of Features +# +# The following ECMAScript 5.1 features, which are not supported in JSON, have +# been extended to JSON5. +# +# Objects +# +# 1) Object keys may be an ECMAScript 5.1 IdentifierName. +# 2) Objects may have a single trailing comma. +# +# Arrays +# +# 3) Arrays may have a single trailing comma. +# +# Strings +# +# 4) Strings may be single quoted. +# 5) Strings may span multiple lines by escaping new line characters. +# 6) Strings may include character escapes. +# +# Numbers +# +# 7) Numbers may be hexadecimal. +# 8) Numbers may have a leading or trailing decimal point. +# 9) Numbers may be IEEE 754 positive infinity, negative infinity, and NaN. +# 10) Numbers may begin with an explicit plus sign. +# +# Comments +# +# 11) Single and multi-line comments are allowed. +# +# White Space +# +# 12) Additional white space characters are allowed. +#----------------------------------------------------------------------------- +# +# Test number in this file are of the form X.Y where X is one of the item +# numbers in the feature list above and Y is the test sequence number. +# + +############################################################################### +# 1) Object keys may be an ECMAScript 5.1 IdentifierName. +do_execsql_test 1.1 { + WITH c(x) AS (VALUES('{a:5,b:6}')) + SELECT x->>'a', json(x), json_valid(x), json_valid5(x) FROM c; +} {5 {{"a":5,"b":6}} 0 1} +do_execsql_test 1.2 { + SELECT '[7,null,{a:5,b:6},[8,9]]'->>'$[2].b'; +} {6} +do_execsql_test 1.3 { + SELECT '{ $123 : 789 }'->>'$."$123"'; +} 789 +do_execsql_test 1.4 { + SELECT '{ _123$xyz : 789 }'->>'$."_123$xyz"'; +} 789 +do_execsql_test 1.5 { + SELECT '{ MNO_123$xyz : 789 }'->>'$."MNO_123$xyz"'; +} 789 + +do_execsql_test 1.6 { + SELECT json('{ MNO_123$xyz : 789 }'); +} [list {{"MNO_123$xyz":789}}] + +do_catchsql_test 1.10 { + SELECT json('{ MNO_123/xyz : 789 }'); +} {1 {malformed JSON}} + +# Contra the JSON5 standard, SQLite does not allow non-ASCII characters in +# an unquoted object label. +# +do_catchsql_test 1.11 { + SELECT json('{ MNO_123æxyz : 789 }'); +} {1 {malformed JSON}} + +############################################################################### +# 2) Objects may have a single trailing comma. + +do_execsql_test 2.1 { + WITH c(x) AS (VALUES('{"a":5, "b":6, }')) + SELECT x->>'b', json(x), json_valid(x), json_valid5(x) FROM c; +} {6 {{"a":5,"b":6}} 0 1} +do_execsql_test 2.2 { + SELECT '{a:5, b:6 , }'->>'b'; +} 6 +do_catchsql_test 2.3 { + SELECT '{a:5, b:6 ,, }'->>'b'; +} {1 {malformed JSON}} +do_catchsql_test 2.4 { + SELECT '{a:5, b:6, ,}'->>'b'; +} {1 {malformed JSON}} + +############################################################################### +# 3) Arrays may have a single trailing comma. + +do_execsql_test 3.1 { + WITH c(x) AS (VALUES('[5, 6,]')) + SELECT x->>1, json(x), json_valid(x), json_valid5(x) FROM c; +} {6 {[5,6]} 0 1} +do_execsql_test 3.2 { + SELECT '[5, 6 , ]'->>1; +} 6 +do_catchsql_test 3.3 { + SELECT '[5, 6,,]'->>1; +} {1 {malformed JSON}} +do_catchsql_test 3.4 { + SELECT '[5, 6 , , ]'->>1; +} {1 {malformed JSON}} + +############################################################################### +# 4) Strings may be single quoted. + +do_execsql_test 4.1 { + WITH c(x) AS (VALUES('{"a": ''abcd''}')) + SELECT x->>'a', json(x), json_valid(x), json_valid5(x) FROM c; +} {abcd {{"a":"abcd"}} 0 1} +do_execsql_test 4.2 { + SELECT '{b: 123, ''a'': ''ab\''cd''}'->>'a'; +} {ab'cd} + +############################################################################### +# 5) Strings may span multiple lines by escaping new line characters. + +do_execsql_test 5.1 { + WITH c(x) AS (VALUES('{a: "abc'||char(0x5c,0x0a)||'xyz"}')) + SELECT x->>'a', json(x), json_valid(x), json_valid5(x) FROM c; +} {abcxyz {{"a":"abcxyz"}} 0 1} +do_execsql_test 5.2 { + SELECT ('{a: "abc'||char(0x5c,0x0d)||'xyz"}')->>'a'; +} {abcxyz} +do_execsql_test 5.3 { + SELECT ('{a: "abc'||char(0x5c,0x0d,0x0a)||'xyz"}')->>'a'; +} {abcxyz} +do_execsql_test 5.4 { + SELECT ('{a: "abc'||char(0x5c,0x2028)||'xyz"}')->>'a'; +} {abcxyz} +do_execsql_test 5.5 { + SELECT ('{a: "abc'||char(0x5c,0x2029)||'xyz"}')->>'a'; +} {abcxyz} + + +############################################################################### +# 6) Strings may include character escapes. + +do_execsql_test 6.1 { + SELECT ('{a: "abc'||char(0x5c,0x27)||'xyz"}')->>'a'; +} {abc'xyz} +do_execsql_test 6.2 { + SELECT ('{a: "abc'||char(0x5c,0x22)||'xyz"}')->>'a'; +} {abc"xyz} +do_execsql_test 6.3 { + SELECT ('{a: "abc'||char(0x5c,0x5c)||'xyz"}')->>'a'; +} {{abc\xyz}} +do_execsql_test 6.4 { + SELECT hex(('{a: "abc\bxyz"}')->>'a'); +} {6162630878797A} +do_execsql_test 6.5 { + SELECT hex(('{a: "abc\f\n\r\t\vxyz"}')->>'a'); +} {6162630C0A0D090B78797A} +do_execsql_test 6.6 { + SELECT hex(('{a: "abc\0xyz"}')->>'a'); +} {6162630078797A} +do_execsql_test 6.7 { + SELECT '{a: "abc\x35\x4f\x6Exyz"}'->>'a'; +} {abc5Onxyz} +do_execsql_test 6.8 { + SELECT '{a: "\x6a\x6A\x6b\x6B\x6c\x6C\x6d\x6D\x6e\x6E\x6f\x6F"}'->>'a'; +} {jjkkllmmnnoo} + +############################################################################### +# 7) Numbers may be hexadecimal. + +do_execsql_test 7.1 { + SELECT '{a: 0x0}'->>'a'; +} 0 +do_execsql_test 7.2 { + SELECT '{a: -0x0}'->>'a'; +} 0 +do_execsql_test 7.3 { + SELECT '{a: +0x0}'->>'a'; +} 0 +do_execsql_test 7.4 { + SELECT '{a: 0xabcdef}'->>'a'; +} 11259375 +do_execsql_test 7.5 { + SELECT '{a: -0xaBcDeF}'->>'a'; +} -11259375 +do_execsql_test 7.6 { + SELECT '{a: +0xABCDEF}'->>'a'; +} 11259375 + +############################################################################### +# 8) Numbers may have a leading or trailing decimal point. + +do_execsql_test 8.1 { + WITH c(x) AS (VALUES('{x: 4.}')) SELECT x->>'x', json(x) FROM c; +} {4.0 {{"x":4.0}}} +do_execsql_test 8.2 { + WITH c(x) AS (VALUES('{x: +4.}')) SELECT x->>'x', json(x) FROM c; +} {4.0 {{"x":4.0}}} +do_execsql_test 8.3 { + WITH c(x) AS (VALUES('{x: -4.}')) SELECT x->>'x', json(x) FROM c; +} {-4.0 {{"x":-4.0}}} +do_execsql_test 8.3 { + WITH c(x) AS (VALUES('{x: .5}')) SELECT x->>'x', json(x) FROM c; +} {0.5 {{"x":0.5}}} +do_execsql_test 8.4 { + WITH c(x) AS (VALUES('{x: -.5}')) SELECT x->>'x', json(x) FROM c; +} {-0.5 {{"x":-0.5}}} +do_execsql_test 8.5 { + WITH c(x) AS (VALUES('{x: +.5}')) SELECT x->>'x', json(x) FROM c; +} {0.5 {{"x":0.5}}} +do_execsql_test 8.6 { + WITH c(x) AS (VALUES('{x: 4.e0}')) SELECT x->>'x', json(x) FROM c; +} {4.0 {{"x":4.0e0}}} +do_execsql_test 8.7 { + WITH c(x) AS (VALUES('{x: +4.e1}')) SELECT x->>'x', json(x) FROM c; +} {40.0 {{"x":4.0e1}}} +do_execsql_test 8.8 { + WITH c(x) AS (VALUES('{x: -4.e2}')) SELECT x->>'x', json(x) FROM c; +} {-400.0 {{"x":-4.0e2}}} +do_execsql_test 8.9 { + WITH c(x) AS (VALUES('{x: .5e3}')) SELECT x->>'x', json(x) FROM c; +} {500.0 {{"x":0.5e3}}} +do_execsql_test 8.10 { + WITH c(x) AS (VALUES('{x: -.5e-1}')) SELECT x->>'x', json(x) FROM c; +} {-0.05 {{"x":-0.5e-1}}} +do_execsql_test 8.11 { + WITH c(x) AS (VALUES('{x: +.5e-2}')) SELECT x->>'x', json(x) FROM c; +} {0.005 {{"x":0.5e-2}}} + + +############################################################################### +# 9) Numbers may be IEEE 754 positive infinity, negative infinity, and NaN. + +do_execsql_test 9.1 { + WITH c(x) AS (VALUES('{x: +Infinity}')) SELECT x->>'x', json(x) FROM c; +} {Inf {{"x":9.0e999}}} +do_execsql_test 9.2 { + WITH c(x) AS (VALUES('{x: -Infinity}')) SELECT x->>'x', json(x) FROM c; +} {-Inf {{"x":-9.0e999}}} +do_execsql_test 9.3 { + WITH c(x) AS (VALUES('{x: Infinity}')) SELECT x->>'x', json(x) FROM c; +} {Inf {{"x":9.0e999}}} +do_execsql_test 9.4 { + WITH c(x) AS (VALUES('{x: NaN}')) SELECT x->>'x', json(x) FROM c; +} {{} {{"x":null}}} + +############################################################################### +# 10) Numbers may begin with an explicit plus sign. + +do_execsql_test 10.1 { + SELECT '{a: +123}'->'a'; +} 123 + +############################################################################### +# 11) Single and multi-line comments are allowed. + +do_execsql_test 11.1 { + SELECT ' /* abc */ { /*def*/ aaa /* xyz */ : // to the end of line + 123 /* xyz */ , /* 123 */ }'->>'aaa'; +} 123 + +############################################################################### +# 12) Additional white space characters are allowed. + +do_execsql_test 12.1 { + SELECT (char(0x09,0x0a,0x0b,0x0c,0x0d,0x20,0xa0,0x2028,0x2029) + || '{a: "xyz"}')->>'a'; +} xyz +do_execsql_test 12.2 { + SELECT ('{a:' || char(0x09,0x0a,0x0b,0x0c,0x0d,0x20,0xa0,0x2028,0x2029) + || '"xyz"}')->>'a'; +} xyz +do_execsql_test 12.3 { + SELECT (char(0x1680,0x2000,0x2001,0x2002,0x2003,0x2004,0x2005, + 0x2006,0x2007,0x2008,0x2009,0x200a,0x3000,0xfeff) + || '{a: "xyz"}')->>'a'; +} xyz +do_execsql_test 12.4 { + SELECT ('{a: ' ||char(0x1680,0x2000,0x2001,0x2002,0x2003,0x2004,0x2005, + 0x2006,0x2007,0x2008,0x2009,0x200a,0x3000,0xfeff) + || ' "xyz"}')->>'a'; +} xyz + + +finish_test From d5de1f39b51ba95aab5ce51387eef9dfe8dff093 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 23:40:03 +0000 Subject: [PATCH 297/341] Remove an ALWAYS() macro that can in fact sometimes be false. [forum:/forumpost/f0e872fcee|Forum post f0e872fcee]. FossilOrigin-Name: d85880e4ed89e2198beea642818051f7a91a71fea9929be0a7ed0603b493ce47 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1307580539..546858402e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompilation\sissues\sseen\swith\solder\sversions\sof\sMSVC. -D 2023-04-27T22:17:12.102 +C Remove\san\sALWAYS()\smacro\sthat\scan\sin\sfact\ssometimes\sbe\sfalse.\n[forum:/forumpost/f0e872fcee|Forum\spost\sf0e872fcee]. +D 2023-04-27T23:40:03.444 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -703,7 +703,7 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c b1d8d87c4c8c77e70f48c43f91444fd66d91532693573b70b837afd572010176 F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd -F src/vdbe.c 1d3e3b10ce83d1470bc349e9fe842ddded303591bae4c49e3292975f3c4d9beb +F src/vdbe.c 94d5520d2a287216c47e6fb641ee88ffd934b0d40c235d693d38bcd0e0750357 F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1eb4573e9092d350d00d05ba716cb0272e2dbb1ecbaadc2a7cc22de0b8cc2bda -R 1210fb0fa6a88a694d8f60efc7074476 -U mistachkin -Z e10e84fb88698cc9750bcbdb28b62bd0 +P 4ba2a651120391d779a3db9881c0fb17e1322cc9d8738f915a46e31a2f313578 +R 17f49f4919f4d8f5c0f5d76a60091614 +U drh +Z 328809aebe8b52d41e5cb3bc142e5a4c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc0f04b096..a9c8c64d27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4ba2a651120391d779a3db9881c0fb17e1322cc9d8738f915a46e31a2f313578 \ No newline at end of file +d85880e4ed89e2198beea642818051f7a91a71fea9929be0a7ed0603b493ce47 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 738596daea..ea2e6fc5b1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2745,7 +2745,7 @@ case OP_IfNullRow: { /* jump */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - if( ALWAYS(pC) && pC->nullRow ){ + if( pC && pC->nullRow ){ sqlite3VdbeMemSetNull(aMem + pOp->p3); goto jump_to_p2; } From bfd28f9e01ae9a8436095a033ce96cf16b9c6f5b Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 23:44:38 +0000 Subject: [PATCH 298/341] Remove a faulty assert() from the CLI. [forum:/forumpost/726c4f7db0|Forum post 726c4f7db0]. FossilOrigin-Name: 4bbebb6bfb9910265d91b777c1711b3b8e0732bcf299f7459b20c4ea110422bd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 546858402e..bbffdf19e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sALWAYS()\smacro\sthat\scan\sin\sfact\ssometimes\sbe\sfalse.\n[forum:/forumpost/f0e872fcee|Forum\spost\sf0e872fcee]. -D 2023-04-27T23:40:03.444 +C Remove\sa\sfaulty\sassert()\sfrom\sthe\sCLI.\n[forum:/forumpost/726c4f7db0|Forum\spost\s726c4f7db0]. +D 2023-04-27T23:44:38.716 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -634,7 +634,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in 5dcbcbcd8d984d05737172bcc23678ec34367905871ca1fdec4ca831bfd806fa +F src/shell.c.in 3b5633b7c78e51df21af693ce27cccca28e6118480060876d6be968c2f7a577c F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4ba2a651120391d779a3db9881c0fb17e1322cc9d8738f915a46e31a2f313578 -R 17f49f4919f4d8f5c0f5d76a60091614 +P d85880e4ed89e2198beea642818051f7a91a71fea9929be0a7ed0603b493ce47 +R c57990c4154d9f0a3a4f65c13761b58d U drh -Z 328809aebe8b52d41e5cb3bc142e5a4c +Z d40988ca57efd61e3977061d11afa616 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a9c8c64d27..accb12491c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d85880e4ed89e2198beea642818051f7a91a71fea9929be0a7ed0603b493ce47 \ No newline at end of file +4bbebb6bfb9910265d91b777c1711b3b8e0732bcf299f7459b20c4ea110422bd \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index a6d67f4095..d7621c0c9e 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -10516,7 +10516,8 @@ static int do_meta_command(char *zLine, ShellState *p){ if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); if( lrc!=SQLITE_OK ){ - assert(lrc==SQLITE_NOMEM); + /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the + ** user does cruel and unnatural things like ".limit expr_depth 0". */ rc = 1; }else{ if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); From 93b4c3beb8f005e1c0ce6f27b1675e4b2f24acc4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Apr 2023 23:59:51 +0000 Subject: [PATCH 299/341] Use a new technique to detect fresh OOM faults in columnName() that does not rely on there being no OOMs prior to entry into columnName(), as [forum/forumpost/fb6811c2f9|forum post fb6811c2f9] demonstrates a technique which could cause an OOM prior to entry into columnName(). FossilOrigin-Name: a63346d6a0c0ca7ba4c87499de2e461be9c77e9b5d98f2bebf308cdb6599f33c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bbffdf19e1..cbe41d4e9d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sfaulty\sassert()\sfrom\sthe\sCLI.\n[forum:/forumpost/726c4f7db0|Forum\spost\s726c4f7db0]. -D 2023-04-27T23:44:38.716 +C Use\sa\snew\stechnique\sto\sdetect\sfresh\sOOM\sfaults\sin\scolumnName()\sthat\sdoes\snot\nrely\son\sthere\sbeing\sno\sOOMs\sprior\sto\sentry\sinto\scolumnName(),\sas\n[forum/forumpost/fb6811c2f9|forum\spost\sfb6811c2f9]\sdemonstrates\sa\stechnique\nwhich\scould\scause\san\sOOM\sprior\sto\sentry\sinto\scolumnName(). +D 2023-04-27T23:59:51.813 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -706,7 +706,7 @@ F src/vacuum.c 84ce7f01f8a7a08748e107a441db83bcec13970190ddcb0c9ff522adbc1c23fd F src/vdbe.c 94d5520d2a287216c47e6fb641ee88ffd934b0d40c235d693d38bcd0e0750357 F src/vdbe.h 637ae853b7d42ae3951034cc63ab7c8af837861f79504cdb5399552fcd89a884 F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb -F src/vdbeapi.c 1a95162e26d5eda3b7b46fbe4fcbc33eb7f801529d66fc2e14c52094a5523339 +F src/vdbeapi.c b4982cde547054c4f7341198db3c3008a48e1eb028f757601bf5bf2fc026cbcf F src/vdbeaux.c 6ee48db408d4c297a363f1e31145c09793a580e7c508bb36063dd017d67117a2 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd F src/vdbemem.c 1d9a0f37b0097fbb53f0d7ba081f7181b83cee2c6f46364706ea0c3896bd8ec0 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d85880e4ed89e2198beea642818051f7a91a71fea9929be0a7ed0603b493ce47 -R c57990c4154d9f0a3a4f65c13761b58d +P 4bbebb6bfb9910265d91b777c1711b3b8e0732bcf299f7459b20c4ea110422bd +R 59c8d7a772734f630b5bf6fd205de302 U drh -Z d40988ca57efd61e3977061d11afa616 +Z d29ba302c14b4aed5db886f9a3fc4bd5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index accb12491c..b9ba76d6e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4bbebb6bfb9910265d91b777c1711b3b8e0732bcf299f7459b20c4ea110422bd \ No newline at end of file +a63346d6a0c0ca7ba4c87499de2e461be9c77e9b5d98f2bebf308cdb6599f33c \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index d1e1232b1a..d8fcda96df 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1337,9 +1337,9 @@ static const void *columnName( assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N=0 ){ + u8 prior_mallocFailed = db->mallocFailed; N += useType*n; sqlite3_mutex_enter(db->mutex); - assert( db->mallocFailed==0 ); #ifndef SQLITE_OMIT_UTF16 if( useUtf16 ){ ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); @@ -1351,7 +1351,8 @@ static const void *columnName( /* A malloc may have failed inside of the _text() call. If this ** is the case, clear the mallocFailed flag and return NULL. */ - if( db->mallocFailed ){ + assert( db->mallocFailed==0 || db->mallocFailed==1 ); + if( db->mallocFailed > prior_mallocFailed ){ sqlite3OomClear(db); ret = 0; } From fe03773305fa7c52e26df3a5794475037fbf456a Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 28 Apr 2023 00:19:55 +0000 Subject: [PATCH 300/341] In the CLI, early out from the output_quoted_string() routine if the input string is NULL. [forum:/forumpost/8a22c0bfa7|forum post 8a22c0bfa7]. FossilOrigin-Name: 2881978d7b7014738a21c590b0430c4fa40e6b00b9547d0284b7656124523ae9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index cbe41d4e9d..8dfeca3498 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sa\snew\stechnique\sto\sdetect\sfresh\sOOM\sfaults\sin\scolumnName()\sthat\sdoes\snot\nrely\son\sthere\sbeing\sno\sOOMs\sprior\sto\sentry\sinto\scolumnName(),\sas\n[forum/forumpost/fb6811c2f9|forum\spost\sfb6811c2f9]\sdemonstrates\sa\stechnique\nwhich\scould\scause\san\sOOM\sprior\sto\sentry\sinto\scolumnName(). -D 2023-04-27T23:59:51.813 +C In\sthe\sCLI,\searly\sout\sfrom\sthe\soutput_quoted_string()\sroutine\sif\sthe\ninput\sstring\sis\sNULL.\s\s[forum:/forumpost/8a22c0bfa7|forum\spost\s8a22c0bfa7]. +D 2023-04-28T00:19:55.495 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -634,7 +634,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in 3b5633b7c78e51df21af693ce27cccca28e6118480060876d6be968c2f7a577c +F src/shell.c.in dbd5bfe1c773115f4e526103807dc3c1484283cd8afe9cf759d64a338408f19f F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4bbebb6bfb9910265d91b777c1711b3b8e0732bcf299f7459b20c4ea110422bd -R 59c8d7a772734f630b5bf6fd205de302 +P a63346d6a0c0ca7ba4c87499de2e461be9c77e9b5d98f2bebf308cdb6599f33c +R e1382566976c275dfea2ac91ab17df60 U drh -Z d29ba302c14b4aed5db886f9a3fc4bd5 +Z 06ecc62941bb9eb7ee780cf940024968 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b9ba76d6e8..250b4f032f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a63346d6a0c0ca7ba4c87499de2e461be9c77e9b5d98f2bebf308cdb6599f33c \ No newline at end of file +2881978d7b7014738a21c590b0430c4fa40e6b00b9547d0284b7656124523ae9 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index d7621c0c9e..18690214b5 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1860,6 +1860,7 @@ static void output_quoted_string(FILE *out, const char *z){ int i; char c; setBinaryMode(out, 1); + if( z==0 ) return; for(i=0; (c = z[i])!=0 && c!='\''; i++){} if( c==0 ){ utf8_printf(out,"'%s'",z); From fb016bf25733207d0875302bac7cfb27c7e901b9 Mon Sep 17 00:00:00 2001 From: larrybr Date: Fri, 28 Apr 2023 00:28:06 +0000 Subject: [PATCH 301/341] Make CLI .load slightly harder to abuse. FossilOrigin-Name: 2779f9270cc431786d0e16ef05ec05b0f22dda2307f3bf17467ad0b25304e160 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8dfeca3498..27e6d37f50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sCLI,\searly\sout\sfrom\sthe\soutput_quoted_string()\sroutine\sif\sthe\ninput\sstring\sis\sNULL.\s\s[forum:/forumpost/8a22c0bfa7|forum\spost\s8a22c0bfa7]. -D 2023-04-28T00:19:55.495 +C Make\sCLI\s.load\sslightly\sharder\sto\sabuse. +D 2023-04-28T00:28:06.476 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -634,7 +634,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in dbd5bfe1c773115f4e526103807dc3c1484283cd8afe9cf759d64a338408f19f +F src/shell.c.in 09097e1b9df1f8092e85bf89979e12ca7b608d7efc84551b5d0c8de4dded7797 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a63346d6a0c0ca7ba4c87499de2e461be9c77e9b5d98f2bebf308cdb6599f33c -R e1382566976c275dfea2ac91ab17df60 -U drh -Z 06ecc62941bb9eb7ee780cf940024968 +P 2881978d7b7014738a21c590b0430c4fa40e6b00b9547d0284b7656124523ae9 +R f31a79597744826157054a8d8987cbe4 +U larrybr +Z 94682afadd89800c43fc7c92687bd9b6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 250b4f032f..df2a5b5785 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2881978d7b7014738a21c590b0430c4fa40e6b00b9547d0284b7656124523ae9 \ No newline at end of file +2779f9270cc431786d0e16ef05ec05b0f22dda2307f3bf17467ad0b25304e160 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 18690214b5..7285067bef 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -9192,7 +9192,8 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zFile, *zProc; char *zErrMsg = 0; failIfSafeMode(p, "cannot run .load in safe mode"); - if( nArg<2 ){ + if( nArg<2 || azArg[1][0]==0 ){ + /* Must have a non-empty FILE. (Will not load self.) */ raw_printf(stderr, "Usage: .load FILE ?ENTRYPOINT?\n"); rc = 1; goto meta_command_exit; From 90f549a0cfb94c25e2126e27ce358618e11e8bd6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 28 Apr 2023 10:10:52 +0000 Subject: [PATCH 302/341] Do not assert() a bad string representation in an sqlite3_value after an OOM. dbsqlfuzz c822a17a23c524a0ac7cfb203c7198209da15de8. FossilOrigin-Name: 91fee79a01971259b21478e60a069a711a00efc79ddfececa6224a152cd8d09a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbemem.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 27e6d37f50..9f1cd11163 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sCLI\s.load\sslightly\sharder\sto\sabuse. -D 2023-04-28T00:28:06.476 +C Do\snot\sassert()\sa\sbad\sstring\srepresentation\sin\san\ssqlite3_value\safter\nan\sOOM.\s\sdbsqlfuzz\sc822a17a23c524a0ac7cfb203c7198209da15de8. +D 2023-04-28T10:10:52.120 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -709,7 +709,7 @@ F src/vdbeInt.h a4147a4ddf613cb1bcb555ace9e9e74a9c099d65facd88155f191b1fb4d74cfb F src/vdbeapi.c b4982cde547054c4f7341198db3c3008a48e1eb028f757601bf5bf2fc026cbcf F src/vdbeaux.c 6ee48db408d4c297a363f1e31145c09793a580e7c508bb36063dd017d67117a2 F src/vdbeblob.c 5e61ce31aca17db8fb60395407457a8c1c7fb471dde405e0cd675974611dcfcd -F src/vdbemem.c 1d9a0f37b0097fbb53f0d7ba081f7181b83cee2c6f46364706ea0c3896bd8ec0 +F src/vdbemem.c 1cac4028c0dabbf1f3259f107440e2780e05ac9fe419e9709e6eb4e166ba714b F src/vdbesort.c 43756031ca7430f7aec3ef904824a7883c4ede783e51f280d99b9b65c0796e35 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2881978d7b7014738a21c590b0430c4fa40e6b00b9547d0284b7656124523ae9 -R f31a79597744826157054a8d8987cbe4 -U larrybr -Z 94682afadd89800c43fc7c92687bd9b6 +P 2779f9270cc431786d0e16ef05ec05b0f22dda2307f3bf17467ad0b25304e160 +R fcf6991862458e9816409ec8d717ddc8 +U drh +Z d3df2b888a0fbd6bd36c6662515c45b3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index df2a5b5785..e2c06ba976 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2779f9270cc431786d0e16ef05ec05b0f22dda2307f3bf17467ad0b25304e160 \ No newline at end of file +91fee79a01971259b21478e60a069a711a00efc79ddfececa6224a152cd8d09a \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 7b3b970f3f..d3cd55ba9f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -157,6 +157,7 @@ int sqlite3VdbeMemValidStrRep(Mem *p){ char *z; int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; + if( p->db && p->db->mallocFailed ) return 1; if( p->flags & MEM_Term ){ /* Insure that the string is properly zero-terminated. Pay particular ** attention to the case where p->n is odd */ From 1170fb52e12c5476ac8cf03216609c6824953982 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 28 Apr 2023 11:02:01 +0000 Subject: [PATCH 303/341] Fix json_tree() so that it is able to deal with the JNODE_RAW labels of a JSON5 object. FossilOrigin-Name: f56528d413d8e622f7c4f18b2f9f2e620bfb441c020461299b35a90072ee6c13 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/json.c | 26 +++++++++++--------------- test/json502.test | 25 +++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 22 deletions(-) create mode 100644 test/json502.test diff --git a/manifest b/manifest index b8b8e777ef..90577f373d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sthe\slatest\strunk\sfixes\sand\senhancements\sinto\sthe\sjson5\sbranch. -D 2023-04-28T10:23:01.697 +C Fix\sjson_tree()\sso\sthat\sit\sis\sable\sto\sdeal\swith\sthe\sJNODE_RAW\slabels\sof\na\sJSON5\sobject. +D 2023-04-28T11:02:01.468 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -593,7 +593,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 76c7d11f7a5f29eaf3180c6ffcdf7fa808e931f519295d0725a736be8956a5cd +F src/json.c 7d03a1185d7aee74f53a3f2381355ff4b71d955f287d29d30c0b0979afc96b67 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -1260,6 +1260,7 @@ F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a888011 F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d F test/json501.test 43614faefc9ac708e29d55cdc290b5cea336c151030196885708ea3d132fce63 +F test/json502.test 66d150cc098674b8bf4354526a8dd411b926f43ca892306bcb3b6d3f93fef7be F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c feb4358fb022da8ebd098c45811f2f6507688bb6c43aa72b3e840df19026317b F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 @@ -2065,8 +2066,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bc84a82e4ddc1b71025c56c49e62a44f0b12fa87a6417ad61967d9d3121a0d4e 91fee79a01971259b21478e60a069a711a00efc79ddfececa6224a152cd8d09a -R 454da0ecccffebadc86f13fb2a470b64 +P b5ca15cfc19380cf870b70be6a86e70f2026cc3d6d89005b45891d58c4f11c2d +R fca31e8533435392332b7b8d5d313d4f U drh -Z dbd4148e71a7855c43311d70f48a2c21 +Z 70e5d0234b5092ca91c1a96a87fa5d9d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index eb3f235ab7..3b090148dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5ca15cfc19380cf870b70be6a86e70f2026cc3d6d89005b45891d58c4f11c2d \ No newline at end of file +f56528d413d8e622f7c4f18b2f9f2e620bfb441c020461299b35a90072ee6c13 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 6bac63ec51..1152d9f916 100644 --- a/src/json.c +++ b/src/json.c @@ -726,17 +726,11 @@ static void jsonReturn( break; } case JSON_STRING: { -#if 0 /* Never happens because JNODE_RAW is only set by json_set(), - ** json_insert() and json_replace() and those routines do not - ** call jsonReturn() */ if( pNode->jnFlags & JNODE_RAW ){ assert( pNode->eU==1 ); sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, SQLITE_TRANSIENT); - }else -#endif - assert( (pNode->jnFlags & JNODE_RAW)==0 ); - if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ + }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ assert( pNode->eU==1 ); sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, @@ -2872,14 +2866,16 @@ static void jsonAppendObjectPathElement( assert( pNode->eU==1 ); z = pNode->u.zJContent; nn = pNode->n; - assert( nn>=2 ); - assert( z[0]=='"' ); - assert( z[nn-1]=='"' ); - if( nn>2 && sqlite3Isalpha(z[1]) ){ - for(jj=2; jjjnFlags & JNODE_RAW)==0 ){ + assert( nn>=2 ); + assert( z[0]=='"' ); + assert( z[nn-1]=='"' ); + if( nn>2 && sqlite3Isalpha(z[1]) ){ + for(jj=2; jj Date: Fri, 28 Apr 2023 13:25:35 +0000 Subject: [PATCH 304/341] Fix indentation and omit trailing whitespace in the random JSON generator script. FossilOrigin-Name: 629db09fceb7bf37561b52ccee06ebf4df261291e9a8ffcca82b243f6db5ff07 --- manifest | 12 +- manifest.uuid | 2 +- test/json/json-generator.tcl | 540 +++++++++++++++++------------------ 3 files changed, 277 insertions(+), 277 deletions(-) diff --git a/manifest b/manifest index 90577f373d..e9188432cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sjson_tree()\sso\sthat\sit\sis\sable\sto\sdeal\swith\sthe\sJNODE_RAW\slabels\sof\na\sJSON5\sobject. -D 2023-04-28T11:02:01.468 +C Fix\sindentation\sand\somit\strailing\swhitespace\sin\sthe\srandom\sJSON\sgenerator\nscript. +D 2023-04-28T13:25:35.641 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1251,7 +1251,7 @@ F test/jrnlmode.test 9b5bc01dac22223cb60ec2d5f97acf568d73820794386de5634dcadbea9 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/json/README.md 506af1f54574b524106acb50d1a341ab5ddfa6d83fe25095007892b07e663e85 -F test/json/json-generator.tcl 229bd293f1865f787c160886cadd282631721925cca2947aaa54bbcd7f65cef7 +F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x F test/json101.test de9c93169b84ac96fd5836c638a2ae1f00e4afbd4003c6b596692d7f05e1cd69 @@ -2066,8 +2066,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b5ca15cfc19380cf870b70be6a86e70f2026cc3d6d89005b45891d58c4f11c2d -R fca31e8533435392332b7b8d5d313d4f +P f56528d413d8e622f7c4f18b2f9f2e620bfb441c020461299b35a90072ee6c13 +R 5da141bb06598ed103d496559df7ae36 U drh -Z 70e5d0234b5092ca91c1a96a87fa5d9d +Z 790f8e2b554134402e5889a19faba52a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3b090148dc..d1f6aa531d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f56528d413d8e622f7c4f18b2f9f2e620bfb441c020461299b35a90072ee6c13 \ No newline at end of file +629db09fceb7bf37561b52ccee06ebf4df261291e9a8ffcca82b243f6db5ff07 \ No newline at end of file diff --git a/test/json/json-generator.tcl b/test/json/json-generator.tcl index e99d6bccb0..d499bc7300 100644 --- a/test/json/json-generator.tcl +++ b/test/json/json-generator.tcl @@ -10,276 +10,276 @@ # expr srand(12345678) set wordlist { -ability able abroad access account act - action active actor add address adept - adroit advance advice affect age ageless - agency agent agile agree air airfare - airline airport alert almond alpha always - amend amount amplify analyst anchor angel - angelic angle ankle annual answer antique - anybody anyhow appeal apple apricot apt - area argon arm army arrival arsenic - art artful article arugula aside ask - aspect assist assume atom atone attempt - author autumn average avocado award awl - azure back bacon bag bagel bake - baker balance ball balloon bamboo banana - band banjo bank barium base basil - basin basis basket bass bat bath - battery beach beak bean bear bearcub - beauty beef beet beige being bell - belly belt bench bend benefit best - beta better beyond bicycle bid big - bike bill bird biscuit bismuth bisque - bit black blank blest blind bliss - block bloom blue board boat body - bokchoy bone bonus book bookish boot - border boron boss bossy bottle bottom - bow bowl bowtie box brain brainy - branch brave bravely bread break breath - breezy brick bridge brie brief briefly - bright broad broil bromine bronze brother - brow brown brush buddy budget buffalo - bug bugle bull bunch burger burly - burrito bus busy butter button buy - buyer byte cab cabbage cabinet cable - cadet cadmium caesium cake calcium caliper - call caller calm calmly camera camp - can canary cancel candle candy cap - capable caper capital captain car carbon - card care career careful carp carpet - carrot carry case cash cassava casual - cat catch catfish catsear catsup cause - cave celery cell century chain chair - chalk chance change channel chapter chard - charge charity chart check cheddar cheery - cheese chicken chicory chiffon child chin - chip chives choice chowder chum church - circle city claim clam class classic - classy clay clean cleaner clear clearly - clerk click client climate clock clorine - closet clothes cloud clown club clue - cluster coach coast coat cobbler cobolt - cod code coffee colby cold collar - college comb combine comet comfort command - comment common company complex concept concern - concert conduit consist contact contest context - control convert cook cookie copilot copper - copy coral cordial corn corner corny - correct cost count counter country county - couple courage course court cover cow - cowbird crab crack craft crash crazy - cream credit creek cress crevice crew - crimson croaker crop cross crowd cube - cuckoo cuisine culture cup current curve - cut cyan cycle dagger daily dance - dare darter data date day daylily - deal dear dearly debate debit decade - decimal deep deft deftly degree delay - deluxe deposit depth design desk detail - device dew diamond diet dig dill - dinner dip direct dirt dish disk - display diver divide divine doctor dodger - donut door dot double dough draft - drag dragon drama draw drawer drawing - dream drill drink drive driver drop - drum dry dryer drywall duck due - dump dusk dust duty dye eagle - ear earring earth ease east easy - eat economy edge editor eel effect - effort egg eight elbow elegant element - elf elk email emerald employ end - endive endless energy engine enjoy enter - entry equal equip error escape essay - eternal evening event exam example excuse - exit expert extent extreme eye face - fact factor factual fail failure fair - fajita fall family fan fang farm - farmer fat fault feature feed feel - feeling fench fennel festive few fiber - field fig figure file fill film - filter final finance finding finger finish - fire fish fishing fit fitting five - fix flier flight floor floral florine - flour flow flower fly flying focus - fold folding food foot force forest - forever forgive form formal format fortune - forum frame free freedom freely fresh - friend frog front fruit fuchsia fuel - fun funny future gain galaxy gallium - game gamma gap garage garden garlic - gas gate gather gauge gear gem - gene general gentle gently gherkin ghost - gift give glad glass gleeful glossy - glove glue goal goat goby gold - goldeye golf good gouda goulash gourd - grab grace grade gram grand grape - grapes grass gravy gray great green - grits grocery ground group grouper grout - growth guard guave guess guest guide - guitar gumbo guppy habit hacksaw haddock - hafnium hagfish hair half halibut hall - hammer hand handle handy hanger happy - hat havarti hay haybale head health - healthy hearing heart hearty heat heavy - heel height helium hello help helpful - herald herring hide high highly highway - hill hip hipster hire history hit - hoki hold hole holiday holly home - honest honey hook hope hopeful horizon - horn horse host hotel hour house - housing human humane humor hunt hurry - ice icecube icefish icy idea ideal - image impact impress inch income indigo - initial inkpen insect inside intense invite - iodine iridium iron island issue item - ivory jacket jargon javelin jello jelly - jewel job jocund join joint joke - jovial joy joyful joyous judge juice - jump junior jury just justice kale - keel keep kelp ketchup key keyhole - keyway khaki kick kid kidney kiloohm - kind kindly king kitchen kite kiwi - knee knife krill krypton kumquat lab - lace lack ladder lake lamp lamprey - land laser laugh law lawn lawyer - layer lead leader leading leaf leafy - league leather leave lecture leek leg - lemon length lentil lesson let letter - lettuce level library life lift light - lily lime limit line linen link - lip list listen lithium lively living - lizard load loan lobster local lock - log long longfin look lotus love - lovely loving low lucid luck luffa - lunch lung machine magenta magnet mail - main major make mall manager mango - manner many map march market maroon - martian master match math matter maximum - maybe meal meaning meat media medium - meet meeting melody melon member memory - mention menu mercury merry mess message - messy metal meter method micron middle - might mile milk mind mine minimum - minnow minor mint minute mirror miss - mission misty mix mixer mixture mobile - mode model moment monitor monk month - moon moray morning most motor mouse - mouth move mover movie much mud - mudfish muffin mullet munster muon muscle - music mustard nail name nation native - natural nature navy neat neatly nebula - neck needle neon nerve net network - neutron news nibble nice nickel night - niobium nobody noise noodle normal north - nose note nothing notice nova novel - number nurse nursery oar object offer - office officer oil okay okra old - olive one onion open opening opinion - option orange orbit orchid order oregano - other ounce outcome outside oven owner - oxygen oyster pace pack package page - pager paint pair pale pan pancake - papaya paper pardon parent park parking - parsley parsnip part partner party pass - passage past pasta path patient pattern - pause pay pea peace peach peacock - peahen peak peanut pear pearl pen - penalty pencil pension people pepper perch - perfect period permit person phase phone - photo phrase physics piano pick picture - pie piece pigeon pike pilot pin - pink pinkie pious pipe pitch pizza - place plan plane planet plant planter - plastic plate play player playful plenty - pliers plum pod poem poet poetry - point police policy pollock pony pool - pop popover poptart pork port portal - post pot potato pound powder power - present press price pride primary print - prior private prize problem process produce - product profile profit program project promise - prompt proof proper protein proton public - puff puffer pull pumpkin pup pupfish - pure purple purpose push put quality - quark quarter quiet quill quit quote - rabbit raccoon race radiant radio radish - radium radon rain rainbow raise ramp - ranch range rasp rate ratio ray - razor reach read reading real reality - reason recipe record recover red redeem - reed reef refuse region regret regular - relaxed release relief relish remote remove - rent repair repeat reply report request - reserve resist resolve resort rest result - return reveal review reward ribbon rice - rich ride ridge right ring rise - risk river rivet road roast rock - rocket role roll roof room rope - rose rough roughy round row royal - rub ruby rudder ruin rule run - runner rush rust sacred saddle safe - safety sail salad salami sale salmon - salt sample sand sander sandy sauce - save saving saw scale scampi scene - scheme school score screen script sea - search season seat second secret sector - seemly self sell senate senior sense - series serve set shake shape share - shark shell shift shine shiny ship - shock shoe shoot shop shovel show - side sign signal silk silly silver - simple sing singer single sink site - size skill skin sky slate sleep - sleepy slice slide slip smart smell - smelt smile smoke smooth snap snipe - snow snowy sock socket sodium soft - softly soil sole solid song sorrel - sort soul sound soup source south - space spare speech speed spell spend - sphere spice spider spirit spite split - spoon sport spot spray spread spring - squab square squash stable staff stage - stand staple star start state status - stay steak steel step stern stew - stick still stock stone stop store - storm story strain street stress strike - string stroke strong studio study stuff - style sugar suit sulfur summer sun - sunny sunset super superb surf survey - sweet swim swing switch symbol system - table tackle tail tale talk tan - tank tap tape target task taste - tau tea teach teal team tear - tell ten tender tennis tent term - test tetra text thanks theme theory - thing think thread throat thumb ticket - tidy tie tiger till time timely - tin tip title toast today toe - tomato tone tongue tool tooth top - topic total touch tough tour towel - tower town track trade train trash - travel tray treat tree trick trip - trout trowel truck trupet trust truth - try tube tuna tune turf turkey - turn turnip tutor tux tweet twist - two type union unique unit upbeat - upper use useful user usual valley - value van vase vast veil vein - velvet verse very vessel vest video - view violet visit visual vivid voice - volume vowel voyage waffle wait wake - walk wall warm warmth wasabi wash - watch water wave wax way wealth - wear web wedge week weekly weight - west whale what wheat wheel when - where while who whole why will - win wind window wing winner winter - wire wish witty wolf wonder wood - wool woolly word work worker world - worry worth worthy wrap wrench wrist - writer xenon yak yam yard yarrow - year yearly yellow yew yogurt young + ability able abroad access account act + action active actor add address adept + adroit advance advice affect age ageless + agency agent agile agree air airfare + airline airport alert almond alpha always + amend amount amplify analyst anchor angel + angelic angle ankle annual answer antique + anybody anyhow appeal apple apricot apt + area argon arm army arrival arsenic + art artful article arugula aside ask + aspect assist assume atom atone attempt + author autumn average avocado award awl + azure back bacon bag bagel bake + baker balance ball balloon bamboo banana + band banjo bank barium base basil + basin basis basket bass bat bath + battery beach beak bean bear bearcub + beauty beef beet beige being bell + belly belt bench bend benefit best + beta better beyond bicycle bid big + bike bill bird biscuit bismuth bisque + bit black blank blest blind bliss + block bloom blue board boat body + bokchoy bone bonus book bookish boot + border boron boss bossy bottle bottom + bow bowl bowtie box brain brainy + branch brave bravely bread break breath + breezy brick bridge brie brief briefly + bright broad broil bromine bronze brother + brow brown brush buddy budget buffalo + bug bugle bull bunch burger burly + burrito bus busy butter button buy + buyer byte cab cabbage cabinet cable + cadet cadmium caesium cake calcium caliper + call caller calm calmly camera camp + can canary cancel candle candy cap + capable caper capital captain car carbon + card care career careful carp carpet + carrot carry case cash cassava casual + cat catch catfish catsear catsup cause + cave celery cell century chain chair + chalk chance change channel chapter chard + charge charity chart check cheddar cheery + cheese chicken chicory chiffon child chin + chip chives choice chowder chum church + circle city claim clam class classic + classy clay clean cleaner clear clearly + clerk click client climate clock clorine + closet clothes cloud clown club clue + cluster coach coast coat cobbler cobolt + cod code coffee colby cold collar + college comb combine comet comfort command + comment common company complex concept concern + concert conduit consist contact contest context + control convert cook cookie copilot copper + copy coral cordial corn corner corny + correct cost count counter country county + couple courage course court cover cow + cowbird crab crack craft crash crazy + cream credit creek cress crevice crew + crimson croaker crop cross crowd cube + cuckoo cuisine culture cup current curve + cut cyan cycle dagger daily dance + dare darter data date day daylily + deal dear dearly debate debit decade + decimal deep deft deftly degree delay + deluxe deposit depth design desk detail + device dew diamond diet dig dill + dinner dip direct dirt dish disk + display diver divide divine doctor dodger + donut door dot double dough draft + drag dragon drama draw drawer drawing + dream drill drink drive driver drop + drum dry dryer drywall duck due + dump dusk dust duty dye eagle + ear earring earth ease east easy + eat economy edge editor eel effect + effort egg eight elbow elegant element + elf elk email emerald employ end + endive endless energy engine enjoy enter + entry equal equip error escape essay + eternal evening event exam example excuse + exit expert extent extreme eye face + fact factor factual fail failure fair + fajita fall family fan fang farm + farmer fat fault feature feed feel + feeling fench fennel festive few fiber + field fig figure file fill film + filter final finance finding finger finish + fire fish fishing fit fitting five + fix flier flight floor floral florine + flour flow flower fly flying focus + fold folding food foot force forest + forever forgive form formal format fortune + forum frame free freedom freely fresh + friend frog front fruit fuchsia fuel + fun funny future gain galaxy gallium + game gamma gap garage garden garlic + gas gate gather gauge gear gem + gene general gentle gently gherkin ghost + gift give glad glass gleeful glossy + glove glue goal goat goby gold + goldeye golf good gouda goulash gourd + grab grace grade gram grand grape + grapes grass gravy gray great green + grits grocery ground group grouper grout + growth guard guave guess guest guide + guitar gumbo guppy habit hacksaw haddock + hafnium hagfish hair half halibut hall + hammer hand handle handy hanger happy + hat havarti hay haybale head health + healthy hearing heart hearty heat heavy + heel height helium hello help helpful + herald herring hide high highly highway + hill hip hipster hire history hit + hoki hold hole holiday holly home + honest honey hook hope hopeful horizon + horn horse host hotel hour house + housing human humane humor hunt hurry + ice icecube icefish icy idea ideal + image impact impress inch income indigo + initial inkpen insect inside intense invite + iodine iridium iron island issue item + ivory jacket jargon javelin jello jelly + jewel job jocund join joint joke + jovial joy joyful joyous judge juice + jump junior jury just justice kale + keel keep kelp ketchup key keyhole + keyway khaki kick kid kidney kiloohm + kind kindly king kitchen kite kiwi + knee knife krill krypton kumquat lab + lace lack ladder lake lamp lamprey + land laser laugh law lawn lawyer + layer lead leader leading leaf leafy + league leather leave lecture leek leg + lemon length lentil lesson let letter + lettuce level library life lift light + lily lime limit line linen link + lip list listen lithium lively living + lizard load loan lobster local lock + log long longfin look lotus love + lovely loving low lucid luck luffa + lunch lung machine magenta magnet mail + main major make mall manager mango + manner many map march market maroon + martian master match math matter maximum + maybe meal meaning meat media medium + meet meeting melody melon member memory + mention menu mercury merry mess message + messy metal meter method micron middle + might mile milk mind mine minimum + minnow minor mint minute mirror miss + mission misty mix mixer mixture mobile + mode model moment monitor monk month + moon moray morning most motor mouse + mouth move mover movie much mud + mudfish muffin mullet munster muon muscle + music mustard nail name nation native + natural nature navy neat neatly nebula + neck needle neon nerve net network + neutron news nibble nice nickel night + niobium nobody noise noodle normal north + nose note nothing notice nova novel + number nurse nursery oar object offer + office officer oil okay okra old + olive one onion open opening opinion + option orange orbit orchid order oregano + other ounce outcome outside oven owner + oxygen oyster pace pack package page + pager paint pair pale pan pancake + papaya paper pardon parent park parking + parsley parsnip part partner party pass + passage past pasta path patient pattern + pause pay pea peace peach peacock + peahen peak peanut pear pearl pen + penalty pencil pension people pepper perch + perfect period permit person phase phone + photo phrase physics piano pick picture + pie piece pigeon pike pilot pin + pink pinkie pious pipe pitch pizza + place plan plane planet plant planter + plastic plate play player playful plenty + pliers plum pod poem poet poetry + point police policy pollock pony pool + pop popover poptart pork port portal + post pot potato pound powder power + present press price pride primary print + prior private prize problem process produce + product profile profit program project promise + prompt proof proper protein proton public + puff puffer pull pumpkin pup pupfish + pure purple purpose push put quality + quark quarter quiet quill quit quote + rabbit raccoon race radiant radio radish + radium radon rain rainbow raise ramp + ranch range rasp rate ratio ray + razor reach read reading real reality + reason recipe record recover red redeem + reed reef refuse region regret regular + relaxed release relief relish remote remove + rent repair repeat reply report request + reserve resist resolve resort rest result + return reveal review reward ribbon rice + rich ride ridge right ring rise + risk river rivet road roast rock + rocket role roll roof room rope + rose rough roughy round row royal + rub ruby rudder ruin rule run + runner rush rust sacred saddle safe + safety sail salad salami sale salmon + salt sample sand sander sandy sauce + save saving saw scale scampi scene + scheme school score screen script sea + search season seat second secret sector + seemly self sell senate senior sense + series serve set shake shape share + shark shell shift shine shiny ship + shock shoe shoot shop shovel show + side sign signal silk silly silver + simple sing singer single sink site + size skill skin sky slate sleep + sleepy slice slide slip smart smell + smelt smile smoke smooth snap snipe + snow snowy sock socket sodium soft + softly soil sole solid song sorrel + sort soul sound soup source south + space spare speech speed spell spend + sphere spice spider spirit spite split + spoon sport spot spray spread spring + squab square squash stable staff stage + stand staple star start state status + stay steak steel step stern stew + stick still stock stone stop store + storm story strain street stress strike + string stroke strong studio study stuff + style sugar suit sulfur summer sun + sunny sunset super superb surf survey + sweet swim swing switch symbol system + table tackle tail tale talk tan + tank tap tape target task taste + tau tea teach teal team tear + tell ten tender tennis tent term + test tetra text thanks theme theory + thing think thread throat thumb ticket + tidy tie tiger till time timely + tin tip title toast today toe + tomato tone tongue tool tooth top + topic total touch tough tour towel + tower town track trade train trash + travel tray treat tree trick trip + trout trowel truck trupet trust truth + try tube tuna tune turf turkey + turn turnip tutor tux tweet twist + two type union unique unit upbeat + upper use useful user usual valley + value van vase vast veil vein + velvet verse very vessel vest video + view violet visit visual vivid voice + volume vowel voyage waffle wait wake + walk wall warm warmth wasabi wash + watch water wave wax way wealth + wear web wedge week weekly weight + west whale what wheat wheel when + where while who whole why will + win wind window wing winner winter + wire wish witty wolf wonder wood + wool woolly word work worker world + worry worth worthy wrap wrench wrist + writer xenon yak yam yard yarrow + year yearly yellow yew yogurt young youth zebra zephyr zinc zone zoo } set nwordlist [llength $wordlist] From 272ae627c535b1ee11b82ba045c641945e34bdc5 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 28 Apr 2023 14:48:11 +0000 Subject: [PATCH 305/341] Add the json_error(X) function that returns the 1-based character offset to the first syntax error in JSON5 string X, or 0 if there are no errors. FossilOrigin-Name: 901ad995d5a722ca2672516205ff488e9acd703a828ca5fc43f11fca5f2af120 --- manifest | 12 +++--- manifest.uuid | 2 +- src/json.c | 115 +++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 111 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index e9188432cd..77de439258 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sindentation\sand\somit\strailing\swhitespace\sin\sthe\srandom\sJSON\sgenerator\nscript. -D 2023-04-28T13:25:35.641 +C Add\sthe\sjson_error(X)\sfunction\sthat\sreturns\sthe\s1-based\scharacter\soffset\sto\nthe\sfirst\ssyntax\serror\sin\sJSON5\sstring\sX,\sor\s0\sif\sthere\sare\sno\serrors. +D 2023-04-28T14:48:11.699 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -593,7 +593,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 7d03a1185d7aee74f53a3f2381355ff4b71d955f287d29d30c0b0979afc96b67 +F src/json.c b532d42d310e570d7f620a692a26f21dc7306063b866088c4e43c647a17118b3 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2066,8 +2066,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f56528d413d8e622f7c4f18b2f9f2e620bfb441c020461299b35a90072ee6c13 -R 5da141bb06598ed103d496559df7ae36 +P 629db09fceb7bf37561b52ccee06ebf4df261291e9a8ffcca82b243f6db5ff07 +R cda10fe16274848e02e4d5ea3713e4b9 U drh -Z 790f8e2b554134402e5889a19faba52a +Z 80f17384e6d82e3c8657040553f2124e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d1f6aa531d..0b6e554ed2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -629db09fceb7bf37561b52ccee06ebf4df261291e9a8ffcca82b243f6db5ff07 \ No newline at end of file +901ad995d5a722ca2672516205ff488e9acd703a828ca5fc43f11fca5f2af120 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 1152d9f916..d1a057b48e 100644 --- a/src/json.c +++ b/src/json.c @@ -1073,7 +1073,10 @@ json_parse_restart: iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); if( iThis<0 ) return -1; for(j=i+1;;j++){ - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = j; + return -1; + } x = jsonParseValue(pParse, j); if( x<=0 ){ if( x==(-2) ){ @@ -1090,6 +1093,7 @@ json_parse_restart: pParse->has5 = 1; x = k; }else{ + pParse->iErr = j; return -1; } } @@ -1109,13 +1113,19 @@ json_parse_restart: } } x = jsonParseValue(pParse, j); - if( x!=(-5) ) return -1; + if( x!=(-5) ){ + pParse->iErr = j; + return -1; + } j = pParse->iErr+1; } parse_object_value: x = jsonParseValue(pParse, j); pParse->iDepth--; - if( x<=0 ) return -1; + if( x<=0 ){ + pParse->iErr = j; + return -1; + } j = x; if( z[j]==',' ){ continue; @@ -1140,6 +1150,7 @@ json_parse_restart: break; } } + pParse->iErr = j; return -1; } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; @@ -1151,7 +1162,10 @@ json_parse_restart: if( iThis<0 ) return -1; memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); for(j=i+1;;j++){ - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = j; + return -1; + } x = jsonParseValue(pParse, j); pParse->iDepth--; if( x<=0 ){ @@ -1160,6 +1174,7 @@ json_parse_restart: if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; break; } + pParse->iErr = j; return -1; } j = x; @@ -1186,6 +1201,7 @@ json_parse_restart: break; } } + pParse->iErr = j; return -1; } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; @@ -1207,6 +1223,7 @@ json_parse_restart: c = z[j]; if( (c & ~0x1f)==0 ){ /* Control characters are not allowed in strings */ + pParse->iErr = j; return -1; } if( c=='\\' ){ @@ -1227,6 +1244,7 @@ json_parse_restart: jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); pParse->has5 = 1; }else{ + pParse->iErr = j; return -1; } }else if( c==cDelim ){ @@ -1242,6 +1260,7 @@ json_parse_restart: jsonParseAddNode(pParse, JSON_NULL, 0, 0); return i+4; } + pParse->iErr = i; return -1; } case 't': { @@ -1249,6 +1268,7 @@ json_parse_restart: jsonParseAddNode(pParse, JSON_TRUE, 0, 0); return i+4; } + pParse->iErr = i; return -1; } case 'f': { @@ -1256,6 +1276,7 @@ json_parse_restart: jsonParseAddNode(pParse, JSON_FALSE, 0, 0); return i+5; } + pParse->iErr = i; return -1; } case '+': { @@ -1271,6 +1292,7 @@ json_parse_restart: seenDP = JSON_REAL; goto parse_number_2; } + pParse->iErr = i; return -1; case '-': case '0': @@ -1338,6 +1360,7 @@ json_parse_restart: jnFlags |= JNODE_JSON5; goto parse_number_2; } + pParse->iErr = i; return -1; } if( z[i+1]=='0' ){ @@ -1358,7 +1381,10 @@ json_parse_restart: c = z[j]; if( sqlite3Isdigit(c) ) continue; if( c=='.' ){ - if( seenDP==JSON_REAL ) return -1; + if( seenDP==JSON_REAL ){ + pParse->iErr = j; + return -1; + } seenDP = JSON_REAL; continue; } @@ -1368,10 +1394,14 @@ json_parse_restart: pParse->has5 = 1; jnFlags |= JNODE_JSON5; }else{ + pParse->iErr = j; return -1; } } - if( seenE ) return -1; + if( seenE ){ + pParse->iErr = j; + return -1; + } seenDP = JSON_REAL; seenE = 1; c = z[j+1]; @@ -1379,7 +1409,10 @@ json_parse_restart: j++; c = z[j+1]; } - if( c<'0' || c>'9' ) return -1; + if( c<'0' || c>'9' ){ + pParse->iErr = j; + return -1; + } continue; } break; @@ -1389,6 +1422,7 @@ json_parse_restart: pParse->has5 = 1; jnFlags |= JNODE_JSON5; }else{ + pParse->iErr = j; return -1; } } @@ -1402,6 +1436,7 @@ json_parse_restart: pParse->has5 = 1; return i+3; } + pParse->iErr = i; return -1; } case 'I': { @@ -1410,6 +1445,7 @@ json_parse_restart: pParse->has5 = 1; return i+8; } + pParse->iErr = i; return -1; } case '}': { @@ -1453,6 +1489,7 @@ json_parse_restart: pParse->has5 = 1; goto json_parse_restart; } + pParse->iErr = i; return -1; } default: { @@ -1471,6 +1508,7 @@ json_parse_restart: return i + nn; } #endif + pParse->iErr = i; return -1; /* Syntax error */ } } /* End switch(z[i]) */ @@ -1575,6 +1613,15 @@ static int jsonParseFindParents(JsonParse *pParse){ ** is no longer valid, parse the JSON again and return the new parse, ** and also register the new parse so that it will be available for ** future sqlite3_get_auxdata() calls. +** +** If an error occurs and pErrCtx!=0 then report the error on pErrCtx +** and return NULL. +** +** If an error occurs and pErrCtx==0 then return the Parse object with +** JsonParse.nErr non-zero. If the caller invokes this routine with +** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller +** is responsible for invoking jsonParseFree() on the returned value. +** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0. */ static JsonParse *jsonParseCached( sqlite3_context *pCtx, @@ -1624,6 +1671,10 @@ static JsonParse *jsonParseCached( p->zJson = (char*)&p[1]; memcpy((char*)p->zJson, zJson, nJson+1); if( jsonParse(p, pErrCtx, p->zJson) ){ + if( pErrCtx==0 ){ + p->nErr = 1; + return p; + } sqlite3_free(p); return 0; } @@ -2501,8 +2552,8 @@ static void jsonTypeFunc( /* ** json_valid(JSON) ** -** Return 1 if JSON is a well-formed JSON string according to RFC-7159. -** Return 0 otherwise. +** Return 1 if JSON is a well-formed canonical JSON string according +** to RFC-7159. Return 0 otherwise. */ static void jsonValidFunc( sqlite3_context *ctx, @@ -2512,8 +2563,17 @@ static void jsonValidFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); p = jsonParseCached(ctx, argv, 0); - sqlite3_result_int(ctx, p!=0 && p->has5==0); + sqlite3_result_int(ctx, p!=0 && p->nErr==0 && p->has5==0); + if( p!=0 && p->nErr ) jsonParseFree(p); } + + +/* +** json_valid5(JSON) +** +** Return 1 if JSON is a well-formed JSON5 string. +** Return 0 otherwise. +*/ static void jsonValid5Func( sqlite3_context *ctx, int argc, @@ -2522,7 +2582,39 @@ static void jsonValid5Func( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); p = jsonParseCached(ctx, argv, 0); - sqlite3_result_int(ctx, p!=0); + sqlite3_result_int(ctx, p!=0 && p->nErr==0); + if( p!=0 && p->nErr ) jsonParseFree(p); +} + + + +/* +** json_error(JSON) +** +** If JSON is not a well-formed JSON5 string, then return the 1-based +** character offset to the location of the first error in that string. +** Return 0 otherwise. +*/ +static void jsonErrorFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + UNUSED_PARAMETER(argc); + p = jsonParseCached(ctx, argv, 0); + if( p==0 || p->oom || p->nErr==0 ){ + sqlite3_result_int(ctx, 0); + }else{ + int n = 0; + int i; + const char *z = p->zJson; + for(i=0; iiErr && z[i]; i++){ + if( (z[i]&0xc0)!=80 ) n++; + } + sqlite3_result_int(ctx, n); + jsonParseFree(p); + } } @@ -3235,6 +3327,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_array, -1, 0, jsonArrayFunc), JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), + JFUNCTION(json_error, 1, 0, jsonErrorFunc), JFUNCTION(json_extract, -1, 0, jsonExtractFunc), JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), From 8e7efe37645453fac5ca98f8a9837f34d14f75b6 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 28 Apr 2023 17:38:35 +0000 Subject: [PATCH 306/341] Improvements to the accuracy of json_error(). Add the extension SQL functions random_json(SEED) and random_json5(SEED). FossilOrigin-Name: 8d09dc1c45a8026b94f70273d064e47939f30cadedc17548b5a26ba054a8d3a7 --- ext/misc/randomjson.c | 202 ++++++++++++++++++++++++++++++++++++++++++ manifest | 13 +-- manifest.uuid | 2 +- src/json.c | 8 +- 4 files changed, 214 insertions(+), 11 deletions(-) create mode 100644 ext/misc/randomjson.c diff --git a/ext/misc/randomjson.c b/ext/misc/randomjson.c new file mode 100644 index 0000000000..3a6f545fe6 --- /dev/null +++ b/ext/misc/randomjson.c @@ -0,0 +1,202 @@ +/* +** 2023-04-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements a the random_json(SEED) and +** random_json5(SEED) functions. Given a numeric SEED value, these +** routines generate pseudo-random JSON or JSON5, respectively. The +** same value is always generated for the same seed. +** +** These SQL functions are intended for testing. They do not have any +** practical real-world use, that we know of. +** +** COMPILE: +** +** gcc --shared -fPIC -o randomjson.so -I. ext/misc/randomjson.c +** +** USING FROM THE CLI: +** +** .load ./randomjson +** SELECT random_json(1); +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* Pseudo-random number generator */ +typedef struct Prng { + unsigned int x, y; +} Prng; + +/* Reseed the PRNG */ +static void prngSeed(Prng *p, unsigned int iSeed){ + p->x = iSeed | 1; + p->y = iSeed; +} + +/* Extract a random number */ +static unsigned int prngInt(Prng *p){ + p->x = (p->x>>1) ^ ((1+~(p->x&1)) & 0xd0000001); + p->y = p->y*1103515245 + 12345; + return p->x ^ p->y; +} + +static const char *azJsonAtoms[] = { + /* JSON /* JSON-5 */ + "0", "0", + "1", "1", + "-1", "-1", + "2", "+2", + "3", "3", + "2.5", "2.5", + "0.75", ".75", + "-4.0e2", "-4.e2", + "5.0e-3", "+5e-3", + "0", "0x0", + "512", "0x200", + "256", "+0x100", + "-2748", "-0xabc", + "true", "true", + "false", "false", + "null", "null", + "9.0e999", "Infinity", + "-9.0e999", "-Infinity", + "9.0e999", "+Infinity", + "null", "NaN", + "-0.0005123", "-0.0005123", + "4.35e-3", "+4.35e-3", + "\"gem\\\"hay\"", "\"gem\\\"hay\"", + "\"icy'joy\"", "'icy\\'joy\'", + "\"keylog\"", "\"key\\\nlog\"", + "\"mix\\\\\\tnet\"", "\"mix\\\\\\tnet\"", + "{}", "{}", + "[]", "[]", + "[]", "[/*empty*/]", + "{}", "{//empty\n}", + "\"ask\"", "\"ask\"", + "\"bag\"", "\"bag\"", + "\"can\"", "\"can\"", + "\"day\"", "\"day\"", + "\"end\"", "'end'", + "\"fly\"", "\"fly\"", + "\"\"", "\"\"", +}; +static const char *azJsonTemplate[] = { + /* JSON JSON-5 */ + "{\"a\":%,\"b\":%,\"c\":%}", "{a:%,b:%,c:%}", + "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"e\":%}", "{a:%,b:%,c:%,d:%,e:%}", + "{\"a\":%,\"b\":%,\"c\":%,\"d\":%,\"\":%}", "{a:%,b:%,c:%,d:%,\"\":%}", + "{\"d\":%}", "{d:%}", + "{\"eeee\":%, \"ffff\":%}", "{eeee:% /*and*/, ffff:%}", + "{\"$g\":%,\"_h_\":%}", "{$g:%,_h_:%,}", + "{\"x\":%,\n \"y\":%}", "{\"x\":%,\n \"y\":%}", + "{\"a b c d\":%,\"e\":%,\"f\":%,\"x\":%,\"y\":%}", + "{\"a b c d\":%,e:%,f:%,x:%,y:%}", + "{\"Z\":%}", "{Z:%,}", + "[%]", "[%,]", + "[%,%]", "[%,%]", + "[%,%,%]", "[%,%,%,]", + "[%,%,%,%]", "[%,%,%,%]", + "[%,%,%,%,%]", "[%,%,%,%,%]", +}; + +#define count(X) (sizeof(X)/sizeof(X[0])) + +#define STRSZ 10000 + +static void jsonExpand( + const char *zSrc, + char *zDest, + Prng *p, + int eType, /* 0 for JSON, 1 for JSON5 */ + unsigned int r /* Growth probability 0..1000. 0 means no growth */ +){ + unsigned int i, j, k; + const char *z; + size_t n; + + j = 0; + if( zSrc==0 ){ + k = prngInt(p)%(count(azJsonTemplate)/2); + k = k*2 + eType; + zSrc = azJsonTemplate[k]; + } + if( strlen(zSrc)>=STRSZ/10 ) r = 0; + for(i=0; zSrc[i]; i++){ + if( zSrc[i]!='%' ){ + if( jhas5 = 1; x = k; }else{ - pParse->iErr = j; + if( x!=-1 ) pParse->iErr = j; return -1; } } @@ -1114,7 +1114,7 @@ json_parse_restart: } x = jsonParseValue(pParse, j); if( x!=(-5) ){ - pParse->iErr = j; + if( x!=(-1) ) pParse->iErr = j; return -1; } j = pParse->iErr+1; @@ -1123,7 +1123,7 @@ json_parse_restart: x = jsonParseValue(pParse, j); pParse->iDepth--; if( x<=0 ){ - pParse->iErr = j; + if( x!=(-1) ) pParse->iErr = j; return -1; } j = x; @@ -1174,7 +1174,7 @@ json_parse_restart: if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; break; } - pParse->iErr = j; + if( x!=(-1) ) pParse->iErr = j; return -1; } j = x; From b0d46fcd72feb143d1ad5d585df943c7a4bc5ee3 Mon Sep 17 00:00:00 2001 From: larrybr Date: Fri, 28 Apr 2023 21:25:03 +0000 Subject: [PATCH 307/341] Revise generate_series() extension (in CLI) to address overflow reported in [forum:754e2d4db2a5|forum post #754e2d4db2a5] and to make behavior better match the like-named PostgreSQL function. FossilOrigin-Name: beeea3e1b010dace9789f27172462b912819d0f8142a67e3e1e7335211e0e9a8 --- ext/misc/series.c | 182 ++++++++++++++++++++++++++++++++++------------ manifest | 17 +++-- manifest.uuid | 2 +- 3 files changed, 146 insertions(+), 55 deletions(-) diff --git a/ext/misc/series.c b/ext/misc/series.c index 3941d96c47..f484cba2dc 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -1,5 +1,5 @@ /* -** 2015-08-18 +** 2015-08-18, 2023-04-28 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -12,7 +12,19 @@ ** ** This file demonstrates how to create a table-valued-function using ** a virtual table. This demo implements the generate_series() function -** which gives similar results to the eponymous function in PostgreSQL. +** which gives the same results as the eponymous function in PostgreSQL, +** within the limitation that its arguments are signed 64-bit integers. +** +** Considering its equivalents to generate_series(start,stop,step): A +** value V[n] sequence is produced for integer n ascending from 0 where +** ( V[n] == start + n * step && sgn(V[n] - stop) * sgn(step) >= 0 ) +** for each produced value (independent of production time ordering.) +** +** All parameters must be either integer or convertable to integer. +** The start parameter is required. +** The stop parameter defaults to (1<<32)-1 (aka 4294967295 or 0xffffffff) +** The step parameter defaults to 1 and 0 is treated as 1. +** ** Examples: ** ** SELECT * FROM generate_series(0,100,5); @@ -28,6 +40,14 @@ ** ** Integers 20 through 29. ** +** SELECT * FROM generate_series(0,-100,-5); +** +** Integers 0 -5 -10 ... -100. +** +** SELECT * FROM generate_series(0,-1); +** +** Empty sequence. +** ** HOW IT WORKS ** ** The generate_series "function" is really a virtual table with the @@ -40,6 +60,9 @@ ** step HIDDEN ** ); ** +** The virtual table also has a rowid, logically equivalent to n+1 where +** "n" is the ascending integer in the aforesaid production definition. +** ** Function arguments in queries against this virtual table are translated ** into equality constraints against successive hidden columns. In other ** words, the following pairs of queries are equivalent to each other: @@ -72,9 +95,93 @@ SQLITE_EXTENSION_INIT1 #include #include +#include #ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return that member of a generate_series(...) sequence whose 0-based +** index is ix. The 0th member is given by smBase. The sequence members +** progress per ix increment by smStep. +*/ +static sqlite3_int64 genSeqMember(sqlite3_int64 smBase, + sqlite3_int64 smStep, + sqlite3_uint64 ix){ + if( ix>=(sqlite3_uint64)LLONG_MAX ){ + /* Get ix into signed i64 range. */ + ix -= (sqlite3_uint64)LLONG_MAX; + smBase += LLONG_MAX * smStep; + } + return smBase + ((sqlite3_int64)ix)*smStep; +} +typedef unsigned char u8; + +typedef struct SequenceSpec { + sqlite3_int64 iBase; /* Starting value ("start") */ + sqlite3_int64 iTerm; /* Given terminal value ("stop") */ + sqlite3_int64 iStep; /* Increment ("step") */ + sqlite3_uint64 uMaxRowidM1; /* maximum rowid minus 1 */ + sqlite3_uint64 uRidCurrent; /* Current rowid-1 during generation */ + sqlite3_int64 iValueCurrent; /* Current value during generation */ + u8 isNotEOF; /* Sequence generation not exhausted */ + u8 isReversing; /* Sequence is being reverse generated */ +} SequenceSpec; + +/* +** Prepare a SequenceSpec for use in generating an integer series +** given initialized iBase, iTerm and iStep values. Sequence is +** initialized per given isReversing. Other members are computed. +*/ +void setupSequence( SequenceSpec *pss ){ + pss->uMaxRowidM1 = 0; + pss->isNotEOF = 0; + if( pss->iTerm < pss->iBase ){ + sqlite3_uint64 nuspan = (sqlite3_uint64)(pss->iBase-pss->iTerm); + if( pss->iStep<0 ){ + pss->isNotEOF = 1; + if( nuspan==ULONG_MAX ){ + pss->uMaxRowidM1 = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; + }else if( pss->iStep>LLONG_MIN ){ + pss->uMaxRowidM1 = nuspan/-pss->iStep; + } + } + }else if( pss->iTerm > pss->iBase ){ + sqlite3_uint64 puspan = (sqlite3_uint64)(pss->iTerm-pss->iBase); + if( pss->iStep>0 ){ + pss->isNotEOF = 1; + pss->uMaxRowidM1 = puspan/pss->iStep; + } + } + pss->uRidCurrent = (pss->isReversing)? pss->uMaxRowidM1 : 0; + pss->iValueCurrent = (pss->isReversing) + ? genSeqMember(pss->iBase, pss->iStep, pss->uMaxRowidM1) + : pss->iBase; +} + +/* +** Progress sequence generator to yield next value, if any. +** Leave its state to either yield next value or be at EOF. +** Return whether there is a next value, or 0 at EOF. +*/ +int progressSequence( SequenceSpec *pss ){ + if( !pss->isNotEOF ) return 0; + if( pss->isReversing ){ + if( pss->uRidCurrent > 0 ){ + pss->uRidCurrent--; + pss->iValueCurrent -= pss->iStep; + }else{ + pss->isNotEOF = 0; + } + }else{ + if( pss->uRidCurrent < pss->uMaxRowidM1 ){ + pss->uRidCurrent++; + pss->iValueCurrent += pss->iStep; + }else{ + pss->isNotEOF = 0; + } + } + return pss->isNotEOF; +} /* series_cursor is a subclass of sqlite3_vtab_cursor which will ** serve as the underlying representation of a cursor that scans @@ -83,12 +190,7 @@ SQLITE_EXTENSION_INIT1 typedef struct series_cursor series_cursor; struct series_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - int isDesc; /* True to count down rather than up */ - sqlite3_int64 iRowid; /* The rowid */ - sqlite3_int64 iValue; /* Current value ("value") */ - sqlite3_int64 mnValue; /* Mimimum value ("start") */ - sqlite3_int64 mxValue; /* Maximum value ("stop") */ - sqlite3_int64 iStep; /* Increment ("step") */ + SequenceSpec ss; /* (this) Derived class data */ }; /* @@ -170,12 +272,7 @@ static int seriesClose(sqlite3_vtab_cursor *cur){ */ static int seriesNext(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - if( pCur->isDesc ){ - pCur->iValue -= pCur->iStep; - }else{ - pCur->iValue += pCur->iStep; - } - pCur->iRowid++; + progressSequence( & pCur->ss ); return SQLITE_OK; } @@ -191,10 +288,10 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->mnValue; break; - case SERIES_COLUMN_STOP: x = pCur->mxValue; break; - case SERIES_COLUMN_STEP: x = pCur->iStep; break; - default: x = pCur->iValue; break; + case SERIES_COLUMN_START: x = pCur->ss.iBase; break; + case SERIES_COLUMN_STOP: x = pCur->ss.iTerm; break; + case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; + default: x = pCur->ss.iValueCurrent; break; } sqlite3_result_int64(ctx, x); return SQLITE_OK; @@ -207,7 +304,7 @@ static int seriesColumn( */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - *pRowid = pCur->iRowid; + *pRowid = ((sqlite3_int64)pCur->ss.uRidCurrent + 1); return SQLITE_OK; } @@ -217,14 +314,10 @@ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int seriesEof(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - if( pCur->isDesc ){ - return pCur->iValue < pCur->mnValue; - }else{ - return pCur->iValue > pCur->mxValue; - } + return !pCur->ss.isNotEOF; } -/* True to cause run-time checking of the start=, stop=, and/or step= +/* True to cause run-time checking of the start=, stop=, and/or step= ** parameters. The only reason to do this is for testing the ** constraint checking logic for virtual tables in the SQLite core. */ @@ -235,7 +328,7 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ /* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least -** once prior to any call to seriesColumn() or seriesRowid() or +** once prior to any call to seriesColumn() or seriesRowid() or ** seriesEof(). ** ** The query plan selected by seriesBestIndex is passed in the idxNum @@ -255,7 +348,7 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ ** (so that seriesEof() will return true) if the table is empty. */ static int seriesFilter( - sqlite3_vtab_cursor *pVtabCursor, + sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStrUnused, int argc, sqlite3_value **argv ){ @@ -263,46 +356,41 @@ static int seriesFilter( int i = 0; (void)idxStrUnused; if( idxNum & 1 ){ - pCur->mnValue = sqlite3_value_int64(argv[i++]); + pCur->ss.iBase = sqlite3_value_int64(argv[i++]); }else{ - pCur->mnValue = 0; + pCur->ss.iBase = 0; } if( idxNum & 2 ){ - pCur->mxValue = sqlite3_value_int64(argv[i++]); + pCur->ss.iTerm = sqlite3_value_int64(argv[i++]); }else{ - pCur->mxValue = 0xffffffff; + pCur->ss.iTerm = 0xffffffff; } if( idxNum & 4 ){ - pCur->iStep = sqlite3_value_int64(argv[i++]); - if( pCur->iStep==0 ){ - pCur->iStep = 1; - }else if( pCur->iStep<0 ){ - pCur->iStep = -pCur->iStep; + pCur->ss.iStep = sqlite3_value_int64(argv[i++]); + if( pCur->ss.iStep==0 ){ + pCur->ss.iStep = 1; + }else if( pCur->ss.iStep<0 ){ if( (idxNum & 16)==0 ) idxNum |= 8; } }else{ - pCur->iStep = 1; + pCur->ss.iStep = 1; } for(i=0; imnValue = 1; - pCur->mxValue = 0; + pCur->ss.iBase = 1; + pCur->ss.iTerm = 0; + pCur->ss.iStep = 1; break; } } if( idxNum & 8 ){ - pCur->isDesc = 1; - pCur->iValue = pCur->mxValue; - if( pCur->iStep>0 ){ - pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep; - } + pCur->ss.isReversing = pCur->ss.iStep > 0; }else{ - pCur->isDesc = 0; - pCur->iValue = pCur->mnValue; + pCur->ss.isReversing = pCur->ss.iStep < 0; } - pCur->iRowid = 1; + setupSequence( &pCur->ss ); return SQLITE_OK; } diff --git a/manifest b/manifest index cbe41d4e9d..6ae7a8b8e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sa\snew\stechnique\sto\sdetect\sfresh\sOOM\sfaults\sin\scolumnName()\sthat\sdoes\snot\nrely\son\sthere\sbeing\sno\sOOMs\sprior\sto\sentry\sinto\scolumnName(),\sas\n[forum/forumpost/fb6811c2f9|forum\spost\sfb6811c2f9]\sdemonstrates\sa\stechnique\nwhich\scould\scause\san\sOOM\sprior\sto\sentry\sinto\scolumnName(). -D 2023-04-27T23:59:51.813 +C Revise\sgenerate_series()\sextension\s(in\sCLI)\sto\saddress\soverflow\sreported\sin\s[forum:754e2d4db2a5|forum\spost\s#754e2d4db2a5]\sand\sto\smake\sbehavior\sbetter\smatch\sthe\slike-named\sPostgreSQL\sfunction. +D 2023-04-28T21:25:03.297 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -305,7 +305,7 @@ F ext/misc/regexp.c f50ab59bfa8934b7ed98de069d2c74c187f2ef523fb09e85f8840f6459a9 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c 8d79354f2c3d46b95ee21272a07cf0bcabb58d1f2b06d9e7b8a31dca1dacb3e5 +F ext/misc/series.c 9553821737ea55abcf534b22a63e9d29c2c9c07c1f82dd56cfa2b18e4d59d046 F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d F ext/misc/shathree.c 543af7ce71d391cd3a9ab6924a6a1124efc63211fd0f2e240dc4b56077ba88ac F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 @@ -2060,8 +2060,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4bbebb6bfb9910265d91b777c1711b3b8e0732bcf299f7459b20c4ea110422bd -R 59c8d7a772734f630b5bf6fd205de302 -U drh -Z d29ba302c14b4aed5db886f9a3fc4bd5 +P a63346d6a0c0ca7ba4c87499de2e461be9c77e9b5d98f2bebf308cdb6599f33c +R 13e2ac79f26c72ca3ffff8649a5a99a9 +T *branch * generate_series-revamp +T *sym-generate_series-revamp * +T -sym-trunk * +U larrybr +Z d68099c137797424f3a3da0f35ed98b8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b9ba76d6e8..9ae26b7d44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a63346d6a0c0ca7ba4c87499de2e461be9c77e9b5d98f2bebf308cdb6599f33c \ No newline at end of file +beeea3e1b010dace9789f27172462b912819d0f8142a67e3e1e7335211e0e9a8 \ No newline at end of file From 6e2de57963198789e6df238962351fa4b840851e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 28 Apr 2023 23:19:11 +0000 Subject: [PATCH 308/341] Bug fix in the offset computation for json_error(). FossilOrigin-Name: 8f7ab5d9210490eb451534df246b82081a5aa252a1bd662659df3da429290837 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c10ebff82d..0d0cafc8eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\saccuracy\sof\sjson_error().\s\sAdd\sthe\sextension\sSQL\nfunctions\srandom_json(SEED)\sand\srandom_json5(SEED). -D 2023-04-28T17:38:35.385 +C Bug\sfix\sin\sthe\soffset\scomputation\sfor\sjson_error(). +D 2023-04-28T23:19:11.271 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -594,7 +594,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 03eba427c0e8700db2895b785642f4dba8440020658d8eef97ec4fadf3836cdf +F src/json.c cc36f7df741f617624abc73adba6ffea1f3358196e1b78b43124265e941d332a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 901ad995d5a722ca2672516205ff488e9acd703a828ca5fc43f11fca5f2af120 -R c710d125f016d21291562e41606e0d2f +P 8d09dc1c45a8026b94f70273d064e47939f30cadedc17548b5a26ba054a8d3a7 +R 85ded77d7e850b8c4e5d880f9f34a021 U drh -Z 98bfef108d9db21095967c25b17439f3 +Z 6b839a1d682f3ab7c8bf4f474d197593 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2d0bc4ddaf..7bde04d486 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d09dc1c45a8026b94f70273d064e47939f30cadedc17548b5a26ba054a8d3a7 \ No newline at end of file +8f7ab5d9210490eb451534df246b82081a5aa252a1bd662659df3da429290837 \ No newline at end of file diff --git a/src/json.c b/src/json.c index f76d7dc5a7..83b02bf5a4 100644 --- a/src/json.c +++ b/src/json.c @@ -2606,11 +2606,11 @@ static void jsonErrorFunc( if( p==0 || p->oom || p->nErr==0 ){ sqlite3_result_int(ctx, 0); }else{ - int n = 0; + int n = 1; int i; const char *z = p->zJson; for(i=0; iiErr && z[i]; i++){ - if( (z[i]&0xc0)!=80 ) n++; + if( (z[i]&0xc0)!=0x80 ) n++; } sqlite3_result_int(ctx, n); jsonParseFree(p); From 8f3fe2ead2103c766380081520b1579481f661d4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 28 Apr 2023 23:38:54 +0000 Subject: [PATCH 309/341] Bug fixes in the logic to skip over JSON5 comments. FossilOrigin-Name: c736b77a2fc5f8f2ba8f770adf05443dfdc3a9847ee519f411e661f76756d611 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 16 +++++++++++++--- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0d0cafc8eb..e5775f3c07 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfix\sin\sthe\soffset\scomputation\sfor\sjson_error(). -D 2023-04-28T23:19:11.271 +C Bug\sfixes\sin\sthe\slogic\sto\sskip\sover\sJSON5\scomments. +D 2023-04-28T23:38:54.056 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -594,7 +594,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c cc36f7df741f617624abc73adba6ffea1f3358196e1b78b43124265e941d332a +F src/json.c 92a84ce1c933ef09296a0a9fdaea95c07d5548ef2f1e540197cfbdef9b38a60a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d09dc1c45a8026b94f70273d064e47939f30cadedc17548b5a26ba054a8d3a7 -R 85ded77d7e850b8c4e5d880f9f34a021 +P 8f7ab5d9210490eb451534df246b82081a5aa252a1bd662659df3da429290837 +R cb0f54422021b51368018226423f13cc U drh -Z 6b839a1d682f3ab7c8bf4f474d197593 +Z 7078eb5b44d703d85936a37156ec4b2c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7bde04d486..88b9340307 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f7ab5d9210490eb451534df246b82081a5aa252a1bd662659df3da429290837 \ No newline at end of file +c736b77a2fc5f8f2ba8f770adf05443dfdc3a9847ee519f411e661f76756d611 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 83b02bf5a4..f2856a841e 100644 --- a/src/json.c +++ b/src/json.c @@ -963,12 +963,22 @@ static int json5Whitespace(const char *zIn){ for(j=n+3; z[j]!='/' || z[j-1]!='*'; j++){ if( z[j]==0 ) goto whitespace_done; } - n += j+1; + n = j+1; break; }else if( z[n+1]=='/' ){ int j; - for(j=n+2; z[j] && z[j]!='\n'; j++){} - n += j; + char c; + for(j=n+2; (c = z[j])!=0; j++){ + if( c=='\n' || c=='\r' ) break; + if( 0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2]) + ){ + j += 2; + break; + } + } + n = j; + if( z[n] ) n++; break; } goto whitespace_done; From 7667f5f12fbc5af6899ff4c33a3c1d6b8fe6ab87 Mon Sep 17 00:00:00 2001 From: larrybr Date: Fri, 28 Apr 2023 23:39:16 +0000 Subject: [PATCH 310/341] Add tests for generate_series(), and change an existing test that enforced varying rowid-to-value mapping with query ordering. (That no longer varies.) FossilOrigin-Name: 9605db4ae37ef383f8e5568af4543a9e6f336bace6ed80ff80fe4c0b0f437078 --- ext/misc/series.c | 3 +++ manifest | 19 ++++++++----------- manifest.uuid | 2 +- test/shell2.test | 29 +++++++++++++++++++++++++++++ test/tabfunc01.test | 2 +- 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/ext/misc/series.c b/ext/misc/series.c index f484cba2dc..c1365881db 100644 --- a/ext/misc/series.c +++ b/ext/misc/series.c @@ -151,6 +151,9 @@ void setupSequence( SequenceSpec *pss ){ pss->isNotEOF = 1; pss->uMaxRowidM1 = puspan/pss->iStep; } + }else if( pss->iTerm == pss->iBase ){ + pss->isNotEOF = 1; + pss->uMaxRowidM1 = 0; } pss->uRidCurrent = (pss->isReversing)? pss->uMaxRowidM1 : 0; pss->iValueCurrent = (pss->isReversing) diff --git a/manifest b/manifest index 6ae7a8b8e8..7b5128c516 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revise\sgenerate_series()\sextension\s(in\sCLI)\sto\saddress\soverflow\sreported\sin\s[forum:754e2d4db2a5|forum\spost\s#754e2d4db2a5]\sand\sto\smake\sbehavior\sbetter\smatch\sthe\slike-named\sPostgreSQL\sfunction. -D 2023-04-28T21:25:03.297 +C Add\stests\sfor\sgenerate_series(),\sand\schange\san\sexisting\stest\sthat\senforced\svarying\srowid-to-value\smapping\swith\squery\sordering.\s(That\sno\slonger\svaries.) +D 2023-04-28T23:39:16.250 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -305,7 +305,7 @@ F ext/misc/regexp.c f50ab59bfa8934b7ed98de069d2c74c187f2ef523fb09e85f8840f6459a9 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946 -F ext/misc/series.c 9553821737ea55abcf534b22a63e9d29c2c9c07c1f82dd56cfa2b18e4d59d046 +F ext/misc/series.c bc7cb3bb6165a15d2e8a37dd5a261c0f11d55e00155f631db22d63fe519b753e F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d F ext/misc/shathree.c 543af7ce71d391cd3a9ab6924a6a1124efc63211fd0f2e240dc4b56077ba88ac F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 @@ -1493,7 +1493,7 @@ F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8e F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test 291c5c4b313adbe44d847af78b730a3bbfa1598c450e09ffe5601170bf7f28e7 -F test/shell2.test 35c0c19d3198ee7669a748c1aedcce27a776ee575cc76128f8fcf665b79672f7 +F test/shell2.test b330df173012f7047593dff8aca049a83f401c991e8a3f1ee6e9ed7753cd3a66 F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b @@ -1565,7 +1565,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 668e5e39108ce84b2f24e402f9382daecc0417edb9c78c6c8f58170584e80c91 +F test/tabfunc01.test 54f27eacd054aa528a8b6e3331192c484104f30aaee351ad035f2b39a00f87c4 F test/table.test eb3463b7add9f16a5bb836badf118cf391b809d09fdccd1f79684600d07ec132 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -2060,11 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a63346d6a0c0ca7ba4c87499de2e461be9c77e9b5d98f2bebf308cdb6599f33c -R 13e2ac79f26c72ca3ffff8649a5a99a9 -T *branch * generate_series-revamp -T *sym-generate_series-revamp * -T -sym-trunk * +P beeea3e1b010dace9789f27172462b912819d0f8142a67e3e1e7335211e0e9a8 +R 52e53a4d072b5936841d520d44c7af9b U larrybr -Z d68099c137797424f3a3da0f35ed98b8 +Z 346788f1a790d4959913907c4ebea4ca # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9ae26b7d44..05fa02dda8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -beeea3e1b010dace9789f27172462b912819d0f8142a67e3e1e7335211e0e9a8 \ No newline at end of file +9605db4ae37ef383f8e5568af4543a9e6f336bace6ed80ff80fe4c0b0f437078 \ No newline at end of file diff --git a/test/shell2.test b/test/shell2.test index aba04a1490..d20e6f8c60 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -216,5 +216,34 @@ do_test shell2-1.4.9 { done 2}} +# Verify that generate_series stays sane near 64-bit range boundaries. +# See overflow report at https://sqlite.org/forum/forumpost/5d34ce5280 +do_test shell2-1.4.10 { + set res [catchcmd :memory: [string trim { + SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); + SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); + SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); + SELECT * FROM generate_series(-9223372036854775808,9223372036854775807, + 9223372036854775807); + SELECT value,rowid FROM generate_series(-4611686018427387904, + 4611686018427387904, 4611686018427387904) ORDER BY value DESC; + SELECT * FROM generate_series(0,-2,-1); + SELECT * FROM generate_series(0,-2); + SELECT * FROM generate_series(0,2) LIMIT 3;}]] +} {0 {9223372036854775807 +9223372036854775807 +9223372036854775807 +-9223372036854775808 +-1 +9223372036854775806 +4611686018427387904|3 +0|2 +-4611686018427387904|1 +0 +-1 +-2 +0 +1 +2}} finish_test diff --git a/test/tabfunc01.test b/test/tabfunc01.test index 8b7386bfec..8e90c549fa 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -61,7 +61,7 @@ do_execsql_test tabfunc01-1.8 { } {30 25 20 15 10 5 0} do_execsql_test tabfunc01-1.9 { SELECT rowid, * FROM generate_series(0,32,5) ORDER BY value DESC; -} {1 30 2 25 3 20 4 15 5 10 6 5 7 0} +} {7 30 6 25 5 20 4 15 3 10 2 5 1 0} do_execsql_test tabfunc01-1.10 { SELECT rowid, * FROM generate_series(0,32,5) ORDER BY +value DESC; } {7 30 6 25 5 20 4 15 3 10 2 5 1 0} From f176776bd73a54e96aa83d9f2fefa4a919cea876 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 29 Apr 2023 00:59:22 +0000 Subject: [PATCH 311/341] Correctly recognize an isolated U+feff as a space character in JSON5. FossilOrigin-Name: 4473dc8e3ad18bb6185ed4a819baf881a7d26ac74bc70fae6ba23a0030be8316 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e5775f3c07..5893cfd206 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfixes\sin\sthe\slogic\sto\sskip\sover\sJSON5\scomments. -D 2023-04-28T23:38:54.056 +C Correctly\srecognize\san\sisolated\sU+feff\sas\sa\sspace\scharacter\sin\sJSON5. +D 2023-04-29T00:59:22.682 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -594,7 +594,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 92a84ce1c933ef09296a0a9fdaea95c07d5548ef2f1e540197cfbdef9b38a60a +F src/json.c 3535ddb77b6bea2374a1ca5dad44cf616969bf202f8143c76bc9d1e38d1b587a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8f7ab5d9210490eb451534df246b82081a5aa252a1bd662659df3da429290837 -R cb0f54422021b51368018226423f13cc +P c736b77a2fc5f8f2ba8f770adf05443dfdc3a9847ee519f411e661f76756d611 +R dbfd99e9ae655ae239188884ec879742 U drh -Z 7078eb5b44d703d85936a37156ec4b2c +Z 9ebd8716513a39a289a7550b0949e896 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 88b9340307..dc0a2012c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c736b77a2fc5f8f2ba8f770adf05443dfdc3a9847ee519f411e661f76756d611 \ No newline at end of file +4473dc8e3ad18bb6185ed4a819baf881a7d26ac74bc70fae6ba23a0030be8316 \ No newline at end of file diff --git a/src/json.c b/src/json.c index f2856a841e..deeb107671 100644 --- a/src/json.c +++ b/src/json.c @@ -1492,7 +1492,8 @@ json_parse_restart: case 0xc2: case 0xe1: case 0xe2: - case 0xe3: { + case 0xe3: + case 0xef: { j = json5Whitespace(&z[i]); if( j>0 ){ i += j; From 64953f0fb8b5c0fdd2ffdb0ed144921c98303909 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 29 Apr 2023 12:13:27 +0000 Subject: [PATCH 312/341] Minor fixes to to the JSON% parser. FossilOrigin-Name: 2fe684cdcdc3cab4ec3348ca5aa5948e4472c562b739c29faebcb77397f8d969 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 21 +++++++++------------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 5893cfd206..db8aa3f3d8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correctly\srecognize\san\sisolated\sU+feff\sas\sa\sspace\scharacter\sin\sJSON5. -D 2023-04-29T00:59:22.682 +C Minor\sfixes\sto\sto\sthe\sJSON%\sparser. +D 2023-04-29T12:13:27.600 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -594,7 +594,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 3535ddb77b6bea2374a1ca5dad44cf616969bf202f8143c76bc9d1e38d1b587a +F src/json.c a25afd49aba4a2ac6b5c0fdabd82f376beccdec18f1946e66143bdb51fb1be33 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c736b77a2fc5f8f2ba8f770adf05443dfdc3a9847ee519f411e661f76756d611 -R dbfd99e9ae655ae239188884ec879742 +P 4473dc8e3ad18bb6185ed4a819baf881a7d26ac74bc70fae6ba23a0030be8316 +R c6f6cd1aa30c85d050651bc2038ace97 U drh -Z 9ebd8716513a39a289a7550b0949e896 +Z 73c28292d33e23cb810b7dd1773fc296 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dc0a2012c6..fc7993d006 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4473dc8e3ad18bb6185ed4a819baf881a7d26ac74bc70fae6ba23a0030be8316 \ No newline at end of file +2fe684cdcdc3cab4ec3348ca5aa5948e4472c562b739c29faebcb77397f8d969 \ No newline at end of file diff --git a/src/json.c b/src/json.c index deeb107671..32a52cf1ee 100644 --- a/src/json.c +++ b/src/json.c @@ -377,10 +377,9 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ int rc = sqlite3DecOrHexToI64(zIn, &i); if( rc<=1 ){ jsonPrintf(100,p,"%lld",i); - }else if( rc==2 ){ - jsonAppendRaw(p, "9.0e999", 7); }else{ - jsonAppendRaw(p, "9223372036854775808", 18); + assert( rc==2 ); + jsonAppendRaw(p, "9.0e999", 7); } return; } @@ -1109,7 +1108,10 @@ json_parse_restart: } if( pParse->oom ) return -1; pNode = &pParse->aNode[pParse->nNode-1]; - if( pNode->eType!=JSON_STRING ) return -1; + if( pNode->eType!=JSON_STRING ){ + pParse->iErr = j; + return -1; + } pNode->jnFlags |= JNODE_LABEL; j = x; if( z[j]==':' ){ @@ -1249,7 +1251,6 @@ json_parse_restart: jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); pParse->has5 = 1; }else if( c=='\r' ){ - j++; if( z[j+1]=='\n' ) j++; jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); pParse->has5 = 1; @@ -1339,10 +1340,7 @@ json_parse_restart: } }else{ if( !sqlite3Isdigit(z[i+1]) ){ - if( z[i+1]=='I' - && (c=='-' || c=='+') - && strncmp(&z[i+1], "Infinity",8)==0 - ){ + if( z[i+1]=='I' && strncmp(&z[i+1], "Infinity",8)==0 ){ if( z[i]=='-' ){ jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); }else{ @@ -1354,7 +1352,6 @@ json_parse_restart: /* Non-standard JSON and JSON5: Allow "Inf" as an alternative ** spelling for "Infinity" and allow it to be in any case. */ if( (z[i+1]=='I' || z[i+1]=='i') - && (c=='-' || c=='+') && sqlite3StrNICmp(&z[i+1], "inf",3)==0 ){ if( z[i]=='-' ){ @@ -2971,8 +2968,8 @@ static void jsonAppendObjectPathElement( nn = pNode->n; if( (pNode->jnFlags & JNODE_RAW)==0 ){ assert( nn>=2 ); - assert( z[0]=='"' ); - assert( z[nn-1]=='"' ); + assert( z[0]=='"' || z[0]=='\'' ); + assert( z[nn-1]=='"' || z[0]=='\'' ); if( nn>2 && sqlite3Isalpha(z[1]) ){ for(jj=2; jj Date: Sat, 29 Apr 2023 15:29:29 +0000 Subject: [PATCH 313/341] Replace duped generate_series() test with another corner case. FossilOrigin-Name: fb2f08783fdfef4d5123ad66792de2265d39272d40d7bf1be7dd21876e8ffd73 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/shell2.test | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ca40aba97d..0204d4830f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Sync\sw/trunk,\simprove\sgenerate_series()\svariable\snames. -D 2023-04-29T12:29:15.001 +C Replace\sduped\sgenerate_series()\stest\swith\sanother\scorner\scase. +D 2023-04-29T15:29:29.280 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1493,7 +1493,7 @@ F test/sharedB.test 1a84863d7a2204e0d42f2e1606577c5e92e4473fa37ea0f5bdf829e4bf8e F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test 291c5c4b313adbe44d847af78b730a3bbfa1598c450e09ffe5601170bf7f28e7 -F test/shell2.test b330df173012f7047593dff8aca049a83f401c991e8a3f1ee6e9ed7753cd3a66 +F test/shell2.test 4872069935ca1bf28e86db1b4ed67a4eec04f47a64cf0ab009cce495531a0c61 F test/shell3.test 91febeac0412812bf6370abb8ed72700e32bf8f9878849414518f662dfd55e8a F test/shell4.test 9abd0c12a7e20a4c49e84d5be208d2124fa6c09e728f56f1f4bee0f02853935f F test/shell5.test c8b6c54f26ec537f8558273d7ed293ca3725ef42e6b12b8f151718628bd1473b @@ -2060,8 +2060,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9605db4ae37ef383f8e5568af4543a9e6f336bace6ed80ff80fe4c0b0f437078 91fee79a01971259b21478e60a069a711a00efc79ddfececa6224a152cd8d09a -R 6f82de0e55204c1ff598ad7fbb0d9ee4 +P ecbf7e13e6454ee879c178c53ffa6face789e93ccbebb0cfac8fde72cb3a4952 +R a43c7ce3100f5e83fb209ebd1799722b U larrybr -Z ad756d0591608cac149890e372e9c3a9 +Z 192fa0c0899cb09731a369a2377e480a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b7c744f9ca..c84ef5f502 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecbf7e13e6454ee879c178c53ffa6face789e93ccbebb0cfac8fde72cb3a4952 \ No newline at end of file +fb2f08783fdfef4d5123ad66792de2265d39272d40d7bf1be7dd21876e8ffd73 \ No newline at end of file diff --git a/test/shell2.test b/test/shell2.test index d20e6f8c60..e50a35d12b 100644 --- a/test/shell2.test +++ b/test/shell2.test @@ -222,7 +222,8 @@ do_test shell2-1.4.10 { set res [catchcmd :memory: [string trim { SELECT * FROM generate_series(9223372036854775807,9223372036854775807,1); SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); - SELECT * FROM generate_series(9223372036854775807,9223372036854775807,-1); + SELECT avg(rowid),min(value),max(value) FROM generate_series( + -9223372036854775808,9223372036854775807,1085102592571150095); SELECT * FROM generate_series(-9223372036854775808,9223372036854775807, 9223372036854775807); SELECT value,rowid FROM generate_series(-4611686018427387904, @@ -232,7 +233,7 @@ do_test shell2-1.4.10 { SELECT * FROM generate_series(0,2) LIMIT 3;}]] } {0 {9223372036854775807 9223372036854775807 -9223372036854775807 +9.5|-9223372036854775808|9223372036854775807 -9223372036854775808 -1 9223372036854775806 From d76d954de3643e515f930c448dd31280fecb62e8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 29 Apr 2023 16:00:20 +0000 Subject: [PATCH 314/341] Do not allow leading zeros on non-zero numeric literals in JSON. FossilOrigin-Name: 3e91494390ba88498eb243f61ce4ef4efa23b58326108a769bc72331d7d7d75b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 14 +++++++------- test/json102.test | 12 ++++++------ 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index db8aa3f3d8..bd3882b93b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sfixes\sto\sto\sthe\sJSON%\sparser. -D 2023-04-29T12:13:27.600 +C Do\snot\sallow\sleading\szeros\son\snon-zero\snumeric\sliterals\sin\sJSON. +D 2023-04-29T16:00:20.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -594,7 +594,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c a25afd49aba4a2ac6b5c0fdabd82f376beccdec18f1946e66143bdb51fb1be33 +F src/json.c 1de9706f7bc22865237fa46a2f3e0de320e0a6afa24ff7b854c7522c0cc57c48 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -1256,7 +1256,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286 F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x F test/json101.test de9c93169b84ac96fd5836c638a2ae1f00e4afbd4003c6b596692d7f05e1cd69 -F test/json102.test 1f61f469d763ff26430dbee76bc75e0aa73084ca84f10e58744fdb899e56de3e +F test/json102.test 3bdf097757faefadfd88c5695ba8ab076ca8d8680eb8f33d73d44c07cbbc666d F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4473dc8e3ad18bb6185ed4a819baf881a7d26ac74bc70fae6ba23a0030be8316 -R c6f6cd1aa30c85d050651bc2038ace97 +P 2fe684cdcdc3cab4ec3348ca5aa5948e4472c562b739c29faebcb77397f8d969 +R ab5a5b4f8c2b54ec0b250293ed522d33 U drh -Z 73c28292d33e23cb810b7dd1773fc296 +Z 69b4a1c1b9499f71fc838cb1772a2121 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fc7993d006..b758f33f29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2fe684cdcdc3cab4ec3348ca5aa5948e4472c562b739c29faebcb77397f8d969 \ No newline at end of file +3e91494390ba88498eb243f61ce4ef4efa23b58326108a769bc72331d7d7d75b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 32a52cf1ee..f603a82bf7 100644 --- a/src/json.c +++ b/src/json.c @@ -333,7 +333,7 @@ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ jsonAppendRaw(p, "\\u0000", 6); break; case '\r': - if( N>=3 && zIn[2]=='\n' ){ + if( zIn[2]=='\n' ){ zIn++; N--; } @@ -1328,15 +1328,15 @@ json_parse_restart: if( c<='0' ){ if( c=='0' ){ - if( sqlite3Isdigit(z[i+1]) ){ - pParse->has5 = 1; - jnFlags = JNODE_JSON5; - }else if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ + if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ assert( seenDP==JSON_INT ); pParse->has5 = 1; jnFlags |= JNODE_JSON5; for(j=i+3; sqlite3Isxdigit(z[j]); j++){} goto parse_number_finish; + }else if( sqlite3Isdigit(z[i+1]) ){ + pParse->iErr = i+1; + return -1; } }else{ if( !sqlite3Isdigit(z[i+1]) ){ @@ -1372,8 +1372,8 @@ json_parse_restart: } if( z[i+1]=='0' ){ if( sqlite3Isdigit(z[i+2]) ){ - pParse->has5 = 1; - jnFlags = JNODE_JSON5; + pParse->iErr = i+1; + return -1; }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ pParse->has5 = 1; jnFlags |= JNODE_JSON5; diff --git a/test/json102.test b/test/json102.test index d45ec11bf1..2594feafa1 100644 --- a/test/json102.test +++ b/test/json102.test @@ -302,18 +302,18 @@ for {set i 0} {$i<100} {incr i} { # fixed. # foreach {id j x0 x5} { - 1401 {'{"x":01}'} 0 1 - 1402 {'{"x":-01}'} 0 1 + 1401 {'{"x":01}'} 0 0 + 1402 {'{"x":-01}'} 0 0 1403 {'{"x":0}'} 1 1 1404 {'{"x":-0}'} 1 1 1405 {'{"x":0.1}'} 1 1 1406 {'{"x":-0.1}'} 1 1 1407 {'{"x":0.0000}'} 1 1 1408 {'{"x":-0.0000}'} 1 1 - 1409 {'{"x":01.5}'} 0 1 - 1410 {'{"x":-01.5}'} 0 1 - 1411 {'{"x":00}'} 0 1 - 1412 {'{"x":-00}'} 0 1 + 1409 {'{"x":01.5}'} 0 0 + 1410 {'{"x":-01.5}'} 0 0 + 1411 {'{"x":00}'} 0 0 + 1412 {'{"x":-00}'} 0 0 1413 {'{"x":+0}'} 0 1 1414 {'{"x":+5}'} 0 1 1415 {'{"x":+5.5}'} 0 1 From 399875f6c568bf7e674948c09686c0c7cabb21d9 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 29 Apr 2023 16:31:08 +0000 Subject: [PATCH 315/341] Simplification of the logic that normalizes JSON5 integer literals into canonical JSON integer literals. Improved reporting of OOM. FossilOrigin-Name: 01ee613c07fcb87e7d7b7f1b1387982715d1343418f37f4a1dc90e43a76d20e8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 47 ++++++++++++++++++++++++++++------------------- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index bd3882b93b..3a9b1fb48b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sleading\szeros\son\snon-zero\snumeric\sliterals\sin\sJSON. -D 2023-04-29T16:00:20.514 +C Simplification\sof\sthe\slogic\sthat\snormalizes\sJSON5\sinteger\sliterals\sinto\ncanonical\sJSON\sinteger\sliterals.\s\sImproved\sreporting\sof\sOOM. +D 2023-04-29T16:31:08.205 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -594,7 +594,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 1de9706f7bc22865237fa46a2f3e0de320e0a6afa24ff7b854c7522c0cc57c48 +F src/json.c 6029d4ce8705ebad5817386a326b620772dcc165e1b23a8e4645061621679538 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2fe684cdcdc3cab4ec3348ca5aa5948e4472c562b739c29faebcb77397f8d969 -R ab5a5b4f8c2b54ec0b250293ed522d33 +P 3e91494390ba88498eb243f61ce4ef4efa23b58326108a769bc72331d7d7d75b +R 9934ae2414640783ca534819ca587694 U drh -Z 69b4a1c1b9499f71fc838cb1772a2121 +Z fec85eb0c6f9c0d91d036f5cadb9198f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b758f33f29..beb328db1e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e91494390ba88498eb243f61ce4ef4efa23b58326108a769bc72331d7d7d75b \ No newline at end of file +01ee613c07fcb87e7d7b7f1b1387982715d1343418f37f4a1dc90e43a76d20e8 \ No newline at end of file diff --git a/src/json.c b/src/json.c index f603a82bf7..17908fd976 100644 --- a/src/json.c +++ b/src/json.c @@ -371,21 +371,17 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ zIn++; N--; } - while( zIn[0]=='0' && N>1 ){ - if( zIn[1]=='x' || zIn[1]=='X' ){ - sqlite3_int64 i = 0; - int rc = sqlite3DecOrHexToI64(zIn, &i); - if( rc<=1 ){ - jsonPrintf(100,p,"%lld",i); - }else{ - assert( rc==2 ); - jsonAppendRaw(p, "9.0e999", 7); - } - return; + if( zIn[0]=='0' && (zIn[1]=='x' || zIn[1]=='X') ){ + sqlite3_int64 i = 0; + int rc = sqlite3DecOrHexToI64(zIn, &i); + if( rc<=1 ){ + jsonPrintf(100,p,"%lld",i); + }else{ + assert( rc==2 ); + jsonAppendRaw(p, "9.0e999", 7); } - zIn++; - N--; - } + return; + } jsonAppendRaw(p, zIn, N); } @@ -2571,8 +2567,13 @@ static void jsonValidFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); p = jsonParseCached(ctx, argv, 0); - sqlite3_result_int(ctx, p!=0 && p->nErr==0 && p->has5==0); - if( p!=0 && p->nErr ) jsonParseFree(p); + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(ctx); + sqlite3_free(p); + }else{ + sqlite3_result_int(ctx, p->nErr==0 && p->has5==0); + if( p->nErr ) jsonParseFree(p); + } } @@ -2590,8 +2591,13 @@ static void jsonValid5Func( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); p = jsonParseCached(ctx, argv, 0); - sqlite3_result_int(ctx, p!=0 && p->nErr==0); - if( p!=0 && p->nErr ) jsonParseFree(p); + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(ctx); + sqlite3_free(p); + }else{ + sqlite3_result_int(ctx, p->nErr==0); + if( p->nErr ) jsonParseFree(p); + } } @@ -2611,7 +2617,10 @@ static void jsonErrorFunc( JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); p = jsonParseCached(ctx, argv, 0); - if( p==0 || p->oom || p->nErr==0 ){ + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(ctx); + sqlite3_free(p); + }else if( p->nErr==0 ){ sqlite3_result_int(ctx, 0); }else{ int n = 1; From f8ae2d2a2a23e4ad3f937712d30e2daf8d3b0c79 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 29 Apr 2023 17:13:39 +0000 Subject: [PATCH 316/341] Add ALWAYS macros on unreachable branches in the new JSON5 logic. FossilOrigin-Name: 91e15ed9d93e51b750172a074dbbcf206128e9476571ce6189b8ea0ba5ab4135 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3a9b1fb48b..f13e0d0240 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sof\sthe\slogic\sthat\snormalizes\sJSON5\sinteger\sliterals\sinto\ncanonical\sJSON\sinteger\sliterals.\s\sImproved\sreporting\sof\sOOM. -D 2023-04-29T16:31:08.205 +C Add\sALWAYS\smacros\son\sunreachable\sbranches\sin\sthe\snew\sJSON5\slogic. +D 2023-04-29T17:13:39.696 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -594,7 +594,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 6029d4ce8705ebad5817386a326b620772dcc165e1b23a8e4645061621679538 +F src/json.c 0de525ac85b275cfbe6a7eb565c9c7849f884331153f60024715a47a18ed781e F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3e91494390ba88498eb243f61ce4ef4efa23b58326108a769bc72331d7d7d75b -R 9934ae2414640783ca534819ca587694 +P 01ee613c07fcb87e7d7b7f1b1387982715d1343418f37f4a1dc90e43a76d20e8 +R 8691ebe0c22e2def3817b9ef68bc683b U drh -Z fec85eb0c6f9c0d91d036f5cadb9198f +Z 4bc62005656847ff8857de80fff89c9b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index beb328db1e..8139537284 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01ee613c07fcb87e7d7b7f1b1387982715d1343418f37f4a1dc90e43a76d20e8 \ No newline at end of file +91e15ed9d93e51b750172a074dbbcf206128e9476571ce6189b8ea0ba5ab4135 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 17908fd976..7876bdcbc4 100644 --- a/src/json.c +++ b/src/json.c @@ -1393,7 +1393,7 @@ json_parse_restart: } if( c=='e' || c=='E' ){ if( z[j-1]<'0' ){ - if( z[j-1]=='.' && j-2>=i && sqlite3Isdigit(z[j-2]) ){ + if( ALWAYS(z[j-1]=='.') && j-2>=i && sqlite3Isdigit(z[j-2]) ){ pParse->has5 = 1; jnFlags |= JNODE_JSON5; }else{ @@ -1421,7 +1421,7 @@ json_parse_restart: break; } if( z[j-1]<'0' ){ - if( z[j-1]=='.' && j-2>=i && sqlite3Isdigit(z[j-2]) ){ + if( ALWAYS(z[j-1]=='.') && j-2>=i && sqlite3Isdigit(z[j-2]) ){ pParse->has5 = 1; jnFlags |= JNODE_JSON5; }else{ @@ -2626,7 +2626,7 @@ static void jsonErrorFunc( int n = 1; int i; const char *z = p->zJson; - for(i=0; iiErr && z[i]; i++){ + for(i=0; iiErr && ALWAYS(z[i]); i++){ if( (z[i]&0xc0)!=0x80 ) n++; } sqlite3_result_int(ctx, n); From 440f7b483306038f4a3ced85bc440c9debe46c54 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 29 Apr 2023 17:35:29 +0000 Subject: [PATCH 317/341] More ALWAYS() macros. FossilOrigin-Name: 770b09f7a795956be63a06847059370db8dfc88654878d742b7b826947029962 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f13e0d0240..2bc4528324 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sALWAYS\smacros\son\sunreachable\sbranches\sin\sthe\snew\sJSON5\slogic. -D 2023-04-29T17:13:39.696 +C More\sALWAYS()\smacros. +D 2023-04-29T17:35:29.727 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -594,7 +594,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 0de525ac85b275cfbe6a7eb565c9c7849f884331153f60024715a47a18ed781e +F src/json.c f76772b2d780f9711896e10bb6efa32d839d9c3fef225d3476e6f5b6683a04f3 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2067,8 +2067,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 01ee613c07fcb87e7d7b7f1b1387982715d1343418f37f4a1dc90e43a76d20e8 -R 8691ebe0c22e2def3817b9ef68bc683b +P 91e15ed9d93e51b750172a074dbbcf206128e9476571ce6189b8ea0ba5ab4135 +R b78f203ca73d8ea097a8cd3d3031aec7 U drh -Z 4bc62005656847ff8857de80fff89c9b +Z 43d50e401ed22df6c70dd6bacd839b80 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8139537284..6cdf339ed2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91e15ed9d93e51b750172a074dbbcf206128e9476571ce6189b8ea0ba5ab4135 \ No newline at end of file +770b09f7a795956be63a06847059370db8dfc88654878d742b7b826947029962 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 7876bdcbc4..a78aa8e614 100644 --- a/src/json.c +++ b/src/json.c @@ -1393,7 +1393,7 @@ json_parse_restart: } if( c=='e' || c=='E' ){ if( z[j-1]<'0' ){ - if( ALWAYS(z[j-1]=='.') && j-2>=i && sqlite3Isdigit(z[j-2]) ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ pParse->has5 = 1; jnFlags |= JNODE_JSON5; }else{ @@ -1421,7 +1421,7 @@ json_parse_restart: break; } if( z[j-1]<'0' ){ - if( ALWAYS(z[j-1]=='.') && j-2>=i && sqlite3Isdigit(z[j-2]) ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ pParse->has5 = 1; jnFlags |= JNODE_JSON5; }else{ From e01e3b5b9491759108c205d2fe6c1b1c1be60120 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Apr 2023 18:31:44 +0000 Subject: [PATCH 318/341] Fix a crash that could occur in fts5 'secure-delete' mode when operating on corrupt records. FossilOrigin-Name: 2e85b0e3dcae0915aa6472a3654c8ac72a6b2083c11747f3f657c79bbdaf530b --- ext/fts5/fts5_index.c | 3 + ext/fts5/test/fts5corrupt7.test | 99 +++++++++++++++++++++++++++++++++ manifest | 16 +++--- manifest.uuid | 2 +- 4 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 ext/fts5/test/fts5corrupt7.test diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 186c421af8..36ac22bfa7 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4634,6 +4634,9 @@ static void fts5SecureDeleteOverflow( pLeaf = 0; }else if( bDetailNone ){ break; + }else if( iNext>=pLeaf->szLeaf || iNext<4 ){ + p->rc = FTS5_CORRUPT; + break; }else{ int nShift = iNext - 4; int nPg; diff --git a/ext/fts5/test/fts5corrupt7.test b/ext/fts5/test/fts5corrupt7.test new file mode 100644 index 0000000000..75995a7c03 --- /dev/null +++ b/ext/fts5/test/fts5corrupt7.test @@ -0,0 +1,99 @@ +# 2023 April 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5corrupt7 + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} +sqlite3_fts5_may_be_corrupt 1 + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); +} + +set doc [string repeat "a b " 30] + +do_execsql_test 1.1 { + BEGIN; + INSERT INTO t1(rowid, x) VALUES(123, $doc); + INSERT INTO t1(rowid, x) VALUES(124, $doc); + COMMIT; +} + +execsql_pp { + SELECT id, fts5_decode(id, block), quote(block) FROM t1_data +} + +set rows [db eval { SELECT rowid FROM t1_data }] +db_save_and_close + +foreach r $rows { + db_restore_and_reopen + + proc edit_block {b} { + binary scan $b c* in + set out [lreplace $in 0 1 255 255] + binary format c* $out + } + db func edit_block edit_block + + do_execsql_test 1.2.$r.1 { + UPDATE t1_data SET block = edit_block(block) WHERE rowid=$r; + } + + do_execsql_test 1.2.$r.2 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } + + do_test 1.2.$r.3 { + catchsql { DELETE FROM t1 WHERE rowid=123; } + catchsql { DELETE FROM t1 WHERE rowid=124; } + set {} {} + } {} + + db close +} + +foreach r $rows { +set r 137438953475 + db_restore_and_reopen + + proc edit_block {b} { + binary scan $b c* in + set out [lreplace $in end end 127] + binary format c* $out + } + db func edit_block edit_block + + do_execsql_test 1.2.$r.1 { + UPDATE t1_data SET block = edit_block(block) WHERE rowid=$r; + } + + do_execsql_test 1.2.$r.2 { + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + } + + do_test 1.2.$r.3 { + catchsql { DELETE FROM t1 WHERE rowid=124; } + catchsql { DELETE FROM t1 WHERE rowid=123; } + set {} {} + } {} + + db close +} + +finish_test diff --git a/manifest b/manifest index 0e94e62118..5b05d8a9f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cure\sCLI\sgenerate_series()\soverflow\sbug\s(forum:754e2d4db2a5|noted\sby\sforum\spost\s#754e2d4db2a5)\sand\sbring\sbehavior\swith\snegative\sstep\sarguments\scloser\sto\sas-documented\sand\seponymous\sfunction\sin\sPostgreSQL. -D 2023-04-29T15:42:46.941 +C Fix\sa\scrash\sthat\scould\soccur\sin\sfts5\s'secure-delete'\smode\swhen\soperating\son\scorrupt\srecords. +D 2023-04-29T18:31:44.051 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 7d298d76ea010c339b26ca47f6f69e9aef520ea46c083deaa4e83e87cf0e94b1 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c 17dca8e874df04182bc45063dc0b761acc242b91f1264d2257b7e37bd4e4c2ad +F ext/fts5/fts5_index.c a84273c6dd091bca6f78444797e49da1b07fabb8eb41ae42add91bba9d1bbd02 F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -137,6 +137,7 @@ F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36 F ext/fts5/test/fts5corrupt4.test f4c08e2182a48d8b70975fd869ee5391855c06d8a0ff87b6a2529e7c5a88a1d3 F ext/fts5/test/fts5corrupt5.test 550d0884c14424f9acad051a741f1dd99ec9342277d938e91ff3daf9123d1209 F ext/fts5/test/fts5corrupt6.test bf8eeae07825b088b9665d9d8e4accbd8dc9bf3cb85b6c64cf6c9e18ccc420a4 +F ext/fts5/test/fts5corrupt7.test f3e68673af2514e31dd67a2ed163f7f597252ab683dec155b8db0cdc0b668342 F ext/fts5/test/fts5delete.test 619295b20dbc1d840b403ee07c878f52378849c3c02e44f2ee143b3e978a0aa7 F ext/fts5/test/fts5detail.test 54015e9c43ec4ba542cfb93268abdf280e0300f350efd08ee411284b03595cc4 F ext/fts5/test/fts5determin.test 1b77879b2ae818b5b71c859e534ee334dac088b7cf3ff3bf76a2c82b1c788d11 @@ -2060,9 +2061,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 91fee79a01971259b21478e60a069a711a00efc79ddfececa6224a152cd8d09a fb2f08783fdfef4d5123ad66792de2265d39272d40d7bf1be7dd21876e8ffd73 -R a43c7ce3100f5e83fb209ebd1799722b -T +closed fb2f08783fdfef4d5123ad66792de2265d39272d40d7bf1be7dd21876e8ffd73 -U larrybr -Z 7a730e984ce1c44cc3481738a9a4584b +P 07383758d68e05021ccd393a69c1fa94836dfb02331326c082495a531a384363 +R f659959a34cd18cd5104e952e98d50d8 +U dan +Z b8e56280f98e35721f913e42d252102b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 109201fac3..8d196fa4ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07383758d68e05021ccd393a69c1fa94836dfb02331326c082495a531a384363 \ No newline at end of file +2e85b0e3dcae0915aa6472a3654c8ac72a6b2083c11747f3f657c79bbdaf530b \ No newline at end of file From 7be1473ccbd55a9b27d90a3cdf20e44d39c6e7a1 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 30 Apr 2023 19:34:41 +0000 Subject: [PATCH 319/341] Omit the json_valid() function. Change the name of json_error() to json_error_position(). Use "NOT json_error_position(X)" as a substitute for "json_valid5(X)". FossilOrigin-Name: 34c4e900a9cc51630eeaf01deef74bf5b18d66e0ab1dc61a2023ac8f837a5197 --- manifest | 18 ++++----- manifest.uuid | 2 +- src/json.c | 94 ++++++++++++++++++++++------------------------- test/json101.test | 26 ++++++------- test/json102.test | 2 +- test/json501.test | 10 ++--- 6 files changed, 73 insertions(+), 79 deletions(-) diff --git a/manifest b/manifest index ac7fedd268..c7969eb46e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sthe\slatest\strunk\senhancements\sinto\sthe\sJSON5\sbranch\sto\nfacilitate\stesting\sof\sthe\sJSON5\sbranch. -D 2023-04-29T18:40:48.921 +C Omit\sthe\sjson_valid()\sfunction.\s\sChange\sthe\sname\sof\sjson_error()\sto\njson_error_position().\s\sUse\s"NOT\sjson_error_position(X)"\sas\sa\ssubstitute\nfor\s"json_valid5(X)". +D 2023-04-30T19:34:41.159 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -595,7 +595,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c f76772b2d780f9711896e10bb6efa32d839d9c3fef225d3476e6f5b6683a04f3 +F src/json.c d216510dbb7900072e4a0cbb1273a21e0a20db57da405b4397abb236a2f30392 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -1256,12 +1256,12 @@ F test/json/README.md 506af1f54574b524106acb50d1a341ab5ddfa6d83fe25095007892b07e F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd28656fb261bddc8a3f F test/json/json-q1.txt 335a7c8ab291d354f33b7decc9559e99a2823d4142291c4be7aa339a631f3c2d F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3b1f1511d21af6737 x -F test/json101.test de9c93169b84ac96fd5836c638a2ae1f00e4afbd4003c6b596692d7f05e1cd69 -F test/json102.test 3bdf097757faefadfd88c5695ba8ab076ca8d8680eb8f33d73d44c07cbbc666d +F test/json101.test 211d75638782370c07e10a847bd66e501ea1537f39c7da4447bfa055c0f261db +F test/json102.test 13dc9e7b7f359ecb861e02f9bd7019f7342a63d1c354273b0a8f3904050560a8 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d -F test/json501.test 43614faefc9ac708e29d55cdc290b5cea336c151030196885708ea3d132fce63 +F test/json501.test 9b14d14f26659287277a1f6fcf2a932a0a2351e1aae522586b530498c977ce45 F test/json502.test 66d150cc098674b8bf4354526a8dd411b926f43ca892306bcb3b6d3f93fef7be F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c feb4358fb022da8ebd098c45811f2f6507688bb6c43aa72b3e840df19026317b @@ -2068,8 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 770b09f7a795956be63a06847059370db8dfc88654878d742b7b826947029962 2e85b0e3dcae0915aa6472a3654c8ac72a6b2083c11747f3f657c79bbdaf530b -R 81b69133758950851391ffde4b6eeead +P 30d12edebad9b097cd5f0da355304d1cb2f8b70d7c7dff378fd7ad7c8ebf9279 +R 5a03c0d318d1b5343d158677e6158a89 U drh -Z 5f3a6d866b7662144df62f2a5c006f47 +Z 50768d32c83122dec00455a3ce19a1c8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 1007d98b66..fa442e6283 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30d12edebad9b097cd5f0da355304d1cb2f8b70d7c7dff378fd7ad7c8ebf9279 \ No newline at end of file +34c4e900a9cc51630eeaf01deef74bf5b18d66e0ab1dc61a2023ac8f837a5197 \ No newline at end of file diff --git a/src/json.c b/src/json.c index a78aa8e614..8fa08d2443 100644 --- a/src/json.c +++ b/src/json.c @@ -134,7 +134,7 @@ struct JsonParse { u16 iDepth; /* Nesting depth */ u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ - u8 has5; /* True if input has JSON5 features */ + u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ int nJson; /* Length of the zJson string in bytes */ u32 iErr; /* Error location in zJson[] */ u32 iHold; /* Replace cache line with the lowest iHold value */ @@ -1086,7 +1086,7 @@ json_parse_restart: if( x<=0 ){ if( x==(-2) ){ j = pParse->iErr; - if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; + if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; pParse->iDepth--; break; } @@ -1095,7 +1095,7 @@ json_parse_restart: int k; for(k=j+1; sqlite3Isalnum(z[k]) || z[k]=='_' || z[k]=='$'; k++){} jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), k-j, &z[j]); - pParse->has5 = 1; + pParse->hasNonstd = 1; x = k; }else{ if( x!=-1 ) pParse->iErr = j; @@ -1179,7 +1179,7 @@ json_parse_restart: if( x<=0 ){ if( x==(-3) ){ j = pParse->iErr; - if( pParse->nNode!=(u32)iThis+1 ) pParse->has5 = 1; + if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; break; } if( x!=(-1) ) pParse->iErr = j; @@ -1218,7 +1218,7 @@ json_parse_restart: case '\'': { u8 jnFlags; char cDelim; - pParse->has5 = 1; + pParse->hasNonstd = 1; jnFlags = JNODE_JSON5; goto parse_string; case '"': @@ -1245,11 +1245,11 @@ json_parse_restart: && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) || (c=='x' && jsonIs2Hex(&z[j+1])) ){ jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); - pParse->has5 = 1; + pParse->hasNonstd = 1; }else if( c=='\r' ){ if( z[j+1]=='\n' ) j++; jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); - pParse->has5 = 1; + pParse->hasNonstd = 1; }else{ pParse->iErr = j; return -1; @@ -1288,12 +1288,12 @@ json_parse_restart: } case '+': { u8 seenDP, seenE, jnFlags; - pParse->has5 = 1; + pParse->hasNonstd = 1; jnFlags = JNODE_JSON5; goto parse_number; case '.': if( sqlite3Isdigit(z[i+1]) ){ - pParse->has5 = 1; + pParse->hasNonstd = 1; jnFlags = JNODE_JSON5; seenE = 0; seenDP = JSON_REAL; @@ -1326,7 +1326,7 @@ json_parse_restart: if( c=='0' ){ if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ assert( seenDP==JSON_INT ); - pParse->has5 = 1; + pParse->hasNonstd = 1; jnFlags |= JNODE_JSON5; for(j=i+3; sqlite3Isxdigit(z[j]); j++){} goto parse_number_finish; @@ -1359,7 +1359,7 @@ json_parse_restart: } #endif if( z[i+1]=='.' ){ - pParse->has5 = 1; + pParse->hasNonstd = 1; jnFlags |= JNODE_JSON5; goto parse_number_2; } @@ -1371,7 +1371,7 @@ json_parse_restart: pParse->iErr = i+1; return -1; }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ - pParse->has5 = 1; + pParse->hasNonstd = 1; jnFlags |= JNODE_JSON5; for(j=i+4; sqlite3Isxdigit(z[j]); j++){} goto parse_number_finish; @@ -1394,7 +1394,7 @@ json_parse_restart: if( c=='e' || c=='E' ){ if( z[j-1]<'0' ){ if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ - pParse->has5 = 1; + pParse->hasNonstd = 1; jnFlags |= JNODE_JSON5; }else{ pParse->iErr = j; @@ -1422,7 +1422,7 @@ json_parse_restart: } if( z[j-1]<'0' ){ if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ - pParse->has5 = 1; + pParse->hasNonstd = 1; jnFlags |= JNODE_JSON5; }else{ pParse->iErr = j; @@ -1436,7 +1436,7 @@ json_parse_restart: case 'N': { if( strncmp(&z[i],"NaN",3)==0 ){ jsonParseAddNode(pParse, JSON_NULL, 4, "null"); - pParse->has5 = 1; + pParse->hasNonstd = 1; return i+3; } pParse->iErr = i; @@ -1445,7 +1445,7 @@ json_parse_restart: case 'I': { if( strncmp(&z[i],"Infinity",8)==0 ){ jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); - pParse->has5 = 1; + pParse->hasNonstd = 1; return i+8; } pParse->iErr = i; @@ -1490,7 +1490,7 @@ json_parse_restart: j = json5Whitespace(&z[i]); if( j>0 ){ i += j; - pParse->has5 = 1; + pParse->hasNonstd = 1; goto json_parse_restart; } pParse->iErr = i; @@ -1545,7 +1545,7 @@ static int jsonParse( jsonParseReset(pParse); return 1; } - pParse->has5 = 1; + pParse->hasNonstd = 1; } } if( i<=0 ){ @@ -2571,43 +2571,38 @@ static void jsonValidFunc( sqlite3_result_error_nomem(ctx); sqlite3_free(p); }else{ - sqlite3_result_int(ctx, p->nErr==0 && p->has5==0); + sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0); if( p->nErr ) jsonParseFree(p); } } - /* -** json_valid5(JSON) +** json_error_position(JSON) ** -** Return 1 if JSON is a well-formed JSON5 string. -** Return 0 otherwise. -*/ -static void jsonValid5Func( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse *p; /* The parse */ - UNUSED_PARAMETER(argc); - p = jsonParseCached(ctx, argv, 0); - if( p==0 || p->oom ){ - sqlite3_result_error_nomem(ctx); - sqlite3_free(p); - }else{ - sqlite3_result_int(ctx, p->nErr==0); - if( p->nErr ) jsonParseFree(p); - } -} - - - -/* -** json_error(JSON) +** If the argument is not an interpretable JSON string, then return the 1-based +** character position at which the parser first recognized that the input +** was in error. The left-most character is 1. If the string is valid +** JSON, then return 0. ** -** If JSON is not a well-formed JSON5 string, then return the 1-based -** character offset to the location of the first error in that string. -** Return 0 otherwise. +** Note that json_valid() is only true for strictly conforming canonical JSON. +** But this routine returns zero if the input contains extension. Thus: +** +** (1) If the input X is strictly conforming canonical JSON: +** +** json_valid(X) returns true +** json_error_position(X) returns 0 +** +** (2) If the input X is JSON but it includes extension (such as JSON5) that +** are not part of RFC-8259: +** +** json_valid(X) returns false +** json_error_position(X) return 0 +** +** (3) If the input X cannot be interpreted as JSON even taking extensions +** into account: +** +** json_valid(X) return false +** json_error_position(X) returns 1 or more */ static void jsonErrorFunc( sqlite3_context *ctx, @@ -3344,7 +3339,7 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_array, -1, 0, jsonArrayFunc), JFUNCTION(json_array_length, 1, 0, jsonArrayLengthFunc), JFUNCTION(json_array_length, 2, 0, jsonArrayLengthFunc), - JFUNCTION(json_error, 1, 0, jsonErrorFunc), + JFUNCTION(json_error_position,1, 0, jsonErrorFunc), JFUNCTION(json_extract, -1, 0, jsonExtractFunc), JFUNCTION(->, 2, JSON_JSON, jsonExtractFunc), JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), @@ -3358,7 +3353,6 @@ void sqlite3RegisterJsonFunctions(void){ JFUNCTION(json_type, 1, 0, jsonTypeFunc), JFUNCTION(json_type, 2, 0, jsonTypeFunc), JFUNCTION(json_valid, 1, 0, jsonValidFunc), - JFUNCTION(json_valid5, 1, 0, jsonValid5Func), #if SQLITE_DEBUG JFUNCTION(json_parse, 1, 0, jsonParseFunc), JFUNCTION(json_test1, 1, 0, jsonTest1Func), diff --git a/test/json101.test b/test/json101.test index 3fc3923da3..298dc3bdbc 100644 --- a/test/json101.test +++ b/test/json101.test @@ -310,8 +310,8 @@ do_execsql_test json-6.1 { SELECT json_valid('{"a":55,"b":72,}'); } {0} do_execsql_test json-6.2 { - SELECT json_valid5('{"a":55,"b":72,}'); -} {1} + SELECT json_error_position('{"a":55,"b":72,}'); +} {0} do_execsql_test json-6.3 { SELECT json_valid(json('{"a":55,"b":72,}')); } {1} @@ -319,23 +319,23 @@ do_execsql_test json-6.4 { SELECT json_valid('{"a":55,"b":72 , }'); } {0} do_execsql_test json-6.5 { - SELECT json_valid5('{"a":55,"b":72 , }'); -} {1} -do_execsql_test json-6.6 { - SELECT json_valid5('{"a":55,"b":72,,}'); + SELECT json_error_position('{"a":55,"b":72 , }'); } {0} +do_execsql_test json-6.6 { + SELECT json_error_position('{"a":55,"b":72,,}'); +} {16} do_execsql_test json-6.7 { SELECT json_valid('{"a":55,"b":72}'); } {1} do_execsql_test json-6.8 { - SELECT json_valid5('["a",55,"b",72,]'); -} {1} -do_execsql_test json-6.9 { - SELECT json_valid5('["a",55,"b",72 , ]'); -} {1} -do_execsql_test json-6.10 { - SELECT json_valid5('["a",55,"b",72,,]'); + SELECT json_error_position('["a",55,"b",72,]'); } {0} +do_execsql_test json-6.9 { + SELECT json_error_position('["a",55,"b",72 , ]'); +} {0} +do_execsql_test json-6.10 { + SELECT json_error_position('["a",55,"b",72,,]'); +} {16} do_execsql_test json-6.11 { SELECT json_valid('["a",55,"b",72]'); } {1} diff --git a/test/json102.test b/test/json102.test index 2594feafa1..bfd5e7ed07 100644 --- a/test/json102.test +++ b/test/json102.test @@ -319,7 +319,7 @@ foreach {id j x0 x5} { 1415 {'{"x":+5.5}'} 0 1 } { do_execsql_test json102-$id " - SELECT json_valid($j), json_valid5($j); + SELECT json_valid($j), NOT json_error_position($j); " [list $x0 $x5] } diff --git a/test/json501.test b/test/json501.test index fbf7bfe9db..4d6a058936 100644 --- a/test/json501.test +++ b/test/json501.test @@ -63,7 +63,7 @@ set testprefix json501 # 1) Object keys may be an ECMAScript 5.1 IdentifierName. do_execsql_test 1.1 { WITH c(x) AS (VALUES('{a:5,b:6}')) - SELECT x->>'a', json(x), json_valid(x), json_valid5(x) FROM c; + SELECT x->>'a', json(x), json_valid(x), NOT json_error_position(x) FROM c; } {5 {{"a":5,"b":6}} 0 1} do_execsql_test 1.2 { SELECT '[7,null,{a:5,b:6},[8,9]]'->>'$[2].b'; @@ -98,7 +98,7 @@ do_catchsql_test 1.11 { do_execsql_test 2.1 { WITH c(x) AS (VALUES('{"a":5, "b":6, }')) - SELECT x->>'b', json(x), json_valid(x), json_valid5(x) FROM c; + SELECT x->>'b', json(x), json_valid(x), NOT json_error_position(x) FROM c; } {6 {{"a":5,"b":6}} 0 1} do_execsql_test 2.2 { SELECT '{a:5, b:6 , }'->>'b'; @@ -115,7 +115,7 @@ do_catchsql_test 2.4 { do_execsql_test 3.1 { WITH c(x) AS (VALUES('[5, 6,]')) - SELECT x->>1, json(x), json_valid(x), json_valid5(x) FROM c; + SELECT x->>1, json(x), json_valid(x), NOT json_error_position(x) FROM c; } {6 {[5,6]} 0 1} do_execsql_test 3.2 { SELECT '[5, 6 , ]'->>1; @@ -132,7 +132,7 @@ do_catchsql_test 3.4 { do_execsql_test 4.1 { WITH c(x) AS (VALUES('{"a": ''abcd''}')) - SELECT x->>'a', json(x), json_valid(x), json_valid5(x) FROM c; + SELECT x->>'a', json(x), json_valid(x), NOT json_error_position(x) FROM c; } {abcd {{"a":"abcd"}} 0 1} do_execsql_test 4.2 { SELECT '{b: 123, ''a'': ''ab\''cd''}'->>'a'; @@ -143,7 +143,7 @@ do_execsql_test 4.2 { do_execsql_test 5.1 { WITH c(x) AS (VALUES('{a: "abc'||char(0x5c,0x0a)||'xyz"}')) - SELECT x->>'a', json(x), json_valid(x), json_valid5(x) FROM c; + SELECT x->>'a', json(x), json_valid(x), NOT json_error_position(x) FROM c; } {abcxyz {{"a":"abcxyz"}} 0 1} do_execsql_test 5.2 { SELECT ('{a: "abc'||char(0x5c,0x0d)||'xyz"}')->>'a'; From 4a398c317d919321205668b010a4955e2325242c Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 30 Apr 2023 19:45:25 +0000 Subject: [PATCH 320/341] All JSON to understand floating point literals "Inf" and "QNaN" and "SNaN" in any case, without the SQLITE_EXTENDED_NAN_INF compile-time option. This extension is always available. FossilOrigin-Name: fb551145e0d84213b3343dc1bc7db70c898b9dea24a72b968240617f4b52d821 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 42 ++++++------------------------------------ 3 files changed, 13 insertions(+), 43 deletions(-) diff --git a/manifest b/manifest index c7969eb46e..0b4e7c7276 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\sjson_valid()\sfunction.\s\sChange\sthe\sname\sof\sjson_error()\sto\njson_error_position().\s\sUse\s"NOT\sjson_error_position(X)"\sas\sa\ssubstitute\nfor\s"json_valid5(X)". -D 2023-04-30T19:34:41.159 +C All\sJSON\sto\sunderstand\sfloating\spoint\sliterals\s"Inf"\sand\s"QNaN"\sand\s"SNaN"\sin\nany\scase,\swithout\sthe\sSQLITE_EXTENDED_NAN_INF\scompile-time\soption.\s\sThis\nextension\sis\salways\savailable. +D 2023-04-30T19:45:25.975 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -595,7 +595,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c d216510dbb7900072e4a0cbb1273a21e0a20db57da405b4397abb236a2f30392 +F src/json.c bd22944b15d786a7a17f0faa672cdb3abeef1b59c505684df64caaca2a18a03a F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2068,8 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 30d12edebad9b097cd5f0da355304d1cb2f8b70d7c7dff378fd7ad7c8ebf9279 -R 5a03c0d318d1b5343d158677e6158a89 +P 34c4e900a9cc51630eeaf01deef74bf5b18d66e0ab1dc61a2023ac8f837a5197 +R 6addb4498a8cf4456a809d957c22b7a6 U drh -Z 50768d32c83122dec00455a3ce19a1c8 +Z 050299ed4706f61a98d79f7a5528f481 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index fa442e6283..db3c346403 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34c4e900a9cc51630eeaf01deef74bf5b18d66e0ab1dc61a2023ac8f837a5197 \ No newline at end of file +fb551145e0d84213b3343dc1bc7db70c898b9dea24a72b968240617f4b52d821 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 8fa08d2443..c221193282 100644 --- a/src/json.c +++ b/src/json.c @@ -1029,8 +1029,6 @@ static int json5Whitespace(const char *zIn){ return n; } - -#ifdef SQLITE_EXTENDED_NAN_INF /* ** Extra floating-point literals to allow in JSON. */ @@ -1049,7 +1047,6 @@ static const struct NanInfName { { 'q', 'Q', 4, JSON_NULL, 4, "QNaN", "null" }, { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" }, }; -#endif /* SQLITE_EXTENDED_NAN_INF */ /* ** Parse a single JSON value which begins at pParse->zJson[i]. Return the @@ -1336,28 +1333,20 @@ json_parse_restart: } }else{ if( !sqlite3Isdigit(z[i+1]) ){ - if( z[i+1]=='I' && strncmp(&z[i+1], "Infinity",8)==0 ){ - if( z[i]=='-' ){ - jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); - }else{ - jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); - } - return i+9; - } -#ifdef SQLITE_EXTENDED_NAN_INF - /* Non-standard JSON and JSON5: Allow "Inf" as an alternative - ** spelling for "Infinity" and allow it to be in any case. */ + /* JSON5 allows for "+Infinity" and "-Infinity" using exactly + ** that case. SQLite also allows these in any case and it allows + ** "+inf" and "-inf". */ if( (z[i+1]=='I' || z[i+1]=='i') && sqlite3StrNICmp(&z[i+1], "inf",3)==0 ){ + pParse->hasNonstd = 1; if( z[i]=='-' ){ jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); }else{ jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); } - return i+4; + return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); } -#endif if( z[i+1]=='.' ){ pParse->hasNonstd = 1; jnFlags |= JNODE_JSON5; @@ -1433,24 +1422,6 @@ json_parse_restart: jsonParseAddNode(pParse, seenDP | (jnFlags<<8), j - i, &z[i]); return j; } - case 'N': { - if( strncmp(&z[i],"NaN",3)==0 ){ - jsonParseAddNode(pParse, JSON_NULL, 4, "null"); - pParse->hasNonstd = 1; - return i+3; - } - pParse->iErr = i; - return -1; - } - case 'I': { - if( strncmp(&z[i],"Infinity",8)==0 ){ - jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); - pParse->hasNonstd = 1; - return i+8; - } - pParse->iErr = i; - return -1; - } case '}': { pParse->iErr = i; return -2; /* End of {...} */ @@ -1497,7 +1468,6 @@ json_parse_restart: return -1; } default: { -#ifdef SQLITE_EXTENDED_NAN_INF int k, nn; c = z[i]; for(k=0; khasNonstd = 1; return i + nn; } -#endif pParse->iErr = i; return -1; /* Syntax error */ } From dae7ae359ed3632b56907435a9606c92e0d0070b Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 30 Apr 2023 20:37:49 +0000 Subject: [PATCH 321/341] Accept the full ECMAScript 5.1 IdentifyName syntax for keys in objects. FossilOrigin-Name: 9be2c87518b33713210e3e2fa56924888e19415c71329141d18150b275f6f25e --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/global.c | 2 +- src/json.c | 22 +++++++++++++++++----- src/sqliteInt.h | 4 ++++ test/json501.test | 9 +++------ 6 files changed, 35 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 0b4e7c7276..5f77a9a3c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C All\sJSON\sto\sunderstand\sfloating\spoint\sliterals\s"Inf"\sand\s"QNaN"\sand\s"SNaN"\sin\nany\scase,\swithout\sthe\sSQLITE_EXTENDED_NAN_INF\scompile-time\soption.\s\sThis\nextension\sis\salways\savailable. -D 2023-04-30T19:45:25.975 +C Accept\sthe\sfull\sECMAScript\s5.1\sIdentifyName\ssyntax\sfor\skeys\sin\sobjects. +D 2023-04-30T20:37:49.462 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -589,13 +589,13 @@ F src/expr.c 6353f4d92d9f67ec3466d8e6978cd31a45e34cb755c4d11e689077f03f7c0a15 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 03c134cc8bffe54835f742ddea0b72ebfc8f6b32773d175c71b8afeea6cb5c83 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 -F src/global.c 428d2580a1cdf5dbe1f356d1feab83710ae0cc862ece0fb57bc8259e43838c74 +F src/global.c bd0892ade7289f6e20bff44c07d06371f2ff9b53cea359e7854b9b72f65adc30 F src/hash.c c6af5f96a7a76d000f07c5402c48c318c2566beecdee9e78b9d9f60ce7119565 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c bd22944b15d786a7a17f0faa672cdb3abeef1b59c505684df64caaca2a18a03a +F src/json.c 0915244c5eae7f80d4a7418ddac3186add8f7cc7185fd9fa7a64f5d9d3a5507b F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -640,7 +640,7 @@ F src/shell.c.in 09097e1b9df1f8092e85bf89979e12ca7b608d7efc84551b5d0c8de4dded779 F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 -F src/sqliteInt.h bf15f7db635d2e64a227bbf86845bc19755dbd932021a6461d6dd15b0da2cfd3 +F src/sqliteInt.h 6766b36c215e33d2cbfd48e632b7cc516092273d909672fdc095bcbd8c005ba1 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -1261,7 +1261,7 @@ F test/json102.test 13dc9e7b7f359ecb861e02f9bd7019f7342a63d1c354273b0a8f39040505 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d -F test/json501.test 9b14d14f26659287277a1f6fcf2a932a0a2351e1aae522586b530498c977ce45 +F test/json501.test f71710f60fa45b19dc336fbaac9e8362f70f80cf81badefdb845ed3f7c7c2ccc F test/json502.test 66d150cc098674b8bf4354526a8dd411b926f43ca892306bcb3b6d3f93fef7be F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/kvtest.c feb4358fb022da8ebd098c45811f2f6507688bb6c43aa72b3e840df19026317b @@ -2068,8 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 34c4e900a9cc51630eeaf01deef74bf5b18d66e0ab1dc61a2023ac8f837a5197 -R 6addb4498a8cf4456a809d957c22b7a6 +P fb551145e0d84213b3343dc1bc7db70c898b9dea24a72b968240617f4b52d821 +R b58bc780aaf2d8c5cd999008db262a63 U drh -Z 050299ed4706f61a98d79f7a5528f481 +Z 3139cc4432c28195fb82dc0d5445fb75 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index db3c346403..302a6bf45d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb551145e0d84213b3343dc1bc7db70c898b9dea24a72b968240617f4b52d821 \ No newline at end of file +9be2c87518b33713210e3e2fa56924888e19415c71329141d18150b275f6f25e \ No newline at end of file diff --git a/src/global.c b/src/global.c index b018c50027..fcba7d7fa2 100644 --- a/src/global.c +++ b/src/global.c @@ -97,7 +97,7 @@ const unsigned char *sqlite3aGTb = &sqlite3UpperToLower[256+12-OP_Ne]; ** isalnum() 0x06 ** isxdigit() 0x08 ** toupper() 0x20 -** SQLite identifier character 0x40 +** SQLite identifier character 0x40 $, _, or non-ascii ** Quote character 0x80 ** ** Bit 0x20 is set if the mapped character requires translation to upper diff --git a/src/json.c b/src/json.c index c221193282..1c89cb43ce 100644 --- a/src/json.c +++ b/src/json.c @@ -552,7 +552,13 @@ static void jsonRenderNode( case JSON_STRING: { assert( pNode->eU==1 ); if( pNode->jnFlags & JNODE_RAW ){ - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + if( pNode->jnFlags & JNODE_LABEL ){ + jsonAppendChar(pOut, '"'); + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + jsonAppendChar(pOut, '"'); + }else{ + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + } }else if( pNode->jnFlags & JNODE_JSON5 ){ jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); }else{ @@ -1046,7 +1052,7 @@ static const struct NanInfName { { 'n', 'N', 3, JSON_NULL, 4, "NaN", "null" }, { 'q', 'Q', 4, JSON_NULL, 4, "QNaN", "null" }, { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" }, -}; +}; /* ** Parse a single JSON value which begins at pParse->zJson[i]. Return the @@ -1088,9 +1094,15 @@ json_parse_restart: break; } j += json5Whitespace(&z[j]); - if( sqlite3Isalpha(z[j]) || z[j]=='_' || z[j]=='$' ){ - int k; - for(k=j+1; sqlite3Isalnum(z[k]) || z[k]=='_' || z[k]=='$'; k++){} + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && z[j+1]=='u' && jsonIs4Hex(&z[j+2])) + ){ + int k = j+1; + while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) + || (z[k]=='\\' && z[k+1]=='u' && jsonIs4Hex(&z[k+2])) + ){ + k++; + } jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), k-j, &z[j]); pParse->hasNonstd = 1; x = k; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4739951a5d..b06e192553 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4456,6 +4456,8 @@ int sqlite3CantopenError(int); # define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) # define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) # define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) +# define sqlite3JsonId1(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x42) +# define sqlite3JsonId2(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x46) #else # define sqlite3Toupper(x) toupper((unsigned char)(x)) # define sqlite3Isspace(x) isspace((unsigned char)(x)) @@ -4465,6 +4467,8 @@ int sqlite3CantopenError(int); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) # define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') +# define sqlite3JsonId1(x) (sqlite3IsIdChar(x)&&(x)<'0') +# define sqlite3JsonId2(x) sqlite3IsIdChar(x) #endif int sqlite3IsIdChar(u8); diff --git a/test/json501.test b/test/json501.test index 4d6a058936..a37326973a 100644 --- a/test/json501.test +++ b/test/json501.test @@ -86,12 +86,9 @@ do_catchsql_test 1.10 { SELECT json('{ MNO_123/xyz : 789 }'); } {1 {malformed JSON}} -# Contra the JSON5 standard, SQLite does not allow non-ASCII characters in -# an unquoted object label. -# -do_catchsql_test 1.11 { - SELECT json('{ MNO_123æxyz : 789 }'); -} {1 {malformed JSON}} +do_execsql_test 1.11 { + SELECT '{ MNO_123æxyz : 789 }'->>'MNO_123æxyz'; +} {789} ############################################################################### # 2) Objects may have a single trailing comma. From 5196d9350556cbd301d7bd89a342a068207a4869 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sun, 30 Apr 2023 23:52:55 +0000 Subject: [PATCH 322/341] Fix a problem with json_patch() when one side or the other is JSON5. dbsqlfuzz bc10593a4ba8e7a7862593532285be31f00f8e41 FossilOrigin-Name: e18c0899cc774f6f20d2a73381fa3ab3af9355cf4d108c7612db259eadbb8b96 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/json.c | 15 +++++++++++---- test/json104.test | 42 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 5f77a9a3c6..5733fbf0e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Accept\sthe\sfull\sECMAScript\s5.1\sIdentifyName\ssyntax\sfor\skeys\sin\sobjects. -D 2023-04-30T20:37:49.462 +C Fix\sa\sproblem\swith\sjson_patch()\swhen\sone\sside\sor\sthe\sother\sis\sJSON5.\ndbsqlfuzz\sbc10593a4ba8e7a7862593532285be31f00f8e41 +D 2023-04-30T23:52:55.182 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -595,7 +595,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 0915244c5eae7f80d4a7418ddac3186add8f7cc7185fd9fa7a64f5d9d3a5507b +F src/json.c 359e99789cbc5db24e14b70e8f615155f5ae0138528eb863bdc114e6c6205c32 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -1259,7 +1259,7 @@ F test/json/json-speed-check.sh 8b7babf530faa58bd59d6d362cec8e9036a68c5457ff46f3 F test/json101.test 211d75638782370c07e10a847bd66e501ea1537f39c7da4447bfa055c0f261db F test/json102.test 13dc9e7b7f359ecb861e02f9bd7019f7342a63d1c354273b0a8f3904050560a8 F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a8880112dbe -F test/json104.test a502dc01853aada95d721b3b275afbe2dc18fffdac1fea6e96fb20c13586bbb5 +F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test 11670a4387f4308ae0318cadcbd6a918ea7edcd19fbafde020720a073952675d F test/json501.test f71710f60fa45b19dc336fbaac9e8362f70f80cf81badefdb845ed3f7c7c2ccc F test/json502.test 66d150cc098674b8bf4354526a8dd411b926f43ca892306bcb3b6d3f93fef7be @@ -2068,8 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb551145e0d84213b3343dc1bc7db70c898b9dea24a72b968240617f4b52d821 -R b58bc780aaf2d8c5cd999008db262a63 +P 9be2c87518b33713210e3e2fa56924888e19415c71329141d18150b275f6f25e +R 7a6b3c9a24ecf3fbc1a69d64509d805d U drh -Z 3139cc4432c28195fb82dc0d5445fb75 +Z 5033e057782cc2feec974bc4cc7a7826 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 302a6bf45d..3e1e31f59e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9be2c87518b33713210e3e2fa56924888e19415c71329141d18150b275f6f25e \ No newline at end of file +e18c0899cc774f6f20d2a73381fa3ab3af9355cf4d108c7612db259eadbb8b96 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 1c89cb43ce..7e112c30dc 100644 --- a/src/json.c +++ b/src/json.c @@ -1675,7 +1675,7 @@ static JsonParse *jsonParseCached( ** Compare the OBJECT label at pNode against zKey,nKey. Return true on ** a match. */ -static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ +static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ assert( pNode->eU==1 ); if( pNode->jnFlags & JNODE_RAW ){ if( pNode->n!=nKey ) return 0; @@ -1685,6 +1685,15 @@ static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; } } +static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ + if( p1->jnFlags & JNODE_RAW ){ + return jsonLabelCompare(p2, p1->u.zJContent, p1->n); + }else if( p2->jnFlags & JNODE_RAW ){ + return jsonLabelCompare(p1, p2->u.zJContent, p2->n); + }else{ + return p1->n==p2->n && strncmp(p1->u.zJContent,p2->u.zJContent,p1->n)==0; + } +} /* forward declaration */ static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); @@ -2246,12 +2255,10 @@ static JsonNode *jsonMergePatch( assert( pPatch[i].eU==1 ); nKey = pPatch[i].n; zKey = pPatch[i].u.zJContent; - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ assert( pTarget[j].eType==JSON_STRING ); assert( pTarget[j].jnFlags & JNODE_LABEL ); - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); - if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){ if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; if( pPatch[i+1].eType==JSON_NULL ){ pTarget[j+1].jnFlags |= JNODE_REMOVE; diff --git a/test/json104.test b/test/json104.test index 56dd2738c3..c3c43d1e98 100644 --- a/test/json104.test +++ b/test/json104.test @@ -30,6 +30,48 @@ do_execsql_test json104-100 { } }'); } {{{"a":"z","c":{"d":"e"}}}} +do_execsql_test json104-101 { + SELECT json_patch('{ + "a": "b", + "c": { + "d": "e", + "f": "g" + } + }','{ + a:"z", + c: { + f: null + } + }'); +} {{{"a":"z","c":{"d":"e"}}}} +do_execsql_test json104-102 { + SELECT json_patch('{ + a: "b", + c: { + d: "e", + f: "g" + } + }','{ + "a":"z", + "c": { + "f": null + } + }'); +} {{{"a":"z","c":{"d":"e"}}}} +do_execsql_test json104-103 { + SELECT json_patch('{ + a: "b", + c: { + d: "e", + f: "g" + } + }','{ + a:"z", + c: { + f: null + } + }'); +} {{{"a":"z","c":{"d":"e"}}}} # This is the example from pages 4 and 5 of RFC-7396 From e210c9390321379f9a5c0b7a205e67128ab12192 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 1 May 2023 03:56:48 +0000 Subject: [PATCH 323/341] Fix abbreviated paths for objects such that they work even if the object key begins with '$'. FossilOrigin-Name: 1b991c78141a9915ae9350ecb347a758e50d7d25c8a0f4cc098ae10d47c27043 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5733fbf0e5..28fca241a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sjson_patch()\swhen\sone\sside\sor\sthe\sother\sis\sJSON5.\ndbsqlfuzz\sbc10593a4ba8e7a7862593532285be31f00f8e41 -D 2023-04-30T23:52:55.182 +C Fix\sabbreviated\spaths\sfor\sobjects\ssuch\sthat\sthey\swork\seven\sif\sthe\sobject\nkey\sbegins\swith\s'$'. +D 2023-05-01T03:56:48.067 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -595,7 +595,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c 359e99789cbc5db24e14b70e8f615155f5ae0138528eb863bdc114e6c6205c32 +F src/json.c dd76caa9c9f1b417f7d7cdee06baabcbfc4c5ee18a578150d8f19c28260abeac F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -2068,8 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9be2c87518b33713210e3e2fa56924888e19415c71329141d18150b275f6f25e -R 7a6b3c9a24ecf3fbc1a69d64509d805d +P e18c0899cc774f6f20d2a73381fa3ab3af9355cf4d108c7612db259eadbb8b96 +R d7c6513749c47dbdaa7fb128b7fd565e U drh -Z 5033e057782cc2feec974bc4cc7a7826 +Z 1d9a6af1cfa9e670ad28e827814897c7 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3e1e31f59e..4bf24a906a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e18c0899cc774f6f20d2a73381fa3ab3af9355cf4d108c7612db259eadbb8b96 \ No newline at end of file +1b991c78141a9915ae9350ecb347a758e50d7d25c8a0f4cc098ae10d47c27043 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 7e112c30dc..aae48c291a 100644 --- a/src/json.c +++ b/src/json.c @@ -2164,7 +2164,7 @@ static void jsonExtractFunc( zPath = (const char*)sqlite3_value_text(argv[1]); if( zPath==0 ) return; if( flags & JSON_ABPATH ){ - if( zPath[0]!='$' ){ + if( zPath[0]!='$' || (zPath[1]!='.' && zPath[1]!='[' && zPath[1]!=0) ){ /* The -> and ->> operators accept abbreviated PATH arguments. This ** is mostly for compatibility with PostgreSQL, but also for ** convenience. From 908dec740443f3253ca74a325c17d8fc36c8a046 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 1 May 2023 11:24:35 +0000 Subject: [PATCH 324/341] Do not apply the "AND false" optimization if either operand comes from the ON clause of a join. Fix for the problem identified by [forum:/forumpost/96cd4a7e9e|forum post 96cd4a7e9e]. FossilOrigin-Name: d095da0e7a24e3bcab6495d964f76a86d7a5910d2d6edddc6e8092bfa6084fe6 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 23 +++++++++++++---------- test/join.test | 18 ++++++++++++++++++ 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 5b05d8a9f2..8207804f11 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scrash\sthat\scould\soccur\sin\sfts5\s'secure-delete'\smode\swhen\soperating\son\scorrupt\srecords. -D 2023-04-29T18:31:44.051 +C Do\snot\sapply\sthe\s"AND\sfalse"\soptimization\sif\seither\soperand\scomes\sfrom\sthe\nON\sclause\sof\sa\sjoin.\s\sFix\sfor\sthe\sproblem\sidentified\sby\n[forum:/forumpost/96cd4a7e9e|forum\spost\s96cd4a7e9e]. +D 2023-05-01T11:24:35.203 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -584,7 +584,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c a9c6d3f51c0a31e9b831e0a0580a98d702904b42d216fee530940e40dec34873 -F src/expr.c 6353f4d92d9f67ec3466d8e6978cd31a45e34cb755c4d11e689077f03f7c0a15 +F src/expr.c 871cfd80c516ee39d90414b2d3da2b5bc9c9e21fe87b7eb787ea7ae4b6461758 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 03c134cc8bffe54835f742ddea0b72ebfc8f6b32773d175c71b8afeea6cb5c83 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122 @@ -1229,7 +1229,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test ed1daf99958fed1b9f017e56bae2bb6b49339a1ec0b70b9e8f7259960c6bf387 +F test/join.test aea7a4f55b2d9eb8ef3434ea78f55b15bd688ab6136a11105c9c52f77424f199 F test/join2.test 8561fe82ce434ac96de91544072e578dc2cadddf2d9bc9cd802f866a9b92502e F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -2061,8 +2061,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07383758d68e05021ccd393a69c1fa94836dfb02331326c082495a531a384363 -R f659959a34cd18cd5104e952e98d50d8 -U dan -Z b8e56280f98e35721f913e42d252102b +P 2e85b0e3dcae0915aa6472a3654c8ac72a6b2083c11747f3f657c79bbdaf530b +R 13838bb1c49ef42cf7d6d72f16ec581a +U drh +Z e3cbaf1612201f89ba80e24327377d58 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8d196fa4ed..7b30323184 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e85b0e3dcae0915aa6472a3654c8ac72a6b2083c11747f3f657c79bbdaf530b \ No newline at end of file +d095da0e7a24e3bcab6495d964f76a86d7a5910d2d6edddc6e8092bfa6084fe6 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index ebaf13af17..9ffc3baded 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1113,9 +1113,9 @@ Select *sqlite3ExprListToValues(Parse *pParse, int nElem, ExprList *pEList){ ** Join two expressions using an AND operator. If either expression is ** NULL, then just return the other expression. ** -** If one side or the other of the AND is known to be false, then instead -** of returning an AND expression, just return a constant expression with -** a value of false. +** If one side or the other of the AND is known to be false, and neither side +** is part of an ON clause, then instead of returning an AND expression, +** just return a constant expression with a value of false. */ Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ sqlite3 *db = pParse->db; @@ -1123,14 +1123,17 @@ Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ return pRight; }else if( pRight==0 ){ return pLeft; - }else if( (ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight)) - && !IN_RENAME_OBJECT - ){ - sqlite3ExprDeferredDelete(pParse, pLeft); - sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3Expr(db, TK_INTEGER, "0"); }else{ - return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + u32 f = pLeft->flags | pRight->flags; + if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse + && !IN_RENAME_OBJECT + ){ + sqlite3ExprDeferredDelete(pParse, pLeft); + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3Expr(db, TK_INTEGER, "0"); + }else{ + return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + } } } diff --git a/test/join.test b/test/join.test index d87982e718..44bfb3bef2 100644 --- a/test/join.test +++ b/test/join.test @@ -1245,4 +1245,22 @@ do_eqp_test join-28.2 { # |--SCAN t4 # `--SEARCH t3 USING AUTOMATIC COVERING INDEX (a=?) + +# 2023-05-01 https://sqlite.org/forum/forumpost/96cd4a7e9e +# +reset_db +db null NULL +do_execsql_test join-29.1 { + CREATE TABLE t0(a INT); INSERT INTO t0(a) VALUES (1); + CREATE TABLE t1(b INT); INSERT INTO t1(b) VALUES (2); + CREATE VIEW v2(c) AS SELECT 3 FROM t1; + SELECT * FROM t1 JOIN v2 ON 0 FULL OUTER JOIN t0 ON true; +} {NULL NULL 1} +do_execsql_test join-29.2 { + SELECT * FROM t1 JOIN v2 ON 1=0 FULL OUTER JOIN t0 ON true; +} {NULL NULL 1} +do_execsql_test join-29.3 { + SELECT * FROM t1 JOIN v2 ON false FULL OUTER JOIN t0 ON true; +} {NULL NULL 1} + finish_test From b5fab681621cfb7279adf1a86945cc3dfe45690b Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 1 May 2023 14:50:54 +0000 Subject: [PATCH 325/341] Fix typos in documentation comments in sqlite3.h. FossilOrigin-Name: 6a8b00788e7911287a8287277eb603e983e3324c9d62de0889ce6f0b2d848f4a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 24 +++++++++++++----------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 8207804f11..e67b1b4eb6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sapply\sthe\s"AND\sfalse"\soptimization\sif\seither\soperand\scomes\sfrom\sthe\nON\sclause\sof\sa\sjoin.\s\sFix\sfor\sthe\sproblem\sidentified\sby\n[forum:/forumpost/96cd4a7e9e|forum\spost\s96cd4a7e9e]. -D 2023-05-01T11:24:35.203 +C Fix\stypos\sin\sdocumentation\scomments\sin\ssqlite3.h. +D 2023-05-01T14:50:54.874 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -636,7 +636,7 @@ F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 F src/shell.c.in 09097e1b9df1f8092e85bf89979e12ca7b608d7efc84551b5d0c8de4dded7797 -F src/sqlite.h.in 4fff9c6cc5d4cbba9532a668112efb6dc469c425e1a2196664d7c07d508363ef +F src/sqlite.h.in db063a877be70f44c05722c92b137d007c960b1152656e3e05870f7aa80140f1 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 F src/sqliteInt.h bf15f7db635d2e64a227bbf86845bc19755dbd932021a6461d6dd15b0da2cfd3 @@ -2061,8 +2061,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2e85b0e3dcae0915aa6472a3654c8ac72a6b2083c11747f3f657c79bbdaf530b -R 13838bb1c49ef42cf7d6d72f16ec581a +P d095da0e7a24e3bcab6495d964f76a86d7a5910d2d6edddc6e8092bfa6084fe6 +R f2f5771bd049d567e34841ebfeb665ab U drh -Z e3cbaf1612201f89ba80e24327377d58 +Z e03bcd2c96e85f3280181572692b47be # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7b30323184..37b6c013b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d095da0e7a24e3bcab6495d964f76a86d7a5910d2d6edddc6e8092bfa6084fe6 \ No newline at end of file +6a8b00788e7911287a8287277eb603e983e3324c9d62de0889ce6f0b2d848f4a \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 117a61f8ca..becaf706a2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2398,7 +2398,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_DQS_DML]] -**
SQLITE_DBCONFIG_DQS_DML +**
SQLITE_DBCONFIG_DQS_DML
**
The SQLITE_DBCONFIG_DQS_DML option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DML statements ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The @@ -2407,7 +2407,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_DQS_DDL]] -**
SQLITE_DBCONFIG_DQS_DDL +**
SQLITE_DBCONFIG_DQS_DDL
**
The SQLITE_DBCONFIG_DQS option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DDL statements, ** such as CREATE TABLE and CREATE INDEX. The @@ -2416,7 +2416,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] -**
SQLITE_DBCONFIG_TRUSTED_SCHEMA +**
SQLITE_DBCONFIG_TRUSTED_SCHEMA
**
The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to ** assume that database schemas are untainted by malicious content. ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite @@ -2436,7 +2436,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] -**
SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +**
SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
**
The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly ** created database file to have a schema format version number (the 4-byte @@ -2458,23 +2458,25 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] -**
SQLITE_DBCONFIG_STMT_SCANSTATUS +**
SQLITE_DBCONFIG_STMT_SCANSTATUS
**
The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in ** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears ** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() ** statistics. For statistics to be collected, the flag must be set on ** the database handle both when the SQL statement is prepared and when it ** is stepped. The flag is set (collection of statistics is enabled) -** by default.
+** by default. +** ** ** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] -**
SQLITE_DBCONFIG_REVERSE_SCANORDER +**
SQLITE_DBCONFIG_REVERSE_SCANORDER
**
The SQLITE_DBCONFIG_REVERSE_SCANORDER option change the default order ** in which tables and indexes are scanned so that the scans start at the end ** and work toward the beginning rather than starting at the beginning and -** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the -** same as setting [PRAGMA reverse_unordered_selects]. This configuration option -** is useful for application testing.
+** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. +** This configuration option is useful for application testing. +** ** ** */ @@ -2496,7 +2498,7 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ #define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ #define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ From d45bbbb05c5f5f497892d3f67cc80ede35aa072e Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 1 May 2023 15:15:11 +0000 Subject: [PATCH 326/341] Implement full xSync() for the OPFS VFS. The previous implementation was not correct for journal files. Reported in [forum:647d2f811dbc2dfe|forum post 647d2f811dbc2dfe]. FossilOrigin-Name: a371374148a2874be6e48890781aa5952229056b146a50fa4d035693341c5636 --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 5 ++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index 13747ffecf..e74279a556 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -780,7 +780,10 @@ const installOpfsVfs = function callee(options){ }, xSync: function(pFile,flags){ ++metrics.xSync.count; - return 0; // impl'd in xFileControl() + mTimeStart('xSync'); + const rc = opRun('xSync', pFile, flags); + mTimeEnd(); + return rc; }, xTruncate: function(pFile,sz64){ mTimeStart('xTruncate'); diff --git a/manifest b/manifest index e67b1b4eb6..997b302e77 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\sdocumentation\scomments\sin\ssqlite3.h. -D 2023-05-01T14:50:54.874 +C Implement\sfull\sxSync()\sfor\sthe\sOPFS\sVFS.\sThe\sprevious\simplementation\swas\snot\scorrect\sfor\sjournal\sfiles.\sReported\sin\s[forum:647d2f811dbc2dfe|forum\spost\s647d2f811dbc2dfe]. +D 2023-05-01T15:15:11.340 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -498,7 +498,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 14130dc3f0830cd994162684eee3c42615aa3bca15950d2a7b48fe5c4b6e657f +F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 98c8baa06f584d344cfb3fa2b044850298c99f38de93308cbfc82d0be3f21c2a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c c42413ca9f3e64c424b2bbfc5decf639670ca38bc8f7afb7760d5379398c9307 F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc @@ -2061,8 +2061,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d095da0e7a24e3bcab6495d964f76a86d7a5910d2d6edddc6e8092bfa6084fe6 -R f2f5771bd049d567e34841ebfeb665ab -U drh -Z e03bcd2c96e85f3280181572692b47be +P 6a8b00788e7911287a8287277eb603e983e3324c9d62de0889ce6f0b2d848f4a +R a2d3684a8de39a1ecf23e8e0e43ca91c +U stephan +Z aea06ba82a9b4dee27236c5c06825c43 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 37b6c013b4..baa63e8461 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a8b00788e7911287a8287277eb603e983e3324c9d62de0889ce6f0b2d848f4a \ No newline at end of file +a371374148a2874be6e48890781aa5952229056b146a50fa4d035693341c5636 \ No newline at end of file From 043353483e1bb376669ab056374e57d85e491731 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 1 May 2023 15:42:59 +0000 Subject: [PATCH 327/341] Expose the SQLITE_DBCONFIG_STMT_SCANSTATUS and SQLITE_DBCONFIG_REVERSE_SCANORDER sqlite3_db_config() options to JS. FossilOrigin-Name: 0a7024af3f0e1dddc1a60c67bc4084e46b297dc16fe753bd5e6a45a59fd1cfa4 --- ext/wasm/api/sqlite3-api-prologue.js | 5 +++++ ext/wasm/api/sqlite3-wasm.c | 4 ++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 61c8a4c62e..c882d5b247 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -1537,6 +1537,9 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( Full docs: https://sqlite.org/c3ref/db_config.html Returns capi.SQLITE_MISUSE if op is not a valid operation ID. + + The variants which take `(int, int*)` arguments treat a + missing or falsy pointer argument as 0. */ capi.sqlite3_db_config = function(pDb, op, ...args){ if(!this.s){ @@ -1565,6 +1568,8 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( case capi.SQLITE_DBCONFIG_ENABLE_VIEW: case capi.SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: case capi.SQLITE_DBCONFIG_TRUSTED_SCHEMA: + case capi.SQLITE_DBCONFIG_STMT_SCANSTATUS: + case capi.SQLITE_DBCONFIG_REVERSE_SCANORDER: return this.ip(pDb, op, args[0], args[1] || 0); case capi.SQLITE_DBCONFIG_LOOKASIDE: return this.pii(pDb, op, args[0], args[1], args[2]); diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index 61ebe084da..dbe594dc3d 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -567,6 +567,8 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_DBCONFIG_ENABLE_VIEW); DefInt(SQLITE_DBCONFIG_LEGACY_FILE_FORMAT); DefInt(SQLITE_DBCONFIG_TRUSTED_SCHEMA); + DefInt(SQLITE_DBCONFIG_STMT_SCANSTATUS); + DefInt(SQLITE_DBCONFIG_REVERSE_SCANORDER); DefInt(SQLITE_DBCONFIG_MAX); } _DefGroup; @@ -1545,6 +1547,8 @@ int sqlite3_wasm_db_config_ip(sqlite3 *pDb, int op, int arg1, int* pArg2){ case SQLITE_DBCONFIG_ENABLE_VIEW: case SQLITE_DBCONFIG_LEGACY_FILE_FORMAT: case SQLITE_DBCONFIG_TRUSTED_SCHEMA: + case SQLITE_DBCONFIG_STMT_SCANSTATUS: + case SQLITE_DBCONFIG_REVERSE_SCANORDER: return sqlite3_db_config(pDb, op, arg1, pArg2); default: return SQLITE_MISUSE; } diff --git a/manifest b/manifest index 997b302e77..9736d31489 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\sfull\sxSync()\sfor\sthe\sOPFS\sVFS.\sThe\sprevious\simplementation\swas\snot\scorrect\sfor\sjournal\sfiles.\sReported\sin\s[forum:647d2f811dbc2dfe|forum\spost\s647d2f811dbc2dfe]. -D 2023-05-01T15:15:11.340 +C Expose\sthe\sSQLITE_DBCONFIG_STMT_SCANSTATUS\sand\sSQLITE_DBCONFIG_REVERSE_SCANORDER\ssqlite3_db_config()\soptions\sto\sJS. +D 2023-05-01T15:42:59.473 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -493,14 +493,14 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af F ext/wasm/api/sqlite3-api-cleanup.js cc21e3486da748463e02bbe51e2464c6ac136587cdfd5aa00cd0b5385f6ca808 F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 2691a34a741015127b210954a1b9586764d3ff0c8a20f00fd15c00f339ecc79f -F ext/wasm/api/sqlite3-api-prologue.js 461ffa5a95f4c1935b3970a58790d3cdca62b16e9b9a6a8d993a2a47d7561f51 +F ext/wasm/api/sqlite3-api-prologue.js 17f4ec398ba34c5c666fea8e8c4eb82064a35b302f2f2eb355283cd8d3f68ed5 F ext/wasm/api/sqlite3-api-worker1.js 40a5b1813fcbe789f23ae196c833432c8c83e7054d660194ddfc51eab1c5b9bf F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 70914ae97784d3028150bbf252e07a423056c42cc345903c81b5fae661ce512f F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 98c8baa06f584d344cfb3fa2b044850298c99f38de93308cbfc82d0be3f21c2a F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c c42413ca9f3e64c424b2bbfc5decf639670ca38bc8f7afb7760d5379398c9307 +F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js 2710a06a59620c6bf7ce298ab1fb6c9ce825b9f9379728b74c486db6613beecc F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75 F ext/wasm/batch-runner.html 4deeed44fe41496dc6898d9fb17938ea3291f40f4bfb977e29d0cef96fbbe4c8 @@ -2061,8 +2061,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6a8b00788e7911287a8287277eb603e983e3324c9d62de0889ce6f0b2d848f4a -R a2d3684a8de39a1ecf23e8e0e43ca91c +P a371374148a2874be6e48890781aa5952229056b146a50fa4d035693341c5636 +R d452f64d5ce046943954f32e4affde81 U stephan -Z aea06ba82a9b4dee27236c5c06825c43 +Z 96e100114fe8b633cd672db89ee75258 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index baa63e8461..edbcb1a4b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a371374148a2874be6e48890781aa5952229056b146a50fa4d035693341c5636 \ No newline at end of file +0a7024af3f0e1dddc1a60c67bc4084e46b297dc16fe753bd5e6a45a59fd1cfa4 \ No newline at end of file From 4cc6f869a45aec35804599b8e5941429540391fa Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 1 May 2023 15:57:46 +0000 Subject: [PATCH 328/341] New enhancements to the sqlite3_db_config() constants documentation. FossilOrigin-Name: d74011a3c495719fe1816e15251269824396ac2a40e41f7b96f0dd507c9be609 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 20 +++++++++++++++----- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 9736d31489..42a4c3cf28 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expose\sthe\sSQLITE_DBCONFIG_STMT_SCANSTATUS\sand\sSQLITE_DBCONFIG_REVERSE_SCANORDER\ssqlite3_db_config()\soptions\sto\sJS. -D 2023-05-01T15:42:59.473 +C New\senhancements\sto\sthe\ssqlite3_db_config()\sconstants\sdocumentation. +D 2023-05-01T15:57:46.245 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -636,7 +636,7 @@ F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 F src/shell.c.in 09097e1b9df1f8092e85bf89979e12ca7b608d7efc84551b5d0c8de4dded7797 -F src/sqlite.h.in db063a877be70f44c05722c92b137d007c960b1152656e3e05870f7aa80140f1 +F src/sqlite.h.in 6066996620c2a97193518148ab2d3cedf37d8ee7667dafa96c207a86152b2cfb F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 F src/sqliteInt.h bf15f7db635d2e64a227bbf86845bc19755dbd932021a6461d6dd15b0da2cfd3 @@ -2061,8 +2061,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a371374148a2874be6e48890781aa5952229056b146a50fa4d035693341c5636 -R d452f64d5ce046943954f32e4affde81 -U stephan -Z 96e100114fe8b633cd672db89ee75258 +P 0a7024af3f0e1dddc1a60c67bc4084e46b297dc16fe753bd5e6a45a59fd1cfa4 +R 3926c1c37fb11b88e72adbce8d76f377 +U drh +Z 66cedfafa0d44bbe2c7f93043ff6be34 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index edbcb1a4b0..6891853f66 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a7024af3f0e1dddc1a60c67bc4084e46b297dc16fe753bd5e6a45a59fd1cfa4 \ No newline at end of file +d74011a3c495719fe1816e15251269824396ac2a40e41f7b96f0dd507c9be609 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index becaf706a2..e31c899b63 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2445,7 +2445,7 @@ struct sqlite3_mem_methods { ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, ** newly created databases are generally not understandable by SQLite versions ** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there -** is now scarcely any need to generated database files that are compatible +** is now scarcely any need to generate database files that are compatible ** all the way back to version 3.0.0, and so this setting is of little ** practical use, but is provided so that SQLite can continue to claim the ** ability to generate new database files that are compatible with version @@ -2465,17 +2465,27 @@ struct sqlite3_mem_methods { ** statistics. For statistics to be collected, the flag must be set on ** the database handle both when the SQL statement is prepared and when it ** is stepped. The flag is set (collection of statistics is enabled) -** by default. +** by default. This option takes two arguments: an integer and a pointer to +** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the statement scanstatus option. If the second argument +** is not NULL, then the value of the statement scanstatus setting after +** processing the first argument is written into the integer that the second +** argument points to. ** ** ** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] **
SQLITE_DBCONFIG_REVERSE_SCANORDER
-**
The SQLITE_DBCONFIG_REVERSE_SCANORDER option change the default order +**
The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order ** in which tables and indexes are scanned so that the scans start at the end ** and work toward the beginning rather than starting at the beginning and ** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the -** same as setting [PRAGMA reverse_unordered_selects]. -** This configuration option is useful for application testing. +** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** two arguments which are an integer and a pointer to an integer. The first +** argument is 1, 0, or -1 to enable, disable, or leave unchanged the +** reverse scan order flag, respectively. If the second argument is not NULL, +** then 0 or 1 is written into the integer that the second argument points to +** depending on if the reverse scan order flag is set after processing the +** first argument. **
** ** From 5ecee3dae30dc88248f501868dca975582a77f42 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 May 2023 15:59:20 +0000 Subject: [PATCH 329/341] Fix typos in comments in sqlite3session.h preventing documentation from being correctly generated. FossilOrigin-Name: ab75170d5609c477613466e8880f20b74b0069281db6536fe09db06ea9d7cff3 --- ext/session/sqlite3session.h | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h index b5be316478..1ea90dce47 100644 --- a/ext/session/sqlite3session.h +++ b/ext/session/sqlite3session.h @@ -80,16 +80,20 @@ int sqlite3session_create( void sqlite3session_delete(sqlite3_session *pSession); /* -** CAPIREF: Conigure a Session Object +** CAPI3REF: Configure a Session Object ** METHOD: sqlite3_session ** ** This method is used to configure a session object after it has been -** created. At present the only valid value for the second parameter is -** [SQLITE_SESSION_OBJCONFIG_SIZE]. +** created. At present the only valid values for the second parameter are +** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID]. ** -** Arguments for sqlite3session_object_config() +*/ +int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); + +/* +** CAPI3REF: Options for sqlite3session_object_config ** -** The following values may passed as the the 4th parameter to +** The following values may passed as the the 2nd parameter to ** sqlite3session_object_config(). ** **
SQLITE_SESSION_OBJCONFIG_SIZE
@@ -118,10 +122,6 @@ void sqlite3session_delete(sqlite3_session *pSession); ** It is an error (SQLITE_MISUSE) to attempt to modify this setting after ** the first table has been attached to the session object. */ -int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); - -/* -*/ #define SQLITE_SESSION_OBJCONFIG_SIZE 1 #define SQLITE_SESSION_OBJCONFIG_ROWID 2 diff --git a/manifest b/manifest index 42a4c3cf28..141110069e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\senhancements\sto\sthe\ssqlite3_db_config()\sconstants\sdocumentation. -D 2023-05-01T15:57:46.245 +C Fix\stypos\sin\scomments\sin\ssqlite3session.h\spreventing\sdocumentation\sfrom\sbeing\scorrectly\sgenerated. +D 2023-05-01T15:59:20.750 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -472,7 +472,7 @@ F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63b F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc F ext/session/sqlite3session.c c30768077832f04f6ae98ab5a5347eae78b4f840bc958e16ae8aa970261a315e -F ext/session/sqlite3session.h 24299a3b64f11afc4422ce92d030ffdb2d3181851a1763b4a0432e195b2a8a16 +F ext/session/sqlite3session.h 653e9d49c4edae231df8a4c8d69c2145195aedb32462d4b44229dbee7d2680fb F ext/session/test_session.c 5285482f83cd92b4c1fe12fcf88210566a18312f4f2aa110f6399dae46aeccbb F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 @@ -2061,8 +2061,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a7024af3f0e1dddc1a60c67bc4084e46b297dc16fe753bd5e6a45a59fd1cfa4 -R 3926c1c37fb11b88e72adbce8d76f377 -U drh -Z 66cedfafa0d44bbe2c7f93043ff6be34 +P d74011a3c495719fe1816e15251269824396ac2a40e41f7b96f0dd507c9be609 +R 94e403657c9a5bd8b38c0149c27e975a +U dan +Z fbd061bc3bd6b18ca241f57c968aece1 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6891853f66..00af3754c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d74011a3c495719fe1816e15251269824396ac2a40e41f7b96f0dd507c9be609 \ No newline at end of file +ab75170d5609c477613466e8880f20b74b0069281db6536fe09db06ea9d7cff3 \ No newline at end of file From 59fe13e39a118e64558291e3fe211203043b84d4 Mon Sep 17 00:00:00 2001 From: drh <> Date: Mon, 1 May 2023 18:52:35 +0000 Subject: [PATCH 330/341] Fix harmless compiler warnings. FossilOrigin-Name: 0a926994691ccb02ea4f0ba3c2e8f728871ae67d24c705fbc0f7af81f38d5f47 --- ext/fts5/fts5_index.c | 4 ++-- ext/session/sqlite3session.c | 2 +- manifest | 19 +++++++++---------- manifest.uuid | 2 +- src/json.c | 6 +++--- src/shell.c.in | 2 +- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 36ac22bfa7..42065aeacf 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4809,7 +4809,7 @@ static void fts5DoSecureDelete( for(iIdx=0, iKeyOff=0; iIdxiStart ) break; + if( (iKeyOff+iVal)>(u32)iStart ) break; iKeyOff += iVal; } @@ -4937,7 +4937,7 @@ static void fts5FlushSecureDelete( i64 iRowid ){ const int f = FTS5INDEX_QUERY_SKIPHASH; - int nTerm = strlen(zTerm); + int nTerm = (int)strlen(zTerm); Fts5Iter *pIter = 0; /* Used to find term instance */ fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); diff --git a/ext/session/sqlite3session.c b/ext/session/sqlite3session.c index 89cffac199..c74c6b4578 100644 --- a/ext/session/sqlite3session.c +++ b/ext/session/sqlite3session.c @@ -1102,7 +1102,7 @@ static int sessionTableInfo( i = 0; if( bRowid ){ - int nName = strlen(SESSIONS_ROWID); + size_t nName = strlen(SESSIONS_ROWID); memcpy(pAlloc, SESSIONS_ROWID, nName+1); azCol[i] = (char*)pAlloc; pAlloc += nName+1; diff --git a/manifest b/manifest index a64e693499..42930021d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sJSON5. -D 2023-05-01T18:28:48.240 +C Fix\sharmless\scompiler\swarnings. +D 2023-05-01T18:52:35.233 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -92,7 +92,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 46af0b3c3c3f00bfc5bdd307434d7c5f0fa0678a034dd48345cd83b20068efbd F ext/fts5/fts5_expr.c 7d298d76ea010c339b26ca47f6f69e9aef520ea46c083deaa4e83e87cf0e94b1 F ext/fts5/fts5_hash.c d4fb70940359f2120ccd1de7ffe64cc3efe65de9e8995b822cd536ff64c96982 -F ext/fts5/fts5_index.c a84273c6dd091bca6f78444797e49da1b07fabb8eb41ae42add91bba9d1bbd02 +F ext/fts5/fts5_index.c a626b7838fbe38db8826dd5776c3e0d1fb9a9ce7b3411dbed7be5e39c39121a7 F ext/fts5/fts5_main.c b4dba04a36aaf9b8e8cef0100b6dbb422cc74753eacc11d6401cac7a87c0f38d F ext/fts5/fts5_storage.c 76c6085239eb44424004c022e9da17a5ecd5aaec859fba90ad47d3b08f4c8082 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -472,7 +472,7 @@ F ext/session/sessionrowid.test 6323ba831721205fd729929745038fd54e9d128c66c654b8 F ext/session/sessionsize.test 8fcf4685993c3dbaa46a24183940ab9f5aa9ed0d23e5fb63bfffbdb56134b795 F ext/session/sessionstat1.test b039e38e2ba83767b464baf39b297cc0b1cc6f3292255cb467ea7e12d0d0280c F ext/session/sessionwor.test 6fd9a2256442cebde5b2284936ae9e0d54bde692d0f5fd009ecef8511f4cf3fc -F ext/session/sqlite3session.c c30768077832f04f6ae98ab5a5347eae78b4f840bc958e16ae8aa970261a315e +F ext/session/sqlite3session.c d14d3043d4e04b080090d92b46e7b01ed25f49985161c8cb4fba05eefaf85850 F ext/session/sqlite3session.h 653e9d49c4edae231df8a4c8d69c2145195aedb32462d4b44229dbee7d2680fb F ext/session/test_session.c 5285482f83cd92b4c1fe12fcf88210566a18312f4f2aa110f6399dae46aeccbb F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 @@ -595,7 +595,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h b638809e083b601b618df877b2e89cb87c2a47a01f4def10be4c4ebb54664ac7 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c a8de1db43335fc4946370a7a7e47d89975ad678ddb15078a150e993ba2fb37d4 -F src/json.c dd76caa9c9f1b417f7d7cdee06baabcbfc4c5ee18a578150d8f19c28260abeac +F src/json.c 77686019edebe229dfaabada4180b4dca6d3122ea82f981e178b77b7a9340a16 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c be5af440f3192c58681b5d43167dbca3ccbfce394d89faa22378a14264781136 F src/main.c 09bc5191f75dc48fc4dfddda143cb864c0c3dbc3297eb9a9c8e01fea58ff847d @@ -636,7 +636,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 3e53e02ce87c9582bd7e7d22f13f4094a271678d9dc72820fa257a2abb5e4032 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c f879cef11c462a2c37a8c906932781e384c3bb32042c355a704a043029c90d27 -F src/shell.c.in 09097e1b9df1f8092e85bf89979e12ca7b608d7efc84551b5d0c8de4dded7797 +F src/shell.c.in 3984d857a1a5d76eff261dfe8d7bef1f0e51e586af0215365301b5c0bd5ffec4 F src/sqlite.h.in 6066996620c2a97193518148ab2d3cedf37d8ee7667dafa96c207a86152b2cfb F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2068,9 +2068,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ab75170d5609c477613466e8880f20b74b0069281db6536fe09db06ea9d7cff3 1b991c78141a9915ae9350ecb347a758e50d7d25c8a0f4cc098ae10d47c27043 -R d47f0442b266c4b31be73caa106551c0 -T +closed 1b991c78141a9915ae9350ecb347a758e50d7d25c8a0f4cc098ae10d47c27043 +P f8c3ed23a6931b1da3b93b3274b132387078112a5c8e8d06b5312c47987d3937 +R 403020ca8f01732564c548c59e1a6a0f U drh -Z 86cda36754b86452b62afe6022ca194a +Z 5b27a0031a8a680d5f1735b2f157795b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0d7ef66824..fe62931d14 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8c3ed23a6931b1da3b93b3274b132387078112a5c8e8d06b5312c47987d3937 \ No newline at end of file +0a926994691ccb02ea4f0ba3c2e8f728871ae67d24c705fbc0f7af81f38d5f47 \ No newline at end of file diff --git a/src/json.c b/src/json.c index aae48c291a..b7cd6cbdf2 100644 --- a/src/json.c +++ b/src/json.c @@ -303,7 +303,7 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ ** features. */ static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ - int i; + u32 i; jsonAppendChar(p, '"'); zIn++; N -= 2; @@ -391,7 +391,7 @@ static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ ** features. */ static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ - int i; + u32 i; if( zIn[0]=='+' ){ zIn++; N--; @@ -2608,7 +2608,7 @@ static void jsonErrorFunc( sqlite3_result_int(ctx, 0); }else{ int n = 1; - int i; + u32 i; const char *z = p->zJson; for(i=0; iiErr && ALWAYS(z[i]); i++){ if( (z[i]&0xc0)!=0x80 ) n++; diff --git a/src/shell.c.in b/src/shell.c.in index 7285067bef..36f7b38cb8 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -5757,7 +5757,7 @@ static int sql_trace_callback( break; } case SQLITE_TRACE_PROFILE: { - sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; + sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0; utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); break; } From 58d40363b955ea096f903326bc30b8e265a74e0a Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 1 May 2023 19:28:07 +0000 Subject: [PATCH 331/341] Remove xFileControl() handling from the OPFS VFS altogether, re-routing all syncing through xSync() instead. This eliminates superfluous syncing introduced in [a371374148a2], as discussed in [forum:647d2f811dbc2dfe|forum thread 647d2f811dbc2dfe]. FossilOrigin-Name: f809de7f232c8c2731a877c7bf1ad39fda5b326b2e45e4748cab4dc5e5b535bd --- ext/wasm/api/sqlite3-vfs-opfs.c-pp.js | 12 ++++-------- ext/wasm/speedtest1-worker.html | 9 ++++----- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js index e74279a556..5c584702d8 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js @@ -353,7 +353,6 @@ const installOpfsVfs = function callee(options){ state.opIds.xClose = i++; state.opIds.xDelete = i++; state.opIds.xDeleteNoWait = i++; - state.opIds.xFileControl = i++; state.opIds.xFileSize = i++; state.opIds.xLock = i++; state.opIds.xOpen = i++; @@ -718,12 +717,9 @@ const installOpfsVfs = function callee(options){ return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; }, xFileControl: function(pFile, opId, pArg){ - mTimeStart('xFileControl'); - const rc = (capi.SQLITE_FCNTL_SYNC===opId) - ? opRun('xSync', pFile, 0) - : capi.SQLITE_NOTFOUND; - mTimeEnd(); - return rc; + /*mTimeStart('xFileControl'); + mTimeEnd();*/ + return capi.SQLITE_NOTFOUND; }, xFileSize: function(pFile,pSz64){ mTimeStart('xFileSize'); @@ -779,8 +775,8 @@ const installOpfsVfs = function callee(options){ return rc; }, xSync: function(pFile,flags){ - ++metrics.xSync.count; mTimeStart('xSync'); + ++metrics.xSync.count; const rc = opRun('xSync', pFile, flags); mTimeEnd(); return rc; diff --git a/ext/wasm/speedtest1-worker.html b/ext/wasm/speedtest1-worker.html index cd0fdb027c..0d88049b95 100644 --- a/ext/wasm/speedtest1-worker.html +++ b/ext/wasm/speedtest1-worker.html @@ -23,7 +23,7 @@
Downloading...
- +