From 04ab586bf3959da012113b399f09013afe553503 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 Dec 2018 21:13:41 +0000 Subject: [PATCH 01/52] Very slightly smaller and faster. FossilOrigin-Name: 27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/build.c | 2 +- src/insert.c | 1 + 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 85ce65b7a1..13139d928b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.26.0 -D 2018-12-01T12:34:55.966 +C Very\sslightly\ssmaller\sand\sfaster. +D 2018-12-01T21:13:41.707 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -450,7 +450,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c ba7c7eef4461790f37c309936bfc5d0d6ba9b194b02d3c8ff1fd53b420ea6d3b F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 -F src/build.c 127d33ad57b455a9339e9fabff41284c8b030cc6247ca7a2a6c0ad7abfc1ce85 +F src/build.c fce47a9789704e65c63299b01be8153745faee7919f5137d3f29b7c3c0b549bd F src/callback.c 789bd33d188146f66c0dd8306472a72d1c05f71924b24a91caf6bd45cf9aba73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@ -467,7 +467,7 @@ F src/hash.c 931ec82d7e070654a8facb42549bbb3a25720171d73ba94c3d3160580d01ef1f F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 6b81aae27b196925d8ff78824f4bbd435d6a40cd38dc324685e21735bb402109 +F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 9050dd153b5583804184be9c9dee9ebb554178d6db1f8ac280899e8aad9060e6 F src/main.c 4cfb3913cc9e65d3ac649b1785ac753fc225d29425d5437e012f7eac0cefe0eb @@ -1779,10 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23684cb841ac2cb0d69e5470253bd96feb733762a7553b952a08470834fe85fa -R dcc394af10af6094607f56a6086fb748 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.26.0 * +P bf8c1b2b7a5960c282e543b9c293686dccff272512d08865f4600fb58238b4f9 +R 6e4818c911aa4ab70efb9a2cf2ccb395 U drh -Z 560a5decbb241d53eb97e72267b6e6e8 +Z 8e5b93098d46384d408591fc5bd49035 diff --git a/manifest.uuid b/manifest.uuid index fd08abb29b..a4307176ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf8c1b2b7a5960c282e543b9c293686dccff272512d08865f4600fb58238b4f9 \ No newline at end of file +27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b \ No newline at end of file diff --git a/src/build.c b/src/build.c index e0fed8a83f..ad1421d195 100644 --- a/src/build.c +++ b/src/build.c @@ -227,7 +227,7 @@ void sqlite3FinishCoding(Parse *pParse){ if( v && pParse->nErr==0 && !db->mallocFailed ){ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ - if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; + assert( pParse->pAinc==0 || pParse->nTab>0 ); sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; }else{ diff --git a/src/insert.c b/src/insert.c index 7a9413901f..0c036e494f 100644 --- a/src/insert.c +++ b/src/insert.c @@ -319,6 +319,7 @@ void sqlite3AutoincrementBegin(Parse *pParse){ aOp[7].p2 = memId+2; aOp[7].p1 = memId; aOp[10].p2 = memId; + if( pParse->nTab==0 ) pParse->nTab = 1; } } From fdc75c9f5326dc96c18664674ad00f382adf2177 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 2 Dec 2018 01:15:16 +0000 Subject: [PATCH 02/52] Omit a line of code that has no affect on the outcome. FossilOrigin-Name: 5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/resolve.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 13139d928b..58d3a04285 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Very\sslightly\ssmaller\sand\sfaster. -D 2018-12-01T21:13:41.707 +C Omit\sa\sline\sof\scode\sthat\shas\sno\saffect\son\sthe\soutcome. +D 2018-12-02T01:15:16.933 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -504,7 +504,7 @@ F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 4cfc44def0f0690ceaab8f6481f5d76284d7f9509aab6e218a679b4836a54614 +F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f F src/shell.c.in 482e23a370cbe5b0d4c73a0f0f5fce34f7caa08a14a8d75e12f0225c4e14915c @@ -1779,7 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bf8c1b2b7a5960c282e543b9c293686dccff272512d08865f4600fb58238b4f9 -R 6e4818c911aa4ab70efb9a2cf2ccb395 +P 27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b +R fc5314ff4e5a8de1b725ac2ced7d9c02 U drh -Z 8e5b93098d46384d408591fc5bd49035 +Z dc803f80871250c47b1ead21e4c26856 diff --git a/manifest.uuid b/manifest.uuid index a4307176ea..aac9cd9145 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b \ No newline at end of file +5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 0c7dfc0b25..effbe646fe 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -80,7 +80,7 @@ static void resolveAlias( if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } - ExprSetProperty(pDup, EP_Alias); +// ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, From 4a5cff73a730a65e7ee4a8f2d131d591e9bd74c5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Dec 2018 01:47:41 +0000 Subject: [PATCH 03/52] Fix a parser bug in the use of parentheses around table-valued functions. FossilOrigin-Name: 58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/parse.y | 6 ++++++ test/json101.test | 19 ++++++++++++++++++- test/tabfunc01.test | 5 +++++ 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 58d3a04285..2d4789227a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sa\sline\sof\scode\sthat\shas\sno\saffect\son\sthe\soutcome. -D 2018-12-02T01:15:16.933 +C Fix\sa\sparser\sbug\sin\sthe\suse\sof\sparentheses\saround\stable-valued\sfunctions. +D 2018-12-03T01:47:41.293 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -495,7 +495,7 @@ F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73 F src/pager.h 217921e81eb5fe455caa5cda96061959706bcdd29ddb57166198645ef7822ac3 -F src/parse.y 6840fe7c0b5eb4dd25ee5d075213bc8255ed4c0678d71bfb6744d0520d91c179 +F src/parse.y 5cf85c2b9dfac38ac4e2bf2776484705186ce2eda8631e65cc0b04bf566c1173 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c bf9fcea656dce1cd2cca6b77a1d1d3552050d55a31c98bf0d9f405930a83bc95 @@ -1053,7 +1053,7 @@ F test/journal3.test c9c29883f5bf535ae82ae21c472df6263806a22e467b6db7cd0d6d54530 F test/jrnlmode.test a6693f2bed4541a21e703aaa37bb3e10de154130645952933b82b2dec0a8b539 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test b40a9f5395d8e669b0bc3eb550ad2ae9e5ada01fbce23c446c2a30a305a6d575 +F test/json101.test 8f8977b00ba02f9a26c1d1f52f29f540f6d5eb162cbd5eb78bb805366d4ab26d F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1 F test/json103.test aff6b7a4c17d5a20b487a7bc1a274bfdc63b829413bdfb83bedac42ec7f67e3b F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa @@ -1343,7 +1343,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 54300134f76db817685194d2f0e63e3fbf7380b45e0d426e00a9aee752497cfb +F test/tabfunc01.test 20e98ffe55f35d8d33fd834ca8bf9d4b637e560af8fcd00464b4154d90a4db45 F test/table.test eb3463b7add9f16a5bb836badf118cf391b809d09fdccd1f79684600d07ec132 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1779,7 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b -R fc5314ff4e5a8de1b725ac2ced7d9c02 +P 5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 +R 0af9de2350de25f0ebcc1d0eef1387db U drh -Z dc803f80871250c47b1ead21e4c26856 +Z 15852f1b01d70da9fb801cb007d16bf9 diff --git a/manifest.uuid b/manifest.uuid index aac9cd9145..1d5b14e4de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 \ No newline at end of file +58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index b150c73e1f..3bb28ab5f5 100644 --- a/src/parse.y +++ b/src/parse.y @@ -664,6 +664,12 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; + if( pOld->fg.isTabFunc ){ + pNew->u1.pFuncArg = pOld->u1.pFuncArg; + pOld->u1.pFuncArg = 0; + pOld->fg.isTabFunc = 0; + pNew->fg.isTabFunc = 1; + } pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } diff --git a/test/json101.test b/test/json101.test index 9a93ee739f..534478df93 100644 --- a/test/json101.test +++ b/test/json101.test @@ -813,6 +813,23 @@ do_execsql_test json-14.170 { SELECT fullkey FROM json_tree('null'); } {$} - +# 2018-12-03 +# Make sure the table-valued functions contained within parentheses +# work correctly. +# +# Bug reported via private email. See TH3 for more information. +# +do_execsql_test json-15.100 { + SELECT * FROM JSON_EACH('{"a":1, "b":2}'); +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +do_execsql_test json-15.110 { + SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz; +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +do_execsql_test json-15.120 { + SELECT * FROM (JSON_EACH('{"a":1, "b":2}')); +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +do_execsql_test json-15.130 { + SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz; +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} finish_test diff --git a/test/tabfunc01.test b/test/tabfunc01.test index dfe3190b52..49f0df889e 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -125,6 +125,11 @@ do_execsql_test tabfunc01-4.3 { SELECT * FROM aux1.generate_series(1,4) } {1 2 3 4} +# 2018-12-03: Fix bug reported by by private email. +do_execsql_test tabfunc01-4.4 { + SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10; +} {1 3 5} + # The next series of tests is verifying that virtual table are able # to optimize the IN operator, even on terms that are not marked "omit". # When the generate_series virtual table is compiled for the testfixture, From de72d2a81a5911c7dcb5fb52ea489c94e336721f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Dec 2018 01:58:02 +0000 Subject: [PATCH 04/52] Remove two lines of unnecessary code, for a very small performance increase and size decrease. FossilOrigin-Name: 15824ccda0f110794a479b58fbf36082d8c383f34bae9dc0921d96547fb37869 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache1.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2d4789227a..2b309eede2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sparser\sbug\sin\sthe\suse\sof\sparentheses\saround\stable-valued\sfunctions. -D 2018-12-03T01:47:41.293 +C Remove\stwo\slines\sof\sunnecessary\scode,\sfor\sa\svery\ssmall\sperformance\sincrease\nand\ssize\sdecrease. +D 2018-12-03T01:58:02.016 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -498,7 +498,7 @@ F src/pager.h 217921e81eb5fe455caa5cda96061959706bcdd29ddb57166198645ef7822ac3 F src/parse.y 5cf85c2b9dfac38ac4e2bf2776484705186ce2eda8631e65cc0b04bf566c1173 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 -F src/pcache1.c bf9fcea656dce1cd2cca6b77a1d1d3552050d55a31c98bf0d9f405930a83bc95 +F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69 @@ -1779,7 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 -R 0af9de2350de25f0ebcc1d0eef1387db +P 58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 +R a804252de7d9904d6bfbaabce768affa U drh -Z 15852f1b01d70da9fb801cb007d16bf9 +Z e3cffb3b13ba67932ad3575ad4d1aeec diff --git a/manifest.uuid b/manifest.uuid index 1d5b14e4de..d71f9abd0c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 \ No newline at end of file +15824ccda0f110794a479b58fbf36082d8c383f34bae9dc0921d96547fb37869 \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index 2880c2c5e6..6df0f15d13 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -102,6 +102,7 @@ struct PgHdr1 { PCache1 *pCache; /* Cache that currently owns this page */ PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ }; /* @@ -570,7 +571,8 @@ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; - pPage->pLruPrev = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */ assert( pPage->isAnchor==0 ); assert( pPage->pCache->pGroup->lru.isAnchor==1 ); pPage->pCache->nRecyclable--; @@ -908,8 +910,9 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; pPage->pCache = pCache; - pPage->pLruPrev = 0; pPage->pLruNext = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */ *(void **)pPage->page.pExtra = 0; pCache->apHash[h] = pPage; if( iKey>pCache->iMaxKey ){ @@ -1069,7 +1072,7 @@ static void pcache1Unpin( /* It is an error to call this function if the page is already ** part of the PGroup LRU list. */ - assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); + assert( pPage->pLruNext==0 ); assert( PAGE_IS_PINNED(pPage) ); if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){ From 8c53b4e7f61218a63d64878453a6441bb8fe5815 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 3 Dec 2018 14:58:07 +0000 Subject: [PATCH 05/52] Update the autoconf makefile for MSVC. FossilOrigin-Name: 675aba1f8b989cfd99370704ecb09031026dc3321cccad122ea91d816e02fdba --- autoconf/Makefile.msc | 2 ++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 270c83c230..a131d4c6e7 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -283,6 +283,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1 !ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF @@ -937,6 +938,7 @@ LIBRESOBJS = SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1 !ENDIF diff --git a/manifest b/manifest index 2b309eede2..ea374024b8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\stwo\slines\sof\sunnecessary\scode,\sfor\sa\svery\ssmall\sperformance\sincrease\nand\ssize\sdecrease. -D 2018-12-03T01:58:02.016 +C Update\sthe\sautoconf\smakefile\sfor\sMSVC. +D 2018-12-03T14:58:07.611 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -14,7 +14,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am e14b629addaa1ce372b72043f28f40de2e32b7e211b6e0fc18dbb87989197e40 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.msc 9e73d9abaadb7a4951ddd0e947c5c791770f23bb1e04bfa50b43c01bee0575f2 +F autoconf/Makefile.msc ee8ab2b6b5d712f68acc4c186e3fad7b831b87a2b118e0a123043ea8a2c8b016 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac 308de24343e76ecfbe9a67f8fcd4c5216b790d230c5d9ce10210b7d5965d6192 @@ -1779,7 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 -R a804252de7d9904d6bfbaabce768affa -U drh -Z e3cffb3b13ba67932ad3575ad4d1aeec +P 15824ccda0f110794a479b58fbf36082d8c383f34bae9dc0921d96547fb37869 +R 6917ee4bf0492fb65a72a63e44f83083 +U mistachkin +Z 0fd7a981eb53d48862b14fa362aff3ba diff --git a/manifest.uuid b/manifest.uuid index d71f9abd0c..65c9608a7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15824ccda0f110794a479b58fbf36082d8c383f34bae9dc0921d96547fb37869 \ No newline at end of file +675aba1f8b989cfd99370704ecb09031026dc3321cccad122ea91d816e02fdba \ No newline at end of file From e89feee5c3028dc1232bec97086cd1b407fb188a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Dec 2018 16:14:49 +0000 Subject: [PATCH 06/52] Add the "remove_diacritics=2" option to the unicode61 tokenizer in both FTS5 and FTS3/4. FossilOrigin-Name: 06177f3f114b5d804b84c27ac843740282e2176fdf0f7a999feda0e1b624adec --- ext/fts3/fts3_unicode.c | 13 +++--- ext/fts3/fts3_unicode2.c | 62 ++++++++++++++++++----------- ext/fts3/unicode/mkunicode.tcl | 36 +++++++++++------ ext/fts3/unicode/parseunicode.tcl | 31 +++++++++++++-- ext/fts5/fts5_tokenize.c | 20 +++++++--- ext/fts5/fts5_unicode2.c | 66 +++++++++++++++++++------------ ext/fts5/test/fts5tokenizer.test | 2 +- ext/fts5/test/fts5umlaut.test | 65 ++++++++++++++++++++++++++++++ ext/fts5/test/fts5unicode3.test | 22 ++++++++--- manifest | 30 +++++++------- manifest.uuid | 2 +- test/fts4umlaut.test | 65 ++++++++++++++++++++++++++++++ 12 files changed, 320 insertions(+), 94 deletions(-) create mode 100644 ext/fts5/test/fts5umlaut.test create mode 100644 test/fts4umlaut.test diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index dfb2680c50..c02ea3990c 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -82,7 +82,7 @@ typedef struct unicode_cursor unicode_cursor; struct unicode_tokenizer { sqlite3_tokenizer base; - int bRemoveDiacritic; + int eRemoveDiacritic; int nException; int *aiException; }; @@ -227,17 +227,20 @@ static int unicodeCreate( pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); if( pNew==NULL ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(unicode_tokenizer)); - pNew->bRemoveDiacritic = 1; + pNew->eRemoveDiacritic = 1; for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; + pNew->eRemoveDiacritic = 1; } else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ - pNew->bRemoveDiacritic = 0; + pNew->eRemoveDiacritic = 0; + } + else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){ + pNew->eRemoveDiacritic = 2; } else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); @@ -350,7 +353,7 @@ static int unicodeNext( /* Write the folded case of the last character read to the output */ zEnd = z; - iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic); + iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic); if( iOut ){ WRITE_UTF8(zOut, iOut); } diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index da7251ed0c..41027b2546 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -159,32 +159,47 @@ int sqlite3FtsUnicodeIsalnum(int c){ ** E"). The resuls of passing a codepoint that corresponds to an ** uppercase letter are undefined. */ -static int remove_diacritic(int c){ +static int remove_diacritic(int c, int bComplex){ unsigned short aDia[] = { 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, }; char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', + '\0', 'a'|0x00, 'c'|0x00, 'e'|0x00, 'i'|0x00, 'n'|0x00, + 'o'|0x00, 'u'|0x00, 'y'|0x00, 'y'|0x00, 'a'|0x00, 'c'|0x00, + 'd'|0x00, 'e'|0x00, 'e'|0x00, 'g'|0x00, 'h'|0x00, 'i'|0x00, + 'j'|0x00, 'k'|0x00, 'l'|0x00, 'n'|0x00, 'o'|0x00, 'r'|0x00, + 's'|0x00, 't'|0x00, 'u'|0x00, 'u'|0x00, 'w'|0x00, 'y'|0x00, + 'z'|0x00, 'o'|0x00, 'u'|0x00, 'a'|0x00, 'i'|0x00, 'o'|0x00, + 'u'|0x00, 'u'|0x80, 'a'|0x80, 'g'|0x00, 'k'|0x00, 'o'|0x00, + 'o'|0x80, 'j'|0x00, 'g'|0x00, 'n'|0x00, 'a'|0x80, 'a'|0x00, + 'e'|0x00, 'i'|0x00, 'o'|0x00, 'r'|0x00, 'u'|0x00, 's'|0x00, + 't'|0x00, 'h'|0x00, 'a'|0x00, 'e'|0x00, 'o'|0x80, 'o'|0x00, + 'o'|0x80, 'y'|0x00, '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a'|0x00, 'b'|0x00, + 'c'|0x80, 'd'|0x00, 'd'|0x00, 'e'|0x80, 'e'|0x00, 'e'|0x80, + 'f'|0x00, 'g'|0x00, 'h'|0x00, 'h'|0x00, 'i'|0x00, 'i'|0x80, + 'k'|0x00, 'l'|0x00, 'l'|0x80, 'l'|0x00, 'm'|0x00, 'n'|0x00, + 'o'|0x80, 'p'|0x00, 'r'|0x00, 'r'|0x80, 'r'|0x00, 's'|0x00, + 's'|0x80, 't'|0x00, 'u'|0x00, 'u'|0x80, 'v'|0x00, 'w'|0x00, + 'w'|0x00, 'x'|0x00, 'y'|0x00, 'z'|0x00, 'h'|0x00, 't'|0x00, + 'w'|0x00, 'y'|0x00, 'a'|0x00, 'a'|0x80, 'a'|0x80, 'a'|0x80, + 'e'|0x00, 'e'|0x80, 'e'|0x80, 'i'|0x00, 'o'|0x00, 'o'|0x80, + 'o'|0x80, 'o'|0x80, 'u'|0x00, 'u'|0x80, 'u'|0x80, 'y'|0x00, }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; @@ -201,7 +216,8 @@ static int remove_diacritic(int c){ } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); } @@ -228,7 +244,7 @@ int sqlite3FtsUnicodeIsdiacritic(int c){ ** The results are undefined if the value passed to this function ** is less than zero. */ -int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ +int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ /* Each entry in the following array defines a rule for folding a range ** of codepoints to lower case. The rule applies to a range of nRange ** codepoints starting at codepoint iCode. @@ -351,7 +367,9 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ assert( ret>0 ); } - if( bRemoveDiacritic ) ret = remove_diacritic(ret); + if( eRemoveDiacritic ){ + ret = remove_diacritic(ret, eRemoveDiacritic==2); + } } else if( c>=66560 && c<66600 ){ diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 84b8ddc80b..36663beebd 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -9,11 +9,12 @@ proc print_rd {map} { set nRange 1 set iFirst [lindex $map 0 0] set cPrev [lindex $map 0 1] + set fPrev [lindex $map 0 2] foreach m [lrange $map 1 end] { - foreach {i c} $m {} + foreach {i c f} $m {} - if {$cPrev == $c} { + if {$cPrev == $c && $fPrev==$f} { for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} { if {[info exists tl_lookup_table($j)]==0} break } @@ -29,13 +30,16 @@ proc print_rd {map} { lappend lRange [list $iFirst $nRange] lappend aChar $cPrev + lappend aFlag $fPrev set iFirst $i set cPrev $c + set fPrev $f set nRange 1 } lappend lRange [list $iFirst $nRange] lappend aChar $cPrev + lappend aFlag $fPrev puts "/*" puts "** If the argument is a codepoint corresponding to a lowercase letter" @@ -45,7 +49,7 @@ proc print_rd {map} { puts "** E\"). The resuls of passing a codepoint that corresponds to an" puts "** uppercase letter are undefined." puts "*/" - puts "static int ${::remove_diacritic}(int c)\{" + puts "static int ${::remove_diacritic}(int c, int bComplex)\{" puts " unsigned short aDia\[\] = \{" puts -nonewline " 0, " set i 1 @@ -60,13 +64,17 @@ proc print_rd {map} { puts "" puts " \};" puts " char aChar\[\] = \{" - puts -nonewline " '\\0', " + puts -nonewline " '\\0', " set i 1 - foreach c $aChar { - set str "'$c', " - if {$c == ""} { set str "'\\0', " } + foreach c $aChar f $aFlag { + if { $f } { + set str "'$c'|0x80, " + } else { + set str "'$c'|0x00, " + } + if {$c == ""} { set str "'\\0', " } - if {($i % 12)==0} {puts "" ; puts -nonewline " " } + if {($i % 6)==0} {puts "" ; puts -nonewline " " } incr i puts -nonewline "$str" } @@ -87,7 +95,8 @@ proc print_rd {map} { } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);} + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);} puts "\}" } @@ -95,7 +104,8 @@ proc print_isdiacritic {zFunc map} { set lCode [list] foreach m $map { - foreach {code char} $m {} + foreach {code char flag} $m {} + if {$flag} continue if {$code && $char == ""} { lappend lCode $code } } set lCode [lsort -integer $lCode] @@ -472,7 +482,7 @@ proc print_fold {zFunc} { puts "** The results are undefined if the value passed to this function" puts "** is less than zero." puts "*/" - puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{" + puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{" set liOff [tl_generate_ioff_table $lRecord] tl_print_table_header @@ -516,7 +526,9 @@ proc print_fold {zFunc} { assert( ret>0 ); } - if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret); + if( eRemoveDiacritic ){ + ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2); + } } }] diff --git a/ext/fts3/unicode/parseunicode.tcl b/ext/fts3/unicode/parseunicode.tcl index 966d7bdd3a..7c246a4a09 100644 --- a/ext/fts3/unicode/parseunicode.tcl +++ b/ext/fts3/unicode/parseunicode.tcl @@ -7,12 +7,24 @@ # character that it should be replaced with, or an empty string if the # codepoint should simply be removed from the input. Examples: # -# { 224 a } (replace codepoint 224 to "a") -# { 769 "" } (remove codepoint 769 from input) +# { 224 a 0 } (replace codepoint 224 to "a") +# { 769 "" 0 } (remove codepoint 769 from input) # # Mappings are only returned for non-upper case codepoints. It is assumed # that the input has already been folded to lower case. # +# The third value in the list is always either 0 or 1. 0 if the +# UnicodeData.txt file maps the codepoint to a single ASCII character and +# a diacritic, or 1 if the mapping is indirect. For example, consider the +# two entries: +# +# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC +# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8 +# +# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a +# diacritic). The second is an indirect mapping, as it maps to the +# first codepoint plus 0302 (a diacritic). +# proc rd_load_unicodedata_text {zName} { global tl_lookup_table @@ -53,18 +65,29 @@ proc rd_load_unicodedata_text {zName} { set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"] set iDia [expr "0x[lindex $character_decomposition_mapping 1]"] + # Filter out upper-case characters, as they will be mapped to their + # lower-case equivalents before this data is used. if {[info exists tl_lookup_table($iCode)]} continue + # Check if this is an indirect mapping. If so, set bIndirect to true + # and change $iAscii to the indirectly mappped ASCII character. + set bIndirect 0 + if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} { + set iAscii $mapping($iAscii) + set bIndirect 1 + } + if { ($iAscii >= 97 && $iAscii <= 122) || ($iAscii >= 65 && $iAscii <= 90) } { - lappend lRet [list $iCode [string tolower [format %c $iAscii]]] + lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect] + set mapping($iCode) $iAscii set dia($iDia) 1 } } foreach d [array names dia] { - lappend lRet [list $d ""] + lappend lRet [list $d "" 0] } set lRet [lsort -integer -index 0 $lRet] diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index af2bc222f2..35526a3898 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -234,13 +234,18 @@ struct Unicode61Tokenizer { unsigned char aTokenChar[128]; /* ASCII range token characters */ char *aFold; /* Buffer to fold text into */ int nFold; /* Size of aFold[] in bytes */ - int bRemoveDiacritic; /* True if remove_diacritics=1 is set */ + int eRemoveDiacritic; /* True if remove_diacritics=1 is set */ int nException; int *aiException; unsigned char aCategory[32]; /* True for token char categories */ }; +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */ +#define FTS5_REMOVE_DIACRITICS_NONE 0 +#define FTS5_REMOVE_DIACRITICS_SIMPLE 1 +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2 + static int fts5UnicodeAddExceptions( Unicode61Tokenizer *p, /* Tokenizer object */ const char *z, /* Characters to treat as exceptions */ @@ -361,7 +366,7 @@ static int fts5UnicodeCreate( int i; memset(p, 0, sizeof(Unicode61Tokenizer)); - p->bRemoveDiacritic = 1; + p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE; p->nFold = 64; p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); if( p->aFold==0 ){ @@ -382,10 +387,15 @@ static int fts5UnicodeCreate( for(i=0; rc==SQLITE_OK && ieRemoveDiacritic = (zArg[0] - '0'); + assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX + ); } - p->bRemoveDiacritic = (zArg[0]=='1'); }else if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ rc = fts5UnicodeAddExceptions(p, zArg, 1); @@ -499,7 +509,7 @@ static int fts5UnicodeTokenize( READ_UTF8(zCsr, zTerm, iCode); if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){ non_ascii_tokenchar: - iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic); + iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic); if( iCode ) WRITE_UTF8(zOut, iCode); }else{ break; diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c index 8c48aaa49b..985b37cc68 100644 --- a/ext/fts5/fts5_unicode2.c +++ b/ext/fts5/fts5_unicode2.c @@ -28,32 +28,47 @@ ** E"). The resuls of passing a codepoint that corresponds to an ** uppercase letter are undefined. */ -static int fts5_remove_diacritic(int c){ +static int fts5_remove_diacritic(int c, int bComplex){ unsigned short aDia[] = { 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, }; char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', + '\0', 'a'|0x00, 'c'|0x00, 'e'|0x00, 'i'|0x00, 'n'|0x00, + 'o'|0x00, 'u'|0x00, 'y'|0x00, 'y'|0x00, 'a'|0x00, 'c'|0x00, + 'd'|0x00, 'e'|0x00, 'e'|0x00, 'g'|0x00, 'h'|0x00, 'i'|0x00, + 'j'|0x00, 'k'|0x00, 'l'|0x00, 'n'|0x00, 'o'|0x00, 'r'|0x00, + 's'|0x00, 't'|0x00, 'u'|0x00, 'u'|0x00, 'w'|0x00, 'y'|0x00, + 'z'|0x00, 'o'|0x00, 'u'|0x00, 'a'|0x00, 'i'|0x00, 'o'|0x00, + 'u'|0x00, 'u'|0x80, 'a'|0x80, 'g'|0x00, 'k'|0x00, 'o'|0x00, + 'o'|0x80, 'j'|0x00, 'g'|0x00, 'n'|0x00, 'a'|0x80, 'a'|0x00, + 'e'|0x00, 'i'|0x00, 'o'|0x00, 'r'|0x00, 'u'|0x00, 's'|0x00, + 't'|0x00, 'h'|0x00, 'a'|0x00, 'e'|0x00, 'o'|0x80, 'o'|0x00, + 'o'|0x80, 'y'|0x00, '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a'|0x00, 'b'|0x00, + 'c'|0x80, 'd'|0x00, 'd'|0x00, 'e'|0x80, 'e'|0x00, 'e'|0x80, + 'f'|0x00, 'g'|0x00, 'h'|0x00, 'h'|0x00, 'i'|0x00, 'i'|0x80, + 'k'|0x00, 'l'|0x00, 'l'|0x80, 'l'|0x00, 'm'|0x00, 'n'|0x00, + 'o'|0x80, 'p'|0x00, 'r'|0x00, 'r'|0x80, 'r'|0x00, 's'|0x00, + 's'|0x80, 't'|0x00, 'u'|0x00, 'u'|0x80, 'v'|0x00, 'w'|0x00, + 'w'|0x00, 'x'|0x00, 'y'|0x00, 'z'|0x00, 'h'|0x00, 't'|0x00, + 'w'|0x00, 'y'|0x00, 'a'|0x00, 'a'|0x80, 'a'|0x80, 'a'|0x80, + 'e'|0x00, 'e'|0x80, 'e'|0x80, 'i'|0x00, 'o'|0x00, 'o'|0x80, + 'o'|0x80, 'o'|0x80, 'u'|0x00, 'u'|0x80, 'u'|0x80, 'y'|0x00, }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; @@ -70,7 +85,8 @@ static int fts5_remove_diacritic(int c){ } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); } @@ -97,7 +113,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){ ** The results are undefined if the value passed to this function ** is less than zero. */ -int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ +int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){ /* Each entry in the following array defines a rule for folding a range ** of codepoints to lower case. The rule applies to a range of nRange ** codepoints starting at codepoint iCode. @@ -220,7 +236,9 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ assert( ret>0 ); } - if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret); + if( eRemoveDiacritic ){ + ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2); + } } else if( c>=66560 && c<66600 ){ @@ -231,11 +249,9 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ } -#if 0 int sqlite3Fts5UnicodeNCat(void) { return 32; } -#endif int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ aArray[0] = 1; @@ -756,7 +772,7 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ]; int n = (aFts5UnicodeData[iTbl] >> 5) + i; for(; i<128 && i Date: Mon, 3 Dec 2018 17:40:46 +0000 Subject: [PATCH 07/52] Remove the unused sqlite3Fts5UnicodeNCat() function. FossilOrigin-Name: 7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 --- ext/fts3/unicode/mkunicode.tcl | 4 ---- ext/fts5/fts5_unicode2.c | 6 ------ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 36663beebd..8465262033 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -617,10 +617,6 @@ proc print_categories {lMap} { set nCat [expr [llength [array names C]] + 1] puts [code { - int sqlite3Fts5UnicodeNCat(void) { - return $nCat; - } - int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ aArray[0] = 1; switch( zCat[0] ){ diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c index 985b37cc68..9d3ffdc972 100644 --- a/ext/fts5/fts5_unicode2.c +++ b/ext/fts5/fts5_unicode2.c @@ -248,11 +248,6 @@ int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){ return ret; } - -int sqlite3Fts5UnicodeNCat(void) { - return 32; -} - int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ aArray[0] = 1; switch( zCat[0] ){ @@ -777,4 +772,3 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ iTbl++; } } - diff --git a/manifest b/manifest index 4fad7bcfe3..faaabc7b06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"remove_diacritics=2"\soption\sto\sthe\sunicode61\stokenizer\sin\sboth\sFTS5\nand\sFTS3/4. -D 2018-12-03T16:14:49.664 +C Remove\sthe\sunused\ssqlite3Fts5UnicodeNCat()\sfunction. +D 2018-12-03T17:40:46.225 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -105,7 +105,7 @@ F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87 F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl 106bb4ff6365b36301fa4a009e5b4bf6ed02a2fbe9156349be9dfd9a92697cde +F ext/fts3/unicode/mkunicode.tcl 2ea30d8122ccf1e33142c9cc8913d8cad9eb6668db359a228f10aeb37e2ab863 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 5edc74ca603d71284d475886e6e91b5c5cf2e8e93e9ba3a36ba2f2440ee97948 @@ -122,7 +122,7 @@ F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6 F ext/fts5/fts5_test_tok.c 80de1a4b1a3caa216c3be8862440f0117a8357dd9b7cfc5a2a2ce11fe6eb64ae F ext/fts5/fts5_tokenize.c ca2b6a033794945ac505241a86b0aa978709c23aa2e6121984d3e3ede96003c8 -F ext/fts5/fts5_unicode2.c 051f207a76a90890009a8b5009ca0c9a327342ec4d10c2145b61a334784b713a +F ext/fts5/fts5_unicode2.c 26519f47d567c5a9b08c376edd38f1b8a773b45ff29f1ffd81d9edfe20f96e18 F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738 F ext/fts5/fts5_vocab.c fbe38044889b2d2d99babeeef239c620fb0332bb928a84506ac748d81500b354 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 @@ -1781,7 +1781,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 675aba1f8b989cfd99370704ecb09031026dc3321cccad122ea91d816e02fdba -R 7123bc66be985a6ec633c5a7bea2ac49 -U dan -Z 3c4b639682109864205c768a98540aa1 +P 06177f3f114b5d804b84c27ac843740282e2176fdf0f7a999feda0e1b624adec +R 0096a940ee50b37b4b72306aa1bf1dc3 +U drh +Z 71af346a1a6924b681515df9ade7b33a diff --git a/manifest.uuid b/manifest.uuid index 3654f68d6b..a264e6b5c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06177f3f114b5d804b84c27ac843740282e2176fdf0f7a999feda0e1b624adec \ No newline at end of file +7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 \ No newline at end of file From cfc45b1021d779cd3031bdfb8480173bb84f0e47 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Dec 2018 23:57:27 +0000 Subject: [PATCH 08/52] Reduce the size of the parser tables generated by Lemon by splitting the yyRuleInfo structure into separate yyRuleInfoLhs and yyRuleInfoNRhs arrays. FossilOrigin-Name: 70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/lemon.c | 11 +++++++++-- tool/lempar.c | 26 ++++++++++++++------------ 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index faaabc7b06..da9de6931b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sunused\ssqlite3Fts5UnicodeNCat()\sfunction. -D 2018-12-03T17:40:46.225 +C Reduce\sthe\ssize\sof\sthe\sparser\stables\sgenerated\sby\sLemon\sby\ssplitting\sthe\nyyRuleInfo\sstructure\sinto\sseparate\syyRuleInfoLhs\sand\syyRuleInfoNRhs\sarrays. +D 2018-12-03T23:57:27.083 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -1700,8 +1700,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c 60d1e1eb0f7ebae709f68f1472d77fbf291c5345cd98ff417219da7e74fd09e9 -F tool/lempar.c 452f12d40229847634a160e5666b6c4ec4392fd81941c3443861b48d497054cc +F tool/lemon.c c9ba01f6729c892ae3e0f55c8a2d694a7e6ec3dd3aa280b411a008ef69b410cd +F tool/lempar.c 61af95b8fac2bfd59c09d55330e78f3f5e352d7aa80bf37404b96ef795be3fdc F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1781,7 +1781,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 06177f3f114b5d804b84c27ac843740282e2176fdf0f7a999feda0e1b624adec -R 0096a940ee50b37b4b72306aa1bf1dc3 +P 7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 +R 457633af26ab8489c6f274ab3a7ad9fc U drh -Z 71af346a1a6924b681515df9ade7b33a +Z 7836770a12b5a3ad0791dca5d89f1095 diff --git a/manifest.uuid b/manifest.uuid index a264e6b5c4..30bc9f4cbf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 \ No newline at end of file +70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 1fca8b9755..7f0e557535 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -4590,13 +4590,20 @@ void ReportTable( tplt_print(out,lemp,lemp->overflow,&lineno); tplt_xfer(lemp->name,in,out,&lineno); - /* Generate the table of rule information + /* Generate the tables of rule information. yyRuleInfoLhs[] and + ** yyRuleInfoNRhs[]. ** ** Note: This code depends on the fact that rules are number ** sequentually beginning with 0. */ for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ - fprintf(out," { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i); + fprintf(out," %4d, /* (%d) ", rp->lhs->index, i); + rule_print(out, rp); + fprintf(out," */\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + fprintf(out," %3d, /* (%d) ", -rp->nrhs, i); rule_print(out, rp); fprintf(out," */\n"); lineno++; } diff --git a/tool/lempar.c b/tool/lempar.c index 325b0e5418..94c0a3162a 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -686,13 +686,15 @@ static void yy_shift( yyTraceShift(yypParser, yyNewState, "Shift"); } -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - signed char nrhs; /* Negative of the number of RHS symbols in the rule */ -} yyRuleInfo[] = { +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { +%% +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { %% }; @@ -725,7 +727,7 @@ static YYACTIONTYPE yy_reduce( yymsp = yypParser->yytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; + yysize = yyRuleInfoNRhs[yyruleno]; if( yysize ){ fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", yyTracePrompt, @@ -740,7 +742,7 @@ static YYACTIONTYPE yy_reduce( /* Check that the stack is large enough to grow by a single entry ** if the RHS of the rule is empty. This ensures that there is room ** enough on the stack to push the LHS value */ - if( yyRuleInfo[yyruleno].nrhs==0 ){ + if( yyRuleInfoNRhs[yyruleno]==0 ){ #ifdef YYTRACKMAXSTACKDEPTH if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -782,9 +784,9 @@ static YYACTIONTYPE yy_reduce( %% /********** End reduce actions ************************************************/ }; - assert( yyruleno Date: Tue, 4 Dec 2018 13:51:26 +0000 Subject: [PATCH 09/52] Small performance increase in sqlite3_step() for the common case where it returns SQLITE_ROW. FossilOrigin-Name: 893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 16 +++++++++------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index da9de6931b..0b06eaadfa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\ssize\sof\sthe\sparser\stables\sgenerated\sby\sLemon\sby\ssplitting\sthe\nyyRuleInfo\sstructure\sinto\sseparate\syyRuleInfoLhs\sand\syyRuleInfoNRhs\sarrays. -D 2018-12-03T23:57:27.083 +C Small\sperformance\sincrease\sin\ssqlite3_step()\sfor\sthe\scommon\scase\swhere\nit\sreturns\sSQLITE_ROW. +D 2018-12-04T13:51:26.090 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -581,7 +581,7 @@ F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 -F src/vdbeapi.c ecccfce6f614c33a95952efeec969d163e8349eac314ee2b7b163eda921b5eb0 +F src/vdbeapi.c 666993b7939530b3e16b21d07425809d4aaa5d75917ba34c513e91ee36c83467 F src/vdbeaux.c f547901b1aa9e2d81c63f06893f633648e434180666a827aacb547d7d6c8a601 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 @@ -1781,7 +1781,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 -R 457633af26ab8489c6f274ab3a7ad9fc +P 70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c +R 86cab02da9395ff714960a3873202ed5 U drh -Z 7836770a12b5a3ad0791dca5d89f1095 +Z a2cf828add1aa9e207a374e8188d7cf1 diff --git a/manifest.uuid b/manifest.uuid index 30bc9f4cbf..16705cde8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c \ No newline at end of file +893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 59327bed38..a4f4baf61e 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -629,16 +629,18 @@ static int sqlite3Step(Vdbe *p){ db->nVdbeExec--; } + if( rc!=SQLITE_ROW ){ #ifndef SQLITE_OMIT_TRACE - /* If the statement completed successfully, invoke the profile callback */ - if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); + /* If the statement completed successfully, invoke the profile callback */ + checkProfileCallback(db, p); #endif - if( rc==SQLITE_DONE && db->autoCommit ){ - assert( p->rc==SQLITE_OK ); - p->rc = doWalCallbacks(db); - if( p->rc!=SQLITE_OK ){ - rc = SQLITE_ERROR; + if( rc==SQLITE_DONE && db->autoCommit ){ + assert( p->rc==SQLITE_OK ); + p->rc = doWalCallbacks(db); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + } } } From 04c6747a804862554811113524b8a70d3a0675e1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Dec 2018 14:33:02 +0000 Subject: [PATCH 10/52] Performance improvement in sqlite3_step() by creating a new mTrace flag for the legacy xProfile pointer that is set by sqlite3_profile(). FossilOrigin-Name: e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/main.c | 6 ++++++ src/sqliteInt.h | 9 +++++++-- src/vdbeapi.c | 6 ++++-- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 0b06eaadfa..b69e0f957a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\sincrease\sin\ssqlite3_step()\sfor\sthe\scommon\scase\swhere\nit\sreturns\sSQLITE_ROW. -D 2018-12-04T13:51:26.090 +C Performance\simprovement\sin\ssqlite3_step()\sby\screating\sa\snew\smTrace\sflag\nfor\sthe\slegacy\sxProfile\spointer\sthat\sis\sset\sby\ssqlite3_profile(). +D 2018-12-04T14:33:02.434 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -471,7 +471,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 9050dd153b5583804184be9c9dee9ebb554178d6db1f8ac280899e8aad9060e6 -F src/main.c 4cfb3913cc9e65d3ac649b1785ac753fc225d29425d5437e012f7eac0cefe0eb +F src/main.c 5a94791735ddd51804b75b7ef4645c25cbe6fe1d0e8f25851b2ed1211ad3d076 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -512,7 +512,7 @@ F src/shell.c.in 482e23a370cbe5b0d4c73a0f0f5fce34f7caa08a14a8d75e12f0225c4e14915 F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h 1161f7579cdd6217737a66517ef27f4016426603eff492e9b31f45a7d7d4c61f +F src/sqliteInt.h f54ee1ef1e0f99d27af20561df72aac9158ed420cfc3a2e330fdee40672daf37 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -581,7 +581,7 @@ F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 -F src/vdbeapi.c 666993b7939530b3e16b21d07425809d4aaa5d75917ba34c513e91ee36c83467 +F src/vdbeapi.c dc825a6ec99a5066c1aa0d9824509057c0510f03cc8c72f81ba074553f8a5ae8 F src/vdbeaux.c f547901b1aa9e2d81c63f06893f633648e434180666a827aacb547d7d6c8a601 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 @@ -1781,7 +1781,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c -R 86cab02da9395ff714960a3873202ed5 +P 893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 +R 8ed134deac4c27b14c838ed0e6319d34 U drh -Z a2cf828add1aa9e207a374e8188d7cf1 +Z 84b5f74d4a6ae62e21d6700a38fd119f diff --git a/manifest.uuid b/manifest.uuid index 16705cde8c..07b135a092 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 \ No newline at end of file +e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e \ No newline at end of file diff --git a/src/main.c b/src/main.c index 46c83463c6..27a4206a4f 100644 --- a/src/main.c +++ b/src/main.c @@ -1996,6 +1996,7 @@ void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; db->xTrace = (int(*)(u32,void*,void*,void*))xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); @@ -2020,6 +2021,9 @@ int sqlite3_trace_v2( if( mTrace==0 ) xTrace = 0; if( xTrace==0 ) mTrace = 0; db->mTrace = mTrace; +#ifndef SQLITE_OMIT_DEPRECATED + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; +#endif db->xTrace = xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); @@ -2052,6 +2056,8 @@ void *sqlite3_profile( pOld = db->pProfileArg; db->xProfile = xProfile; db->pProfileArg = pArg; + db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK; + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; sqlite3_mutex_leave(db->mutex); return pOld; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 051aa40382..9f9d03a605 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1356,10 +1356,13 @@ void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing ** in the style of sqlite3_trace() */ -#define SQLITE_TRACE_LEGACY 0x80 +#define SQLITE_TRACE_LEGACY 0x40 /* Use the legacy xTrace */ +#define SQLITE_TRACE_XPROFILE 0x80 /* Use the legacy xProfile */ #else -#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_XPROFILE 0 #endif /* SQLITE_OMIT_DEPRECATED */ +#define SQLITE_TRACE_NONLEGACY_MASK 0x0f /* Normal flags */ /* @@ -1418,8 +1421,10 @@ struct sqlite3 { void **aExtension; /* Array of shared library handles */ int (*xTrace)(u32,void*,void*,void*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ +#ifndef SQLITE_OMIT_DEPRECATED void (*xProfile)(void*,const char*,u64); /* Profiling function */ void *pProfileArg; /* Argument to profile function */ +#endif void *pCommitArg; /* Argument to xCommitCallback() */ int (*xCommitCallback)(void*); /* Invoked at every commit. */ void *pRollbackArg; /* Argument to xRollbackCallback() */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index a4f4baf61e..3b040a5b01 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -62,14 +62,16 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ sqlite3_int64 iNow; sqlite3_int64 iElapse; assert( p->startTime>0 ); - assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 ); + assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 ); assert( db->init.busy==0 ); assert( p->zSql!=0 ); sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); iElapse = (iNow - p->startTime)*1000000; +#ifndef SQLITE_OMIT_DEPRECATED if( db->xProfile ){ db->xProfile(db->pProfileArg, p->zSql, iElapse); } +#endif if( db->mTrace & SQLITE_TRACE_PROFILE ){ db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse); } @@ -602,7 +604,7 @@ static int sqlite3Step(Vdbe *p){ ); #ifndef SQLITE_OMIT_TRACE - if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0) + if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 && !db->init.busy && p->zSql ){ sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); }else{ From 731dd6ebdaa4463286ed5843cd6d3bc6d506e117 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Dec 2018 16:51:42 +0000 Subject: [PATCH 11/52] Add the "index_usage" utility program. FossilOrigin-Name: df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 --- Makefile.in | 3 + Makefile.msc | 4 ++ main.mk | 4 ++ manifest | 17 ++--- manifest.uuid | 2 +- tool/index_usage.c | 164 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 185 insertions(+), 9 deletions(-) create mode 100644 tool/index_usage.c diff --git a/Makefile.in b/Makefile.in index d2f9710059..028f615cd7 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1306,6 +1306,9 @@ showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo showshm$(TEXE): $(TOP)/tool/showshm.c $(LTLINK) -o $@ $(TOP)/tool/showshm.c +index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS) + changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index e3feffab35..3b95d088d2 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2444,6 +2444,10 @@ showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) showshm.exe: $(TOP)\tool\showshm.c $(LTLINK) $(NO_WARN) $(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS) +index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ + $(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \ diff --git a/main.mk b/main.mk index d18313bdc7..7a85619732 100644 --- a/main.mk +++ b/main.mk @@ -995,6 +995,10 @@ showwal$(EXE): $(TOP)/tool/showwal.c sqlite3.o showshm$(EXE): $(TOP)/tool/showshm.c $(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c +index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \ + $(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB) + changeset$(EXE): $(TOP)/ext/session/changeset.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \ $(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index b69e0f957a..f8f7ebdbb6 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Performance\simprovement\sin\ssqlite3_step()\sby\screating\sa\snew\smTrace\sflag\nfor\sthe\slegacy\sxProfile\spointer\sthat\sis\sset\sby\ssqlite3_profile(). -D 2018-12-04T14:33:02.434 +C Add\sthe\s"index_usage"\sutility\sprogram. +D 2018-12-04T16:51:42.708 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea -F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c +F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 0d6831ff7951b302e888d86d4c469e2ec3c22f59eba4118b8c38d5a51d9e2d4f +F Makefile.msc b7d4a710fa3f0b8cfc532ff195b85dc1ba2a8ad34343cb3d67639f28f0a24306 F README.md 377233394b905d3b2e2b33741289e093bc93f2e7adbe00923b2c5958c9a9edee F VERSION 654da1d4053fb09ffc33a3910e6d427182a7dcdc67e934fa83de2849ac83fccb F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -429,7 +429,7 @@ F ext/userauth/userauth.c f81aa5a3ecacf406f170c62a144405858f6f6de51dbdc0920134e6 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk eeaa279fa6acdcfa6555058548075569a06f891fd67f5901b1e7700d18052fda +F main.mk 55f94164ecc194b067d9c55e106f37fd3c9b39f9668e8b568c98f008b6f9ec90 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1699,6 +1699,7 @@ F tool/fuzzershell.c e1d90a03ca790d7c331c2aae08ca46ff435f1ae1faa6cb9cc48f4687c18 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce +F tool/index_usage.c 8fd515b97522ae4b1aa6ca9847439f005d4cbaf6eb0eb416b694dba77c0263f0 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c c9ba01f6729c892ae3e0f55c8a2d694a7e6ec3dd3aa280b411a008ef69b410cd F tool/lempar.c 61af95b8fac2bfd59c09d55330e78f3f5e352d7aa80bf37404b96ef795be3fdc @@ -1781,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 -R 8ed134deac4c27b14c838ed0e6319d34 +P e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e +R 14a419221d3a581ec01dad82979f03fc U drh -Z 84b5f74d4a6ae62e21d6700a38fd119f +Z 0158a53fd6ddbd828c2808dfe15d0a38 diff --git a/manifest.uuid b/manifest.uuid index 07b135a092..19384dd637 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e \ No newline at end of file +df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 \ No newline at end of file diff --git a/tool/index_usage.c b/tool/index_usage.c new file mode 100644 index 0000000000..a86202425e --- /dev/null +++ b/tool/index_usage.c @@ -0,0 +1,164 @@ +/* +** 2018-12-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a utility program used to help determine which +** indexes in a database schema are used and unused, and how often specific +** indexes are used. +*/ +#include "sqlite3.h" +#include +#include +#include +#include + +static void usage(const char *argv0){ + printf("Usage: %s DATABASE LOG\n\n", argv0); + printf( + "DATABASE is an SQLite database against which various statements\n" + "have been run. The SQL text is stored in LOG. LOG is an SQLite\n" + "database with this schema:\n" + "\n" + " CREATE TABLE sqllog(sql TEXT);\n" + "\n" + "This utility program analyzes statements contained in LOG and prints\n" + "a report showing how many times each index in DATABASE is used by the\n" + "statements in LOG.\n" + "\n" + "DATABASE only needs to contain the schema used by the statements in\n" + "LOG. The content can be removed from DATABASE.\n" + ); + printf("\nAnalysis will be done by SQLite version %s dated %.20s\n" + "checkin number %.40s. Different versions\n" + "of SQLite might use different indexes.\n", + sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21); + exit(1); +} + +int main(int argc, char **argv){ + sqlite3 *db = 0; /* The main database */ + sqlite3_stmt *pStmt = 0; /* a query */ + char *zSql; + int nErr = 0; + int rc; + + if( argc!=3 ) usage(argv[0]); + rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0); + if( rc ){ + printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0); + if( rc ){ + printf("Cannot read the schema from \"%s\" - %s\n", argv[1], + sqlite3_errmsg(db)); + goto errorOut; + } + sqlite3_finalize(pStmt); + pStmt = 0; + rc = sqlite3_exec(db, + "CREATE TABLE temp.idxu(\n" + " tbl TEXT,\n" + " idx TEXT,\n" + " cnt INT,\n" + " PRIMARY KEY(idx)\n" + ") WITHOUT ROWID;", 0, 0, 0); + if( rc ){ + printf("Cannot create the result table - %s\n", + sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_exec(db, + "INSERT INTO temp.idxu(tbl,idx,cnt)" + " SELECT tbl_name, name, 0 FROM sqlite_master" + " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0); + + /* Open the LOG database */ + zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( rc ){ + printf("Cannot open the LOG database \"%s\" - %s\n", + argv[2], sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog", + -1, &pStmt, 0); + if( rc ){ + printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n", + argv[2], sqlite3_errmsg(db)); + goto errorOut; + } + + /* Update the counts based on LOG */ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zLog = (const char*)sqlite3_column_text(pStmt, 0); + sqlite3_stmt *pS2; + if( zLog==0 ) continue; + zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog); + rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0); + sqlite3_free(zSql); + if( rc ){ + printf("Cannot compile LOG entry %d (%s): %s\n", + sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db)); + nErr++; + }else{ + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zExplain = (const char*)sqlite3_column_text(pS2,3); + const char *z1, *z2; + int n; + /* printf("EXPLAIN: %s\n", zExplain); */ + z1 = strstr(zExplain, " USING INDEX "); + if( z1==0 ) continue; + z1 += 13; + for(z2=z1+1; z2[1] && z2[1]!='('; z2++){} + n = z2 - z1; + zSql = sqlite3_mprintf( + "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1 + ); + /* printf("sql: %s\n", zSql); */ + sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + } + sqlite3_finalize(pS2); + } + sqlite3_finalize(pStmt); + + /* Generate the report */ + rc = sqlite3_prepare_v2(db, + "SELECT tbl, idx, cnt, " + " (SELECT group_concat(name,',') FROM pragma_index_info(idx))" + " FROM temp.idxu, main.sqlite_master" + " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name" + " AND temp.idxu.idx=main.sqlite_master.name" + " ORDER BY cnt DESC, tbl, idx", + -1, &pStmt, 0); + if( rc ){ + printf("Cannot query the result table - %s\n", + sqlite3_errmsg(db)); + goto errorOut; + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("%10d %s on %s(%s)\n", + sqlite3_column_int(pStmt, 2), + sqlite3_column_text(pStmt, 1), + sqlite3_column_text(pStmt, 0), + sqlite3_column_text(pStmt, 3)); + } + sqlite3_finalize(pStmt); + pStmt = 0; + +errorOut: + sqlite3_finalize(pStmt); + sqlite3_close(db); + return nErr; +} From 707821ff72aa67bc1fbf7ed6df1617d3e19d372b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Dec 2018 13:39:06 +0000 Subject: [PATCH 12/52] Enhance the sqlite3_normalize_sql() interface so that it works even if the prepared statement was not initially compiled using SQLITE_PREPARE_NORMALIZED. Enhance the ".trace" command in the CLI so that it is able to access the full scope of functionality provided by sqlite3_trace_v2() and in particular so that it is able to show normalized SQL output using the newly enhanced sqlite3_normalize_sql() interface. FossilOrigin-Name: 7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5 --- manifest | 26 ++++----- manifest.uuid | 2 +- src/hash.c | 2 +- src/hash.h | 3 ++ src/prepare.c | 41 ++++++-------- src/shell.c.in | 136 +++++++++++++++++++++++++++++++++++++++-------- src/sqliteInt.h | 2 +- src/vdbeapi.c | 6 ++- src/vdbeaux.c | 2 +- test/shell4.test | 8 +-- 10 files changed, 159 insertions(+), 69 deletions(-) diff --git a/manifest b/manifest index f8f7ebdbb6..68a14a5b76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"index_usage"\sutility\sprogram. -D 2018-12-04T16:51:42.708 +C Enhance\sthe\ssqlite3_normalize_sql()\sinterface\sso\sthat\sit\sworks\seven\sif\sthe\nprepared\sstatement\swas\snot\sinitially\scompiled\susing\nSQLITE_PREPARE_NORMALIZED.\s\sEnhance\sthe\s".trace"\scommand\sin\sthe\sCLI\sso\sthat\nit\sis\sable\sto\saccess\sthe\sfull\sscope\sof\sfunctionality\sprovided\sby\s\nsqlite3_trace_v2()\sand\sin\sparticular\sso\sthat\sit\sis\sable\sto\sshow\snormalized\nSQL\soutput\susing\sthe\snewly\senhanced\ssqlite3_normalize_sql()\sinterface. +D 2018-12-05T13:39:06.092 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -464,8 +464,8 @@ F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812 F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f F src/global.c 8291eee0782b83124de14ec0389ec9fd6ae1873358a6b0d9469fe17a46ad803b -F src/hash.c 931ec82d7e070654a8facb42549bbb3a25720171d73ba94c3d3160580d01ef1f -F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 +F src/hash.c 6d2f67276469384fb8784fb8e962deeaae6832955626468325d705a01b999594 +F src/hash.h eebf2250e56b5d5353b873406557e373d2888cf51f111e28917666456d479e85 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 @@ -502,17 +502,17 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69 +F src/prepare.c 277c8af17124bd4f67ae6e1a8e795a91f12bf7ce3a4b344c7eac46d360250c20 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f -F src/shell.c.in 482e23a370cbe5b0d4c73a0f0f5fce34f7caa08a14a8d75e12f0225c4e14915c +F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h f54ee1ef1e0f99d27af20561df72aac9158ed420cfc3a2e330fdee40672daf37 +F src/sqliteInt.h 128f2ef4cd6037300cb5510dd3c882417b225d6f4ed6210bddfd920104c7341f F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -581,8 +581,8 @@ F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 -F src/vdbeapi.c dc825a6ec99a5066c1aa0d9824509057c0510f03cc8c72f81ba074553f8a5ae8 -F src/vdbeaux.c f547901b1aa9e2d81c63f06893f633648e434180666a827aacb547d7d6c8a601 +F src/vdbeapi.c 9ac7e3946a2762b79c314922c84c7de30731e0ed1d7c94ac82266b795221d9d7 +F src/vdbeaux.c 8e2fe020824b743090025ff6f9ffeec3ca4624523ddb2d1af1b1f61abaab3db4 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f @@ -1280,7 +1280,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test d2bf5daeb6f449f0169c9ef3094db17a16a02199c5dcf1a635a3e79b07eb0edd F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494 -F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d +F test/shell4.test a6881d0ae226ded0df8ebdfa574c5aa6dc28d6884ccba1089dc56ed08b9e5ef4 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e -R 14a419221d3a581ec01dad82979f03fc +P df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 +R e65d818662ab9003f3be5e2ff4aec20c U drh -Z 0158a53fd6ddbd828c2808dfe15d0a38 +Z 66f377ac461c094ae8c28a5487556c27 diff --git a/manifest.uuid b/manifest.uuid index 19384dd637..fab3fff857 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 \ No newline at end of file +7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5 \ No newline at end of file diff --git a/src/hash.c b/src/hash.c index fba9dc9f80..e725f2aa5e 100644 --- a/src/hash.c +++ b/src/hash.c @@ -72,7 +72,7 @@ static unsigned int strHashN(const char *z, int n){ /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). ** 0x9e3779b1 is 2654435761 which is the closest prime number to ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ - h += sqlite3UpperToLower[z[i]]; + h += sqlite3UpperToLower[(unsigned char)z[i]]; h *= 0x9e3779b1; } return h; diff --git a/src/hash.h b/src/hash.h index 90540eda50..cf773663b7 100644 --- a/src/hash.h +++ b/src/hash.h @@ -68,6 +68,9 @@ struct HashElem { void sqlite3HashInit(Hash*); void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); void *sqlite3HashFind(const Hash*, const char *pKey); +#ifdef SQLITE_ENABLE_NORMALIZE +void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey); +#endif void sqlite3HashClear(Hash*); /* diff --git a/src/prepare.c b/src/prepare.c index 4d33f0b1e1..86afac5350 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -791,8 +791,7 @@ done: */ static int estimateNormalizedSize( const char *zSql, /* The original SQL string */ - int nSql, /* Length of original SQL string */ - u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */ + int nSql /* Length of original SQL string */ ){ int nOut = nSql + 4; const char *z = zSql; @@ -847,18 +846,14 @@ static void copyNormalizedToken( } /* -** Perform normalization of the SQL contained in the prepared statement and -** store the result in the zNormSql field. The schema for the associated -** databases are consulted while performing the normalization in order to -** determine if a token appears to be an identifier. All identifiers are -** left intact in the normalized SQL and all literals are replaced with a -** single '?'. +** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return +** the normalization in space obtained from sqlite3DbMalloc(). Or return +** NULL if anything goes wrong or if zSql is NULL. */ -void sqlite3Normalize( +char *sqlite3Normalize( Vdbe *pVdbe, /* VM being reprepared */ const char *zSql, /* The original SQL string */ - int nSql, /* Size of the input string in bytes */ - u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */ + int nSql /* Size of the input string in bytes */ ){ sqlite3 *db; /* Database handle. */ char *z; /* The output string */ @@ -873,11 +868,10 @@ void sqlite3Normalize( db = sqlite3VdbeDb(pVdbe); assert( db!=0 ); - assert( pVdbe->zNormSql==0 ); - if( zSql==0 ) return; - nZ = estimateNormalizedSize(zSql, nSql, prepFlags); + if( zSql==0 ) return 0; + nZ = estimateNormalizedSize(zSql, nSql); z = sqlite3DbMallocRawNN(db, nZ); - if( z==0 ) return; + if( z==0 ) goto normalizeError; sqlite3HashInit(&inHash); for(i=j=0; i0 && z[j-1]!=';' ){ z[j++] = ';'; } z[j] = 0; assert( jzNormSql = z; sqlite3HashClear(&inHash); + return z; + +normalizeError: + sqlite3DbFree(db, z); + sqlite3HashClear(&inHash); + return 0; } #endif /* SQLITE_ENABLE_NORMALIZE */ diff --git a/src/shell.c.in b/src/shell.c.in index ad5a1498be..887c2fcf59 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1007,6 +1007,7 @@ struct ShellState { u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ u8 nEqpLevel; /* Depth of the EQP output graph */ + u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ @@ -1066,6 +1067,12 @@ struct ShellState { #define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ #define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ +/* Allowed values for ShellState.eTraceType +*/ +#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ +#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ +#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ + /* ** These are the allowed shellFlgs values */ @@ -3492,7 +3499,22 @@ static const char *(azHelp[]) = { ".testcase NAME Begin redirecting output to 'testcase-out.txt'", ".timeout MS Try opening locked tables for MS milliseconds", ".timer on|off Turn SQL timer on or off", - ".trace FILE|off Output each SQL statement as it is run", +#ifndef SQLITE_OMIT_TRACE + ".trace ?OPTIONS? Output each SQL statement as it is run", + " FILE Send output to FILE", + " stdout Send output to stdout", + " stderr Send output to stderr", + " off Disable tracing", + " --expanded Expand query parameters", +#ifdef SQLITE_ENABLE_NORMALIZE + " --normalized Normal the SQL statements", +#endif + " --plain Show SQL as it is input", + " --stmt Trace statement execution (SQLITE_TRACE_STMT)", + " --profile Profile statements (SQLITE_TRACE_PROFILE)", + " --row Trace each row (SQLITE_TRACE_ROW)", + " --close Trace connection close (SQLITE_TRACE_CLOSE)", +#endif /* SQLITE_OMIT_TRACE */ ".vfsinfo ?AUX? Information about the top-level VFS", ".vfslist List all available VFSes", ".vfsname ?AUX? Print the name of the VFS stack", @@ -3999,24 +4021,60 @@ static FILE *output_file_open(const char *zFile, int bTextMode){ return f; } -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) +#ifndef SQLITE_OMIT_TRACE /* ** A routine for handling output from sqlite3_trace(). */ static int sql_trace_callback( - unsigned mType, - void *pArg, - void *pP, - void *pX + unsigned mType, /* The trace type */ + void *pArg, /* The ShellState pointer */ + void *pP, /* Usually a pointer to sqlite_stmt */ + void *pX /* Auxiliary output */ ){ - FILE *f = (FILE*)pArg; - UNUSED_PARAMETER(mType); - UNUSED_PARAMETER(pP); - if( f ){ - const char *z = (const char*)pX; - int i = strlen30(z); - while( i>0 && z[i-1]==';' ){ i--; } - utf8_printf(f, "%.*s;\n", i, z); + ShellState *p = (ShellState*)pArg; + sqlite3_stmt *pStmt; + const char *zSql; + int nSql; + if( p->traceOut==0 ) return 0; + if( mType==SQLITE_TRACE_CLOSE ){ + utf8_printf(p->traceOut, "-- closing database connection\n"); + return 0; + } + if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + zSql = (const char*)pX; + }else{ + pStmt = (sqlite3_stmt*)pP; + switch( p->eTraceType ){ + case SHELL_TRACE_EXPANDED: { + zSql = sqlite3_expanded_sql(pStmt); + break; + } +#ifdef SQLITE_ENABLE_NORMALIZE + case SHELL_TRACE_NORMALIZED: { + zSql = sqlite3_normalized_sql(pStmt); + break; + } +#endif + default: { + zSql = sqlite3_sql(pStmt); + break; + } + } + } + if( zSql==0 ) return 0; + nSql = strlen30(zSql); + while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } + switch( mType ){ + case SQLITE_TRACE_ROW: + case SQLITE_TRACE_STMT: { + utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + break; + } + case SQLITE_TRACE_PROFILE: { + sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; + utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + break; + } } return 0; } @@ -7838,23 +7896,55 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else +#ifndef SQLITE_OMIT_TRACE if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + int mType = 0; + int jj; open_db(p, 0); - if( nArg!=2 ){ - raw_printf(stderr, "Usage: .trace FILE|off\n"); - rc = 1; - goto meta_command_exit; + for(jj=1; jjeTraceType = SHELL_TRACE_EXPANDED; + } +#ifdef SQLITE_ENABLE_NORMALIZE + else if( optionMatch(z, "normalized") ){ + p->eTraceType = SHELL_TRACE_NORMALIZED; + } +#endif + else if( optionMatch(z, "plain") ){ + p->eTraceType = SHELL_TRACE_PLAIN; + } + else if( optionMatch(z, "profile") ){ + mType |= SQLITE_TRACE_PROFILE; + } + else if( optionMatch(z, "row") ){ + mType |= SQLITE_TRACE_ROW; + } + else if( optionMatch(z, "stmt") ){ + mType |= SQLITE_TRACE_STMT; + } + else if( optionMatch(z, "close") ){ + mType |= SQLITE_TRACE_CLOSE; + } + else { + raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z); + rc = 1; + goto meta_command_exit; + } + }else{ + output_file_close(p->traceOut); + p->traceOut = output_file_open(azArg[1], 0); + } } - output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) if( p->traceOut==0 ){ sqlite3_trace_v2(p->db, 0, 0, 0); }else{ - sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut); + if( mType==0 ) mType = SQLITE_TRACE_STMT; + sqlite3_trace_v2(p->db, mType, sql_trace_callback, p); } -#endif }else +#endif /* !defined(SQLITE_OMIT_TRACE) */ #if SQLITE_USER_AUTHENTICATION if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9f9d03a605..fc52fecfe2 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4424,7 +4424,7 @@ int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); void sqlite3ParserReset(Parse*); #ifdef SQLITE_ENABLE_NORMALIZE -void sqlite3Normalize(Vdbe*, const char*, int, u8); +char *sqlite3Normalize(Vdbe*, const char*, int); #endif int sqlite3Reprepare(Vdbe*); void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 3b040a5b01..ca073589f7 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1712,7 +1712,11 @@ char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){ */ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; - return p ? p->zNormSql : 0; + if( p==0 ) return 0; + if( p->zNormSql==0 && p->zSql!=0 ){ + p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql)); + } + return p->zNormSql; } #endif /* SQLITE_ENABLE_NORMALIZE */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f1496a3abf..0ed9e81d90 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -67,7 +67,7 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){ #ifdef SQLITE_ENABLE_NORMALIZE assert( p->zNormSql==0 ); if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){ - sqlite3Normalize(p, p->zSql, n, prepFlags); + p->zNormSql = sqlite3Normalize(p, p->zSql, n); assert( p->zNormSql!=0 || p->db->mallocFailed ); } #endif diff --git a/test/shell4.test b/test/shell4.test index 88e5e69a28..2210e74a67 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -107,14 +107,14 @@ SELECT 1; } {1 1 1} do_test shell4-2.1 { - catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace" -} {1 {Usage: .trace FILE|off}} + catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown" +} {1 {Unknown option "--unknown" on ".trace"}} do_test shell4-2.2 { catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n" } {0 {}} do_test shell4-2.3 { - catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n" -} {/^1 {PRAGMA.*Usage:.*}$/} + catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n" +} {/^0 {PRAGMA.*}$/} ifcapable trace { do_test shell4-2.4 { catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;" From ea41251eb03c3fb9b59a70712b4dbd3a9dc782fe Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Dec 2018 13:49:04 +0000 Subject: [PATCH 13/52] Ensure that ALTER TABLE modifies table and column names embedded in WITH clauses that are part of views and triggers. FossilOrigin-Name: f44bc7a8b3fac82aa5598e9bdaf65ea4dd3c331cf90c1d5ba26ff1698e92c230 --- manifest | 16 ++++++------- manifest.uuid | 2 +- src/alter.c | 22 +++++++++++++++-- test/altertab2.test | 58 ++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 86 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 68a14a5b76..fabc9f14b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\ssqlite3_normalize_sql()\sinterface\sso\sthat\sit\sworks\seven\sif\sthe\nprepared\sstatement\swas\snot\sinitially\scompiled\susing\nSQLITE_PREPARE_NORMALIZED.\s\sEnhance\sthe\s".trace"\scommand\sin\sthe\sCLI\sso\sthat\nit\sis\sable\sto\saccess\sthe\sfull\sscope\sof\sfunctionality\sprovided\sby\s\nsqlite3_trace_v2()\sand\sin\sparticular\sso\sthat\sit\sis\sable\sto\sshow\snormalized\nSQL\soutput\susing\sthe\snewly\senhanced\ssqlite3_normalize_sql()\sinterface. -D 2018-12-05T13:39:06.092 +C Ensure\sthat\sALTER\sTABLE\smodifies\stable\sand\scolumn\snames\sembedded\sin\sWITH\nclauses\sthat\sare\spart\sof\sviews\sand\striggers. +D 2018-12-05T13:49:04.148 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -441,7 +441,7 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F src/alter.c f886160da189e4e99093cd5a2aca625652cc9b027d5100b87f81c175d1056387 +F src/alter.c 80747854ca90dadc7c51cc1da302e29081ca5f78a2a87b3d44f65d39db834ff5 F src/analyze.c 3dc6b98cf007b005af89df165c966baaa48e8124f38c87b4d2b276fe7f0b9eb9 F src/attach.c 92b51739a885da8bd84bc9a05485f1e48148bce5c15432f059b45af98fff75cd F src/auth.c 0fac71038875693a937e506bceb492c5f136dd7b1249fbd4ae70b4e8da14f9df @@ -615,7 +615,7 @@ F test/alterlegacy.test 82022721ce0de29cedc9a7af63bc9fcc078b0ee000f8283b4b6ea9c3 F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74a3fb80c9 F test/altermalloc2.test fa7b1c1139ea39b8dec407cf1feb032ca8e0076bd429574969b619175ad0174b F test/altertab.test 17e46baa6b2234048c91891a303141afceca4da95a36ee1a0a9fec6ccef1f4da -F test/altertab2.test 0d64de5632ca5de13b023839cfe5b8952d029e4622befcea1433adaa93883220 +F test/altertab2.test 814369c72a7ed777ab2acf3f17fcff5ecb724816eb7c6659f40ef87b09521c99 F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 7168c8bffa5d5cbc53c05b7e9c7fcdd24b365a1bc5046ce80c45efa3c02e6b7c F test/analyze3.test ff62d9029e6deb2c914490c6b00caf7fae47cc85cdc046e4a0d0a4d4b87c71d8 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 -R e65d818662ab9003f3be5e2ff4aec20c -U drh -Z 66f377ac461c094ae8c28a5487556c27 +P 7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5 +R 5be4b63edcb6a436be7d6ac2ccab0473 +U dan +Z 758395cc3e6658700c83b35c2e31d2cc diff --git a/manifest.uuid b/manifest.uuid index fab3fff857..7bb95e4443 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5 \ No newline at end of file +f44bc7a8b3fac82aa5598e9bdaf65ea4dd3c331cf90c1d5ba26ff1698e92c230 \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index 1280e90559..3c46406d79 100644 --- a/src/alter.c +++ b/src/alter.c @@ -780,14 +780,31 @@ static void renameTokenFind(Parse *pParse, struct RenameCtx *pCtx, void *pPtr){ } } +/* +** Iterate through the Select objects that are part of WITH clauses attached +** to select statement pSelect. +*/ +static void renameWalkWith(Walker *pWalker, Select *pSelect){ + if( pSelect->pWith ){ + int i; + for(i=0; ipWith->nCte; i++){ + Select *p = pSelect->pWith->a[i].pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pWalker->pParse; + sqlite3SelectPrep(sNC.pParse, p, &sNC); + sqlite3WalkSelect(pWalker, p); + } + } +} + /* ** This is a Walker select callback. It does nothing. It is only required ** because without a dummy callback, sqlite3WalkExpr() and similar do not ** descend into sub-select statements. */ static int renameColumnSelectCb(Walker *pWalker, Select *p){ - UNUSED_PARAMETER(pWalker); - UNUSED_PARAMETER(p); + renameWalkWith(pWalker, p); return WRC_Continue; } @@ -1364,6 +1381,7 @@ static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ renameTokenFind(pWalker->pParse, p, pItem->zName); } } + renameWalkWith(pWalker, pSelect); return WRC_Continue; } diff --git a/test/altertab2.test b/test/altertab2.test index 2e4212c35e..45a91537d0 100644 --- a/test/altertab2.test +++ b/test/altertab2.test @@ -12,7 +12,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix altertab +set testprefix altertab2 # If SQLITE_OMIT_ALTERTABLE is defined, omit this file. ifcapable !altertable { @@ -85,5 +85,61 @@ do_execsql_test 2.3 { {CREATE TABLE c3(x, FOREIGN KEY (x) REFERENCES "p3"(a))} } +#------------------------------------------------------------------------- +# Table name in WITH clauses that are part of views or triggers. +# +foreach {tn schema} { + 1 { + CREATE TABLE log_entry(col1, y); + CREATE INDEX i1 ON log_entry(col1); + } + + 2 { + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(x); + CREATE TABLE log_entry(col1); + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN + INSERT INTO t2 SELECT col1 FROM log_entry; + END; + } + + 3 { + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(x); + CREATE TABLE log_entry(col1); + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN + INSERT INTO t2 + WITH xyz(x) AS (SELECT col1 FROM log_entry) + SELECT x FROM xyz; + END; + } + + 4 { + CREATE TABLE log_entry(col1); + CREATE VIEW ttt AS + WITH xyz(x) AS (SELECT col1 FROM log_entry) + SELECT x FROM xyz; + } +} { + reset_db + do_execsql_test 3.$tn.1 $schema + set expect [db eval "SELECT sql FROM sqlite_master"] + set expect [string map {log_entry {"newname"}} $expect] + + do_execsql_test 3.$tn.2 { + ALTER TABLE log_entry RENAME TO newname; + SELECT sql FROM sqlite_master; + } $expect + + reset_db + do_execsql_test 3.$tn.3 $schema + set expect [db eval "SELECT sql FROM sqlite_master"] + set expect [string map {col1 newname} $expect] + + do_execsql_test 3.$tn.4 { + ALTER TABLE log_entry RENAME col1 TO newname; + SELECT sql FROM sqlite_master; + } $expect +} finish_test From 19efd0db4133f2d0f9985856ae9b4b5572abf733 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Dec 2018 17:48:57 +0000 Subject: [PATCH 14/52] Simplifications to the sqlite3_normalized_sql() implementation. FossilOrigin-Name: 94ea6379178e3ff6a0d1d5819ca4ac558bdadb1ca8a3637c797079db7dc0cd61 --- manifest | 22 +++++++++---------- manifest.uuid | 2 +- src/callback.c | 21 +++---------------- src/hash.c | 56 ------------------------------------------------- src/hash.h | 3 --- src/prepare.c | 27 ++++++++++++++++++------ src/sqliteInt.h | 4 +--- 7 files changed, 37 insertions(+), 98 deletions(-) diff --git a/manifest b/manifest index fabc9f14b7..06972a00ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sALTER\sTABLE\smodifies\stable\sand\scolumn\snames\sembedded\sin\sWITH\nclauses\sthat\sare\spart\sof\sviews\sand\striggers. -D 2018-12-05T13:49:04.148 +C Simplifications\sto\sthe\ssqlite3_normalized_sql()\simplementation. +D 2018-12-05T17:48:57.422 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -452,7 +452,7 @@ F src/btree.c ba7c7eef4461790f37c309936bfc5d0d6ba9b194b02d3c8ff1fd53b420ea6d3b F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 F src/build.c fce47a9789704e65c63299b01be8153745faee7919f5137d3f29b7c3c0b549bd -F src/callback.c 789bd33d188146f66c0dd8306472a72d1c05f71924b24a91caf6bd45cf9aba73 +F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 @@ -464,8 +464,8 @@ F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812 F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f F src/global.c 8291eee0782b83124de14ec0389ec9fd6ae1873358a6b0d9469fe17a46ad803b -F src/hash.c 6d2f67276469384fb8784fb8e962deeaae6832955626468325d705a01b999594 -F src/hash.h eebf2250e56b5d5353b873406557e373d2888cf51f111e28917666456d479e85 +F src/hash.c a12580e143f10301ed5166ea4964ae2853d3905a511d4e0c44497245c7ce1f7a +F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 @@ -502,7 +502,7 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c 277c8af17124bd4f67ae6e1a8e795a91f12bf7ce3a4b344c7eac46d360250c20 +F src/prepare.c f77ebf812f46a174e805d939cec0017771904e79049d04edf66dbbb3ec25ad3e F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 @@ -512,7 +512,7 @@ F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883 F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h 128f2ef4cd6037300cb5510dd3c882417b225d6f4ed6210bddfd920104c7341f +F src/sqliteInt.h e96520f27310016d0dabb496e13346244bbea83a6adc3134810e026f1d4a02b6 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5 -R 5be4b63edcb6a436be7d6ac2ccab0473 -U dan -Z 758395cc3e6658700c83b35c2e31d2cc +P f44bc7a8b3fac82aa5598e9bdaf65ea4dd3c331cf90c1d5ba26ff1698e92c230 +R 14f7302635d9f1332de56d5fb4d8f1a4 +U drh +Z b51bb6387d49f37b8a1232489a1c6ba3 diff --git a/manifest.uuid b/manifest.uuid index 7bb95e4443..6334ab25e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f44bc7a8b3fac82aa5598e9bdaf65ea4dd3c331cf90c1d5ba26ff1698e92c230 \ No newline at end of file +94ea6379178e3ff6a0d1d5819ca4ac558bdadb1ca8a3637c797079db7dc0cd61 \ No newline at end of file diff --git a/src/callback.c b/src/callback.c index faf6d520c4..ad53bd4d3c 100644 --- a/src/callback.c +++ b/src/callback.c @@ -283,7 +283,7 @@ static int matchQuality( ** Search a FuncDefHash for a function with the given name. Return ** a pointer to the matching FuncDef if found, or 0 if there is no match. */ -static FuncDef *functionSearch( +FuncDef *sqlite3FunctionSearch( int h, /* Hash of the name */ const char *zFunc /* Name of function */ ){ @@ -295,21 +295,6 @@ static FuncDef *functionSearch( } return 0; } -#ifdef SQLITE_ENABLE_NORMALIZE -FuncDef *sqlite3FunctionSearchN( - int h, /* Hash of the name */ - const char *zFunc, /* Name of function */ - int nFunc /* Length of the name */ -){ - FuncDef *p; - for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ - if( sqlite3StrNICmp(p->zName, zFunc, nFunc)==0 ){ - return p; - } - } - return 0; -} -#endif /* SQLITE_ENABLE_NORMALIZE */ /* ** Insert a new FuncDef into a FuncDefHash hash table. @@ -325,7 +310,7 @@ void sqlite3InsertBuiltinFuncs( int nName = sqlite3Strlen30(zName); int h = SQLITE_FUNC_HASH(zName[0], nName); assert( zName[0]>='a' && zName[0]<='z' ); - pOther = functionSearch(h, zName); + pOther = sqlite3FunctionSearch(h, zName); if( pOther ){ assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] ); aDef[i].pNext = pOther->pNext; @@ -403,7 +388,7 @@ FuncDef *sqlite3FindFunction( if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){ bestScore = 0; h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName); - p = functionSearch(h, zName); + p = sqlite3FunctionSearch(h, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ diff --git a/src/hash.c b/src/hash.c index e725f2aa5e..79bb85ac01 100644 --- a/src/hash.c +++ b/src/hash.c @@ -64,20 +64,6 @@ static unsigned int strHash(const char *z){ } return h; } -#ifdef SQLITE_ENABLE_NORMALIZE -static unsigned int strHashN(const char *z, int n){ - unsigned int h = 0; - int i; - for(i=0; iht ){ /*OPTIMIZATION-IF-TRUE*/ - struct _ht *pEntry; - h = strHashN(pKey, nKey) % pH->htsize; - pEntry = &pH->ht[h]; - elem = pEntry->chain; - count = pEntry->count; - }else{ - h = 0; - elem = pH->first; - count = pH->count; - } - if( pHash ) *pHash = h; - while( count-- ){ - assert( elem!=0 ); - if( sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ - return elem; - } - elem = elem->next; - } - return &nullElement; -} -#endif /* SQLITE_ENABLE_NORMALIZE */ /* Remove a single entry from the hash table given a pointer to that ** element and a hash on the element's key. @@ -267,14 +219,6 @@ void *sqlite3HashFind(const Hash *pH, const char *pKey){ assert( pKey!=0 ); return findElementWithHash(pH, pKey, 0)->data; } -#ifdef SQLITE_ENABLE_NORMALIZE -void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey){ - assert( pH!=0 ); - assert( pKey!=0 ); - assert( nKey>=0 ); - return findElementWithHashN(pH, pKey, nKey, 0)->data; -} -#endif /* SQLITE_ENABLE_NORMALIZE */ /* Insert an element into the hash table pH. The key is pKey ** and the data is "data". diff --git a/src/hash.h b/src/hash.h index cf773663b7..90540eda50 100644 --- a/src/hash.h +++ b/src/hash.h @@ -68,9 +68,6 @@ struct HashElem { void sqlite3HashInit(Hash*); void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); void *sqlite3HashFind(const Hash*, const char *pKey); -#ifdef SQLITE_ENABLE_NORMALIZE -void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey); -#endif void sqlite3HashClear(Hash*); /* diff --git a/src/prepare.c b/src/prepare.c index 86afac5350..7ac96a68f6 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -728,27 +728,41 @@ static int shouldTreatAsIdentifier( Hash *pHash; /* Hash table of tables for current database. */ HashElem *e; /* Hash element for hash table iteration. */ Table *pTab; /* Database table for columns being checked. */ + char *zId; /* Zero terminated name of the identifier */ + char zSpace[50]; /* Static space for the zero-terminated name */ - if( sqlite3IsRowidN(zToken, nToken) ){ - return 1; + if( nToken0 ){ int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken); - if( sqlite3FunctionSearchN(hash, zToken, nToken) ) return 1; + if( sqlite3FunctionSearch(hash, zId) ){ + bFound = 1; + goto done; + } } assert( db!=0 ); sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ pHash = &db->aFunc; - if( sqlite3HashFindN(pHash, zToken, nToken) ){ + if( sqlite3HashFind(pHash, zId) ){ bFound = 1; break; } pSchema = db->aDb[i].pSchema; if( pSchema==0 ) continue; pHash = &pSchema->tblHash; - if( sqlite3HashFindN(pHash, zToken, nToken) ){ + if( sqlite3HashFind(pHash, zId) ){ bFound = 1; break; } @@ -770,7 +784,7 @@ static int shouldTreatAsIdentifier( goto done; } } - if( pHash && sqlite3HashFindN(pHash, zToken, nToken) ){ + if( pHash && sqlite3HashFind(pHash, zId) ){ bFound = 1; goto done; } @@ -779,6 +793,7 @@ static int shouldTreatAsIdentifier( done: sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); + if( zId!=zSpace ) sqlite3DbFree(db, zId); return bFound; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index fc52fecfe2..9373accf8d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4056,9 +4056,7 @@ ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); IdList *sqlite3IdListDup(sqlite3*,IdList*); Select *sqlite3SelectDup(sqlite3*,Select*,int); -#ifdef SQLITE_ENABLE_NORMALIZE -FuncDef *sqlite3FunctionSearchN(int,const char*,int); -#endif +FuncDef *sqlite3FunctionSearch(int,const char*); void sqlite3InsertBuiltinFuncs(FuncDef*,int); FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); void sqlite3RegisterBuiltinFunctions(void); From 8a65380de906819e3ccba31ffaa72fbfb3ed044b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Dec 2018 19:42:59 +0000 Subject: [PATCH 15/52] Increase the version number to 3.27.0 for the next development cycle. FossilOrigin-Name: 8f8d682825d065bcaebda4cba63f8b67789b923d8b62eb7c576a5bc01536f577 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 419ede3b9c..8c53120442 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.26.0 +3.27.0 diff --git a/configure b/configure index 51653aa599..c711d432ed 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.26.0. +# Generated by GNU Autoconf 2.69 for sqlite 3.27.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -726,8 +726,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.26.0' -PACKAGE_STRING='sqlite 3.26.0' +PACKAGE_VERSION='3.27.0' +PACKAGE_STRING='sqlite 3.27.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1466,7 +1466,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.26.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.27.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1531,7 +1531,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.26.0:";; + short | recursive ) echo "Configuration of sqlite 3.27.0:";; esac cat <<\_ACEOF @@ -1657,7 +1657,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.26.0 +sqlite configure 3.27.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2076,7 +2076,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.26.0, which was +It was created by sqlite $as_me 3.27.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -12232,7 +12232,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.26.0, which was +This file was extended by sqlite $as_me 3.27.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -12298,7 +12298,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.26.0 +sqlite config.status 3.27.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/manifest b/manifest index 06972a00ce..f14cf47203 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Simplifications\sto\sthe\ssqlite3_normalized_sql()\simplementation. -D 2018-12-05T17:48:57.422 +C Increase\sthe\sversion\snumber\sto\s3.27.0\sfor\sthe\snext\sdevelopment\scycle. +D 2018-12-05T19:42:59.951 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc b7d4a710fa3f0b8cfc532ff195b85dc1ba2a8ad34343cb3d67639f28f0a24306 F README.md 377233394b905d3b2e2b33741289e093bc93f2e7adbe00923b2c5958c9a9edee -F VERSION 654da1d4053fb09ffc33a3910e6d427182a7dcdc67e934fa83de2849ac83fccb +F VERSION 453e2f4529ca208196d5567db28d549d7151f79efd33f6e6cfe6e613e583a0be F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 @@ -33,7 +33,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 6376abec766e9a0785178b1823b5a587e9f1ccbc F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 5811ffcd4866902d1706dcf8e0527f89165ec52859659942c9649bb1d3e4cc7b x +F configure e2b4021dcb8ee6f9a91e7f4011316957868e8065019ac994f68e27363120c272 x F configure.ac 3552d3aecade98a9d4b64bceb48ffb7726cbc85902efde956812942f060fbd0a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f44bc7a8b3fac82aa5598e9bdaf65ea4dd3c331cf90c1d5ba26ff1698e92c230 -R 14f7302635d9f1332de56d5fb4d8f1a4 +P 94ea6379178e3ff6a0d1d5819ca4ac558bdadb1ca8a3637c797079db7dc0cd61 +R 07b88da37b5c39140160539d8ff1a5aa U drh -Z b51bb6387d49f37b8a1232489a1c6ba3 +Z 2d432cc7c63d4590326f1d8d2ea75b30 diff --git a/manifest.uuid b/manifest.uuid index 6334ab25e4..c71e9a627a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94ea6379178e3ff6a0d1d5819ca4ac558bdadb1ca8a3637c797079db7dc0cd61 \ No newline at end of file +8f8d682825d065bcaebda4cba63f8b67789b923d8b62eb7c576a5bc01536f577 \ No newline at end of file From d4a4dd6ebcbd1d43f916fcd7f8d91d91ca762c08 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Dec 2018 21:55:39 +0000 Subject: [PATCH 16/52] Fix an imbalanced lock problem in sqlite3Normalize() introduced by the simplification in [94ea6379178e3ff6a]. FossilOrigin-Name: f69624373e33c4d370a9264a317ffdb0adbce967a950f243de2b41161a8c7ded --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/prepare.c | 11 ++++++----- test/normalize.test | 2 +- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index f14cf47203..3c6584f6a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.27.0\sfor\sthe\snext\sdevelopment\scycle. -D 2018-12-05T19:42:59.951 +C Fix\san\simbalanced\slock\sproblem\sin\ssqlite3Normalize()\sintroduced\sby\sthe\nsimplification\sin\s[94ea6379178e3ff6a]. +D 2018-12-05T21:55:39.342 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -502,7 +502,7 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c f77ebf812f46a174e805d939cec0017771904e79049d04edf66dbbb3ec25ad3e +F src/prepare.c 733d34a557cf2fb81af31fcf7c8df6a8070fcbef1a535ae120d2ff4fda99452d F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 @@ -1142,7 +1142,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 F test/nockpt.test 8c43b25af63b0bd620cf1b003529e37b6f1dc53bd22690e96a1bd73f78dde53a F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e -F test/normalize.test 6a80564d2000702b5919ed2c1069fef0f95762142bc96a71b4c124a845165713 +F test/normalize.test 17ff5c732bc16b100f4e479da3a59735ed24f4ed574fccf8e46279fad182f81e F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 94ea6379178e3ff6a0d1d5819ca4ac558bdadb1ca8a3637c797079db7dc0cd61 -R 07b88da37b5c39140160539d8ff1a5aa +P 8f8d682825d065bcaebda4cba63f8b67789b923d8b62eb7c576a5bc01536f577 +R 54e3066009cd3484077d0fae620be2a9 U drh -Z 2d432cc7c63d4590326f1d8d2ea75b30 +Z dbd77411550a9172a53b37194b8bf655 diff --git a/manifest.uuid b/manifest.uuid index c71e9a627a..87911d8f7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f8d682825d065bcaebda4cba63f8b67789b923d8b62eb7c576a5bc01536f577 \ No newline at end of file +f69624373e33c4d370a9264a317ffdb0adbce967a950f243de2b41161a8c7ded \ No newline at end of file diff --git a/src/prepare.c b/src/prepare.c index 7ac96a68f6..20e10511b5 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -741,13 +741,13 @@ static int shouldTreatAsIdentifier( } if( sqlite3IsRowid(zId) ){ bFound = 1; - goto done; + goto done1; } if( nToken>0 ){ int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken); if( sqlite3FunctionSearch(hash, zId) ){ bFound = 1; - goto done; + goto done1; } } assert( db!=0 ); @@ -781,18 +781,19 @@ static int shouldTreatAsIdentifier( }else{ *pRc = SQLITE_NOMEM_BKPT; bFound = 0; - goto done; + goto done2; } } if( pHash && sqlite3HashFind(pHash, zId) ){ bFound = 1; - goto done; + goto done2; } } } -done: +done2: sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); +done1: if( zId!=zSpace ) sqlite3DbFree(db, zId); return bFound; } diff --git a/test/normalize.test b/test/normalize.test index e85bd00332..35c995fe75 100644 --- a/test/normalize.test +++ b/test/normalize.test @@ -86,7 +86,7 @@ do_test 201 { } {} do_test 202 { sqlite3_normalized_sql $STMT -} {} +} {SELECT a,b FROM t1 WHERE b=?ORDER BY a;} do_test 203 { sqlite3_finalize $STMT } {SQLITE_OK} From 253a6af532438efbb27a23d775d75d748679a861 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Dec 2018 23:45:46 +0000 Subject: [PATCH 17/52] The sqlite3_normalized_sql() interface should not be transforming quoted identifier names into wildcards. Fix this, and at the same time simplify the code substantially. FossilOrigin-Name: e8540377ec66fa5f9ae3c93bedb5c094057698199c37fc211f7ea95429e815e4 --- manifest | 18 ++++----- manifest.uuid | 2 +- src/expr.c | 8 ---- src/prepare.c | 96 +-------------------------------------------- src/sqliteInt.h | 3 -- test/normalize.test | 2 +- 6 files changed, 12 insertions(+), 117 deletions(-) diff --git a/manifest b/manifest index 3c6584f6a4..a8462726b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\simbalanced\slock\sproblem\sin\ssqlite3Normalize()\sintroduced\sby\sthe\nsimplification\sin\s[94ea6379178e3ff6a]. -D 2018-12-05T21:55:39.342 +C The\ssqlite3_normalized_sql()\sinterface\sshould\snot\sbe\stransforming\squoted\nidentifier\snames\sinto\swildcards.\s\sFix\sthis,\sand\sat\sthe\ssame\stime\ssimplify\nthe\scode\ssubstantially. +D 2018-12-05T23:45:46.950 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -459,7 +459,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7 F src/dbstat.c 3c8bd4e77f0244fd2bd7cc90acf116ad2f8e82d70e536637f35ac2bc99b726f9 F src/delete.c f7938125847e8ef485448db5fbad29acb2991381a02887dd854c1617315ab9fb -F src/expr.c 9aacc0b72348ba90010b672dcbbbe2fa56e1182043bc917a3a147b2bc57a5497 +F src/expr.c a3e90131d7f92b77a6a6ae2e71f0d2882237041ed11dbe3c914e5e278ddc2d24 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812 F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f @@ -502,7 +502,7 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c 733d34a557cf2fb81af31fcf7c8df6a8070fcbef1a535ae120d2ff4fda99452d +F src/prepare.c 90099721296d9646addc5dda092464da21546e7410406818dde1600467b120c4 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 @@ -512,7 +512,7 @@ F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883 F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h e96520f27310016d0dabb496e13346244bbea83a6adc3134810e026f1d4a02b6 +F src/sqliteInt.h 8a9a4c1cce0cfd95d0fa8527ee12dbba0e7e1ec9983109626d4932cd48c5d571 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1142,7 +1142,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 F test/nockpt.test 8c43b25af63b0bd620cf1b003529e37b6f1dc53bd22690e96a1bd73f78dde53a F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e -F test/normalize.test 17ff5c732bc16b100f4e479da3a59735ed24f4ed574fccf8e46279fad182f81e +F test/normalize.test 7fe7674d83e37c635ea2e88a27b54a0fe1afddb192d636d5672b1639859800f1 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8f8d682825d065bcaebda4cba63f8b67789b923d8b62eb7c576a5bc01536f577 -R 54e3066009cd3484077d0fae620be2a9 +P f69624373e33c4d370a9264a317ffdb0adbce967a950f243de2b41161a8c7ded +R f3aa45194b952de0676e91bd1b04c704 U drh -Z dbd77411550a9172a53b37194b8bf655 +Z 5f1f0ed88784c364d2e88ebae6a00a1f diff --git a/manifest.uuid b/manifest.uuid index 87911d8f7f..8c94029b26 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f69624373e33c4d370a9264a317ffdb0adbce967a950f243de2b41161a8c7ded \ No newline at end of file +e8540377ec66fa5f9ae3c93bedb5c094057698199c37fc211f7ea95429e815e4 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index d4eb9de62d..fb44a7ef75 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2149,14 +2149,6 @@ int sqlite3IsRowid(const char *z){ if( sqlite3StrICmp(z, "OID")==0 ) return 1; return 0; } -#ifdef SQLITE_ENABLE_NORMALIZE -int sqlite3IsRowidN(const char *z, int n){ - if( sqlite3StrNICmp(z, "_ROWID_", n)==0 ) return 1; - if( sqlite3StrNICmp(z, "ROWID", n)==0 ) return 1; - if( sqlite3StrNICmp(z, "OID", n)==0 ) return 1; - return 0; -} -#endif /* ** pX is the RHS of an IN operator. If pX is a SELECT statement diff --git a/src/prepare.c b/src/prepare.c index 20e10511b5..76598e25d2 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -710,93 +710,6 @@ static int sqlite3LockAndPrepare( } #ifdef SQLITE_ENABLE_NORMALIZE -/* -** Checks if the specified token is a table, column, or function name, -** based on the databases associated with the statement being prepared. -** If the function fails, zero is returned and pRc is filled with the -** error code. -*/ -static int shouldTreatAsIdentifier( - sqlite3 *db, /* Database handle. */ - const char *zToken, /* Pointer to start of token to be checked */ - int nToken, /* Length of token to be checked */ - int *pRc /* Pointer to error code upon failure */ -){ - int bFound = 0; /* Non-zero if token is an identifier name. */ - int i, j; /* Database and column loop indexes. */ - Schema *pSchema; /* Schema for current database. */ - Hash *pHash; /* Hash table of tables for current database. */ - HashElem *e; /* Hash element for hash table iteration. */ - Table *pTab; /* Database table for columns being checked. */ - char *zId; /* Zero terminated name of the identifier */ - char zSpace[50]; /* Static space for the zero-terminated name */ - - if( nToken0 ){ - int hash = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zToken[0]], nToken); - if( sqlite3FunctionSearch(hash, zId) ){ - bFound = 1; - goto done1; - } - } - assert( db!=0 ); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - pHash = &db->aFunc; - if( sqlite3HashFind(pHash, zId) ){ - bFound = 1; - break; - } - pSchema = db->aDb[i].pSchema; - if( pSchema==0 ) continue; - pHash = &pSchema->tblHash; - if( sqlite3HashFind(pHash, zId) ){ - bFound = 1; - break; - } - for(e=sqliteHashFirst(pHash); e; e=sqliteHashNext(e)){ - pTab = sqliteHashData(e); - if( pTab==0 ) continue; - pHash = pTab->pColHash; - if( pHash==0 ){ - pTab->pColHash = pHash = sqlite3_malloc(sizeof(Hash)); - if( pHash ){ - sqlite3HashInit(pHash); - for(j=0; jnCol; j++){ - Column *pCol = &pTab->aCol[j]; - sqlite3HashInsert(pHash, pCol->zName, pCol); - } - }else{ - *pRc = SQLITE_NOMEM_BKPT; - bFound = 0; - goto done2; - } - } - if( pHash && sqlite3HashFind(pHash, zId) ){ - bFound = 1; - goto done2; - } - } - } -done2: - sqlite3BtreeLeaveAll(db); - sqlite3_mutex_leave(db->mutex); -done1: - if( zId!=zSpace ) sqlite3DbFree(db, zId); - return bFound; -} /* ** Attempt to estimate the final output buffer size needed for the fully @@ -972,19 +885,12 @@ char *sqlite3Normalize( z[j++] = ' '; } if( tokenType==TK_ID ){ - int i2 = i, n2 = n, rc = SQLITE_OK; + int i2 = i, n2 = n; if( nParen>0 ){ assert( nParen Date: Wed, 5 Dec 2018 23:56:02 +0000 Subject: [PATCH 18/52] Get rid of the hash table used to track IN operators in the sqlite3_normalized_sql() implementation. Use simple integer variables instead. FossilOrigin-Name: 272dc74fd0304d6a28aaa8798d13e2f950c1a24d92d17519e3c32aef86714586 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/prepare.c | 28 ++++++++++------------------ 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index a8462726b9..41eeb7cb19 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\ssqlite3_normalized_sql()\sinterface\sshould\snot\sbe\stransforming\squoted\nidentifier\snames\sinto\swildcards.\s\sFix\sthis,\sand\sat\sthe\ssame\stime\ssimplify\nthe\scode\ssubstantially. -D 2018-12-05T23:45:46.950 +C Get\srid\sof\sthe\shash\stable\sused\sto\strack\sIN\soperators\sin\sthe\nsqlite3_normalized_sql()\simplementation.\s\sUse\ssimple\sinteger\svariables\ninstead. +D 2018-12-05T23:56:02.639 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -502,7 +502,7 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c 90099721296d9646addc5dda092464da21546e7410406818dde1600467b120c4 +F src/prepare.c 66b5f9791a3ef57131cbba58c33104ebea814a70a5cfcafabf5aed5a3e3858fe F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f69624373e33c4d370a9264a317ffdb0adbce967a950f243de2b41161a8c7ded -R f3aa45194b952de0676e91bd1b04c704 +P e8540377ec66fa5f9ae3c93bedb5c094057698199c37fc211f7ea95429e815e4 +R d3461cbb797157ae329a2dd1a53cba4d U drh -Z 5f1f0ed88784c364d2e88ebae6a00a1f +Z ad7226daa41778e03500de0c126598a5 diff --git a/manifest.uuid b/manifest.uuid index 8c94029b26..41f2aab1d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e8540377ec66fa5f9ae3c93bedb5c094057698199c37fc211f7ea95429e815e4 \ No newline at end of file +272dc74fd0304d6a28aaa8798d13e2f950c1a24d92d17519e3c32aef86714586 \ No newline at end of file diff --git a/src/prepare.c b/src/prepare.c index 76598e25d2..c470a54bda 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -793,7 +793,8 @@ char *sqlite3Normalize( int prevTokenType = 0; /* Type of the previous token, except spaces */ int n; /* Size of the next token */ int nParen = 0; /* Nesting level of parenthesis */ - Hash inHash; /* Table of parenthesis levels to output index. */ + int iStartIN = 0; /* Start of RHS of IN operator in z[] */ + int nParenAtIN = 0; /* Value of nParent at start of RHS of IN operator */ db = sqlite3VdbeDb(pVdbe); assert( db!=0 ); @@ -801,7 +802,6 @@ char *sqlite3Normalize( nZ = estimateNormalizedSize(zSql, nSql); z = sqlite3DbMallocRawNN(db, nZ); if( z==0 ) goto normalizeError; - sqlite3HashInit(&inHash); for(i=j=0; i0 ){ - sqlite3HashInsert(&inHash, zSql+nParen, 0); - assert( jj+60 && nParen==nParenAtIN ){ + assert( iStartIN+6=0 ); assert( nZ-1-j0 ){ - assert( nParen0 && z[j-1]!=';' ){ z[j++] = ';'; } z[j] = 0; assert( j Date: Thu, 6 Dec 2018 01:08:58 +0000 Subject: [PATCH 19/52] Fix a missing mutex in the sqlite3_normalized_sql() interface when it is called on a prepared statement that did not previously have a computed normalization of the input SQL. FossilOrigin-Name: 1a1a59c6c570879c8f318bbc132103c0285088ba233dd8c32e049b86419347b0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 41eeb7cb19..799319cf7b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\srid\sof\sthe\shash\stable\sused\sto\strack\sIN\soperators\sin\sthe\nsqlite3_normalized_sql()\simplementation.\s\sUse\ssimple\sinteger\svariables\ninstead. -D 2018-12-05T23:56:02.639 +C Fix\sa\smissing\smutex\sin\sthe\ssqlite3_normalized_sql()\sinterface\swhen\sit\sis\ncalled\son\sa\sprepared\sstatement\sthat\sdid\snot\spreviously\shave\sa\scomputed\nnormalization\sof\sthe\sinput\sSQL. +D 2018-12-06T01:08:58.519 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -581,7 +581,7 @@ F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 -F src/vdbeapi.c 9ac7e3946a2762b79c314922c84c7de30731e0ed1d7c94ac82266b795221d9d7 +F src/vdbeapi.c 9709452bee82963e1f7f1f5d0c71db823d553f8dbb2c47a911c4983d537a1947 F src/vdbeaux.c 8e2fe020824b743090025ff6f9ffeec3ca4624523ddb2d1af1b1f61abaab3db4 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e8540377ec66fa5f9ae3c93bedb5c094057698199c37fc211f7ea95429e815e4 -R d3461cbb797157ae329a2dd1a53cba4d +P 272dc74fd0304d6a28aaa8798d13e2f950c1a24d92d17519e3c32aef86714586 +R b7800b30e5f18a6dd0b8f343735cdee2 U drh -Z ad7226daa41778e03500de0c126598a5 +Z 5df8bf04779626cf856fa0c34564f045 diff --git a/manifest.uuid b/manifest.uuid index 41f2aab1d9..54e405c549 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -272dc74fd0304d6a28aaa8798d13e2f950c1a24d92d17519e3c32aef86714586 \ No newline at end of file +1a1a59c6c570879c8f318bbc132103c0285088ba233dd8c32e049b86419347b0 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index ca073589f7..d8a463ae3b 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1714,7 +1714,9 @@ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; if( p==0 ) return 0; if( p->zNormSql==0 && p->zSql!=0 ){ + sqlite3_mutex_enter(p->db->mutex); p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql)); + sqlite3_mutex_leave(p->db->mutex); } return p->zNormSql; } From 974b24896ac3a1c2f5862c69539a269b7b0a07b4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 01:53:12 +0000 Subject: [PATCH 20/52] Fix a potential NULL pointer dereference that can occur in ALTER TABLE following an OOM. Test case in TH3. FossilOrigin-Name: ea50815bf80ab0d36891518f209acbcd3d6d8d84529741bffd4e841874c78aff --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/alter.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 799319cf7b..2a3da09056 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smissing\smutex\sin\sthe\ssqlite3_normalized_sql()\sinterface\swhen\sit\sis\ncalled\son\sa\sprepared\sstatement\sthat\sdid\snot\spreviously\shave\sa\scomputed\nnormalization\sof\sthe\sinput\sSQL. -D 2018-12-06T01:08:58.519 +C Fix\sa\spotential\sNULL\spointer\sdereference\sthat\scan\soccur\sin\sALTER\sTABLE\nfollowing\san\sOOM.\s\sTest\scase\sin\sTH3. +D 2018-12-06T01:53:12.207 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -441,7 +441,7 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F src/alter.c 80747854ca90dadc7c51cc1da302e29081ca5f78a2a87b3d44f65d39db834ff5 +F src/alter.c 87c9057f5eaa012da23b8e50848eee5e99088c3c478555f9ed255485b61ab5aa F src/analyze.c 3dc6b98cf007b005af89df165c966baaa48e8124f38c87b4d2b276fe7f0b9eb9 F src/attach.c 92b51739a885da8bd84bc9a05485f1e48148bce5c15432f059b45af98fff75cd F src/auth.c 0fac71038875693a937e506bceb492c5f136dd7b1249fbd4ae70b4e8da14f9df @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 272dc74fd0304d6a28aaa8798d13e2f950c1a24d92d17519e3c32aef86714586 -R b7800b30e5f18a6dd0b8f343735cdee2 +P 1a1a59c6c570879c8f318bbc132103c0285088ba233dd8c32e049b86419347b0 +R da4c56411a9ea719449f0bfabf9dd28c U drh -Z 5df8bf04779626cf856fa0c34564f045 +Z f66e0b178b75d5d7e91c658f98db7d44 diff --git a/manifest.uuid b/manifest.uuid index 54e405c549..18d994c2ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a1a59c6c570879c8f318bbc132103c0285088ba233dd8c32e049b86419347b0 \ No newline at end of file +ea50815bf80ab0d36891518f209acbcd3d6d8d84529741bffd4e841874c78aff \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index 3c46406d79..0a60918bea 100644 --- a/src/alter.c +++ b/src/alter.c @@ -1375,6 +1375,10 @@ static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ int i; RenameCtx *p = pWalker->u.pRename; SrcList *pSrc = pSelect->pSrc; + if( pSrc==0 ){ + assert( pWalker->pParse->db->mallocFailed ); + return WRC_Abort; + } for(i=0; inSrc; i++){ struct SrcList_item *pItem = &pSrc->a[i]; if( pItem->pTab==p->pTab ){ From fc08a1aef8b2750f0f6721dc1937c42a514fbab3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 02:01:17 +0000 Subject: [PATCH 21/52] Remove the unused pColHash field from the Table object. FossilOrigin-Name: 3a2c047989facc3461c63a2f9eed412014c951035a80da47c52a70139fb552de --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 6 ------ src/sqliteInt.h | 3 --- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 2a3da09056..cdcaa64e13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sNULL\spointer\sdereference\sthat\scan\soccur\sin\sALTER\sTABLE\nfollowing\san\sOOM.\s\sTest\scase\sin\sTH3. -D 2018-12-06T01:53:12.207 +C Remove\sthe\sunused\spColHash\sfield\sfrom\sthe\sTable\sobject. +D 2018-12-06T02:01:17.509 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -451,7 +451,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c ba7c7eef4461790f37c309936bfc5d0d6ba9b194b02d3c8ff1fd53b420ea6d3b F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 -F src/build.c fce47a9789704e65c63299b01be8153745faee7919f5137d3f29b7c3c0b549bd +F src/build.c ef9d7dc73e40dd9d10c28848343e21e8bc1baaab92cfb75eda893fff4fbf6b55 F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@ -512,7 +512,7 @@ F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883 F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h 8a9a4c1cce0cfd95d0fa8527ee12dbba0e7e1ec9983109626d4932cd48c5d571 +F src/sqliteInt.h 97adda953e7f118d47b8135f76f88c6420ff6707285782616f393a9ea255d577 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a1a59c6c570879c8f318bbc132103c0285088ba233dd8c32e049b86419347b0 -R da4c56411a9ea719449f0bfabf9dd28c +P ea50815bf80ab0d36891518f209acbcd3d6d8d84529741bffd4e841874c78aff +R 95857723bf3583344511ca7005311df0 U drh -Z f66e0b178b75d5d7e91c658f98db7d44 +Z 273a3b08446feca43c8c62181fb35feb diff --git a/manifest.uuid b/manifest.uuid index 18d994c2ee..5c54503b56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea50815bf80ab0d36891518f209acbcd3d6d8d84529741bffd4e841874c78aff \ No newline at end of file +3a2c047989facc3461c63a2f9eed412014c951035a80da47c52a70139fb552de \ No newline at end of file diff --git a/src/build.c b/src/build.c index ad1421d195..01fca6237e 100644 --- a/src/build.c +++ b/src/build.c @@ -636,12 +636,6 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ /* Delete the Table structure itself. */ -#ifdef SQLITE_ENABLE_NORMALIZE - if( pTable->pColHash ){ - sqlite3HashClear(pTable->pColHash); - sqlite3_free(pTable->pColHash); - } -#endif sqlite3DeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index fad351ec5e..bdf0d7fcbb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1957,9 +1957,6 @@ struct VTable { struct Table { char *zName; /* Name of the table or view */ Column *aCol; /* Information about each column */ -#ifdef SQLITE_ENABLE_NORMALIZE - Hash *pColHash; /* All columns indexed by name */ -#endif Index *pIndex; /* List of SQL indexes on this table. */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ FKey *pFKey; /* Linked list of all foreign keys in this table */ From 3e2d47d49fbf897f28971845671d3fd6aa3cc5b3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 03:59:25 +0000 Subject: [PATCH 22/52] Invoking the sqlite3_trace() or sqlite3_trace_v2() interfaces cancels any sqlite3_profile() that is running. FossilOrigin-Name: ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 4 ---- src/sqlite.h.in | 6 +++--- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index cdcaa64e13..720ed6ca6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sunused\spColHash\sfield\sfrom\sthe\sTable\sobject. -D 2018-12-06T02:01:17.509 +C Invoking\sthe\ssqlite3_trace()\sor\ssqlite3_trace_v2()\sinterfaces\scancels\nany\ssqlite3_profile()\sthat\sis\srunning. +D 2018-12-06T03:59:25.918 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -471,7 +471,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 9050dd153b5583804184be9c9dee9ebb554178d6db1f8ac280899e8aad9060e6 -F src/main.c 5a94791735ddd51804b75b7ef4645c25cbe6fe1d0e8f25851b2ed1211ad3d076 +F src/main.c 156076e2fc4eea09523323bae1d5f177a95a777a3b22e49e173437556fe3571c F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -509,7 +509,7 @@ F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d -F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3 +F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 F src/sqliteInt.h 97adda953e7f118d47b8135f76f88c6420ff6707285782616f393a9ea255d577 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ea50815bf80ab0d36891518f209acbcd3d6d8d84529741bffd4e841874c78aff -R 95857723bf3583344511ca7005311df0 +P 3a2c047989facc3461c63a2f9eed412014c951035a80da47c52a70139fb552de +R e33573679d2b83476b49b39d100df865 U drh -Z 273a3b08446feca43c8c62181fb35feb +Z 9b63335dca170db3a75ac781c84ad8b2 diff --git a/manifest.uuid b/manifest.uuid index 5c54503b56..787485caf5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a2c047989facc3461c63a2f9eed412014c951035a80da47c52a70139fb552de \ No newline at end of file +ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f \ No newline at end of file diff --git a/src/main.c b/src/main.c index 27a4206a4f..aa4546684c 100644 --- a/src/main.c +++ b/src/main.c @@ -1996,7 +1996,6 @@ void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; - if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; db->xTrace = (int(*)(u32,void*,void*,void*))xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); @@ -2021,9 +2020,6 @@ int sqlite3_trace_v2( if( mTrace==0 ) xTrace = 0; if( xTrace==0 ) mTrace = 0; db->mTrace = mTrace; -#ifndef SQLITE_OMIT_DEPRECATED - if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; -#endif db->xTrace = xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 85ef3635a4..4f33bb9ca9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2991,9 +2991,9 @@ int sqlite3_set_authorizer( ** time is in units of nanoseconds, however the current implementation ** is only capable of millisecond resolution so the six least significant ** digits in the time are meaningless. Future versions of SQLite -** might provide greater resolution on the profiler callback. The -** sqlite3_profile() function is considered experimental and is -** subject to change in future versions of SQLite. +** might provide greater resolution on the profiler callback. Invoking +** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the +** profile callback. */ SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); From ec8fc62c42902496b3955c7447f865a6142edfeb Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 16:11:14 +0000 Subject: [PATCH 23/52] Issue a warning whenever a double-quoted string literal is used. FossilOrigin-Name: ac9ad5043026b30394812457e1535df2759aea0d4510029561e92e386672796f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/resolve.c | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 720ed6ca6b..462c918e79 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Invoking\sthe\ssqlite3_trace()\sor\ssqlite3_trace_v2()\sinterfaces\scancels\nany\ssqlite3_profile()\sthat\sis\srunning. -D 2018-12-06T03:59:25.918 +C Issue\sa\swarning\swhenever\sa\sdouble-quoted\sstring\sliteral\sis\sused. +D 2018-12-06T16:11:14.066 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -505,7 +505,7 @@ F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 F src/prepare.c 66b5f9791a3ef57131cbba58c33104ebea814a70a5cfcafabf5aed5a3e3858fe F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 +F src/resolve.c e0408228bad5d13937a626521cba42c6f768643a6353712218d7e01fb201b202 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a2c047989facc3461c63a2f9eed412014c951035a80da47c52a70139fb552de -R e33573679d2b83476b49b39d100df865 +P ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f +R 3200e72b511607c15352eb3a60112ed6 U drh -Z 9b63335dca170db3a75ac781c84ad8b2 +Z 06f8f012c6fe6f33f55c7a711e9e6cc5 diff --git a/manifest.uuid b/manifest.uuid index 787485caf5..2cc1a0e981 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f \ No newline at end of file +ac9ad5043026b30394812457e1535df2759aea0d4510029561e92e386672796f \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index effbe646fe..39c2039f59 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -474,6 +474,22 @@ static int lookupName( if( cnt==0 && zTab==0 ){ assert( pExpr->op==TK_ID ); if( ExprHasProperty(pExpr,EP_DblQuoted) ){ + /* If a double-quoted identifier does not match any known column name, + ** then treat it as a string. + ** + ** This hack was added in the early days of SQLite in a misguided attempt + ** to be compatible with MySQL 3.x, which used double-quotes for strings. + ** I now sorely regret putting in this hack. The effect of this hack is + ** that misspelled identifier names are silently converted into strings + ** rather than causing an error, to the frustration of countless + ** programmers. To all those frustrated programmers, my apologies. + ** + ** Someday, I hope to get rid of this hack. Unfortunately there is + ** a huge amount of legacy SQL that uses it. So for now, we just + ** issue a warning. + */ + sqlite3_log(SQLITE_WARNING, + "double-quoted string literal: \"%w\"", zCol); pExpr->op = TK_STRING; pExpr->y.pTab = 0; return WRC_Prune; From 70d5dfba687ac41bf79d30198f7dd3d58cb6645e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 16:50:55 +0000 Subject: [PATCH 24/52] When saving off the value of sqlite3.flags, take care to preserve all 64 bits. FossilOrigin-Name: 9c6dbcfab5952cf4e54de30cf9cee48f988b91a35dc3f04d64d6d994dd84a076 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/main.c | 2 +- src/select.c | 2 +- src/vacuum.c | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 462c918e79..46d74a407e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Issue\sa\swarning\swhenever\sa\sdouble-quoted\sstring\sliteral\sis\sused. -D 2018-12-06T16:11:14.066 +C When\ssaving\soff\sthe\svalue\sof\ssqlite3.flags,\stake\scare\sto\spreserve\sall\s64\nbits. +D 2018-12-06T16:50:55.170 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -471,7 +471,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 9050dd153b5583804184be9c9dee9ebb554178d6db1f8ac280899e8aad9060e6 -F src/main.c 156076e2fc4eea09523323bae1d5f177a95a777a3b22e49e173437556fe3571c +F src/main.c cd8fefa30afc0b480b2881c216465f6ba5ea185b1ef4acb9f3593c12dc3b05a8 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -507,7 +507,7 @@ F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c e0408228bad5d13937a626521cba42c6f768643a6353712218d7e01fb201b202 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 -F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f +F src/select.c 44e04ad75cbbe2d2f6a30c5f586f344bbc94b80e456623b8602750202e484717 F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -577,7 +577,7 @@ F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274 F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 -F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d +F src/vacuum.c 675a4bdae41b115062327544e3c55c2dc302677be2231b004d28a33dd418684d F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec63d3506bd429560077f82a4c5ed9d189780789fe1c134fff4f3b8733be1a3f -R 3200e72b511607c15352eb3a60112ed6 +P ac9ad5043026b30394812457e1535df2759aea0d4510029561e92e386672796f +R c007d58f48241c5155abb3e9bb7962bc U drh -Z 06f8f012c6fe6f33f55c7a711e9e6cc5 +Z cdbf42bed7dc66ac0253bfff8bfb898d diff --git a/manifest.uuid b/manifest.uuid index 2cc1a0e981..4ec17b4483 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac9ad5043026b30394812457e1535df2759aea0d4510029561e92e386672796f \ No newline at end of file +9c6dbcfab5952cf4e54de30cf9cee48f988b91a35dc3f04d64d6d994dd84a076 \ No newline at end of file diff --git a/src/main.c b/src/main.c index aa4546684c..a4049f23f5 100644 --- a/src/main.c +++ b/src/main.c @@ -843,7 +843,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ if( aFlagOp[i].op==op ){ int onoff = va_arg(ap, int); int *pRes = va_arg(ap, int*); - u32 oldFlags = db->flags; + u64 oldFlags = db->flags; if( onoff>0 ){ db->flags |= aFlagOp[i].mask; }else if( onoff==0 ){ diff --git a/src/select.c b/src/select.c index c60ff27001..68f2c9f627 100644 --- a/src/select.c +++ b/src/select.c @@ -2077,7 +2077,7 @@ void sqlite3SelectAddColumnTypeAndCollation( Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ Table *pTab; sqlite3 *db = pParse->db; - int savedFlags; + u64 savedFlags; savedFlags = db->flags; db->flags &= ~SQLITE_FullColNames; diff --git a/src/vacuum.c b/src/vacuum.c index 5fb048916b..5017dbffde 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -136,8 +136,8 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ Btree *pTemp; /* The temporary database we vacuum into */ - u16 saved_mDbFlags; /* Saved value of db->mDbFlags */ - u32 saved_flags; /* Saved value of db->flags */ + u32 saved_mDbFlags; /* Saved value of db->mDbFlags */ + u64 saved_flags; /* Saved value of db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ u8 saved_mTrace; /* Saved trace settings */ From d597b966d9ffe95daa6a7143091d87916c6225f1 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 6 Dec 2018 16:54:44 +0000 Subject: [PATCH 25/52] Add test file wal2snapshot.test that should have been added as part of an earlier commit. FossilOrigin-Name: f6baa7e1163ed5f61375b0554337030fec23e8a9f60c6412e1b5d626961e93f9 --- manifest | 11 +++--- manifest.uuid | 2 +- test/wal2snapshot.test | 84 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 test/wal2snapshot.test diff --git a/manifest b/manifest index 5b4bda667c..07d0505ca6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\sscript\sproblem\son\sthis\sbranch. -D 2018-12-05T17:31:24.866 +C Add\stest\sfile\swal2snapshot.test\sthat\sshould\shave\sbeen\sadded\sas\spart\sof\san\nearlier\scommit. +D 2018-12-06T16:54:44.030 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -1595,6 +1595,7 @@ F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal2simple.test 8c9dfb8f1bca01a0deb57f7074cdb83865c2292e89b13f7a51a1c160dca3f5f4 +F test/wal2snapshot.test 95a919e1c73dee0e0212d10931d03cc1116f68a0ff603163e551aaa5ac7025c1 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 9c11da7aeccd83a46d79a556ad11a18d3cb15aa9 @@ -1782,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1d8d4f689653ce80157740e339f7f1b42479bf90d82b176d8202d0a49f428398 -R ea40d6e9e89bd0f6f2c98f8376b01df3 +P 285d1c5904dd607457c6b5ff7be3d3191a90ca8b1fb0837327663c9910f978ac +R c6176f33f0706e46f32e37a55f5b7c74 U dan -Z 1339c3788ed308ea20667b613925c238 +Z dba2fba0086f2ef35808e528fd424f75 diff --git a/manifest.uuid b/manifest.uuid index defc4724b4..60435bf318 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -285d1c5904dd607457c6b5ff7be3d3191a90ca8b1fb0837327663c9910f978ac \ No newline at end of file +f6baa7e1163ed5f61375b0554337030fec23e8a9f60c6412e1b5d626961e93f9 \ No newline at end of file diff --git a/test/wal2snapshot.test b/test/wal2snapshot.test new file mode 100644 index 0000000000..9f461db7d6 --- /dev/null +++ b/test/wal2snapshot.test @@ -0,0 +1,84 @@ +# 2018 December 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix wal2snapshot +ifcapable !wal {finish_test ; return } +ifcapable !snapshot {finish_test; return} + +foreach {tn mode} {1 wal 2 wal2} { + reset_db + do_execsql_test $tn.1 "PRAGMA journal_mode = $mode" $mode + + do_execsql_test $tn.2 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + BEGIN; + } + + # Check that sqlite3_snapshot_get() is an error for a wal2 db. + # + if {$tn==1} { + do_test 1.3 { + set S [sqlite3_snapshot_get db main] + sqlite3_snapshot_free $S + } {} + } else { + do_test 2.3 { + list [catch { sqlite3_snapshot_get db main } msg] $msg + } {1 SQLITE_ERROR} + } + + # Check that sqlite3_snapshot_recover() is an error for a wal2 db. + # + do_execsql_test $tn.4 COMMIT + if {$tn==1} { + do_test 1.5 { + sqlite3_snapshot_recover db main + } {} + } else { + do_test 2.5 { + list [catch { sqlite3_snapshot_recover db main } msg] $msg + } {1 SQLITE_ERROR} + } + + # Check that sqlite3_snapshot_open() is an error for a wal2 db. + # + if {$tn==1} { + do_test 1.6 { + execsql BEGIN + set SNAPSHOT [sqlite3_snapshot_get_blob db main] + sqlite3_snapshot_open_blob db main $SNAPSHOT + execsql COMMIT + } {} + } else { + do_test 2.6 { + execsql BEGIN + set res [ + list [catch { sqlite3_snapshot_open_blob db main $SNAPSHOT } msg] $msg + ] + execsql COMMIT + set res + } {1 SQLITE_ERROR} + } +} + + +finish_test + + From d5b44d60c7c79ff828e790a6fe31c80cdf10e47c Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 17:06:02 +0000 Subject: [PATCH 26/52] When masking bits off of sqlite3.flags, make sure the mask is 64 bits in size so as not to accidentally mask of high-order bits. FossilOrigin-Name: 53d3b169d8e1892163526caff2c843302c92e280fdeff6831e23a9bb15b82be3 --- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/btree.c | 2 +- src/loadext.c | 2 +- src/main.c | 4 ++-- src/pragma.c | 2 +- src/prepare.c | 2 +- src/select.c | 2 +- src/vacuum.c | 2 +- src/vdbe.c | 2 +- src/vdbeaux.c | 2 +- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 46d74a407e..035bff0044 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\ssaving\soff\sthe\svalue\sof\ssqlite3.flags,\stake\scare\sto\spreserve\sall\s64\nbits. -D 2018-12-06T16:50:55.170 +C When\smasking\sbits\soff\sof\ssqlite3.flags,\smake\ssure\sthe\smask\sis\s64\sbits\nin\ssize\sso\sas\snot\sto\saccidentally\smask\sof\shigh-order\sbits. +D 2018-12-06T17:06:02.839 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -448,7 +448,7 @@ F src/auth.c 0fac71038875693a937e506bceb492c5f136dd7b1249fbd4ae70b4e8da14f9df F src/backup.c 78d3cecfbe28230a3a9a1793e2ead609f469be43e8f486ca996006be551857ab F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c ba7c7eef4461790f37c309936bfc5d0d6ba9b194b02d3c8ff1fd53b420ea6d3b +F src/btree.c 4377d0d9a0b969c30b2bc343a12140a53ba6ab0dbf34c1686f232e67d87a557b F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 F src/build.c ef9d7dc73e40dd9d10c28848343e21e8bc1baaab92cfb75eda893fff4fbf6b55 @@ -470,8 +470,8 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e -F src/loadext.c 9050dd153b5583804184be9c9dee9ebb554178d6db1f8ac280899e8aad9060e6 -F src/main.c cd8fefa30afc0b480b2881c216465f6ba5ea185b1ef4acb9f3593c12dc3b05a8 +F src/loadext.c e6f10875d52aca3b7e57ce1ec174aeafc9b6c00b43000cd30d791f9cb490b7a6 +F src/main.c 473d149b51d4f5376e3308e11d5bc07f7570cf596eacecd148c6f7ae63a649ce F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -500,14 +500,14 @@ F src/parse.y 5cf85c2b9dfac38ac4e2bf2776484705186ce2eda8631e65cc0b04bf566c1173 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc -F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 +F src/pragma.c 96ce7dce4dc9cb2b7aa0e1b2ce7536870bdc00b10becc278245e775489447ea0 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c 66b5f9791a3ef57131cbba58c33104ebea814a70a5cfcafabf5aed5a3e3858fe +F src/prepare.c 5fe8eb586d0e9d4009609490866525f242a3d0df01e39d967bf2611dc8081088 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c e0408228bad5d13937a626521cba42c6f768643a6353712218d7e01fb201b202 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 -F src/select.c 44e04ad75cbbe2d2f6a30c5f586f344bbc94b80e456623b8602750202e484717 +F src/select.c 06a122decd9268c747e9110448b0c84249a4623ca9d78a73618ce648f7af8869 F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -577,12 +577,12 @@ F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274 F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 -F src/vacuum.c 675a4bdae41b115062327544e3c55c2dc302677be2231b004d28a33dd418684d -F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b +F src/vacuum.c c67085526fb51e141a85be3f7c9b91a2d38705bbb0ef911d8dec1e2d36cbdce1 +F src/vdbe.c c7312c909df5032b4ed27996c2ed8f5bf948f6a982458f85398520aaba0ccf81 F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 F src/vdbeapi.c 9709452bee82963e1f7f1f5d0c71db823d553f8dbb2c47a911c4983d537a1947 -F src/vdbeaux.c 8e2fe020824b743090025ff6f9ffeec3ca4624523ddb2d1af1b1f61abaab3db4 +F src/vdbeaux.c d71429f70e95e21aaf15ee287a78beefcda4bdc7d1b6d3c704dfeba820a792e0 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ac9ad5043026b30394812457e1535df2759aea0d4510029561e92e386672796f -R c007d58f48241c5155abb3e9bb7962bc +P 9c6dbcfab5952cf4e54de30cf9cee48f988b91a35dc3f04d64d6d994dd84a076 +R b1b5c125b5cc5dd847ef51923c461312 U drh -Z cdbf42bed7dc66ac0253bfff8bfb898d +Z 9423d60b113ec33e61947be8b42c640e diff --git a/manifest.uuid b/manifest.uuid index 4ec17b4483..8f6dcb9997 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c6dbcfab5952cf4e54de30cf9cee48f988b91a35dc3f04d64d6d994dd84a076 \ No newline at end of file +53d3b169d8e1892163526caff2c843302c92e280fdeff6831e23a9bb15b82be3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 137500eedc..9046a72d32 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9856,7 +9856,7 @@ char *sqlite3BtreeIntegrityCheck( } #endif testcase( pBt->db->flags & SQLITE_CellSizeCk ); - pBt->db->flags &= ~SQLITE_CellSizeCk; + pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)iflags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; }else{ - db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); + db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc); } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; diff --git a/src/main.c b/src/main.c index a4049f23f5..627d24e0b1 100644 --- a/src/main.c +++ b/src/main.c @@ -847,7 +847,7 @@ int sqlite3_db_config(sqlite3 *db, int op, ...){ if( onoff>0 ){ db->flags |= aFlagOp[i].mask; }else if( onoff==0 ){ - db->flags &= ~aFlagOp[i].mask; + db->flags &= ~(u64)aFlagOp[i].mask; } if( oldFlags!=db->flags ){ sqlite3ExpirePreparedStatements(db, 0); @@ -1310,7 +1310,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ /* Any deferred constraint violations have now been resolved. */ db->nDeferredCons = 0; db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + db->flags &= ~(u64)SQLITE_DeferFKs; /* If one has been configured, invoke the rollback-hook callback */ if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ diff --git a/src/pragma.c b/src/pragma.c index ada652cf14..2f27d11005 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -816,7 +816,7 @@ void sqlite3Pragma( if( sqlite3GetBoolean(zRight, size!=0) ){ db->flags |= SQLITE_CacheSpill; }else{ - db->flags &= ~SQLITE_CacheSpill; + db->flags &= ~(u64)SQLITE_CacheSpill; } setAllPagerFlags(db); } diff --git a/src/prepare.c b/src/prepare.c index c470a54bda..ad07eca45a 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -293,7 +293,7 @@ int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){ ** indices that the user might have created. */ if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){ - db->flags &= ~SQLITE_LegacyFileFmt; + db->flags &= ~(u64)SQLITE_LegacyFileFmt; } /* Read the schema information out of the schema tables diff --git a/src/select.c b/src/select.c index 68f2c9f627..a275cb4b4d 100644 --- a/src/select.c +++ b/src/select.c @@ -2080,7 +2080,7 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ u64 savedFlags; savedFlags = db->flags; - db->flags &= ~SQLITE_FullColNames; + db->flags &= ~(u64)SQLITE_FullColNames; db->flags |= SQLITE_ShortColNames; sqlite3SelectPrep(pParse, pSelect, 0); if( pParse->nErr ) return 0; diff --git a/src/vacuum.c b/src/vacuum.c index 5017dbffde..8416b16729 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -166,7 +166,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ saved_mTrace = db->mTrace; db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; - db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder + db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_Defensive | SQLITE_CountRows); db->mTrace = 0; diff --git a/src/vdbe.c b/src/vdbe.c index 14f3e1bd3c..2803330b34 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -7086,7 +7086,7 @@ case OP_VRename: { rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); if( rc ) goto abort_due_to_error; rc = pVtab->pModule->xRename(pVtab, pName->z); - if( isLegacy==0 ) db->flags &= ~SQLITE_LegacyAlter; + if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter; sqlite3VtabImportErrmsg(p, pVtab); p->expired = 0; if( rc ) goto abort_due_to_error; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0ed9e81d90..40a63319db 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2855,7 +2855,7 @@ int sqlite3VdbeHalt(Vdbe *p){ }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } }else{ From 11df7d2839a44e4814ebb110718e658c5ad9014a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 19:15:36 +0000 Subject: [PATCH 27/52] Simplify the query flattener so that it does not duplicate the WHERE clause of subquery that is being incorporated into the outer query - copies it directly. This is more efficient. And it also fixes the specific test case show for ticket [f09fcd17810f65f71789525] but it does not resolve the more general problem that sqlite3ExprDup() does not correctly duplicate expressions that contain subqueries with window functions. FossilOrigin-Name: f1b18d44ff855573542c1947a42d4025dff54cacc13dac14d044521762ea9736 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/select.c | 4 +++- test/window1.test | 13 +++++++++++++ 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 035bff0044..8674aa7a68 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\smasking\sbits\soff\sof\ssqlite3.flags,\smake\ssure\sthe\smask\sis\s64\sbits\nin\ssize\sso\sas\snot\sto\saccidentally\smask\sof\shigh-order\sbits. -D 2018-12-06T17:06:02.839 +C Simplify\sthe\squery\sflattener\sso\sthat\sit\sdoes\snot\sduplicate\sthe\sWHERE\sclause\nof\ssubquery\sthat\sis\sbeing\sincorporated\sinto\sthe\souter\squery\s-\scopies\sit\ndirectly.\s\sThis\sis\smore\sefficient.\s\sAnd\sit\salso\sfixes\sthe\sspecific\stest\scase\nshow\sfor\sticket\s[f09fcd17810f65f71789525]\sbut\sit\sdoes\snot\sresolve\sthe\smore\ngeneral\sproblem\sthat\ssqlite3ExprDup()\sdoes\snot\scorrectly\sduplicate\sexpressions\nthat\scontain\ssubqueries\swith\swindow\sfunctions. +D 2018-12-06T19:15:36.453 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -507,7 +507,7 @@ F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c e0408228bad5d13937a626521cba42c6f768643a6353712218d7e01fb201b202 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 -F src/select.c 06a122decd9268c747e9110448b0c84249a4623ca9d78a73618ce648f7af8869 +F src/select.c 8c7317d5ee920516a56b8b4ca79fbfca70a1f8b52d67e884c808ea3a016c04e3 F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1655,7 +1655,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test 02e481ac48c445b43bab7b3cf1e4115165b5127a1aa29e14f5372922c836f1a4 +F test/window1.test d55c41a0b7a3246ed3882e87c540ac2e4bf7a8f7774e0c2e81690b4f4bedc088 F test/window2.tcl 9bfa842d8a62b0d36dc8c1b5972206393c43847433c6d75940b87fec93ce3143 F test/window2.test 8e6d2a1b9f54dfebee1cde961c8590cd87b4db45c50f44947a211e1b63c2a05e F test/window3.tcl 577a3b1ff913208e5248c04dab9df17fd760ce159a752789e26d0cb4a5f91823 @@ -1782,7 +1782,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9c6dbcfab5952cf4e54de30cf9cee48f988b91a35dc3f04d64d6d994dd84a076 -R b1b5c125b5cc5dd847ef51923c461312 +P 53d3b169d8e1892163526caff2c843302c92e280fdeff6831e23a9bb15b82be3 +R a2d2ec8d34c71e6abf08e50ec579efcc +T *branch * ticket-f09fcd17810f +T *sym-ticket-f09fcd17810f * +T -sym-trunk * U drh -Z 9423d60b113ec33e61947be8b42c640e +Z 5018814e4a4daf9bdeea4c03d1c78115 diff --git a/manifest.uuid b/manifest.uuid index 8f6dcb9997..9922fca4fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53d3b169d8e1892163526caff2c843302c92e280fdeff6831e23a9bb15b82be3 \ No newline at end of file +f1b18d44ff855573542c1947a42d4025dff54cacc13dac14d044521762ea9736 \ No newline at end of file diff --git a/src/select.c b/src/select.c index a275cb4b4d..5e30504d4c 100644 --- a/src/select.c +++ b/src/select.c @@ -3461,6 +3461,7 @@ static Expr *substExpr( ifNullRow.iTable = pSubst->iNewTable; pCopy = &ifNullRow; } + testcase( ExprHasProperty(pCopy, EP_Subquery) ); pNew = sqlite3ExprDup(db, pCopy, 0); if( pNew && pSubst->isLeftJoin ){ ExprSetProperty(pNew, EP_CanBeNull); @@ -4025,7 +4026,8 @@ static int flattenSubquery( pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; } - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); + pWhere = pSub->pWhere; + pSub->pWhere = 0; if( isLeftJoin>0 ){ setJoinExpr(pWhere, iNewParent); } diff --git a/test/window1.test b/test/window1.test index a8399a8606..c24bbaa20f 100644 --- a/test/window1.test +++ b/test/window1.test @@ -594,4 +594,17 @@ do_execsql_test 13.5 { } { } +# 2018-12-06 +# https://www.sqlite.org/src/info/f09fcd17810f65f7 +# Assertion fault when window functions are used +# +sqlite3 db :memory: +do_execsql_test 14.0 { + SELECT * FROM( + SELECT * FROM (SELECT 1 AS c) WHERE c IN ( + SELECT (row_number() OVER()) FROM (VALUES (0)) + ) + ); +} {1} + finish_test From bb383df7c625daa4c7a0db4c7eed2296dc171bde Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 19:56:20 +0000 Subject: [PATCH 28/52] New test case that still hits an assertion fault, just to prove that the previous checkin merely made the problem more obscure and did not completely fix it. FossilOrigin-Name: a9a3b532643b5f106509bb29c3e6bc9d41ec5b2da5a0cd4067898f376eb626a2 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- test/window1.test | 16 +++++++++++++++- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8674aa7a68..066d87f7c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\squery\sflattener\sso\sthat\sit\sdoes\snot\sduplicate\sthe\sWHERE\sclause\nof\ssubquery\sthat\sis\sbeing\sincorporated\sinto\sthe\souter\squery\s-\scopies\sit\ndirectly.\s\sThis\sis\smore\sefficient.\s\sAnd\sit\salso\sfixes\sthe\sspecific\stest\scase\nshow\sfor\sticket\s[f09fcd17810f65f71789525]\sbut\sit\sdoes\snot\sresolve\sthe\smore\ngeneral\sproblem\sthat\ssqlite3ExprDup()\sdoes\snot\scorrectly\sduplicate\sexpressions\nthat\scontain\ssubqueries\swith\swindow\sfunctions. -D 2018-12-06T19:15:36.453 +C New\stest\scase\sthat\sstill\shits\san\sassertion\sfault,\sjust\sto\sprove\sthat\sthe\nprevious\scheckin\smerely\smade\sthe\sproblem\smore\sobscure\sand\sdid\snot\scompletely\nfix\sit. +D 2018-12-06T19:56:20.155 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -1655,7 +1655,7 @@ F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2 F test/win32lock.test fbf107c91d8f5512be5a5b87c4c42ab9fdd54972 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/win32nolock.test ac4f08811a562e45a5755e661f45ca85892bdbbc -F test/window1.test d55c41a0b7a3246ed3882e87c540ac2e4bf7a8f7774e0c2e81690b4f4bedc088 +F test/window1.test 1003e19bebe06be286a38139ea5fc010b30c055cc1527824b09d609f89bbd93b F test/window2.tcl 9bfa842d8a62b0d36dc8c1b5972206393c43847433c6d75940b87fec93ce3143 F test/window2.test 8e6d2a1b9f54dfebee1cde961c8590cd87b4db45c50f44947a211e1b63c2a05e F test/window3.tcl 577a3b1ff913208e5248c04dab9df17fd760ce159a752789e26d0cb4a5f91823 @@ -1782,10 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 53d3b169d8e1892163526caff2c843302c92e280fdeff6831e23a9bb15b82be3 -R a2d2ec8d34c71e6abf08e50ec579efcc -T *branch * ticket-f09fcd17810f -T *sym-ticket-f09fcd17810f * -T -sym-trunk * +P f1b18d44ff855573542c1947a42d4025dff54cacc13dac14d044521762ea9736 +R 999c00703256c76e5513029bb0036a91 U drh -Z 5018814e4a4daf9bdeea4c03d1c78115 +Z 08d2025b0cdb73a3ed0863948c64547c diff --git a/manifest.uuid b/manifest.uuid index 9922fca4fa..3ba1ff645b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1b18d44ff855573542c1947a42d4025dff54cacc13dac14d044521762ea9736 \ No newline at end of file +a9a3b532643b5f106509bb29c3e6bc9d41ec5b2da5a0cd4067898f376eb626a2 \ No newline at end of file diff --git a/test/window1.test b/test/window1.test index c24bbaa20f..5f9b5dbb75 100644 --- a/test/window1.test +++ b/test/window1.test @@ -596,7 +596,12 @@ do_execsql_test 13.5 { # 2018-12-06 # https://www.sqlite.org/src/info/f09fcd17810f65f7 -# Assertion fault when window functions are used +# Assertion fault when window functions are used. +# +# Root cause is the query flattener invoking sqlite3ExprDup() on +# expressions that contain subqueries with window functions. The +# sqlite3ExprDup() routine is not making correctly initializing +# Select.pWin field of the subqueries. # sqlite3 db :memory: do_execsql_test 14.0 { @@ -606,5 +611,14 @@ do_execsql_test 14.0 { ) ); } {1} +do_execsql_test 14.1 { + CREATE TABLE t1(x); INSERT INTO t1(x) VALUES(12345); + CREATE TABLE t2(c); INSERT INTO t2(c) VALUES(1); + SELECT y, y+1, y+2 FROM ( + SELECT c IN ( + SELECT (row_number() OVER()) FROM t1 + ) AS y FROM t2 + ); +} {1 2 3} finish_test From a83899754423bea16d4efeb4f39482b57d79c083 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 22:04:19 +0000 Subject: [PATCH 29/52] Fix the sqlite3ExprDup() routine so that it makes complete duplications of subqueries containing window functions. FossilOrigin-Name: 940174543e87184a0278fcd02e8a096a11510174d9c1d65d21878819790ddaff --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 27 +++++++++++++++++++++++++++ src/window.c | 1 + 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 066d87f7c1..73d0947676 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scase\sthat\sstill\shits\san\sassertion\sfault,\sjust\sto\sprove\sthat\sthe\nprevious\scheckin\smerely\smade\sthe\sproblem\smore\sobscure\sand\sdid\snot\scompletely\nfix\sit. -D 2018-12-06T19:56:20.155 +C Fix\sthe\ssqlite3ExprDup()\sroutine\sso\sthat\sit\smakes\scomplete\sduplications\sof\nsubqueries\scontaining\swindow\sfunctions. +D 2018-12-06T22:04:19.327 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -459,7 +459,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7 F src/dbstat.c 3c8bd4e77f0244fd2bd7cc90acf116ad2f8e82d70e536637f35ac2bc99b726f9 F src/delete.c f7938125847e8ef485448db5fbad29acb2991381a02887dd854c1617315ab9fb -F src/expr.c a3e90131d7f92b77a6a6ae2e71f0d2882237041ed11dbe3c914e5e278ddc2d24 +F src/expr.c 348e3877784d3e3a8b053a6b2e32d00b986fc4420ab484a41bf38e6d31e3b84b F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812 F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f @@ -596,7 +596,7 @@ F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e F src/whereInt.h f125f29fca80890768e0b2caa14f95db74b2dacd3a122a168f97aa7b64d6968f F src/wherecode.c c45f03aefc2266b990df0fc4d7acc4e27f56f881f4fc0fc355b7cbc4d7189da5 F src/whereexpr.c 491f0894ad9903750cdecb7894437a0cabdffdd88f574d2b1c9ac85d14fe4b9c -F src/window.c 6550e2850ebced51100ef83d49b00a1cf03f81a482dafedafb0320df647ed8fc +F src/window.c ea81ecd031ed2cbc14b7db6fd7f4bee2471b894feae5fea0547b15b1e2dd8fb2 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f1b18d44ff855573542c1947a42d4025dff54cacc13dac14d044521762ea9736 -R 999c00703256c76e5513029bb0036a91 +P a9a3b532643b5f106509bb29c3e6bc9d41ec5b2da5a0cd4067898f376eb626a2 +R 24973ceb08f0bf40a95db4b2c4fd154c U drh -Z 08d2025b0cdb73a3ed0863948c64547c +Z f52863a824efeb9518162f1a9aff57dd diff --git a/manifest.uuid b/manifest.uuid index 3ba1ff645b..282cc9c5e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9a3b532643b5f106509bb29c3e6bc9d41ec5b2da5a0cd4067898f376eb626a2 \ No newline at end of file +940174543e87184a0278fcd02e8a096a11510174d9c1d65d21878819790ddaff \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index fb44a7ef75..a6bb441d56 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1330,6 +1330,32 @@ static With *withDup(sqlite3 *db, With *p){ # define withDup(x,y) 0 #endif +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** The gatherSelectWindows() procedure and its helper routine +** gatherSelectWindowsCallback() are used to scan all the expressions +** an a newly duplicated SELECT statement and gather all of the Window +** objects found there, assembling them onto the linked list at Select->pWin. +*/ +static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_FUNCTION && pExpr->y.pWin!=0 ){ + assert( ExprHasProperty(pExpr, EP_WinFunc) ); + pExpr->y.pWin->pNextWin = pWalker->u.pSelect->pWin; + pWalker->u.pSelect->pWin = pExpr->y.pWin; + } + return WRC_Continue; +} +static void gatherSelectWindows(Select *p){ + Walker w; + w.xExprCallback = gatherSelectWindowsCallback; + w.xSelectCallback = 0; + w.u.pSelect = p; + sqlite3WalkSelectExpr(&w, p); + sqlite3WalkSelectFrom(&w, p); +} +#endif + + /* ** The following group of routines make deep copies of expressions, ** expression lists, ID lists, and select statements. The copies can @@ -1497,6 +1523,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){ #ifndef SQLITE_OMIT_WINDOWFUNC pNew->pWin = 0; pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn); + if( p->pWin ) gatherSelectWindows(pNew); #endif pNew->selId = p->selId; *pp = pNew; diff --git a/src/window.c b/src/window.c index f5deae9a6e..f3e274d6e3 100644 --- a/src/window.c +++ b/src/window.c @@ -2133,6 +2133,7 @@ Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){ if( pNew ){ pNew->zName = sqlite3DbStrDup(db, p->zName); pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0); + pNew->pFunc = p->pFunc; pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0); pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0); pNew->eType = p->eType; From a37b6a5e31f341412dc0c454569fea5cd0f88cef Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Dec 2018 22:12:18 +0000 Subject: [PATCH 30/52] Performance improvement: Avoid using sqlite3WalkerSelectExpr() and sqlite3WalkerSelectFrom() twice, so that the compiler will in-line their implementation. FossilOrigin-Name: 2b9258b8b0342330ebe8c22b59ec276fd042a05547d15b24fdf29e16280868de --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 73d0947676..4b2190f9c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3ExprDup()\sroutine\sso\sthat\sit\smakes\scomplete\sduplications\sof\nsubqueries\scontaining\swindow\sfunctions. -D 2018-12-06T22:04:19.327 +C Performance\simprovement:\s\sAvoid\susing\ssqlite3WalkerSelectExpr()\nand\ssqlite3WalkerSelectFrom()\stwice,\sso\sthat\sthe\scompiler\swill\sin-line\stheir\nimplementation. +D 2018-12-06T22:12:18.890 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -459,7 +459,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7 F src/dbstat.c 3c8bd4e77f0244fd2bd7cc90acf116ad2f8e82d70e536637f35ac2bc99b726f9 F src/delete.c f7938125847e8ef485448db5fbad29acb2991381a02887dd854c1617315ab9fb -F src/expr.c 348e3877784d3e3a8b053a6b2e32d00b986fc4420ab484a41bf38e6d31e3b84b +F src/expr.c b83e2056086e364293ce741509d885d1646a32d62727fe3169b23231c7e4f62f F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812 F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a9a3b532643b5f106509bb29c3e6bc9d41ec5b2da5a0cd4067898f376eb626a2 -R 24973ceb08f0bf40a95db4b2c4fd154c +P 940174543e87184a0278fcd02e8a096a11510174d9c1d65d21878819790ddaff +R 49ae07edd1762ce880f34b83601e0b9e U drh -Z f52863a824efeb9518162f1a9aff57dd +Z bcb12cb10c9264efa5baeffb2512d3d5 diff --git a/manifest.uuid b/manifest.uuid index 282cc9c5e5..40b4d9a6d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -940174543e87184a0278fcd02e8a096a11510174d9c1d65d21878819790ddaff \ No newline at end of file +2b9258b8b0342330ebe8c22b59ec276fd042a05547d15b24fdf29e16280868de \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index a6bb441d56..6886d30b2d 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1345,13 +1345,16 @@ static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){ } return WRC_Continue; } +static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){ + return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune; +} static void gatherSelectWindows(Select *p){ Walker w; w.xExprCallback = gatherSelectWindowsCallback; - w.xSelectCallback = 0; + w.xSelectCallback = gatherSelectWindowsSelectCallback; + w.xSelectCallback2 = 0; w.u.pSelect = p; - sqlite3WalkSelectExpr(&w, p); - sqlite3WalkSelectFrom(&w, p); + sqlite3WalkSelect(&w, p); } #endif From 62a88294e8fb48d3ed8e0fdbbc346004ed09b168 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Dec 2018 03:01:07 +0000 Subject: [PATCH 31/52] Fix dbfuzz2.c so that it works with -DSQLITE_OMIT_INIT FossilOrigin-Name: 9ad796a8822f1b7e1e99b73c2cc5be59dbfd85e9c27f3e795c29a2c002c611d1 --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/dbfuzz2.c | 1 + 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fae9ecaa07..4087d03b83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3ExprDup()\sfunction\sso\sthat\sit\scorrectly\sduplicates\sthe\nWindow\sobject\slist\son\sa\sSelect\sthat\scontains\swindow\sfunctions.\s\sFix\nfor\sticket\s[f09fcd17810f65f717]. -D 2018-12-07T01:56:26.338 +C Fix\sdbfuzz2.c\sso\sthat\sit\sworks\swith\s-DSQLITE_OMIT_INIT +D 2018-12-07T03:01:07.990 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -774,7 +774,7 @@ F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee -F test/dbfuzz2.c 652f85bac1770e927da139db513234a3eba308f72ac2f8b32f0093d7d19def70 +F test/dbfuzz2.c b8ed9b32a1f287505e55970e55203bedcb9170f137ecefa2254033c9faccdfba F test/dbpage.test 650234ba683b9d82b899c6c51439819787e7609f17a0cc40e0080a7b6443bc38 F test/dbstatus.test cd83aa623b8aab477269bc94cf8aa90c1e195a144561dd04a1620770aaa8524e F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef @@ -1782,8 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 53d3b169d8e1892163526caff2c843302c92e280fdeff6831e23a9bb15b82be3 2b9258b8b0342330ebe8c22b59ec276fd042a05547d15b24fdf29e16280868de -R 49ae07edd1762ce880f34b83601e0b9e -T +closed 2b9258b8b0342330ebe8c22b59ec276fd042a05547d15b24fdf29e16280868de +P db5ed2268eda2e6c1df15cd8df4176463d89103b8fda33ba9a0604f0d92bd4da +R 2969a28b361c45f45888cf9a9ac52a9e U drh -Z 8b11b2c511f2560f5e30a67bc4544388 +Z 1170e81cf6857f6b3144fe1a49be0ee2 diff --git a/manifest.uuid b/manifest.uuid index bf0d95f7f0..55f1f4ff1e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db5ed2268eda2e6c1df15cd8df4176463d89103b8fda33ba9a0604f0d92bd4da \ No newline at end of file +9ad796a8822f1b7e1e99b73c2cc5be59dbfd85e9c27f3e795c29a2c002c611d1 \ No newline at end of file diff --git a/test/dbfuzz2.c b/test/dbfuzz2.c index 0833f03868..9e3aca2527 100644 --- a/test/dbfuzz2.c +++ b/test/dbfuzz2.c @@ -78,6 +78,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *aData, size_t nByte){ printf("************** nByte=%d ***************\n", (int)nByte); fflush(stdout); } + if( sqlite3_initialize() ) return 0; rc = sqlite3_open(0, &db); if( rc ) return 1; a = sqlite3_malloc64(nByte+1); From 893bd3751b30460c9556b3c9ae51c4ac4f26b6af Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Dec 2018 16:32:11 +0000 Subject: [PATCH 32/52] Fix the sqlite3_normalized_sql() interface so that it renders double-quoted string literals as "?". FossilOrigin-Name: 0d8e150434bbd179696f1ffe71d1e06cb3d43e6468496c7e481fca8486387bad --- manifest | 22 +++++++++--------- manifest.uuid | 2 +- src/prepare.c | 8 +++++-- src/resolve.c | 3 +++ src/vdbe.h | 4 ++++ src/vdbeInt.h | 19 ++++++++++++++- src/vdbeaux.c | 56 ++++++++++++++++++++++++++++++++++++++++++++- test/normalize.test | 2 +- 8 files changed, 99 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 4087d03b83..f4fb1dff1b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sdbfuzz2.c\sso\sthat\sit\sworks\swith\s-DSQLITE_OMIT_INIT -D 2018-12-07T03:01:07.990 +C Fix\sthe\ssqlite3_normalized_sql()\sinterface\sso\sthat\sit\srenders\ndouble-quoted\sstring\sliterals\sas\s"?". +D 2018-12-07T16:32:11.362 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -502,10 +502,10 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 96ce7dce4dc9cb2b7aa0e1b2ce7536870bdc00b10becc278245e775489447ea0 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c 5fe8eb586d0e9d4009609490866525f242a3d0df01e39d967bf2611dc8081088 +F src/prepare.c be449edb106a16f1ad95f9b798bdc2337f8c3f83b96c284f417c0a26d43f0c1b F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c e0408228bad5d13937a626521cba42c6f768643a6353712218d7e01fb201b202 +F src/resolve.c 095d1d41d7a981ee9db8bfeae25ed0d6a8a5e5e3d66b0f4efd71877ed7b56132 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 8c7317d5ee920516a56b8b4ca79fbfca70a1f8b52d67e884c808ea3a016c04e3 F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d @@ -579,10 +579,10 @@ F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c c67085526fb51e141a85be3f7c9b91a2d38705bbb0ef911d8dec1e2d36cbdce1 F src/vdbe.c c7312c909df5032b4ed27996c2ed8f5bf948f6a982458f85398520aaba0ccf81 -F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 -F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 +F src/vdbe.h d82f323d581b36b8e147d650257ef34e0e93790039b6cbda45c321c275f7595e +F src/vdbeInt.h 73f5051923f3f29779bfc374c0c68e23b8e5e3792def2e33e51b427edb890abd F src/vdbeapi.c 9709452bee82963e1f7f1f5d0c71db823d553f8dbb2c47a911c4983d537a1947 -F src/vdbeaux.c d71429f70e95e21aaf15ee287a78beefcda4bdc7d1b6d3c704dfeba820a792e0 +F src/vdbeaux.c 9a9617666124e18cbd6e936740f7469dcf0d82867b1abf9ed828694500930b64 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f @@ -1142,7 +1142,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 F test/nockpt.test 8c43b25af63b0bd620cf1b003529e37b6f1dc53bd22690e96a1bd73f78dde53a F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e -F test/normalize.test 7fe7674d83e37c635ea2e88a27b54a0fe1afddb192d636d5672b1639859800f1 +F test/normalize.test 17ff5c732bc16b100f4e479da3a59735ed24f4ed574fccf8e46279fad182f81e F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db5ed2268eda2e6c1df15cd8df4176463d89103b8fda33ba9a0604f0d92bd4da -R 2969a28b361c45f45888cf9a9ac52a9e +P 9ad796a8822f1b7e1e99b73c2cc5be59dbfd85e9c27f3e795c29a2c002c611d1 +R 75d9d47dfde008764fbf9239d7b4fada U drh -Z 1170e81cf6857f6b3144fe1a49be0ee2 +Z 20abf3696cef97c5bdf0b2ae41ceaaa6 diff --git a/manifest.uuid b/manifest.uuid index 55f1f4ff1e..30608cbb37 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ad796a8822f1b7e1e99b73c2cc5be59dbfd85e9c27f3e795c29a2c002c611d1 \ No newline at end of file +0d8e150434bbd179696f1ffe71d1e06cb3d43e6468496c7e481fca8486387bad \ No newline at end of file diff --git a/src/prepare.c b/src/prepare.c index ad07eca45a..811ef95aea 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -882,9 +882,13 @@ char *sqlite3Normalize( z[j++] = ' '; } if( tokenType==TK_ID ){ - int i2 = i, n2 = n; + if( zSql[i]=='"' + && sqlite3VdbeUsesDoubleQuotedString(db,pVdbe,zSql+i,n) + ){ + z[j++] = '?'; + break; + } if( nParen==nParenAtIN ) iStartIN = 0; - if( flags&SQLITE_TOKEN_QUOTED ){ i2++; n2-=2; } } copyNormalizedToken(zSql, i, n, flags, z, &j); break; diff --git a/src/resolve.c b/src/resolve.c index 39c2039f59..a9262a45ad 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -490,6 +490,9 @@ static int lookupName( */ sqlite3_log(SQLITE_WARNING, "double-quoted string literal: \"%w\"", zCol); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3VdbeAddDblquoteStr(db,pParse->pVdbe, zCol); +#endif pExpr->op = TK_STRING; pExpr->y.pTab = 0; return WRC_Prune; diff --git a/src/vdbe.h b/src/vdbe.h index d42acc0cca..1712b8b224 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -251,6 +251,10 @@ void sqlite3VdbeCountChanges(Vdbe*); sqlite3 *sqlite3VdbeDb(Vdbe*); u8 sqlite3VdbePrepareFlags(Vdbe*); void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8); +#ifdef SQLITE_ENABLE_NORMALIZE +void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*); +int sqlite3VdbeUsesDoubleQuotedString(sqlite3*,Vdbe*,const char*,int); +#endif void sqlite3VdbeSwap(Vdbe*,Vdbe*); VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 5c719923d7..70543526c1 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -335,6 +335,9 @@ struct sqlite3_context { */ typedef unsigned bft; /* Bit Field Type */ +/* The ScanStatus object holds a single value for the +** sqlite3_stmt_scanstatus() interface. +*/ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */ @@ -345,6 +348,19 @@ struct ScanStatus { char *zName; /* Name of table or index */ }; +/* The DblquoteStr object holds the text of a double-quoted +** string for a prepared statement. A linked list of these objects +** is constructed during statement parsing and is held on Vdbe.pDblStr. +** When computing a normalized SQL statement for an SQL statement, that +** list is consulted for each double-quoted identifier to see if the +** identifier should really be a string literal. +*/ +typedef struct DblquoteStr DblquoteStr; +struct DblquoteStr { + DblquoteStr *pNextStr; /* Next string literal in the list */ + char z[8]; /* Dequoted value for the string */ +}; + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. @@ -364,7 +380,7 @@ struct Vdbe { int pc; /* The program counter */ int rc; /* Value to return */ int nChange; /* Number of db changes made since last reset */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ + int iStatement; /* Statement number (or 0 if has no opened stmt) */ i64 iCurrentTime; /* Value of julianday('now') for this statement */ i64 nFkConstraint; /* Number of imm. FK constraints this VM */ i64 nStmtDefCons; /* Number of def. constraints when stmt started */ @@ -408,6 +424,7 @@ struct Vdbe { char *zSql; /* Text of the SQL statement that generated this */ #ifdef SQLITE_ENABLE_NORMALIZE char *zNormSql; /* Normalization of the associated SQL statement */ + DblquoteStr *pDblStr; /* List of double-quoted string literals */ #endif void *pFree; /* Free this when deleting the vdbe */ VdbeFrame *pFrame; /* Parent frame */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 40a63319db..2f45c217f1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -73,6 +73,53 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){ #endif } +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Add a new element to the Vdbe->pDblStr list. +*/ +void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){ + if( p ){ + int n = sqlite3Strlen30(z); + DblquoteStr *pStr = sqlite3DbMallocRawNN(db, + sizeof(*pStr)+n+1-sizeof(pStr->z)); + if( pStr ){ + pStr->pNextStr = p->pDblStr; + p->pDblStr = pStr; + memcpy(pStr->z, z, n+1); + } + } +} +#endif + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** zId of length nId is a double-quoted identifier. Check to see if +** that identifier is really used as a string literal. +*/ +int sqlite3VdbeUsesDoubleQuotedString( + sqlite3 *db, /* Used for transient malloc */ + Vdbe *pVdbe, /* The prepared statement */ + const char *zId, /* The double-quoted identifier */ + int nId /* Bytes in zId, which is not zero-terminated */ +){ + char *z; + DblquoteStr *pStr; + assert( zId!=0 ); + assert( zId[0]=='"' ); + assert( nId>=2 ); + assert( zId[nId-1]=='"' ); + if( pVdbe->pDblStr==0 ) return 0; + z = sqlite3DbStrNDup(db, zId, nId); + if( z==0 ) return 0; + sqlite3Dequote(z); + for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){ + if( strcmp(z, pStr->z)==0 ) break; + } + sqlite3DbFree(db, z); + return pStr!=0; +} +#endif + /* ** Swap all content between two VDBE structures. */ @@ -92,7 +139,7 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ zTmp = pA->zSql; pA->zSql = pB->zSql; pB->zSql = zTmp; -#ifdef SQLITE_ENABLE_NORMALIZE +#if 0 zTmp = pA->zNormSql; pA->zNormSql = pB->zNormSql; pB->zNormSql = zTmp; @@ -3170,6 +3217,13 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->zSql); #ifdef SQLITE_ENABLE_NORMALIZE sqlite3DbFree(db, p->zNormSql); + { + DblquoteStr *pThis, *pNext; + for(pThis=p->pDblStr; pThis; pThis=pNext){ + pNext = pThis->pNextStr; + sqlite3DbFree(db, pThis); + } + } #endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS { diff --git a/test/normalize.test b/test/normalize.test index c5b69336e9..35c995fe75 100644 --- a/test/normalize.test +++ b/test/normalize.test @@ -207,7 +207,7 @@ foreach {tnum sql flags norm} { 430 {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');} 0x2 - {0 {SELECT"a"FROM t1 WHERE"x"IN("1","2",?);}} + {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}} 440 {SELECT 'a' FROM t1 WHERE 'x';} From b0b7db91b8cc880ae7d5f3c334607fda17364d62 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Dec 2018 17:28:28 +0000 Subject: [PATCH 33/52] Prototype implementation for the VACUUM INTO command. FossilOrigin-Name: 036e3320a4af36c1311b25b2e504b0079c8b33df8ad7b7e5fddad07150e6f87d --- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/parse.y | 7 +++++-- src/sqliteInt.h | 4 ++-- src/vacuum.c | 37 ++++++++++++++++++++++++++----------- src/vdbe.c | 8 ++++++-- 6 files changed, 52 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index f4fb1dff1b..f3b956e8c9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3_normalized_sql()\sinterface\sso\sthat\sit\srenders\ndouble-quoted\sstring\sliterals\sas\s"?". -D 2018-12-07T16:32:11.362 +C Prototype\simplementation\sfor\sthe\sVACUUM\sINTO\scommand. +D 2018-12-07T17:28:28.770 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -496,7 +496,7 @@ F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73 F src/pager.h 217921e81eb5fe455caa5cda96061959706bcdd29ddb57166198645ef7822ac3 -F src/parse.y 5cf85c2b9dfac38ac4e2bf2776484705186ce2eda8631e65cc0b04bf566c1173 +F src/parse.y 3bf1b720e70a3cfb08f491913f61b181d71576879146d68ee845f6d77c6bdc70 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc @@ -512,7 +512,7 @@ F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883 F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h 97adda953e7f118d47b8135f76f88c6420ff6707285782616f393a9ea255d577 +F src/sqliteInt.h fa86b5689fae6d3b446fcae0430e4d1f3757bf5e72d0ca2e34d1287a2c3643f2 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -577,8 +577,8 @@ F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274 F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 -F src/vacuum.c c67085526fb51e141a85be3f7c9b91a2d38705bbb0ef911d8dec1e2d36cbdce1 -F src/vdbe.c c7312c909df5032b4ed27996c2ed8f5bf948f6a982458f85398520aaba0ccf81 +F src/vacuum.c 76b92b5d6afc83a8f396b59bcc042955bf563f07b6aa60eea3d57f7be5336180 +F src/vdbe.c 872bdd34338548242b36df18c49c90b34689e41c0b4e5c197e83bb82a38ce8dd F src/vdbe.h d82f323d581b36b8e147d650257ef34e0e93790039b6cbda45c321c275f7595e F src/vdbeInt.h 73f5051923f3f29779bfc374c0c68e23b8e5e3792def2e33e51b427edb890abd F src/vdbeapi.c 9709452bee82963e1f7f1f5d0c71db823d553f8dbb2c47a911c4983d537a1947 @@ -1782,7 +1782,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ad796a8822f1b7e1e99b73c2cc5be59dbfd85e9c27f3e795c29a2c002c611d1 -R 75d9d47dfde008764fbf9239d7b4fada +P 0d8e150434bbd179696f1ffe71d1e06cb3d43e6468496c7e481fca8486387bad +R 319ff574feecdbd7034aa017b5a3dd35 +T *branch * vacuum-into +T *sym-vacuum-into * +T -sym-trunk * U drh -Z 20abf3696cef97c5bdf0b2ae41ceaaa6 +Z 59d199189f2c579663aad138ddb0acd8 diff --git a/manifest.uuid b/manifest.uuid index 30608cbb37..3de4b7164f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d8e150434bbd179696f1ffe71d1e06cb3d43e6468496c7e481fca8486387bad \ No newline at end of file +036e3320a4af36c1311b25b2e504b0079c8b33df8ad7b7e5fddad07150e6f87d \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 3bb28ab5f5..dbbd40775e 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1367,8 +1367,11 @@ cmd ::= DROP INDEX ifexists(E) fullname(X). {sqlite3DropIndex(pParse, X, E);} // %ifndef SQLITE_OMIT_VACUUM %ifndef SQLITE_OMIT_ATTACH -cmd ::= VACUUM. {sqlite3Vacuum(pParse,0);} -cmd ::= VACUUM nm(X). {sqlite3Vacuum(pParse,&X);} +%type vinto {Token} +cmd ::= VACUUM vinto(Y). {sqlite3Vacuum(pParse,0,&Y);} +cmd ::= VACUUM nm(X) vinto(Y). {sqlite3Vacuum(pParse,&X,&Y);} +vinto(A) ::= INTO nm(X). {A = X;} +vinto(A) ::= . {A.z = 0;} %endif SQLITE_OMIT_ATTACH %endif SQLITE_OMIT_VACUUM diff --git a/src/sqliteInt.h b/src/sqliteInt.h index bdf0d7fcbb..e175de9518 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3985,8 +3985,8 @@ Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *); Index *sqlite3FindIndex(sqlite3*,const char*, const char*); void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); -void sqlite3Vacuum(Parse*,Token*); -int sqlite3RunVacuum(char**, sqlite3*, int); +void sqlite3Vacuum(Parse*,Token*,Token*); +int sqlite3RunVacuum(char**, sqlite3*, int, const char*); char *sqlite3NameFromToken(sqlite3*, Token*); int sqlite3ExprCompare(Parse*,Expr*, Expr*, int); int sqlite3ExprCompareSkip(Expr*, Expr*, int); diff --git a/src/vacuum.c b/src/vacuum.c index 8416b16729..04958ab53f 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -102,9 +102,10 @@ static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ ** transient would cause the database file to appear to be deleted ** following reboot. */ -void sqlite3Vacuum(Parse *pParse, Token *pNm){ +void sqlite3Vacuum(Parse *pParse, Token *pNm, Token *pInto){ Vdbe *v = sqlite3GetVdbe(pParse); int iDb = 0; + assert( pInto!=0 ); if( v==0 ) return; if( pNm ){ #ifndef SQLITE_BUG_COMPATIBLE_20160819 @@ -125,6 +126,10 @@ void sqlite3Vacuum(Parse *pParse, Token *pNm){ if( iDb!=1 ){ sqlite3VdbeAddOp1(v, OP_Vacuum, iDb); sqlite3VdbeUsesBtree(v, iDb); + if( pInto->z ){ + char *zName = sqlite3NameFromToken(pParse->db, pInto); + sqlite3VdbeChangeP4(v, -1, zName, P4_DYNAMIC); + } } return; } @@ -132,7 +137,7 @@ void sqlite3Vacuum(Parse *pParse, Token *pNm){ /* ** This routine implements the OP_Vacuum opcode of the VDBE. */ -int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ +int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb, const char *zOut){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ Btree *pTemp; /* The temporary database we vacuum into */ @@ -189,7 +194,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ ** to write the journal header file. */ nDb = db->nDb; - rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db"); + rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut ? zOut : ""); if( rc!=SQLITE_OK ) goto end_of_vacuum; assert( (db->nDb-1)==nDb ); pDb = &db->aDb[nDb]; @@ -225,7 +230,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ */ rc = execSql(db, pzErrMsg, "BEGIN"); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = sqlite3BtreeBeginTrans(pMain, 2, 0); + rc = sqlite3BtreeBeginTrans(pMain, zOut==0 ? 2 : 0, 0); if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Do not attempt to change the page size for a WAL database */ @@ -320,7 +325,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); - assert( 1==sqlite3BtreeIsInTrans(pMain) ); + assert( zOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) ); /* Copy Btree meta values */ for(i=0; iflags */ @@ -368,9 +381,11 @@ end_of_vacuum: pDb->pSchema = 0; } - /* This both clears the schemas and reduces the size of the db->aDb[] - ** array. */ - sqlite3ResetAllSchemasOfConnection(db); + if( zOut==0 ){ + /* This both clears the schemas and reduces the size of the db->aDb[] + ** array. */ + sqlite3ResetAllSchemasOfConnection(db); + } return rc; } diff --git a/src/vdbe.c b/src/vdbe.c index 2803330b34..179189eea7 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6684,14 +6684,18 @@ case OP_JournalMode: { /* out2 */ #endif /* SQLITE_OMIT_PRAGMA */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* Opcode: Vacuum P1 * * * * +/* Opcode: Vacuum P1 * * P4 * ** ** Vacuum the entire database P1. P1 is 0 for "main", and 2 or more ** for an attached database. The "temp" database may not be vacuumed. +** +** If P4 is not a NULL pointer, then it is a UTF8 string which is the +** name of a file into which to write the result of the vacuum. When +** P4 is NULL, the result of vacuum overwrites the original database. */ case OP_Vacuum: { assert( p->readOnly==0 ); - rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1); + rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1, pOp->p4.z); if( rc ) goto abort_due_to_error; break; } From 83da91b7bd6a6253080fa3b821e4adc9002a3541 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Dec 2018 20:26:29 +0000 Subject: [PATCH 34/52] Remove a line of code from VACUUM that is no longer needed, due to the change that allows ATTACH to run within a transaction. FossilOrigin-Name: a92c398fc5df142ff1459c1be4a6832f2219bc7fabe5789535be3bbd41a4269b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vacuum.c | 7 ------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index f4fb1dff1b..46031d6ed1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3_normalized_sql()\sinterface\sso\sthat\sit\srenders\ndouble-quoted\sstring\sliterals\sas\s"?". -D 2018-12-07T16:32:11.362 +C Remove\sa\sline\sof\scode\sfrom\sVACUUM\sthat\sis\sno\slonger\sneeded,\sdue\sto\sthe\nchange\sthat\sallows\sATTACH\sto\srun\swithin\sa\stransaction. +D 2018-12-07T20:26:29.949 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -577,7 +577,7 @@ F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274 F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 -F src/vacuum.c c67085526fb51e141a85be3f7c9b91a2d38705bbb0ef911d8dec1e2d36cbdce1 +F src/vacuum.c b25dbace5b6727cd906441e87857b828eb2377f7b1b2ba3eec36155ce727730c F src/vdbe.c c7312c909df5032b4ed27996c2ed8f5bf948f6a982458f85398520aaba0ccf81 F src/vdbe.h d82f323d581b36b8e147d650257ef34e0e93790039b6cbda45c321c275f7595e F src/vdbeInt.h 73f5051923f3f29779bfc374c0c68e23b8e5e3792def2e33e51b427edb890abd @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9ad796a8822f1b7e1e99b73c2cc5be59dbfd85e9c27f3e795c29a2c002c611d1 -R 75d9d47dfde008764fbf9239d7b4fada +P 0d8e150434bbd179696f1ffe71d1e06cb3d43e6468496c7e481fca8486387bad +R 4426cbb4a0e989d65734a42a486b3458 U drh -Z 20abf3696cef97c5bdf0b2ae41ceaaa6 +Z 888d594bf1edeab12c4e75c4f15da299 diff --git a/manifest.uuid b/manifest.uuid index 30608cbb37..5c7c601c94 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d8e150434bbd179696f1ffe71d1e06cb3d43e6468496c7e481fca8486387bad \ No newline at end of file +a92c398fc5df142ff1459c1be4a6832f2219bc7fabe5789535be3bbd41a4269b \ No newline at end of file diff --git a/src/vacuum.c b/src/vacuum.c index 8416b16729..4570371fb0 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -195,13 +195,6 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ pDb = &db->aDb[nDb]; assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); pTemp = pDb->pBt; - - /* The call to execSql() to attach the temp database has left the file - ** locked (as there was more than one active statement when the transaction - ** to read the schema was concluded. Unlock it here so that this doesn't - ** cause problems for the call to BtreeSetPageSize() below. */ - sqlite3BtreeCommit(pTemp); - nRes = sqlite3BtreeGetOptimalReserve(pMain); /* A VACUUM cannot change the pagesize of an encrypted database. */ From 7464f578eda14dbf56d4e37d72aae2af03a5930c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Dec 2018 23:48:41 +0000 Subject: [PATCH 35/52] Do not allow VACUUM INTO into a file that already exists. FossilOrigin-Name: 92f70e0fa3c9de7fde046f11cc0a7c2800511bb5ace8e68c845133931607616e --- manifest | 13 +++++----- manifest.uuid | 2 +- src/vacuum.c | 17 +++++++++---- test/vacuum-into.test | 58 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 12 deletions(-) create mode 100644 test/vacuum-into.test diff --git a/manifest b/manifest index c59fcc0427..635f30acfa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sVACUUM\ssimplification\sfrom\strunk. -D 2018-12-07T20:40:12.098 +C Do\snot\sallow\sVACUUM\sINTO\sinto\sa\sfile\sthat\salready\sexists. +D 2018-12-07T23:48:41.554 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -577,7 +577,7 @@ F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274 F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 -F src/vacuum.c 7b74a106ecac182900124e7708b3657377a5b2b18812efc31f100caca1c0eae3 +F src/vacuum.c 5d49497308883e608bb420c112e87c10417b576af2a6d1dd0dba62d4a494ffd7 F src/vdbe.c 872bdd34338548242b36df18c49c90b34689e41c0b4e5c197e83bb82a38ce8dd F src/vdbe.h d82f323d581b36b8e147d650257ef34e0e93790039b6cbda45c321c275f7595e F src/vdbeInt.h 73f5051923f3f29779bfc374c0c68e23b8e5e3792def2e33e51b427edb890abd @@ -1562,6 +1562,7 @@ F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae +F test/vacuum-into.test dab9e498ea303f27370e71eb7e9a3369eb71ae90f4ea1afa741a1347ece39f5d F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d @@ -1782,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 036e3320a4af36c1311b25b2e504b0079c8b33df8ad7b7e5fddad07150e6f87d a92c398fc5df142ff1459c1be4a6832f2219bc7fabe5789535be3bbd41a4269b -R 5e2deff1e10a62e804983a88b3d403b2 +P 93d92a0a5d21a1856316c0205ecaa253691b6e5349b552d43027005676d14820 +R 4b91cbcd00a28fb8ade4cd8b52c5b6e4 U drh -Z a260dfd202744b5f58f4cbc1728d27d6 +Z b3677de0205f375bf2be187f033f6ce8 diff --git a/manifest.uuid b/manifest.uuid index 94144aeb81..4f71499d96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -93d92a0a5d21a1856316c0205ecaa253691b6e5349b552d43027005676d14820 \ No newline at end of file +92f70e0fa3c9de7fde046f11cc0a7c2800511bb5ace8e68c845133931607616e \ No newline at end of file diff --git a/src/vacuum.c b/src/vacuum.c index 4f4fcf410c..a28541cc58 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -200,6 +200,15 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb, const char *zOut){ pDb = &db->aDb[nDb]; assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); pTemp = pDb->pBt; + if( zOut!=0 ){ + sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); + i64 sz = 0; + if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ + rc = SQLITE_ERROR; + sqlite3SetString(pzErrMsg, db, "output file already exists"); + goto end_of_vacuum; + } + } nRes = sqlite3BtreeGetOptimalReserve(pMain); /* A VACUUM cannot change the pagesize of an encrypted database. */ @@ -374,11 +383,9 @@ end_of_vacuum: pDb->pSchema = 0; } - if( zOut==0 ){ - /* This both clears the schemas and reduces the size of the db->aDb[] - ** array. */ - sqlite3ResetAllSchemasOfConnection(db); - } + /* This both clears the schemas and reduces the size of the db->aDb[] + ** array. */ + sqlite3ResetAllSchemasOfConnection(db); return rc; } diff --git a/test/vacuum-into.test b/test/vacuum-into.test new file mode 100644 index 0000000000..f04c0becac --- /dev/null +++ b/test/vacuum-into.test @@ -0,0 +1,58 @@ +# 2018-12-07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the VACUUM INTO statement. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If the VACUUM statement is disabled in the current build, skip all +# the tests in this file. +# +ifcapable {!vacuum} { + omit_test vacuum.test {Compiled with SQLITE_OMIT_VACUUM} + finish_test + return +} + +forcedelete out.db +do_execsql_test vacuum-into-100 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b) SELECT x, randomblob(600) FROM c; + CREATE INDEX t1b ON t1(b); + DELETE FROM t1 WHERE a%2; + SELECT count(*), sum(a), sum(length(b)) FROM t1; +} {50 2550 30000} +do_execsql_test vacuum-into-110 { + VACUUM main INTO 'out.db'; +} {} +sqlite3 db2 out.db +do_test vacuum-into-120 { + db2 eval {SELECT count(*), sum(a), sum(length(b)) FROM t1} +} {50 2550 30000} +do_catchsql_test vacuum-into-130 { + VACUUM INTO 'out.db'; +} {1 {output file already exists}} +forcedelete out2.db +do_catchsql_test vacuum-into-140 { + VACUUM INTO 'out2.db'; +} {0 {}} +do_catchsql_test vacuum-into-150 { + VACUUM INTO 'out2.db'; +} {1 {output file already exists}} + +do_catchsql_test vacuum-into-200 { + VACUUM main INTO ':memory:'; +} {0 {}} + +finish_test From 2f6239ed4e7d8e152de77d00aae4678d62d7e4f3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 8 Dec 2018 00:43:08 +0000 Subject: [PATCH 36/52] Allow the INTO clause of VACUUM to be a text-valued expression. FossilOrigin-Name: af172b53b46759f491f522356e14c5e2374d3f25ec70fbc1e100cadded8f9b22 --- manifest | 20 ++++++++-------- manifest.uuid | 2 +- src/parse.y | 11 +++++---- src/sqliteInt.h | 4 ++-- src/vacuum.c | 53 +++++++++++++++++++++++++++---------------- src/vdbe.c | 11 +++++---- test/vacuum-into.test | 11 +++++++++ 7 files changed, 70 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 635f30acfa..52fef86043 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sVACUUM\sINTO\sinto\sa\sfile\sthat\salready\sexists. -D 2018-12-07T23:48:41.554 +C Allow\sthe\sINTO\sclause\sof\sVACUUM\sto\sbe\sa\stext-valued\sexpression. +D 2018-12-08T00:43:08.028 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -496,7 +496,7 @@ F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73 F src/pager.h 217921e81eb5fe455caa5cda96061959706bcdd29ddb57166198645ef7822ac3 -F src/parse.y 3bf1b720e70a3cfb08f491913f61b181d71576879146d68ee845f6d77c6bdc70 +F src/parse.y a3c0db595bc642c6ee1d72869842f7f5b0b6ebeb91c21d0a7cba631d27e7afbd F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc @@ -512,7 +512,7 @@ F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883 F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h fa86b5689fae6d3b446fcae0430e4d1f3757bf5e72d0ca2e34d1287a2c3643f2 +F src/sqliteInt.h 70ce5e14c887554d3c51f2045f5a95b6e83de745d7f6448e79e49fdd8dfc2d5c F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -577,8 +577,8 @@ F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274 F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 -F src/vacuum.c 5d49497308883e608bb420c112e87c10417b576af2a6d1dd0dba62d4a494ffd7 -F src/vdbe.c 872bdd34338548242b36df18c49c90b34689e41c0b4e5c197e83bb82a38ce8dd +F src/vacuum.c 3ffe64ecfc94b7528c5d7bdb1c3a19d72fec63f2aa846e3b90f8de5dbbddf5aa +F src/vdbe.c 55bafc424748d9ed505ab2680736e51d1bb05c01e9885cbb3b287b51dc8b47ec F src/vdbe.h d82f323d581b36b8e147d650257ef34e0e93790039b6cbda45c321c275f7595e F src/vdbeInt.h 73f5051923f3f29779bfc374c0c68e23b8e5e3792def2e33e51b427edb890abd F src/vdbeapi.c 9709452bee82963e1f7f1f5d0c71db823d553f8dbb2c47a911c4983d537a1947 @@ -1562,7 +1562,7 @@ F test/uri.test 3481026f00ade6dfe8adb7acb6e1e47b04369568 F test/uri2.test 9d3ba7a53ee167572d53a298ee4a5d38ec4a8fb7 F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae -F test/vacuum-into.test dab9e498ea303f27370e71eb7e9a3369eb71ae90f4ea1afa741a1347ece39f5d +F test/vacuum-into.test 181a8ae8c2479d88ebc118076e8cfbc062ad8f8a51b56a139bd12870a8a84c34 F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d @@ -1783,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 93d92a0a5d21a1856316c0205ecaa253691b6e5349b552d43027005676d14820 -R 4b91cbcd00a28fb8ade4cd8b52c5b6e4 +P 92f70e0fa3c9de7fde046f11cc0a7c2800511bb5ace8e68c845133931607616e +R f797d08299c49f27762cbb498a2316b3 U drh -Z b3677de0205f375bf2be187f033f6ce8 +Z 0107ff29eb1054c07981156ef518bc62 diff --git a/manifest.uuid b/manifest.uuid index 4f71499d96..d2641d7679 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92f70e0fa3c9de7fde046f11cc0a7c2800511bb5ace8e68c845133931607616e \ No newline at end of file +af172b53b46759f491f522356e14c5e2374d3f25ec70fbc1e100cadded8f9b22 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index dbbd40775e..3f9384c86c 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1367,11 +1367,12 @@ cmd ::= DROP INDEX ifexists(E) fullname(X). {sqlite3DropIndex(pParse, X, E);} // %ifndef SQLITE_OMIT_VACUUM %ifndef SQLITE_OMIT_ATTACH -%type vinto {Token} -cmd ::= VACUUM vinto(Y). {sqlite3Vacuum(pParse,0,&Y);} -cmd ::= VACUUM nm(X) vinto(Y). {sqlite3Vacuum(pParse,&X,&Y);} -vinto(A) ::= INTO nm(X). {A = X;} -vinto(A) ::= . {A.z = 0;} +%type vinto {Expr*} +%destructor vinto {sqlite3ExprDelete(pParse->db, $$);} +cmd ::= VACUUM vinto(Y). {sqlite3Vacuum(pParse,0,Y);} +cmd ::= VACUUM nm(X) vinto(Y). {sqlite3Vacuum(pParse,&X,Y);} +vinto(A) ::= INTO expr(X). {A = X;} +vinto(A) ::= . {A = 0;} %endif SQLITE_OMIT_ATTACH %endif SQLITE_OMIT_VACUUM diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e175de9518..0412a33523 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3985,8 +3985,8 @@ Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_item *); Index *sqlite3FindIndex(sqlite3*,const char*, const char*); void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); -void sqlite3Vacuum(Parse*,Token*,Token*); -int sqlite3RunVacuum(char**, sqlite3*, int, const char*); +void sqlite3Vacuum(Parse*,Token*,Expr*); +int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*); char *sqlite3NameFromToken(sqlite3*, Token*); int sqlite3ExprCompare(Parse*,Expr*, Expr*, int); int sqlite3ExprCompareSkip(Expr*, Expr*, int); diff --git a/src/vacuum.c b/src/vacuum.c index a28541cc58..b4041b6d97 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -102,17 +102,16 @@ static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ ** transient would cause the database file to appear to be deleted ** following reboot. */ -void sqlite3Vacuum(Parse *pParse, Token *pNm, Token *pInto){ +void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){ Vdbe *v = sqlite3GetVdbe(pParse); int iDb = 0; - assert( pInto!=0 ); - if( v==0 ) return; + if( v==0 ) goto build_vacuum_end; if( pNm ){ #ifndef SQLITE_BUG_COMPATIBLE_20160819 /* Default behavior: Report an error if the argument to VACUUM is ** not recognized */ iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm); - if( iDb<0 ) return; + if( iDb<0 ) goto build_vacuum_end; #else /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments ** to VACUUM are silently ignored. This is a back-out of a bug fix that @@ -124,20 +123,28 @@ void sqlite3Vacuum(Parse *pParse, Token *pNm, Token *pInto){ #endif } if( iDb!=1 ){ - sqlite3VdbeAddOp1(v, OP_Vacuum, iDb); - sqlite3VdbeUsesBtree(v, iDb); - if( pInto->z ){ - char *zName = sqlite3NameFromToken(pParse->db, pInto); - sqlite3VdbeChangeP4(v, -1, zName, P4_DYNAMIC); + int iIntoReg = 0; + if( pInto ){ + iIntoReg = ++pParse->nMem; + sqlite3ExprCode(pParse, pInto, iIntoReg); } + sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg); + sqlite3VdbeUsesBtree(v, iDb); } +build_vacuum_end: + sqlite3ExprDelete(pParse->db, pInto); return; } /* ** This routine implements the OP_Vacuum opcode of the VDBE. */ -int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb, const char *zOut){ +int sqlite3RunVacuum( + char **pzErrMsg, /* Write error message here */ + sqlite3 *db, /* Database connection */ + int iDb, /* Which attached DB to vacuum */ + sqlite3_value *pOut /* Write results here, if not NULL */ +){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ Btree *pTemp; /* The temporary database we vacuum into */ @@ -151,6 +158,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb, const char *zOut){ int nRes; /* Bytes of reserved space at the end of each page */ int nDb; /* Number of attached databases */ const char *zDbMain; /* Schema name of database to vacuum */ + const char *zOut; /* Name of output file */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); @@ -160,6 +168,15 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb, const char *zOut){ sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); return SQLITE_ERROR; } + if( pOut ){ + if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){ + sqlite3SetString(pzErrMsg, db, "non-text filename"); + return SQLITE_ERROR; + } + zOut = (const char*)sqlite3_value_text(pOut); + }else{ + zOut = ""; + } /* Save the current value of the database flags so that it can be ** restored before returning. Then set the writable-schema flag, and @@ -194,13 +211,13 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb, const char *zOut){ ** to write the journal header file. */ nDb = db->nDb; - rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut ? zOut : ""); + rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut); if( rc!=SQLITE_OK ) goto end_of_vacuum; assert( (db->nDb-1)==nDb ); pDb = &db->aDb[nDb]; assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); pTemp = pDb->pBt; - if( zOut!=0 ){ + if( pOut ){ sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); i64 sz = 0; if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ @@ -232,7 +249,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb, const char *zOut){ */ rc = execSql(db, pzErrMsg, "BEGIN"); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = sqlite3BtreeBeginTrans(pMain, zOut==0 ? 2 : 0, 0); + rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0); if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Do not attempt to change the page size for a WAL database */ @@ -327,7 +344,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb, const char *zOut){ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); - assert( zOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) ); + assert( pOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) ); /* Copy Btree meta values */ for(i=0; ireadOnly==0 ); - rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1, pOp->p4.z); + rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1, + pOp->p2 ? &aMem[pOp->p2] : 0); if( rc ) goto abort_due_to_error; break; } diff --git a/test/vacuum-into.test b/test/vacuum-into.test index f04c0becac..cb91abc5d9 100644 --- a/test/vacuum-into.test +++ b/test/vacuum-into.test @@ -55,4 +55,15 @@ do_catchsql_test vacuum-into-200 { VACUUM main INTO ':memory:'; } {0 {}} +# The INTO argument can be an arbitrary expression. +# +do_execsql_test vacuum-into-300 { + CREATE TABLE t2(name TEXT); + INSERT INTO t2 VALUES(':memory:'); + VACUUM main INTO (SELECT name FROM t2); +} {} +do_catchsql_test vacuum-into-310 { + VACUUM INTO null; +} {1 {non-text filename}} + finish_test From a50bffb85ec1246b95468341ccac3748577d9b31 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 8 Dec 2018 01:09:14 +0000 Subject: [PATCH 37/52] Add the --async option to the ".backup" command in the CLI. FossilOrigin-Name: 7b6a605b1883dfcbe237b0a58f6f0c233a109e88b521c3b3d97198526c8a0317 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 11 ++++++++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 52fef86043..c01c4e39ca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sINTO\sclause\sof\sVACUUM\sto\sbe\sa\stext-valued\sexpression. -D 2018-12-08T00:43:08.028 +C Add\sthe\s--async\soption\sto\sthe\s".backup"\scommand\sin\sthe\sCLI. +D 2018-12-08T01:09:14.691 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -508,7 +508,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 095d1d41d7a981ee9db8bfeae25ed0d6a8a5e5e3d66b0f4efd71877ed7b56132 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 8c7317d5ee920516a56b8b4ca79fbfca70a1f8b52d67e884c808ea3a016c04e3 -F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d +F src/shell.c.in 5f38bd0e127c2cc4e506b5c3565c10879ddfae6c2d867bb5972563e40717c19c F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 @@ -1783,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 92f70e0fa3c9de7fde046f11cc0a7c2800511bb5ace8e68c845133931607616e -R f797d08299c49f27762cbb498a2316b3 +P af172b53b46759f491f522356e14c5e2374d3f25ec70fbc1e100cadded8f9b22 +R df568823aa698ca1a561c86a7815b31b U drh -Z 0107ff29eb1054c07981156ef518bc62 +Z 361de2f1352642d2a8077d8c1209b182 diff --git a/manifest.uuid b/manifest.uuid index d2641d7679..d46907ba61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af172b53b46759f491f522356e14c5e2374d3f25ec70fbc1e100cadded8f9b22 \ No newline at end of file +7b6a605b1883dfcbe237b0a58f6f0c233a109e88b521c3b3d97198526c8a0317 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 887c2fcf59..fd9cefa7a8 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3377,6 +3377,7 @@ static const char *(azHelp[]) = { #endif ".backup ?DB? FILE Backup DB (default \"main\") to FILE", " --append Use the appendvfs", + " --async Write to FILE without a journal and without fsync()", ".bail on|off Stop after hitting an error. Default OFF", ".binary on|off Turn binary output on or off. Default OFF", ".cd DIRECTORY Change the working directory to DIRECTORY", @@ -5840,6 +5841,7 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3 *pDest; sqlite3_backup *pBackup; int j; + int bAsync = 0; const char *zVfs = 0; for(j=1; jdb, zDb); if( pBackup==0 ){ From d4a4a36171fdc93a9fc117af2f957c86624b50da Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 8 Dec 2018 20:30:31 +0000 Subject: [PATCH 38/52] Do not use a partial index as a table scan in an IN operator. Fix for ticket [1d958d90596593a77420e59]. FossilOrigin-Name: 15bc915dd7ae25bfdfbe357c3792db3639799e4d121c6b5b24297af2d6d7129e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 1 + test/index6.test | 19 ++++++++++++++++++- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 46031d6ed1..600419b00c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sline\sof\scode\sfrom\sVACUUM\sthat\sis\sno\slonger\sneeded,\sdue\sto\sthe\nchange\sthat\sallows\sATTACH\sto\srun\swithin\sa\stransaction. -D 2018-12-07T20:26:29.949 +C Do\snot\suse\sa\spartial\sindex\sas\sa\stable\sscan\sin\san\sIN\soperator.\nFix\sfor\sticket\s[1d958d90596593a77420e59]. +D 2018-12-08T20:30:31.225 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -459,7 +459,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7 F src/dbstat.c 3c8bd4e77f0244fd2bd7cc90acf116ad2f8e82d70e536637f35ac2bc99b726f9 F src/delete.c f7938125847e8ef485448db5fbad29acb2991381a02887dd854c1617315ab9fb -F src/expr.c b83e2056086e364293ce741509d885d1646a32d62727fe3169b23231c7e4f62f +F src/expr.c 494a778ee193d99127c72adefcec3e8e133f482acf5f1c267652a8f51029e9ae F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812 F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f @@ -1015,7 +1015,7 @@ F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 F test/index3.test 51685f39345462b84fcf77eb8537af847fdf438cc96b05c45d6aaca4e473ade0 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 -F test/index6.test d07ea75b8c21f125c6f325522e8df8c05c91e9251ec923a31d0582b2ba4a617d +F test/index6.test 7cb66c4e54da73f51e874dcf4a761411c0f8f3f5a4fdf7022d8a505b0194296b F test/index7.test 72b59b8ddc5c13f4962886b4011eb9975014317d17ef36c6297921362fb7dd98 F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0d8e150434bbd179696f1ffe71d1e06cb3d43e6468496c7e481fca8486387bad -R 4426cbb4a0e989d65734a42a486b3458 +P a92c398fc5df142ff1459c1be4a6832f2219bc7fabe5789535be3bbd41a4269b +R b6f349da580def7412aedbdbd15fbe9b U drh -Z 888d594bf1edeab12c4e75c4f15da299 +Z 66d01b11a28cdf2981ee67c4d1459164 diff --git a/manifest.uuid b/manifest.uuid index 5c7c601c94..e603f26372 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a92c398fc5df142ff1459c1be4a6832f2219bc7fabe5789535be3bbd41a4269b \ No newline at end of file +15bc915dd7ae25bfdfbe357c3792db3639799e4d121c6b5b24297af2d6d7129e \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 6886d30b2d..0b2e797752 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2443,6 +2443,7 @@ int sqlite3FindInIndex( Bitmask colUsed; /* Columns of the index used */ Bitmask mCol; /* Mask for the current column */ if( pIdx->nColumnpPartIdxWhere!=0 ) continue; /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute ** BITMASK(nExpr) without overflowing */ testcase( pIdx->nColumn==BMS-2 ); diff --git a/test/index6.test b/test/index6.test index 4ddce453fd..33c2ed3382 100644 --- a/test/index6.test +++ b/test/index6.test @@ -389,6 +389,23 @@ do_execsql_test index6-11.1 { do_execsql_test index6-11.2 { EXPLAIN QUERY PLAN SELECT a FROM t11 WHERE b<>99 AND c<>98; } {/USING INDEX t11x/} - + +# 2018-12-08 +# Ticket https://www.sqlite.org/src/info/1d958d90596593a7 +# NOT IN operator fails when using a partial index. +# +do_execsql_test index6-12.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(1,1); + INSERT INTO t1 VALUES(2,2); + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(1); + INSERT INTO t2 VALUES(2); + SELECT 'one', * FROM t2 WHERE x NOT IN (SELECT a FROM t1); + CREATE INDEX t1a ON t1(a) WHERE b=1; + SELECT 'two', * FROM t2 WHERE x NOT IN (SELECT a FROM t1); +} {} finish_test From ce706c2cf340f91bdeaa42dda03db1a2eafe8d6f Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 9 Dec 2018 18:55:05 +0000 Subject: [PATCH 39/52] New test case for ticket [1d958d90596593a77420e59]. FossilOrigin-Name: b7bf3c9832bde26b359f4d58ebcf7788c500586d311387d022192ed65a126252 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/index6.test | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 600419b00c..c70fc83c3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sa\spartial\sindex\sas\sa\stable\sscan\sin\san\sIN\soperator.\nFix\sfor\sticket\s[1d958d90596593a77420e59]. -D 2018-12-08T20:30:31.225 +C New\stest\scase\sfor\sticket\s[1d958d90596593a77420e59]. +D 2018-12-09T18:55:05.721 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -1015,7 +1015,7 @@ F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 F test/index3.test 51685f39345462b84fcf77eb8537af847fdf438cc96b05c45d6aaca4e473ade0 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 -F test/index6.test 7cb66c4e54da73f51e874dcf4a761411c0f8f3f5a4fdf7022d8a505b0194296b +F test/index6.test 6b3e6cd4bef343ed4541e74c55936ed112962a6552c085242612b598e12910a4 F test/index7.test 72b59b8ddc5c13f4962886b4011eb9975014317d17ef36c6297921362fb7dd98 F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/index9.test 0aa3e509dddf81f93380396e40e9bb386904c1054924ba8fa9bcdfe85a8e7721 @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a92c398fc5df142ff1459c1be4a6832f2219bc7fabe5789535be3bbd41a4269b -R b6f349da580def7412aedbdbd15fbe9b +P 15bc915dd7ae25bfdfbe357c3792db3639799e4d121c6b5b24297af2d6d7129e +R c0db1af5f2bfbd2fba4377002cf1535d U drh -Z 66d01b11a28cdf2981ee67c4d1459164 +Z e19bb708feb379cc8644d3f55b45ddb6 diff --git a/manifest.uuid b/manifest.uuid index e603f26372..1e7c3a3d08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15bc915dd7ae25bfdfbe357c3792db3639799e4d121c6b5b24297af2d6d7129e \ No newline at end of file +b7bf3c9832bde26b359f4d58ebcf7788c500586d311387d022192ed65a126252 \ No newline at end of file diff --git a/test/index6.test b/test/index6.test index 33c2ed3382..29b73f72d4 100644 --- a/test/index6.test +++ b/test/index6.test @@ -407,5 +407,7 @@ do_execsql_test index6-12.1 { CREATE INDEX t1a ON t1(a) WHERE b=1; SELECT 'two', * FROM t2 WHERE x NOT IN (SELECT a FROM t1); } {} - +do_execsql_test index6-12.2 { + SELECT x FROM t2 WHERE x IN (SELECT a FROM t1) ORDER BY +x; +} {1 2} finish_test From 4d342827d60d9bb7a63771395c5f36ac53f7f9f8 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Dec 2018 00:41:28 +0000 Subject: [PATCH 40/52] Fix the shell1.test test for the new format of the .backup command. FossilOrigin-Name: 9748d7995bc1dd632d66d2d326048d50e29c6f5e6993d5a6294d14421d2cb72f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/shell1.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 033a02476a..5e8b9f496f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfixes\sfrom\strunk. -D 2018-12-10T00:40:43.719 +C Fix\sthe\sshell1.test\stest\sfor\sthe\snew\sformat\sof\sthe\s.backup\scommand. +D 2018-12-10T00:41:28.076 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -1277,7 +1277,7 @@ F test/sharedA.test 49d87ec54ab640fbbc3786ee3c01de94aaa482a3a9f834ad3fe92770eb69 F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test d2bf5daeb6f449f0169c9ef3094db17a16a02199c5dcf1a635a3e79b07eb0edd +F test/shell1.test 0378c4e9e800da6fbb3c86c0c8f2cf5efc9e4155b4b6447d27dc71de648fc0a1 F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494 F test/shell4.test a6881d0ae226ded0df8ebdfa574c5aa6dc28d6884ccba1089dc56ed08b9e5ef4 @@ -1783,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7b6a605b1883dfcbe237b0a58f6f0c233a109e88b521c3b3d97198526c8a0317 b7bf3c9832bde26b359f4d58ebcf7788c500586d311387d022192ed65a126252 -R 9c5048fab1535ae69a9ca5a527426080 +P ab1e58fac9e3f9d7d90cefb39f8f145d211d38ec0cb393208c4531cc646056c1 +R f78f1915a996819be59599569cf251be U drh -Z 4be57a3eceec619de5e7431288dd440b +Z 8250db6890dbee78d842dff888889f8d diff --git a/manifest.uuid b/manifest.uuid index 9ce083fd00..d3b869dc52 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab1e58fac9e3f9d7d90cefb39f8f145d211d38ec0cb393208c4531cc646056c1 \ No newline at end of file +9748d7995bc1dd632d66d2d326048d50e29c6f5e6993d5a6294d14421d2cb72f \ No newline at end of file diff --git a/test/shell1.test b/test/shell1.test index aaf7addf5c..f105bac5b7 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -256,7 +256,7 @@ do_test shell1-3.1.3 { do_test shell1-3.1.4 { # too many arguments catchcmd "test.db" ".backup FOO BAR BAD" -} {1 {Usage: .backup ?DB? ?--append? FILENAME}} +} {1 {Usage: .backup ?DB? ?OPTIONS? FILENAME}} # .bail ON|OFF Stop after hitting an error. Default OFF do_test shell1-3.2.1 { From dc6214ee3910093d930962ba026e1f8329b6ef98 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 10 Dec 2018 08:41:32 +0000 Subject: [PATCH 41/52] Fix a problem with using "-vacuum" (the default) as the state database when resuming an RBU vacuum. FossilOrigin-Name: c878d741733a897d47a1a0b57b454bc9095ce302b48cc01089336d5d20e2a24c --- ext/rbu/rbu_common.tcl | 13 +++++++------ ext/rbu/rbuvacuum2.test | 27 ++++++++++++++++++--------- ext/rbu/sqlite3rbu.c | 8 +++++++- ext/rbu/sqlite3rbu.h | 6 +++++- ext/rbu/test_rbu.c | 1 + manifest | 23 +++++++++++------------ manifest.uuid | 2 +- 7 files changed, 50 insertions(+), 30 deletions(-) diff --git a/ext/rbu/rbu_common.tcl b/ext/rbu/rbu_common.tcl index 2b263b7660..b5e63aafe5 100644 --- a/ext/rbu/rbu_common.tcl +++ b/ext/rbu/rbu_common.tcl @@ -86,12 +86,13 @@ proc step_rbu_legacy {target rbu} { set rc } -proc do_rbu_vacuum_test {tn step} { - forcedelete state.db - uplevel [list do_test $tn.1 { - if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db } +proc do_rbu_vacuum_test {tn step {statedb state.db}} { + forcedelete $statedb + if {$statedb=="" && $step==1} breakpoint + uplevel [list do_test $tn.1 [string map [list %state% $statedb] { + if {$step==0} { sqlite3rbu_vacuum rbu test.db {%state%}} while 1 { - if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db } + if {$step==1} { sqlite3rbu_vacuum rbu test.db {%state%}} set state [rbu state] check_prestep_state test.db $state set rc [rbu step] @@ -100,7 +101,7 @@ proc do_rbu_vacuum_test {tn step} { if {$step==1} { rbu close } } rbu close - } {SQLITE_DONE}] + }] {SQLITE_DONE}] uplevel [list do_execsql_test $tn.2 { PRAGMA integrity_check diff --git a/ext/rbu/rbuvacuum2.test b/ext/rbu/rbuvacuum2.test index 4713580c70..6d6dfde9b9 100644 --- a/ext/rbu/rbuvacuum2.test +++ b/ext/rbu/rbuvacuum2.test @@ -16,8 +16,11 @@ source [file join [file dirname [info script]] rbu_common.tcl] -foreach step {0 1} { - set ::testprefix rbuvacuum2-$step +foreach {step} {0 1} { +foreach {ttt state} { + s state.db t test.db-vacuum n {} +} { + set ::testprefix rbuvacuum2-$step$ttt #------------------------------------------------------------------------- # Test that a database that contains fts3 tables can be vacuumed. @@ -29,7 +32,7 @@ foreach step {0 1} { INSERT INTO t1 VALUES('fix this issue', 'at some point'); } - do_rbu_vacuum_test 1.2 $step + do_rbu_vacuum_test 1.2 $step $state do_execsql_test 1.3 { SELECT * FROM t1; @@ -46,7 +49,7 @@ foreach step {0 1} { INSERT INTO t1 VALUES('a b c', 'x y z'); } - do_rbu_vacuum_test 1.6 $step + do_rbu_vacuum_test 1.6 $step $state do_execsql_test 1.7 { INSERT INTO t1(t1) VALUES('integrity-check'); SELECT * FROM t1; @@ -67,7 +70,7 @@ foreach step {0 1} { INSERT INTO t1 VALUES('fix this issue', 'at some point'); } - do_rbu_vacuum_test 2.2 $step + do_rbu_vacuum_test 2.2 $step $state do_execsql_test 2.3 { SELECT * FROM t1; @@ -84,7 +87,7 @@ foreach step {0 1} { INSERT INTO t1 VALUES('a b c', 'x y z'); } - do_rbu_vacuum_test 2.6 $step + do_rbu_vacuum_test 2.6 $step $state do_execsql_test 2.7 { INSERT INTO t1(t1) VALUES('integrity-check'); SELECT * FROM t1; @@ -107,7 +110,7 @@ foreach step {0 1} { INSERT INTO rt VALUES(3, 55, 65); } - do_rbu_vacuum_test 3.2 $step + do_rbu_vacuum_test 3.2 $step $state do_execsql_test 3.3 { SELECT * FROM rt; @@ -120,7 +123,7 @@ foreach step {0 1} { SELECT rowid FROM rt WHERE x2>59 AND x1 < 59 } {2 3} - do_rbu_vacuum_test 3.5 $step + do_rbu_vacuum_test 3.5 $step $state do_execsql_test 3.6.1 { SELECT rowid FROM rt WHERE x2>51 AND x1 < 51 @@ -147,7 +150,7 @@ foreach step {0 1} { trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END} } - do_rbu_vacuum_test 4.3 $step + do_rbu_vacuum_test 4.3 $step $state do_execsql_test 4.4 { SELECT * FROM sqlite_master; } { @@ -157,6 +160,7 @@ foreach step {0 1} { } } } +} #------------------------------------------------------------------------- # Test that passing a NULL value as the second argument to @@ -231,4 +235,9 @@ do_test 6.3 { execsql { PRAGMA integrity_check } } {ok} +do_test 6.4 { + sqlite3rbu_vacuum rbu test.db test.db-vactmp + list [catch { rbu close } msg] $msg +} {1 SQLITE_MISUSE} + finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index cd2f96c51b..7683c3464b 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2477,7 +2477,7 @@ static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){ if( *zExtra=='\0' ) zExtra = 0; } - zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", + zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", sqlite3_db_filename(p->dbRbu, "main"), (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) ); @@ -3743,6 +3743,12 @@ sqlite3rbu *sqlite3rbu_vacuum( const char *zState ){ if( zTarget==0 ){ return rbuMisuseError(); } + if( zState ){ + int n = strlen(zState); + if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){ + return rbuMisuseError(); + } + } /* TODO: Check that both arguments are non-NULL */ return openRbuHandle(0, zTarget, zState); } diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index 1acbcca469..69d89500a0 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -333,7 +333,11 @@ SQLITE_API sqlite3rbu *sqlite3rbu_open( ** name of the state database is "-vacuum", where ** is the name of the target database file. In this case, on UNIX, if the ** state database is not already present in the file-system, it is created -** with the same permissions as the target db is made. +** with the same permissions as the target db is made. +** +** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the +** state database ends with "-vactmp". This name is reserved for internal +** use. ** ** This function does not delete the state database after an RBU vacuum ** is completed, even if it created it. However, if the call to diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index e0b4d77af0..6d04bfe8cc 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -273,6 +273,7 @@ static int SQLITE_TCLAPI test_sqlite3rbu_vacuum( zCmd = Tcl_GetString(objv[1]); zTarget = Tcl_GetString(objv[2]); if( objc==4 ) zStateDb = Tcl_GetString(objv[3]); + if( zStateDb && zStateDb[0]=='\0' ) zStateDb = 0; pRbu = sqlite3rbu_vacuum(zTarget, zStateDb); Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0); diff --git a/manifest b/manifest index 544f8a6898..a24f9cc5e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sthe\sVACUUM\sINTO\scommand. -D 2018-12-10T01:48:29.276 +C Fix\sa\sproblem\swith\susing\s"-vacuum"\s(the\sdefault)\sas\sthe\sstate\sdatabase\nwhen\sresuming\san\sRBU\svacuum. +D 2018-12-10T08:41:32.074 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -330,7 +330,7 @@ F ext/rbu/rbu9.test 0e4d985e25620d61920597e8ea69c871c9e8c1f5a0be2ae9fa70bb641d74 F ext/rbu/rbuA.test b34a90cb495682c25b5fc03a9d5e7a4fc99541c29256f25e2e2a4f6542b4f5b3 F ext/rbu/rbuB.test 52b07158824c6927b7e25554ace92a695cdebfc296ae3d308ac386984aded9bc F ext/rbu/rbuC.test 80f1cc2fb74f44b1128fd0ed8eedab3a76fefeb72a947860e2869ef76fc8dc6b -F ext/rbu/rbu_common.tcl acfb7fbbaf8d46a9f6f6a5ec795616c84d705e1565d918afe43f0ff53ea0efa5 +F ext/rbu/rbu_common.tcl 4b3d033b3e3844292ae3a1aefc0e524e64b0db5a0e4310657919e4504ac3073f F ext/rbu/rbucollate.test cac528a9a46318cba42e61258bb42660bbbf4fdb9a8c863de5a54ad0c658d197 F ext/rbu/rbucrash.test 000981a1fe8a6e4d9a684232f6a129e66a3ef595f5ed74655e2f9c68ffa613b4 F ext/rbu/rbucrash2.test efa143cc94228eb0266d3f1abfbee60a5838a84cef7cc3fcb8c145b74d96fd41 @@ -348,10 +348,10 @@ F ext/rbu/rbusave.test f4190a1a86fccf84f723af5c93813365ae33feda35845ba107b59683d F ext/rbu/rbusplit.test b37e7b40b38760881dc9c854bd40b4744c6b6cd74990754eca3bda0f407051e8 F ext/rbu/rbutemplimit.test 7f408f49b90fa0a720d7599f3aec74a3c85e6cd78e56fdf726ce00af9147a341 F ext/rbu/rbuvacuum.test 55e101e90168c2b31df6c9638fe73dc7f7cc666b6142266d1563697d79f73534 -F ext/rbu/rbuvacuum2.test 0a7669bbabdaeed915f02f59f33fe20e13d4932ba2086fe00a82064d9424c80b -F ext/rbu/sqlite3rbu.c 71f8c09948d09ec9c5a8dbe7127e8ef61ef0853e698b2650be2485ac7b9c75c8 -F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2 -F ext/rbu/test_rbu.c baa23eb28457580673d2175e5f0c29ced0cd320ee819b13ad362398c53b96e90 +F ext/rbu/rbuvacuum2.test b8e5b51dc8b2c0153373d024c0936be3f66f9234acbd6d0baab0869d56b14e6b +F ext/rbu/sqlite3rbu.c f722ed4177c9fb73f2f6f116240687ac7603735fa95ea63bff71827929d4c192 +F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 +F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd F ext/repair/checkindex.c 7d28c01a2e012ac64257d230fc452b2cafb78311a91a343633d01d95220f66f3 @@ -1783,8 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b7bf3c9832bde26b359f4d58ebcf7788c500586d311387d022192ed65a126252 9748d7995bc1dd632d66d2d326048d50e29c6f5e6993d5a6294d14421d2cb72f -R f78f1915a996819be59599569cf251be -T +closed 9748d7995bc1dd632d66d2d326048d50e29c6f5e6993d5a6294d14421d2cb72f -U drh -Z ee3d9a1eeb178b5acf328ec54a590f24 +P 77f150b8b46761f4f62f9d8926c10a95a70589a4525393fc16b321bd98c083a7 +R 958323adeafdf09cbb584115dfab1b22 +U dan +Z 3e8a12745bc34fa21f3c5c886b9f537f diff --git a/manifest.uuid b/manifest.uuid index fb0635c7f2..afb7891c77 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77f150b8b46761f4f62f9d8926c10a95a70589a4525393fc16b321bd98c083a7 \ No newline at end of file +c878d741733a897d47a1a0b57b454bc9095ce302b48cc01089336d5d20e2a24c \ No newline at end of file From 643d855da9625dd18f784674792dadd9697bb3ad Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Dec 2018 16:00:57 +0000 Subject: [PATCH 42/52] Refactor the sqlite3_normalized_sql() implementation. This is a work-in-progress. There are still issues. FossilOrigin-Name: a4c890b0af9786295e6df05022009d8946550adb873535c610be805c2b7a4083 --- manifest | 24 +++--- manifest.uuid | 2 +- src/prepare.c | 198 ------------------------------------------------ src/sqliteInt.h | 3 - src/tokenize.c | 197 +++++++++++++++++++++++++++++++---------------- src/vdbe.h | 2 +- src/vdbeaux.c | 16 +--- 7 files changed, 148 insertions(+), 294 deletions(-) diff --git a/manifest b/manifest index 544f8a6898..c0a5842691 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sthe\sVACUUM\sINTO\scommand. -D 2018-12-10T01:48:29.276 +C Refactor\sthe\ssqlite3_normalized_sql()\simplementation.\sThis\sis\sa\nwork-in-progress.\sThere\sare\sstill\sissues. +D 2018-12-10T16:00:57.538 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -502,7 +502,7 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 96ce7dce4dc9cb2b7aa0e1b2ce7536870bdc00b10becc278245e775489447ea0 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c be449edb106a16f1ad95f9b798bdc2337f8c3f83b96c284f417c0a26d43f0c1b +F src/prepare.c 0e8fc0deaf36da104e08d07ce7d97bc09ab57d078b399381532fec3fa1d3f2bb F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 095d1d41d7a981ee9db8bfeae25ed0d6a8a5e5e3d66b0f4efd71877ed7b56132 @@ -512,7 +512,7 @@ F src/shell.c.in 5f38bd0e127c2cc4e506b5c3565c10879ddfae6c2d867bb5972563e40717c19 F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h 70ce5e14c887554d3c51f2045f5a95b6e83de745d7f6448e79e49fdd8dfc2d5c +F src/sqliteInt.h 369d4774d97643e26085e0ea4cdee6afeadaa27670193b9eebaea8c95687fad0 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -570,7 +570,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 9e781e1ca80eefe7b5d6a9e2cd5c678c847da55fd6f093781fad7950934d4c83 +F src/tokenize.c a43ecbafefceb475522f17d6f92d64193a755fe3f35615dfca13340c81ca3872 F src/treeview.c 7b12ac059de54c939b6eb0dbffc9410c29c80d2470cee5cbe07d5ff9ea2d9253 F src/trigger.c d3d78568f37fb2e6cdcc2d1e7b60156f15b0b600adec55b83c5d42f6cad250bd F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274 @@ -579,10 +579,10 @@ F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 F src/util.c d9eb0a6c4aae1b00a7369eadd7ca0bbe946cb4c953b6751aa20d357c2f482157 F src/vacuum.c 3ffe64ecfc94b7528c5d7bdb1c3a19d72fec63f2aa846e3b90f8de5dbbddf5aa F src/vdbe.c 55bafc424748d9ed505ab2680736e51d1bb05c01e9885cbb3b287b51dc8b47ec -F src/vdbe.h d82f323d581b36b8e147d650257ef34e0e93790039b6cbda45c321c275f7595e +F src/vdbe.h 8990d668a89890a33326b0a29b992c4014b72f3b6cdcd9ee0e190593c247f9b0 F src/vdbeInt.h 73f5051923f3f29779bfc374c0c68e23b8e5e3792def2e33e51b427edb890abd F src/vdbeapi.c 9709452bee82963e1f7f1f5d0c71db823d553f8dbb2c47a911c4983d537a1947 -F src/vdbeaux.c 9a9617666124e18cbd6e936740f7469dcf0d82867b1abf9ed828694500930b64 +F src/vdbeaux.c c72fc6015e52c212d6c6db7cee04bfbfbd681106f551c9296812a73082fb09d6 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f @@ -1783,8 +1783,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b7bf3c9832bde26b359f4d58ebcf7788c500586d311387d022192ed65a126252 9748d7995bc1dd632d66d2d326048d50e29c6f5e6993d5a6294d14421d2cb72f -R f78f1915a996819be59599569cf251be -T +closed 9748d7995bc1dd632d66d2d326048d50e29c6f5e6993d5a6294d14421d2cb72f +P 77f150b8b46761f4f62f9d8926c10a95a70589a4525393fc16b321bd98c083a7 +R d755f31eec9cdb1a3415665c2aacb5ab +T *branch * normalize-refactor +T *sym-normalize-refactor * +T -sym-trunk * U drh -Z ee3d9a1eeb178b5acf328ec54a590f24 +Z f90c6b85a4f2311f0e8b3d6a605ae025 diff --git a/manifest.uuid b/manifest.uuid index fb0635c7f2..5d73daf1c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77f150b8b46761f4f62f9d8926c10a95a70589a4525393fc16b321bd98c083a7 \ No newline at end of file +a4c890b0af9786295e6df05022009d8946550adb873535c610be805c2b7a4083 \ No newline at end of file diff --git a/src/prepare.c b/src/prepare.c index 811ef95aea..e06b7cb6ad 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -709,204 +709,6 @@ static int sqlite3LockAndPrepare( return rc; } -#ifdef SQLITE_ENABLE_NORMALIZE - -/* -** Attempt to estimate the final output buffer size needed for the fully -** normalized version of the specified SQL string. This should take into -** account any potential expansion that could occur (e.g. via IN clauses -** being expanded, etc). This size returned is the total number of bytes -** including the NUL terminator. -*/ -static int estimateNormalizedSize( - const char *zSql, /* The original SQL string */ - int nSql /* Length of original SQL string */ -){ - int nOut = nSql + 4; - const char *z = zSql; - while( nOut0 ){ - zOut[j++] = '"'; - continue; - }else if( k==nToken-1 ){ - zOut[j++] = '"'; - continue; - } - } - if( bKeyword ){ - zOut[j++] = sqlite3Toupper(zSql[iIn+k]); - }else{ - zOut[j++] = sqlite3Tolower(zSql[iIn+k]); - } - } - *piOut = j; -} - -/* -** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return -** the normalization in space obtained from sqlite3DbMalloc(). Or return -** NULL if anything goes wrong or if zSql is NULL. -*/ -char *sqlite3Normalize( - Vdbe *pVdbe, /* VM being reprepared */ - const char *zSql, /* The original SQL string */ - int nSql /* Size of the input string in bytes */ -){ - sqlite3 *db; /* Database handle. */ - char *z; /* The output string */ - int nZ; /* Size of the output string in bytes */ - int i; /* Next character to read from zSql[] */ - int j; /* Next character to fill in on z[] */ - int tokenType = 0; /* Type of the next token */ - int prevTokenType = 0; /* Type of the previous token, except spaces */ - int n; /* Size of the next token */ - int nParen = 0; /* Nesting level of parenthesis */ - int iStartIN = 0; /* Start of RHS of IN operator in z[] */ - int nParenAtIN = 0; /* Value of nParent at start of RHS of IN operator */ - - db = sqlite3VdbeDb(pVdbe); - assert( db!=0 ); - if( zSql==0 ) return 0; - nZ = estimateNormalizedSize(zSql, nSql); - z = sqlite3DbMallocRawNN(db, nZ); - if( z==0 ) goto normalizeError; - for(i=j=0; i0 && nParen==nParenAtIN ){ - assert( iStartIN+6=0 ); - assert( nZ-1-j=0 ); - /* Fall through */ - } - case TK_MINUS: - case TK_SEMI: - case TK_PLUS: - case TK_STAR: - case TK_SLASH: - case TK_REM: - case TK_EQ: - case TK_LE: - case TK_NE: - case TK_LSHIFT: - case TK_LT: - case TK_RSHIFT: - case TK_GT: - case TK_GE: - case TK_BITOR: - case TK_CONCAT: - case TK_COMMA: - case TK_BITAND: - case TK_BITNOT: - case TK_DOT: - case TK_IN: - case TK_IS: - case TK_NOT: - case TK_NULL: - case TK_ID: { - if( tokenType==TK_NULL ){ - if( prevTokenType==TK_IS || prevTokenType==TK_NOT ){ - /* NULL is a keyword in this case, not a literal value */ - }else{ - /* Here the NULL is a literal value */ - z[j++] = '?'; - break; - } - } - if( j>0 && sqlite3IsIdChar(z[j-1]) && sqlite3IsIdChar(zSql[i]) ){ - z[j++] = ' '; - } - if( tokenType==TK_ID ){ - if( zSql[i]=='"' - && sqlite3VdbeUsesDoubleQuotedString(db,pVdbe,zSql+i,n) - ){ - z[j++] = '?'; - break; - } - if( nParen==nParenAtIN ) iStartIN = 0; - } - copyNormalizedToken(zSql, i, n, flags, z, &j); - break; - } - } - } - assert( j0 && z[j-1]==' ' ){ j--; } - if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; } - z[j] = 0; - assert( jrc!=SQLITE_OK ); return nErr; } + + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Insert a single space character into pStr if the current string +** ends with an identifier +*/ +static void addSpaceSeparator(sqlite3_str *pStr){ + if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){ + sqlite3_str_append(pStr, " ", 1); + } +} + +/* +** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return +** the normalization in space obtained from sqlite3DbMalloc(). Or return +** NULL if anything goes wrong or if zSql is NULL. +*/ +char *sqlite3Normalize( + Vdbe *pVdbe, /* VM being reprepared */ + const char *zSql, /* The original SQL string */ + int nSql /* Size of the input string in bytes */ +){ + sqlite3 *db; /* The database connection */ + int i; /* Next unread byte of zSql[] */ + int n; /* length of current token */ + int tokenType; /* type of current token */ + int prevType; /* Previous non-whitespace token */ + int nParen; /* Number of nested levels of parentheses */ + int iStartIN; /* Start of RHS of IN operator in z[] */ + int nParenAtIN; /* Value of nParent at start of RHS of IN operator */ + int j; /* Bytes of normalized SQL generated so far */ + sqlite3_str *pStr; /* The normalized SQL string under construction */ + + if( zSql==0 || nSql==0 ) return 0; + db = sqlite3VdbeDb(pVdbe); + tokenType = -1; + nParen = iStartIN = nParenAtIN = 0; + pStr = sqlite3_str_new(db); + for(i=0; iaccError==0; i+=n){ + if( tokenType!=TK_SPACE ){ + prevType = tokenType; + } + n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); + if( NEVER(n<=0) ) break; + switch( tokenType ){ + case TK_SPACE: { + break; + } + case TK_NULL: { + if( prevType==TK_IS || prevType==TK_NOT ){ + sqlite3_str_append(pStr, " NULL", 5); + break; + } + /* Fall through */ + } + case TK_STRING: + case TK_INTEGER: + case TK_FLOAT: + case TK_VARIABLE: + case TK_BLOB: { + sqlite3_str_append(pStr, "?", 1); + break; + } + case TK_LP: { + nParen++; + if( prevType==TK_IN ){ + iStartIN = pStr->nChar; + nParenAtIN = nParen; + } + sqlite3_str_append(pStr, "(", 1); + break; + } + case TK_RP: { + if( iStartIN>0 && nParen==nParenAtIN ){ + assert( pStr->nChar>=iStartIN ); + pStr->nChar = iStartIN+1; + sqlite3_str_append(pStr, "?,?,?", 5); + iStartIN = 0; + } + nParen--; + sqlite3_str_append(pStr, ")", 1); + break; + } + case TK_ID: { + j = pStr->nChar; + if( sqlite3Isquote(zSql[i]) ){ + char *zId = sqlite3DbStrNDup(db, zSql+i, n); + int nId; + int eType = 0; + if( zId==0 ) break; + sqlite3Dequote(zId); + if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){ + sqlite3_str_append(pStr, "?", 1); + sqlite3DbFree(db, zId); + break; + } + nId = sqlite3Strlen30(zId); + if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zId, nId); + }else{ + sqlite3_str_appendf(pStr, "\"%w\"", zId); + } + sqlite3DbFree(db, zId); + }else{ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zSql+i, n); + } + while( jnChar ){ + pStr->zText[j] = sqlite3Tolower(pStr->zText[j]); + j++; + } + break; + } + default: { + if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); + j = pStr->nChar; + sqlite3_str_append(pStr, zSql+i, n); + while( jnChar ){ + pStr->zText[j] = sqlite3Toupper(pStr->zText[j]); + j++; + } + break; + } + } + } + return sqlite3_str_finish(pStr); +} +#endif /* SQLITE_ENABLE_NORMALIZE */ diff --git a/src/vdbe.h b/src/vdbe.h index 1712b8b224..f4d360e49e 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -253,7 +253,7 @@ u8 sqlite3VdbePrepareFlags(Vdbe*); void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8); #ifdef SQLITE_ENABLE_NORMALIZE void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*); -int sqlite3VdbeUsesDoubleQuotedString(sqlite3*,Vdbe*,const char*,int); +int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*); #endif void sqlite3VdbeSwap(Vdbe*,Vdbe*); VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 2f45c217f1..ac7bf6a65b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -97,26 +97,16 @@ void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){ ** that identifier is really used as a string literal. */ int sqlite3VdbeUsesDoubleQuotedString( - sqlite3 *db, /* Used for transient malloc */ Vdbe *pVdbe, /* The prepared statement */ - const char *zId, /* The double-quoted identifier */ - int nId /* Bytes in zId, which is not zero-terminated */ + const char *zId /* The double-quoted identifier, already dequoted */ ){ - char *z; DblquoteStr *pStr; assert( zId!=0 ); - assert( zId[0]=='"' ); - assert( nId>=2 ); - assert( zId[nId-1]=='"' ); if( pVdbe->pDblStr==0 ) return 0; - z = sqlite3DbStrNDup(db, zId, nId); - if( z==0 ) return 0; - sqlite3Dequote(z); for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){ - if( strcmp(z, pStr->z)==0 ) break; + if( strcmp(zId, pStr->z)==0 ) return 1; } - sqlite3DbFree(db, z); - return pStr!=0; + return 0; } #endif From 9042ff214b1b7f42d77cff6d8c1d1c22aa0b5428 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Dec 2018 16:49:33 +0000 Subject: [PATCH 43/52] Fix issues with the new normalizer. FossilOrigin-Name: 057d7d40c56b7416a59a79cb627b2b3e5837eca2d6dbcb50127baaadf1941f51 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/tokenize.c | 6 ++++++ test/normalize.test | 16 ++++++++-------- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index c0a5842691..56e2ba7b0a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\ssqlite3_normalized_sql()\simplementation.\sThis\sis\sa\nwork-in-progress.\sThere\sare\sstill\sissues. -D 2018-12-10T16:00:57.538 +C Fix\sissues\swith\sthe\snew\snormalizer. +D 2018-12-10T16:49:33.233 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -570,7 +570,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c a43ecbafefceb475522f17d6f92d64193a755fe3f35615dfca13340c81ca3872 +F src/tokenize.c 8ae201c2e6955325e3d16e7bba4a74fb857d09c1e89274857e7d4b1cdb1f9af4 F src/treeview.c 7b12ac059de54c939b6eb0dbffc9410c29c80d2470cee5cbe07d5ff9ea2d9253 F src/trigger.c d3d78568f37fb2e6cdcc2d1e7b60156f15b0b600adec55b83c5d42f6cad250bd F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274 @@ -1142,7 +1142,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test 437d40e6d0778b050d7750726c0cbd2c9936b81962926e8f8c48ca698f00f4d1 F test/nockpt.test 8c43b25af63b0bd620cf1b003529e37b6f1dc53bd22690e96a1bd73f78dde53a F test/nolock.test f196cf8b8fbea4e2ca345140a2b3f3b0da45c76e -F test/normalize.test 17ff5c732bc16b100f4e479da3a59735ed24f4ed574fccf8e46279fad182f81e +F test/normalize.test 422027884ffb67ebba32bb78487c67cf67643496d19c077b07044bdba071a3f6 F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 @@ -1783,10 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 77f150b8b46761f4f62f9d8926c10a95a70589a4525393fc16b321bd98c083a7 -R d755f31eec9cdb1a3415665c2aacb5ab -T *branch * normalize-refactor -T *sym-normalize-refactor * -T -sym-trunk * +P a4c890b0af9786295e6df05022009d8946550adb873535c610be805c2b7a4083 +R c326114a802cb731930ed1e3cc6ed39d U drh -Z f90c6b85a4f2311f0e8b3d6a605ae025 +Z 6f97af92ba72d549a5865cb69df69952 diff --git a/manifest.uuid b/manifest.uuid index 5d73daf1c8..56b266680d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4c890b0af9786295e6df05022009d8946550adb873535c610be805c2b7a4083 \ No newline at end of file +057d7d40c56b7416a59a79cb627b2b3e5837eca2d6dbcb50127baaadf1941f51 \ No newline at end of file diff --git a/src/tokenize.c b/src/tokenize.c index 61a24caf29..499fae6f19 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -799,6 +799,7 @@ char *sqlite3Normalize( break; } case TK_ID: { + iStartIN = 0; j = pStr->nChar; if( sqlite3Isquote(zSql[i]) ){ char *zId = sqlite3DbStrNDup(db, zSql+i, n); @@ -829,6 +830,10 @@ char *sqlite3Normalize( } break; } + case TK_SELECT: { + iStartIN = 0; + /* fall through */ + } default: { if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); j = pStr->nChar; @@ -841,6 +846,7 @@ char *sqlite3Normalize( } } } + if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1); return sqlite3_str_finish(pStr); } #endif /* SQLITE_ENABLE_NORMALIZE */ diff --git a/test/normalize.test b/test/normalize.test index 35c995fe75..8b0c7ed8db 100644 --- a/test/normalize.test +++ b/test/normalize.test @@ -207,7 +207,7 @@ foreach {tnum sql flags norm} { 430 {SELECT "a" FROM t1 WHERE "x" IN ("1","2",'3');} 0x2 - {0 {SELECT"a"FROM t1 WHERE"x"IN(?,?,?);}} + {0 {SELECT a FROM t1 WHERE x IN(?,?,?);}} 440 {SELECT 'a' FROM t1 WHERE 'x';} @@ -217,7 +217,7 @@ foreach {tnum sql flags norm} { 450 {SELECT [a] FROM t1 WHERE [x];} 0x2 - {0 {SELECT"a"FROM t1 WHERE"x";}} + {0 {SELECT a FROM t1 WHERE x;}} 460 {SELECT * FROM t1 WHERE x IN (x);} @@ -232,12 +232,12 @@ foreach {tnum sql flags norm} { 480 {SELECT * FROM t1 WHERE x IN ([x],"a");} 0x2 - {0 {SELECT*FROM t1 WHERE x IN("x","a");}} + {0 {SELECT*FROM t1 WHERE x IN(x,a);}} 500 {SELECT * FROM t1 WHERE x IN ([x],"a",'b',sqlite_version());} 0x2 - {0 {SELECT*FROM t1 WHERE x IN("x","a",?,sqlite_version());}} + {0 {SELECT*FROM t1 WHERE x IN(x,a,?,sqlite_version());}} 520 {SELECT * FROM t1 WHERE x IN (SELECT x FROM t1);} @@ -247,12 +247,12 @@ foreach {tnum sql flags norm} { 540 {SELECT * FROM t1 WHERE x IN ((SELECT x FROM t1));} 0x2 - {0 {SELECT*FROM t1 WHERE x IN(?,?,?);}} + {0 {SELECT*FROM t1 WHERE x IN((SELECT x FROM t1));}} 550 {SELECT a, a+1, a||'b', a+"b" FROM t1;} 0x2 - {0 {SELECT a,a+?,a||?,a+"b"FROM t1;}} + {0 {SELECT a,a+?,a||?,a+b FROM t1;}} 570 {SELECT * FROM t1 WHERE x IN (1);} @@ -316,7 +316,7 @@ foreach {tnum sql flags norm} { 680 {SELECT a, "col f" FROM t1 LEFT OUTER JOIN t2 ON [t1].[col f] == [t2].[col y];} 0x2 - {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON"t1"."col f"=="t2"."col y";}} + {0 {SELECT a,"col f"FROM t1 LEFT OUTER JOIN t2 ON t1."col f"==t2."col y";}} 690 {SELECT * FROM ( WITH x AS ( SELECT * FROM t1 WHERE x IN ( 1)) SELECT 10);} @@ -346,7 +346,7 @@ foreach {tnum sql flags norm} { 760 {SELECT x FROM t1 WHERE x IN ([x] IS NOT NULL, NULL, 1, 'a', "b", x'00');} 0x2 - {0 {SELECT x FROM t1 WHERE x IN("x"IS NOT NULL,?,?,?,"b",?);}} + {0 {SELECT x FROM t1 WHERE x IN(x IS NOT NULL,?,?,?,b,?);}} } { do_test $tnum { set code [catch { From 1a6c2b1d383009aeb306f53f1915a443ab79de0f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Dec 2018 20:01:40 +0000 Subject: [PATCH 44/52] Further refinements to the sqlite3_normalized_sql() interface. TH3 now gives 100% MC/DC on that interface. FossilOrigin-Name: c96bf6cca220e363b099455ce35195ce7e89d374a52dc787f56e7b11e587bced --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/resolve.c | 2 +- src/sqlite.h.in | 15 +++++++-------- src/sqliteInt.h | 2 +- src/tokenize.c | 7 +++---- src/vdbeapi.c | 4 ++-- src/vdbeaux.c | 7 ------- 8 files changed, 26 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index 52a10db3c6..79aa8f42ae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\ssqlite3_normalized_sql()\simplementation. -D 2018-12-10T18:15:44.685 +C Further\srefinements\sto\sthe\ssqlite3_normalized_sql()\sinterface.\s\sTH3\snow\ngives\s100%\sMC/DC\son\sthat\sinterface. +D 2018-12-10T20:01:40.741 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -505,14 +505,14 @@ F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 F src/prepare.c 0e8fc0deaf36da104e08d07ce7d97bc09ab57d078b399381532fec3fa1d3f2bb F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 095d1d41d7a981ee9db8bfeae25ed0d6a8a5e5e3d66b0f4efd71877ed7b56132 +F src/resolve.c abd65c518c198400193c6319a70c0d722fa30a35be89dc898917ff6489edf017 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 8c7317d5ee920516a56b8b4ca79fbfca70a1f8b52d67e884c808ea3a016c04e3 F src/shell.c.in 5f38bd0e127c2cc4e506b5c3565c10879ddfae6c2d867bb5972563e40717c19c -F src/sqlite.h.in 908ec406feefc4c7e1486a2e3dc30a8bfb51c5a345a8e8130ac201962db171c4 +F src/sqlite.h.in 92fd656c26cc76de9fa8c5bf1a473066e3b5c6da345a447679f0f44de1aa4edd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h 369d4774d97643e26085e0ea4cdee6afeadaa27670193b9eebaea8c95687fad0 +F src/sqliteInt.h 3dda7ba0ea00f591c18405e5061d10041e0fcd5934e2542f29f8c8cffd73c242 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -570,7 +570,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9 F src/test_window.c cdae419fdcea5bad6dcd9368c685abdad6deb59e9fc8b84b153de513d394ba3f F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c -F src/tokenize.c 8ae201c2e6955325e3d16e7bba4a74fb857d09c1e89274857e7d4b1cdb1f9af4 +F src/tokenize.c c8af4feebd8bf5a4d60a14018d91f61013f658ec864dfce7661bae73d86b3191 F src/treeview.c 7b12ac059de54c939b6eb0dbffc9410c29c80d2470cee5cbe07d5ff9ea2d9253 F src/trigger.c d3d78568f37fb2e6cdcc2d1e7b60156f15b0b600adec55b83c5d42f6cad250bd F src/update.c 1816d56c1bca1ba4e0ef98cac2f49be62858e9df1dc08844c7067eb41cc44274 @@ -581,8 +581,8 @@ F src/vacuum.c 3ffe64ecfc94b7528c5d7bdb1c3a19d72fec63f2aa846e3b90f8de5dbbddf5aa F src/vdbe.c 55bafc424748d9ed505ab2680736e51d1bb05c01e9885cbb3b287b51dc8b47ec F src/vdbe.h 8990d668a89890a33326b0a29b992c4014b72f3b6cdcd9ee0e190593c247f9b0 F src/vdbeInt.h 73f5051923f3f29779bfc374c0c68e23b8e5e3792def2e33e51b427edb890abd -F src/vdbeapi.c 9709452bee82963e1f7f1f5d0c71db823d553f8dbb2c47a911c4983d537a1947 -F src/vdbeaux.c c72fc6015e52c212d6c6db7cee04bfbfbd681106f551c9296812a73082fb09d6 +F src/vdbeapi.c 43ccaa96e28ebe56a46616c300218e5073911eff618e9d0ab7526983cda939b1 +F src/vdbeaux.c f00d9b32a250b829a3c00140255a1c37a6463d726bb87ed6bbb80a1ce76a56bd F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f @@ -1783,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c878d741733a897d47a1a0b57b454bc9095ce302b48cc01089336d5d20e2a24c 057d7d40c56b7416a59a79cb627b2b3e5837eca2d6dbcb50127baaadf1941f51 -R ac6b36f113296d9388b94571dafb8554 +P 06e955e5d2b857b9ce94a73fb174b620c101bdb160842f9a0e958bdb48e8c987 +R 692e5dc4224a0726be01a748426768ab U drh -Z db9c09c70c1ad5ac23765ae8ea064291 +Z 6406ae7cdd7eefde2a741c866c8156da diff --git a/manifest.uuid b/manifest.uuid index c11c3dbd1c..963e5257c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06e955e5d2b857b9ce94a73fb174b620c101bdb160842f9a0e958bdb48e8c987 \ No newline at end of file +c96bf6cca220e363b099455ce35195ce7e89d374a52dc787f56e7b11e587bced \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index a9262a45ad..6dc42aef7a 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -491,7 +491,7 @@ static int lookupName( sqlite3_log(SQLITE_WARNING, "double-quoted string literal: \"%w\"", zCol); #ifdef SQLITE_ENABLE_NORMALIZE - sqlite3VdbeAddDblquoteStr(db,pParse->pVdbe, zCol); + sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol); #endif pExpr->op = TK_STRING; pExpr->y.pTab = 0; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 4f33bb9ca9..08d499037a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3629,14 +3629,13 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** deplete the limited store of lookaside memory. Future versions of ** SQLite may act on this hint differently. ** -** [[SQLITE_PREPARE_NORMALIZE]] ^(
SQLITE_PREPARE_NORMALIZE
-**
The SQLITE_PREPARE_NORMALIZE flag indicates that a normalized -** representation of the SQL statement should be calculated and then -** associated with the prepared statement, which can be obtained via -** the [sqlite3_normalized_sql()] interface.)^ The semantics used to -** normalize a SQL statement are unspecified and subject to change. -** At a minimum, literal values will be replaced with suitable -** placeholders. +** [[SQLITE_PREPARE_NORMALIZE]]
SQLITE_PREPARE_NORMALIZE
+**
The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used +** to be required for any prepared statement that wanted to use the +** [sqlite3_normalized_sql()] interface. However, the +** [sqlite3_normalized_sql()] interface is now available to all +** prepared statements, regardless of whether or not they use this +** flag. ** */ #define SQLITE_PREPARE_PERSISTENT 0x01 diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b8f771488d..7dd2584828 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4413,7 +4413,7 @@ int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); void sqlite3ParserReset(Parse*); #ifdef SQLITE_ENABLE_NORMALIZE -char *sqlite3Normalize(Vdbe*, const char*, int); +char *sqlite3Normalize(Vdbe*, const char*); #endif int sqlite3Reprepare(Vdbe*); void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); diff --git a/src/tokenize.c b/src/tokenize.c index 499fae6f19..297f9ab002 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -734,8 +734,7 @@ static void addSpaceSeparator(sqlite3_str *pStr){ */ char *sqlite3Normalize( Vdbe *pVdbe, /* VM being reprepared */ - const char *zSql, /* The original SQL string */ - int nSql /* Size of the input string in bytes */ + const char *zSql /* The original SQL string */ ){ sqlite3 *db; /* The database connection */ int i; /* Next unread byte of zSql[] */ @@ -748,12 +747,12 @@ char *sqlite3Normalize( int j; /* Bytes of normalized SQL generated so far */ sqlite3_str *pStr; /* The normalized SQL string under construction */ - if( zSql==0 || nSql==0 ) return 0; db = sqlite3VdbeDb(pVdbe); tokenType = -1; nParen = iStartIN = nParenAtIN = 0; pStr = sqlite3_str_new(db); - for(i=0; iaccError==0; i+=n){ + assert( pStr!=0 ); /* sqlite3_str_new() never returns NULL */ + for(i=0; zSql[i] && pStr->accError==0; i+=n){ if( tokenType!=TK_SPACE ){ prevType = tokenType; } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index d8a463ae3b..91379a1469 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1713,9 +1713,9 @@ char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; if( p==0 ) return 0; - if( p->zNormSql==0 && p->zSql!=0 ){ + if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){ sqlite3_mutex_enter(p->db->mutex); - p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql)); + p->zNormSql = sqlite3Normalize(p, p->zSql); sqlite3_mutex_leave(p->db->mutex); } return p->zNormSql; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ac7bf6a65b..5225cbc73f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -64,13 +64,6 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){ } assert( p->zSql==0 ); p->zSql = sqlite3DbStrNDup(p->db, z, n); -#ifdef SQLITE_ENABLE_NORMALIZE - assert( p->zNormSql==0 ); - if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){ - p->zNormSql = sqlite3Normalize(p, p->zSql, n); - assert( p->zNormSql!=0 || p->db->mallocFailed ); - } -#endif } #ifdef SQLITE_ENABLE_NORMALIZE From 368bfe8b008d1c4de12059c13750954db1b21dc9 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Dec 2018 12:20:41 +0000 Subject: [PATCH 45/52] Small performance improvement in the sqlite3_bind() family of interfaces. FossilOrigin-Name: 1dc0c3df32572f09265fc73eeaa0748159e0e9523b9b2d6768b834a29c909968 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 2 +- src/vdbeapi.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 79aa8f42ae..825488a2d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\srefinements\sto\sthe\ssqlite3_normalized_sql()\sinterface.\s\sTH3\snow\ngives\s100%\sMC/DC\son\sthat\sinterface. -D 2018-12-10T20:01:40.741 +C Small\sperformance\simprovement\sin\sthe\ssqlite3_bind()\sfamily\sof\sinterfaces. +D 2018-12-11T12:20:41.473 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -471,7 +471,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c e6f10875d52aca3b7e57ce1ec174aeafc9b6c00b43000cd30d791f9cb490b7a6 -F src/main.c 473d149b51d4f5376e3308e11d5bc07f7570cf596eacecd148c6f7ae63a649ce +F src/main.c 008f45dd39cdc7fb9e7e1f61d75a5c97397f8e07427a8879be23e27747d5f948 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -581,7 +581,7 @@ F src/vacuum.c 3ffe64ecfc94b7528c5d7bdb1c3a19d72fec63f2aa846e3b90f8de5dbbddf5aa F src/vdbe.c 55bafc424748d9ed505ab2680736e51d1bb05c01e9885cbb3b287b51dc8b47ec F src/vdbe.h 8990d668a89890a33326b0a29b992c4014b72f3b6cdcd9ee0e190593c247f9b0 F src/vdbeInt.h 73f5051923f3f29779bfc374c0c68e23b8e5e3792def2e33e51b427edb890abd -F src/vdbeapi.c 43ccaa96e28ebe56a46616c300218e5073911eff618e9d0ab7526983cda939b1 +F src/vdbeapi.c d006df501bc7f39204a4a500dba54fcf5a3925fabe25f9a8853c79adb2897a6b F src/vdbeaux.c f00d9b32a250b829a3c00140255a1c37a6463d726bb87ed6bbb80a1ce76a56bd F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 @@ -1783,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 06e955e5d2b857b9ce94a73fb174b620c101bdb160842f9a0e958bdb48e8c987 -R 692e5dc4224a0726be01a748426768ab +P c96bf6cca220e363b099455ce35195ce7e89d374a52dc787f56e7b11e587bced +R 86dccf908eed64e76085197f6613cec4 U drh -Z 6406ae7cdd7eefde2a741c866c8156da +Z df1a85b6e63863e8c0c59775b55fa8b0 diff --git a/manifest.uuid b/manifest.uuid index 963e5257c1..707eb7f983 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c96bf6cca220e363b099455ce35195ce7e89d374a52dc787f56e7b11e587bced \ No newline at end of file +1dc0c3df32572f09265fc73eeaa0748159e0e9523b9b2d6768b834a29c909968 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 627d24e0b1..efffc2b633 100644 --- a/src/main.c +++ b/src/main.c @@ -2405,7 +2405,7 @@ const char *sqlite3_errmsg(sqlite3 *db){ z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); }else{ testcase( db->pErr==0 ); - z = (char*)sqlite3_value_text(db->pErr); + z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0; assert( !db->mallocFailed ); if( z==0 ){ z = sqlite3ErrStr(db->errCode); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 91379a1469..c43646a89e 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1286,7 +1286,7 @@ static int vdbeUnbind(Vdbe *p, int i){ pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; - sqlite3Error(p->db, SQLITE_OK); + p->db->errCode = SQLITE_OK; /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. From cfc078318cc39ab2f98369a1be8fdc221ea63851 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Dec 2018 12:51:46 +0000 Subject: [PATCH 46/52] Small performance improvement in sqlite3_step(). FossilOrigin-Name: d1db8d5894450b24bb0335983503d9bbf6cc48a0ae4b83291283fb2d32b6b25b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 825488a2d0..c56ba9e8ee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sin\sthe\ssqlite3_bind()\sfamily\sof\sinterfaces. -D 2018-12-11T12:20:41.473 +C Small\sperformance\simprovement\sin\ssqlite3_step(). +D 2018-12-11T12:51:46.910 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -581,7 +581,7 @@ F src/vacuum.c 3ffe64ecfc94b7528c5d7bdb1c3a19d72fec63f2aa846e3b90f8de5dbbddf5aa F src/vdbe.c 55bafc424748d9ed505ab2680736e51d1bb05c01e9885cbb3b287b51dc8b47ec F src/vdbe.h 8990d668a89890a33326b0a29b992c4014b72f3b6cdcd9ee0e190593c247f9b0 F src/vdbeInt.h 73f5051923f3f29779bfc374c0c68e23b8e5e3792def2e33e51b427edb890abd -F src/vdbeapi.c d006df501bc7f39204a4a500dba54fcf5a3925fabe25f9a8853c79adb2897a6b +F src/vdbeapi.c 57a2d794a8833f269b878dbc24e955369bdb379af6c4e93ebc5ce1a20fa3daf4 F src/vdbeaux.c f00d9b32a250b829a3c00140255a1c37a6463d726bb87ed6bbb80a1ce76a56bd F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 @@ -1783,7 +1783,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c96bf6cca220e363b099455ce35195ce7e89d374a52dc787f56e7b11e587bced -R 86dccf908eed64e76085197f6613cec4 +P 1dc0c3df32572f09265fc73eeaa0748159e0e9523b9b2d6768b834a29c909968 +R 3a0f12a0379dcf87f0d4dac5a67e82c1 U drh -Z df1a85b6e63863e8c0c59775b55fa8b0 +Z a9bab00c03f66c3893504302653c1fce diff --git a/manifest.uuid b/manifest.uuid index 707eb7f983..e4dca32cf8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1dc0c3df32572f09265fc73eeaa0748159e0e9523b9b2d6768b834a29c909968 \ No newline at end of file +d1db8d5894450b24bb0335983503d9bbf6cc48a0ae4b83291283fb2d32b6b25b \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index c43646a89e..23b19273b6 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -585,7 +585,7 @@ static int sqlite3Step(Vdbe *p){ return SQLITE_NOMEM_BKPT; } - if( p->pc<=0 && p->expired ){ + if( p->pc<0 && p->expired ){ p->rc = SQLITE_SCHEMA; rc = SQLITE_ERROR; goto end_of_step; @@ -662,9 +662,9 @@ end_of_step: || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); - if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 - && rc!=SQLITE_ROW - && rc!=SQLITE_DONE + if( rc!=SQLITE_ROW + && rc!=SQLITE_DONE + && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ /* If this statement was prepared using saved SQL and an ** error has occurred, then return the error code in p->rc to the From f17e510ee599c7f5521ab8f2ea49563b0010c17e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 11 Dec 2018 17:56:23 +0000 Subject: [PATCH 47/52] Change the way wal2 locks work to ensure a reader only ever has to lock a single slot. FossilOrigin-Name: 18b2c23ac53d985ccc5798ea2d92fb75644b857c373fb490e0d04d5d0194a3d5 --- manifest | 14 +-- manifest.uuid | 2 +- src/wal.c | 205 ++++++++++++++++++++++--------------------- test/wal2simple.test | 2 - 4 files changed, 115 insertions(+), 108 deletions(-) diff --git a/manifest b/manifest index 1e2d309518..b182fb73af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2018-12-11T13:44:59.901 +C Change\sthe\sway\swal2\slocks\swork\sto\sensure\sa\sreader\sonly\sever\shas\sto\slock\sa\nsingle\sslot. +D 2018-12-11T17:56:23.255 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -590,7 +590,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c c01f62932ae8458e77aed8846e819e167a52224c0a13f972c9e7d90354205911 +F src/wal.c 22850fcd2efa9b13e237f6097f931fe7dbd9e50f401a659288d70ae3951396b8 F src/wal.h d2a69695c84137f76e19a247a342cb02ab0131001b6f58153d94b71195bbd84d F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1597,7 +1597,7 @@ F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c -F test/wal2simple.test 8c9dfb8f1bca01a0deb57f7074cdb83865c2292e89b13f7a51a1c160dca3f5f4 +F test/wal2simple.test ce30899ab9c396d0d91905f401bb9a591f12bdf0b6b6bd5d1a62b6f560eac6e3 F test/wal2snapshot.test 95a919e1c73dee0e0212d10931d03cc1116f68a0ff603163e551aaa5ac7025c1 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1787,7 +1787,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f6baa7e1163ed5f61375b0554337030fec23e8a9f60c6412e1b5d626961e93f9 d1db8d5894450b24bb0335983503d9bbf6cc48a0ae4b83291283fb2d32b6b25b -R 01aa34cbfc00cc4424502da843942732 +P d8dd98a39ea061dea264a7d81153f7b1be2f81b554b30d0ce289897c802209bd +R 7649521e36559a5f6fcaed801161c713 U dan -Z d3ebc7a4959beab3fb6b05d6d91630aa +Z 45541997f2aea4861e4215e9e1638ee2 diff --git a/manifest.uuid b/manifest.uuid index 6216c305ae..5aa435a1c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8dd98a39ea061dea264a7d81153f7b1be2f81b554b30d0ce289897c802209bd \ No newline at end of file +18b2c23ac53d985ccc5798ea2d92fb75644b857c373fb490e0d04d5d0194a3d5 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index ee5d2e1779..634de7ec6f 100644 --- a/src/wal.c +++ b/src/wal.c @@ -362,7 +362,6 @@ ** recovery procedure still takes the same exclusive lock on the entire ** range of SQLITE_SHM_NLOCK shm-locks. This works because the read-locks ** above use four of the six read-locking slots used by legacy wal mode. -** See the header comment for function walLockReader() for details. ** ** STARTUP/RECOVERY ** @@ -473,16 +472,20 @@ int sqlite3WalTrace = 0; ** is held, or else is the index of the read-mark on which a lock is ** held. ** -** In wal2 mode, Wal.readLock must be set to one of the following values. -** A value of -1 still indicates that no read-lock is held, but the other -** values are symbolic. See the implementation of walLockReader() for -** details of how the symbols map to OS level locks. +** In wal2 mode, a value of -1 still indicates that no read-lock is held. +** And a non-zero value still represents the index of the read-mark on +** which a lock is held. There are two differences: +** +** 1. wal2 mode never uses read-mark 0. +** +** 2. locks on each read-mark have a different interpretation, as +** indicated by the symbolic names below. */ #define WAL_LOCK_NONE -1 #define WAL_LOCK_PART1 1 #define WAL_LOCK_PART1_FULL2 2 -#define WAL_LOCK_PART2 3 -#define WAL_LOCK_PART2_FULL1 4 +#define WAL_LOCK_PART2_FULL1 3 +#define WAL_LOCK_PART2 4 /* ** This constant is used in wal2 mode only. @@ -1111,36 +1114,6 @@ static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ walLockName(lockIdx), n)); } -/* -** This function is used to take and release read-locks in wal2 mode. -** -** Use of WAL_READ_LOCK(x) slots for (1<=x<=4). -** -** 1) Partial read of *-wal-1 (blocks checkpointer from checkpointing) -** 2) Full read of *-wal-2 (blocks writer from writing) -** 3) Partial read of *-wal-2 (blocks checkpointer from checkpointing) -** 4) Full read of *-wal-1 (blocks writer from writing) -*/ -static int walLockReader(Wal *pWal, int eLock, int bLock){ - int i; /* Index of first readmark to lock */ - int n; /* Number of readmarks to lock */ - - assert( pWal->hdr.iVersion==WAL_VERSION2 ); - if( pWal->exclusiveMode ) return SQLITE_OK; - - switch( eLock ){ - case WAL_LOCK_PART1 : i = 1; n = 1; break; - case WAL_LOCK_PART1_FULL2: i = 1; n = 2; break; - case WAL_LOCK_PART2 : i = 3; n = 1; break; - case WAL_LOCK_PART2_FULL1: i = 3; n = 2; break; - default: assert( !"cannot happen" ); - } - - return sqlite3OsShmLock(pWal->pDbFd, WAL_READ_LOCK(i), n, - SQLITE_SHM_SHARED | (bLock ? SQLITE_SHM_LOCK : SQLITE_SHM_UNLOCK) - ); -} - /* ** Compute a hash on a page number. The resulting hash value must land ** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances @@ -2251,6 +2224,68 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ assert( pInfo->aReadMark[0]==0 ); } +/* +** This function is used in wal2 mode. +** +** This function is called when writer pWal is just about to start +** writing out frames. Parameter iApp is the current wal file. The "other" wal +** file (wal file !iApp) has been fully checkpointed. This function returns +** SQLITE_OK if there are no readers preventing the writer from switching to +** the other wal file. Or SQLITE_BUSY if there are. +*/ +static int wal2RestartOk(Wal *pWal, int iApp){ + /* The other wal file (wal file !iApp) can be overwritten if there + ** are no readers reading from it - no "full" or "partial" locks. + ** Technically speaking it is not possible for any reader to hold + ** a "part" lock, as this would have prevented the file from being + ** checkpointed. But checking anyway doesn't hurt. The following + ** is equivalent to: + ** + ** if( iApp==0 ) eLock = WAL_LOCK_PART1_FULL2; + ** if( iApp==1 ) eLock = WAL_LOCK_PART1; + */ + int eLock = 1 + (iApp==0); + + assert( WAL_LOCK_PART1==1 ); + assert( WAL_LOCK_PART1_FULL2==2 ); + assert( WAL_LOCK_PART2_FULL1==3 ); + assert( WAL_LOCK_PART2==4 ); + + assert( iApp!=0 || eLock==WAL_LOCK_PART1_FULL2 ); + assert( iApp!=1 || eLock==WAL_LOCK_PART1 ); + + return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 3); +} +static void wal2RestartFinished(Wal *pWal, int iApp){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iApp==0)), 3); +} + +/* +** This function is used in wal2 mode. +** +** This function is called when a checkpointer wishes to checkpoint wal +** file iCkpt. It takes the required lock and, if successful, returns +** SQLITE_OK. Otherwise, an SQLite error code (e.g. SQLITE_BUSY). If this +** function returns SQLITE_OK, it is the responsibility of the caller +** to invoke wal2CheckpointFinished() to release the lock. +*/ +static int wal2CheckpointOk(Wal *pWal, int iCkpt){ + int eLock = 1 + (iCkpt*2); + + assert( WAL_LOCK_PART1==1 ); + assert( WAL_LOCK_PART1_FULL2==2 ); + assert( WAL_LOCK_PART2_FULL1==3 ); + assert( WAL_LOCK_PART2==4 ); + + assert( iCkpt!=0 || eLock==WAL_LOCK_PART1 ); + assert( iCkpt!=1 || eLock==WAL_LOCK_PART2_FULL1 ); + + return walLockExclusive(pWal, WAL_READ_LOCK(eLock), 2); +} +static void wal2CheckpointFinished(Wal *pWal, int iCkpt){ + walUnlockExclusive(pWal, WAL_READ_LOCK(1 + (iCkpt*2)), 2); +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -2318,7 +2353,7 @@ static int walCheckpoint( ** preventing this checkpoint operation. If one is found, return ** early. */ if( bWal2 ){ - rc = walLockExclusive(pWal, WAL_READ_LOCK(1 + iCkpt*2), 1); + rc = wal2CheckpointOk(pWal, iCkpt); if( rc!=SQLITE_OK ) return rc; } @@ -2367,9 +2402,9 @@ static int walCheckpoint( assert( rc==SQLITE_OK || pIter==0 ); } - if( pIter - && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK - ){ + if( pIter && (bWal2 + || (rc = walBusyLock(pWal, xBusy, pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK + )){ u32 nBackfill = pInfo->nBackfill; assert( bWal2==0 || nBackfill==0 ); @@ -2434,7 +2469,9 @@ static int walCheckpoint( } /* Release the reader lock held while backfilling */ - walUnlockExclusive(pWal, WAL_READ_LOCK(bWal2 ? 1 + iCkpt*2 : 0), 1); + if( bWal2==0 ){ + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); + } } if( rc==SQLITE_BUSY ){ @@ -2442,6 +2479,7 @@ static int walCheckpoint( ** just because there are active readers. */ rc = SQLITE_OK; } + if( bWal2 ) wal2CheckpointFinished(pWal, iCkpt); } /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the @@ -3060,18 +3098,27 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ assert( pWal->apWiData[0]!=0 ); pInfo = walCkptInfo(pWal); if( isWalMode2(pWal) ){ - int eLock = 1 + (walidxGetFile(&pWal->hdr)*2); - if( pInfo->nBackfill==0 ){ - eLock += walidxGetMxFrame(&pWal->hdr, !walidxGetFile(&pWal->hdr))>0; - } - rc = walLockReader(pWal, eLock, 1); + /* This connection needs a "part" lock on the current wal file and, + ** unless pInfo->nBackfill is set to indicate that it has already been + ** checkpointed, a "full" lock on the other wal file. */ + int iWal = walidxGetFile(&pWal->hdr); + int nBackfill = pInfo->nBackfill || walidxGetMxFrame(&pWal->hdr, !iWal)==0; + int eLock = 1 + (iWal*2) + (nBackfill==iWal); + + assert( nBackfill==0 || nBackfill==1 ); + assert( iWal==0 || iWal==1 ); + assert( iWal!=0 || nBackfill!=1 || eLock==WAL_LOCK_PART1 ); + assert( iWal!=0 || nBackfill!=0 || eLock==WAL_LOCK_PART1_FULL2 ); + assert( iWal!=1 || nBackfill!=1 || eLock==WAL_LOCK_PART2 ); + assert( iWal!=1 || nBackfill!=0 || eLock==WAL_LOCK_PART2_FULL1 ); + + rc = walLockShared(pWal, WAL_READ_LOCK(eLock)); if( rc!=SQLITE_OK ){ return rc; } - walShmBarrier(pWal); if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ - walLockReader(pWal, eLock, 0); + walUnlockShared(pWal, WAL_READ_LOCK(eLock)); return WAL_RETRY; }else{ pWal->readLock = eLock; @@ -3403,11 +3450,7 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ void sqlite3WalEndReadTransaction(Wal *pWal){ sqlite3WalEndWriteTransaction(pWal); if( pWal->readLock!=WAL_LOCK_NONE ){ - if( isWalMode2(pWal) ){ - (void)walLockReader(pWal, pWal->readLock, 0); - }else{ - walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - } + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = WAL_LOCK_NONE; } } @@ -3794,33 +3837,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ return rc; } -/* -** This function is used in wal2 mode. -** -** This function is called when writer pWal is just about to start -** writing out frames. The "other" wal file (wal file !pWal->hdr.iAppend) -** has been fully checkpointed. This function returns SQLITE_OK if there -** are no readers preventing the writer from switching to the other wal -** file. Or SQLITE_BUSY if there are. -*/ -static int walRestartOk(Wal *pWal){ - int rc; /* Return code */ - int iApp = walidxGetFile(&pWal->hdr); /* Current WAL file */ - - /* No reader can be doing a "partial" read of wal file !iApp - in that - ** case it would not have been possible to checkpoint the file. So - ** it is only necessary to test for "full" readers. See the comment - ** above walLockReader() function for exactly what this means in terms - ** of locks. */ - int i = (iApp==0) ? 2 : 4; - - rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1); - if( rc==SQLITE_OK ){ - walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); - } - return rc; -} - /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending @@ -3848,19 +3864,20 @@ static int walRestartLog(Wal *pWal){ if( walidxGetMxFrame(&pWal->hdr, iApp)>=nWalSize ){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); if( walidxGetMxFrame(&pWal->hdr, !iApp)==0 || pInfo->nBackfill ){ - rc = walRestartOk(pWal); + rc = wal2RestartOk(pWal, iApp); if( rc==SQLITE_OK ){ - iApp = !iApp; + int iNew = !iApp; pWal->nCkpt++; - walidxSetFile(&pWal->hdr, iApp); - walidxSetMxFrame(&pWal->hdr, iApp, 0); + walidxSetFile(&pWal->hdr, iNew); + walidxSetMxFrame(&pWal->hdr, iNew, 0); sqlite3Put4byte((u8*)&pWal->hdr.aSalt[0], pWal->hdr.aFrameCksum[0]); sqlite3Put4byte((u8*)&pWal->hdr.aSalt[1], pWal->hdr.aFrameCksum[1]); walIndexWriteHdr(pWal); pInfo->nBackfill = 0; - walLockReader(pWal, pWal->readLock, 0); - pWal->readLock = iApp ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2; - rc = walLockReader(pWal, pWal->readLock, 1); + wal2RestartFinished(pWal, iApp); + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + pWal->readLock = iNew ? WAL_LOCK_PART2_FULL1 : WAL_LOCK_PART1_FULL2; + rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock)); }else if( rc==SQLITE_BUSY ){ rc = SQLITE_OK; } @@ -4490,11 +4507,7 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){ if( op==0 ){ if( pWal->exclusiveMode ){ pWal->exclusiveMode = WAL_NORMAL_MODE; - if( isWalMode2(pWal) ){ - rc = walLockReader(pWal, pWal->readLock, 1); - }else{ - rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - } + rc = walLockShared(pWal, WAL_READ_LOCK(pWal->readLock)); if( rc!=SQLITE_OK ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } @@ -4506,11 +4519,7 @@ int sqlite3WalExclusiveMode(Wal *pWal, int op){ }else if( op>0 ){ assert( pWal->exclusiveMode==WAL_NORMAL_MODE ); assert( pWal->readLock>=0 ); - if( isWalMode2(pWal) ){ - walLockReader(pWal, pWal->readLock, 0); - }else{ - walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - } + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; rc = 1; }else{ diff --git a/test/wal2simple.test b/test/wal2simple.test index becbb232fc..28e13bf162 100644 --- a/test/wal2simple.test +++ b/test/wal2simple.test @@ -243,8 +243,6 @@ do_test 6.1 { } } {} -puts "[file size test.db-wal] [file size test.db-wal2]" - do_test 6.2.1 { foreach f [glob -nocomplain test.db2*] { forcedelete $f } forcecopy test.db-wal2 test.db2-wal2 From 2c4328ed870ce007ffb2181393cfc999d618406c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Dec 2018 19:04:19 +0000 Subject: [PATCH 48/52] Add tests to ensure that each of the 4 wal read-locks does what it is supposed to. FossilOrigin-Name: 4d5779f31d4931edb8bb8952d8886625ead5e51f0c308e5763e519427f6609e1 --- manifest | 12 +-- manifest.uuid | 2 +- test/wal2simple.test | 186 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 192 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b182fb73af..4cb34c27b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sway\swal2\slocks\swork\sto\sensure\sa\sreader\sonly\sever\shas\sto\slock\sa\nsingle\sslot. -D 2018-12-11T17:56:23.255 +C Add\stests\sto\sensure\sthat\seach\sof\sthe\s4\swal\sread-locks\sdoes\swhat\sit\sis\ssupposed\sto. +D 2018-12-12T19:04:19.425 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -1597,7 +1597,7 @@ F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c -F test/wal2simple.test ce30899ab9c396d0d91905f401bb9a591f12bdf0b6b6bd5d1a62b6f560eac6e3 +F test/wal2simple.test 96206c98bf64ab20ec00a1c0f6c709e258b98b39f2149889361f31966ce5a703 F test/wal2snapshot.test 95a919e1c73dee0e0212d10931d03cc1116f68a0ff603163e551aaa5ac7025c1 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1787,7 +1787,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d8dd98a39ea061dea264a7d81153f7b1be2f81b554b30d0ce289897c802209bd -R 7649521e36559a5f6fcaed801161c713 +P 18b2c23ac53d985ccc5798ea2d92fb75644b857c373fb490e0d04d5d0194a3d5 +R 234c1705f2bf2a9abb44249d0ee7aea0 U dan -Z 45541997f2aea4861e4215e9e1638ee2 +Z d3854fa2ddea650fc625b1a32d28af78 diff --git a/manifest.uuid b/manifest.uuid index 5aa435a1c6..60eb3eb58e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18b2c23ac53d985ccc5798ea2d92fb75644b857c373fb490e0d04d5d0194a3d5 \ No newline at end of file +4d5779f31d4931edb8bb8952d8886625ead5e51f0c308e5763e519427f6609e1 \ No newline at end of file diff --git a/test/wal2simple.test b/test/wal2simple.test index 28e13bf162..3d35b026c6 100644 --- a/test/wal2simple.test +++ b/test/wal2simple.test @@ -114,7 +114,6 @@ do_execsql_test 3.0 { PRAGMA journal_mode = wal2; } {wal2} -breakpoint do_test 3.1 { execsql BEGIN for {set i 1} {$i < 1000} {incr i} { @@ -287,4 +286,189 @@ do_test 6.4.2 { } {wal2} db2 close +#------------------------------------------------------------------------- +reset_db +sqlite3 db2 test.db +do_execsql_test 7.0 { + PRAGMA journal_size_limit = 10000; + PRAGMA journal_mode = wal2; + PRAGMA wal_autocheckpoint = 0; + BEGIN; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES( randomblob(8000) ); + COMMIT; +} {10000 wal2 0} + +do_test 7.1 { + list [file size test.db-wal] [file size test.db-wal2] +} {9464 0} + +# Connection db2 is holding a PART1 lock. +# +# 7.2.2: Test that the PART1 does not prevent db from switching to the +# other wal file. +# +# 7.2.3: Test that the PART1 does prevent a checkpoint of test.db-wal. +# +# 7.2.4: Test that after the PART1 is released the checkpoint is possible. +# +do_test 7.2.1 { + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 +} {1} +do_test 7.2.2 { + execsql { + INSERT INTO t1 VALUES( randomblob(800) ); + INSERT INTO t1 VALUES( randomblob(800) ); + } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {13656 3176 1024} +do_test 7.2.3 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {13656 3176 1024} +do_test 7.2.4 { + execsql { END } db2 + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {13656 3176 11264} + +# Connection db2 is holding a PART2_FULL1 lock. +# +# 7.3.2: Test that the lock does not prevent checkpointing. +# +# 7.3.3: Test that the lock does prevent the writer from overwriting +# test.db-wal. +# +# 7.3.4: Test that after the PART2_FULL1 is released the writer can +# switch wal files and overwrite test.db-wal +# +db close +db2 close +sqlite3 db test.db +sqlite3 db2 test.db +do_test 7.3.1 { + execsql { + PRAGMA wal_autocheckpoint = 0; + PRAGMA journal_size_limit = 10000; + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(500)); + } + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 3176 11264} +do_test 7.3.2 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 3176 21504} +do_test 7.3.3 { + execsql { + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(500)); + } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 18896 21504} +do_test 7.3.4 { + execsql END db2 + execsql { INSERT INTO t1 VALUES(randomblob(5000)); } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 18896 21504} + +# Connection db2 is holding a PART2 lock. +# +# 7.4.2: Test that the lock does not prevent writer switching to test.db-wal. +# +# 7.3.3: Test that the lock does prevent checkpointing of test.db-wal2. +# +# 7.3.4: Test that after the PART2 is released test.db-wal2 can be +# checkpointed. +# +db close +db2 close +sqlite3 db test.db +sqlite3 db2 test.db +do_test 7.4.1 { + execsql { + PRAGMA wal_autocheckpoint = 0; + PRAGMA journal_size_limit = 10000; + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(10000)); + PRAGMA wal_checkpoint; + } + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 44032} +do_test 7.4.2 { + execsql { + INSERT INTO t1 VALUES(randomblob(5000)); + } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 44032} +do_test 7.4.3 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 44032} +do_test 7.4.4 { + execsql END db2 + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 54272} + +# Connection db2 is holding a PART1_FULL2 lock. +# +# 7.5.2: Test that the lock does not prevent a checkpoint of test.db-wal2. +# +# 7.5.3: Test that the lock does prevent the writer from overwriting +# test.db-wal2. +# +# 7.5.4: Test that after the PART1_FULL2 lock is released, the writer +# can switch to test.db-wal2. +# +db close +db2 close +sqlite3 db test.db +sqlite3 db2 test.db +do_test 7.5.1 { + execsql { + PRAGMA wal_autocheckpoint = 0; + PRAGMA journal_size_limit = 10000; + INSERT INTO t1 VALUES(randomblob(10000)); + INSERT INTO t1 VALUES(randomblob(10000)); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(randomblob(5000)); + } + execsql { + BEGIN; + SELECT count(*) FROM t1; + } db2 + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 64512} +do_test 7.5.2 { + execsql { PRAGMA wal_checkpoint } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {12608 12608 75776} +do_test 7.5.3.1 { + execsql { INSERT INTO t1 VALUES(randomblob(5000)) } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {14704 12608 75776} +do_test 7.5.3.2 { + execsql { INSERT INTO t1 VALUES(randomblob(5000)) } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {22040 12608 75776} +do_test 7.5.4 { + execsql END db2 + execsql { INSERT INTO t1 VALUES(randomblob(5000)) } + list [file size test.db-wal] [file size test.db-wal2] [file size test.db] +} {22040 12608 75776} + + finish_test + From eeb778d506e429549090fd78b75282b579908e8a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Dec 2018 20:39:38 +0000 Subject: [PATCH 49/52] Add new test file wal2big.test. FossilOrigin-Name: e3e50bcdab8c91e003942d84430b3e580e034141236d19dda0e8af4ecf0e085b --- manifest | 13 ++++++----- manifest.uuid | 2 +- src/wal.c | 9 ++------ test/wal2big.test | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 14 deletions(-) create mode 100644 test/wal2big.test diff --git a/manifest b/manifest index 4cb34c27b4..8249da62fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\sensure\sthat\seach\sof\sthe\s4\swal\sread-locks\sdoes\swhat\sit\sis\ssupposed\sto. -D 2018-12-12T19:04:19.425 +C Add\snew\stest\sfile\swal2big.test. +D 2018-12-12T20:39:38.782 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -590,7 +590,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 22850fcd2efa9b13e237f6097f931fe7dbd9e50f401a659288d70ae3951396b8 +F src/wal.c 7106d08dce79c9c5338c2b194139c418f21cf30dc2440145ce788893dd9b49cc F src/wal.h d2a69695c84137f76e19a247a342cb02ab0131001b6f58153d94b71195bbd84d F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1596,6 +1596,7 @@ F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bd F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 +F test/wal2big.test b6e78336b7aa517bf81953e118d73cd405549fb95159fab7d2b67a8eb5990ade F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal2simple.test 96206c98bf64ab20ec00a1c0f6c709e258b98b39f2149889361f31966ce5a703 F test/wal2snapshot.test 95a919e1c73dee0e0212d10931d03cc1116f68a0ff603163e551aaa5ac7025c1 @@ -1787,7 +1788,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 18b2c23ac53d985ccc5798ea2d92fb75644b857c373fb490e0d04d5d0194a3d5 -R 234c1705f2bf2a9abb44249d0ee7aea0 +P 4d5779f31d4931edb8bb8952d8886625ead5e51f0c308e5763e519427f6609e1 +R 4985894a481d77e7477a50c3c152405c U dan -Z d3854fa2ddea650fc625b1a32d28af78 +Z 72ed7e3951a42a191c8ff32f1f42f36f diff --git a/manifest.uuid b/manifest.uuid index 60eb3eb58e..65c0861b34 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d5779f31d4931edb8bb8952d8886625ead5e51f0c308e5763e519427f6609e1 \ No newline at end of file +e3e50bcdab8c91e003942d84430b3e580e034141236d19dda0e8af4ecf0e085b \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 634de7ec6f..4cc37d0700 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1204,14 +1204,9 @@ static int walExternalDecode(u32 iExternal, u32 *piRead){ iExternal - (iHash/2) * HASHTABLE_NPAGE; return 0; } - if( iHash==0 ){ - *piRead = iExternal; - return 0; - }else{ - *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE; - } - return (iHash % 2); + *piRead = iExternal - HASHTABLE_NPAGE_ONE - ((iHash-1)/2) * HASHTABLE_NPAGE; + return 1; } /* diff --git a/test/wal2big.test b/test/wal2big.test new file mode 100644 index 0000000000..2758108479 --- /dev/null +++ b/test/wal2big.test @@ -0,0 +1,59 @@ +# 2017 September 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2big +ifcapable !wal {finish_test ; return } + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000000; + + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000 + ) + INSERT INTO t1 SELECT random(), random(), random() FROM s; +} {wal2 10000000} + +do_execsql_test 1.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200000 + ) + INSERT INTO t1 SELECT random(), random(), random() FROM s; +} + +do_test 1.1 { + list [expr [file size test.db-wal]>10000000] \ + [expr [file size test.db-wal2]>10000000] +} {1 1} + +do_test 1.2 { + sqlite3 db2 test.db + execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } db2 +} {400000 ok} + +finish_test From 597bf19266ad8b148d8ff9d88b99a9fdf032f9d7 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 13 Dec 2018 16:26:43 +0000 Subject: [PATCH 50/52] Add tests cases for recovery in wal2 mode. FossilOrigin-Name: 34f56f8a4239781071edf6317723d0a7333249551c2c1ce7ed39fc116d53d780 --- manifest | 15 ++-- manifest.uuid | 2 +- src/wal.c | 8 +- test/wal2big.test | 16 +++- test/wal2recover.test | 167 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 14 deletions(-) create mode 100644 test/wal2recover.test diff --git a/manifest b/manifest index 8249da62fe..06ffd7fd7e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\swal2big.test. -D 2018-12-12T20:39:38.782 +C Add\stests\scases\sfor\srecovery\sin\swal2\smode. +D 2018-12-13T16:26:43.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -590,7 +590,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 7106d08dce79c9c5338c2b194139c418f21cf30dc2440145ce788893dd9b49cc +F src/wal.c 93389c399264ca3e0728b7bc2565b4ee6a9099b78929e633ffad3bcd493e415e F src/wal.h d2a69695c84137f76e19a247a342cb02ab0131001b6f58153d94b71195bbd84d F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1596,7 +1596,8 @@ F test/vtab_err.test dcc8b7b9cb67522b3fe7a272c73856829dae4ab7fdb30399aea1b6981bd F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 -F test/wal2big.test b6e78336b7aa517bf81953e118d73cd405549fb95159fab7d2b67a8eb5990ade +F test/wal2big.test 0b4ec526f9ca4bbabc355042c38045ae2e253fb46eb327bb7693d0122bc6968b +F test/wal2recover.test 4f14625c65cdb6132e061f85d2d441f8bb96aea4b20d4261bcf14041d8e1a735 F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal2simple.test 96206c98bf64ab20ec00a1c0f6c709e258b98b39f2149889361f31966ce5a703 F test/wal2snapshot.test 95a919e1c73dee0e0212d10931d03cc1116f68a0ff603163e551aaa5ac7025c1 @@ -1788,7 +1789,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4d5779f31d4931edb8bb8952d8886625ead5e51f0c308e5763e519427f6609e1 -R 4985894a481d77e7477a50c3c152405c +P e3e50bcdab8c91e003942d84430b3e580e034141236d19dda0e8af4ecf0e085b +R e3b17b10c3f7dc5dae3bc5cc7a744091 U dan -Z 72ed7e3951a42a191c8ff32f1f42f36f +Z 6979d16521822ff458ef9e063c382318 diff --git a/manifest.uuid b/manifest.uuid index 65c0861b34..edb729f46a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3e50bcdab8c91e003942d84430b3e580e034141236d19dda0e8af4ecf0e085b \ No newline at end of file +34f56f8a4239781071edf6317723d0a7333249551c2c1ce7ed39fc116d53d780 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 4cc37d0700..7d334ed50b 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1656,8 +1656,8 @@ static int walIndexRecover(Wal *pWal){ /* The case where *-wal2 may follow *-wal */ if( nCkpt2<=0x0F && nCkpt2==nCkpt1+1 ){ if( sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[0]))==hdr.aFrameCksum[0] - && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1] - ){ + && sqlite3Get4byte((u8*)(&pWal->hdr.aSalt[1]))==hdr.aFrameCksum[1] + ){ walidxSetFile(&pWal->hdr, 1); walidxSetMxFrame(&pWal->hdr, 1, pWal->hdr.mxFrame); walidxSetMxFrame(&pWal->hdr, 0, hdr.mxFrame); @@ -1715,9 +1715,9 @@ static int walIndexRecover(Wal *pWal){ if( pWal->hdr.nPage ){ if( isWalMode2(pWal) ){ sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, - "recovered (%d,%d) frames from WAL files %s[2] (%s mode)", + "recovered (%d,%d) frames from WAL files %s[2] (wal2 mode)", walidxGetMxFrame(&pWal->hdr, 0), walidxGetMxFrame(&pWal->hdr, 1), - pWal->zWalName, isWalMode2(pWal) ? "wal2" : "wal" + pWal->zWalName ); }else{ sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, diff --git a/test/wal2big.test b/test/wal2big.test index 2758108479..53552e3639 100644 --- a/test/wal2big.test +++ b/test/wal2big.test @@ -43,12 +43,12 @@ do_execsql_test 1.1 { INSERT INTO t1 SELECT random(), random(), random() FROM s; } -do_test 1.1 { +do_test 1.2 { list [expr [file size test.db-wal]>10000000] \ [expr [file size test.db-wal2]>10000000] } {1 1} -do_test 1.2 { +do_test 1.3 { sqlite3 db2 test.db execsql { SELECT count(*) FROM t1; @@ -56,4 +56,16 @@ do_test 1.2 { } db2 } {400000 ok} +do_test 1.4 { + db2 close + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } +} {400000 ok} + finish_test diff --git a/test/wal2recover.test b/test/wal2recover.test new file mode 100644 index 0000000000..fecd85829c --- /dev/null +++ b/test/wal2recover.test @@ -0,0 +1,167 @@ +# 2018 December 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2recover +ifcapable !wal {finish_test ; return } + +proc db_copy {from to} { + forcecopy $from $to + forcecopy ${from}-wal ${to}-wal + forcecopy ${from}-wal2 ${to}-wal2 +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 15000; + PRAGMA wal_autocheckpoint = 0; +} {wal2 15000 0} + +do_test 1.1 { + for {set i 1} {$i <= 1000} {incr i} { + execsql { INSERT INTO t1 VALUES(random(), random(), random()) } + db_copy test.db test.db2 + sqlite3 db2 test.db + set res [execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } db2] + db2 close + if {$res != [list $i ok]} { + error "failure on iteration $i" + } + } + set {} {} +} {} + +#-------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(x UNIQUE); + CREATE TABLE t2(x UNIQUE); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000; + PRAGMA wal_autocheckpoint = 0; + BEGIN; + INSERT INTO t1 VALUES(randomblob(4000)); + INSERT INTO t1 VALUES(randomblob(4000)); + INSERT INTO t1 VALUES(randomblob(4000)); + COMMIT; + BEGIN; + INSERT INTO t2 VALUES(randomblob(4000)); + INSERT INTO t2 VALUES(randomblob(4000)); + INSERT INTO t2 VALUES(randomblob(4000)); + COMMIT; +} {wal2 10000 0} +do_test 2.0.1 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {5120 28328 28328} + +# Test recovery with both wal files intact. +# +do_test 2.1 { + db_copy test.db test.db2 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 3 ok} + +do_test 2.2 { + db2 close + db_copy test.db test.db2 + hexio_write test.db2-wal 16 12345678 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + } db2 +} {0 3} + +do_test 2.3 { + db2 close + db_copy test.db test.db2 + hexio_write test.db2-wal2 16 12345678 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 0 ok} + +do_test 2.4 { + db2 close + db_copy test.db test.db2 + forcecopy test.db-wal test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 0 ok} + +do_test 2.5 { + db2 close + db_copy test.db test.db2 + forcecopy test.db-wal test.db2-wal2 + forcecopy test.db-wal2 test.db2-wal + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 3 ok} + +do_test 2.6 { + db2 close + db_copy test.db test.db2 + forcecopy test.db-wal test.db2-wal2 + close [open test.db-wal w] + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {3 0 ok} + +do_test 2.7 { + db2 close + db_copy test.db test.db2 + forcedelete test.db2-wal + sqlite3 db2 test.db2 + execsql { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; + PRAGMA integrity_check; + } db2 +} {0 0 ok} + +finish_test + From e698c1f79ea8ab498fa863f7dab2bbc91946de6c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Dec 2018 19:53:39 +0000 Subject: [PATCH 51/52] Add further tests for wal2 mode. FossilOrigin-Name: 54e628f902a5508badc8941ceda1bec12fe8f58969c91c670a37888ef2681880 --- manifest | 17 +++---- manifest.uuid | 2 +- src/wal.c | 2 +- test/permutations.test | 14 ++++-- test/wal2recover.test | 99 +++++++++++++++++++++++++++++++++++++++++ test/wal2savepoint.test | 60 +++++++++++++++++++++++++ 6 files changed, 180 insertions(+), 14 deletions(-) create mode 100644 test/wal2savepoint.test diff --git a/manifest b/manifest index 06ffd7fd7e..88e15baab9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\scases\sfor\srecovery\sin\swal2\smode. -D 2018-12-13T16:26:43.514 +C Add\sfurther\stests\sfor\swal2\smode. +D 2018-12-14T19:53:39.654 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -590,7 +590,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 93389c399264ca3e0728b7bc2565b4ee6a9099b78929e633ffad3bcd493e415e +F src/wal.c 1deb1d320b5de56407172ac21577d2bbf5c6ea52dcc0f5315a8d926d4839146e F src/wal.h d2a69695c84137f76e19a247a342cb02ab0131001b6f58153d94b71195bbd84d F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1182,7 +1182,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 57c9f829d72c91c89ae08febe205b73ec22751f476b4dafeafb614dfd263d74c +F test/permutations.test f97a5613430a62ef5fb5aafeb37d4e857dea44e47c31064f154c7049c25b292b F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30 F test/pragma.test c267bf02742c823a191960895b3d52933cebd7beee26757d1ed694f213fcd867 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -1597,8 +1597,9 @@ F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 F test/wal2big.test 0b4ec526f9ca4bbabc355042c38045ae2e253fb46eb327bb7693d0122bc6968b -F test/wal2recover.test 4f14625c65cdb6132e061f85d2d441f8bb96aea4b20d4261bcf14041d8e1a735 +F test/wal2recover.test 64155a1f13ac434ebb374f109b9daa2b52c3ca69db9199be0bcbabb9727cae4e F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c +F test/wal2savepoint.test 29725383e7eb8b78fed46333d69dc45b4ed321db43ecc81766aadea991874bef F test/wal2simple.test 96206c98bf64ab20ec00a1c0f6c709e258b98b39f2149889361f31966ce5a703 F test/wal2snapshot.test 95a919e1c73dee0e0212d10931d03cc1116f68a0ff603163e551aaa5ac7025c1 F test/wal3.test 2a93004bc0fb2b5c29888964024695bade278ab2 @@ -1789,7 +1790,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e3e50bcdab8c91e003942d84430b3e580e034141236d19dda0e8af4ecf0e085b -R e3b17b10c3f7dc5dae3bc5cc7a744091 +P 34f56f8a4239781071edf6317723d0a7333249551c2c1ce7ed39fc116d53d780 +R cabac54018fcd32b9b9c7e5679bc38f9 U dan -Z 6979d16521822ff458ef9e063c382318 +Z be9d783248b67e630efcf341b29cd59e diff --git a/manifest.uuid b/manifest.uuid index edb729f46a..0fc73c3fc5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34f56f8a4239781071edf6317723d0a7333249551c2c1ce7ed39fc116d53d780 \ No newline at end of file +54e628f902a5508badc8941ceda1bec12fe8f58969c91c670a37888ef2681880 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 7d334ed50b..4ce5a67585 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1532,7 +1532,7 @@ static int walIndexRecoverOne(Wal *pWal, int iWal, u32 *pnCkpt, int *pbZero){ } sqlite3_free(aFrame); - }else if( pbZero && nSize==0 ){ + }else if( pbZero ){ *pbZero = 1; } } diff --git a/test/permutations.test b/test/permutations.test index 7e8f38ed1e..75f6ecbc02 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -430,10 +430,16 @@ lappend ::testsuitelist xxx test_suite "coverage-wal" -description { Coverage tests for file wal.c. } -files { - wal.test wal2.test wal3.test walmode.test - walbak.test walhook.test walcrash2.test walcksum.test - walfault.test walbig.test walnoshm.test - wal5.test +wal2big.test wal2recover.test wal2rewrite.test +wal2simple.test wal2snapshot.test wal2.test +wal3.test wal4.test wal5.test +wal64k.test wal6.test wal7.test wal8.test wal9.test +walbak.test walbig.test walblock.test walcksum.test +walfault.test walhook.test walmode.test walnoshm.test +waloverwrite.test walpersist.test walprotocol2.test +walprotocol.test walro2.test walrofault.test walro.test +walshared.test walslow.test wal.test +wal2savepoint.test } test_suite "coverage-pager" -description { diff --git a/test/wal2recover.test b/test/wal2recover.test index fecd85829c..e8cecc1bf7 100644 --- a/test/wal2recover.test +++ b/test/wal2recover.test @@ -163,5 +163,104 @@ do_test 2.7 { } db2 } {0 0 ok} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE t1(a TEXT, b TEXT, c TEXT); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 5; +} {wal2 10000 0} + +do_execsql_test 3.1 { + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200) + INSERT INTO t1 SELECT i, i, i FROM s; + + INSERT INTO t1 VALUES(201, 201, 201); +} {} + +do_test 3.2 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {5120 15752 4224} + +do_test 3.3 { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + PRAGMA journal_size_limit = 10000; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 5; + BEGIN; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200) + INSERT INTO t1 SELECT i, i, i FROM s; + } db2 + list [file size test.db2] [file size test.db2-wal] [file size test.db2-wal2] +} {5120 15752 23088} + +do_test 3.4 { + set fd [open test.db2-shm] + fconfigure $fd -encoding binary -translation binary + set data [read $fd] + close $fd + + set fd [open test.db-shm w] + fconfigure $fd -encoding binary -translation binary + puts -nonewline $fd $data + close $fd + + execsql { + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10) + INSERT INTO t1 SELECT i, i, i FROM s; + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } +} {211 ok} + +do_test 3.5 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {5120 15752 18896} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + PRAGMA journal_mode = wal2; + CREATE TABLE xyz(x, y, z); + INSERT INTO xyz VALUES('x', 'y', 'z'); +} {wal2} +db close +do_test 4.1 { + close [open test.db-wal w] + file mkdir test.db-wal2 + sqlite3 db test.db + catchsql { SELECT * FROM xyz } +} {1 {unable to open database file}} +db close +file delete test.db-wal2 + +do_test 4.2 { + sqlite3 db test.db + execsql { + INSERT INTO xyz VALUES('a', 'b', 'c'); + } + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcedelete test.db2-wal2 + file mkdir test.db2-wal2 + sqlite3 db2 test.db2 + catchsql { SELECT * FROM xyz } db2 +} {1 {unable to open database file}} +db2 close +file delete test.db2-wal2 + + + finish_test diff --git a/test/wal2savepoint.test b/test/wal2savepoint.test new file mode 100644 index 0000000000..c875af88c4 --- /dev/null +++ b/test/wal2savepoint.test @@ -0,0 +1,60 @@ +# 2018 December 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2savepoint +ifcapable !wal {finish_test ; return } + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 15000; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 5; +} {wal2 15000 0} + +do_execsql_test 1.1 { + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 200) + INSERT INTO t1 SELECT random(), random(), random() FROM s; +} {} + +do_test 1.2 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {5120 23088 0} + +do_execsql_test 1.3 { + BEGIN; + SAVEPOINT abc; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 100) + INSERT INTO t1 SELECT random(), random(), random() FROM s; + ROLLBACK TO abc; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s where i < 10) + INSERT INTO t1 SELECT random(), random(), random() FROM s; + COMMIT; + SELECT count(*) FROM t1; + PRAGMA integrity_check; +} {210 ok} + + +finish_test + From 93543215e03af2d0c3b0b85712a7a1b6e4381e4b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 15 Dec 2018 20:20:13 +0000 Subject: [PATCH 52/52] Further test cases for wal2 mode. FossilOrigin-Name: 9cb5f8dab685f5ea36ad142cfa588dee82e87f3a89e8dcf84e0ee124bb29bc7f --- manifest | 20 ++-- manifest.uuid | 2 +- src/test_hexio.c | 22 +++- src/wal.c | 2 +- test/permutations.test | 2 +- test/wal2lock.test | 106 +++++++++++++++++++ test/wal2recover.test | 34 ------ test/wal2recover2.test | 227 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 364 insertions(+), 51 deletions(-) create mode 100644 test/wal2lock.test create mode 100644 test/wal2recover2.test diff --git a/manifest b/manifest index 88e15baab9..3d69ddd17c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\swal2\smode. -D 2018-12-14T19:53:39.654 +C Further\stest\scases\sfor\swal2\smode. +D 2018-12-15T20:20:13.338 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -539,7 +539,7 @@ F src/test_demovfs.c a0c3bdd45ed044115c2c9f7779e56eafff18741e F src/test_devsym.c 1960abbb234b97e9b920f07e99503fc04b443f62bbc3c6ff2c2cea2133e3b8a2 F src/test_fs.c 35a2f7dd8a915900873386331386d9ba1ae1b5026d74fd20c2807bc76221f291 F src/test_func.c d12d805953bcb3bb19f71d29cdc93383b7b7a3369504d2b7e398a1bd77376294 -F src/test_hexio.c 1d4469ca61ab202a1fcec6543f584d2407205e8d +F src/test_hexio.c 1a70ba996c8c8c028469b428ceca0d2c9c84fae68589984022ed3fc6e706eaf7 F src/test_init.c 4413c211a94b62157ca4c145b3f27c497f03c664 F src/test_intarray.c 39b4181662a0f33a427748d87218e7578d913e683dc27eab7098bb41617cac71 F src/test_intarray.h d57ae92f420cda25e22790dac474d60961bd0c500cbaa3338a05152d4a669ef7 @@ -590,7 +590,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 1deb1d320b5de56407172ac21577d2bbf5c6ea52dcc0f5315a8d926d4839146e +F src/wal.c beca1487f7cecd1d6a965ba899a5a6b8868b3eb80580d33b2a73f8e37ab4901b F src/wal.h d2a69695c84137f76e19a247a342cb02ab0131001b6f58153d94b71195bbd84d F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1182,7 +1182,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test f97a5613430a62ef5fb5aafeb37d4e857dea44e47c31064f154c7049c25b292b +F test/permutations.test c7090c0136224e2ca400f1e0a085dad13a4f40a03387881e26f1a4cd1344d557 F test/pg_common.tcl 301ac19c1a52fd55166d26db929b3b89165c634d52b5f8ad76ea8cb06960db30 F test/pragma.test c267bf02742c823a191960895b3d52933cebd7beee26757d1ed694f213fcd867 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -1597,7 +1597,9 @@ F test/vtab_shared.test 5253bff2355a9a3f014c15337da7e177ab0ef8ad F test/wal.test 613efec03e517e1775d86b993a54877d2e29a477 F test/wal2.test 155b9efa999bdb38ce1cd729b9a4fcdbffd6b88be27f039bad1d2929d287d918 F test/wal2big.test 0b4ec526f9ca4bbabc355042c38045ae2e253fb46eb327bb7693d0122bc6968b -F test/wal2recover.test 64155a1f13ac434ebb374f109b9daa2b52c3ca69db9199be0bcbabb9727cae4e +F test/wal2lock.test 0ef98d72dc6bcf7711dedd684760488400d9a9a6eec0dc5d3822060437793552 +F test/wal2recover.test da390d40c955a74b8a049d41248f92f928a5a5ee6bbcca33cb665c95b736c11c +F test/wal2recover2.test 98749381c2e61574e181a2e288295a3bf93d12f57769db50ecea7fc211d9ad5c F test/wal2rewrite.test 6ca6f631ffcf871240beab5f02608913fd075c6d0d31310b026c8383c65c9f9c F test/wal2savepoint.test 29725383e7eb8b78fed46333d69dc45b4ed321db43ecc81766aadea991874bef F test/wal2simple.test 96206c98bf64ab20ec00a1c0f6c709e258b98b39f2149889361f31966ce5a703 @@ -1790,7 +1792,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 34f56f8a4239781071edf6317723d0a7333249551c2c1ce7ed39fc116d53d780 -R cabac54018fcd32b9b9c7e5679bc38f9 +P 54e628f902a5508badc8941ceda1bec12fe8f58969c91c670a37888ef2681880 +R e2c9e8287795c59cafc4b6dc4a5ee2a2 U dan -Z be9d783248b67e630efcf341b29cd59e +Z 964094b4a0f5d261b4d47067580a5ce8 diff --git a/manifest.uuid b/manifest.uuid index 0fc73c3fc5..c6c9f8cdaa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54e628f902a5508badc8941ceda1bec12fe8f58969c91c670a37888ef2681880 \ No newline at end of file +9cb5f8dab685f5ea36ad142cfa588dee82e87f3a89e8dcf84e0ee124bb29bc7f \ No newline at end of file diff --git a/src/test_hexio.c b/src/test_hexio.c index 7b62ea08bc..1ffedec7f0 100644 --- a/src/test_hexio.c +++ b/src/test_hexio.c @@ -190,7 +190,7 @@ static int SQLITE_TCLAPI hexio_write( } /* -** USAGE: hexio_get_int HEXDATA +** USAGE: hexio_get_int [-littleendian] HEXDATA ** ** Interpret the HEXDATA argument as a big-endian integer. Return ** the value of that integer. HEXDATA can contain between 2 and 8 @@ -207,12 +207,20 @@ static int SQLITE_TCLAPI hexio_get_int( const unsigned char *zIn; unsigned char *aOut; unsigned char aNum[4]; + int bLittle = 0; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "HEXDATA"); + if( objc==3 ){ + int n; + char *z = Tcl_GetStringFromObj(objv[1], &n); + if( n>=2 && n<=13 && memcmp(z, "-littleendian", n)==0 ){ + bLittle = 1; + } + } + if( (objc-bLittle)!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "[-littleendian] HEXDATA"); return TCL_ERROR; } - zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1], &nIn); + zIn = (const unsigned char *)Tcl_GetStringFromObj(objv[1+bLittle], &nIn); aOut = sqlite3_malloc( nIn/2 ); if( aOut==0 ){ return TCL_ERROR; @@ -225,7 +233,11 @@ static int SQLITE_TCLAPI hexio_get_int( memcpy(&aNum[4-nOut], aOut, nOut); } sqlite3_free(aOut); - val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3]; + if( bLittle ){ + val = (aNum[3]<<24) | (aNum[2]<<16) | (aNum[1]<<8) | aNum[0]; + }else{ + val = (aNum[0]<<24) | (aNum[1]<<16) | (aNum[2]<<8) | aNum[3]; + } Tcl_SetObjResult(interp, Tcl_NewIntObj(val)); return TCL_OK; } diff --git a/src/wal.c b/src/wal.c index 4ce5a67585..6200d59c9a 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3109,7 +3109,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ rc = walLockShared(pWal, WAL_READ_LOCK(eLock)); if( rc!=SQLITE_OK ){ - return rc; + return (rc==SQLITE_BUSY ? WAL_RETRY : rc); } walShmBarrier(pWal); if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ diff --git a/test/permutations.test b/test/permutations.test index 75f6ecbc02..4530baf88b 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -439,7 +439,7 @@ walfault.test walhook.test walmode.test walnoshm.test waloverwrite.test walpersist.test walprotocol2.test walprotocol.test walro2.test walrofault.test walro.test walshared.test walslow.test wal.test -wal2savepoint.test +wal2savepoint.test wal2lock.test wal2recover2.test } test_suite "coverage-pager" -description { diff --git a/test/wal2lock.test b/test/wal2lock.test new file mode 100644 index 0000000000..f86cf87925 --- /dev/null +++ b/test/wal2lock.test @@ -0,0 +1,106 @@ +# 2018 December 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2lock +ifcapable !wal {finish_test ; return } + +db close +testvfs tvfs +sqlite3 db test.db -vfs tvfs + +do_execsql_test 1.0 { + PRAGMA journal_mode = wal2; + CREATE TABLE y1(y, yy); + CREATE INDEX y1y ON y1(y); + CREATE INDEX y1yy ON y1(yy); + INSERT INTO y1 VALUES(1, 2), (3, 4), (5, 6); +} {wal2} + +tvfs script vfs_callback +tvfs filter xShmLock + +set ::lock [list] +proc vfs_callback {func file name lock} { + lappend ::lock $lock + return SQLITE_OK +} + +do_execsql_test 1.1.1 { + SELECT * FROM y1 +} {1 2 3 4 5 6} +do_test 1.1.2 { + set ::lock +} {{4 1 lock shared} {4 1 unlock shared}} + +set ::bFirst 1 +proc vfs_callback {func file name lock} { + if {$::bFirst} { + set ::bFirst 0 + return SQLITE_BUSY + } + return SQLITE_OK +} +do_execsql_test 1.2 { + SELECT * FROM y1 +} {1 2 3 4 5 6} + +set ::bFirst 1 +proc vfs_callback {func file name lock} { + if {$::bFirst} { + set ::bFirst 0 + return SQLITE_IOERR + } + return SQLITE_OK +} +do_catchsql_test 1.3 { + SELECT * FROM y1 +} {1 {disk I/O error}} + +puts "# Warning: This next test case causes SQLite to call xSleep(1) 100 times." +puts "# Normally this equates to a delay of roughly 10 seconds, but if SQLite" +puts "# is built on unix without HAVE_USLEEP defined, it may be much longer." +proc vfs_callback {func file name lock} { return SQLITE_BUSY } +do_catchsql_test 1.4 { + SELECT * FROM y1 +} {1 {locking protocol}} +proc vfs_callback {func file name lock} { return SQLITE_OK } + +sqlite3 db2 test.db -vfs tvfs +set ::bFirst 1 + +proc vfs_callback {func file name lock} { + if {$::bFirst} { + set ::bFirst 0 + db2 eval { INSERT INTO y1 VALUES(7, 8) } + } +} + +do_execsql_test 1.5.1 { + SELECT * FROM y1 +} {1 2 3 4 5 6 7 8} +do_execsql_test 1.5.2 { + SELECT * FROM y1 +} {1 2 3 4 5 6 7 8} + +db close +db2 close +tvfs delete +finish_test diff --git a/test/wal2recover.test b/test/wal2recover.test index e8cecc1bf7..8fd08b8905 100644 --- a/test/wal2recover.test +++ b/test/wal2recover.test @@ -227,40 +227,6 @@ do_test 3.5 { list [file size test.db] [file size test.db-wal] [file size test.db-wal2] } {5120 15752 18896} -#------------------------------------------------------------------------- -# -reset_db -do_execsql_test 4.0 { - PRAGMA journal_mode = wal2; - CREATE TABLE xyz(x, y, z); - INSERT INTO xyz VALUES('x', 'y', 'z'); -} {wal2} -db close -do_test 4.1 { - close [open test.db-wal w] - file mkdir test.db-wal2 - sqlite3 db test.db - catchsql { SELECT * FROM xyz } -} {1 {unable to open database file}} -db close -file delete test.db-wal2 - -do_test 4.2 { - sqlite3 db test.db - execsql { - INSERT INTO xyz VALUES('a', 'b', 'c'); - } - forcecopy test.db test.db2 - forcecopy test.db-wal test.db2-wal - forcedelete test.db2-wal2 - file mkdir test.db2-wal2 - sqlite3 db2 test.db2 - catchsql { SELECT * FROM xyz } db2 -} {1 {unable to open database file}} -db2 close -file delete test.db2-wal2 - - finish_test diff --git a/test/wal2recover2.test b/test/wal2recover2.test new file mode 100644 index 0000000000..11804c34a3 --- /dev/null +++ b/test/wal2recover2.test @@ -0,0 +1,227 @@ +# 2018 December 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL2" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/malloc_common.tcl +source $testdir/wal_common.tcl + +set testprefix wal2recover2 +ifcapable !wal {finish_test ; return } + +do_execsql_test 1.0 { + CREATE TABLE t1(x); + CREATE TABLE t2(x); + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 ) + INSERT INTO t1 SELECT i FROM s; + WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<1500 ) + INSERT INTO t2 SELECT i FROM s; + + PRAGMA journal_mode = wal2; + PRAGMA journal_size_limit = 10000; +} {wal2 10000} + +set ::L 1125750 +set ::M 1126500 +set ::H 1127250 + +do_execsql_test 1.1 { + UPDATE t1 SET x=x+1; + UPDATE t2 SET x=x+1 WHERE rowid<=750; + + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; +} [list $H $M] + +do_test 1.2 { + list [file size test.db] [file size test.db-wal] [file size test.db-wal2] +} {31744 14704 7368} + +proc cksum {zIn data} { + if {[string length $zIn]==0} { + set s0 0 + set s1 0 + } else { + set s0 [hexio_get_int [string range $zIn 0 7]] + set s1 [hexio_get_int [string range $zIn 8 15]] + } + set n [expr [string length $data] / 8] + + for {set i 0} {$i < $n} {incr i 2} { + set x0 [hexio_get_int -l [string range $data [expr $i*8] [expr $i*8+7]]] + set x1 [hexio_get_int -l [string range $data [expr $i*8+8] [expr $i*8+8+7]]] + + set s0 [expr ($s0 + $x0 + $s1) & 0xFFFFFFFF] + set s1 [expr ($s1 + $x1 + $s0) & 0xFFFFFFFF] + } + + return "[hexio_render_int32 $s0][hexio_render_int32 $s1]" +} + +proc fix_wal_cksums {file} { + # Fix the checksum on the wal header. + set data [hexio_read $file 0 32] + set cksum [cksum {} [string range $data 0 47]] + set salt [hexio_read $file 16 8] + hexio_write $file 24 $cksum + + # Fix the checksums for all pages in the wal file. + set pgsz [hexio_get_int [hexio_read $file 8 4]] + set sz [file size $file] + for {set off 32} {$off < $sz} {incr off [expr $pgsz+24]} { + set e [hexio_read $file $off 8] + set cksum [cksum $cksum $e] + + set p [hexio_read $file [expr $off+24] $pgsz] + set cksum [cksum $cksum $p] + + hexio_write $file [expr $off+8] $salt + hexio_write $file [expr $off+16] $cksum + } +} + +proc wal_incr_hdrfield {file field} { + switch -- $field { + nCkpt { set offset 12 } + salt0 { set offset 16 } + salt1 { set offset 20 } + default { + error "unknown field $field - should be \"nCkpt\", \"salt0\" or \"salt1\"" + } + } + + # Increment the value in the wal header. + set v [hexio_get_int [hexio_read $file $offset 4]] + incr v + hexio_write $file $offset [hexio_render_int32 $v] + + # Fix various checksums + fix_wal_cksums $file +} + +proc wal_set_nckpt {file val} { + # Increment the value in the wal header. + hexio_write $file 12 [hexio_render_int32 $val] + + # Fix various checksums + fix_wal_cksums $file +} + +proc wal_set_follow {file prevfile} { + set pgsz [hexio_get_int [hexio_read $prevfile 8 4]] + set sz [file size $prevfile] + set cksum [hexio_read $prevfile [expr $sz-$pgsz-8] 8] + + hexio_write $file 16 $cksum + fix_wal_cksums $file +} + +foreach {tn file field} { + 1 test.db2-wal salt0 + 2 test.db2-wal salt1 + 3 test.db2-wal nCkpt + 4 test.db2-wal2 salt0 + 5 test.db2-wal2 salt1 + 6 test.db2-wal2 nCkpt +} { + do_test 1.3.$tn { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + forcecopy test.db-wal2 test.db2-wal2 + wal_incr_hdrfield $file $field + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 + } [list $H $L] + db2 close +} + +do_test 1.4 { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcedelete test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 +} [list $L $M] + +do_test 1.5 { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcecopy test.db-wal test.db2-wal2 + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 +} [list $H $M] + +foreach {tn file field} { + 1 test.db2-wal salt0 + 2 test.db2-wal salt1 + 3 test.db2-wal2 salt0 + 4 test.db2-wal2 salt1 +} { + do_test 1.6.$tn { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcecopy test.db-wal test.db2-wal2 + wal_incr_hdrfield $file $field + breakpoint + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 + } [list $H $L] + db2 close +} + +foreach {tn nCkpt1 nCkpt2 res} [list \ + 1 2 1 "$H $M" \ + 2 2 2 "$L $M" \ + 3 3 1 "$H $L" \ + 4 15 14 "$H $M" \ + 5 0 15 "$H $M" \ + 6 1 15 "$L $M" \ +] { + do_test 1.7.$tn { + forcecopy test.db test.db2 + forcecopy test.db-wal2 test.db2-wal + forcecopy test.db-wal test.db2-wal2 + + wal_set_nckpt test.db2-wal2 $nCkpt2 + wal_set_nckpt test.db2-wal $nCkpt1 + wal_set_follow test.db2-wal test.db2-wal2 + + if {$tn==1} breakpoint + + sqlite3 db2 test.db2 + execsql { + SELECT sum(x) FROM t1; + SELECT sum(x) FROM t2; + } db2 + } $res + db2 close +} + + +finish_test +