From 04ab586bf3959da012113b399f09013afe553503 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 Dec 2018 21:13:41 +0000 Subject: [PATCH 01/15] Very slightly smaller and faster. FossilOrigin-Name: 27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/build.c | 2 +- src/insert.c | 1 + 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 85ce65b7a1..13139d928b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.26.0 -D 2018-12-01T12:34:55.966 +C Very\sslightly\ssmaller\sand\sfaster. +D 2018-12-01T21:13:41.707 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -450,7 +450,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c ba7c7eef4461790f37c309936bfc5d0d6ba9b194b02d3c8ff1fd53b420ea6d3b F src/btree.h febb2e817be499570b7a2e32a9bbb4b607a9234f6b84bb9ae84916d4806e96f2 F src/btreeInt.h 620ab4c7235f43572cf3ac2ac8723cbdf68073be4d29da24897c7b77dda5fd96 -F src/build.c 127d33ad57b455a9339e9fabff41284c8b030cc6247ca7a2a6c0ad7abfc1ce85 +F src/build.c fce47a9789704e65c63299b01be8153745faee7919f5137d3f29b7c3c0b549bd F src/callback.c 789bd33d188146f66c0dd8306472a72d1c05f71924b24a91caf6bd45cf9aba73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@ -467,7 +467,7 @@ F src/hash.c 931ec82d7e070654a8facb42549bbb3a25720171d73ba94c3d3160580d01ef1f F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 -F src/insert.c 6b81aae27b196925d8ff78824f4bbd435d6a40cd38dc324685e21735bb402109 +F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 9050dd153b5583804184be9c9dee9ebb554178d6db1f8ac280899e8aad9060e6 F src/main.c 4cfb3913cc9e65d3ac649b1785ac753fc225d29425d5437e012f7eac0cefe0eb @@ -1779,10 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 23684cb841ac2cb0d69e5470253bd96feb733762a7553b952a08470834fe85fa -R dcc394af10af6094607f56a6086fb748 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.26.0 * +P bf8c1b2b7a5960c282e543b9c293686dccff272512d08865f4600fb58238b4f9 +R 6e4818c911aa4ab70efb9a2cf2ccb395 U drh -Z 560a5decbb241d53eb97e72267b6e6e8 +Z 8e5b93098d46384d408591fc5bd49035 diff --git a/manifest.uuid b/manifest.uuid index fd08abb29b..a4307176ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf8c1b2b7a5960c282e543b9c293686dccff272512d08865f4600fb58238b4f9 \ No newline at end of file +27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b \ No newline at end of file diff --git a/src/build.c b/src/build.c index e0fed8a83f..ad1421d195 100644 --- a/src/build.c +++ b/src/build.c @@ -227,7 +227,7 @@ void sqlite3FinishCoding(Parse *pParse){ if( v && pParse->nErr==0 && !db->mallocFailed ){ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ - if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; + assert( pParse->pAinc==0 || pParse->nTab>0 ); sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; }else{ diff --git a/src/insert.c b/src/insert.c index 7a9413901f..0c036e494f 100644 --- a/src/insert.c +++ b/src/insert.c @@ -319,6 +319,7 @@ void sqlite3AutoincrementBegin(Parse *pParse){ aOp[7].p2 = memId+2; aOp[7].p1 = memId; aOp[10].p2 = memId; + if( pParse->nTab==0 ) pParse->nTab = 1; } } From fdc75c9f5326dc96c18664674ad00f382adf2177 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 2 Dec 2018 01:15:16 +0000 Subject: [PATCH 02/15] Omit a line of code that has no affect on the outcome. FossilOrigin-Name: 5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/resolve.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 13139d928b..58d3a04285 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Very\sslightly\ssmaller\sand\sfaster. -D 2018-12-01T21:13:41.707 +C Omit\sa\sline\sof\scode\sthat\shas\sno\saffect\son\sthe\soutcome. +D 2018-12-02T01:15:16.933 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -504,7 +504,7 @@ F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 4cfc44def0f0690ceaab8f6481f5d76284d7f9509aab6e218a679b4836a54614 +F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f F src/shell.c.in 482e23a370cbe5b0d4c73a0f0f5fce34f7caa08a14a8d75e12f0225c4e14915c @@ -1779,7 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bf8c1b2b7a5960c282e543b9c293686dccff272512d08865f4600fb58238b4f9 -R 6e4818c911aa4ab70efb9a2cf2ccb395 +P 27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b +R fc5314ff4e5a8de1b725ac2ced7d9c02 U drh -Z 8e5b93098d46384d408591fc5bd49035 +Z dc803f80871250c47b1ead21e4c26856 diff --git a/manifest.uuid b/manifest.uuid index a4307176ea..aac9cd9145 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b \ No newline at end of file +5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 0c7dfc0b25..effbe646fe 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -80,7 +80,7 @@ static void resolveAlias( if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } - ExprSetProperty(pDup, EP_Alias); +// ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, From 4a5cff73a730a65e7ee4a8f2d131d591e9bd74c5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Dec 2018 01:47:41 +0000 Subject: [PATCH 03/15] Fix a parser bug in the use of parentheses around table-valued functions. FossilOrigin-Name: 58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/parse.y | 6 ++++++ test/json101.test | 19 ++++++++++++++++++- test/tabfunc01.test | 5 +++++ 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 58d3a04285..2d4789227a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sa\sline\sof\scode\sthat\shas\sno\saffect\son\sthe\soutcome. -D 2018-12-02T01:15:16.933 +C Fix\sa\sparser\sbug\sin\sthe\suse\sof\sparentheses\saround\stable-valued\sfunctions. +D 2018-12-03T01:47:41.293 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -495,7 +495,7 @@ F src/os_win.c 85d9e532d0444ab6c16d7431490c2e279e282aa0917b0e988996b1ae0de5c5a0 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 75e0f3cfa3962c714f519f8a3d1e67ecca1c91de0e010a036b988e40ce9e4c73 F src/pager.h 217921e81eb5fe455caa5cda96061959706bcdd29ddb57166198645ef7822ac3 -F src/parse.y 6840fe7c0b5eb4dd25ee5d075213bc8255ed4c0678d71bfb6744d0520d91c179 +F src/parse.y 5cf85c2b9dfac38ac4e2bf2776484705186ce2eda8631e65cc0b04bf566c1173 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c bf9fcea656dce1cd2cca6b77a1d1d3552050d55a31c98bf0d9f405930a83bc95 @@ -1053,7 +1053,7 @@ F test/journal3.test c9c29883f5bf535ae82ae21c472df6263806a22e467b6db7cd0d6d54530 F test/jrnlmode.test a6693f2bed4541a21e703aaa37bb3e10de154130645952933b82b2dec0a8b539 F test/jrnlmode2.test 8759a1d4657c064637f8b079592651530db738419e1d649c6df7048cd724363d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/json101.test b40a9f5395d8e669b0bc3eb550ad2ae9e5ada01fbce23c446c2a30a305a6d575 +F test/json101.test 8f8977b00ba02f9a26c1d1f52f29f540f6d5eb162cbd5eb78bb805366d4ab26d F test/json102.test eeb54efa221e50b74a2d6fb9259963b48d7414dca3ce2fdfdeed45cb28487bc1 F test/json103.test aff6b7a4c17d5a20b487a7bc1a274bfdc63b829413bdfb83bedac42ec7f67e3b F test/json104.test 877d5845f6303899b7889ea5dd1bea99076e3100574d5c536082245c5805dcaa @@ -1343,7 +1343,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039 F test/syscall.test a39d9a36f852ae6e4800f861bc2f2e83f68bbc2112d9399931ecfadeabd2d69d F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04 -F test/tabfunc01.test 54300134f76db817685194d2f0e63e3fbf7380b45e0d426e00a9aee752497cfb +F test/tabfunc01.test 20e98ffe55f35d8d33fd834ca8bf9d4b637e560af8fcd00464b4154d90a4db45 F test/table.test eb3463b7add9f16a5bb836badf118cf391b809d09fdccd1f79684600d07ec132 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 @@ -1779,7 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 27798f17f567ad065f8a99effcb287bc241df7b450330ef890d192c70528e62b -R fc5314ff4e5a8de1b725ac2ced7d9c02 +P 5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 +R 0af9de2350de25f0ebcc1d0eef1387db U drh -Z dc803f80871250c47b1ead21e4c26856 +Z 15852f1b01d70da9fb801cb007d16bf9 diff --git a/manifest.uuid b/manifest.uuid index aac9cd9145..1d5b14e4de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 \ No newline at end of file +58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index b150c73e1f..3bb28ab5f5 100644 --- a/src/parse.y +++ b/src/parse.y @@ -664,6 +664,12 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; + if( pOld->fg.isTabFunc ){ + pNew->u1.pFuncArg = pOld->u1.pFuncArg; + pOld->u1.pFuncArg = 0; + pOld->fg.isTabFunc = 0; + pNew->fg.isTabFunc = 1; + } pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } diff --git a/test/json101.test b/test/json101.test index 9a93ee739f..534478df93 100644 --- a/test/json101.test +++ b/test/json101.test @@ -813,6 +813,23 @@ do_execsql_test json-14.170 { SELECT fullkey FROM json_tree('null'); } {$} - +# 2018-12-03 +# Make sure the table-valued functions contained within parentheses +# work correctly. +# +# Bug reported via private email. See TH3 for more information. +# +do_execsql_test json-15.100 { + SELECT * FROM JSON_EACH('{"a":1, "b":2}'); +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +do_execsql_test json-15.110 { + SELECT xyz.* FROM JSON_EACH('{"a":1, "b":2}') AS xyz; +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +do_execsql_test json-15.120 { + SELECT * FROM (JSON_EACH('{"a":1, "b":2}')); +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} +do_execsql_test json-15.130 { + SELECT xyz.* FROM (JSON_EACH('{"a":1, "b":2}')) AS xyz; +} {a 1 integer 1 2 {} {$.a} {$} b 2 integer 2 4 {} {$.b} {$}} finish_test diff --git a/test/tabfunc01.test b/test/tabfunc01.test index dfe3190b52..49f0df889e 100644 --- a/test/tabfunc01.test +++ b/test/tabfunc01.test @@ -125,6 +125,11 @@ do_execsql_test tabfunc01-4.3 { SELECT * FROM aux1.generate_series(1,4) } {1 2 3 4} +# 2018-12-03: Fix bug reported by by private email. +do_execsql_test tabfunc01-4.4 { + SELECT * FROM (generate_series(1,5,2)) AS x LIMIT 10; +} {1 3 5} + # The next series of tests is verifying that virtual table are able # to optimize the IN operator, even on terms that are not marked "omit". # When the generate_series virtual table is compiled for the testfixture, From de72d2a81a5911c7dcb5fb52ea489c94e336721f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Dec 2018 01:58:02 +0000 Subject: [PATCH 04/15] Remove two lines of unnecessary code, for a very small performance increase and size decrease. FossilOrigin-Name: 15824ccda0f110794a479b58fbf36082d8c383f34bae9dc0921d96547fb37869 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache1.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2d4789227a..2b309eede2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sparser\sbug\sin\sthe\suse\sof\sparentheses\saround\stable-valued\sfunctions. -D 2018-12-03T01:47:41.293 +C Remove\stwo\slines\sof\sunnecessary\scode,\sfor\sa\svery\ssmall\sperformance\sincrease\nand\ssize\sdecrease. +D 2018-12-03T01:58:02.016 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -498,7 +498,7 @@ F src/pager.h 217921e81eb5fe455caa5cda96061959706bcdd29ddb57166198645ef7822ac3 F src/parse.y 5cf85c2b9dfac38ac4e2bf2776484705186ce2eda8631e65cc0b04bf566c1173 F src/pcache.c 696a01f1a6370c1b50a09c15972bc3bee3333f8fcd1f2da8e9a76b1b062c59ee F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 -F src/pcache1.c bf9fcea656dce1cd2cca6b77a1d1d3552050d55a31c98bf0d9f405930a83bc95 +F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69 @@ -1779,7 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5d933aa659eb7a13f9ab44fe7762be292a1c3c75b957a3b7e0bc6188257b62f4 -R 0af9de2350de25f0ebcc1d0eef1387db +P 58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 +R a804252de7d9904d6bfbaabce768affa U drh -Z 15852f1b01d70da9fb801cb007d16bf9 +Z e3cffb3b13ba67932ad3575ad4d1aeec diff --git a/manifest.uuid b/manifest.uuid index 1d5b14e4de..d71f9abd0c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 \ No newline at end of file +15824ccda0f110794a479b58fbf36082d8c383f34bae9dc0921d96547fb37869 \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index 2880c2c5e6..6df0f15d13 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -102,6 +102,7 @@ struct PgHdr1 { PCache1 *pCache; /* Cache that currently owns this page */ PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ }; /* @@ -570,7 +571,8 @@ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; - pPage->pLruPrev = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */ assert( pPage->isAnchor==0 ); assert( pPage->pCache->pGroup->lru.isAnchor==1 ); pPage->pCache->nRecyclable--; @@ -908,8 +910,9 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; pPage->pCache = pCache; - pPage->pLruPrev = 0; pPage->pLruNext = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */ *(void **)pPage->page.pExtra = 0; pCache->apHash[h] = pPage; if( iKey>pCache->iMaxKey ){ @@ -1069,7 +1072,7 @@ static void pcache1Unpin( /* It is an error to call this function if the page is already ** part of the PGroup LRU list. */ - assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); + assert( pPage->pLruNext==0 ); assert( PAGE_IS_PINNED(pPage) ); if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){ From 8c53b4e7f61218a63d64878453a6441bb8fe5815 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 3 Dec 2018 14:58:07 +0000 Subject: [PATCH 05/15] Update the autoconf makefile for MSVC. FossilOrigin-Name: 675aba1f8b989cfd99370704ecb09031026dc3321cccad122ea91d816e02fdba --- autoconf/Makefile.msc | 2 ++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/autoconf/Makefile.msc b/autoconf/Makefile.msc index 270c83c230..a131d4c6e7 100644 --- a/autoconf/Makefile.msc +++ b/autoconf/Makefile.msc @@ -283,6 +283,7 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DBSTAT_VTAB=1 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_INTROSPECTION_PRAGMAS=1 +OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_DESERIALIZE=1 !ENDIF OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1 !ENDIF @@ -937,6 +938,7 @@ LIBRESOBJS = SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1 +SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_DESERIALIZE=1 !ENDIF diff --git a/manifest b/manifest index 2b309eede2..ea374024b8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\stwo\slines\sof\sunnecessary\scode,\sfor\sa\svery\ssmall\sperformance\sincrease\nand\ssize\sdecrease. -D 2018-12-03T01:58:02.016 +C Update\sthe\sautoconf\smakefile\sfor\sMSVC. +D 2018-12-03T14:58:07.611 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -14,7 +14,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F autoconf/INSTALL 83e4a25da9fd053c7b3665eaaaf7919707915903 F autoconf/Makefile.am e14b629addaa1ce372b72043f28f40de2e32b7e211b6e0fc18dbb87989197e40 F autoconf/Makefile.fallback 22fe523eb36dfce31e0f6349f782eb084e86a5620b2b0b4f84a2d6133f53f5ac -F autoconf/Makefile.msc 9e73d9abaadb7a4951ddd0e947c5c791770f23bb1e04bfa50b43c01bee0575f2 +F autoconf/Makefile.msc ee8ab2b6b5d712f68acc4c186e3fad7b831b87a2b118e0a123043ea8a2c8b016 F autoconf/README.first 6c4f34fe115ff55d4e8dbfa3cecf04a0188292f7 F autoconf/README.txt 4f04b0819303aabaa35fff5f7b257fb0c1ef95f1 F autoconf/configure.ac 308de24343e76ecfbe9a67f8fcd4c5216b790d230c5d9ce10210b7d5965d6192 @@ -1779,7 +1779,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 58a51123d1a6381cc67d3c64ba3468ec5a92c299ad6fd86de0b843d0ffafb846 -R a804252de7d9904d6bfbaabce768affa -U drh -Z e3cffb3b13ba67932ad3575ad4d1aeec +P 15824ccda0f110794a479b58fbf36082d8c383f34bae9dc0921d96547fb37869 +R 6917ee4bf0492fb65a72a63e44f83083 +U mistachkin +Z 0fd7a981eb53d48862b14fa362aff3ba diff --git a/manifest.uuid b/manifest.uuid index d71f9abd0c..65c9608a7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15824ccda0f110794a479b58fbf36082d8c383f34bae9dc0921d96547fb37869 \ No newline at end of file +675aba1f8b989cfd99370704ecb09031026dc3321cccad122ea91d816e02fdba \ No newline at end of file From e89feee5c3028dc1232bec97086cd1b407fb188a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Dec 2018 16:14:49 +0000 Subject: [PATCH 06/15] Add the "remove_diacritics=2" option to the unicode61 tokenizer in both FTS5 and FTS3/4. FossilOrigin-Name: 06177f3f114b5d804b84c27ac843740282e2176fdf0f7a999feda0e1b624adec --- ext/fts3/fts3_unicode.c | 13 +++--- ext/fts3/fts3_unicode2.c | 62 ++++++++++++++++++----------- ext/fts3/unicode/mkunicode.tcl | 36 +++++++++++------ ext/fts3/unicode/parseunicode.tcl | 31 +++++++++++++-- ext/fts5/fts5_tokenize.c | 20 +++++++--- ext/fts5/fts5_unicode2.c | 66 +++++++++++++++++++------------ ext/fts5/test/fts5tokenizer.test | 2 +- ext/fts5/test/fts5umlaut.test | 65 ++++++++++++++++++++++++++++++ ext/fts5/test/fts5unicode3.test | 22 ++++++++--- manifest | 30 +++++++------- manifest.uuid | 2 +- test/fts4umlaut.test | 65 ++++++++++++++++++++++++++++++ 12 files changed, 320 insertions(+), 94 deletions(-) create mode 100644 ext/fts5/test/fts5umlaut.test create mode 100644 test/fts4umlaut.test diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index dfb2680c50..c02ea3990c 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -82,7 +82,7 @@ typedef struct unicode_cursor unicode_cursor; struct unicode_tokenizer { sqlite3_tokenizer base; - int bRemoveDiacritic; + int eRemoveDiacritic; int nException; int *aiException; }; @@ -227,17 +227,20 @@ static int unicodeCreate( pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); if( pNew==NULL ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(unicode_tokenizer)); - pNew->bRemoveDiacritic = 1; + pNew->eRemoveDiacritic = 1; for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; + pNew->eRemoveDiacritic = 1; } else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ - pNew->bRemoveDiacritic = 0; + pNew->eRemoveDiacritic = 0; + } + else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){ + pNew->eRemoveDiacritic = 2; } else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); @@ -350,7 +353,7 @@ static int unicodeNext( /* Write the folded case of the last character read to the output */ zEnd = z; - iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic); + iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic); if( iOut ){ WRITE_UTF8(zOut, iOut); } diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index da7251ed0c..41027b2546 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -159,32 +159,47 @@ int sqlite3FtsUnicodeIsalnum(int c){ ** E"). The resuls of passing a codepoint that corresponds to an ** uppercase letter are undefined. */ -static int remove_diacritic(int c){ +static int remove_diacritic(int c, int bComplex){ unsigned short aDia[] = { 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, }; char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', + '\0', 'a'|0x00, 'c'|0x00, 'e'|0x00, 'i'|0x00, 'n'|0x00, + 'o'|0x00, 'u'|0x00, 'y'|0x00, 'y'|0x00, 'a'|0x00, 'c'|0x00, + 'd'|0x00, 'e'|0x00, 'e'|0x00, 'g'|0x00, 'h'|0x00, 'i'|0x00, + 'j'|0x00, 'k'|0x00, 'l'|0x00, 'n'|0x00, 'o'|0x00, 'r'|0x00, + 's'|0x00, 't'|0x00, 'u'|0x00, 'u'|0x00, 'w'|0x00, 'y'|0x00, + 'z'|0x00, 'o'|0x00, 'u'|0x00, 'a'|0x00, 'i'|0x00, 'o'|0x00, + 'u'|0x00, 'u'|0x80, 'a'|0x80, 'g'|0x00, 'k'|0x00, 'o'|0x00, + 'o'|0x80, 'j'|0x00, 'g'|0x00, 'n'|0x00, 'a'|0x80, 'a'|0x00, + 'e'|0x00, 'i'|0x00, 'o'|0x00, 'r'|0x00, 'u'|0x00, 's'|0x00, + 't'|0x00, 'h'|0x00, 'a'|0x00, 'e'|0x00, 'o'|0x80, 'o'|0x00, + 'o'|0x80, 'y'|0x00, '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a'|0x00, 'b'|0x00, + 'c'|0x80, 'd'|0x00, 'd'|0x00, 'e'|0x80, 'e'|0x00, 'e'|0x80, + 'f'|0x00, 'g'|0x00, 'h'|0x00, 'h'|0x00, 'i'|0x00, 'i'|0x80, + 'k'|0x00, 'l'|0x00, 'l'|0x80, 'l'|0x00, 'm'|0x00, 'n'|0x00, + 'o'|0x80, 'p'|0x00, 'r'|0x00, 'r'|0x80, 'r'|0x00, 's'|0x00, + 's'|0x80, 't'|0x00, 'u'|0x00, 'u'|0x80, 'v'|0x00, 'w'|0x00, + 'w'|0x00, 'x'|0x00, 'y'|0x00, 'z'|0x00, 'h'|0x00, 't'|0x00, + 'w'|0x00, 'y'|0x00, 'a'|0x00, 'a'|0x80, 'a'|0x80, 'a'|0x80, + 'e'|0x00, 'e'|0x80, 'e'|0x80, 'i'|0x00, 'o'|0x00, 'o'|0x80, + 'o'|0x80, 'o'|0x80, 'u'|0x00, 'u'|0x80, 'u'|0x80, 'y'|0x00, }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; @@ -201,7 +216,8 @@ static int remove_diacritic(int c){ } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); } @@ -228,7 +244,7 @@ int sqlite3FtsUnicodeIsdiacritic(int c){ ** The results are undefined if the value passed to this function ** is less than zero. */ -int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ +int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ /* Each entry in the following array defines a rule for folding a range ** of codepoints to lower case. The rule applies to a range of nRange ** codepoints starting at codepoint iCode. @@ -351,7 +367,9 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ assert( ret>0 ); } - if( bRemoveDiacritic ) ret = remove_diacritic(ret); + if( eRemoveDiacritic ){ + ret = remove_diacritic(ret, eRemoveDiacritic==2); + } } else if( c>=66560 && c<66600 ){ diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 84b8ddc80b..36663beebd 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -9,11 +9,12 @@ proc print_rd {map} { set nRange 1 set iFirst [lindex $map 0 0] set cPrev [lindex $map 0 1] + set fPrev [lindex $map 0 2] foreach m [lrange $map 1 end] { - foreach {i c} $m {} + foreach {i c f} $m {} - if {$cPrev == $c} { + if {$cPrev == $c && $fPrev==$f} { for {set j [expr $iFirst+$nRange]} {$j<$i} {incr j} { if {[info exists tl_lookup_table($j)]==0} break } @@ -29,13 +30,16 @@ proc print_rd {map} { lappend lRange [list $iFirst $nRange] lappend aChar $cPrev + lappend aFlag $fPrev set iFirst $i set cPrev $c + set fPrev $f set nRange 1 } lappend lRange [list $iFirst $nRange] lappend aChar $cPrev + lappend aFlag $fPrev puts "/*" puts "** If the argument is a codepoint corresponding to a lowercase letter" @@ -45,7 +49,7 @@ proc print_rd {map} { puts "** E\"). The resuls of passing a codepoint that corresponds to an" puts "** uppercase letter are undefined." puts "*/" - puts "static int ${::remove_diacritic}(int c)\{" + puts "static int ${::remove_diacritic}(int c, int bComplex)\{" puts " unsigned short aDia\[\] = \{" puts -nonewline " 0, " set i 1 @@ -60,13 +64,17 @@ proc print_rd {map} { puts "" puts " \};" puts " char aChar\[\] = \{" - puts -nonewline " '\\0', " + puts -nonewline " '\\0', " set i 1 - foreach c $aChar { - set str "'$c', " - if {$c == ""} { set str "'\\0', " } + foreach c $aChar f $aFlag { + if { $f } { + set str "'$c'|0x80, " + } else { + set str "'$c'|0x00, " + } + if {$c == ""} { set str "'\\0', " } - if {($i % 12)==0} {puts "" ; puts -nonewline " " } + if {($i % 6)==0} {puts "" ; puts -nonewline " " } incr i puts -nonewline "$str" } @@ -87,7 +95,8 @@ proc print_rd {map} { } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);} + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);} puts "\}" } @@ -95,7 +104,8 @@ proc print_isdiacritic {zFunc map} { set lCode [list] foreach m $map { - foreach {code char} $m {} + foreach {code char flag} $m {} + if {$flag} continue if {$code && $char == ""} { lappend lCode $code } } set lCode [lsort -integer $lCode] @@ -472,7 +482,7 @@ proc print_fold {zFunc} { puts "** The results are undefined if the value passed to this function" puts "** is less than zero." puts "*/" - puts "int ${zFunc}\(int c, int bRemoveDiacritic)\{" + puts "int ${zFunc}\(int c, int eRemoveDiacritic)\{" set liOff [tl_generate_ioff_table $lRecord] tl_print_table_header @@ -516,7 +526,9 @@ proc print_fold {zFunc} { assert( ret>0 ); } - if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret); + if( eRemoveDiacritic ){ + ret = ${::remove_diacritic}(ret, eRemoveDiacritic==2); + } } }] diff --git a/ext/fts3/unicode/parseunicode.tcl b/ext/fts3/unicode/parseunicode.tcl index 966d7bdd3a..7c246a4a09 100644 --- a/ext/fts3/unicode/parseunicode.tcl +++ b/ext/fts3/unicode/parseunicode.tcl @@ -7,12 +7,24 @@ # character that it should be replaced with, or an empty string if the # codepoint should simply be removed from the input. Examples: # -# { 224 a } (replace codepoint 224 to "a") -# { 769 "" } (remove codepoint 769 from input) +# { 224 a 0 } (replace codepoint 224 to "a") +# { 769 "" 0 } (remove codepoint 769 from input) # # Mappings are only returned for non-upper case codepoints. It is assumed # that the input has already been folded to lower case. # +# The third value in the list is always either 0 or 1. 0 if the +# UnicodeData.txt file maps the codepoint to a single ASCII character and +# a diacritic, or 1 if the mapping is indirect. For example, consider the +# two entries: +# +# 1ECD;LATIN SMALL LETTER O WITH DOT BELOW;Ll;0;L;006F 0323;;;;N;;;1ECC;;1ECC +# 1ED9;LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW;Ll;0;L;1ECD 0302;;;;N;;;1ED8;;1ED8 +# +# The first codepoint is a direct mapping (as 006F is ASCII and 0323 is a +# diacritic). The second is an indirect mapping, as it maps to the +# first codepoint plus 0302 (a diacritic). +# proc rd_load_unicodedata_text {zName} { global tl_lookup_table @@ -53,18 +65,29 @@ proc rd_load_unicodedata_text {zName} { set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"] set iDia [expr "0x[lindex $character_decomposition_mapping 1]"] + # Filter out upper-case characters, as they will be mapped to their + # lower-case equivalents before this data is used. if {[info exists tl_lookup_table($iCode)]} continue + # Check if this is an indirect mapping. If so, set bIndirect to true + # and change $iAscii to the indirectly mappped ASCII character. + set bIndirect 0 + if {[info exists dia($iDia)] && [info exists mapping($iAscii)]} { + set iAscii $mapping($iAscii) + set bIndirect 1 + } + if { ($iAscii >= 97 && $iAscii <= 122) || ($iAscii >= 65 && $iAscii <= 90) } { - lappend lRet [list $iCode [string tolower [format %c $iAscii]]] + lappend lRet [list $iCode [string tolower [format %c $iAscii]] $bIndirect] + set mapping($iCode) $iAscii set dia($iDia) 1 } } foreach d [array names dia] { - lappend lRet [list $d ""] + lappend lRet [list $d "" 0] } set lRet [lsort -integer -index 0 $lRet] diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index af2bc222f2..35526a3898 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -234,13 +234,18 @@ struct Unicode61Tokenizer { unsigned char aTokenChar[128]; /* ASCII range token characters */ char *aFold; /* Buffer to fold text into */ int nFold; /* Size of aFold[] in bytes */ - int bRemoveDiacritic; /* True if remove_diacritics=1 is set */ + int eRemoveDiacritic; /* True if remove_diacritics=1 is set */ int nException; int *aiException; unsigned char aCategory[32]; /* True for token char categories */ }; +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */ +#define FTS5_REMOVE_DIACRITICS_NONE 0 +#define FTS5_REMOVE_DIACRITICS_SIMPLE 1 +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2 + static int fts5UnicodeAddExceptions( Unicode61Tokenizer *p, /* Tokenizer object */ const char *z, /* Characters to treat as exceptions */ @@ -361,7 +366,7 @@ static int fts5UnicodeCreate( int i; memset(p, 0, sizeof(Unicode61Tokenizer)); - p->bRemoveDiacritic = 1; + p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE; p->nFold = 64; p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); if( p->aFold==0 ){ @@ -382,10 +387,15 @@ static int fts5UnicodeCreate( for(i=0; rc==SQLITE_OK && ieRemoveDiacritic = (zArg[0] - '0'); + assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX + ); } - p->bRemoveDiacritic = (zArg[0]=='1'); }else if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ rc = fts5UnicodeAddExceptions(p, zArg, 1); @@ -499,7 +509,7 @@ static int fts5UnicodeTokenize( READ_UTF8(zCsr, zTerm, iCode); if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){ non_ascii_tokenchar: - iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic); + iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic); if( iCode ) WRITE_UTF8(zOut, iCode); }else{ break; diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c index 8c48aaa49b..985b37cc68 100644 --- a/ext/fts5/fts5_unicode2.c +++ b/ext/fts5/fts5_unicode2.c @@ -28,32 +28,47 @@ ** E"). The resuls of passing a codepoint that corresponds to an ** uppercase letter are undefined. */ -static int fts5_remove_diacritic(int c){ +static int fts5_remove_diacritic(int c, int bComplex){ unsigned short aDia[] = { 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, }; char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', + '\0', 'a'|0x00, 'c'|0x00, 'e'|0x00, 'i'|0x00, 'n'|0x00, + 'o'|0x00, 'u'|0x00, 'y'|0x00, 'y'|0x00, 'a'|0x00, 'c'|0x00, + 'd'|0x00, 'e'|0x00, 'e'|0x00, 'g'|0x00, 'h'|0x00, 'i'|0x00, + 'j'|0x00, 'k'|0x00, 'l'|0x00, 'n'|0x00, 'o'|0x00, 'r'|0x00, + 's'|0x00, 't'|0x00, 'u'|0x00, 'u'|0x00, 'w'|0x00, 'y'|0x00, + 'z'|0x00, 'o'|0x00, 'u'|0x00, 'a'|0x00, 'i'|0x00, 'o'|0x00, + 'u'|0x00, 'u'|0x80, 'a'|0x80, 'g'|0x00, 'k'|0x00, 'o'|0x00, + 'o'|0x80, 'j'|0x00, 'g'|0x00, 'n'|0x00, 'a'|0x80, 'a'|0x00, + 'e'|0x00, 'i'|0x00, 'o'|0x00, 'r'|0x00, 'u'|0x00, 's'|0x00, + 't'|0x00, 'h'|0x00, 'a'|0x00, 'e'|0x00, 'o'|0x80, 'o'|0x00, + 'o'|0x80, 'y'|0x00, '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a'|0x00, 'b'|0x00, + 'c'|0x80, 'd'|0x00, 'd'|0x00, 'e'|0x80, 'e'|0x00, 'e'|0x80, + 'f'|0x00, 'g'|0x00, 'h'|0x00, 'h'|0x00, 'i'|0x00, 'i'|0x80, + 'k'|0x00, 'l'|0x00, 'l'|0x80, 'l'|0x00, 'm'|0x00, 'n'|0x00, + 'o'|0x80, 'p'|0x00, 'r'|0x00, 'r'|0x80, 'r'|0x00, 's'|0x00, + 's'|0x80, 't'|0x00, 'u'|0x00, 'u'|0x80, 'v'|0x00, 'w'|0x00, + 'w'|0x00, 'x'|0x00, 'y'|0x00, 'z'|0x00, 'h'|0x00, 't'|0x00, + 'w'|0x00, 'y'|0x00, 'a'|0x00, 'a'|0x80, 'a'|0x80, 'a'|0x80, + 'e'|0x00, 'e'|0x80, 'e'|0x80, 'i'|0x00, 'o'|0x00, 'o'|0x80, + 'o'|0x80, 'o'|0x80, 'u'|0x00, 'u'|0x80, 'u'|0x80, 'y'|0x00, }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; @@ -70,7 +85,8 @@ static int fts5_remove_diacritic(int c){ } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); } @@ -97,7 +113,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){ ** The results are undefined if the value passed to this function ** is less than zero. */ -int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ +int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){ /* Each entry in the following array defines a rule for folding a range ** of codepoints to lower case. The rule applies to a range of nRange ** codepoints starting at codepoint iCode. @@ -220,7 +236,9 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ assert( ret>0 ); } - if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret); + if( eRemoveDiacritic ){ + ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2); + } } else if( c>=66560 && c<66600 ){ @@ -231,11 +249,9 @@ int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ } -#if 0 int sqlite3Fts5UnicodeNCat(void) { return 32; } -#endif int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ aArray[0] = 1; @@ -756,7 +772,7 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ]; int n = (aFts5UnicodeData[iTbl] >> 5) + i; for(; i<128 && i Date: Mon, 3 Dec 2018 17:40:46 +0000 Subject: [PATCH 07/15] Remove the unused sqlite3Fts5UnicodeNCat() function. FossilOrigin-Name: 7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 --- ext/fts3/unicode/mkunicode.tcl | 4 ---- ext/fts5/fts5_unicode2.c | 6 ------ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 36663beebd..8465262033 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -617,10 +617,6 @@ proc print_categories {lMap} { set nCat [expr [llength [array names C]] + 1] puts [code { - int sqlite3Fts5UnicodeNCat(void) { - return $nCat; - } - int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ aArray[0] = 1; switch( zCat[0] ){ diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c index 985b37cc68..9d3ffdc972 100644 --- a/ext/fts5/fts5_unicode2.c +++ b/ext/fts5/fts5_unicode2.c @@ -248,11 +248,6 @@ int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){ return ret; } - -int sqlite3Fts5UnicodeNCat(void) { - return 32; -} - int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ aArray[0] = 1; switch( zCat[0] ){ @@ -777,4 +772,3 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ iTbl++; } } - diff --git a/manifest b/manifest index 4fad7bcfe3..faaabc7b06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"remove_diacritics=2"\soption\sto\sthe\sunicode61\stokenizer\sin\sboth\sFTS5\nand\sFTS3/4. -D 2018-12-03T16:14:49.664 +C Remove\sthe\sunused\ssqlite3Fts5UnicodeNCat()\sfunction. +D 2018-12-03T17:40:46.225 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -105,7 +105,7 @@ F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87 F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl 106bb4ff6365b36301fa4a009e5b4bf6ed02a2fbe9156349be9dfd9a92697cde +F ext/fts3/unicode/mkunicode.tcl 2ea30d8122ccf1e33142c9cc8913d8cad9eb6668db359a228f10aeb37e2ab863 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 5edc74ca603d71284d475886e6e91b5c5cf2e8e93e9ba3a36ba2f2440ee97948 @@ -122,7 +122,7 @@ F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6 F ext/fts5/fts5_test_tok.c 80de1a4b1a3caa216c3be8862440f0117a8357dd9b7cfc5a2a2ce11fe6eb64ae F ext/fts5/fts5_tokenize.c ca2b6a033794945ac505241a86b0aa978709c23aa2e6121984d3e3ede96003c8 -F ext/fts5/fts5_unicode2.c 051f207a76a90890009a8b5009ca0c9a327342ec4d10c2145b61a334784b713a +F ext/fts5/fts5_unicode2.c 26519f47d567c5a9b08c376edd38f1b8a773b45ff29f1ffd81d9edfe20f96e18 F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738 F ext/fts5/fts5_vocab.c fbe38044889b2d2d99babeeef239c620fb0332bb928a84506ac748d81500b354 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 @@ -1781,7 +1781,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 675aba1f8b989cfd99370704ecb09031026dc3321cccad122ea91d816e02fdba -R 7123bc66be985a6ec633c5a7bea2ac49 -U dan -Z 3c4b639682109864205c768a98540aa1 +P 06177f3f114b5d804b84c27ac843740282e2176fdf0f7a999feda0e1b624adec +R 0096a940ee50b37b4b72306aa1bf1dc3 +U drh +Z 71af346a1a6924b681515df9ade7b33a diff --git a/manifest.uuid b/manifest.uuid index 3654f68d6b..a264e6b5c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06177f3f114b5d804b84c27ac843740282e2176fdf0f7a999feda0e1b624adec \ No newline at end of file +7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 \ No newline at end of file From 51883dfc8bcbba9f409d2d1efe8cadbd501000c3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Dec 2018 19:29:37 +0000 Subject: [PATCH 08/15] Cherrypick a couple of fixes from begin-concurrent-pnu into this branch. The differences between the two branches are now that this one does not have "PRAGMA noop_update" or the mutex-free PRNG. FossilOrigin-Name: a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 --- manifest | 19 +++++++++----- manifest.uuid | 2 +- src/btree.c | 50 +++++++++++++++++++++++++++++++++--- test/concurrent6.test | 60 +++++++++++++++++++++++++++++++++++++++++++ test/concurrent7.test | 52 +++++++++++++++++++++++++++++++++++++ 5 files changed, 172 insertions(+), 11 deletions(-) create mode 100644 test/concurrent6.test create mode 100644 test/concurrent7.test diff --git a/manifest b/manifest index eadf9a7db7..b00fa69f09 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bring\sup\sto\sdate\swith\sversion\s3.26.0. -D 2018-12-03T18:15:52.710 +C Cherrypick\sa\scouple\sof\sfixes\sfrom\sbegin-concurrent-pnu\sinto\sthis\sbranch.\sThe\ndifferences\sbetween\sthe\stwo\sbranches\sare\snow\sthat\sthis\sone\sdoes\snot\shave\n"PRAGMA\snoop_update"\sor\sthe\smutex-free\sPRNG. +D 2018-12-03T19:29:37.766 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -451,7 +451,7 @@ F src/auth.c 0fac71038875693a937e506bceb492c5f136dd7b1249fbd4ae70b4e8da14f9df F src/backup.c 78d3cecfbe28230a3a9a1793e2ead609f469be43e8f486ca996006be551857ab F src/bitvec.c 8433d9e98dd6f2ea3286e0d2fe5d65de1bfc18a706486eb2026b01be066b5806 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 4ad4c92bbd327b72cc7b3444de6b6d0f416e29ef0b21b2c3ec16165ba4070804 +F src/btree.c 4a2184be69d491d4b0228d4e397d67cb0802bbec06e7615b485ea1af69a131f6 F src/btree.h 1ed41c71481a1196a520064f2282bc13d768bbd8ae2850e319a3048f8ee7cb3d F src/btreeInt.h 6c65e6c96f561596f6870c79a64d4706af81613881d7947e3f063e923f14115f F src/build.c 5e04fb8528a4a915ed9af94b5cd068e53f2cfa9824d37464c4059279ea9bc8a0 @@ -735,6 +735,8 @@ F test/concurrent2.test 9dfbeb0a323733fe1d13443371734bb94a674dbf777f464365475903 F test/concurrent3.test 530671ac706f6a1d0f4992dbdd33a86408330d03cd90fb9e82ecb1b27f5fd081 F test/concurrent4.test e0b12cd467137e50259df3b4f837507e82aaa07c35941c88664dc8ed1d089c44 F test/concurrent5.test 0c16cbf7446af162a14e6def30445e94016064eb994e5aa4ebb2bebc59554176 +F test/concurrent6.test a7860e9ca13bb5fb76bcf41c5524fbfa9c37e6e258ecf84ffb5748a272488c67 +F test/concurrent7.test b96fa5c4cfdf8d5c0bc66b6934214500bad0260884a736f054ccc76e81aae85d F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1790,7 +1792,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 28a615a2e0f48b0fee3eaf7841ff902e069fa6c221df6ad9a57b8709c88561fb bf8c1b2b7a5960c282e543b9c293686dccff272512d08865f4600fb58238b4f9 -R ad7ac98b07a5e19a9d448dcba578a20a -U drh -Z e005195a2c11998dc5b1c6dec19b9447 +P f0ddb358cc68e5ec6d9e758893ab3da058a3b2e705124a7449279c992e672a4a +Q +50c8952c92b9f0c61935fb0df04ed1426d9e266a812071b7bf5b0215c5552757 +Q +570233716032f258b878d52c4d5a47e07292d66fa84e3a85c0388ec15efee625 +Q +dc0fc2aa7cbefeb5f0ba8c992fd3e9adcfb5a4d61e2321c1bd93f4d36ba9aafc +R 0b6ca72792de50021b48db0500a10b60 +U dan +Z e7051457303763dcda2174967ac468b5 diff --git a/manifest.uuid b/manifest.uuid index d0b8f54408..13eb6ed24f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0ddb358cc68e5ec6d9e758893ab3da058a3b2e705124a7449279c992e672a4a \ No newline at end of file +a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 828d63c180..f4f7365b92 100644 --- a/src/btree.c +++ b/src/btree.c @@ -656,11 +656,48 @@ static void btreePtrmapDelete(BtShared *pBt){ pBt->pMap = 0; } } + +/* +** Check that the pointer-map does not contain any entries with a parent +** page of 0. Call sqlite3_log() multiple times to output the entire +** data structure if it does. +*/ +static void btreePtrmapCheck(BtShared *pBt, Pgno nPage){ + Pgno i; + int bProblem = 0; + BtreePtrmap *p = pBt->pMap; + + for(i=p->iFirst; i<=nPage; i++){ + PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst]; + if( pEntry->eType==PTRMAP_OVERFLOW1 + || pEntry->eType==PTRMAP_OVERFLOW2 + || pEntry->eType==PTRMAP_BTREE + ){ + if( pEntry->parent==0 ){ + bProblem = 1; + break; + } + } + } + + if( bProblem ){ + for(i=p->iFirst; i<=nPage; i++){ + PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst]; + sqlite3_log(SQLITE_CORRUPT, + "btreePtrmapCheck: pgno=%d eType=%d parent=%d", + (int)i, (int)pEntry->eType, (int)pEntry->parent + ); + } + abort(); + } +} + #else /* SQLITE_OMIT_CONCURRENT */ # define btreePtrmapAllocate(x) SQLITE_OK # define btreePtrmapDelete(x) # define btreePtrmapBegin(x,y) SQLITE_OK # define btreePtrmapEnd(x,y,z) +# define btreePtrmapCheck(y,z) #endif /* SQLITE_OMIT_CONCURRENT */ static void releasePage(MemPage *pPage); /* Forward reference */ @@ -4152,7 +4189,10 @@ static int btreeRelocateRange( if( pEntry->eType==PTRMAP_FREEPAGE ){ Pgno dummy; rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT); - releasePage(pFree); + if( pFree ){ + assert( sqlite3PagerPageRefcount(pFree->pDbPage)==1 ); + sqlite3PcacheDrop(pFree->pDbPage); + } assert( rc!=SQLITE_OK || dummy==iPg ); }else if( pnCurrent ){ btreeGetPage(pBt, iPg, &pPg, 0); @@ -4210,6 +4250,8 @@ static int btreeFixUnlocked(Btree *p){ Pgno iHTrunk = get4byte(&p1[32]); u32 nHFree = get4byte(&p1[36]); + btreePtrmapCheck(pBt, nPage); + /* Attach the head database free list to the end of the current ** transactions free-list (if any). */ if( iTrunk!=0 ){ @@ -4236,10 +4278,12 @@ static int btreeFixUnlocked(Btree *p){ /* The current transaction allocated pages pMap->iFirst through ** nPage (inclusive) at the end of the database file. Meanwhile, ** other transactions have allocated (iFirst..nHPage). So move - ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */ + ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */ Pgno iLast = MIN(nPage, nHPage); /* Last page to move */ Pgno nCurrent; /* Current size of db */ + nCurrent = MAX(nPage, nHPage); + pBt->nPage = nCurrent; rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent); /* There are now no collisions with the snapshot at the head of the @@ -6229,7 +6273,7 @@ static int allocateBtreePage( ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); - if( ISCONCURRENT==0 && n>=mxPage ){ + if( n>=mxPage ){ return SQLITE_CORRUPT_BKPT; } diff --git a/test/concurrent6.test b/test/concurrent6.test new file mode 100644 index 0000000000..44718b7dbc --- /dev/null +++ b/test/concurrent6.test @@ -0,0 +1,60 @@ +# 2017 May 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set ::testprefix concurrent6 + +ifcapable !concurrent { + finish_test + return +} + +sqlite3 db2 test.db + +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + CREATE TABLE t2(x); + CREATE TABLE t3(x); + CREATE TABLE t4(x); + + INSERT INTO t1 VALUES(zeroblob(1500)); +} {wal} + +do_execsql_test -db db2 1.1 { + BEGIN CONCURRENT; + INSERT INTO t3 VALUES(zeroblob(4000)); + DELETE FROM t1; +} + +do_execsql_test 1.2 { + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t2 SELECT zeroblob(1000) FROM s; + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100) + INSERT INTO t4 SELECT zeroblob(1000) FROM s; + + DELETE FROM t4; +} + +do_execsql_test -db db2 1.3 { + COMMIT; +} + + +finish_test + diff --git a/test/concurrent7.test b/test/concurrent7.test new file mode 100644 index 0000000000..871e428031 --- /dev/null +++ b/test/concurrent7.test @@ -0,0 +1,52 @@ +# 2018 Jan 5 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix concurrent7 + +sqlite3 db2 test.db + +do_execsql_test 1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + CREATE TABLE t2(x); +} {wal} + +do_execsql_test -db db2 2 { + SELECT * FROM t1; +} + +do_execsql_test 3 { + BEGIN CONCURRENT; + INSERT INTO t1 VALUES(randomblob(1500)); + INSERT INTO t1 VALUES(randomblob(1500)); + DELETE FROM t1 WHERE rowid = 1; +} + +do_execsql_test -db db2 4 { + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + INSERT INTO t2 VALUES(randomblob(1500)); + DELETE FROM t2 WHERE rowid IN (1, 2); +} + +do_execsql_test 5 { + COMMIT; + PRAGMA integrity_check; +} {ok} + +finish_test + + From 834c48c279f8fc016acb0b79e356d3a7f98abc6d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Dec 2018 20:38:15 +0000 Subject: [PATCH 09/15] Minor change to wal.c on this branch to make it more similar to trunk. FossilOrigin-Name: 6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 --- manifest | 16 +++++------ manifest.uuid | 2 +- src/wal.c | 73 +++++++++++++++++++-------------------------------- 3 files changed, 35 insertions(+), 56 deletions(-) diff --git a/manifest b/manifest index b00fa69f09..801f2cffb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cherrypick\sa\scouple\sof\sfixes\sfrom\sbegin-concurrent-pnu\sinto\sthis\sbranch.\sThe\ndifferences\sbetween\sthe\stwo\sbranches\sare\snow\sthat\sthis\sone\sdoes\snot\shave\n"PRAGMA\snoop_update"\sor\sthe\smutex-free\sPRNG. -D 2018-12-03T19:29:37.766 +C Minor\schange\sto\swal.c\son\sthis\sbranch\sto\smake\sit\smore\ssimilar\sto\strunk. +D 2018-12-03T20:38:15.728 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -592,7 +592,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 41de67424237a3dd097a093057ba454ccfef26c452f6263fecd5591a15dd5a9a +F src/wal.c 8a12219e699ed737fa4dd8f0449f1bb5dee765502b48710dbd5be747ea58e65b F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1792,10 +1792,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f0ddb358cc68e5ec6d9e758893ab3da058a3b2e705124a7449279c992e672a4a -Q +50c8952c92b9f0c61935fb0df04ed1426d9e266a812071b7bf5b0215c5552757 -Q +570233716032f258b878d52c4d5a47e07292d66fa84e3a85c0388ec15efee625 -Q +dc0fc2aa7cbefeb5f0ba8c992fd3e9adcfb5a4d61e2321c1bd93f4d36ba9aafc -R 0b6ca72792de50021b48db0500a10b60 +P a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 +R d66e284ab90799b5d6b7d2957b4e83a3 +T +closed 0d12f49feb78a94a1b188e80379b51dfe9bf6c8e60225134e15216192cabed21 U dan -Z e7051457303763dcda2174967ac468b5 +Z 035c0f4c166e3fab14296a764b5a28cd diff --git a/manifest.uuid b/manifest.uuid index 13eb6ed24f..dd8dc8b081 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 \ No newline at end of file +6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 031389ea6f..673a159b19 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2854,19 +2854,37 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Search the hash tables for an entry matching page number pgno. Ignore -** any entries that lie after frame iLast within the wal file. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. +** +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -static int walFindFrame( - Wal *pWal, - Pgno pgno, - u32 iLast, - u32 *piRead +int sqlite3WalFindFrame( + Wal *pWal, /* WAL handle */ + Pgno pgno, /* Database page number to read data for */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ + u32 iRead = 0; /* If !=0, WAL frame to return data from */ + u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ - u32 iRead = 0; int iMinHash; + /* This routine is only be called from within a read transaction. */ + assert( pWal->readLock>=0 || pWal->lockError ); + + /* If the "last page" field of the wal-index header snapshot is 0, then + ** no data will be read from the wal under any circumstances. Return early + ** in this case as an optimization. Likewise, if pWal->readLock==0, + ** then the WAL is ignored by the reader so return early, as if the + ** WAL were empty. + */ + if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ + *piRead = 0; + return SQLITE_OK; + } + /* Each iteration of the following for() loop searches one ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). ** @@ -2917,43 +2935,6 @@ static int walFindFrame( if( iRead ) break; } - *piRead = iRead; - return SQLITE_OK; -} - -/* -** Search the wal file for page pgno. If found, set *piRead to the frame that -** contains the page. Otherwise, if pgno is not in the wal file, set *piRead -** to zero. -** -** Return SQLITE_OK if successful, or an error code if an error occurs. If an -** error does occur, the final value of *piRead is undefined. -*/ -int sqlite3WalFindFrame( - Wal *pWal, /* WAL handle */ - Pgno pgno, /* Database page number to read data for */ - u32 *piRead /* OUT: Frame number (or zero) */ -){ - u32 iRead = 0; /* If !=0, WAL frame to return data from */ - u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ - int rc; - - /* This routine is only be called from within a read transaction. */ - assert( pWal->readLock>=0 || pWal->lockError ); - - /* If the "last page" field of the wal-index header snapshot is 0, then - ** no data will be read from the wal under any circumstances. Return early - ** in this case as an optimization. Likewise, if pWal->readLock==0, - ** then the WAL is ignored by the reader so return early, as if the - ** WAL were empty. - */ - if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ - *piRead = 0; - return SQLITE_OK; - } - - rc = walFindFrame(pWal, pgno, iLast, &iRead); - #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* If expensive assert() statements are available, do a linear search ** of the wal-index file content. Make sure the results agree with the @@ -2973,7 +2954,7 @@ int sqlite3WalFindFrame( #endif *piRead = iRead; - return rc; + return SQLITE_OK; } /* From cfc45b1021d779cd3031bdfb8480173bb84f0e47 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Dec 2018 23:57:27 +0000 Subject: [PATCH 10/15] Reduce the size of the parser tables generated by Lemon by splitting the yyRuleInfo structure into separate yyRuleInfoLhs and yyRuleInfoNRhs arrays. FossilOrigin-Name: 70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/lemon.c | 11 +++++++++-- tool/lempar.c | 26 ++++++++++++++------------ 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index faaabc7b06..da9de6931b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sunused\ssqlite3Fts5UnicodeNCat()\sfunction. -D 2018-12-03T17:40:46.225 +C Reduce\sthe\ssize\sof\sthe\sparser\stables\sgenerated\sby\sLemon\sby\ssplitting\sthe\nyyRuleInfo\sstructure\sinto\sseparate\syyRuleInfoLhs\sand\syyRuleInfoNRhs\sarrays. +D 2018-12-03T23:57:27.083 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -1700,8 +1700,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c 60d1e1eb0f7ebae709f68f1472d77fbf291c5345cd98ff417219da7e74fd09e9 -F tool/lempar.c 452f12d40229847634a160e5666b6c4ec4392fd81941c3443861b48d497054cc +F tool/lemon.c c9ba01f6729c892ae3e0f55c8a2d694a7e6ec3dd3aa280b411a008ef69b410cd +F tool/lempar.c 61af95b8fac2bfd59c09d55330e78f3f5e352d7aa80bf37404b96ef795be3fdc F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1781,7 +1781,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 06177f3f114b5d804b84c27ac843740282e2176fdf0f7a999feda0e1b624adec -R 0096a940ee50b37b4b72306aa1bf1dc3 +P 7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 +R 457633af26ab8489c6f274ab3a7ad9fc U drh -Z 71af346a1a6924b681515df9ade7b33a +Z 7836770a12b5a3ad0791dca5d89f1095 diff --git a/manifest.uuid b/manifest.uuid index a264e6b5c4..30bc9f4cbf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 \ No newline at end of file +70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 1fca8b9755..7f0e557535 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -4590,13 +4590,20 @@ void ReportTable( tplt_print(out,lemp,lemp->overflow,&lineno); tplt_xfer(lemp->name,in,out,&lineno); - /* Generate the table of rule information + /* Generate the tables of rule information. yyRuleInfoLhs[] and + ** yyRuleInfoNRhs[]. ** ** Note: This code depends on the fact that rules are number ** sequentually beginning with 0. */ for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ - fprintf(out," { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i); + fprintf(out," %4d, /* (%d) ", rp->lhs->index, i); + rule_print(out, rp); + fprintf(out," */\n"); lineno++; + } + tplt_xfer(lemp->name,in,out,&lineno); + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + fprintf(out," %3d, /* (%d) ", -rp->nrhs, i); rule_print(out, rp); fprintf(out," */\n"); lineno++; } diff --git a/tool/lempar.c b/tool/lempar.c index 325b0e5418..94c0a3162a 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -686,13 +686,15 @@ static void yy_shift( yyTraceShift(yypParser, yyNewState, "Shift"); } -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - signed char nrhs; /* Negative of the number of RHS symbols in the rule */ -} yyRuleInfo[] = { +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { +%% +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { %% }; @@ -725,7 +727,7 @@ static YYACTIONTYPE yy_reduce( yymsp = yypParser->yytos; #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; + yysize = yyRuleInfoNRhs[yyruleno]; if( yysize ){ fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", yyTracePrompt, @@ -740,7 +742,7 @@ static YYACTIONTYPE yy_reduce( /* Check that the stack is large enough to grow by a single entry ** if the RHS of the rule is empty. This ensures that there is room ** enough on the stack to push the LHS value */ - if( yyRuleInfo[yyruleno].nrhs==0 ){ + if( yyRuleInfoNRhs[yyruleno]==0 ){ #ifdef YYTRACKMAXSTACKDEPTH if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ yypParser->yyhwm++; @@ -782,9 +784,9 @@ static YYACTIONTYPE yy_reduce( %% /********** End reduce actions ************************************************/ }; - assert( yyruleno Date: Tue, 4 Dec 2018 13:51:26 +0000 Subject: [PATCH 11/15] Small performance increase in sqlite3_step() for the common case where it returns SQLITE_ROW. FossilOrigin-Name: 893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 16 +++++++++------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index da9de6931b..0b06eaadfa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\ssize\sof\sthe\sparser\stables\sgenerated\sby\sLemon\sby\ssplitting\sthe\nyyRuleInfo\sstructure\sinto\sseparate\syyRuleInfoLhs\sand\syyRuleInfoNRhs\sarrays. -D 2018-12-03T23:57:27.083 +C Small\sperformance\sincrease\sin\ssqlite3_step()\sfor\sthe\scommon\scase\swhere\nit\sreturns\sSQLITE_ROW. +D 2018-12-04T13:51:26.090 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -581,7 +581,7 @@ F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 -F src/vdbeapi.c ecccfce6f614c33a95952efeec969d163e8349eac314ee2b7b163eda921b5eb0 +F src/vdbeapi.c 666993b7939530b3e16b21d07425809d4aaa5d75917ba34c513e91ee36c83467 F src/vdbeaux.c f547901b1aa9e2d81c63f06893f633648e434180666a827aacb547d7d6c8a601 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 @@ -1781,7 +1781,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7149dacf1d440a19f62808b4591c3fa8da202b2ec742d5490a63f2ec005ff9e7 -R 457633af26ab8489c6f274ab3a7ad9fc +P 70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c +R 86cab02da9395ff714960a3873202ed5 U drh -Z 7836770a12b5a3ad0791dca5d89f1095 +Z a2cf828add1aa9e207a374e8188d7cf1 diff --git a/manifest.uuid b/manifest.uuid index 30bc9f4cbf..16705cde8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c \ No newline at end of file +893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 59327bed38..a4f4baf61e 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -629,16 +629,18 @@ static int sqlite3Step(Vdbe *p){ db->nVdbeExec--; } + if( rc!=SQLITE_ROW ){ #ifndef SQLITE_OMIT_TRACE - /* If the statement completed successfully, invoke the profile callback */ - if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); + /* If the statement completed successfully, invoke the profile callback */ + checkProfileCallback(db, p); #endif - if( rc==SQLITE_DONE && db->autoCommit ){ - assert( p->rc==SQLITE_OK ); - p->rc = doWalCallbacks(db); - if( p->rc!=SQLITE_OK ){ - rc = SQLITE_ERROR; + if( rc==SQLITE_DONE && db->autoCommit ){ + assert( p->rc==SQLITE_OK ); + p->rc = doWalCallbacks(db); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + } } } From 50232dd821f419f3f0a667f0f9699e714e567641 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Dec 2018 13:51:43 +0000 Subject: [PATCH 12/15] Fix a problem with SQLITE_ENABLE_EXPENSIVE_ASSERT builds on this branch. FossilOrigin-Name: ddb4a6fbf8619db058b5eb8fcd687084ed4b65a6f69810357e324158257a911f --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 801f2cffb1..6115220691 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schange\sto\swal.c\son\sthis\sbranch\sto\smake\sit\smore\ssimilar\sto\strunk. -D 2018-12-03T20:38:15.728 +C Fix\sa\sproblem\swith\sSQLITE_ENABLE_EXPENSIVE_ASSERT\sbuilds\son\sthis\sbranch. +D 2018-12-04T13:51:43.293 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -592,7 +592,7 @@ F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 70188a745dc4e57d26e942681ff4b2912b7c8249ad5de3f60f0677b4337bcfaa F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 8a12219e699ed737fa4dd8f0449f1bb5dee765502b48710dbd5be747ea58e65b +F src/wal.c 6675d84d7bc5d04879e487d79248fb79d9bdad46811cc5ed5ef6233bdf1b70a1 F src/wal.h f325a5856b669f5ba449157485915816103857c8574efc746ac55eba3335c5e0 F src/walker.c fb94aadc9099ff9c6506d0a8b88d51266005bcaa265403f3d7caf732a562eb66 F src/where.c 3818e8a736a05d2cb194e64399af707e367fbcc5c251d785804d02eaf121288e @@ -1792,8 +1792,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a56506b9387a067ef259504d127694ad20223f4b08781d1676ff7f5fdd9443d8 -R d66e284ab90799b5d6b7d2957b4e83a3 -T +closed 0d12f49feb78a94a1b188e80379b51dfe9bf6c8e60225134e15216192cabed21 +P 6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 +R d464b4876361195a5c5bd9cc28bc9bdf U dan -Z 035c0f4c166e3fab14296a764b5a28cd +Z 0763a5ad9b6082127adcd5e3f801575e diff --git a/manifest.uuid b/manifest.uuid index dd8dc8b081..f3449cc2a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a7af3ead5949c461430c1fa92798dc2bbbc58c8cd504005c5afa38993f0be82 \ No newline at end of file +ddb4a6fbf8619db058b5eb8fcd687084ed4b65a6f69810357e324158257a911f \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 673a159b19..89f475d4d1 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2939,7 +2939,7 @@ int sqlite3WalFindFrame( /* If expensive assert() statements are available, do a linear search ** of the wal-index file content. Make sure the results agree with the ** result obtained using the hash indexes above. */ - if( rc==SQLITE_OK ){ + { u32 iRead2 = 0; u32 iTest; assert( pWal->bShmUnreliable || pWal->minFrame>0 ); From 04c6747a804862554811113524b8a70d3a0675e1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Dec 2018 14:33:02 +0000 Subject: [PATCH 13/15] Performance improvement in sqlite3_step() by creating a new mTrace flag for the legacy xProfile pointer that is set by sqlite3_profile(). FossilOrigin-Name: e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/main.c | 6 ++++++ src/sqliteInt.h | 9 +++++++-- src/vdbeapi.c | 6 ++++-- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 0b06eaadfa..b69e0f957a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\sincrease\sin\ssqlite3_step()\sfor\sthe\scommon\scase\swhere\nit\sreturns\sSQLITE_ROW. -D 2018-12-04T13:51:26.090 +C Performance\simprovement\sin\ssqlite3_step()\sby\screating\sa\snew\smTrace\sflag\nfor\sthe\slegacy\sxProfile\spointer\sthat\sis\sset\sby\ssqlite3_profile(). +D 2018-12-04T14:33:02.434 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c @@ -471,7 +471,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 9050dd153b5583804184be9c9dee9ebb554178d6db1f8ac280899e8aad9060e6 -F src/main.c 4cfb3913cc9e65d3ac649b1785ac753fc225d29425d5437e012f7eac0cefe0eb +F src/main.c 5a94791735ddd51804b75b7ef4645c25cbe6fe1d0e8f25851b2ed1211ad3d076 F src/malloc.c 07295435093ce354c6d9063ac05a2eeae28bd251d2e63c48b3d67c12c76f7e18 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -512,7 +512,7 @@ F src/shell.c.in 482e23a370cbe5b0d4c73a0f0f5fce34f7caa08a14a8d75e12f0225c4e14915 F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h 1161f7579cdd6217737a66517ef27f4016426603eff492e9b31f45a7d7d4c61f +F src/sqliteInt.h f54ee1ef1e0f99d27af20561df72aac9158ed420cfc3a2e330fdee40672daf37 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -581,7 +581,7 @@ F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 -F src/vdbeapi.c 666993b7939530b3e16b21d07425809d4aaa5d75917ba34c513e91ee36c83467 +F src/vdbeapi.c dc825a6ec99a5066c1aa0d9824509057c0510f03cc8c72f81ba074553f8a5ae8 F src/vdbeaux.c f547901b1aa9e2d81c63f06893f633648e434180666a827aacb547d7d6c8a601 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 @@ -1781,7 +1781,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 70fe8ec2ae3099b8773834c7ac2e56768addbecd57956ac523e71a7dc264049c -R 86cab02da9395ff714960a3873202ed5 +P 893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 +R 8ed134deac4c27b14c838ed0e6319d34 U drh -Z a2cf828add1aa9e207a374e8188d7cf1 +Z 84b5f74d4a6ae62e21d6700a38fd119f diff --git a/manifest.uuid b/manifest.uuid index 16705cde8c..07b135a092 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 \ No newline at end of file +e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e \ No newline at end of file diff --git a/src/main.c b/src/main.c index 46c83463c6..27a4206a4f 100644 --- a/src/main.c +++ b/src/main.c @@ -1996,6 +1996,7 @@ void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; db->xTrace = (int(*)(u32,void*,void*,void*))xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); @@ -2020,6 +2021,9 @@ int sqlite3_trace_v2( if( mTrace==0 ) xTrace = 0; if( xTrace==0 ) mTrace = 0; db->mTrace = mTrace; +#ifndef SQLITE_OMIT_DEPRECATED + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; +#endif db->xTrace = xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); @@ -2052,6 +2056,8 @@ void *sqlite3_profile( pOld = db->pProfileArg; db->xProfile = xProfile; db->pProfileArg = pArg; + db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK; + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; sqlite3_mutex_leave(db->mutex); return pOld; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 051aa40382..9f9d03a605 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1356,10 +1356,13 @@ void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing ** in the style of sqlite3_trace() */ -#define SQLITE_TRACE_LEGACY 0x80 +#define SQLITE_TRACE_LEGACY 0x40 /* Use the legacy xTrace */ +#define SQLITE_TRACE_XPROFILE 0x80 /* Use the legacy xProfile */ #else -#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_XPROFILE 0 #endif /* SQLITE_OMIT_DEPRECATED */ +#define SQLITE_TRACE_NONLEGACY_MASK 0x0f /* Normal flags */ /* @@ -1418,8 +1421,10 @@ struct sqlite3 { void **aExtension; /* Array of shared library handles */ int (*xTrace)(u32,void*,void*,void*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ +#ifndef SQLITE_OMIT_DEPRECATED void (*xProfile)(void*,const char*,u64); /* Profiling function */ void *pProfileArg; /* Argument to profile function */ +#endif void *pCommitArg; /* Argument to xCommitCallback() */ int (*xCommitCallback)(void*); /* Invoked at every commit. */ void *pRollbackArg; /* Argument to xRollbackCallback() */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index a4f4baf61e..3b040a5b01 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -62,14 +62,16 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ sqlite3_int64 iNow; sqlite3_int64 iElapse; assert( p->startTime>0 ); - assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 ); + assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 ); assert( db->init.busy==0 ); assert( p->zSql!=0 ); sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); iElapse = (iNow - p->startTime)*1000000; +#ifndef SQLITE_OMIT_DEPRECATED if( db->xProfile ){ db->xProfile(db->pProfileArg, p->zSql, iElapse); } +#endif if( db->mTrace & SQLITE_TRACE_PROFILE ){ db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse); } @@ -602,7 +604,7 @@ static int sqlite3Step(Vdbe *p){ ); #ifndef SQLITE_OMIT_TRACE - if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0) + if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 && !db->init.busy && p->zSql ){ sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); }else{ From 731dd6ebdaa4463286ed5843cd6d3bc6d506e117 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Dec 2018 16:51:42 +0000 Subject: [PATCH 14/15] Add the "index_usage" utility program. FossilOrigin-Name: df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 --- Makefile.in | 3 + Makefile.msc | 4 ++ main.mk | 4 ++ manifest | 17 ++--- manifest.uuid | 2 +- tool/index_usage.c | 164 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 185 insertions(+), 9 deletions(-) create mode 100644 tool/index_usage.c diff --git a/Makefile.in b/Makefile.in index d2f9710059..028f615cd7 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1306,6 +1306,9 @@ showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo showshm$(TEXE): $(TOP)/tool/showshm.c $(LTLINK) -o $@ $(TOP)/tool/showshm.c +index_usage$(TEXE): $(TOP)/tool/index_usage.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/index_usage.c sqlite3.lo $(TLIBS) + changeset$(TEXE): $(TOP)/ext/session/changeset.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/ext/session/changeset.c sqlite3.lo $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index e3feffab35..3b95d088d2 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2444,6 +2444,10 @@ showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(SQLITE3H) showshm.exe: $(TOP)\tool\showshm.c $(LTLINK) $(NO_WARN) $(TOP)\tool\showshm.c /link $(LDFLAGS) $(LTLINKOPTS) +index_usage.exe: $(TOP)\tool\index_usage.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ + $(TOP)\tool\index_usage.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + changeset.exe: $(TOP)\ext\session\changeset.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 \ diff --git a/main.mk b/main.mk index d18313bdc7..7a85619732 100644 --- a/main.mk +++ b/main.mk @@ -995,6 +995,10 @@ showwal$(EXE): $(TOP)/tool/showwal.c sqlite3.o showshm$(EXE): $(TOP)/tool/showshm.c $(TCC) -o showshm$(EXE) $(TOP)/tool/showshm.c +index_usage$(EXE): $(TOP)/tool/index_usage.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_DEPRECATED -o index_usage$(EXE) \ + $(TOP)/tool/index_usage.c sqlite3.o $(THREADLIB) + changeset$(EXE): $(TOP)/ext/session/changeset.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o changeset$(EXE) \ $(TOP)/ext/session/changeset.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index b69e0f957a..f8f7ebdbb6 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Performance\simprovement\sin\ssqlite3_step()\sby\screating\sa\snew\smTrace\sflag\nfor\sthe\slegacy\sxProfile\spointer\sthat\sis\sset\sby\ssqlite3_profile(). -D 2018-12-04T14:33:02.434 +C Add\sthe\s"index_usage"\sutility\sprogram. +D 2018-12-04T16:51:42.708 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea -F Makefile.in a050c8670ea0d7b37b2192306cbb50d392acd9902b84e9b56f3444d006f97a6c +F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 0d6831ff7951b302e888d86d4c469e2ec3c22f59eba4118b8c38d5a51d9e2d4f +F Makefile.msc b7d4a710fa3f0b8cfc532ff195b85dc1ba2a8ad34343cb3d67639f28f0a24306 F README.md 377233394b905d3b2e2b33741289e093bc93f2e7adbe00923b2c5958c9a9edee F VERSION 654da1d4053fb09ffc33a3910e6d427182a7dcdc67e934fa83de2849ac83fccb F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -429,7 +429,7 @@ F ext/userauth/userauth.c f81aa5a3ecacf406f170c62a144405858f6f6de51dbdc0920134e6 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk eeaa279fa6acdcfa6555058548075569a06f891fd67f5901b1e7700d18052fda +F main.mk 55f94164ecc194b067d9c55e106f37fd3c9b39f9668e8b568c98f008b6f9ec90 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1699,6 +1699,7 @@ F tool/fuzzershell.c e1d90a03ca790d7c331c2aae08ca46ff435f1ae1faa6cb9cc48f4687c18 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a5a4f F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce +F tool/index_usage.c 8fd515b97522ae4b1aa6ca9847439f005d4cbaf6eb0eb416b694dba77c0263f0 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c c9ba01f6729c892ae3e0f55c8a2d694a7e6ec3dd3aa280b411a008ef69b410cd F tool/lempar.c 61af95b8fac2bfd59c09d55330e78f3f5e352d7aa80bf37404b96ef795be3fdc @@ -1781,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 893448265299f4c70c32c8e92ea66f8d33c1c213b21701f73fa3815514cd5ef6 -R 8ed134deac4c27b14c838ed0e6319d34 +P e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e +R 14a419221d3a581ec01dad82979f03fc U drh -Z 84b5f74d4a6ae62e21d6700a38fd119f +Z 0158a53fd6ddbd828c2808dfe15d0a38 diff --git a/manifest.uuid b/manifest.uuid index 07b135a092..19384dd637 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e \ No newline at end of file +df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 \ No newline at end of file diff --git a/tool/index_usage.c b/tool/index_usage.c new file mode 100644 index 0000000000..a86202425e --- /dev/null +++ b/tool/index_usage.c @@ -0,0 +1,164 @@ +/* +** 2018-12-04 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a utility program used to help determine which +** indexes in a database schema are used and unused, and how often specific +** indexes are used. +*/ +#include "sqlite3.h" +#include +#include +#include +#include + +static void usage(const char *argv0){ + printf("Usage: %s DATABASE LOG\n\n", argv0); + printf( + "DATABASE is an SQLite database against which various statements\n" + "have been run. The SQL text is stored in LOG. LOG is an SQLite\n" + "database with this schema:\n" + "\n" + " CREATE TABLE sqllog(sql TEXT);\n" + "\n" + "This utility program analyzes statements contained in LOG and prints\n" + "a report showing how many times each index in DATABASE is used by the\n" + "statements in LOG.\n" + "\n" + "DATABASE only needs to contain the schema used by the statements in\n" + "LOG. The content can be removed from DATABASE.\n" + ); + printf("\nAnalysis will be done by SQLite version %s dated %.20s\n" + "checkin number %.40s. Different versions\n" + "of SQLite might use different indexes.\n", + sqlite3_libversion(), sqlite3_sourceid(), sqlite3_sourceid()+21); + exit(1); +} + +int main(int argc, char **argv){ + sqlite3 *db = 0; /* The main database */ + sqlite3_stmt *pStmt = 0; /* a query */ + char *zSql; + int nErr = 0; + int rc; + + if( argc!=3 ) usage(argv[0]); + rc = sqlite3_open_v2(argv[1], &db, SQLITE_OPEN_READONLY, 0); + if( rc ){ + printf("Cannot open \"%s\" for reading: %s\n", argv[1], sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_prepare_v2(db, "SELECT * FROM sqlite_master", -1, &pStmt, 0); + if( rc ){ + printf("Cannot read the schema from \"%s\" - %s\n", argv[1], + sqlite3_errmsg(db)); + goto errorOut; + } + sqlite3_finalize(pStmt); + pStmt = 0; + rc = sqlite3_exec(db, + "CREATE TABLE temp.idxu(\n" + " tbl TEXT,\n" + " idx TEXT,\n" + " cnt INT,\n" + " PRIMARY KEY(idx)\n" + ") WITHOUT ROWID;", 0, 0, 0); + if( rc ){ + printf("Cannot create the result table - %s\n", + sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_exec(db, + "INSERT INTO temp.idxu(tbl,idx,cnt)" + " SELECT tbl_name, name, 0 FROM sqlite_master" + " WHERE type='index' AND sql IS NOT NULL", 0, 0, 0); + + /* Open the LOG database */ + zSql = sqlite3_mprintf("ATTACH %Q AS log", argv[2]); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( rc ){ + printf("Cannot open the LOG database \"%s\" - %s\n", + argv[2], sqlite3_errmsg(db)); + goto errorOut; + } + rc = sqlite3_prepare_v2(db, "SELECT sql, rowid FROM log.sqllog", + -1, &pStmt, 0); + if( rc ){ + printf("Cannot read the SQLLOG table in the LOG database \"%s\" - %s\n", + argv[2], sqlite3_errmsg(db)); + goto errorOut; + } + + /* Update the counts based on LOG */ + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + const char *zLog = (const char*)sqlite3_column_text(pStmt, 0); + sqlite3_stmt *pS2; + if( zLog==0 ) continue; + zSql = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zLog); + rc = sqlite3_prepare_v2(db, zSql, -1, &pS2, 0); + sqlite3_free(zSql); + if( rc ){ + printf("Cannot compile LOG entry %d (%s): %s\n", + sqlite3_column_int(pStmt, 1), zLog, sqlite3_errmsg(db)); + nErr++; + }else{ + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zExplain = (const char*)sqlite3_column_text(pS2,3); + const char *z1, *z2; + int n; + /* printf("EXPLAIN: %s\n", zExplain); */ + z1 = strstr(zExplain, " USING INDEX "); + if( z1==0 ) continue; + z1 += 13; + for(z2=z1+1; z2[1] && z2[1]!='('; z2++){} + n = z2 - z1; + zSql = sqlite3_mprintf( + "UPDATE temp.idxu SET cnt=cnt+1 WHERE idx='%.*q'", n, z1 + ); + /* printf("sql: %s\n", zSql); */ + sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + } + sqlite3_finalize(pS2); + } + sqlite3_finalize(pStmt); + + /* Generate the report */ + rc = sqlite3_prepare_v2(db, + "SELECT tbl, idx, cnt, " + " (SELECT group_concat(name,',') FROM pragma_index_info(idx))" + " FROM temp.idxu, main.sqlite_master" + " WHERE temp.idxu.tbl=main.sqlite_master.tbl_name" + " AND temp.idxu.idx=main.sqlite_master.name" + " ORDER BY cnt DESC, tbl, idx", + -1, &pStmt, 0); + if( rc ){ + printf("Cannot query the result table - %s\n", + sqlite3_errmsg(db)); + goto errorOut; + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("%10d %s on %s(%s)\n", + sqlite3_column_int(pStmt, 2), + sqlite3_column_text(pStmt, 1), + sqlite3_column_text(pStmt, 0), + sqlite3_column_text(pStmt, 3)); + } + sqlite3_finalize(pStmt); + pStmt = 0; + +errorOut: + sqlite3_finalize(pStmt); + sqlite3_close(db); + return nErr; +} From 707821ff72aa67bc1fbf7ed6df1617d3e19d372b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Dec 2018 13:39:06 +0000 Subject: [PATCH 15/15] Enhance the sqlite3_normalize_sql() interface so that it works even if the prepared statement was not initially compiled using SQLITE_PREPARE_NORMALIZED. Enhance the ".trace" command in the CLI so that it is able to access the full scope of functionality provided by sqlite3_trace_v2() and in particular so that it is able to show normalized SQL output using the newly enhanced sqlite3_normalize_sql() interface. FossilOrigin-Name: 7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5 --- manifest | 26 ++++----- manifest.uuid | 2 +- src/hash.c | 2 +- src/hash.h | 3 ++ src/prepare.c | 41 ++++++-------- src/shell.c.in | 136 +++++++++++++++++++++++++++++++++++++++-------- src/sqliteInt.h | 2 +- src/vdbeapi.c | 6 ++- src/vdbeaux.c | 2 +- test/shell4.test | 8 +-- 10 files changed, 159 insertions(+), 69 deletions(-) diff --git a/manifest b/manifest index f8f7ebdbb6..68a14a5b76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"index_usage"\sutility\sprogram. -D 2018-12-04T16:51:42.708 +C Enhance\sthe\ssqlite3_normalize_sql()\sinterface\sso\sthat\sit\sworks\seven\sif\sthe\nprepared\sstatement\swas\snot\sinitially\scompiled\susing\nSQLITE_PREPARE_NORMALIZED.\s\sEnhance\sthe\s".trace"\scommand\sin\sthe\sCLI\sso\sthat\nit\sis\sable\sto\saccess\sthe\sfull\sscope\sof\sfunctionality\sprovided\sby\s\nsqlite3_trace_v2()\sand\sin\sparticular\sso\sthat\sit\sis\sable\sto\sshow\snormalized\nSQL\soutput\susing\sthe\snewly\senhanced\ssqlite3_normalize_sql()\sinterface. +D 2018-12-05T13:39:06.092 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 68d0ba0f0b533d5bc84c78c13a6ce84ee81183a67014caa47a969e67f028fa1c @@ -464,8 +464,8 @@ F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 972a4ba14296bef2303a0abbad1e3d82bc3c61f9e6ce4e8e9528bdee68748812 F src/func.c 7c288b4ce309b5a8b8473514b88e1f8e69a80134509a8c0db8e39c858e367e7f F src/global.c 8291eee0782b83124de14ec0389ec9fd6ae1873358a6b0d9469fe17a46ad803b -F src/hash.c 931ec82d7e070654a8facb42549bbb3a25720171d73ba94c3d3160580d01ef1f -F src/hash.h ab34c5c54a9e9de2e790b24349ba5aab3dbb4fd4 +F src/hash.c 6d2f67276469384fb8784fb8e962deeaae6832955626468325d705a01b999594 +F src/hash.h eebf2250e56b5d5353b873406557e373d2888cf51f111e28917666456d479e85 F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c f12f27eb606d601825be9a229a7390a8d64d40226697883f96de8e088d620055 @@ -502,17 +502,17 @@ F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c ad0ffc5b35b0280d045ac569d34d4b842e3e6a4a118f6396b320987a0957afcc F src/pragma.c 4e056f042683b99c4ea0db395f68d051b1a95833ab40951c40d3ef7e1fee1354 F src/pragma.h fdd03d78a7497f74a3f652909f945328480089189526841ae829ce7313d98d13 -F src/prepare.c f81f8d707e583192c28fea0b2e19385415b7d188123b23f49b038076408d7a69 +F src/prepare.c 277c8af17124bd4f67ae6e1a8e795a91f12bf7ce3a4b344c7eac46d360250c20 F src/printf.c 0f1177cf1dd4d7827bf64d840768514ec76409abecaca9e8b577dbd065150381 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 976e7879286a1eecdc71ceff64f6d1b3f58c8f8096537ba668b3dc0887f410c1 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 61e867a906f140b73baf4ce7a201ad6dcba30820969f5618ee40e9a0d32c6f5f -F src/shell.c.in 482e23a370cbe5b0d4c73a0f0f5fce34f7caa08a14a8d75e12f0225c4e14915c +F src/shell.c.in 1f0819e69fb1ebd2eb44695530dc43936608bf9b752981a0ffd4e2e4a9e3883d F src/sqlite.h.in cce9feede1c1c03923c091b4bbbd081dd77aaf92024cc2cdbf65f712c2f668c3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 960f1b86c3610fa23cb6a267572a97dcf286e77aa0dd3b9b23292ffaa1ea8683 -F src/sqliteInt.h f54ee1ef1e0f99d27af20561df72aac9158ed420cfc3a2e330fdee40672daf37 +F src/sqliteInt.h 128f2ef4cd6037300cb5510dd3c882417b225d6f4ed6210bddfd920104c7341f F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -581,8 +581,8 @@ F src/vacuum.c 836cadc922de866c849e23a75f93d344cdc143d388339305d09a3fed27e8798d F src/vdbe.c 005e691ea4c7d51e6c1a69d9389aeb34700884c85f51681817ddea3fdc2fc39b F src/vdbe.h 5081dcc497777efe5e9ebe7330d283a044a005e4bdda2e2e984f03bf89a0d907 F src/vdbeInt.h 437e6c6af679fdf157867eb83a8adc6cf5145d6774453c2214cfd0bd01d92980 -F src/vdbeapi.c dc825a6ec99a5066c1aa0d9824509057c0510f03cc8c72f81ba074553f8a5ae8 -F src/vdbeaux.c f547901b1aa9e2d81c63f06893f633648e434180666a827aacb547d7d6c8a601 +F src/vdbeapi.c 9ac7e3946a2762b79c314922c84c7de30731e0ed1d7c94ac82266b795221d9d7 +F src/vdbeaux.c 8e2fe020824b743090025ff6f9ffeec3ca4624523ddb2d1af1b1f61abaab3db4 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 F src/vdbemem.c 7b3305bc4a5139f4536ac9b5f61da0f915e49d2e3fdfa87dfdfa9d7aba8bc1e9 F src/vdbesort.c 90aad5a92608f2dd771c96749beabdb562c9d881131a860a7a5bccf66dc3be7f @@ -1280,7 +1280,7 @@ F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test d2bf5daeb6f449f0169c9ef3094db17a16a02199c5dcf1a635a3e79b07eb0edd F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494 -F test/shell4.test 89ad573879a745974ff2df20ff97c5d6ffffbd5d +F test/shell4.test a6881d0ae226ded0df8ebdfa574c5aa6dc28d6884ccba1089dc56ed08b9e5ef4 F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458 F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3 F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f @@ -1782,7 +1782,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e28584e8bc7b7405380064b60523fa6191f827f74075f6d117eb7732d752ba5e -R 14a419221d3a581ec01dad82979f03fc +P df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 +R e65d818662ab9003f3be5e2ff4aec20c U drh -Z 0158a53fd6ddbd828c2808dfe15d0a38 +Z 66f377ac461c094ae8c28a5487556c27 diff --git a/manifest.uuid b/manifest.uuid index 19384dd637..fab3fff857 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df95455213c9d1db7229e94217e78edc05cbf9e40f39528105494ea6ac52be94 \ No newline at end of file +7da617e97eb905cb009c47403786682b911e32a630f266e1c53ea72836fc88b5 \ No newline at end of file diff --git a/src/hash.c b/src/hash.c index fba9dc9f80..e725f2aa5e 100644 --- a/src/hash.c +++ b/src/hash.c @@ -72,7 +72,7 @@ static unsigned int strHashN(const char *z, int n){ /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). ** 0x9e3779b1 is 2654435761 which is the closest prime number to ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ - h += sqlite3UpperToLower[z[i]]; + h += sqlite3UpperToLower[(unsigned char)z[i]]; h *= 0x9e3779b1; } return h; diff --git a/src/hash.h b/src/hash.h index 90540eda50..cf773663b7 100644 --- a/src/hash.h +++ b/src/hash.h @@ -68,6 +68,9 @@ struct HashElem { void sqlite3HashInit(Hash*); void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); void *sqlite3HashFind(const Hash*, const char *pKey); +#ifdef SQLITE_ENABLE_NORMALIZE +void *sqlite3HashFindN(const Hash *pH, const char *pKey, int nKey); +#endif void sqlite3HashClear(Hash*); /* diff --git a/src/prepare.c b/src/prepare.c index 4d33f0b1e1..86afac5350 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -791,8 +791,7 @@ done: */ static int estimateNormalizedSize( const char *zSql, /* The original SQL string */ - int nSql, /* Length of original SQL string */ - u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */ + int nSql /* Length of original SQL string */ ){ int nOut = nSql + 4; const char *z = zSql; @@ -847,18 +846,14 @@ static void copyNormalizedToken( } /* -** Perform normalization of the SQL contained in the prepared statement and -** store the result in the zNormSql field. The schema for the associated -** databases are consulted while performing the normalization in order to -** determine if a token appears to be an identifier. All identifiers are -** left intact in the normalized SQL and all literals are replaced with a -** single '?'. +** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return +** the normalization in space obtained from sqlite3DbMalloc(). Or return +** NULL if anything goes wrong or if zSql is NULL. */ -void sqlite3Normalize( +char *sqlite3Normalize( Vdbe *pVdbe, /* VM being reprepared */ const char *zSql, /* The original SQL string */ - int nSql, /* Size of the input string in bytes */ - u8 prepFlags /* The flags passed to sqlite3_prepare_v3() */ + int nSql /* Size of the input string in bytes */ ){ sqlite3 *db; /* Database handle. */ char *z; /* The output string */ @@ -873,11 +868,10 @@ void sqlite3Normalize( db = sqlite3VdbeDb(pVdbe); assert( db!=0 ); - assert( pVdbe->zNormSql==0 ); - if( zSql==0 ) return; - nZ = estimateNormalizedSize(zSql, nSql, prepFlags); + if( zSql==0 ) return 0; + nZ = estimateNormalizedSize(zSql, nSql); z = sqlite3DbMallocRawNN(db, nZ); - if( z==0 ) return; + if( z==0 ) goto normalizeError; sqlite3HashInit(&inHash); for(i=j=0; i0 && z[j-1]!=';' ){ z[j++] = ';'; } z[j] = 0; assert( jzNormSql = z; sqlite3HashClear(&inHash); + return z; + +normalizeError: + sqlite3DbFree(db, z); + sqlite3HashClear(&inHash); + return 0; } #endif /* SQLITE_ENABLE_NORMALIZE */ diff --git a/src/shell.c.in b/src/shell.c.in index ad5a1498be..887c2fcf59 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1007,6 +1007,7 @@ struct ShellState { u8 openMode; /* SHELL_OPEN_NORMAL, _APPENDVFS, or _ZIPFILE */ u8 doXdgOpen; /* Invoke start/open/xdg-open in output_reset() */ u8 nEqpLevel; /* Depth of the EQP output graph */ + u8 eTraceType; /* SHELL_TRACE_* value for type of trace */ unsigned mEqpLines; /* Mask of veritical lines in the EQP output graph */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ @@ -1066,6 +1067,12 @@ struct ShellState { #define SHELL_OPEN_READONLY 4 /* Open a normal database read-only */ #define SHELL_OPEN_DESERIALIZE 5 /* Open using sqlite3_deserialize() */ +/* Allowed values for ShellState.eTraceType +*/ +#define SHELL_TRACE_PLAIN 0 /* Show input SQL text */ +#define SHELL_TRACE_EXPANDED 1 /* Show expanded SQL text */ +#define SHELL_TRACE_NORMALIZED 2 /* Show normalized SQL text */ + /* ** These are the allowed shellFlgs values */ @@ -3492,7 +3499,22 @@ static const char *(azHelp[]) = { ".testcase NAME Begin redirecting output to 'testcase-out.txt'", ".timeout MS Try opening locked tables for MS milliseconds", ".timer on|off Turn SQL timer on or off", - ".trace FILE|off Output each SQL statement as it is run", +#ifndef SQLITE_OMIT_TRACE + ".trace ?OPTIONS? Output each SQL statement as it is run", + " FILE Send output to FILE", + " stdout Send output to stdout", + " stderr Send output to stderr", + " off Disable tracing", + " --expanded Expand query parameters", +#ifdef SQLITE_ENABLE_NORMALIZE + " --normalized Normal the SQL statements", +#endif + " --plain Show SQL as it is input", + " --stmt Trace statement execution (SQLITE_TRACE_STMT)", + " --profile Profile statements (SQLITE_TRACE_PROFILE)", + " --row Trace each row (SQLITE_TRACE_ROW)", + " --close Trace connection close (SQLITE_TRACE_CLOSE)", +#endif /* SQLITE_OMIT_TRACE */ ".vfsinfo ?AUX? Information about the top-level VFS", ".vfslist List all available VFSes", ".vfsname ?AUX? Print the name of the VFS stack", @@ -3999,24 +4021,60 @@ static FILE *output_file_open(const char *zFile, int bTextMode){ return f; } -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) +#ifndef SQLITE_OMIT_TRACE /* ** A routine for handling output from sqlite3_trace(). */ static int sql_trace_callback( - unsigned mType, - void *pArg, - void *pP, - void *pX + unsigned mType, /* The trace type */ + void *pArg, /* The ShellState pointer */ + void *pP, /* Usually a pointer to sqlite_stmt */ + void *pX /* Auxiliary output */ ){ - FILE *f = (FILE*)pArg; - UNUSED_PARAMETER(mType); - UNUSED_PARAMETER(pP); - if( f ){ - const char *z = (const char*)pX; - int i = strlen30(z); - while( i>0 && z[i-1]==';' ){ i--; } - utf8_printf(f, "%.*s;\n", i, z); + ShellState *p = (ShellState*)pArg; + sqlite3_stmt *pStmt; + const char *zSql; + int nSql; + if( p->traceOut==0 ) return 0; + if( mType==SQLITE_TRACE_CLOSE ){ + utf8_printf(p->traceOut, "-- closing database connection\n"); + return 0; + } + if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + zSql = (const char*)pX; + }else{ + pStmt = (sqlite3_stmt*)pP; + switch( p->eTraceType ){ + case SHELL_TRACE_EXPANDED: { + zSql = sqlite3_expanded_sql(pStmt); + break; + } +#ifdef SQLITE_ENABLE_NORMALIZE + case SHELL_TRACE_NORMALIZED: { + zSql = sqlite3_normalized_sql(pStmt); + break; + } +#endif + default: { + zSql = sqlite3_sql(pStmt); + break; + } + } + } + if( zSql==0 ) return 0; + nSql = strlen30(zSql); + while( nSql>0 && zSql[nSql-1]==';' ){ nSql--; } + switch( mType ){ + case SQLITE_TRACE_ROW: + case SQLITE_TRACE_STMT: { + utf8_printf(p->traceOut, "%.*s;\n", nSql, zSql); + break; + } + case SQLITE_TRACE_PROFILE: { + sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; + utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", nSql, zSql, nNanosec); + break; + } } return 0; } @@ -7838,23 +7896,55 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else +#ifndef SQLITE_OMIT_TRACE if( c=='t' && strncmp(azArg[0], "trace", n)==0 ){ + int mType = 0; + int jj; open_db(p, 0); - if( nArg!=2 ){ - raw_printf(stderr, "Usage: .trace FILE|off\n"); - rc = 1; - goto meta_command_exit; + for(jj=1; jjeTraceType = SHELL_TRACE_EXPANDED; + } +#ifdef SQLITE_ENABLE_NORMALIZE + else if( optionMatch(z, "normalized") ){ + p->eTraceType = SHELL_TRACE_NORMALIZED; + } +#endif + else if( optionMatch(z, "plain") ){ + p->eTraceType = SHELL_TRACE_PLAIN; + } + else if( optionMatch(z, "profile") ){ + mType |= SQLITE_TRACE_PROFILE; + } + else if( optionMatch(z, "row") ){ + mType |= SQLITE_TRACE_ROW; + } + else if( optionMatch(z, "stmt") ){ + mType |= SQLITE_TRACE_STMT; + } + else if( optionMatch(z, "close") ){ + mType |= SQLITE_TRACE_CLOSE; + } + else { + raw_printf(stderr, "Unknown option \"%s\" on \".trace\"\n", z); + rc = 1; + goto meta_command_exit; + } + }else{ + output_file_close(p->traceOut); + p->traceOut = output_file_open(azArg[1], 0); + } } - output_file_close(p->traceOut); - p->traceOut = output_file_open(azArg[1], 0); -#if !defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_OMIT_FLOATING_POINT) if( p->traceOut==0 ){ sqlite3_trace_v2(p->db, 0, 0, 0); }else{ - sqlite3_trace_v2(p->db, SQLITE_TRACE_STMT, sql_trace_callback,p->traceOut); + if( mType==0 ) mType = SQLITE_TRACE_STMT; + sqlite3_trace_v2(p->db, mType, sql_trace_callback, p); } -#endif }else +#endif /* !defined(SQLITE_OMIT_TRACE) */ #if SQLITE_USER_AUTHENTICATION if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9f9d03a605..fc52fecfe2 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4424,7 +4424,7 @@ int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); void sqlite3ParserReset(Parse*); #ifdef SQLITE_ENABLE_NORMALIZE -void sqlite3Normalize(Vdbe*, const char*, int, u8); +char *sqlite3Normalize(Vdbe*, const char*, int); #endif int sqlite3Reprepare(Vdbe*); void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 3b040a5b01..ca073589f7 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1712,7 +1712,11 @@ char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){ */ const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe *)pStmt; - return p ? p->zNormSql : 0; + if( p==0 ) return 0; + if( p->zNormSql==0 && p->zSql!=0 ){ + p->zNormSql = sqlite3Normalize(p, p->zSql, sqlite3Strlen30(p->zSql)); + } + return p->zNormSql; } #endif /* SQLITE_ENABLE_NORMALIZE */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f1496a3abf..0ed9e81d90 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -67,7 +67,7 @@ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){ #ifdef SQLITE_ENABLE_NORMALIZE assert( p->zNormSql==0 ); if( p->zSql && (prepFlags & SQLITE_PREPARE_NORMALIZE)!=0 ){ - sqlite3Normalize(p, p->zSql, n, prepFlags); + p->zNormSql = sqlite3Normalize(p, p->zSql, n); assert( p->zNormSql!=0 || p->db->mallocFailed ); } #endif diff --git a/test/shell4.test b/test/shell4.test index 88e5e69a28..2210e74a67 100644 --- a/test/shell4.test +++ b/test/shell4.test @@ -107,14 +107,14 @@ SELECT 1; } {1 1 1} do_test shell4-2.1 { - catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace" -} {1 {Usage: .trace FILE|off}} + catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace --unknown" +} {1 {Unknown option "--unknown" on ".trace"}} do_test shell4-2.2 { catchcmd ":memory:" "CREATE TABLE t1(x);\n.trace off\n.trace off\n" } {0 {}} do_test shell4-2.3 { - catchcmd ":memory:" ".trace stdout\n.trace\n.trace off\n.dump\n" -} {/^1 {PRAGMA.*Usage:.*}$/} + catchcmd ":memory:" ".trace stdout\n.dump\n.trace off\n" +} {/^0 {PRAGMA.*}$/} ifcapable trace { do_test shell4-2.4 { catchcmd ":memory:" ".trace stdout\nCREATE TABLE t1(x);SELECT * FROM t1;"