From 0619838c5587f09e661ebfab63e309ff60a38237 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 21 Apr 2017 16:04:18 +0000 Subject: [PATCH 01/11] Fix an FTS5 bug that could cause a prefix-query without a prefix-index on a database that contains delete-markers to return extra, non-matching, rows. FossilOrigin-Name: 840042cb2bed2924e2263f21887317f661e5a585c4466c3af25b91ed57a6b49b --- ext/fts5/fts5_index.c | 8 ++------ ext/fts5/test/fts5aa.test | 14 ++++++++++++++ manifest | 15 +++++++-------- manifest.uuid | 2 +- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 955489a35a..3d6616bd91 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2878,7 +2878,8 @@ static void fts5MultiIterNext2( ){ assert( pIter->bSkipEmpty ); if( p->rc==SQLITE_OK ){ - do { + *pbNewTerm = 0; + do{ int iFirst = pIter->aFirst[1].iFirst; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; int bNewTerm = 0; @@ -2891,8 +2892,6 @@ static void fts5MultiIterNext2( fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); *pbNewTerm = 1; - }else{ - *pbNewTerm = 0; } fts5AssertMultiIterSetup(p, pIter); @@ -3845,9 +3844,6 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; -static int nCall = 0; -nCall++; - assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); /* Set the szLeaf header field. */ diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test index 9a95d81cfb..64c5331c64 100644 --- a/ext/fts5/test/fts5aa.test +++ b/ext/fts5/test/fts5aa.test @@ -577,6 +577,20 @@ do_execsql_test 21.1 { COMMIT; } +do_execsql_test 22.0 { + CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%); + INSERT INTO t9(rowid, x) VALUES(2, 'bbb'); + BEGIN; + INSERT INTO t9(rowid, x) VALUES(1, 'aaa'); + DELETE FROM t9 WHERE rowid = 2; + INSERT INTO t9(rowid, x) VALUES(3, 'bbb'); + COMMIT; +} + +do_execsql_test 22.1 { + SELECT rowid FROM t9('a*') +} {1} + } diff --git a/manifest b/manifest index 4ba5cabb77..004e6a67b1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\simprovements\sto\scoverage\sof\sfts3\smodule. -D 2017-04-19T13:25:45.345 +C Fix\san\sFTS5\sbug\sthat\scould\scause\sa\sprefix-query\swithout\sa\sprefix-index\son\sa\ndatabase\sthat\scontains\sdelete-markers\sto\sreturn\sextra,\snon-matching,\srows. +D 2017-04-21T16:04:18.457 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -105,7 +105,7 @@ F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 F ext/fts5/fts5_expr.c f2825f714d91bbe62ab5820aee9ad12e0c94205b2a01725eaa9072415ae9ff1c F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 -F ext/fts5/fts5_index.c 551add2b7762a2857336747def3b9d4fa304df476e188fb323492e2fd851772b +F ext/fts5/fts5_index.c dc25123df20c60492857de491a194dab4b46ace217b8483bda305d357bf6431d F ext/fts5/fts5_main.c 1ba0e7806886c1bc16e20d0dde1c2b535d1aeb98cbbb937c4c3e064af5ac6f03 F ext/fts5/fts5_storage.c 7750986004f3f0c94619a85ecb5dd6cbef53e5e3853488e8a906c269d4d11db6 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a @@ -118,7 +118,7 @@ F ext/fts5/fts5_vocab.c e44fefa7f0c1db252998af071daf06a7147e17e7 F ext/fts5/fts5parse.y a070b538e08ae9e2177d15c337ed2a3464408f0f886e746307098f746efd94ca F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841 -F ext/fts5/test/fts5aa.test b3cb080db4851580705c5e261c9d4308edf030d5cbbc24b5d6668403b73f023b +F ext/fts5/test/fts5aa.test 6dd1bfa4426a8d77e8c8d447aad02515c85118b082f24053ac89802300077ff1 F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c @@ -1575,8 +1575,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ea8a0d2ce0cb1ca3f4f18c72fb780d1c26792799acc87e6726f9eaccf2f178bf -Q +6417c1bbeceaea88ec1972149a1afc74fc40a677c1447160e53dbf4ba0930c42 -R 14e4aea31c2fb4851be140b478cd2183 +P 6b21d0fdebdccfaf63590d9ca9a279c22b8baec07c1a669b9f617f25bd857384 +R cdbe5adc9cdc11c6c21e2042b3458b4a U dan -Z e5cfee329765f9813d3e00bb36cba8b5 +Z 85a5569935df66b83446b4c819d590d0 diff --git a/manifest.uuid b/manifest.uuid index 2d3577ec63..75f1e7c375 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b21d0fdebdccfaf63590d9ca9a279c22b8baec07c1a669b9f617f25bd857384 \ No newline at end of file +840042cb2bed2924e2263f21887317f661e5a585c4466c3af25b91ed57a6b49b \ No newline at end of file From 39c9d3a367f1be20db794879d3038fc0e140fd23 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 21 Apr 2017 17:03:32 +0000 Subject: [PATCH 02/11] Remove a NEVER macro for a condition that may be true as of [a47efb7c]. Problem reported by OSSFuzz. FossilOrigin-Name: bdc50d8d127266d02b291f6bfcd4e35eb07994ce23987d6e3921124cd881929a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vtab.c | 2 +- test/indexexpr2.test | 8 ++++++++ 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 004e6a67b1..ff9be45e17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sFTS5\sbug\sthat\scould\scause\sa\sprefix-query\swithout\sa\sprefix-index\son\sa\ndatabase\sthat\scontains\sdelete-markers\sto\sreturn\sextra,\snon-matching,\srows. -D 2017-04-21T16:04:18.457 +C Remove\sa\sNEVER\smacro\sfor\sa\scondition\sthat\smay\sbe\strue\sas\sof\s[a47efb7c].\nProblem\sreported\sby\sOSSFuzz. +D 2017-04-21T17:03:32.804 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -478,7 +478,7 @@ F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 2c70f8f5de6c71fb99a22c5b83be9fab5c47cdd8e279fa44a8c00cfed06d7e89 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 -F src/vtab.c 007513c2ef52472fcdea6a741683d50662e82790 +F src/vtab.c 35b9bdc2b41de32a417141d12097bcc4e29a77ed7cdb8f836d1d2305d946b61b F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 @@ -882,7 +882,7 @@ F test/index7.test 7feababe16f2091b229c22aff2bcc1d4d6b9d2bb F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985 F test/indexexpr1.test 038b3befa74e5a75126b6e9dd2ae5df61c1c7cf7 -F test/indexexpr2.test bcf694ae72efaaeb97691b990b61e39bf233884e27b9cc6b845a1f0c6bc9f0bb +F test/indexexpr2.test 68ee9dbe83fcf85e50f4d0bd1f742a082496f2ee5153f4be2a1861db84462bf7 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test 38742b5e9601c8f8d76e9b7555f7270288c2d371 @@ -1575,7 +1575,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6b21d0fdebdccfaf63590d9ca9a279c22b8baec07c1a669b9f617f25bd857384 -R cdbe5adc9cdc11c6c21e2042b3458b4a +P 840042cb2bed2924e2263f21887317f661e5a585c4466c3af25b91ed57a6b49b +R deaae08958bfb25b170ee17b0f3a96e7 U dan -Z 85a5569935df66b83446b4c819d590d0 +Z 7063af02b35d7fd8e05a5f53a195e6a5 diff --git a/manifest.uuid b/manifest.uuid index 75f1e7c375..c5ec43f0d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -840042cb2bed2924e2263f21887317f661e5a585c4466c3af25b91ed57a6b49b \ No newline at end of file +bdc50d8d127266d02b291f6bfcd4e35eb07994ce23987d6e3921124cd881929a \ No newline at end of file diff --git a/src/vtab.c b/src/vtab.c index 48891034d5..e603e921e6 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -1053,7 +1053,7 @@ FuncDef *sqlite3VtabOverloadFunction( if( NEVER(pExpr==0) ) return pDef; if( pExpr->op!=TK_COLUMN ) return pDef; pTab = pExpr->pTab; - if( NEVER(pTab==0) ) return pDef; + if( pTab==0 ) return pDef; if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; assert( pVtab!=0 ); diff --git a/test/indexexpr2.test b/test/indexexpr2.test index 140498b124..47d50de3ef 100644 --- a/test/indexexpr2.test +++ b/test/indexexpr2.test @@ -32,5 +32,13 @@ do_execsql_test 1.2 { SELECT 'TWOX' == (b || 'x') COLLATE nocase FROM t1 WHERE (b || 'x')>'onex' } {0 1} +do_execsql_test 2.0 { + CREATE INDEX i2 ON t1(a+1); +} + +do_execsql_test 2.1 { + SELECT a+1, quote(a+1) FROM t1 ORDER BY 1; +} {2 2 3 3 4 4} + finish_test From 0c3233be0bf2663631d38b9c97f445eb7d0a9120 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Apr 2017 00:20:49 +0000 Subject: [PATCH 03/11] Fix an assertion fault found by OSSFuzz. FossilOrigin-Name: e39769f442ea86273f8fb7bd2f249b7d760b2fe4bf9ec191ff91def8231f3f35 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/date.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ff9be45e17..756b2ba9ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sNEVER\smacro\sfor\sa\scondition\sthat\smay\sbe\strue\sas\sof\s[a47efb7c].\nProblem\sreported\sby\sOSSFuzz. -D 2017-04-21T17:03:32.804 +C Fix\san\sassertion\sfault\sfound\sby\sOSSFuzz. +D 2017-04-22T00:20:49.403 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -352,7 +352,7 @@ F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f F src/callback.c 2e76147783386374bf01b227f752c81ec872d730 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c -F src/date.c ee676e7694dfadbdd2fde1a258a71be8360ba5ae +F src/date.c cc42a41c7422389860d40419a5e3bce5eaf6e7835c3ba2677751dc653550a5c7 F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c F src/expr.c f10e35dc50be4c8f82eb99bf5d8530229d1d60957cc3c9473ffe584d0444087c @@ -1575,7 +1575,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 840042cb2bed2924e2263f21887317f661e5a585c4466c3af25b91ed57a6b49b -R deaae08958bfb25b170ee17b0f3a96e7 -U dan -Z 7063af02b35d7fd8e05a5f53a195e6a5 +P bdc50d8d127266d02b291f6bfcd4e35eb07994ce23987d6e3921124cd881929a +R 1b30a3da1c7e16f4c6839737edbe387b +U drh +Z 8bf87a9edded8e3e15721afc254bc71f diff --git a/manifest.uuid b/manifest.uuid index c5ec43f0d9..0e4d408460 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bdc50d8d127266d02b291f6bfcd4e35eb07994ce23987d6e3921124cd881929a \ No newline at end of file +e39769f442ea86273f8fb7bd2f249b7d760b2fe4bf9ec191ff91def8231f3f35 \ No newline at end of file diff --git a/src/date.c b/src/date.c index b49d506bc4..f668ad0a84 100644 --- a/src/date.c +++ b/src/date.c @@ -423,8 +423,10 @@ static void computeYMD(DateTime *p){ p->Y = 2000; p->M = 1; p->D = 1; + }else if( !validJulianDay(p->iJD) ){ + datetimeError(p); + return; }else{ - assert( validJulianDay(p->iJD) ); Z = (int)((p->iJD + 43200000)/86400000); A = (int)((Z - 1867216.25)/36524.25); A = Z + 1 + A - (A/4); From a1188d6d93a72688e95e895b5b67a3cf22bab098 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Apr 2017 14:16:55 +0000 Subject: [PATCH 04/11] When building an ephemeral table of integers, do not assume that the table does not already contain N if the insert cursor points to N-1. Fix for ticket [61fe9745]. FossilOrigin-Name: 1c1b0862236f564283be08338790d6494e705f1198547a59499e3f8c07f2cd99 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 3 --- test/in5.test | 12 ++++++++++++ 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 756b2ba9ab..3f0843de7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassertion\sfault\sfound\sby\sOSSFuzz. -D 2017-04-22T00:20:49.403 +C When\sbuilding\san\sephemeral\stable\sof\sintegers,\sdo\snot\sassume\sthat\sthe\stable\ndoes\snot\salready\scontain\sN\sif\sthe\sinsert\scursor\spoints\sto\sN-1.\sFix\sfor\sticket\n[61fe9745]. +D 2017-04-24T14:16:55.419 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -345,7 +345,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c 24ae5472bd0b53b4130ecdda389deb621af721d1fcb50890b878102b00bd10fa +F src/btree.c a0d9a1c782ff3d22df5d217a4fa7125dd69ad5849caa51c4442c10246ca8ae27 F src/btree.h 80f518c0788be6cec8d9f8e13bd8e380df299d2b5e4ac340dc887b0642647cfc F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610 F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f @@ -860,7 +860,7 @@ F test/in.test 20c5529986998949908f889c8208b2cd894b2cc9 F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 -F test/in5.test 6c006e0bcd7351b69350ef566e65f244023489e9 +F test/in5.test 7ae37fcd4a5e198291c6ab5f31a5bb3d15397efe8b75a6736d7a95a7b8dd9e08 F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822 F test/incrblob2.test a5ce5ed1d0b01e2ed347245a21170372528af0a5 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 @@ -1575,7 +1575,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bdc50d8d127266d02b291f6bfcd4e35eb07994ce23987d6e3921124cd881929a -R 1b30a3da1c7e16f4c6839737edbe387b -U drh -Z 8bf87a9edded8e3e15721afc254bc71f +P e39769f442ea86273f8fb7bd2f249b7d760b2fe4bf9ec191ff91def8231f3f35 +R e0d07d9fe79e0700cdeeda8796e623bf +U dan +Z 6adee9578f00e93307a285e5276adbb5 diff --git a/manifest.uuid b/manifest.uuid index 0e4d408460..cd44d34c9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e39769f442ea86273f8fb7bd2f249b7d760b2fe4bf9ec191ff91def8231f3f35 \ No newline at end of file +1c1b0862236f564283be08338790d6494e705f1198547a59499e3f8c07f2cd99 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 48cfae5c72..35e47173fa 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8124,9 +8124,6 @@ int sqlite3BtreeInsert( ** btreeMoveto() call */ if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){ loc = 0; - }else if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey>0 - && pCur->info.nKey==pX->nKey-1 ){ - loc = -1; }else if( loc==0 ){ rc = sqlite3BtreeMovetoUnpacked(pCur, 0, pX->nKey, flags!=0, &loc); if( rc ) return rc; diff --git a/test/in5.test b/test/in5.test index ac60cfe58e..a5ea7c1784 100644 --- a/test/in5.test +++ b/test/in5.test @@ -237,4 +237,16 @@ do_execsql_test 8.4 { SELECT count(*) FROM n1 WHERE a IN (SELECT +a FROM n1) } 3 +#------------------------------------------------------------------------- +# Test that ticket 61fe97454c is fixed. +# +do_execsql_test 9.0 { + CREATE TABLE t9(a INTEGER PRIMARY KEY); + INSERT INTO t9 VALUES (44), (45); +} +do_execsql_test 9.1 { + SELECT * FROM t9 WHERE a IN (44, 45, 44, 45) +} {44 45} + + finish_test From 69bfa55279f0c765168e61887b7a8f6aa0690393 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Apr 2017 04:32:17 +0000 Subject: [PATCH 05/11] Fix a subtle bug in Lemon discovered and reported on the mailing list by Kelvin Sherlock, who also suggested the correct fix. FossilOrigin-Name: 304689f8acb53d68e1afed6e6c4332e78132e3d57071b8f94df0d13515b3b3d8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/lemon.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3f0843de7d..e629c607d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sbuilding\san\sephemeral\stable\sof\sintegers,\sdo\snot\sassume\sthat\sthe\stable\ndoes\snot\salready\scontain\sN\sif\sthe\sinsert\scursor\spoints\sto\sN-1.\sFix\sfor\sticket\n[61fe9745]. -D 2017-04-24T14:16:55.419 +C Fix\sa\ssubtle\sbug\sin\sLemon\sdiscovered\sand\sreported\son\sthe\smailing\slist\nby\sKelvin\sSherlock,\swho\salso\ssuggested\sthe\scorrect\sfix. +D 2017-04-26T04:32:17.931 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -1501,7 +1501,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c 6b0cdffb377e79c6295c40300677688f3d35c21c0154da635642ef82dc3a5836 +F tool/lemon.c f4f1045743e12f86b132253a3192ef92c94bfceb7f41ac41b8e3b373aa78474e F tool/lempar.c db1bdb4821f2d8fbd76e577cf3ab18642c8d08d1 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 @@ -1575,7 +1575,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e39769f442ea86273f8fb7bd2f249b7d760b2fe4bf9ec191ff91def8231f3f35 -R e0d07d9fe79e0700cdeeda8796e623bf -U dan -Z 6adee9578f00e93307a285e5276adbb5 +P 1c1b0862236f564283be08338790d6494e705f1198547a59499e3f8c07f2cd99 +R f4ca2f4306b5e6dd872a5563d102577a +U drh +Z 2b3a1c4223ca97fbac851b16ecf78a54 diff --git a/manifest.uuid b/manifest.uuid index cd44d34c9a..c1c0608b0f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c1b0862236f564283be08338790d6494e705f1198547a59499e3f8c07f2cd99 \ No newline at end of file +304689f8acb53d68e1afed6e6c4332e78132e3d57071b8f94df0d13515b3b3d8 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index af8e4de12c..37d2c781d9 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -4161,7 +4161,7 @@ void ReportTable( for(i=0; inxstate; i++){ for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){ if( ap->type==REDUCE || ap->type==SHIFTREDUCE ){ - ap->x.rp->doesReduce = i; + ap->x.rp->doesReduce = 1; } } } From b9e6a73866bea489a9a96fe835d7ac64f3e183b3 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 26 Apr 2017 14:34:04 +0000 Subject: [PATCH 06/11] Replace with "fts5.h" in test file fts5_test_tok.c. FossilOrigin-Name: 63d9ca5c7392e1efe3330689fe750310a952287e843b3242178724e8561fae0e --- ext/fts5/fts5_test_tok.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/fts5/fts5_test_tok.c b/ext/fts5/fts5_test_tok.c index 10af126c10..cfc7a8b904 100644 --- a/ext/fts5/fts5_test_tok.c +++ b/ext/fts5/fts5_test_tok.c @@ -40,7 +40,7 @@ */ #if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_FTS5) -#include +#include "fts5.h" #include #include diff --git a/manifest b/manifest index e629c607d5..c75ceffb65 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\ssubtle\sbug\sin\sLemon\sdiscovered\sand\sreported\son\sthe\smailing\slist\nby\sKelvin\sSherlock,\swho\salso\ssuggested\sthe\scorrect\sfix. -D 2017-04-26T04:32:17.931 +C Replace\s\swith\s"fts5.h"\sin\stest\sfile\sfts5_test_tok.c. +D 2017-04-26T14:34:04.050 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -110,7 +110,7 @@ F ext/fts5/fts5_main.c 1ba0e7806886c1bc16e20d0dde1c2b535d1aeb98cbbb937c4c3e064af F ext/fts5/fts5_storage.c 7750986004f3f0c94619a85ecb5dd6cbef53e5e3853488e8a906c269d4d11db6 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc -F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be +F ext/fts5/fts5_test_tok.c f4c27f2e3200653a458f5316f6aaa863f7532d98fff7534ece3670fb7f2c196b F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738 @@ -1575,7 +1575,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1c1b0862236f564283be08338790d6494e705f1198547a59499e3f8c07f2cd99 -R f4ca2f4306b5e6dd872a5563d102577a -U drh -Z 2b3a1c4223ca97fbac851b16ecf78a54 +P 304689f8acb53d68e1afed6e6c4332e78132e3d57071b8f94df0d13515b3b3d8 +R 61b00e8c1460788b4584df3d61fb38a7 +U dan +Z 4318fdde372a0dfeeb363247529017d5 diff --git a/manifest.uuid b/manifest.uuid index c1c0608b0f..2e3189074c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -304689f8acb53d68e1afed6e6c4332e78132e3d57071b8f94df0d13515b3b3d8 \ No newline at end of file +63d9ca5c7392e1efe3330689fe750310a952287e843b3242178724e8561fae0e \ No newline at end of file From 4b1736957983f4248890c79efe693a77e1945418 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 26 Apr 2017 17:21:33 +0000 Subject: [PATCH 07/11] Add new test file cachespill.test. FossilOrigin-Name: 2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 --- manifest | 11 ++++--- manifest.uuid | 2 +- test/cachespill.test | 77 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 test/cachespill.test diff --git a/manifest b/manifest index c75ceffb65..8c73ad3567 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\s\swith\s"fts5.h"\sin\stest\sfile\sfts5_test_tok.c. -D 2017-04-26T14:34:04.050 +C Add\snew\stest\sfile\scachespill.test. +D 2017-04-26T17:21:33.779 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -576,6 +576,7 @@ F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/cacheflush.test af25bb1509df04c1da10e38d8f322d66eceedf61 +F test/cachespill.test 895997f84a25b323b166aecb69baab2d6380ea98f9e0bcc688c4493c535cfab9 F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 F test/capi3.test f0c66919e43d42e1572a69be039e4527a931b00f F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 @@ -1575,7 +1576,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 304689f8acb53d68e1afed6e6c4332e78132e3d57071b8f94df0d13515b3b3d8 -R 61b00e8c1460788b4584df3d61fb38a7 +P 63d9ca5c7392e1efe3330689fe750310a952287e843b3242178724e8561fae0e +R c4490df1c59aee5f8635e7285ee6a7e5 U dan -Z 4318fdde372a0dfeeb363247529017d5 +Z 2b0ae6051b3e0017c3cacc8dc5ffb6e4 diff --git a/manifest.uuid b/manifest.uuid index 2e3189074c..44877c7f80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63d9ca5c7392e1efe3330689fe750310a952287e843b3242178724e8561fae0e \ No newline at end of file +2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 \ No newline at end of file diff --git a/test/cachespill.test b/test/cachespill.test new file mode 100644 index 0000000000..069251354f --- /dev/null +++ b/test/cachespill.test @@ -0,0 +1,77 @@ +# 2017 April 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cachespill + +ifcapable !pager_pragmas { + finish_test + return +} + +#------------------------------------------------------------------------- +# Test that "PRAGMA cache_spill = 0" completely disables cache spilling. +# +do_execsql_test 1.1 { + PRAGMA auto_vacuum = 0; + PRAGMA page_size = 1024; + PRAGMA cache_size = 100; + CREATE TABLE t1(a); +} + +do_test 1.2 { + file size test.db +} {2048} + +do_test 1.3 { + execsql { + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200 + ) INSERT INTO t1 SELECT randomblob(900) FROM s; + } + expr {[file size test.db] > 50000} +} {1} + +do_test 1.4 { + execsql ROLLBACK + file size test.db +} {2048} + +do_test 1.5 { + execsql { + PRAGMA cache_spill = 0; + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200 + ) INSERT INTO t1 SELECT randomblob(900) FROM s; + } + file size test.db +} {2048} + +do_test 1.5 { + execsql { + ROLLBACK; + PRAGMA cache_spill = 1; + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<200 + ) INSERT INTO t1 SELECT randomblob(900) FROM s; + } + expr {[file size test.db] > 50000} +} {1} + +do_execsql_test 1.6 { ROLLBACK } + + +finish_test From 6f654a40e8618e41b96be3d8dc36cac8ed69a56c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 28 Apr 2017 19:59:55 +0000 Subject: [PATCH 08/11] Within a loop that uses a non-covering index test, test non-indexed terms that can be tested without seeking the main table cursor before those that cannot. FossilOrigin-Name: afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444 --- manifest | 16 ++++++---- manifest.uuid | 2 +- src/wherecode.c | 78 ++++++++++++++++++++++++++-------------------- test/pushdown.test | 59 +++++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 40 deletions(-) create mode 100644 test/pushdown.test diff --git a/manifest b/manifest index 8c73ad3567..062de1d7ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\scachespill.test. -D 2017-04-26T17:21:33.779 +C Within\sa\sloop\sthat\suses\sa\snon-covering\sindex\stest,\stest\snon-indexed\sterms\sthat\ncan\sbe\stested\swithout\sseeking\sthe\smain\stable\scursor\sbefore\sthose\sthat\scannot. +D 2017-04-28T19:59:55.785 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -485,7 +485,7 @@ F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 F src/where.c c6352f15be5031907c68bcbde96cad1a6da20e9f4051d10168a59235de9a8566 F src/whereInt.h 2a4b634d63ce488b46d4b0da8f2eaa8f9aeab202bc25ef76f007de5e3fba1f20 -F src/wherecode.c 943e32e9dccd0af802e0683ae11071c8bd808364e5908a5fb66758bd404c8681 +F src/wherecode.c 878a8c75af2bf6103ed4d1df639ee64ddb66f11b3a388f9caf7598c159a2c9af F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1043,6 +1043,7 @@ F test/printf.test b3ff34e73d59124140eaf89f7672e21bc2ca5fcc F test/printf2.test 9e6db85f81c63f2367c34a9d7db384088bd374ad F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc +F test/pushdown.test a5d2e5e66cc94cfb0989a5dd23e77427dd6c4313019155ba9051f52dfc5326aa F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca F test/quick.test 1681febc928d686362d50057c642f77a02c62e57 F test/quota-glob.test 32901e9eed6705d68ca3faee2a06b73b57cb3c26 @@ -1576,7 +1577,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 63d9ca5c7392e1efe3330689fe750310a952287e843b3242178724e8561fae0e -R c4490df1c59aee5f8635e7285ee6a7e5 +P 2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 +R 8c650dff0f0f10ff63f37c52f3854c68 +T *branch * pushdown-optimization +T *sym-pushdown-optimization * +T -sym-trunk * U dan -Z 2b0ae6051b3e0017c3cacc8dc5ffb6e4 +Z 464ba506caa7cc6d49be35980d40d709 diff --git a/manifest.uuid b/manifest.uuid index 44877c7f80..0b47274c6c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 \ No newline at end of file +afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444 \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index d429fbb523..a95889136a 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1129,6 +1129,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ + Index *pIdx = 0; /* Index used by loop (if any) */ pParse = pWInfo->pParse; v = pParse->pVdbe; @@ -1454,7 +1455,6 @@ Bitmask sqlite3WhereCodeOneLoopStart( int endEq; /* True if range end uses ==, >= or <= */ int start_constraints; /* Start of range is constrained */ int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ int iIdxCur; /* The VDBE cursor for the index */ int nExtraReg = 0; /* Number of extra registers needed */ int op; /* Instruction opcode */ @@ -1705,6 +1705,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( }else{ assert( pLevel->p5==0 ); } + if( omitTable ) pIdx = 0; }else #ifndef SQLITE_OMIT_OR_OPTIMIZATION @@ -2022,42 +2023,53 @@ Bitmask sqlite3WhereCodeOneLoopStart( /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. - */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE; - int skipLikeAddr = 0; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - testcase( pWInfo->untestedTerms==0 - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ); - pWInfo->untestedTerms = 1; - continue; - } - pE = pTerm->pExpr; - assert( pE!=0 ); - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ - continue; - } - if( pTerm->wtFlags & TERM_LIKECOND ){ - /* If the TERM_LIKECOND flag is set, that means that the range search - ** is sufficient to guarantee that the LIKE operator is true, so we - ** can skip the call to the like(A,B) function. But this only works - ** for strings. So do not skip the call to the function on the pass - ** that compares BLOBs. */ + ** + ** This loop may run either once (pIdx==0) or twice (pIdx!=0). If + ** it is run twice, then the first iteration codes those sub-expressions + ** that can be computed using columns from pIdx only (without seeking + ** the main table cursor). */ + while( 1 ){ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + int skipLikeAddr = 0; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( pIdx && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ + continue; + } + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + continue; + } + if( pTerm->wtFlags & TERM_LIKECOND ){ + /* If the TERM_LIKECOND flag is set, that means that the range search + ** is sufficient to guarantee that the LIKE operator is true, so we + ** can skip the call to the like(A,B) function. But this only works + ** for strings. So do not skip the call to the function on the pass + ** that compares BLOBs. */ #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS - continue; + continue; #else - u32 x = pLevel->iLikeRepCntr; - assert( x>0 ); - skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)? OP_IfNot : OP_If, (int)(x>>1)); - VdbeCoverage(v); + u32 x = pLevel->iLikeRepCntr; + assert( x>0 ); + skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)?OP_IfNot:OP_If, (int)(x>>1)); + VdbeCoverage(v); #endif + } + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); + pTerm->wtFlags |= TERM_CODED; } - sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); - if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); - pTerm->wtFlags |= TERM_CODED; + if( pIdx==0 ) break; + pIdx = 0; } /* Insert code to test for implied constraints based on transitivity diff --git a/test/pushdown.test b/test/pushdown.test new file mode 100644 index 0000000000..5be6054c08 --- /dev/null +++ b/test/pushdown.test @@ -0,0 +1,59 @@ +# 2017 April 29 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix pushdown + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 'b1', 'c1'); + INSERT INTO t1 VALUES(2, 'b2', 'c2'); + INSERT INTO t1 VALUES(3, 'b3', 'c3'); + INSERT INTO t1 VALUES(4, 'b4', 'c4'); + CREATE INDEX i1 ON t1(a, c); +} + +proc f {val} { + lappend ::L $val + return 0 +} +db func f f + +do_test 1.1 { + set L [list] + execsql { SELECT * FROM t1 WHERE a=2 AND f(b) AND f(c) } + set L +} {c2} + +do_test 1.2 { + set L [list] + execsql { SELECT * FROM t1 WHERE a=3 AND f(c) AND f(b) } + set L +} {c3} + +do_execsql_test 1.3 { + DROP INDEX i1; + CREATE INDEX i1 ON t1(a, b); +} +do_test 1.4 { + set L [list] + execsql { SELECT * FROM t1 WHERE a=2 AND f(b) AND f(c) } + set L +} {b2} + +do_test 1.5 { + set L [list] + execsql { SELECT * FROM t1 WHERE a=3 AND f(c) AND f(b) } + set L +} {b3} + +finish_test From 6ab3eb5d0eade5492508823c7f9405fc1a560987 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Apr 2017 14:56:55 +0000 Subject: [PATCH 09/11] Minor size and performance improvements to the push-down optimization. FossilOrigin-Name: 91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/wherecode.c | 15 +++++++++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 062de1d7ff..6cc1fccff6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Within\sa\sloop\sthat\suses\sa\snon-covering\sindex\stest,\stest\snon-indexed\sterms\sthat\ncan\sbe\stested\swithout\sseeking\sthe\smain\stable\scursor\sbefore\sthose\sthat\scannot. -D 2017-04-28T19:59:55.785 +C Minor\ssize\sand\sperformance\simprovements\sto\sthe\spush-down\soptimization. +D 2017-04-29T14:56:55.202 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -485,7 +485,7 @@ F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c b71a992b413b3a022572eccf29ef4b4890223791 F src/where.c c6352f15be5031907c68bcbde96cad1a6da20e9f4051d10168a59235de9a8566 F src/whereInt.h 2a4b634d63ce488b46d4b0da8f2eaa8f9aeab202bc25ef76f007de5e3fba1f20 -F src/wherecode.c 878a8c75af2bf6103ed4d1df639ee64ddb66f11b3a388f9caf7598c159a2c9af +F src/wherecode.c 8ad48867660519e262a401720845dc76934f86f558ec9606335fafcd7a2554f8 F src/whereexpr.c e913aaa7b73ffcce66abcea5f197e2c538d48b5df78d0b7bba8ff4d73cc2e745 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1577,10 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 -R 8c650dff0f0f10ff63f37c52f3854c68 -T *branch * pushdown-optimization -T *sym-pushdown-optimization * -T -sym-trunk * -U dan -Z 464ba506caa7cc6d49be35980d40d709 +P afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444 +R d3fbd82e46a4092d676255e41f20df5b +U drh +Z bf8e798698af6ed2b80cadbc03ebd2b9 diff --git a/manifest.uuid b/manifest.uuid index 0b47274c6c..2939f423e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afe68f0a8060dc5c92cb1fb32a9f22bd36140cd2c0bb5b6cea853e169c5ed444 \ No newline at end of file +91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d \ No newline at end of file diff --git a/src/wherecode.c b/src/wherecode.c index a95889136a..feda45f911 100644 --- a/src/wherecode.c +++ b/src/wherecode.c @@ -1130,6 +1130,7 @@ Bitmask sqlite3WhereCodeOneLoopStart( int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ Index *pIdx = 0; /* Index used by loop (if any) */ + int loopAgain; /* True if constraint generator loop should repeat */ pParse = pWInfo->pParse; v = pParse->pVdbe; @@ -2027,8 +2028,10 @@ Bitmask sqlite3WhereCodeOneLoopStart( ** This loop may run either once (pIdx==0) or twice (pIdx!=0). If ** it is run twice, then the first iteration codes those sub-expressions ** that can be computed using columns from pIdx only (without seeking - ** the main table cursor). */ - while( 1 ){ + ** the main table cursor). + */ + do{ + loopAgain = 0; for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ Expr *pE; int skipLikeAddr = 0; @@ -2043,10 +2046,11 @@ Bitmask sqlite3WhereCodeOneLoopStart( } pE = pTerm->pExpr; assert( pE!=0 ); - if( pIdx && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ + if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ continue; } - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ + if( pIdx && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ + loopAgain = 1; continue; } if( pTerm->wtFlags & TERM_LIKECOND ){ @@ -2068,9 +2072,8 @@ Bitmask sqlite3WhereCodeOneLoopStart( if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); pTerm->wtFlags |= TERM_CODED; } - if( pIdx==0 ) break; pIdx = 0; - } + }while( loopAgain ); /* Insert code to test for implied constraints based on transitivity ** of the "==" operator. From bb6783b86f507ce41bda979d7a3a581ca29cd19a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Apr 2017 18:02:49 +0000 Subject: [PATCH 10/11] Improvements to opcode documentation in the bytecode engine. No changes to code. FossilOrigin-Name: e54c9f8db5b2fa8ea82c6eab7482255431af16901f7992c9667b56a0e50a9f4f --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/vdbe.c | 42 +++++++++++++++++++++--------------------- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index bf8f046722..b61163712f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Evaluate\sWHERE\sclause\sterms\sthat\sreference\sonly\sthe\sindex\sbefore\sevaluating\nterms\sthat\srequire\sthe\stable,\sand\sthereby\savoid\sseeking\sthe\stable\srow\sif\nindex\sterms\sare\sfalse.\nThis\sis\scalled\sthe\s"push-down"\soptimization\sin\sthe\sMySQL\sworld,\swe\sare\stold. -D 2017-04-29T15:27:04.896 +C Improvements\sto\sopcode\sdocumentation\sin\sthe\sbytecode\sengine.\nNo\schanges\sto\scode. +D 2017-04-29T18:02:49.909 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -469,7 +469,7 @@ F src/update.c c443935c652af9365e033f756550b5032d02e1b06eb2cb890ed7511ae0c051dc F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c 1eba07ffa0e4ff5e14ceca17e366c26f58c11d12cc56e55a9a13157a93533b7e +F src/vdbe.c 349eb6789cf8b03ef1aa42271aa9a17a6a7794c07448d27c5405904ba1b21715 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848 F src/vdbeInt.h c070bc5c8b913bda0ceaa995cd4d939ded5e4fc96cf7c3c1c602d41b871f8ade F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 @@ -1577,8 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d -R d3fbd82e46a4092d676255e41f20df5b -T +closed 91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d +P d7bb79ed3a40419d143fbe35c310e51fe7b384a22f082a61ad788671d2d33ee0 +R 5c1f22cfd852090783654b7cdddcbe29 U drh -Z d8af2da2fe4450264227719b4f172b61 +Z 979167fc5393ee8033851cc4ccc2f64f diff --git a/manifest.uuid b/manifest.uuid index a192425962..8be70e2935 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7bb79ed3a40419d143fbe35c310e51fe7b384a22f082a61ad788671d2d33ee0 \ No newline at end of file +e54c9f8db5b2fa8ea82c6eab7482255431af16901f7992c9667b56a0e50a9f4f \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 5bed6f6f4e..15f015a598 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -764,7 +764,7 @@ jump_to_p2_and_check_for_interrupt: pOp = &aOp[pOp->p2 - 1]; /* Opcodes that are used as the bottom of a loop (OP_Next, OP_Prev, - ** OP_VNext, OP_RowSetNext, or OP_SorterNext) all jump here upon + ** OP_VNext, or OP_SorterNext) all jump here upon ** completion. Check to see if sqlite3_interrupt() has been called ** or if the progress callback needs to be invoked. ** @@ -1567,7 +1567,7 @@ arithmetic_result_is_null: /* Opcode: CollSeq P1 * * P4 ** -** P4 is a pointer to a CollSeq struct. If the next call to a user function +** P4 is a pointer to a CollSeq object. If the next call to a user function ** or aggregate calls sqlite3GetFuncCollSeq(), this collation sequence will ** be returned. This is used by the built-in min(), max() and nullif() ** functions. @@ -1848,11 +1848,11 @@ case OP_RealAffinity: { /* in1 */ ** Force the value in register P1 to be the type defined by P2. ** **
    -**
  • TEXT -**
  • BLOB -**
  • NUMERIC -**
  • INTEGER -**
  • REAL +**
  • P2=='A' → BLOB +**
  • P2=='B' → TEXT +**
  • P2=='C' → NUMERIC +**
  • P2=='D' → INTEGER +**
  • P2=='E' → REAL **
** ** A NULL value is not changed by this routine. It remains NULL. @@ -2723,8 +2723,8 @@ op_column_out: ** ** Apply affinities to a range of P2 registers starting with P1. ** -** P4 is a string that is P2 characters long. The nth character of the -** string indicates the column affinity that should be used for the nth +** P4 is a string that is P2 characters long. The N-th character of the +** string indicates the column affinity that should be used for the N-th ** memory cell in the range. */ case OP_Affinity: { @@ -2751,8 +2751,8 @@ case OP_Affinity: { ** use as a data record in a database table or as a key ** in an index. The OP_Column opcode can decode the record later. ** -** P4 may be a string that is P2 characters long. The nth character of the -** string indicates the column affinity that should be used for the nth +** P4 may be a string that is P2 characters long. The N-th character of the +** string indicates the column affinity that should be used for the N-th ** field of the index key. ** ** The mapping from character to affinity is given by the SQLITE_AFF_ @@ -5748,7 +5748,7 @@ case OP_IntegrityCk: { /* Opcode: RowSetAdd P1 P2 * * * ** Synopsis: rowset(P1)=r[P2] ** -** Insert the integer value held by register P2 into a boolean index +** Insert the integer value held by register P2 into a RowSet object ** held in register P1. ** ** An assertion fails if P2 is not an integer. @@ -5768,8 +5768,9 @@ case OP_RowSetAdd: { /* in1, in2 */ /* Opcode: RowSetRead P1 P2 P3 * * ** Synopsis: r[P3]=rowset(P1) ** -** Extract the smallest value from boolean index P1 and put that value into -** register P3. Or, if boolean index P1 is initially empty, leave P3 +** Extract the smallest value from the RowSet object in P1 +** and put that value into register P3. +** Or, if RowSet object P1 is initially empty, leave P3 ** unchanged and jump to instruction P2. */ case OP_RowSetRead: { /* jump, in1, out3 */ @@ -5800,15 +5801,14 @@ case OP_RowSetRead: { /* jump, in1, out3 */ ** integer in P3 into the RowSet and continue on to the ** next opcode. ** -** The RowSet object is optimized for the case where successive sets -** of integers, where each set contains no duplicates. Each set -** of values is identified by a unique P4 value. The first set -** must have P4==0, the final set P4=-1. P4 must be either -1 or -** non-negative. For non-negative values of P4 only the lower 4 -** bits are significant. +** The RowSet object is optimized for the case where sets of integers +** are inserted in distinct phases, which each set contains no duplicates. +** Each set is identified by a unique P4 value. The first set +** must have P4==0, the final set must have P4==-1, and for all other sets +** must have P4>0. ** ** This allows optimizations: (a) when P4==0 there is no need to test -** the rowset object for P3, as it is guaranteed not to contain it, +** the RowSet object for P3, as it is guaranteed not to contain it, ** (b) when P4==-1 there is no need to insert the value, as it will ** never be tested for, and (c) when a value that is part of set X is ** inserted, there is no need to search to see if the same value was From 7555581fc1ee5be5133efcbe1f46f834abc3dc5f Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Apr 2017 19:29:49 +0000 Subject: [PATCH 11/11] Add a single testcase() macro to the subquery processing logic. FossilOrigin-Name: 4e1df76e3d85922648e0e1cce73a266c3b1ed4511ace259ec0a01d7693af9e6f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b61163712f..22a472ed22 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sopcode\sdocumentation\sin\sthe\sbytecode\sengine.\nNo\schanges\sto\scode. -D 2017-04-29T18:02:49.909 +C Add\sa\ssingle\stestcase()\smacro\sto\sthe\ssubquery\sprocessing\slogic. +D 2017-04-29T19:29:49.700 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -402,7 +402,7 @@ F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c bf8ab605e49717c222136380453cfb7eda564f8e500d5ff6a01341ea59fefe80 +F src/select.c 478e95d424bb86d34c7c95d20872cbd78df97af5f83c3fd7de55d5b2413f927d F src/shell.c 21b79c0e1b93f8e35fd7b4087d6ba438326c3d7e285d0dd51dfd741475f858a1 F src/sqlite.h.in 40233103e3e4e10f8a63523498d0259d232e42aba478e2d3fb914799185aced6 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1577,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d7bb79ed3a40419d143fbe35c310e51fe7b384a22f082a61ad788671d2d33ee0 -R 5c1f22cfd852090783654b7cdddcbe29 +P e54c9f8db5b2fa8ea82c6eab7482255431af16901f7992c9667b56a0e50a9f4f +R 6f21ea13ed166571d20351d2145d6640 U drh -Z 979167fc5393ee8033851cc4ccc2f64f +Z 4a7e86f7be3de966a53a14c1a17e31d5 diff --git a/manifest.uuid b/manifest.uuid index 8be70e2935..45bf2ccb33 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e54c9f8db5b2fa8ea82c6eab7482255431af16901f7992c9667b56a0e50a9f4f \ No newline at end of file +4e1df76e3d85922648e0e1cce73a266c3b1ed4511ace259ec0a01d7693af9e6f \ No newline at end of file diff --git a/src/select.c b/src/select.c index 3def13618e..77824c4aaa 100644 --- a/src/select.c +++ b/src/select.c @@ -5035,6 +5035,10 @@ int sqlite3Select( ** to be invoked again. */ if( pItem->addrFillSub ){ if( pItem->fg.viaCoroutine==0 ){ + /* The subroutine that manifests the view might be a one-time routine, + ** or it might need to be rerun on each iteration because it + ** encodes a correlated subquery. */ + testcase( sqlite3VdbeGetOp(v, pItem->addrFillSub)->opcode==OP_Once ); sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); } continue;