diff --git a/manifest b/manifest index 8b4409d2ff..98dcf120d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\swith\smulti-column\sIN(SELECT...)\sprocessing. -D 2016-07-27T19:33:04.107 +C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. +D 2016-07-28T13:59:21.728 F Makefile.in 6c20d44f72d4564f11652b26291a214c8367e5db F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a @@ -337,7 +337,7 @@ F src/ctime.c 61949e83c4c36e37195a8398ebc752780b534d95 F src/date.c 1cc9fb516ec9932c6fd4d2a0d2f8bc4480145c39 F src/dbstat.c 4f6f7f52b49beb9636ffbd517cfe44a402ba4ad0 F src/delete.c 4aba4214a377ce8ddde2d2e609777bcc8235200f -F src/expr.c bee29fbb017db26ad1189d35b2a7fe6ff8da3722 +F src/expr.c 82bc40bb27bc68b45303b560ab14eb16e39786a7 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c bc4145347595b7770f9a598cff1c848302cf5413 F src/func.c 61a4114cf7004f10c542cfabbab9f2bcb9033045 @@ -388,7 +388,7 @@ F src/shell.c 9351fc6de11e1d908648c0a92d85627138e3dee5 F src/sqlite.h.in c6e68a4a47610631822a4f8f83a44c9f75339331 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 46f300b6e300e0fa916d7d58c44b53415b8471a9 -F src/sqliteInt.h 9a6af8787f03f54ce02ff88bd45f89bb600c3a4c +F src/sqliteInt.h 65cf752e1a62cedb2430ecc0547a09833ea13616 F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c 5b18f9526900f61189ab0b83f1ef41d9f871a2ab F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 @@ -463,7 +463,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c 02eeecc265f6ffd0597378f5d8ae9070b62a406a F src/wal.h 6dd221ed384afdc204bc61e25c23ef7fd5a511f2 F src/walker.c 0f142b5bd3ed2041fc52d773880748b212e63354 -F src/where.c 0b370db8ce08fac93a098f27eaf425d2269be376 +F src/where.c df58d6ad7878a08aa96c652ccbc6d0949f8fa472 F src/whereInt.h 14dd243e13b81cbb0a66063d38b70f93a7d6e613 F src/wherecode.c 3aff7683566af3428f865904aafa7efb1fbd8701 F src/whereexpr.c b896f8ff6a53cbd3daaee84ec33e39098762bb46 @@ -845,6 +845,7 @@ F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test 8621491915800ec274609e42e02a97d67e9b13e7 F test/index6.test 43b4e29258b978fcdab84fc61df4f5212119dd09 F test/index7.test 9c6765a74fc3fcde7aebc5b3bd40d98df14a527c +F test/index8.test bc2e3db70e8e62459aaa1bd7e4a9b39664f8f9d7 F test/indexedby.test 9c4cd331224e57f79fbf411ae245e6272d415985 F test/indexexpr1.test cb71b6586177b840e28110dd952178bb2bdfedc2 F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d @@ -984,7 +985,7 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 84e4cc5cbca285357f7906e99b21be4f2bf5abc0 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 -F test/parser1.test 222b5cbf3e2e659fec1bf7d723488c8b9c94f1d0 +F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff @@ -1029,7 +1030,7 @@ F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test cde525ea3075283eb950cdcdefe23ead4f700daa F test/savepointfault.test f044eac64b59f09746c7020ee261734de82bf9b2 -F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 +F test/scanstatus.test 030acbbdcea6a3fc676fee99edc84f6f16c0cf92 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1510,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d4f3d52c5a11fc7ad7e8cad76361edbcef13a12f 4d59df02d3713b3e3804e1a88e676749b2794286 -R 1d3755c62e27d021f9887ac09a170360 +P 719a3b2035a335ca8b9704646b1d641011e3ea0e 6feff15cae8f0427be790355841d49c479c1c586 +R 3c287799720eaa100d938a8b5fd4645d U dan -Z 2549ddc5292b60c5848a8cde952880f9 +Z 3b75e2ba5b9150b77f0261ae152b7ba4 diff --git a/manifest.uuid b/manifest.uuid index 21c7478815..88af2bcaa6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -719a3b2035a335ca8b9704646b1d641011e3ea0e \ No newline at end of file +9685880f7baeb670739fdcf2d9df08e22abaa699 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 3c95136c5c..26587309f2 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4312,6 +4312,61 @@ int sqlite3ExprImpliesExpr(Expr *pE1, Expr *pE2, int iTab){ return 0; } +/* +** An instance of the following structure is used by the tree walker +** to determine if an expression can be evaluated by reference to the +** index only, without having to do a search for the corresponding +** table entry. The IdxCover.pIdx field is the index. IdxCover.iCur +** is the cursor for the table. +*/ +struct IdxCover { + Index *pIdx; /* The index to be tested for coverage */ + int iCur; /* Cursor number for the table corresponding to the index */ +}; + +/* +** Check to see if there are references to columns in table +** pWalker->u.pIdxCover->iCur can be satisfied using the index +** pWalker->u.pIdxCover->pIdx. +*/ +static int exprIdxCover(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pWalker->u.pIdxCover->iCur + && sqlite3ColumnOfIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Continue; +} + +/* +** Determine if an index pIdx on table with cursor iCur contains will +** the expression pExpr. Return true if the index does cover the +** expression and false if the pExpr expression references table columns +** that are not found in the index pIdx. +** +** An index covering an expression means that the expression can be +** evaluated using only the index and without having to lookup the +** corresponding table entry. +*/ +int sqlite3ExprCoveredByIndex( + Expr *pExpr, /* The index to be tested */ + int iCur, /* The cursor number for the corresponding table */ + Index *pIdx /* The index that might be used for coverage */ +){ + Walker w; + struct IdxCover xcov; + memset(&w, 0, sizeof(w)); + xcov.iCur = iCur; + xcov.pIdx = pIdx; + w.xExprCallback = exprIdxCover; + w.u.pIdxCover = &xcov; + sqlite3WalkExpr(&w, pExpr); + return !w.eCode; +} + + /* ** An instance of the following structure is used by the tree walker ** to count references to table columns in the arguments of an diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7219732bc3..ea41e65755 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3258,6 +3258,7 @@ struct Walker { struct SrcCount *pSrcCount; /* Counting column references */ struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ int *aiCol; /* array of column indexes */ + struct IdxCover *pIdxCover; /* Check for index coverage */ } u; }; @@ -3701,6 +3702,7 @@ int sqlite3ExprListCompare(ExprList*, ExprList*, int); int sqlite3ExprImpliesExpr(Expr*, Expr*, int); void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); +int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); Vdbe *sqlite3GetVdbe(Parse*); #ifndef SQLITE_OMIT_BUILTIN_TEST diff --git a/src/where.c b/src/where.c index 6c039c6b7c..fdf0be7358 100644 --- a/src/where.c +++ b/src/where.c @@ -2539,11 +2539,11 @@ static int whereLoopAddBtreeIndex( pNew->nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + nIter = pProbe->aiRowLogEst[saved_nEq]+1 - pProbe->aiRowLogEst[saved_nEq+1]; pNew->nOut -= nIter; /* TUNING: Because uncertainties in the estimates for skip-scan queries, ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ - nIter += 5; + nIter += 4; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); pNew->nOut = saved_nOut; pNew->u.btree.nEq = saved_nEq; @@ -2838,11 +2838,34 @@ static int whereLoopAddBtree( /* The cost of visiting the index rows is N*K, where K is ** between 1.1 and 3.0, depending on the relative sizes of the - ** index and table rows. If this is a non-covering index scan, - ** also add the cost of visiting table rows (N*3.0). */ + ** index and table rows. */ pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; if( m!=0 ){ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); + /* If this is a non-covering index scan, add in the cost of + ** doing table lookups. The cost will be 3x the number of + ** lookups. Take into account WHERE clause terms that can be + ** satisfied using just the index, and that do not require a + ** table lookup. */ + LogEst nLookup = rSize + 16; /* Base cost: N*3 */ + int ii; + int iCur = pSrc->iCursor; + WhereClause *pWC = &pWInfo->sWC; + for(ii=0; iinTerm; ii++){ + WhereTerm *pTerm = &pWC->a[ii]; + if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){ + break; + } + /* pTerm can be evaluated using just the index. So reduce + ** the expected number of table lookups accordingly */ + if( pTerm->truthProb<=0 ){ + nLookup += pTerm->truthProb; + }else{ + nLookup--; + if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19; + } + } + + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup); } ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); diff --git a/test/index8.test b/test/index8.test new file mode 100644 index 0000000000..bb58228527 --- /dev/null +++ b/test/index8.test @@ -0,0 +1,60 @@ +# 2016-07-27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for ORDER BY and LIMIT on an index scan. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Performance regression reported at +# http://www.mail-archive.com/sqlite-users@mailinglists.sqlite.org/msg98615.html +# +# Caused by the ORDER BY LIMIT optionation for check-in +# https://sqlite.org/src/info/bf46179d44843769 +# +# Fixed on approximately 2016-07-27 by changes that compute a better score +# for index scans by taking into account WHERE clause constraints that can +# be handled by the index and do not require a table lookup. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a,b,c,d); + WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM c WHERE x<100) + INSERT INTO t1(a,b,c,d) + SELECT x/10, x%10, x%19, x FROM c; + CREATE INDEX t1abc ON t1(a,b,c); + SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2; +} {0 4 4 4 2 3 4 23} + +# Prior to the fix, the following EQP would show a table scan and a sort +# rather than an index scan. +# +do_execsql_test 1.0eqp { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2; +} {/SCAN TABLE t1 USING INDEX t1abc/} + +# If we change the index so that it no longer covers the WHERE clause, +# then we should (correctly) revert to using a table scan. +# +do_execsql_test 1.1 { + DROP INDEX t1abc; + CREATE INDEX t1abd ON t1(a,b,d); + SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2; +} {0 4 4 4 2 3 4 23} +do_execsql_test 1.1eqp { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE c=4 ORDER BY a, b LIMIT 2; +} {~/USING INDEX/} + + +finish_test diff --git a/test/parser1.test b/test/parser1.test index 78c1a40c63..c708dded1f 100644 --- a/test/parser1.test +++ b/test/parser1.test @@ -76,4 +76,27 @@ do_catchsql_test parser1-2.2 { SELECT x FROM c; } {1 {syntax error after column name "x"}} +# Verify that the comma between multiple table constraints is +# optional. +# +# The missing comma is technically a syntax error. But we have to support +# it because there might be legacy databases that omit the commas in their +# sqlite_master tables. +# +do_execsql_test parser1-3.1 { + CREATE TABLE t300(id INTEGER PRIMARY KEY); + CREATE TABLE t301( + id INTEGER PRIMARY KEY, + c1 INTEGER NOT NULL, + c2 INTEGER NOT NULL, + c3 BOOLEAN NOT NULL DEFAULT 0, + FOREIGN KEY(c1) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT + /* no comma */ + FOREIGN KEY(c2) REFERENCES t300(id) ON DELETE CASCADE ON UPDATE RESTRICT + /* no comma */ + UNIQUE(c1, c2) + ); + PRAGMA foreign_key_list(t301); +} {0 0 t300 c2 id RESTRICT CASCADE NONE 1 0 t300 c1 id RESTRICT CASCADE NONE} + finish_test diff --git a/test/scanstatus.test b/test/scanstatus.test index ed24d97437..48365a8794 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -333,7 +333,7 @@ do_execsql_test 5.3.2 { SELECT count(*) FROM t2 WHERE y = 'j'; } {19} do_scanstatus_test 5.3.3 { - nLoop 1 nVisit 19 nEst 56.0 zName t2xy zExplain + nLoop 1 nVisit 19 nEst 52.0 zName t2xy zExplain {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} } @@ -349,7 +349,7 @@ do_execsql_test 5.4.2 { do_scanstatus_test 5.4.3 { nLoop 1 nVisit 10 nEst 10.0 zName t1bc zExplain {SCAN TABLE t1 USING COVERING INDEX t1bc} - nLoop 10 nVisit 200 nEst 56.0 zName t2xy + nLoop 10 nVisit 200 nEst 52.0 zName t2xy zExplain {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} }