From ab31a8450b5d4a122c8157d2aaf281b8844b0d6b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Apr 2017 20:53:09 +0000 Subject: [PATCH 01/11] Automatically transfer terms from the HAVING clause to the WHERE clause of an aggregate query in cases where the result of evaluating the term depends only one one or more of the GROUP BY expressions (and on no other inputs). FossilOrigin-Name: 5375a3ce56f1d993b13b469fe33ec7679948f53940f62a15ddbaeb8aaa26a22c --- manifest | 23 ++++++------ manifest.uuid | 2 +- src/expr.c | 45 +++++++++++++++++++++++ src/select.c | 78 ++++++++++++++++++++++++++++++++++++++++ src/sqliteInt.h | 21 ++++++----- test/having.test | 94 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 243 insertions(+), 20 deletions(-) create mode 100644 test/having.test diff --git a/manifest b/manifest index bf8f046722..a30d603c08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Evaluate\sWHERE\sclause\sterms\sthat\sreference\sonly\sthe\sindex\sbefore\sevaluating\nterms\sthat\srequire\sthe\stable,\sand\sthereby\savoid\sseeking\sthe\stable\srow\sif\nindex\sterms\sare\sfalse.\nThis\sis\scalled\sthe\s"push-down"\soptimization\sin\sthe\sMySQL\sworld,\swe\sare\stold. -D 2017-04-29T15:27:04.896 +C Automatically\stransfer\sterms\sfrom\sthe\sHAVING\sclause\sto\sthe\sWHERE\sclause\sof\san\naggregate\squery\sin\scases\swhere\sthe\sresult\sof\sevaluating\sthe\sterm\sdepends\sonly\none\sone\sor\smore\sof\sthe\sGROUP\sBY\sexpressions\s(and\son\sno\sother\sinputs). +D 2017-04-29T20:53:09.360 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -355,7 +355,7 @@ F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c F src/date.c cc42a41c7422389860d40419a5e3bce5eaf6e7835c3ba2677751dc653550a5c7 F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c f10e35dc50be4c8f82eb99bf5d8530229d1d60957cc3c9473ffe584d0444087c +F src/expr.c ed8914c001a24ab25aee965f498c96de577bf2c4a6145022b3688c11c2b29c82 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c db65492ae549c3b548c9ef1f279ce1684f1c473b116e1c56a90878cd5dcf968d F src/func.c 9d52522cc8ae7f5cdadfe14594262f1618bc1f86083c4cd6da861b4cf5af6174 @@ -402,12 +402,12 @@ F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c bf8ab605e49717c222136380453cfb7eda564f8e500d5ff6a01341ea59fefe80 +F src/select.c a2b839b8cfa75d154490839ee80bad3452b9810ad39862ad044d154211292866 F src/shell.c 21b79c0e1b93f8e35fd7b4087d6ba438326c3d7e285d0dd51dfd741475f858a1 F src/sqlite.h.in 40233103e3e4e10f8a63523498d0259d232e42aba478e2d3fb914799185aced6 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28 -F src/sqliteInt.h 9affb53bb405dcea1d86e85198ebaf6232a684cc2b2af6b3c181869f1c8f3e93 +F src/sqliteInt.h aea3aa1b81e0d07d5b1c39b8c5a54a1dc5e4f10136cb63da392aef9eb2a5108b F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -850,6 +850,7 @@ F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 +F test/having.test a03676a754815628a08d3b96d506dd9eda8ffbdd356cd8ea9c2e5368286fbe6a F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5 @@ -1577,8 +1578,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2d0b64316d66a362f5891ceb71a1fd8e4607732274b09b0a8472b97eef68ebc2 91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d -R d3fbd82e46a4092d676255e41f20df5b -T +closed 91dfb61a1a25763bb0b5c1e353a9d584bc6de3f6eb445f54202ffe7f6fee6e8d -U drh -Z d8af2da2fe4450264227719b4f172b61 +P d7bb79ed3a40419d143fbe35c310e51fe7b384a22f082a61ad788671d2d33ee0 +R bd50b7d1e9edcc2d5d20fca02c02c5c7 +T *branch * having-where-optimization +T *sym-having-where-optimization * +T -sym-trunk * +U dan +Z 94cbfc69ef14a78710cd01b746487e79 diff --git a/manifest.uuid b/manifest.uuid index a192425962..57fc2eb5ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7bb79ed3a40419d143fbe35c310e51fe7b384a22f082a61ad788671d2d33ee0 \ No newline at end of file +5375a3ce56f1d993b13b469fe33ec7679948f53940f62a15ddbaeb8aaa26a22c \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 873911633f..73143c51de 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1815,6 +1815,51 @@ int sqlite3ExprIsTableConstant(Expr *p, int iCur){ return exprIsConst(p, 3, iCur); } + +/* +** sqlite3WalkExpr() callback used by sqlite3ExprIsConstantOrGroupBy(). +*/ +static int exprNodeIsConstantOrGroupBy(Walker *pWalker, Expr *pExpr){ + ExprList *pGroupBy = pWalker->u.pGroupBy; + int i; + + /* Check if pExpr is identical to any GROUP BY term. If so, consider + ** it constant. */ + for(i=0; inExpr; i++){ + Expr *p = pGroupBy->a[i].pExpr; + if( sqlite3ExprCompare(pExpr, p, -1)<2 ){ + CollSeq *pColl = sqlite3ExprCollSeq(pWalker->pParse, p); + if( pColl==0 || sqlite3_stricmp("BINARY", pColl->zName)==0 ){ + return WRC_Prune; + } + } + } + + /* Check if pExpr is a sub-select. If so, consider it variable. */ + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + pWalker->eCode = 0; + return WRC_Abort; + } + + return exprNodeIsConstant(pWalker, pExpr); +} + +/* +** Walk the expression tree passed as the first argument. Return non-zero +** if the expression consists entirely of constants or copies of terms +** in pGroupBy that sort with the BINARY collation sequence. +*/ +int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprList *pGroupBy){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 1; + w.xExprCallback = exprNodeIsConstantOrGroupBy; + w.u.pGroupBy = pGroupBy; + w.pParse = pParse; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + /* ** Walk an expression tree. Return non-zero if the expression is constant ** or a function call with constant arguments. Return and 0 if there diff --git a/src/select.c b/src/select.c index 3def13618e..7a195560e6 100644 --- a/src/select.c +++ b/src/select.c @@ -4879,6 +4879,79 @@ static void explainSimpleCount( # define explainSimpleCount(a,b,c) #endif +/* +** Context object for havingToWhereExprCb(). +*/ +struct HavingToWhereCtx { + Expr **ppWhere; + ExprList *pGroupBy; +}; + +/* +** sqlite3WalkExpr() callback used by havingToWhere(). +** +** If the node passed to the callback is a TK_AND node, return +** WRC_Continue to tell sqlite3WalkExpr() to iterate through child nodes. +** +** Otherwise, return WRC_Prune. In this case, also check if the +** sub-expression matches the criteria for being moved to the WHERE +** clause. If so, add it to the WHERE clause and replace the sub-expression +** within the HAVING expression with a constant "1". +*/ +static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ + if( pExpr->op!=TK_AND ){ + struct HavingToWhereCtx *p = pWalker->u.pHavingCtx; + if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, p->pGroupBy) ){ + sqlite3 *db = pWalker->pParse->db; + Expr *pNew = sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[1], 0); + if( pNew ){ + Expr *pWhere = *(p->ppWhere); + SWAP(Expr, *pNew, *pExpr); + if( pWhere ){ + pNew = sqlite3ExprAnd(db, pWhere, pNew); + } + *(p->ppWhere) = pNew; + } + } + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Transfer eligible terms from the HAVING clause of a query, which is +** processed after grouping, to the WHERE clause, which is processed before +** grouping. For example, the query: +** +** SELECT * FROM WHERE a=? GROUP BY b HAVING b=? AND c=? +** +** can be rewritten as: +** +** SELECT * FROM WHERE a=? AND b=? GROUP BY b HAVING c=? +** +** A term of the HAVING expression is eligible for transfer if it consists +** entirely of constants and expressions that are also GROUP BY terms that +** use the "BINARY" collation sequence. +*/ +static void havingToWhere( + Parse *pParse, + ExprList *pGroupBy, + Expr *pHaving, + Expr **ppWhere +){ + struct HavingToWhereCtx sCtx; + Walker sWalker; + + sCtx.ppWhere = ppWhere; + sCtx.pGroupBy = pGroupBy; + + memset(&sWalker, 0, sizeof(sWalker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = havingToWhereExprCb; + sWalker.u.pHavingCtx = &sCtx; + sqlite3WalkExpr(&sWalker, pHaving); +} + /* ** Generate code for the SELECT statement given in the p argument. ** @@ -5343,6 +5416,11 @@ int sqlite3Select( sqlite3ExprAnalyzeAggList(&sNC, pEList); sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); if( pHaving ){ + if( pGroupBy ){ + assert( pWhere==p->pWhere ); + havingToWhere(pParse, pGroupBy, pHaving, &p->pWhere); + pWhere = p->pWhere; + } sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } sAggInfo.nAccumulator = sAggInfo.nColumn; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 40660aed99..fc24885e28 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3316,15 +3316,17 @@ struct Walker { int walkerDepth; /* Number of subqueries */ u8 eCode; /* A small processing code */ union { /* Extra data for callback */ - NameContext *pNC; /* Naming context */ - int n; /* A counter */ - int iCur; /* A cursor number */ - SrcList *pSrcList; /* FROM clause */ - struct SrcCount *pSrcCount; /* Counting column references */ - struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ - int *aiCol; /* array of column indexes */ - struct IdxCover *pIdxCover; /* Check for index coverage */ - struct IdxExprTrans *pIdxTrans; /* Convert indexed expr to column */ + NameContext *pNC; /* Naming context */ + int n; /* A counter */ + int iCur; /* A cursor number */ + SrcList *pSrcList; /* FROM clause */ + struct SrcCount *pSrcCount; /* Counting column references */ + struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ + int *aiCol; /* array of column indexes */ + struct IdxCover *pIdxCover; /* Check for index coverage */ + struct IdxExprTrans *pIdxTrans; /* Convert indexed expr to column */ + ExprList *pGroupBy; /* GROUP BY clause */ + struct HavingToWhereCtx *pHavingCtx; /* HAVING to WHERE clause ctx */ } u; }; @@ -3794,6 +3796,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); +int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); int sqlite3ExprIsTableConstant(Expr*,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS int sqlite3ExprContainsSubquery(Expr*); diff --git a/test/having.test b/test/having.test new file mode 100644 index 0000000000..7763b06850 --- /dev/null +++ b/test/having.test @@ -0,0 +1,94 @@ +# 2017 April 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test the HAVING->WHERE optimization. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix having + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); + INSERT INTO t1 VALUES(1, 3); + INSERT INTO t1 VALUES(2, 4); + INSERT INTO t1 VALUES(1, 5); + INSERT INTO t1 VALUES(2, 6); +} {} + +foreach {tn sql res} { + 1 "SELECT a, sum(b) FROM t1 GROUP BY a HAVING a=2" {2 12} + 2 "SELECT a, sum(b) FROM t1 GROUP BY a HAVING a=2 AND sum(b)>10" {2 12} + 3 "SELECT a, sum(b) FROM t1 GROUP BY a HAVING sum(b)>12" {} +} { + do_execsql_test 1.$tn $sql $res +} + +proc compare_vdbe {sql1 sql2} { + set r1 [list] + set r2 [list] + db eval "explain $sql1" { lappend r1 $opcode $p1 $p2 $p3} + db eval "explain $sql2" { lappend r2 $opcode $p1 $p2 $p3} + return [expr {$r1==$r2}] +} + +proc do_compare_vdbe_test {tn sql1 sql2 res} { + uplevel [list do_test $tn [list compare_vdbe $sql1 $sql2] $res] +} + +do_compare_vdbe_test 2.1 { + SELECT a, sum(b) FROM t1 GROUP BY a HAVING a=2 +} { + SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a +} 1 +do_compare_vdbe_test 2.2 { + SELECT a, sum(b) FROM t1 GROUP BY a+1 HAVING a=2 +} { + SELECT a, sum(b) FROM t1 GROUP BY a+1 HAVING a=2 +} 1 + +foreach {tn sql1 sql2} { + 1 "SELECT a, sum(b) FROM t1 GROUP BY a HAVING a=2" + "SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a" + + 2 "SELECT a, sum(b) FROM t1 GROUP BY a HAVING sum(b)>5 AND a=2" + "SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a HAVING sum(b)>5" + + 3 "SELECT a, sum(b) FROM t1 GROUP BY a COLLATE binary HAVING a=2" + "SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a COLLATE binary" + + 4 { + SELECT x,y FROM ( + SELECT a AS x, sum(b) AS y FROM t1 + GROUP BY a + ) WHERE x BETWEEN 8888 AND 9999 + } { + SELECT x,y FROM ( + SELECT a AS x, sum(b) AS y FROM t1 + WHERE x BETWEEN 8888 AND 9999 + GROUP BY a + ) + } +} { + do_compare_vdbe_test 3.$tn $sql1 $sql2 1 +} + +foreach {tn sql1 sql2} { + 1 "SELECT a, sum(b) FROM t1 GROUP BY a COLLATE nocase HAVING a=2" + "SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a COLLATE nocase" +} { + do_compare_vdbe_test 4.$tn $sql1 $sql2 0 +} + +finish_test + From 181a1167dc24f86e106e62a3d9874099fe2cdebd Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 May 2017 14:09:32 +0000 Subject: [PATCH 02/11] Add extra tests for the optimization on this branch. FossilOrigin-Name: 4921cd9520080f9baff70e548f64a56e2204b398b8397a2d318a98c32ec4b00c --- manifest | 15 ++++---- manifest.uuid | 2 +- test/having.test | 90 ++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 82 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index a30d603c08..d89bebfd83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Automatically\stransfer\sterms\sfrom\sthe\sHAVING\sclause\sto\sthe\sWHERE\sclause\sof\san\naggregate\squery\sin\scases\swhere\sthe\sresult\sof\sevaluating\sthe\sterm\sdepends\sonly\none\sone\sor\smore\sof\sthe\sGROUP\sBY\sexpressions\s(and\son\sno\sother\sinputs). -D 2017-04-29T20:53:09.360 +C Add\sextra\stests\sfor\sthe\soptimization\son\sthis\sbranch. +D 2017-05-01T14:09:32.423 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -850,7 +850,7 @@ F test/fuzzer2.test a85ef814ce071293bce1ad8dffa217cbbaad4c14 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/gcfault.test dd28c228a38976d6336a3fc42d7e5f1ad060cb8c F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 -F test/having.test a03676a754815628a08d3b96d506dd9eda8ffbdd356cd8ea9c2e5368286fbe6a +F test/having.test 30a02b8a9a47cba7bdb5281999c5cbff407c2ac296511ee64dd0b418fe38eb0f F test/hexlit.test 4a6a5f46e3c65c4bf1fa06f5dd5a9507a5627751 F test/hidden.test 23c1393a79e846d68fd902d72c85d5e5dcf98711 F test/hook.test dbc0b87756e1e20e7497b56889c9e9cd2f8cc2b5 @@ -1578,10 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d7bb79ed3a40419d143fbe35c310e51fe7b384a22f082a61ad788671d2d33ee0 -R bd50b7d1e9edcc2d5d20fca02c02c5c7 -T *branch * having-where-optimization -T *sym-having-where-optimization * -T -sym-trunk * +P 5375a3ce56f1d993b13b469fe33ec7679948f53940f62a15ddbaeb8aaa26a22c +R 123e296789286f2ee6ee6640a2471eff U dan -Z 94cbfc69ef14a78710cd01b746487e79 +Z 8431f2088d761010cb187c22f8f8d978 diff --git a/manifest.uuid b/manifest.uuid index 57fc2eb5ff..2428b6a0d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5375a3ce56f1d993b13b469fe33ec7679948f53940f62a15ddbaeb8aaa26a22c \ No newline at end of file +4921cd9520080f9baff70e548f64a56e2204b398b8397a2d318a98c32ec4b00c \ No newline at end of file diff --git a/test/having.test b/test/having.test index 7763b06850..6d2f9fdcc5 100644 --- a/test/having.test +++ b/test/having.test @@ -17,6 +17,8 @@ source $testdir/tester.tcl set testprefix having do_execsql_test 1.0 { + CREATE TABLE t2(c, d); + CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 1); INSERT INTO t1 VALUES(2, 2); @@ -34,11 +36,14 @@ foreach {tn sql res} { do_execsql_test 1.$tn $sql $res } +# Run an EXPLAIN command for both SQL statements. Return true if +# the outputs are identical, or false otherwise. +# proc compare_vdbe {sql1 sql2} { set r1 [list] set r2 [list] - db eval "explain $sql1" { lappend r1 $opcode $p1 $p2 $p3} - db eval "explain $sql2" { lappend r2 $opcode $p1 $p2 $p3} + db eval "explain $sql1" { lappend r1 $opcode $p1 $p2 $p3 $p4 $p5} + db eval "explain $sql2" { lappend r2 $opcode $p1 $p2 $p3 $p4 $p5} return [expr {$r1==$r2}] } @@ -46,17 +51,10 @@ proc do_compare_vdbe_test {tn sql1 sql2 res} { uplevel [list do_test $tn [list compare_vdbe $sql1 $sql2] $res] } -do_compare_vdbe_test 2.1 { - SELECT a, sum(b) FROM t1 GROUP BY a HAVING a=2 -} { - SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a -} 1 -do_compare_vdbe_test 2.2 { - SELECT a, sum(b) FROM t1 GROUP BY a+1 HAVING a=2 -} { - SELECT a, sum(b) FROM t1 GROUP BY a+1 HAVING a=2 -} 1 - +#------------------------------------------------------------------------- +# Test that various statements that are eligible for the optimization +# produce the same VDBE code as optimizing by hand does. +# foreach {tn sql1 sql2} { 1 "SELECT a, sum(b) FROM t1 GROUP BY a HAVING a=2" "SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a" @@ -79,16 +77,78 @@ foreach {tn sql1 sql2} { GROUP BY a ) } + + 5 "SELECT a, sum(b) FROM t1 GROUP BY a COLLATE binary HAVING 0" + "SELECT a, sum(b) FROM t1 WHERE 0 GROUP BY a COLLATE binary" + + 6 "SELECT count(*) FROM t1,t2 WHERE a=c GROUP BY b, d HAVING b=d" + "SELECT count(*) FROM t1,t2 WHERE a=c AND b=d GROUP BY b, d" + + 7 { + SELECT count(*) FROM t1,t2 WHERE a=c GROUP BY b, d + HAVING b=d COLLATE nocase + } { + SELECT count(*) FROM t1,t2 WHERE a=c AND b=d COLLATE nocase + GROUP BY b, d + } + + 8 "SELECT a, sum(b) FROM t1 GROUP BY a||b HAVING substr(a||b, 1, 1)='a'" + "SELECT a, sum(b) FROM t1 WHERE substr(a||b, 1, 1)='a' GROUP BY a||b" } { - do_compare_vdbe_test 3.$tn $sql1 $sql2 1 + do_compare_vdbe_test 2.$tn $sql1 $sql2 1 } +#------------------------------------------------------------------------- +# 1: Test that the optimization is only applied if the GROUP BY term +# uses BINARY collation. +# +# 2: Not applied if there is a non-deterministic function in the HAVING +# term. +# foreach {tn sql1 sql2} { 1 "SELECT a, sum(b) FROM t1 GROUP BY a COLLATE nocase HAVING a=2" "SELECT a, sum(b) FROM t1 WHERE a=2 GROUP BY a COLLATE nocase" + + 2 "SELECT a, sum(b) FROM t1 GROUP BY a HAVING randomblob(a) Date: Mon, 1 May 2017 15:15:41 +0000 Subject: [PATCH 03/11] Initial implementation of an optimization that attempts to reuse the same materialization of a view on a self-join of the view. FossilOrigin-Name: 478c34b9a8b5127d13024e10307aa832f160b89720c46424dd17555bd36f590d --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/select.c | 31 ++++++++++++++++++++++++++++--- src/vdbe.c | 28 ++++++++++++++++++++++++++++ src/vdbeaux.c | 2 +- 5 files changed, 69 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 22a472ed22..14baff5c8e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\ssingle\stestcase()\smacro\sto\sthe\ssubquery\sprocessing\slogic. -D 2017-04-29T19:29:49.700 +C Initial\simplementation\sof\san\soptimization\sthat\sattempts\sto\sreuse\sthe\ssame\nmaterialization\sof\sa\sview\son\sa\sself-join\sof\sthe\sview. +D 2017-05-01T15:15:41.259 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -402,7 +402,7 @@ F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 478e95d424bb86d34c7c95d20872cbd78df97af5f83c3fd7de55d5b2413f927d +F src/select.c 1fba56aa7d6530a37e173e5144df92ad6140c472f9807c168a2ebff096b590c7 F src/shell.c 21b79c0e1b93f8e35fd7b4087d6ba438326c3d7e285d0dd51dfd741475f858a1 F src/sqlite.h.in 40233103e3e4e10f8a63523498d0259d232e42aba478e2d3fb914799185aced6 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -469,11 +469,11 @@ F src/update.c c443935c652af9365e033f756550b5032d02e1b06eb2cb890ed7511ae0c051dc F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c F src/util.c ca8440ede81e155d15cff7c101654f60b55a9ae6 F src/vacuum.c 1fe4555cd8c9b263afb85b5b4ee3a4a4181ad569 -F src/vdbe.c 349eb6789cf8b03ef1aa42271aa9a17a6a7794c07448d27c5405904ba1b21715 +F src/vdbe.c 7c37c36f3d528b10e5a5df76267ca42ed604fb3f15ff1d656afe1f33ff5d1f76 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848 F src/vdbeInt.h c070bc5c8b913bda0ceaa995cd4d939ded5e4fc96cf7c3c1c602d41b871f8ade F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 -F src/vdbeaux.c 6b3f6ce909e206d4c918988b13b7fa687e92b4471d137e0f2a37edac80ec60be +F src/vdbeaux.c bb99698cfee2d2fd9fa2870f8a24153475793b9c85cb626b412d6bad8c3a73a8 F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 2c70f8f5de6c71fb99a22c5b83be9fab5c47cdd8e279fa44a8c00cfed06d7e89 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c @@ -1577,7 +1577,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e54c9f8db5b2fa8ea82c6eab7482255431af16901f7992c9667b56a0e50a9f4f -R 6f21ea13ed166571d20351d2145d6640 +P 4e1df76e3d85922648e0e1cce73a266c3b1ed4511ace259ec0a01d7693af9e6f +R 9a92f7b5439fceb732be32bb3fe5b30d +T *branch * materialized-view-reuse +T *sym-materialized-view-reuse * +T -sym-trunk * U drh -Z 4a7e86f7be3de966a53a14c1a17e31d5 +Z 0a08980cdb3c200bc98cb9ecc97a5f40 diff --git a/manifest.uuid b/manifest.uuid index 45bf2ccb33..5ca2086bf7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e1df76e3d85922648e0e1cce73a266c3b1ed4511ace259ec0a01d7693af9e6f \ No newline at end of file +478c34b9a8b5127d13024e10307aa832f160b89720c46424dd17555bd36f590d \ No newline at end of file diff --git a/src/select.c b/src/select.c index 77824c4aaa..69c75fb96a 100644 --- a/src/select.c +++ b/src/select.c @@ -4879,6 +4879,24 @@ static void explainSimpleCount( # define explainSimpleCount(a,b,c) #endif +/* +** Check to see if the pThis entry of pTabList is a self-join of a prior view. +** If it is, then return the SrcList_item for the prior view. If it is not, +** then return 0. +*/ +static struct SrcList_item *isSelfJoinView( + SrcList *pTabList, /* Search for self-joins in this FROM clause */ + struct SrcList_item *pThis /* Search for prior reference to this subquery */ +){ + struct SrcList_item *pItem; + for(pItem = pTabList->a; pItempSelect==0 ) continue; + if( pItem->fg.viaCoroutine ) continue; + if( sqlite3StrICmp(pItem->zName, pThis->zName)==0 ) return pItem; + } + return 0; +} + /* ** Generate code for the SELECT statement given in the p argument. ** @@ -5113,6 +5131,8 @@ int sqlite3Select( int topAddr; int onceAddr = 0; int retAddr; + struct SrcList_item *pPrior; + assert( pItem->addrFillSub==0 ); pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); @@ -5126,9 +5146,14 @@ int sqlite3Select( }else{ VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); } - sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); - sqlite3Select(pParse, pSub, &dest); + pPrior = isSelfJoinView(pTabList, pItem); + if( pPrior ){ + sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); + }else{ + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + } pItem->pTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); diff --git a/src/vdbe.c b/src/vdbe.c index 15f015a598..ebf80c8beb 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3540,6 +3540,34 @@ open_cursor_set_hints: break; } +/* Opcode: OpenDup P1 P2 * * * +** +** Open a new cursor P1 that points to the same ephemeral table as +** cursor P2. The P2 cursor must have been opened by a prior OP_OpenEphemeral +** opcode. Only ephemeral cursors may be duplicated. +** +** Duplicate ephemeral cursors are used for self-joins of materialized views. +*/ +case OP_OpenDup: { + VdbeCursor *pOrig; /* The original cursor to be duplicated */ + VdbeCursor *pCx; /* The new cursor */ + + pOrig = p->apCsr[pOp->p2]; + assert( pOrig->pBtx!=0 ); /* Only ephemeral cursors can be duplicated */ + + pCx = allocateCursor(p, pOp->p1, pOrig->nField, -1, CURTYPE_BTREE); + if( pCx==0 ) goto no_mem; + pCx->nullRow = 1; + pCx->isEphemeral = 1; + pCx->pKeyInfo = pOrig->pKeyInfo; + pCx->isTable = pOrig->isTable; + rc = sqlite3BtreeCursor(pOrig->pBtx, MASTER_ROOT, BTREE_WRCSR, + pCx->pKeyInfo, pCx->uc.pCursor); + if( rc ) goto abort_due_to_error; + break; +} + + /* Opcode: OpenEphemeral P1 P2 * P4 P5 ** Synopsis: nColumn=P2 ** diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 00a5ec91a9..e903f4bbcd 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2039,7 +2039,7 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ sqlite3BtreeClose(pCx->pBtx); /* The pCx->pCursor will be close automatically, if it exists, by ** the call above. */ - }else{ + }else if( !pCx->isEphemeral ){ assert( pCx->uc.pCursor!=0 ); sqlite3BtreeCloseCursor(pCx->uc.pCursor); } From 33543c237b38936a3beab3475e9664c20dfa4bfe Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 May 2017 16:37:20 +0000 Subject: [PATCH 04/11] Minor bug fixes and performance enhancement. FossilOrigin-Name: b2aae559581d05e4e8caaddfbd137cb275d582e82d58872c6f42e462dc859b1c --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/select.c | 4 +++- src/vdbeaux.c | 6 +++--- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 14baff5c8e..472fdaea3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\simplementation\sof\san\soptimization\sthat\sattempts\sto\sreuse\sthe\ssame\nmaterialization\sof\sa\sview\son\sa\sself-join\sof\sthe\sview. -D 2017-05-01T15:15:41.259 +C Minor\sbug\sfixes\sand\sperformance\senhancement. +D 2017-05-01T16:37:20.261 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -402,7 +402,7 @@ F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c 1fba56aa7d6530a37e173e5144df92ad6140c472f9807c168a2ebff096b590c7 +F src/select.c c9aa9575ccdb85e92167e00cd7b955221533df47679828c4fa6765f11f5dcd12 F src/shell.c 21b79c0e1b93f8e35fd7b4087d6ba438326c3d7e285d0dd51dfd741475f858a1 F src/sqlite.h.in 40233103e3e4e10f8a63523498d0259d232e42aba478e2d3fb914799185aced6 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -473,7 +473,7 @@ F src/vdbe.c 7c37c36f3d528b10e5a5df76267ca42ed604fb3f15ff1d656afe1f33ff5d1f76 F src/vdbe.h f7d1456e28875c2dcb964056589b5b7149ab7edf39edeca801596a39bb3d3848 F src/vdbeInt.h c070bc5c8b913bda0ceaa995cd4d939ded5e4fc96cf7c3c1c602d41b871f8ade F src/vdbeapi.c 5b08d82592bcff4470601fe78aaabebd50837860 -F src/vdbeaux.c bb99698cfee2d2fd9fa2870f8a24153475793b9c85cb626b412d6bad8c3a73a8 +F src/vdbeaux.c b4999c744e59deba7ab8733640219ecbc771721b362d7e26ce4c57db575ad80b F src/vdbeblob.c 359891617358deefc85bef7bcf787fa6b77facb9 F src/vdbemem.c 2c70f8f5de6c71fb99a22c5b83be9fab5c47cdd8e279fa44a8c00cfed06d7e89 F src/vdbesort.c e72fe02a2121386ba767ede8942e9450878b8fc873abf3d1b6824485f092570c @@ -1577,10 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4e1df76e3d85922648e0e1cce73a266c3b1ed4511ace259ec0a01d7693af9e6f -R 9a92f7b5439fceb732be32bb3fe5b30d -T *branch * materialized-view-reuse -T *sym-materialized-view-reuse * -T -sym-trunk * +P 478c34b9a8b5127d13024e10307aa832f160b89720c46424dd17555bd36f590d +R 2a2ab9e2643c70c79e0d330bc99dce06 U drh -Z 0a08980cdb3c200bc98cb9ecc97a5f40 +Z cc122146d9fb91e0d9a08125d9d2cc46 diff --git a/manifest.uuid b/manifest.uuid index 5ca2086bf7..61b372a2ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -478c34b9a8b5127d13024e10307aa832f160b89720c46424dd17555bd36f590d \ No newline at end of file +b2aae559581d05e4e8caaddfbd137cb275d582e82d58872c6f42e462dc859b1c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 69c75fb96a..df1fd7ed14 100644 --- a/src/select.c +++ b/src/select.c @@ -4892,7 +4892,9 @@ static struct SrcList_item *isSelfJoinView( for(pItem = pTabList->a; pItempSelect==0 ) continue; if( pItem->fg.viaCoroutine ) continue; - if( sqlite3StrICmp(pItem->zName, pThis->zName)==0 ) return pItem; + if( pItem->zName==0 ) continue; + if( sqlite3_stricmp(pItem->zDatabase, pThis->zDatabase)!=0 ) continue; + if( sqlite3_stricmp(pItem->zName, pThis->zName)==0 ) return pItem; } return 0; } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e903f4bbcd..b41ed95b2e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2035,11 +2035,11 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ break; } case CURTYPE_BTREE: { - if( pCx->pBtx ){ - sqlite3BtreeClose(pCx->pBtx); + if( pCx->isEphemeral ){ + if( pCx->pBtx ) sqlite3BtreeClose(pCx->pBtx); /* The pCx->pCursor will be close automatically, if it exists, by ** the call above. */ - }else if( !pCx->isEphemeral ){ + }else{ assert( pCx->uc.pCursor!=0 ); sqlite3BtreeCloseCursor(pCx->uc.pCursor); } From ed712980852ab2abf1d4c6ad26172b8358a75567 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 May 2017 17:04:35 +0000 Subject: [PATCH 05/11] Futher bug fixes to the function that determines when a materialized view can be reused. FossilOrigin-Name: c64fe3a1695925693385d313e9ad2a1d8cb37ddaa8cc19920ae0978c91bc4c2c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 472fdaea3a..221fc04ec8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sbug\sfixes\sand\sperformance\senhancement. -D 2017-05-01T16:37:20.261 +C Futher\sbug\sfixes\sto\sthe\sfunction\sthat\sdetermines\swhen\sa\smaterialized\sview\ncan\sbe\sreused. +D 2017-05-01T17:04:35.162 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -402,7 +402,7 @@ F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c c9aa9575ccdb85e92167e00cd7b955221533df47679828c4fa6765f11f5dcd12 +F src/select.c 2546672c84240e9b9870d93fdccd7f2de98392fd6836d9bf6611aa60a5ace85f F src/shell.c 21b79c0e1b93f8e35fd7b4087d6ba438326c3d7e285d0dd51dfd741475f858a1 F src/sqlite.h.in 40233103e3e4e10f8a63523498d0259d232e42aba478e2d3fb914799185aced6 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1577,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 478c34b9a8b5127d13024e10307aa832f160b89720c46424dd17555bd36f590d -R 2a2ab9e2643c70c79e0d330bc99dce06 +P b2aae559581d05e4e8caaddfbd137cb275d582e82d58872c6f42e462dc859b1c +R fb2a58136338dea6025ffd0d391dd6d5 U drh -Z cc122146d9fb91e0d9a08125d9d2cc46 +Z 522d6c582c8e206ece3fcf0a3e5fff13 diff --git a/manifest.uuid b/manifest.uuid index 61b372a2ad..ac2289d07e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2aae559581d05e4e8caaddfbd137cb275d582e82d58872c6f42e462dc859b1c \ No newline at end of file +c64fe3a1695925693385d313e9ad2a1d8cb37ddaa8cc19920ae0978c91bc4c2c \ No newline at end of file diff --git a/src/select.c b/src/select.c index df1fd7ed14..e7a448343f 100644 --- a/src/select.c +++ b/src/select.c @@ -4894,7 +4894,13 @@ static struct SrcList_item *isSelfJoinView( if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; if( sqlite3_stricmp(pItem->zDatabase, pThis->zDatabase)!=0 ) continue; - if( sqlite3_stricmp(pItem->zName, pThis->zName)==0 ) return pItem; + if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; + if( sqlite3ExprCompare(pThis->pSelect->pWhere, pItem->pSelect->pWhere, -1) ){ + /* The view was modified by some other optimization such as + ** pushDownWhereTerms() */ + continue; + } + return pItem; } return 0; } From 874080bbb6cbf78e91aee3515dbd98acd26f233b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 May 2017 18:12:56 +0000 Subject: [PATCH 06/11] Clear the BTCF_ValidNKey flag set if a cursor is moved by sqlite3BtreeInsert(). Fix for [f68dc596c4]. FossilOrigin-Name: a6727eef6d757a39ad23e5c8cbe960f5d909e5d37cb4e90bc3bdbb8bf58cd6f8 --- manifest | 16 +++---- manifest.uuid | 2 +- src/btree.c | 1 + test/conflict3.test | 102 +++++++++++++++++++++++++------------------- 4 files changed, 68 insertions(+), 53 deletions(-) diff --git a/manifest b/manifest index 22a472ed22..3815ce54e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\ssingle\stestcase()\smacro\sto\sthe\ssubquery\sprocessing\slogic. -D 2017-04-29T19:29:49.700 +C Clear\sthe\sBTCF_ValidNKey\sflag\sset\sif\sa\scursor\sis\smoved\sby\nsqlite3BtreeInsert().\sFix\sfor\s[f68dc596c4]. +D 2017-05-01T18:12:56.867 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -345,7 +345,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c a0d9a1c782ff3d22df5d217a4fa7125dd69ad5849caa51c4442c10246ca8ae27 +F src/btree.c ef42df0cd8fa80ab174c83deb13a216465c49555eb01d57abdfd814a8546fa77 F src/btree.h 80f518c0788be6cec8d9f8e13bd8e380df299d2b5e4ac340dc887b0642647cfc F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610 F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f @@ -604,7 +604,7 @@ F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c -F test/conflict3.test dec0634c0f31dec9a4b01c63063e939f0cd21b6b +F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 F test/contrib01.test 2a1cbc0f2f48955d7d073f725765da6fbceda6b4 F test/corrupt.test 141c39ea650c1365e85a49e402fa05cb9617fb97 F test/corrupt2.test e4964cee73dda57a90958e0087a6b388b1d9cb58 @@ -1577,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e54c9f8db5b2fa8ea82c6eab7482255431af16901f7992c9667b56a0e50a9f4f -R 6f21ea13ed166571d20351d2145d6640 -U drh -Z 4a7e86f7be3de966a53a14c1a17e31d5 +P 4e1df76e3d85922648e0e1cce73a266c3b1ed4511ace259ec0a01d7693af9e6f +R 2f44609e52881f5575acf3fe925fc9aa +U dan +Z 5b866fdf8c8937d7431fdae30fbd42bc diff --git a/manifest.uuid b/manifest.uuid index 45bf2ccb33..a351564742 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e1df76e3d85922648e0e1cce73a266c3b1ed4511ace259ec0a01d7693af9e6f \ No newline at end of file +a6727eef6d757a39ad23e5c8cbe960f5d909e5d37cb4e90bc3bdbb8bf58cd6f8 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 35e47173fa..4ba985f931 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8190,6 +8190,7 @@ int sqlite3BtreeInsert( }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); idx = ++pCur->ix; + pCur->curFlags &= ~BTCF_ValidNKey; }else{ assert( pPage->leaf ); } diff --git a/test/conflict3.test b/test/conflict3.test index b51a55ee71..751a442685 100644 --- a/test/conflict3.test +++ b/test/conflict3.test @@ -19,13 +19,14 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix conflict3 ifcapable !conflict { finish_test return } -do_execsql_test conflict-1.1 { +do_execsql_test 1.1 { CREATE TABLE t1( a INTEGER PRIMARY KEY ON CONFLICT REPLACE, b UNIQUE ON CONFLICT IGNORE, @@ -37,7 +38,7 @@ do_execsql_test conflict-1.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-1.2 { +do_execsql_test 1.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -45,16 +46,16 @@ do_execsql_test conflict-1.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-1.3 { +do_test 1.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-1.4 { +do_execsql_test 1.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Replete the tests above, but this time on a table non-INTEGER primary key. # -do_execsql_test conflict-2.1 { +do_execsql_test 2.1 { DROP TABLE t1; CREATE TABLE t1( a INT PRIMARY KEY ON CONFLICT REPLACE, @@ -67,7 +68,7 @@ do_execsql_test conflict-2.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-2.2 { +do_execsql_test 2.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -75,16 +76,16 @@ do_execsql_test conflict-2.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-2.3 { +do_test 2.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-2.4 { +do_execsql_test 2.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Replete again on a WITHOUT ROWID table. # -do_execsql_test conflict-3.1 { +do_execsql_test 3.1 { DROP TABLE t1; CREATE TABLE t1( a INT PRIMARY KEY ON CONFLICT REPLACE, @@ -97,7 +98,7 @@ do_execsql_test conflict-3.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-3.2 { +do_execsql_test 3.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -105,16 +106,16 @@ do_execsql_test conflict-3.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-3.3 { +do_test 3.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-3.4 { +do_execsql_test 3.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Arrange the table rows in a different order and repeat. # -do_execsql_test conflict-4.1 { +do_execsql_test 4.1 { DROP TABLE t1; CREATE TABLE t1( b UNIQUE ON CONFLICT IGNORE, @@ -127,7 +128,7 @@ do_execsql_test conflict-4.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-4.2 { +do_execsql_test 4.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -135,16 +136,16 @@ do_execsql_test conflict-4.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-4.3 { +do_test 4.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-4.4 { +do_execsql_test 4.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Arrange the table rows in a different order and repeat. # -do_execsql_test conflict-5.1 { +do_execsql_test 5.1 { DROP TABLE t1; CREATE TABLE t1( b UNIQUE ON CONFLICT IGNORE, @@ -157,7 +158,7 @@ do_execsql_test conflict-5.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-5.2 { +do_execsql_test 5.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -165,16 +166,16 @@ do_execsql_test conflict-5.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-5.3 { +do_test 5.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-5.4 { +do_execsql_test 5.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Arrange the table rows in a different order and repeat. # -do_execsql_test conflict-6.1 { +do_execsql_test 6.1 { DROP TABLE t1; CREATE TABLE t1( c UNIQUE ON CONFLICT FAIL, @@ -187,7 +188,7 @@ do_execsql_test conflict-6.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-6.2 { +do_execsql_test 6.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -195,16 +196,16 @@ do_execsql_test conflict-6.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-6.3 { +do_test 6.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-6.4 { +do_execsql_test 6.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Change which column is the PRIMARY KEY # -do_execsql_test conflict-7.1 { +do_execsql_test 7.1 { DROP TABLE t1; CREATE TABLE t1( a UNIQUE ON CONFLICT REPLACE, @@ -217,7 +218,7 @@ do_execsql_test conflict-7.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-7.2 { +do_execsql_test 7.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -225,16 +226,16 @@ do_execsql_test conflict-7.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-7.3 { +do_test 7.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-7.4 { +do_execsql_test 7.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Change which column is the PRIMARY KEY # -do_execsql_test conflict-8.1 { +do_execsql_test 8.1 { DROP TABLE t1; CREATE TABLE t1( a UNIQUE ON CONFLICT REPLACE, @@ -247,7 +248,7 @@ do_execsql_test conflict-8.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-8.2 { +do_execsql_test 8.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -255,16 +256,16 @@ do_execsql_test conflict-8.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-8.3 { +do_test 8.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-8.4 { +do_execsql_test 8.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Change which column is the PRIMARY KEY # -do_execsql_test conflict-9.1 { +do_execsql_test 9.1 { DROP TABLE t1; CREATE TABLE t1( a UNIQUE ON CONFLICT REPLACE, @@ -277,7 +278,7 @@ do_execsql_test conflict-9.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-9.2 { +do_execsql_test 9.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -285,16 +286,16 @@ do_execsql_test conflict-9.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-9.3 { +do_test 9.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-9.4 { +do_execsql_test 9.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Change which column is the PRIMARY KEY # -do_execsql_test conflict-10.1 { +do_execsql_test 10.1 { DROP TABLE t1; CREATE TABLE t1( a UNIQUE ON CONFLICT REPLACE, @@ -307,7 +308,7 @@ do_execsql_test conflict-10.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-10.2 { +do_execsql_test 10.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -315,16 +316,16 @@ do_execsql_test conflict-10.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-10.3 { +do_test 10.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-10.4 { +do_execsql_test 10.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} # Change which column is the PRIMARY KEY # -do_execsql_test conflict-11.1 { +do_execsql_test 11.1 { DROP TABLE t1; CREATE TABLE t1( a UNIQUE ON CONFLICT REPLACE, @@ -337,7 +338,7 @@ do_execsql_test conflict-11.1 { # Insert a row that conflicts on column B. The insert should be ignored. # -do_execsql_test conflict-11.2 { +do_execsql_test 11.2 { INSERT INTO t1(a,b,c) VALUES(3,2,5); SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4} @@ -345,12 +346,25 @@ do_execsql_test conflict-11.2 { # Insert two rows where the second conflicts on C. The first row show go # and and then there should be a constraint error. # -do_test conflict-11.3 { +do_test 11.3 { catchsql {INSERT INTO t1(a,b,c) VALUES(4,5,6), (5,6,4);} } {1 {UNIQUE constraint failed: t1.c}} -do_execsql_test conflict-11.4 { +do_execsql_test 11.4 { SELECT a,b,c FROM t1 ORDER BY a; } {1 2 3 2 3 4 4 5 6} +# Check that ticket [f68dc596c4] has been fixed. +# +do_execsql_test 12.1 { + CREATE TABLE t2(a INTEGER PRIMARY KEY, b TEXT); + INSERT INTO t2 VALUES(111, '111'); +} +do_execsql_test 12.2 { + REPLACE INTO t2 VALUES(NULL, '112'), (111, '111B'); +} +do_execsql_test 12.3 { + SELECT * FROM t2; +} {111 111B 112 112} + finish_test From 4b2667c8b87d3fb4a3da8874fb29a5d38926ca7d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 May 2017 18:24:01 +0000 Subject: [PATCH 07/11] Enhance "PRAGMA integrity_check" to detect duplicate rowids within a leaf page. FossilOrigin-Name: adcad37b001d255299be716b16003b56ec1fba349ef73261ee0e543186c4f311 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3815ce54e0..d87364e174 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clear\sthe\sBTCF_ValidNKey\sflag\sset\sif\sa\scursor\sis\smoved\sby\nsqlite3BtreeInsert().\sFix\sfor\s[f68dc596c4]. -D 2017-05-01T18:12:56.867 +C Enhance\s"PRAGMA\sintegrity_check"\sto\sdetect\sduplicate\srowids\swithin\sa\sleaf\npage. +D 2017-05-01T18:24:01.052 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -345,7 +345,7 @@ F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792 F src/backup.c faf17e60b43233c214aae6a8179d24503a61e83b F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca -F src/btree.c ef42df0cd8fa80ab174c83deb13a216465c49555eb01d57abdfd814a8546fa77 +F src/btree.c 8c1fd4cfa2b0bf021386e0a1f4e30b64eea7a2c1bc2e0c3e5901a626b1ab6aa9 F src/btree.h 80f518c0788be6cec8d9f8e13bd8e380df299d2b5e4ac340dc887b0642647cfc F src/btreeInt.h a392d353104b4add58b4a59cb185f5d5693dde832c565b77d8d4c343ed98f610 F src/build.c 4026a9c554b233e50c5e9ad46963e676cf54dd2306d952aa1eaa07a1bc9ce14f @@ -1577,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4e1df76e3d85922648e0e1cce73a266c3b1ed4511ace259ec0a01d7693af9e6f -R 2f44609e52881f5575acf3fe925fc9aa +P a6727eef6d757a39ad23e5c8cbe960f5d909e5d37cb4e90bc3bdbb8bf58cd6f8 +R 90aeb34d2a97f39322f1438c3090df25 U dan -Z 5b866fdf8c8937d7431fdae30fbd42bc +Z f3a1c9f94034eac95802b81d94d576ee diff --git a/manifest.uuid b/manifest.uuid index a351564742..0a84edf303 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6727eef6d757a39ad23e5c8cbe960f5d909e5d37cb4e90bc3bdbb8bf58cd6f8 \ No newline at end of file +adcad37b001d255299be716b16003b56ec1fba349ef73261ee0e543186c4f311 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4ba985f931..e5edf44c4d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9315,6 +9315,7 @@ static int checkTreePage( checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey); } maxKey = info.nKey; + keyCanBeEqual = 0; /* Only the first key on the page may ==maxKey */ } /* Check the content overflow list */ From c478c8daec5b41ac03aae74972b2c6c8785be5a8 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 May 2017 19:53:12 +0000 Subject: [PATCH 08/11] Remove an unnecessary branch. FossilOrigin-Name: a33179596f534067a58d68b77160f11ab13272b29f912d7cbe15ea00bbf03ade --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 4 +--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d89bebfd83..8040d9e110 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sfor\sthe\soptimization\son\sthis\sbranch. -D 2017-05-01T14:09:32.423 +C Remove\san\sunnecessary\sbranch. +D 2017-05-01T19:53:12.478 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -402,7 +402,7 @@ F src/printf.c 8757834f1b54dae512fb25eb1acc8e94a0d15dd2290b58f2563f65973265adb2 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac -F src/select.c a2b839b8cfa75d154490839ee80bad3452b9810ad39862ad044d154211292866 +F src/select.c 2c7ce8872ba623c24ebe2e881c54d7f7dcd3d9a3af04b8e34db0671354082db9 F src/shell.c 21b79c0e1b93f8e35fd7b4087d6ba438326c3d7e285d0dd51dfd741475f858a1 F src/sqlite.h.in 40233103e3e4e10f8a63523498d0259d232e42aba478e2d3fb914799185aced6 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1578,7 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5375a3ce56f1d993b13b469fe33ec7679948f53940f62a15ddbaeb8aaa26a22c -R 123e296789286f2ee6ee6640a2471eff -U dan -Z 8431f2088d761010cb187c22f8f8d978 +P 4921cd9520080f9baff70e548f64a56e2204b398b8397a2d318a98c32ec4b00c +R c6e750f01e8b5a0c9293dd8f991b3ffc +U drh +Z 3a6ca4224cd5b67f49546c5966995b4b diff --git a/manifest.uuid b/manifest.uuid index 2428b6a0d4..622d624499 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4921cd9520080f9baff70e548f64a56e2204b398b8397a2d318a98c32ec4b00c \ No newline at end of file +a33179596f534067a58d68b77160f11ab13272b29f912d7cbe15ea00bbf03ade \ No newline at end of file diff --git a/src/select.c b/src/select.c index 7a195560e6..4e78a49335 100644 --- a/src/select.c +++ b/src/select.c @@ -4907,9 +4907,7 @@ static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ if( pNew ){ Expr *pWhere = *(p->ppWhere); SWAP(Expr, *pNew, *pExpr); - if( pWhere ){ - pNew = sqlite3ExprAnd(db, pWhere, pNew); - } + pNew = sqlite3ExprAnd(db, pWhere, pNew); *(p->ppWhere) = pNew; } } From 8d5cee166e040eadb05ca68411c8919728c429b1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 2 May 2017 01:30:44 +0000 Subject: [PATCH 09/11] Fix typo in comment. No changes to code. FossilOrigin-Name: 6674814afb9e763e7f7060776586e26da28040b3208ce06c8a285dd647e5a53d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d87364e174..34f024e720 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\s"PRAGMA\sintegrity_check"\sto\sdetect\sduplicate\srowids\swithin\sa\sleaf\npage. -D 2017-05-01T18:24:01.052 +C Fix\stypo\sin\scomment.\s\sNo\schanges\sto\scode. +D 2017-05-02T01:30:44.805 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -404,7 +404,7 @@ F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 478e95d424bb86d34c7c95d20872cbd78df97af5f83c3fd7de55d5b2413f927d F src/shell.c 21b79c0e1b93f8e35fd7b4087d6ba438326c3d7e285d0dd51dfd741475f858a1 -F src/sqlite.h.in 40233103e3e4e10f8a63523498d0259d232e42aba478e2d3fb914799185aced6 +F src/sqlite.h.in eeb1da70a61d52e1d58e5b55446b85bbac571699421d3cf857421c56214013ce F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28 F src/sqliteInt.h 9affb53bb405dcea1d86e85198ebaf6232a684cc2b2af6b3c181869f1c8f3e93 @@ -1577,7 +1577,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a6727eef6d757a39ad23e5c8cbe960f5d909e5d37cb4e90bc3bdbb8bf58cd6f8 -R 90aeb34d2a97f39322f1438c3090df25 -U dan -Z f3a1c9f94034eac95802b81d94d576ee +P adcad37b001d255299be716b16003b56ec1fba349ef73261ee0e543186c4f311 +R 333e64e1a0afdbe4625860741bf47c15 +U mistachkin +Z b65b5e3a6f4ab6d1f0aa8214a9325839 diff --git a/manifest.uuid b/manifest.uuid index 0a84edf303..4e8c369818 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adcad37b001d255299be716b16003b56ec1fba349ef73261ee0e543186c4f311 \ No newline at end of file +6674814afb9e763e7f7060776586e26da28040b3208ce06c8a285dd647e5a53d \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 99d381d9d0..9e1eab4a2c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -857,7 +857,7 @@ struct sqlite3_io_methods { ** opcode allows these two values (10 retries and 25 milliseconds of delay) ** to be adjusted. The values are changed for all database connections ** within the same process. The argument is a pointer to an array of two -** integers where the first integer i the new retry count and the second +** integers where the first integer is the new retry count and the second ** integer is the delay. If either integer is negative, then the setting ** is not changed but instead the prior value of that setting is written ** into the array entry, allowing the current retry settings to be From ab314001b80737422a101b6600a0531fb867f38d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 May 2017 16:46:41 +0000 Subject: [PATCH 10/11] Additional comments on the sqlite3ExprIsConstantOrGroupBy() routine. No code changes. FossilOrigin-Name: 8424492eac506866d2918e5fe03c8f65fef960215d56012a3b52ed42789ed35a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8040d9e110..0421bd0099 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sbranch. -D 2017-05-01T19:53:12.478 +C Additional\scomments\son\sthe\ssqlite3ExprIsConstantOrGroupBy()\sroutine.\s\sNo\ncode\schanges. +D 2017-05-02T16:46:41.947 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -355,7 +355,7 @@ F src/ctime.c 47d91a25ad8f199a71a5b1b7b169d6dd0d6e98c5719eca801568798743d1161c F src/date.c cc42a41c7422389860d40419a5e3bce5eaf6e7835c3ba2677751dc653550a5c7 F src/dbstat.c 19ee7a4e89979d4df8e44cfac7a8f905ec89b77d F src/delete.c 0d9d5549d42e79ce4d82ff1db1e6c81e36d2f67c -F src/expr.c ed8914c001a24ab25aee965f498c96de577bf2c4a6145022b3688c11c2b29c82 +F src/expr.c 965f5e6074ee61cf933be079c6a443c88414490c13ec270b5baaacaa920280fa F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c db65492ae549c3b548c9ef1f279ce1684f1c473b116e1c56a90878cd5dcf968d F src/func.c 9d52522cc8ae7f5cdadfe14594262f1618bc1f86083c4cd6da861b4cf5af6174 @@ -1578,7 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4921cd9520080f9baff70e548f64a56e2204b398b8397a2d318a98c32ec4b00c -R c6e750f01e8b5a0c9293dd8f991b3ffc +P a33179596f534067a58d68b77160f11ab13272b29f912d7cbe15ea00bbf03ade +R 23bb18e29d815650d0a53ab5e0119122 U drh -Z 3a6ca4224cd5b67f49546c5966995b4b +Z 17cedac15c0a75bf83e81927ffcddf58 diff --git a/manifest.uuid b/manifest.uuid index 622d624499..eeb0aa3465 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a33179596f534067a58d68b77160f11ab13272b29f912d7cbe15ea00bbf03ade \ No newline at end of file +8424492eac506866d2918e5fe03c8f65fef960215d56012a3b52ed42789ed35a \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 73143c51de..a79b0b7495 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1848,6 +1848,20 @@ static int exprNodeIsConstantOrGroupBy(Walker *pWalker, Expr *pExpr){ ** Walk the expression tree passed as the first argument. Return non-zero ** if the expression consists entirely of constants or copies of terms ** in pGroupBy that sort with the BINARY collation sequence. +** +** This routine is used to determine if a term of the HAVING clause can +** be promoted into the WHERE clause. In order for such a promotion to work, +** the value of the HAVING clause term must be the same for all members of +** a "group". The requirement that the GROUP BY term must be BINARY +** assumes that no other collating sequence will have a finer-grained +** grouping than binary. In other words (A=B COLLATE binary) implies +** A=B in every other collating sequence. The requirement that the +** GROUP BY be BINARY is stricter than necessary. It would also work +** to promote HAVING clauses that use the same alternative collating +** sequence as the GROUP BY term, but that is much harder to check, +** alternative collating sequences are uncommon, and this is only an +** optimization, so we take the easy way out and simply require the +** GROUP BY to use the BINARY collating sequence. */ int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprList *pGroupBy){ Walker w; From e780764b4eb802a382fc59b9b82f68aa18af4ba2 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 May 2017 18:00:31 +0000 Subject: [PATCH 11/11] Fix the fts3EvalAverageDocsize() routine so that it returns errors from sqlite3_reset() rather than always returning SQLITE_OK. FossilOrigin-Name: 430f539cbb3f806fb89191e0b759a5f8b49d9e5b6c95fe9a4b55a1aa0875762a --- ext/fts3/fts3.c | 1 - manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 26b0b00bcd..97cfc6a585 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4787,7 +4787,6 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ ** data stored in all rows of each column of the table, from left ** to right. */ - int rc; Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; sqlite3_stmt *pStmt; sqlite3_int64 nDoc = 0; diff --git a/manifest b/manifest index 1395c271dd..84aba2e596 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reuse\sthe\ssame\smaterialization\sof\sa\sview\swhen\sthat\sview\sappears\sin\sa\squery\nmore\sthan\sonce,\ssuch\sas\sin\sa\sself-join. -D 2017-05-02T17:54:19.760 +C Fix\sthe\sfts3EvalAverageDocsize()\sroutine\sso\sthat\sit\sreturns\serrors\sfrom\nsqlite3_reset()\srather\sthan\salways\sreturning\sSQLITE_OK. +D 2017-05-02T18:00:31.645 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -70,7 +70,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 10fc22119e3d91997eb5820d96ff709ca7c61b6f767e09b360b986b897ad74c6 +F ext/fts3/fts3.c 88ea5c444fdf262b4eb7b5872b9a3192b8b8d0df8cc9735c5aed159ec5dae02a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h eb2502000148e80913b965db3e59f29251266d0a F ext/fts3/fts3_aux.c 9edc3655fcb287f0467d0a4b886a01c6185fe9f1 @@ -1578,8 +1578,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 47cbb471d056c8e1834a5ca72491404a3bfb273b5ff7bdd84b98d263938ea874 c64fe3a1695925693385d313e9ad2a1d8cb37ddaa8cc19920ae0978c91bc4c2c -R 250cb39f933147d7683c248b1fcd12aa -T +closed c64fe3a1695925693385d313e9ad2a1d8cb37ddaa8cc19920ae0978c91bc4c2c +P 9e35c89dbe744312f612e507b51ff9a5bb656def75392d25bc19fc638548cd1e +R ff82b9e0856a749852e0a40a30515627 U drh -Z d1eca8aa444c2ea7562fa3a6e3e0ab31 +Z a7183f9c248f45ecd3f1461fb3e694c7 diff --git a/manifest.uuid b/manifest.uuid index d9ed7e8021..ee0d3109c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9e35c89dbe744312f612e507b51ff9a5bb656def75392d25bc19fc638548cd1e \ No newline at end of file +430f539cbb3f806fb89191e0b759a5f8b49d9e5b6c95fe9a4b55a1aa0875762a \ No newline at end of file