diff --git a/manifest b/manifest index 987d96a26b..7db55b50f8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sfuzz\sinvariant\schecker\sto\sconform\sto\sthe\slatest\schanges\sin\ndbsqlfuzz. -D 2022-09-20T17:21:54.866 +C The\sRIGHT\sJOIN\slogic\sadded\sto\sversion\s3.39.0\sadded\sa\snew\srestriction\son\sthe\nquery\sflattener\swhich\sresulted\sin\sa\sperformance\sregression\sfor\ssome\squeries,\nas\sdocumented\sby\s[forum:/forumpost/96b9e5709cf47cda|forum\spost\s96b9e5709cf47cda].\nThis\schange\sremoves\sthat\srestriction\s(thus\srestoring\sperformance)\sand\sfixes\sthe\nproblem\sthat\sRIGHT\sJOIN\swas\shaving\sin\sa\sdifferent\sway. +D 2022-09-21T09:54:41.195 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -539,7 +539,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e -F src/expr.c 24e828db6b2fab8aabfb5d2c0d83dbdfc5a1972b1147fa893350e317ab7e282f +F src/expr.c 1cbdd76eeedb729ea9060df03e3e6b74a302784a13bfa38794a8194f894641ea F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002 F src/func.c 8f72e88cccdee22185133c10f96ccd61dc34c5ea4b1fa9a73c237ef59b2e64f1 @@ -588,7 +588,7 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764 F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 -F src/select.c dcfd2df77e4601555cef48174e3c61723f72fccfdd5b80caa5da63e23338c040 +F src/select.c bb18acf4eded647fef88d4d543c673874dbebff516fbeba90a85e6c13f2a58cd F src/shell.c.in e7e7c2c69ae86c5ee9e8ad66227203d46ff6dce8700a1b1dababff01c71d33df F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -660,7 +660,7 @@ F src/upsert.c 8789047a8f0a601ea42fa0256d1ba3190c13746b6ba940fe2d25643a7e991937 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c 0be191521ff6d2805995f4910f0b6231b42843678b2efdc1abecaf39929a673f F src/vacuum.c bb346170b0b54c6683bba4a5983aea40485597fdf605c87ec8bc2e199fe88cd8 -F src/vdbe.c d27ec9a57f752fc2acf6a64d43bbf6072d2415efc976184f6d8a146e65819d3b +F src/vdbe.c 0d1e3c658d98a7bb7201532ea7a3e4d59bf9165421c780d5f84c361e372f1179 F src/vdbe.h 64619af62603dc3c4f5ff6ff6d2c8f389abd667a29ce6007ed44bd22b3211cd0 F src/vdbeInt.h 17b7461ffcf9ee760d1341731715a419f6b8c763089a7ece25c2e8098d702b3f F src/vdbeapi.c fc3183daf72808b4311b228989120fdbc2dc44972fb0d77d5c453460cc0e5b2c @@ -675,7 +675,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b9df133a705093da8977da5eb202eaadb844839f1c7297c08d33471f5491843d F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a F src/walker.c f890a3298418d7cba3b69b8803594fdc484ea241206a8dfa99db6dd36f8cbb3b -F src/where.c 424c42590b71968a9b81cd890df2671902028613fee38a50ed4c2f7ca65315d3 +F src/where.c 3774d9831a6f4ac81f201fa9e2ec1bb40c99b7207dce7663c2e39fb73cc1bfd5 F src/whereInt.h 70cd30de9ed784aa33fa6bd1245f060617de7a00d992469b6d8e419eed915743 F src/wherecode.c 6bb1cf9d0a4e3e04dab0bf0ea4a8d936a0dcc05a7e2207beeda6c61aea6dd341 F src/whereexpr.c 55a39f42aaf982574fbf52906371a84cceed98a994422198dfd03db4fce4cc46 @@ -1180,7 +1180,7 @@ F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b F test/istrue.test e7f285bb70282625c258e866ce6337d4c762922f5a300e1b50f958aef6e7d9c9 -F test/join.test 21dbc65ab2476b10ae3ed1dcf64a99fb9a40473caa22f4cec2d6a829933c1442 +F test/join.test e32cb9b1491eed682489e2cde33a22a4eb7611fe5aa3b0aa4b275fe27ab3f3ac F test/join2.test 466b07233820f5deee66a6c3bf6e4500c8bbf7b83649e67606f5f649c07928c0 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -2000,8 +2000,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2647d73bb1876d01851f52cb3fd6af08a962ab336b6d51c160d27b1baf94f75f -R 4c12344a007c9e9bdcdbf323756fa10a +P 8d8c124349d2a25200ace9a60d76751594b722468ad0c39d927a5d2c6e2464e0 c7fbc9b0453050e2746af27e3a11e0c3701bef8c56da8e19173242c6ea3aff8b +R ffb250842cc4325c51d9fb2f2a7cd199 +T +closed c7fbc9b0453050e2746af27e3a11e0c3701bef8c56da8e19173242c6ea3aff8b U drh -Z 7b99665ad0fc78591cd70e35a24e34f9 +Z c4277c5174933f302dd59d96a8e6bf4e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7ec8709a6f..c74cd3d3de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d8c124349d2a25200ace9a60d76751594b722468ad0c39d927a5d2c6e2464e0 \ No newline at end of file +501609eddf2a46d51e8cd56477a22d4df142a166e78fe4322c0db11d1eed6687 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 8572512926..baa0fe6476 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3223,6 +3223,7 @@ void sqlite3CodeRhsOfIN( sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); } if( addrOnce ){ + sqlite3VdbeAddOp1(v, OP_NullRow, iTab); sqlite3VdbeJumpHere(v, addrOnce); /* Subroutine return */ assert( ExprUseYSub(pExpr) ); diff --git a/src/select.c b/src/select.c index 7b7731c234..2876f48ab8 100644 --- a/src/select.c +++ b/src/select.c @@ -3724,7 +3724,7 @@ static int multiSelectOrderBy( ** the left operands of a RIGHT JOIN. In either case, we need to potentially ** bypass the substituted expression with OP_IfNullRow. ** -** Suppose the original expression integer constant. Even though the table +** Suppose the original expression is an integer constant. Even though the table ** has the nullRow flag set, because the expression is an integer constant, ** it will not be NULLed out. So instead, we insert an OP_IfNullRow opcode ** that checks to see if the nullRow flag is set on the table. If the nullRow @@ -4188,11 +4188,6 @@ static void renumberCursors( ** ** (28) The subquery is not a MATERIALIZED CTE. ** -** (29) Either the subquery is not the right-hand operand of a join with an -** ON or USING clause nor the right-hand operand of a NATURAL JOIN, or -** the right-most table within the FROM clause of the subquery -** is not part of an outer join. -** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query @@ -4296,15 +4291,6 @@ static int flattenSubquery( } isOuterJoin = 1; } -#ifdef SQLITE_EXTRA_IFNULLROW - else if( iFrom>0 && !isAgg ){ - /* Setting isOuterJoin to -1 causes OP_IfNullRow opcodes to be generated for - ** every reference to any result column from subquery in a join, even - ** though they are not necessary. This will stress-test the OP_IfNullRow - ** opcode. */ - isOuterJoin = -1; - } -#endif assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */ if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ @@ -4314,33 +4300,6 @@ static int flattenSubquery( return 0; /* (28) */ } - /* Restriction (29): - ** - ** We do not want two constraints on the same FROM-clause term of the - ** flattened query where one constraint has the EP_InnerON flag and the - ** other has the EP_OuterON flag. - ** - ** To prevent this, one or the other of the following conditions must be - ** false: - ** - ** (29a) The right-most entry in the FROM clause of the subquery - ** must not be part of an outer join. - ** - ** (29b) The subquery itself must not be the right operand of a - ** NATURAL join or a join that has an ON or USING clause. - */ - if( pSubSrc->nSrc>=2 - && (pSubSrc->a[pSubSrc->nSrc-1].fg.jointype & JT_OUTER)!=0 - ){ - if( (pSubitem->fg.jointype & JT_NATURAL)!=0 - || pSubitem->fg.isUsing - || NEVER(pSubitem->u3.pOn!=0) /* ON clause already shifted into WHERE */ - || pSubitem->fg.isOn - ){ - return 0; - } - } - /* Restriction (17): If the sub-query is a compound SELECT, then it must ** use only the UNION ALL operator. And none of the simple select queries ** that make up the compound SELECT are allowed to be aggregate or distinct diff --git a/src/vdbe.c b/src/vdbe.c index 686ebf175f..ebf52e67af 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4897,12 +4897,16 @@ case OP_SeekHit: { /* Opcode: IfNotOpen P1 P2 * * * ** Synopsis: if( !csr[P1] ) goto P2 ** -** If cursor P1 is not open, jump to instruction P2. Otherwise, fall through. +** If cursor P1 is not open or if P1 is set to a NULL row using the +** OP_NullRow opcode, then jump to instruction P2. Otherwise, fall through. */ case OP_IfNotOpen: { /* jump */ + VdbeCursor *pCur; + assert( pOp->p1>=0 && pOp->p1nCursor ); - VdbeBranchTaken(p->apCsr[pOp->p1]==0, 2); - if( !p->apCsr[pOp->p1] ){ + pCur = p->apCsr[pOp->p1]; + VdbeBranchTaken(pCur==0 || pCur->nullRow, 2); + if( pCur==0 || pCur->nullRow ){ goto jump_to_p2_and_check_for_interrupt; } break; diff --git a/src/where.c b/src/where.c index 3e90fa9a7c..5811d830d1 100644 --- a/src/where.c +++ b/src/where.c @@ -743,6 +743,38 @@ static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ #define whereTraceIndexInfoOutputs(A) #endif +/* +** We know that pSrc is an operand of an outer join. Return true if +** pTerm is a constraint that is compatible with that join. +** +** pTerm must be EP_OuterON if pSrc is the right operand of an +** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc +** is the left operand of a RIGHT join. +*/ +static int constraintCompatibleWithOuterJoin( + const WhereTerm *pTerm, /* WHERE clause term to check */ + const SrcItem *pSrc /* Table we are trying to access */ +){ + assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */ + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); + testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) + testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); + if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) + || pTerm->pExpr->w.iJoin != pSrc->iCursor + ){ + return 0; + } + if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0 + && ExprHasProperty(pTerm->pExpr, EP_InnerON) + ){ + return 0; + } + return 1; +} + + + #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* ** Return TRUE if the WHERE clause term pTerm is of a form where it @@ -758,16 +790,10 @@ static int termCanDriveIndex( if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; assert( (pSrc->fg.jointype & JT_RIGHT)==0 ); - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){ - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); - testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) - testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); - if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) - || pTerm->pExpr->w.iJoin != pSrc->iCursor - ){ - return 0; /* See tag-20191211-001 */ - } + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + return 0; } if( (pTerm->prereqRight & notReady)!=0 ) return 0; assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); @@ -1184,17 +1210,10 @@ static sqlite3_index_info *allocateIndexInfo( ** right-hand table of a LEFT JOIN nor to the either table of a ** RIGHT JOIN. See tag-20191211-001 for the ** equivalent restriction for ordinary tables. */ - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){ - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_RIGHT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); - testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ); - testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); - if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) - || pTerm->pExpr->w.iJoin != pSrc->iCursor - ){ - continue; - } + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; } nTerm++; pTerm->wtFlags |= TERM_OK; @@ -2865,17 +2884,10 @@ static int whereLoopAddBtreeIndex( ** 2022-06-10: The same condition applies to termCanDriveIndex() above. ** https://sqlite.org/forum/forumpost/51e6959f61 */ - if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ){ - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_RIGHT ); - testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); - testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) - testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); - if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) - || pTerm->pExpr->w.iJoin != pSrc->iCursor - ){ - continue; - } + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; } if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ diff --git a/test/join.test b/test/join.test index 82dbd447f9..f0ac0be8b1 100644 --- a/test/join.test +++ b/test/join.test @@ -1130,5 +1130,81 @@ do_execsql_test join-27.5 { ) AS t99 ON b IN (1,2,3); } {} +db null NULL +do_execsql_test join-27.6 { + INSERT INTO t1 VALUES(3,4,NULL); + INSERT INTO t2 VALUES(1,2); + WITH t99(b) AS ( + SELECT coalesce(b,3) FROM t2 AS x LEFT JOIN t1 ON c IN (SELECT x FROM t3) + ) + SELECT d, e, b FROM t2 JOIN t99 ON b IN (1,2,3) ORDER BY +d; +} {NULL NULL 3 NULL NULL 3 1 2 3 1 2 3} +do_execsql_test join-27.7 { + SELECT d, e, b2 + FROM t2 + JOIN (SELECT coalesce(b,3) AS b2 FROM t2 AS x LEFT JOIN t1 + ON c IN (SELECT x FROM t3)) AS t99 + ON b2 IN (1,2,3) ORDER BY +d; +} {NULL NULL 3 NULL NULL 3 1 2 3 1 2 3} + +do_execsql_test join-27.8 { + DELETE FROM t1; + DELETE FROM t2 WHERE d IS NOT NULL; + DELETE FROM t3; + SELECT * FROM t2 JOIN (SELECT b FROM t2 LEFT JOIN t1 + ON c IN (SELECT x FROM t3)) AS t99 ON b IN (1,2,3); +} {} + +do_execsql_test join-27.9 { + DELETE FROM t1; + DELETE FROM t2; + DELETE FROM t3; + INSERT INTO t1 VALUES(4,3,5); + INSERT INTO t2 VALUES(1,2); + INSERT INTO t3 VALUES(5); + SELECT * FROM t2 JOIN (SELECT b FROM t2 LEFT JOIN t1 + ON c IN (SELECT x FROM t3)) AS t99 ON b IS NULL; +} {} +do_execsql_test join-27.10 { + WITH t99(b) AS ( + SELECT b FROM t2 AS x LEFT JOIN t1 ON c IN (SELECT x FROM t3) + ) + SELECT d, e, b FROM t2 JOIN t99 ON b IS NULL; +} {} + + +# 2022-09-19 https://sqlite.org/forum/forumpost/96b9e5709cf47cda +# Performance regression relative to version 3.38.0 that resulted from +# a new query flattener restriction that was added to fixes the join-27.* +# tests above. The restriction needed to be removed and the join-27.* +# problem fixed another way. +# +reset_db +do_execsql_test join-28.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT, c INT); + CREATE TABLE t2(d INTEGER PRIMARY KEY, e INT); + CREATE VIEW t3(a,b,c,d,e) AS SELECT * FROM t1 LEFT JOIN t2 ON d=c; + CREATE TABLE t4(x INT, y INT); + INSERT INTO t1 VALUES(1,2,3); + INSERT INTO t2 VALUES(1,5); + INSERT INTO t4 VALUES(1,4); + SELECT a, b, y FROM t4 JOIN t3 ON a=x; +} {1 2 4} +do_eqp_test join-28.2 { + SELECT a, b, y FROM t4 JOIN t3 ON a=x; +} { + QUERY PLAN + |--SCAN t4 + `--SEARCH t1 USING INTEGER PRIMARY KEY (rowid=?) +} +# ^^^^^^^ Without the fix (if the query flattening optimization does not +# run) the query plan above would look like this: +# +# QUERY PLAN +# |--MATERIALIZE t3 +# | |--SCAN t1 +# | `--SEARCH t2 USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN +# |--SCAN t4 +# `--SEARCH t3 USING AUTOMATIC COVERING INDEX (a=?) finish_test