Give antijoins made from sublinks a relid

When pull_up_sublinks() converts a sublink to an antijoin, it builds a
JoinExpr with no RTE, on the grounds that nothing above the join can
reference its RHS.  That overlooks PlaceHolderVars.  A PHV with no
Vars of its own is evaluated at its phrels, and for the outputs of a
pulled-up subquery phrels is all the relids in the subquery's
jointree, including the RHS of such a join.

A qual using the PHV thus appears to reference the RHS rels, and can
be pushed down into the RHS as a parameterized join clause.  There it
has the opposite of the intended effect: outer rows that fail the qual
find no match and are emitted, giving wrong results.  For an outer
join, including an antijoin made by reducing a left join, this cannot
happen because the join's relid is among the qual's relids.  Before
v16, the qual's nullable_relids prevented it.

To fix, make a join RTE for an antijoin built from a sublink, so that
it has a relid like any other outer join.  Semijoins are left as they
are: a qual evaluated within a semijoin's RHS does not change the
result.

Back-patch to v16.  Before that, the qual's nullable_relids included
the antijoin's RHS, which kept the qual from being moved there; that
protection went away when nullable_relids was replaced by outer-join
relids.

Author: Richard Guo <guofenglinux@gmail.com>
Reviewed-by: Tom Lane <tgl@sss.pgh.pa.us>
Reviewed-by: wenhui qiu <qiuwenhuifx@gmail.com>
Discussion: https://postgr.es/m/CAMbWs4_rEdcG89BgC-AO7b+Bf2z8w3976+4zP==Urt=CXB1hsg@mail.gmail.com
Backpatch-through: 16

src/backend/optimizer/plan/initsplan.c

@@ -956,19 +956,16 @@ deconstruct_recurse(PlannerInfo *root, Node *jtnode,
 									child_domain->jd_relids);
 				jtitem->qualscope = bms_union(left_item->qualscope,
 											  right_item->qualscope);
-				/* caution: ANTI join derived from SEMI will lack rtindex */
-				if (j->rtindex != 0)
-				{
-					parent_domain->jd_relids =
-						bms_add_member(parent_domain->jd_relids,
-									   j->rtindex);
-					jtitem->qualscope = bms_add_member(jtitem->qualscope,
+				Assert(j->rtindex != 0);
+				parent_domain->jd_relids =
+					bms_add_member(parent_domain->jd_relids,
+								   j->rtindex);
+				jtitem->qualscope = bms_add_member(jtitem->qualscope,
+												   j->rtindex);
+				root->outer_join_rels = bms_add_member(root->outer_join_rels,
 													   j->rtindex);
-					root->outer_join_rels = bms_add_member(root->outer_join_rels,
-														   j->rtindex);
-					mark_rels_nulled_by_join(root, j->rtindex,
-											 right_item->qualscope);
-				}
+				mark_rels_nulled_by_join(root, j->rtindex,
+										 right_item->qualscope);
 				jtitem->inner_join_rels = bms_union(left_item->inner_join_rels,
 													right_item->inner_join_rels);
 				jtitem->left_rels = left_item->qualscope;

src/backend/optimizer/plan/subselect.c

@@ -91,6 +91,7 @@ static bool contain_outer_selfref(Node *node);
 static bool contain_outer_selfref_walker(Node *node, Index *depth);
 static void inline_cte(PlannerInfo *root, CommonTableExpr *cte);
 static bool inline_cte_walker(Node *node, inline_cte_walker_context *context);
+static int	add_antijoin_rte(Query *parse);
 static bool simplify_EXISTS_query(PlannerInfo *root, Query *query);
 static Query *convert_EXISTS_to_ANY(PlannerInfo *root, Query *subselect,
 									Node **testexpr, List **paramIds);
@@ -1528,11 +1529,56 @@ convert_EXISTS_sublink_to_join(PlannerInfo *root, SubLink *sublink,
 	result->join_using_alias = NULL;
 	result->quals = whereClause;
 	result->alias = NULL;
-	result->rtindex = 0;		/* we don't need an RTE for it */
+	/* a semijoin doesn't need an RTE, but see add_antijoin_rte() */
+	result->rtindex = under_not ? add_antijoin_rte(parse) : 0;
 
 	return result;
 }
 
+/*
+ * add_antijoin_rte: make a join RTE for an antijoin built from a sublink
+ *
+ * Giving the antijoin a relid, like an antijoin made by reducing a left join
+ * has, is what tells the rest of the planner that anything coming from above
+ * the join must not be evaluated within its RHS.  Normally nothing above can
+ * reference the RHS at all, but a PlaceHolderVar whose phrels cover the whole
+ * join can make a qual appear to; the join's relid in phrels then keeps such
+ * a qual from being pushed down into the RHS, where it would have the
+ * opposite of the intended effect.  (For a semijoin that would be harmless,
+ * so those are left without a relid.)
+ *
+ * Returns the new RTE's index.
+ */
+static int
+add_antijoin_rte(Query *parse)
+{
+	RangeTblEntry *rte;
+
+	/* Manufacture a join RTE to use. */
+	rte = makeNode(RangeTblEntry);
+	rte->rtekind = RTE_JOIN;
+	rte->jointype = JOIN_ANTI;
+	rte->joinmergedcols = 0;
+	rte->joinaliasvars = NIL;	/* nothing can reference the join's output */
+	rte->joinleftcols = NIL;	/* so it has no columns */
+	rte->joinrightcols = NIL;	/* ditto */
+	rte->join_using_alias = NULL;
+
+	rte->alias = NULL;
+	rte->eref = makeAlias("*ANTI*", NIL);
+	rte->lateral = false;
+	rte->inh = false;
+	rte->inFromCl = false;		/* not something the user wrote in FROM */
+
+	/*
+	 * Add completed RTE to the query's range table list, so that we know its
+	 * index.
+	 */
+	parse->rtable = lappend(parse->rtable, rte);
+
+	return list_length(parse->rtable);
+}
+
 /*
  * simplify_EXISTS_query: remove any useless stuff in an EXISTS's subquery
  *

src/include/nodes/pathnodes.h

@@ -2831,8 +2831,7 @@ typedef struct PlaceHolderVar
  * LEFT, FULL, SEMI, or ANTI.
  *
  * ojrelid is the RT index of the join RTE representing this outer join,
- * if there is one.  It is zero when jointype is INNER or SEMI, and can be
- * zero for jointype ANTI (if the join was transformed from a SEMI join).
+ * if there is one.  It is zero when jointype is INNER or SEMI.
  * One use for this field is that when constructing the output targetlist of a
  * join relation that implements this OJ, we add ojrelid to the varnullingrels
  * and phnullingrels fields of nullable (RHS) output columns, so that the

src/include/nodes/primnodes.h

@@ -2274,8 +2274,9 @@ typedef struct RangeTblRef
  * During parse analysis, an RTE is created for the Join, and its index
  * is filled into rtindex.  This RTE is present mainly so that Vars can
  * be created that refer to the outputs of the join.  The planner sometimes
- * generates JoinExprs internally; these can have rtindex = 0 if there are
- * no join alias variables referencing such joins.
+ * generates JoinExprs internally.  Those for semijoins have rtindex = 0,
+ * since no join alias variables can reference them; but an antijoin is
+ * given an RTE anyway, because it needs a relid as other outer joins do.
  *----------
  */
 typedef struct JoinExpr

src/test/regress/expected/join.out

@@ -9044,3 +9044,32 @@ GROUP BY s.c1, s.c2;
 (7 rows)
 
 DROP TABLE group_tbl;
+--
+-- Test that a qual from above an antijoin made from a sublink is not pushed
+-- down into the antijoin's RHS.
+--
+EXPLAIN (COSTS OFF)
+SELECT s.*
+FROM (VALUES (1)) AS v(z) LEFT JOIN
+  (SELECT 123456 AS c, t1.f1 AS x FROM int4_tbl t1
+   WHERE NOT EXISTS (SELECT 1 FROM tenk1 t2 WHERE t2.unique1 = t1.f1)) s ON true
+WHERE s.x = s.c;
+                      QUERY PLAN                       
+-------------------------------------------------------
+ Nested Loop Anti Join
+   Filter: (t1.f1 = 123456)
+   ->  Seq Scan on int4_tbl t1
+   ->  Index Only Scan using tenk1_unique1 on tenk1 t2
+         Index Cond: (unique1 = t1.f1)
+(5 rows)
+
+SELECT s.*
+FROM (VALUES (1)) AS v(z) LEFT JOIN
+  (SELECT 123456 AS c, t1.f1 AS x FROM int4_tbl t1
+   WHERE NOT EXISTS (SELECT 1 FROM tenk1 t2 WHERE t2.unique1 = t1.f1)) s ON true
+WHERE s.x = s.c;
+   c    |   x    
+--------+--------
+ 123456 | 123456
+(1 row)
+

src/test/regress/sql/join.sql

@@ -3328,3 +3328,21 @@ SELECT 1 FROM group_tbl t1
 GROUP BY s.c1, s.c2;
 
 DROP TABLE group_tbl;
+
+--
+-- Test that a qual from above an antijoin made from a sublink is not pushed
+-- down into the antijoin's RHS.
+--
+
+EXPLAIN (COSTS OFF)
+SELECT s.*
+FROM (VALUES (1)) AS v(z) LEFT JOIN
+  (SELECT 123456 AS c, t1.f1 AS x FROM int4_tbl t1
+   WHERE NOT EXISTS (SELECT 1 FROM tenk1 t2 WHERE t2.unique1 = t1.f1)) s ON true
+WHERE s.x = s.c;
+
+SELECT s.*
+FROM (VALUES (1)) AS v(z) LEFT JOIN
+  (SELECT 123456 AS c, t1.f1 AS x FROM int4_tbl t1
+   WHERE NOT EXISTS (SELECT 1 FROM tenk1 t2 WHERE t2.unique1 = t1.f1)) s ON true
+WHERE s.x = s.c;