Disallow joins whose lateral references can never be satisfied

If a proposed join has lateral references to rels outside it,
join_is_legal() checks that those rels can still be joined to from
outside.  To do that it collects the rels that must end up on the
inner side of an outer join with the proposed join, directly or
indirectly, and rejects the join if any of its lateral references
point to one of them.

That search followed only outer joins, from min_lefthand to
min_righthand.  It failed to notice that a rel that laterally
references the proposed join, or any rel found by the search, cannot
supply a parameter to the proposed join either, and that the same
holds for rels on the inner side of an outer join with such a rel.  As
a result, we could approve a join whose lateral references could never
be satisfied.

Such a join can never appear in a complete plan.  Worse, as the
existing comment warns, its apparent legality could mislead the
clauseless-join heuristics into thinking that some other join rel need
not be formed.  In particular, have_join_order_restriction() would not
force a clauseless outer join if one of its input rels appeared to
have a legal clause join elsewhere, and if that outer join was
required, we would fail with "failed to build any N-way joins".

To fix, also include in the search any input rel of the current join
search that laterally references a rel already found.  The input rels
are taken from root->initial_rels rather than from the query's base
rels, because an input can be the joinrel of a sub-joinlist, as
happens for a full join or when from_collapse_limit or
join_collapse_limit prevents flattening.  Such a joinrel is joined as
a unit, so it has to be included as a whole.

Back-patch to all supported branches.

Author: Richard Guo <guofenglinux@gmail.com>
Reviewed-by: Tender Wang <tndrwang@gmail.com>
Reviewed-by: wenhui qiu <qiuwenhuifx@gmail.com>
Discussion: https://postgr.es/m/CAMbWs4_p6iWbsjNcYpd6-6mWFnog3UJ2fZKrxWAQTsyNoyeQTw@mail.gmail.com
Backpatch-through: 14

src/backend/optimizer/path/allpaths.c

@@ -3348,7 +3348,8 @@ make_rel_from_joinlist(PlannerInfo *root, List *joinlist)
 		 * using a plugin, GEQO, or the regular join search code.
 		 *
 		 * We put the initial_rels list into a PlannerInfo field because
-		 * has_legal_joinclause() needs to look at it (ugly :-().
+		 * has_legal_joinclause() and join_is_legal() need to look at it (ugly
+		 * :-().
 		 */
 		root->initial_rels = initial_rels;
 

src/backend/optimizer/path/joinrels.c

@@ -630,6 +630,9 @@ join_is_legal(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
 		 * to find any plan at all.  We have to consider not only rels that
 		 * are directly on the inner side of an OJ with the joinrel, but also
 		 * ones that are indirectly so, so search to find all such rels.
+		 * Likewise, input rels of the current join search that laterally
+		 * reference the joinrel or any rel found so far cannot supply a
+		 * parameter to the joinrel, so include those too.
 		 */
 		join_lateral_rels = min_join_parameterization(root, joinrelids,
 													  rel1, rel2);
@@ -657,6 +660,24 @@ join_is_legal(PlannerInfo *root, RelOptInfo *rel1, RelOptInfo *rel2,
 						more = true;
 					}
 				}
+
+				/*
+				 * Add input rels that laterally reference any rel found so
+				 * far.  An input rel can be a sub-joinlist's joinrel, which
+				 * must be taken as a whole.
+				 */
+				foreach(l, root->initial_rels)
+				{
+					RelOptInfo *irel = (RelOptInfo *) lfirst(l);
+
+					if (bms_overlap(irel->lateral_relids, join_plus_rhs) &&
+						!bms_is_subset(irel->relids, join_plus_rhs))
+					{
+						join_plus_rhs = bms_add_members(join_plus_rhs,
+														irel->relids);
+						more = true;
+					}
+				}
 			} while (more);
 			if (bms_overlap(join_plus_rhs, join_lateral_rels))
 				return false;	/* will not be able to join to some RHS rel */

src/test/regress/expected/join.out

@@ -4470,6 +4470,114 @@ where ss1.c2 = 0;
 ----+----+----+----+----+----
 (0 rows)
 
+--
+-- check that a join is disallowed when its lateral reference can only be
+-- satisfied by way of a rel that laterally references the join itself
+--
+explain (costs off)
+select 1 from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 left join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+                      QUERY PLAN                       
+-------------------------------------------------------
+ Nested Loop
+   Join Filter: ((t3.q2) > t1.f1)
+   ->  Hash Left Join
+         Hash Cond: ((t2.f1) = t3.q1)
+         ->  Nested Loop
+               ->  Nested Loop Left Join
+                     ->  Seq Scan on int4_tbl t1
+                     ->  Materialize
+                           ->  Seq Scan on int4_tbl t2
+               ->  Result
+         ->  Hash
+               ->  Seq Scan on int8_tbl t3
+   ->  Memoize
+         Cache Key: t3.q2
+         Cache Mode: binary
+         ->  Result
+(16 rows)
+
+select 1 from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 left join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+ ?column? 
+----------
+(0 rows)
+
+-- likewise when the outer join above the lateral referencer is a full join
+explain (costs off)
+select count(*) from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 full join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+                      QUERY PLAN                       
+-------------------------------------------------------
+ Aggregate
+   ->  Nested Loop
+         Join Filter: ((t3.q2) > t1.f1)
+         ->  Nested Loop
+               ->  Nested Loop Left Join
+                     ->  Seq Scan on int4_tbl t1
+                     ->  Materialize
+                           ->  Seq Scan on int4_tbl t2
+               ->  Hash Full Join
+                     Hash Cond: (t3.q1 = (t2.f1))
+                     ->  Seq Scan on int8_tbl t3
+                     ->  Hash
+                           ->  Result
+         ->  Memoize
+               Cache Key: t3.q2
+               Cache Mode: binary
+               ->  Result
+(17 rows)
+
+select count(*) from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 full join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+ count 
+-------
+    80
+(1 row)
+
+-- likewise when the lateral referencer is part of a sub-joinlist
+set from_collapse_limit to 4;
+explain (costs off)
+select count(*) from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+                      QUERY PLAN                       
+-------------------------------------------------------
+ Aggregate
+   ->  Nested Loop
+         Join Filter: ((t3.q2) > t1.f1)
+         ->  Nested Loop
+               ->  Nested Loop Left Join
+                     ->  Seq Scan on int4_tbl t1
+                     ->  Materialize
+                           ->  Seq Scan on int4_tbl t2
+               ->  Hash Join
+                     Hash Cond: (t3.q1 = (t2.f1))
+                     ->  Seq Scan on int8_tbl t3
+                     ->  Hash
+                           ->  Result
+         ->  Result
+(14 rows)
+
+select count(*) from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+ count 
+-------
+     0
+(1 row)
+
+reset from_collapse_limit;
 --
 -- test successful handling of full join underneath left join (bug #14105)
 --

src/test/regress/sql/join.sql

@@ -1523,6 +1523,50 @@ select ss2.* from
   lateral (select i41.*, i8.*, ss1.* from text_tbl limit 1) ss2
 where ss1.c2 = 0;
 
+--
+-- check that a join is disallowed when its lateral reference can only be
+-- satisfied by way of a rel that laterally references the join itself
+--
+
+explain (costs off)
+select 1 from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 left join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+
+select 1 from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 left join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+
+-- likewise when the outer join above the lateral referencer is a full join
+explain (costs off)
+select count(*) from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 full join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+
+select count(*) from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 full join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+
+-- likewise when the lateral referencer is part of a sub-joinlist
+set from_collapse_limit to 4;
+
+explain (costs off)
+select count(*) from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+
+select count(*) from int4_tbl t1 left join int4_tbl t2 on true,
+  lateral (select t2.f1 offset 0) s1 join int8_tbl t3 on s1.f1 = t3.q1,
+  lateral (select t3.q2 offset 0) s2
+where s2.q2 > t1.f1;
+
+reset from_collapse_limit;
+
 --
 -- test successful handling of full join underneath left join (bug #14105)
 --