Set hasSubLinks when expanding a whole-row join alias reference

If a subquery has been flattened into its parent, the joinaliasvars
entries of a join above it can be arbitrary expressions rather than
plain Vars, so expanding a reference to such a join alias may insert a
SubLink into a lower-level subquery.  flatten_join_alias_vars_mutator
detects that and sets the subquery's hasSubLinks flag, but only in the
single-column code path; the whole-row path just asserted in a comment
that its recursive call would handle this, which is true only when the
alias entry is itself a Var referencing another join.

Hence a whole-row reference to such a join appearing in a sub-select
left that sub-select's hasSubLinks false, so preprocess_expression
skipped SS_process_sublinks for it, and the unprocessed SubLink
reached code that is not prepared for one: this produced "cannot
handle unplanned sub-select" from cost_qual_eval, an assertion failure
in preprocess_aggrefs, or "unrecognized node type" at execution,
depending on where in the sub-select the SubLink ended up.

To fix, make the same check in the whole-row expansion path.

Back-patch to all supported branches.

Author: Richard Guo <guofenglinux@gmail.com>
Reviewed-by: Ayush Tiwari <ayushtiwari.slg01@gmail.com>
Discussion: https://postgr.es/m/CAMbWs49PgeNFhQTZ2gSzwatF0_LXyegMPGt++jSfxUN7NZuQ5Q@mail.gmail.com
Backpatch-through: 14

src/backend/optimizer/util/var.c

@@ -790,8 +790,12 @@ flatten_join_alias_vars_mutator(Node *node,
 				if (IsA(newvar, Var))
 					((Var *) newvar)->location = var->location;
 				/* Recurse in case join input is itself a join */
-				/* (also takes care of setting inserted_sublink if needed) */
 				newvar = flatten_join_alias_vars_mutator(newvar, context);
+
+				/* Detect if we are adding a sublink to query */
+				if (context->possible_sublink && !context->inserted_sublink)
+					context->inserted_sublink = checkExprHasSubLink(newvar);
+
 				fields = lappend(fields, newvar);
 				/* We need the names of non-dropped columns, too */
 				colnames = lappend(colnames, copyObject((Node *) lfirst(ln)));

src/test/regress/expected/subselect.out

@@ -706,6 +706,31 @@ from
 -----
 (0 rows)
 
+-- Likewise, but with the sublink inserted into a whole-row reference.
+select 1 from ((select (select 1) as x) ss1 cross join int4_tbl i4) j
+  where (select 1 where j is null) is null;
+ ?column? 
+----------
+        1
+        1
+        1
+        1
+        1
+(5 rows)
+
+select 1 from ((select (select 1) as x) ss1 cross join int4_tbl i4) j
+  where (1, 1) in (select (j is null)::int, count(*) from int4_tbl);
+ ?column? 
+----------
+(0 rows)
+
+select 1 from ((select (select 1) as x) ss1 cross join int4_tbl i4) j
+  where exists (select 1 from int4_tbl
+                tablesample system ((j is null)::int * 100));
+ ?column? 
+----------
+(0 rows)
+
 --
 -- Test case for subselect within UPDATE of INSERT...ON CONFLICT DO UPDATE
 --

src/test/regress/sql/subselect.sql

@@ -406,6 +406,18 @@ from
   join
   int4_tbl i4 on dummy = i4.f1;
 
+-- Likewise, but with the sublink inserted into a whole-row reference.
+
+select 1 from ((select (select 1) as x) ss1 cross join int4_tbl i4) j
+  where (select 1 where j is null) is null;
+
+select 1 from ((select (select 1) as x) ss1 cross join int4_tbl i4) j
+  where (1, 1) in (select (j is null)::int, count(*) from int4_tbl);
+
+select 1 from ((select (select 1) as x) ss1 cross join int4_tbl i4) j
+  where exists (select 1 from int4_tbl
+                tablesample system ((j is null)::int * 100));
+
 --
 -- Test case for subselect within UPDATE of INSERT...ON CONFLICT DO UPDATE
 --