From 876fd37fc8126c4725c2aca709a6be34c2001258 Mon Sep 17 00:00:00 2001 From: Tom Lane Date: Thu, 7 Feb 2019 13:10:46 -0500 Subject: [PATCH] Ensure that foreign scans with lateral refs are planned correctly. As reported in bug #15613 from Srinivasan S A, file_fdw and postgres_fdw neglected to mark plain baserel foreign paths as parameterized when the relation has lateral_relids. Other FDWs have surely copied this mistake, so rather than just patching those two modules, install a band-aid fix in create_foreignscan_path to rectify the mistake centrally. Although the band-aid is enough to fix the visible symptom, correct the calls in file_fdw and postgres_fdw anyway, so that they are valid examples for external FDWs. Also, since the band-aid isn't enough to make this work for parameterized foreign joins, throw an elog(ERROR) if such a case is passed to create_foreignscan_path. This shouldn't pose much of a problem for existing external FDWs, since it's likely they aren't trying to make such paths anyway (though some of them may need a defense against joins with lateral_relids, similar to the one this patch installs into postgres_fdw). Add some assertions in relnode.c to catch future occurrences of the same error --- in particular, as backstop against core-code mistakes like the one fixed by commit bdd9a99aa. Discussion: https://postgr.es/m/15613-092be1be9576c728@postgresql.org --- contrib/file_fdw/file_fdw.c | 6 +- .../postgres_fdw/expected/postgres_fdw.out | 56 +++++++++++++++++++ contrib/postgres_fdw/postgres_fdw.c | 5 +- contrib/postgres_fdw/sql/postgres_fdw.sql | 26 +++++++++ src/backend/optimizer/util/pathnode.c | 23 ++++++++ src/backend/optimizer/util/relnode.c | 9 +++ 6 files changed, 123 insertions(+), 2 deletions(-) diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 5a4d5aac215..f94745c8fdb 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -516,6 +516,10 @@ fileGetForeignPaths(PlannerInfo *root, * Create a ForeignPath node and add it as only possible path. We use the * fdw_private list of the path to carry the convert_selectively option; * it will be propagated into the fdw_private list of the Plan node. + * + * We don't support pushing join clauses into the quals of this path, but + * it could still have required parameterization due to LATERAL refs in + * its tlist. */ add_path(baserel, (Path *) create_foreignscan_path(root, baserel, @@ -523,7 +527,7 @@ fileGetForeignPaths(PlannerInfo *root, startup_cost, total_cost, NIL, /* no pathkeys */ - NULL, /* no outer rel either */ + baserel->lateral_relids, coptions)); /* diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index e46e1a4aa70..20b627b74ce 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -682,6 +682,62 @@ SELECT * FROM ft2 WHERE c1 = ANY (ARRAY(SELECT c1 FROM ft1 WHERE c1 < 5)); 4 | 4 | 00004 | Mon Jan 05 00:00:00 1970 PST | Mon Jan 05 00:00:00 1970 | 4 | 4 | foo (4 rows) +-- bug #15613: bad plan for foreign table scan with lateral reference +EXPLAIN (VERBOSE, COSTS OFF) +SELECT ref_0.c2, subq_1.* +FROM + "S 1"."T 1" AS ref_0, + LATERAL ( + SELECT ref_0."C 1" c1, subq_0.* + FROM (SELECT ref_0.c2, ref_1.c3 + FROM ft1 AS ref_1) AS subq_0 + RIGHT JOIN ft2 AS ref_3 ON (subq_0.c3 = ref_3.c3) + ) AS subq_1 +WHERE ref_0."C 1" < 10 AND subq_1.c3 = '00001' +ORDER BY ref_0."C 1"; + QUERY PLAN +--------------------------------------------------------------------------------------------------------- + Nested Loop + Output: ref_0.c2, ref_0."C 1", (ref_0.c2), ref_1.c3, ref_0."C 1" + -> Nested Loop + Output: ref_0.c2, ref_0."C 1", ref_1.c3, (ref_0.c2) + -> Index Scan using t1_pkey on "S 1"."T 1" ref_0 + Output: ref_0."C 1", ref_0.c2, ref_0.c3, ref_0.c4, ref_0.c5, ref_0.c6, ref_0.c7, ref_0.c8 + Index Cond: (ref_0."C 1" < 10) + -> Foreign Scan on public.ft1 ref_1 + Output: ref_1.c3, ref_0.c2 + Remote SQL: SELECT c3 FROM "S 1"."T 1" WHERE ((c3 = '00001'::text)) + -> Materialize + Output: ref_3.c3 + -> Foreign Scan on public.ft2 ref_3 + Output: ref_3.c3 + Remote SQL: SELECT c3 FROM "S 1"."T 1" WHERE ((c3 = '00001'::text)) +(15 rows) + +SELECT ref_0.c2, subq_1.* +FROM + "S 1"."T 1" AS ref_0, + LATERAL ( + SELECT ref_0."C 1" c1, subq_0.* + FROM (SELECT ref_0.c2, ref_1.c3 + FROM ft1 AS ref_1) AS subq_0 + RIGHT JOIN ft2 AS ref_3 ON (subq_0.c3 = ref_3.c3) + ) AS subq_1 +WHERE ref_0."C 1" < 10 AND subq_1.c3 = '00001' +ORDER BY ref_0."C 1"; + c2 | c1 | c2 | c3 +----+----+----+------- + 1 | 1 | 1 | 00001 + 2 | 2 | 2 | 00001 + 3 | 3 | 3 | 00001 + 4 | 4 | 4 | 00001 + 5 | 5 | 5 | 00001 + 6 | 6 | 6 | 00001 + 7 | 7 | 7 | 00001 + 8 | 8 | 8 | 00001 + 9 | 9 | 9 | 00001 +(9 rows) + -- =================================================================== -- parameterized queries -- =================================================================== diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 5a4a47dbd2b..c5d9ee43dd9 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -547,13 +547,16 @@ postgresGetForeignPaths(PlannerInfo *root, * baserestrict conditions we were able to send to remote, there might * actually be an indexscan happening there). We already did all the work * to estimate cost and size of this path. + * + * Although this path uses no join clauses, it could still have required + * parameterization due to LATERAL refs in its tlist. */ path = create_foreignscan_path(root, baserel, fpinfo->rows, fpinfo->startup_cost, fpinfo->total_cost, NIL, /* no pathkeys */ - NULL, /* no outer rel either */ + baserel->lateral_relids, NIL); /* no fdw_private list */ add_path(baserel, (Path *) path); diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 20871fd47b1..440e69099d7 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -232,6 +232,32 @@ WHERE a.c2 = 6 AND b.c1 = a.c1 AND a.c8 = 'foo' AND b.c7 = upper(a.c7); SELECT * FROM ft1 WHERE c1 = ANY (ARRAY(SELECT c1 FROM ft2 WHERE c1 < 5)); SELECT * FROM ft2 WHERE c1 = ANY (ARRAY(SELECT c1 FROM ft1 WHERE c1 < 5)); +-- bug #15613: bad plan for foreign table scan with lateral reference +EXPLAIN (VERBOSE, COSTS OFF) +SELECT ref_0.c2, subq_1.* +FROM + "S 1"."T 1" AS ref_0, + LATERAL ( + SELECT ref_0."C 1" c1, subq_0.* + FROM (SELECT ref_0.c2, ref_1.c3 + FROM ft1 AS ref_1) AS subq_0 + RIGHT JOIN ft2 AS ref_3 ON (subq_0.c3 = ref_3.c3) + ) AS subq_1 +WHERE ref_0."C 1" < 10 AND subq_1.c3 = '00001' +ORDER BY ref_0."C 1"; + +SELECT ref_0.c2, subq_1.* +FROM + "S 1"."T 1" AS ref_0, + LATERAL ( + SELECT ref_0."C 1" c1, subq_0.* + FROM (SELECT ref_0.c2, ref_1.c3 + FROM ft1 AS ref_1) AS subq_0 + RIGHT JOIN ft2 AS ref_3 ON (subq_0.c3 = ref_3.c3) + ) AS subq_1 +WHERE ref_0."C 1" < 10 AND subq_1.c3 = '00001' +ORDER BY ref_0."C 1"; + -- =================================================================== -- parameterized queries -- =================================================================== diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 9b90be5a3e6..11dc6954965 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1736,6 +1736,29 @@ create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel, { ForeignPath *pathnode = makeNode(ForeignPath); + /* + * Since the path's required_outer should always include all the rel's + * lateral_relids, forcibly add those if necessary. This is a bit of a + * hack, but up till early 2019 the contrib FDWs failed to ensure that, + * and it's likely that the same error has propagated into many external + * FDWs. Don't risk modifying the passed-in relid set here. + */ + if (rel->lateral_relids && !bms_is_subset(rel->lateral_relids, + required_outer)) + required_outer = bms_union(required_outer, rel->lateral_relids); + + /* + * Although this function is only designed to be used for scans of + * baserels, before v12 postgres_fdw abused it to make paths for join and + * upper rels. It will work for such cases as long as required_outer is + * empty (otherwise get_baserel_parampathinfo does the wrong thing), which + * fortunately is the expected case for now. + */ + if (!bms_is_empty(required_outer) && + !(rel->reloptkind == RELOPT_BASEREL || + rel->reloptkind == RELOPT_OTHER_MEMBER_REL)) + elog(ERROR, "parameterized foreign joins are not supported yet"); + pathnode->path.pathtype = T_ForeignScan; pathnode->path.parent = rel; pathnode->path.param_info = get_baserel_parampathinfo(root, rel, diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c index 09cfb458dd6..e16a318d4c2 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c @@ -857,6 +857,9 @@ get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, double rows; ListCell *lc; + /* If rel has LATERAL refs, every path for it should account for them */ + Assert(bms_is_subset(baserel->lateral_relids, required_outer)); + /* Unparameterized paths have no ParamPathInfo */ if (bms_is_empty(required_outer)) return NULL; @@ -956,6 +959,9 @@ get_joinrel_parampathinfo(PlannerInfo *root, RelOptInfo *joinrel, double rows; ListCell *lc; + /* If rel has LATERAL refs, every path for it should account for them */ + Assert(bms_is_subset(joinrel->lateral_relids, required_outer)); + /* Unparameterized paths have no ParamPathInfo or extra join clauses */ if (bms_is_empty(required_outer)) return NULL; @@ -1152,6 +1158,9 @@ get_appendrel_parampathinfo(RelOptInfo *appendrel, Relids required_outer) ParamPathInfo *ppi; ListCell *lc; + /* If rel has LATERAL refs, every path for it should account for them */ + Assert(bms_is_subset(appendrel->lateral_relids, required_outer)); + /* Unparameterized paths have no ParamPathInfo */ if (bms_is_empty(required_outer)) return NULL;