Allow FDWs and custom scan providers to replace joins with scans.

Foreign data wrappers can use this capability for so-called "join
pushdown"; that is, instead of executing two separate foreign scans
and then joining the results locally, they can generate a path which
performs the join on the remote server and then is scanned locally.
This commit does not extend postgres_fdw to take advantage of this
capability; it just provides the infrastructure.

Custom scan providers can use this in a similar way.  Previously,
it was only possible for a custom scan provider to scan a single
relation.  Now, it can scan an entire join tree, provided of course
that it knows how to produce the same results that the join would
have produced if executed normally.

KaiGai Kohei, reviewed by Shigeru Hanada, Ashutosh Bapat, and me.

src/backend/optimizer/path/joinpath.c

@@ -17,10 +17,13 @@
 #include <math.h>
 
 #include "executor/executor.h"
+#include "foreign/fdwapi.h"
 #include "optimizer/cost.h"
 #include "optimizer/pathnode.h"
 #include "optimizer/paths.h"
 
+/* Hook for plugins to get control in add_paths_to_joinrel() */
+set_join_pathlist_hook_type set_join_pathlist_hook = NULL;
 
 #define PATH_PARAM_BY_REL(path, rel)  \
 	((path)->param_info && bms_overlap(PATH_REQ_OUTER(path), (rel)->relids))
@@ -260,6 +263,27 @@ add_paths_to_joinrel(PlannerInfo *root,
 							 restrictlist, jointype,
 							 sjinfo, &semifactors,
 							 param_source_rels, extra_lateral_rels);
+
+	/*
+	 * 5. If both inner and outer relations are managed by the same FDW,
+	 * give it a chance to push down joins.
+	 */
+	if (joinrel->fdwroutine &&
+		joinrel->fdwroutine->GetForeignJoinPaths)
+		joinrel->fdwroutine->GetForeignJoinPaths(root, joinrel,
+												 outerrel, innerrel,
+												 restrictlist, jointype, sjinfo,
+												 &semifactors,
+												 param_source_rels,
+												 extra_lateral_rels);
+	/*
+	 * 6. Finally, give extensions a chance to manipulate the path list.
+	 */
+	if (set_join_pathlist_hook)
+		set_join_pathlist_hook(root, joinrel, outerrel, innerrel,
+							   restrictlist, jointype,
+							   sjinfo, &semifactors,
+							   param_source_rels, extra_lateral_rels);
 }
 
 /*

src/backend/optimizer/plan/createplan.c

@@ -44,7 +44,6 @@
 #include "utils/lsyscache.h"
 
 
-static Plan *create_plan_recurse(PlannerInfo *root, Path *best_path);
 static Plan *create_scan_plan(PlannerInfo *root, Path *best_path);
 static List *build_path_tlist(PlannerInfo *root, Path *path);
 static bool use_physical_tlist(PlannerInfo *root, RelOptInfo *rel);
@@ -220,7 +219,7 @@ create_plan(PlannerInfo *root, Path *best_path)
  * create_plan_recurse
  *	  Recursive guts of create_plan().
  */
-static Plan *
+Plan *
 create_plan_recurse(PlannerInfo *root, Path *best_path)
 {
 	Plan	   *plan;
@@ -1961,16 +1960,25 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path,
 	ForeignScan *scan_plan;
 	RelOptInfo *rel = best_path->path.parent;
 	Index		scan_relid = rel->relid;
-	RangeTblEntry *rte;
+	Oid			rel_oid = InvalidOid;
 	Bitmapset  *attrs_used = NULL;
 	ListCell   *lc;
 	int			i;
 
-	/* it should be a base rel... */
-	Assert(scan_relid > 0);
-	Assert(rel->rtekind == RTE_RELATION);
-	rte = planner_rt_fetch(scan_relid, root);
-	Assert(rte->rtekind == RTE_RELATION);
+	/*
+	 * If we're scanning a base relation, look up the OID.
+	 * (We can skip this if scanning a join relation.)
+	 */
+	if (scan_relid > 0)
+	{
+		RangeTblEntry *rte;
+
+		Assert(rel->rtekind == RTE_RELATION);
+		rte = planner_rt_fetch(scan_relid, root);
+		Assert(rte->rtekind == RTE_RELATION);
+		rel_oid = rte->relid;
+	}
+	Assert(rel->fdwroutine != NULL);
 
 	/*
 	 * Sort clauses into best execution order.  We do this first since the FDW
@@ -1985,13 +1993,39 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path,
 	 * has selected some join clauses for remote use but also wants them
 	 * rechecked locally).
 	 */
-	scan_plan = rel->fdwroutine->GetForeignPlan(root, rel, rte->relid,
+	scan_plan = rel->fdwroutine->GetForeignPlan(root, rel, rel_oid,
 												best_path,
 												tlist, scan_clauses);
+	/*
+	 * Sanity check.  There may be resjunk entries in fdw_ps_tlist that
+	 * are included only to help EXPLAIN deparse plans properly. We require
+	 * that these are at the end, so that when the executor builds the scan
+	 * descriptor based on the non-junk entries, it gets the attribute
+	 * numbers correct.
+	 */
+	if (scan_plan->scan.scanrelid == 0)
+	{
+		bool	found_resjunk = false;
+
+		foreach (lc, scan_plan->fdw_ps_tlist)
+		{
+			TargetEntry	   *tle = lfirst(lc);
+
+			if (tle->resjunk)
+				found_resjunk = true;
+			else if (found_resjunk)
+				elog(ERROR, "junk TLE should not apper prior to valid one");
+		}
+	}
+	/* Set the relids that are represented by this foreign scan for Explain */
+	scan_plan->fdw_relids = best_path->path.parent->relids;
 
 	/* Copy cost data from Path to Plan; no need to make FDW do this */
 	copy_path_costsize(&scan_plan->scan.plan, &best_path->path);
 
+	/* Track FDW server-id; no need to make FDW do this */
+	scan_plan->fdw_handler = rel->fdw_handler;
+
 	/*
 	 * Replace any outer-relation variables with nestloop params in the qual
 	 * and fdw_exprs expressions.  We do this last so that the FDW doesn't
@@ -2053,12 +2087,7 @@ create_customscan_plan(PlannerInfo *root, CustomPath *best_path,
 {
 	CustomScan *cplan;
 	RelOptInfo *rel = best_path->path.parent;
-
-	/*
-	 * Right now, all we can support is CustomScan node which is associated
-	 * with a particular base relation to be scanned.
-	 */
-	Assert(rel && rel->reloptkind == RELOPT_BASEREL);
+	ListCell   *lc;
 
 	/*
 	 * Sort clauses into the best execution order, although custom-scan
@@ -2077,6 +2106,30 @@ create_customscan_plan(PlannerInfo *root, CustomPath *best_path,
 															  scan_clauses);
 	Assert(IsA(cplan, CustomScan));
 
+	/*
+	 * Sanity check.  There may be resjunk entries in custom_ps_tlist that
+	 * are included only to help EXPLAIN deparse plans properly. We require
+	 * that these are at the end, so that when the executor builds the scan
+	 * descriptor based on the non-junk entries, it gets the attribute
+	 * numbers correct.
+	 */
+	if (cplan->scan.scanrelid == 0)
+	{
+		bool	found_resjunk = false;
+
+		foreach (lc, cplan->custom_ps_tlist)
+		{
+			TargetEntry	   *tle = lfirst(lc);
+
+			if (tle->resjunk)
+				found_resjunk = true;
+			else if (found_resjunk)
+				elog(ERROR, "junk TLE should not apper prior to valid one");
+		}
+	}
+	/* Set the relids that are represented by this custom scan for Explain */
+	cplan->custom_relids = best_path->path.parent->relids;
+
 	/*
 	 * Copy cost data from Path to Plan; no need to make custom-plan providers
 	 * do this

src/backend/optimizer/plan/setrefs.c

@@ -86,6 +86,12 @@ static void flatten_unplanned_rtes(PlannerGlobal *glob, RangeTblEntry *rte);
 static bool flatten_rtes_walker(Node *node, PlannerGlobal *glob);
 static void add_rte_to_flat_rtable(PlannerGlobal *glob, RangeTblEntry *rte);
 static Plan *set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset);
+static void set_foreignscan_references(PlannerInfo *root,
+									   ForeignScan *fscan,
+									   int rtoffset);
+static void set_customscan_references(PlannerInfo *root,
+									  CustomScan *cscan,
+									  int rtoffset);
 static Plan *set_indexonlyscan_references(PlannerInfo *root,
 							 IndexOnlyScan *plan,
 							 int rtoffset);
@@ -565,31 +571,11 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset)
 			}
 			break;
 		case T_ForeignScan:
-			{
-				ForeignScan *splan = (ForeignScan *) plan;
-
-				splan->scan.scanrelid += rtoffset;
-				splan->scan.plan.targetlist =
-					fix_scan_list(root, splan->scan.plan.targetlist, rtoffset);
-				splan->scan.plan.qual =
-					fix_scan_list(root, splan->scan.plan.qual, rtoffset);
-				splan->fdw_exprs =
-					fix_scan_list(root, splan->fdw_exprs, rtoffset);
-			}
+			set_foreignscan_references(root, (ForeignScan *) plan, rtoffset);
 			break;
 
 		case T_CustomScan:
-			{
-				CustomScan *splan = (CustomScan *) plan;
-
-				splan->scan.scanrelid += rtoffset;
-				splan->scan.plan.targetlist =
-					fix_scan_list(root, splan->scan.plan.targetlist, rtoffset);
-				splan->scan.plan.qual =
-					fix_scan_list(root, splan->scan.plan.qual, rtoffset);
-				splan->custom_exprs =
-					fix_scan_list(root, splan->custom_exprs, rtoffset);
-			}
+			set_customscan_references(root, (CustomScan *) plan, rtoffset);
 			break;
 
 		case T_NestLoop:
@@ -876,6 +862,121 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset)
 	return plan;
 }
 
+/*
+ * set_foreignscan_references
+ *     Do set_plan_references processing on an ForeignScan
+ */
+static void
+set_foreignscan_references(PlannerInfo *root,
+						   ForeignScan *fscan,
+						   int rtoffset)
+{
+	if (rtoffset > 0)
+	{
+		Bitmapset  *tempset = NULL;
+		int			x = -1;
+
+		while ((x = bms_next_member(fscan->fdw_relids, x)) >= 0)
+			tempset = bms_add_member(tempset, x + rtoffset);
+		fscan->fdw_relids = tempset;
+	}
+
+	if (fscan->scan.scanrelid == 0)
+	{
+		indexed_tlist *pscan_itlist = build_tlist_index(fscan->fdw_ps_tlist);
+
+		fscan->scan.plan.targetlist = (List *)
+			fix_upper_expr(root,
+						   (Node *) fscan->scan.plan.targetlist,
+						   pscan_itlist,
+						   INDEX_VAR,
+						   rtoffset);
+		fscan->scan.plan.qual = (List *)
+			fix_upper_expr(root,
+						   (Node *) fscan->scan.plan.qual,
+						   pscan_itlist,
+						   INDEX_VAR,
+						   rtoffset);
+		fscan->fdw_exprs = (List *)
+			fix_upper_expr(root,
+						   (Node *) fscan->fdw_exprs,
+						   pscan_itlist,
+						   INDEX_VAR,
+						   rtoffset);
+		fscan->fdw_ps_tlist =
+			fix_scan_list(root, fscan->fdw_ps_tlist, rtoffset);
+		pfree(pscan_itlist);
+	}
+	else
+	{
+		fscan->scan.scanrelid += rtoffset;
+		fscan->scan.plan.targetlist =
+			fix_scan_list(root, fscan->scan.plan.targetlist, rtoffset);
+		fscan->scan.plan.qual =
+			fix_scan_list(root, fscan->scan.plan.qual, rtoffset);
+		fscan->fdw_exprs =
+			fix_scan_list(root, fscan->fdw_exprs, rtoffset);
+	}
+}
+
+/*
+ * set_customscan_references
+ *     Do set_plan_references processing on an CustomScan
+ */
+static void
+set_customscan_references(PlannerInfo *root,
+						  CustomScan *cscan,
+						  int rtoffset)
+{
+	if (rtoffset > 0)
+	{
+		Bitmapset  *tempset = NULL;
+		int			x = -1;
+
+		while ((x = bms_next_member(cscan->custom_relids, x)) >= 0)
+			tempset = bms_add_member(tempset, x + rtoffset);
+		cscan->custom_relids = tempset;
+	}
+
+	if (cscan->scan.scanrelid == 0)
+	{
+		indexed_tlist *pscan_itlist =
+			build_tlist_index(cscan->custom_ps_tlist);
+
+		cscan->scan.plan.targetlist = (List *)
+			fix_upper_expr(root,
+						   (Node *) cscan->scan.plan.targetlist,
+						   pscan_itlist,
+						   INDEX_VAR,
+						   rtoffset);
+		cscan->scan.plan.qual = (List *)
+			fix_upper_expr(root,
+						   (Node *) cscan->scan.plan.qual,
+						   pscan_itlist,
+						   INDEX_VAR,
+						   rtoffset);
+		cscan->custom_exprs = (List *)
+			fix_upper_expr(root,
+						   (Node *) cscan->custom_exprs,
+						   pscan_itlist,
+						   INDEX_VAR,
+						   rtoffset);
+		cscan->custom_ps_tlist =
+			fix_scan_list(root, cscan->custom_ps_tlist, rtoffset);
+		pfree(pscan_itlist);
+	}
+	else
+	{
+		cscan->scan.scanrelid += rtoffset;
+		cscan->scan.plan.targetlist =
+			fix_scan_list(root, cscan->scan.plan.targetlist, rtoffset);
+		cscan->scan.plan.qual =
+			fix_scan_list(root, cscan->scan.plan.qual, rtoffset);
+		cscan->custom_exprs =
+			fix_scan_list(root, cscan->custom_exprs, rtoffset);
+	}
+}
+
 /*
  * set_indexonlyscan_references
  *		Do set_plan_references processing on an IndexOnlyScan

src/backend/optimizer/util/plancat.c

@@ -379,10 +379,15 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent,
 
 	/* Grab the fdwroutine info using the relcache, while we have it */
 	if (relation->rd_rel->relkind == RELKIND_FOREIGN_TABLE)
+	{
+		rel->fdw_handler = GetFdwHandlerByRelId(RelationGetRelid(relation));
 		rel->fdwroutine = GetFdwRoutineForRelation(relation, true);
+	}
 	else
+	{
+		rel->fdw_handler = InvalidOid;
 		rel->fdwroutine = NULL;
-
+	}
 	heap_close(relation, NoLock);
 
 	/*

src/backend/optimizer/util/relnode.c

@@ -14,6 +14,7 @@
  */
 #include "postgres.h"
 
+#include "foreign/fdwapi.h"
 #include "optimizer/cost.h"
 #include "optimizer/pathnode.h"
 #include "optimizer/paths.h"
@@ -122,6 +123,7 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptKind reloptkind)
 	rel->subroot = NULL;
 	rel->subplan_params = NIL;
 	rel->fdwroutine = NULL;
+	rel->fdw_handler = InvalidOid;
 	rel->fdw_private = NULL;
 	rel->baserestrictinfo = NIL;
 	rel->baserestrictcost.startup = 0;
@@ -426,6 +428,18 @@ build_join_rel(PlannerInfo *root,
 	set_joinrel_size_estimates(root, joinrel, outer_rel, inner_rel,
 							   sjinfo, restrictlist);
 
+	/*
+	 * Set FDW handler and routine if both outer and inner relation
+	 * are managed by same FDW driver.
+	 */
+	if (OidIsValid(outer_rel->fdw_handler) &&
+		OidIsValid(inner_rel->fdw_handler) &&
+		outer_rel->fdw_handler == inner_rel->fdw_handler)
+	{
+		joinrel->fdw_handler = outer_rel->fdw_handler;
+		joinrel->fdwroutine = GetFdwRoutine(joinrel->fdw_handler);
+	}
+
 	/*
 	 * Add the joinrel to the query's joinrel list, and store it into the
 	 * auxiliary hashtable if there is one.  NB: GEQO requires us to append