Revise FDW planning API, again.

Further reflection shows that a single callback isn't very workable if we
desire to let FDWs generate multiple Paths, because that forces the FDW to
do all work necessary to generate a valid Plan node for each Path.  Instead
split the former PlanForeignScan API into three steps: GetForeignRelSize,
GetForeignPaths, GetForeignPlan.  We had already bit the bullet of breaking
the 9.1 FDW API for 9.2, so this shouldn't cause very much additional pain,
and it's substantially more flexible for complex FDWs.

Add an fdw_private field to RelOptInfo so that the new functions can save
state there rather than possibly having to recalculate information two or
three times.

In addition, we'd not thought through what would be needed to allow an FDW
to set up subexpressions of its choice for runtime execution.  We could
treat ForeignScan.fdw_private as an executable expression but that seems
likely to break existing FDWs unnecessarily (in particular, it would
restrict the set of node types allowable in fdw_private to those supported
by expression_tree_walker).  Instead, invent a separate field fdw_exprs
which will receive the postprocessing appropriate for expression trees.
(One field is enough since it can be a list of expressions; also, we assume
the corresponding expression state tree(s) will be held within fdw_state,
so we don't need to add anything to ForeignScanState.)

Per review of Hanada Shigeru's pgsql_fdw patch.  We may need to tweak this
further as we continue to work on that patch, but to me it feels a lot
closer to being right now.

src/backend/nodes/copyfuncs.c

@@ -591,8 +591,9 @@ _copyForeignScan(const ForeignScan *from)
 	/*
 	 * copy remainder of node
 	 */
-	COPY_SCALAR_FIELD(fsSystemCol);
+	COPY_NODE_FIELD(fdw_exprs);
 	COPY_NODE_FIELD(fdw_private);
+	COPY_SCALAR_FIELD(fsSystemCol);
 
 	return newnode;
 }

src/backend/nodes/outfuncs.c

@@ -559,8 +559,9 @@ _outForeignScan(StringInfo str, const ForeignScan *node)
 
 	_outScanInfo(str, (const Scan *) node);
 
-	WRITE_BOOL_FIELD(fsSystemCol);
+	WRITE_NODE_FIELD(fdw_exprs);
 	WRITE_NODE_FIELD(fdw_private);
+	WRITE_BOOL_FIELD(fsSystemCol);
 }
 
 static void
@@ -1741,6 +1742,7 @@ _outRelOptInfo(StringInfo str, const RelOptInfo *node)
 	WRITE_FLOAT_FIELD(allvisfrac, "%.6f");
 	WRITE_NODE_FIELD(subplan);
 	WRITE_NODE_FIELD(subroot);
+	/* we don't try to print fdwroutine or fdw_private */
 	WRITE_NODE_FIELD(baserestrictinfo);
 	WRITE_NODE_FIELD(joininfo);
 	WRITE_BOOL_FIELD(has_eclass_joins);

src/backend/optimizer/path/allpaths.c

@@ -396,6 +396,12 @@ set_foreign_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte)
 {
 	/* Mark rel with estimated output rows, width, etc */
 	set_foreign_size_estimates(root, rel);
+
+	/* Get FDW routine pointers for the rel */
+	rel->fdwroutine = GetFdwRoutineByRelId(rte->relid);
+
+	/* Let FDW adjust the size estimates, if it can */
+	rel->fdwroutine->GetForeignRelSize(root, rel, rte->relid);
 }
 
 /*
@@ -405,11 +411,8 @@ set_foreign_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte)
 static void
 set_foreign_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte)
 {
-	FdwRoutine *fdwroutine;
-
-	/* Call the FDW's PlanForeignScan function to generate path(s) */
-	fdwroutine = GetFdwRoutineByRelId(rte->relid);
-	fdwroutine->PlanForeignScan(rte->relid, root, rel);
+	/* Call the FDW's GetForeignPaths function to generate path(s) */
+	rel->fdwroutine->GetForeignPaths(root, rel, rte->relid);
 
 	/* Select cheapest path */
 	set_cheapest(rel);

src/backend/optimizer/path/costsize.c

@@ -3745,7 +3745,7 @@ set_cte_size_estimates(PlannerInfo *root, RelOptInfo *rel, Plan *cteplan)
  * using what will be purely datatype-driven estimates from the targetlist.
  * There is no way to do anything sane with the rows value, so we just put
  * a default estimate and hope that the wrapper can improve on it.	The
- * wrapper's PlanForeignScan function will be called momentarily.
+ * wrapper's GetForeignRelSize function will be called momentarily.
  *
  * The rel's targetlist and restrictinfo list must have been constructed
  * already.

src/backend/optimizer/plan/createplan.c

@@ -20,6 +20,7 @@
 #include <math.h>
 
 #include "access/skey.h"
+#include "foreign/fdwapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
 #include "nodes/nodeFuncs.h"
@@ -119,8 +120,6 @@ static CteScan *make_ctescan(List *qptlist, List *qpqual,
 			 Index scanrelid, int ctePlanId, int cteParam);
 static WorkTableScan *make_worktablescan(List *qptlist, List *qpqual,
 				   Index scanrelid, int wtParam);
-static ForeignScan *make_foreignscan(List *qptlist, List *qpqual,
-				 Index scanrelid, bool fsSystemCol, List *fdw_private);
 static BitmapAnd *make_bitmap_and(List *bitmapplans);
 static BitmapOr *make_bitmap_or(List *bitmapplans);
 static NestLoop *make_nestloop(List *tlist,
@@ -1816,7 +1815,6 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path,
 	RelOptInfo *rel = best_path->path.parent;
 	Index		scan_relid = rel->relid;
 	RangeTblEntry *rte;
-	bool		fsSystemCol;
 	int			i;
 
 	/* it should be a base rel... */
@@ -1825,31 +1823,56 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path,
 	rte = planner_rt_fetch(scan_relid, root);
 	Assert(rte->rtekind == RTE_RELATION);
 
-	/* Sort clauses into best execution order */
+	/*
+	 * Sort clauses into best execution order.  We do this first since the
+	 * FDW might have more info than we do and wish to adjust the ordering.
+	 */
 	scan_clauses = order_qual_clauses(root, scan_clauses);
 
-	/* Reduce RestrictInfo list to bare expressions; ignore pseudoconstants */
-	scan_clauses = extract_actual_clauses(scan_clauses, false);
+	/*
+	 * Let the FDW perform its processing on the restriction clauses and
+	 * generate the plan node.  Note that the FDW might remove restriction
+	 * clauses that it intends to execute remotely, or even add more (if it
+	 * has selected some join clauses for remote use but also wants them
+	 * rechecked locally).
+	 */
+	scan_plan = rel->fdwroutine->GetForeignPlan(root, rel, rte->relid,
+												best_path,
+												tlist, scan_clauses);
 
-	/* Detect whether any system columns are requested from rel */
-	fsSystemCol = false;
+	/* Copy cost data from Path to Plan; no need to make FDW do this */
+	copy_path_costsize(&scan_plan->scan.plan, &best_path->path);
+
+	/*
+	 * Replace any outer-relation variables with nestloop params in the qual
+	 * and fdw_exprs expressions.  We do this last so that the FDW doesn't
+	 * have to be involved.  (Note that parts of fdw_exprs could have come
+	 * from join clauses, so doing this beforehand on the scan_clauses
+	 * wouldn't work.)
+	 */
+	if (best_path->path.required_outer)
+	{
+		scan_plan->scan.plan.qual = (List *)
+			replace_nestloop_params(root, (Node *) scan_plan->scan.plan.qual);
+		scan_plan->fdw_exprs = (List *)
+			replace_nestloop_params(root, (Node *) scan_plan->fdw_exprs);
+	}
+
+	/*
+	 * Detect whether any system columns are requested from rel.  This is a
+	 * bit of a kluge and might go away someday, so we intentionally leave it
+	 * out of the API presented to FDWs.
+	 */
+	scan_plan->fsSystemCol = false;
 	for (i = rel->min_attr; i < 0; i++)
 	{
 		if (!bms_is_empty(rel->attr_needed[i - rel->min_attr]))
 		{
-			fsSystemCol = true;
+			scan_plan->fsSystemCol = true;
 			break;
 		}
 	}
 
-	scan_plan = make_foreignscan(tlist,
-								 scan_clauses,
-								 scan_relid,
-								 fsSystemCol,
-								 best_path->fdw_private);
-
-	copy_path_costsize(&scan_plan->scan.plan, &best_path->path);
-
 	return scan_plan;
 }
 
@@ -3183,24 +3206,26 @@ make_worktablescan(List *qptlist,
 	return node;
 }
 
-static ForeignScan *
+ForeignScan *
 make_foreignscan(List *qptlist,
 				 List *qpqual,
 				 Index scanrelid,
-				 bool fsSystemCol,
+				 List *fdw_exprs,
 				 List *fdw_private)
 {
 	ForeignScan *node = makeNode(ForeignScan);
 	Plan	   *plan = &node->scan.plan;
 
-	/* cost should be inserted by caller */
+	/* cost will be filled in by create_foreignscan_plan */
 	plan->targetlist = qptlist;
 	plan->qual = qpqual;
 	plan->lefttree = NULL;
 	plan->righttree = NULL;
 	node->scan.scanrelid = scanrelid;
-	node->fsSystemCol = fsSystemCol;
+	node->fdw_exprs = fdw_exprs;
 	node->fdw_private = fdw_private;
+	/* fsSystemCol will be filled in by create_foreignscan_plan */
+	node->fsSystemCol = false;
 
 	return node;
 }

src/backend/optimizer/plan/setrefs.c

@@ -428,6 +428,8 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset)
 					fix_scan_list(root, splan->scan.plan.targetlist, rtoffset);
 				splan->scan.plan.qual =
 					fix_scan_list(root, splan->scan.plan.qual, rtoffset);
+				splan->fdw_exprs =
+					fix_scan_list(root, splan->fdw_exprs, rtoffset);
 			}
 			break;
 

src/backend/optimizer/plan/subselect.c

@@ -2137,6 +2137,8 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params,
 			break;
 
 		case T_ForeignScan:
+			finalize_primnode((Node *) ((ForeignScan *) plan)->fdw_exprs,
+							  &context);
 			context.paramids = bms_add_members(context.paramids, scan_params);
 			break;
 

src/backend/optimizer/util/pathnode.c

@@ -1767,7 +1767,7 @@ create_worktablescan_path(PlannerInfo *root, RelOptInfo *rel)
  *	  returning the pathnode.
  *
  * This function is never called from core Postgres; rather, it's expected
- * to be called by the PlanForeignScan function of a foreign data wrapper.
+ * to be called by the GetForeignPaths function of a foreign data wrapper.
  * We make the FDW supply all fields of the path, since we do not have any
  * way to calculate them in core.
  */

src/backend/optimizer/util/relnode.c

@@ -113,6 +113,8 @@ build_simple_rel(PlannerInfo *root, int relid, RelOptKind reloptkind)
 	rel->allvisfrac = 0;
 	rel->subplan = NULL;
 	rel->subroot = NULL;
+	rel->fdwroutine = NULL;
+	rel->fdw_private = NULL;
 	rel->baserestrictinfo = NIL;
 	rel->baserestrictcost.startup = 0;
 	rel->baserestrictcost.per_tuple = 0;
@@ -366,6 +368,8 @@ build_join_rel(PlannerInfo *root,
 	joinrel->allvisfrac = 0;
 	joinrel->subplan = NULL;
 	joinrel->subroot = NULL;
+	joinrel->fdwroutine = NULL;
+	joinrel->fdw_private = NULL;
 	joinrel->baserestrictinfo = NIL;
 	joinrel->baserestrictcost.startup = 0;
 	joinrel->baserestrictcost.per_tuple = 0;

src/include/foreign/fdwapi.h

@@ -23,9 +23,20 @@ struct ExplainState;
  * Callback function signatures --- see fdwhandler.sgml for more info.
  */
 
-typedef void (*PlanForeignScan_function) (Oid foreigntableid,
-										  PlannerInfo *root,
-										  RelOptInfo *baserel);
+typedef void (*GetForeignRelSize_function) (PlannerInfo *root,
+											RelOptInfo *baserel,
+											Oid foreigntableid);
+
+typedef void (*GetForeignPaths_function) (PlannerInfo *root,
+										  RelOptInfo *baserel,
+										  Oid foreigntableid);
+
+typedef ForeignScan *(*GetForeignPlan_function) (PlannerInfo *root,
+												 RelOptInfo *baserel,
+												 Oid foreigntableid,
+												 ForeignPath *best_path,
+												 List *tlist,
+												 List *scan_clauses);
 
 typedef void (*ExplainForeignScan_function) (ForeignScanState *node,
 													struct ExplainState *es);
@@ -53,7 +64,9 @@ typedef struct FdwRoutine
 {
 	NodeTag		type;
 
-	PlanForeignScan_function PlanForeignScan;
+	GetForeignRelSize_function GetForeignRelSize;
+	GetForeignPaths_function GetForeignPaths;
+	GetForeignPlan_function GetForeignPlan;
 	ExplainForeignScan_function ExplainForeignScan;
 	BeginForeignScan_function BeginForeignScan;
 	IterateForeignScan_function IterateForeignScan;

src/include/nodes/plannodes.h

@@ -462,13 +462,22 @@ typedef struct WorkTableScan
 
 /* ----------------
  *		ForeignScan node
+ *
+ * fdw_exprs and fdw_private are both under the control of the foreign-data
+ * wrapper, but fdw_exprs is presumed to contain expression trees and will
+ * be post-processed accordingly by the planner; fdw_private won't be.
+ * Note that everything in both lists must be copiable by copyObject().
+ * One way to store an arbitrary blob of bytes is to represent it as a bytea
+ * Const.  Usually, though, you'll be better off choosing a representation
+ * that can be dumped usefully by nodeToString().
  * ----------------
  */
 typedef struct ForeignScan
 {
 	Scan		scan;
-	bool		fsSystemCol;	/* true if any "system column" is needed */
+	List	   *fdw_exprs;		/* expressions that FDW may evaluate */
 	List	   *fdw_private;	/* private data for FDW */
+	bool		fsSystemCol;	/* true if any "system column" is needed */
 } ForeignScan;
 
 

src/include/nodes/relation.h

@@ -334,10 +334,13 @@ typedef struct PlannerInfo
  *		allvisfrac - fraction of disk pages that are marked all-visible
  *		subplan - plan for subquery (NULL if it's not a subquery)
  *		subroot - PlannerInfo for subquery (NULL if it's not a subquery)
+ *		fdwroutine - function hooks for FDW, if foreign table (else NULL)
+ *		fdw_private - private state for FDW, if foreign table (else NULL)
  *
  *		Note: for a subquery, tuples, subplan, subroot are not set immediately
  *		upon creation of the RelOptInfo object; they are filled in when
- *		set_base_rel_pathlist processes the object.
+ *		set_subquery_pathlist processes the object.  Likewise, fdwroutine
+ *		and fdw_private are filled during initial path creation.
  *
  *		For otherrels that are appendrel members, these fields are filled
  *		in just as for a baserel.
@@ -414,8 +417,12 @@ typedef struct RelOptInfo
 	BlockNumber pages;			/* size estimates derived from pg_class */
 	double		tuples;
 	double		allvisfrac;
+	/* use "struct Plan" to avoid including plannodes.h here */
 	struct Plan *subplan;		/* if subquery */
 	PlannerInfo *subroot;		/* if subquery */
+	/* use "struct FdwRoutine" to avoid including fdwapi.h here */
+	struct FdwRoutine *fdwroutine;	/* if foreign table */
+	void	   *fdw_private;	/* if foreign table */
 
 	/* used by various scans and joins: */
 	List	   *baserestrictinfo;		/* RestrictInfo structures (if base
@@ -793,14 +800,13 @@ typedef struct TidPath
 } TidPath;
 
 /*
- * ForeignPath represents a scan of a foreign table
+ * ForeignPath represents a potential scan of a foreign table
  *
- * fdw_private contains FDW private data about the scan, which will be copied
- * to the final ForeignScan plan node so that it is available at execution
- * time.  Note that everything in this list must be copiable by copyObject().
- * One way to store an arbitrary blob of bytes is to represent it as a bytea
- * Const.  Usually, though, you'll be better off choosing a representation
- * that can be dumped usefully by nodeToString().
+ * fdw_private stores FDW private data about the scan.  While fdw_private is
+ * not actually touched by the core code during normal operations, it's
+ * generally a good idea to use a representation that can be dumped by
+ * nodeToString(), so that you can examine the structure during debugging
+ * with tools like pprint().
  */
 typedef struct ForeignPath
 {

src/include/optimizer/planmain.h

@@ -42,6 +42,8 @@ extern Plan *optimize_minmax_aggregates(PlannerInfo *root, List *tlist,
 extern Plan *create_plan(PlannerInfo *root, Path *best_path);
 extern SubqueryScan *make_subqueryscan(List *qptlist, List *qpqual,
 				  Index scanrelid, Plan *subplan);
+extern ForeignScan *make_foreignscan(List *qptlist, List *qpqual,
+				 Index scanrelid, List *fdw_exprs, List *fdw_private);
 extern Append *make_append(List *appendplans, List *tlist);
 extern RecursiveUnion *make_recursive_union(List *tlist,
 					 Plan *lefttree, Plan *righttree, int wtParam,