MERGE SQL Command following SQL:2016

MERGE performs actions that modify rows in the target table
using a source table or query. MERGE provides a single SQL
statement that can conditionally INSERT/UPDATE/DELETE rows
a task that would other require multiple PL statements.
e.g.

MERGE INTO target AS t
USING source AS s
ON t.tid = s.sid
WHEN MATCHED AND t.balance > s.delta THEN
  UPDATE SET balance = t.balance - s.delta
WHEN MATCHED THEN
  DELETE
WHEN NOT MATCHED AND s.delta > 0 THEN
  INSERT VALUES (s.sid, s.delta)
WHEN NOT MATCHED THEN
  DO NOTHING;

MERGE works with regular and partitioned tables, including
column and row security enforcement, as well as support for
row, statement and transition triggers.

MERGE is optimized for OLTP and is parameterizable, though
also useful for large scale ETL/ELT. MERGE is not intended
to be used in preference to existing single SQL commands
for INSERT, UPDATE or DELETE since there is some overhead.
MERGE can be used statically from PL/pgSQL.

MERGE does not yet support inheritance, write rules,
RETURNING clauses, updatable views or foreign tables.
MERGE follows SQL Standard per the most recent SQL:2016.

Includes full tests and documentation, including full
isolation tests to demonstrate the concurrent behavior.

This version written from scratch in 2017 by Simon Riggs,
using docs and tests originally written in 2009. Later work
from Pavan Deolasee has been both complex and deep, leaving
the lead author credit now in his hands.
Extensive discussion of concurrency from Peter Geoghegan,
with thanks for the time and effort contributed.

Various issues reported via sqlsmith by Andreas Seltenreich

Authors: Pavan Deolasee, Simon Riggs
Reviewer: Peter Geoghegan, Amit Langote, Tomas Vondra, Simon Riggs

Discussion:
https://postgr.es/m/CANP8+jKitBSrB7oTgT9CY2i1ObfOt36z0XMraQc+Xrz8QB0nXA@mail.gmail.com
https://postgr.es/m/CAH2-WzkJdBuxj9PO=2QaO9-3h3xGbQPZ34kJH=HukRekwM-GZg@mail.gmail.com

src/backend/optimizer/plan/createplan.c

@@ -288,9 +288,13 @@ static ModifyTable *make_modifytable(PlannerInfo *root,
 				 CmdType operation, bool canSetTag,
 				 Index nominalRelation, List *partitioned_rels,
 				 bool partColsUpdated,
-				 List *resultRelations, List *subplans,
+				 List *resultRelations,
+				 Index mergeTargetRelation,
+				 List *subplans,
 				 List *withCheckOptionLists, List *returningLists,
-				 List *rowMarks, OnConflictExpr *onconflict, int epqParam);
+				 List *rowMarks, OnConflictExpr *onconflict,
+				 List *mergeSourceTargetList,
+				 List *mergeActionList, int epqParam);
 static GatherMerge *create_gather_merge_plan(PlannerInfo *root,
 						 GatherMergePath *best_path);
 
@@ -2446,11 +2450,14 @@ create_modifytable_plan(PlannerInfo *root, ModifyTablePath *best_path)
 							best_path->partitioned_rels,
 							best_path->partColsUpdated,
 							best_path->resultRelations,
+							best_path->mergeTargetRelation,
 							subplans,
 							best_path->withCheckOptionLists,
 							best_path->returningLists,
 							best_path->rowMarks,
 							best_path->onconflict,
+							best_path->mergeSourceTargetList,
+							best_path->mergeActionList,
 							best_path->epqParam);
 
 	copy_generic_path_info(&plan->plan, &best_path->path);
@@ -6517,9 +6524,13 @@ make_modifytable(PlannerInfo *root,
 				 CmdType operation, bool canSetTag,
 				 Index nominalRelation, List *partitioned_rels,
 				 bool partColsUpdated,
-				 List *resultRelations, List *subplans,
+				 List *resultRelations,
+				 Index mergeTargetRelation,
+				 List *subplans,
 				 List *withCheckOptionLists, List *returningLists,
-				 List *rowMarks, OnConflictExpr *onconflict, int epqParam)
+				 List *rowMarks, OnConflictExpr *onconflict,
+				 List *mergeSourceTargetList,
+				 List *mergeActionList, int epqParam)
 {
 	ModifyTable *node = makeNode(ModifyTable);
 	List	   *fdw_private_list;
@@ -6545,6 +6556,7 @@ make_modifytable(PlannerInfo *root,
 	node->partitioned_rels = partitioned_rels;
 	node->partColsUpdated = partColsUpdated;
 	node->resultRelations = resultRelations;
+	node->mergeTargetRelation = mergeTargetRelation;
 	node->resultRelIndex = -1;	/* will be set correctly in setrefs.c */
 	node->rootResultRelIndex = -1;	/* will be set correctly in setrefs.c */
 	node->plans = subplans;
@@ -6577,6 +6589,8 @@ make_modifytable(PlannerInfo *root,
 	node->withCheckOptionLists = withCheckOptionLists;
 	node->returningLists = returningLists;
 	node->rowMarks = rowMarks;
+	node->mergeSourceTargetList = mergeSourceTargetList;
+	node->mergeActionList = mergeActionList;
 	node->epqParam = epqParam;
 
 	/*

src/backend/optimizer/plan/planner.c

@@ -794,6 +794,24 @@ subquery_planner(PlannerGlobal *glob, Query *parse,
 		/* exclRelTlist contains only Vars, so no preprocessing needed */
 	}
 
+	foreach(l, parse->mergeActionList)
+	{
+		MergeAction *action = (MergeAction *) lfirst(l);
+
+		action->targetList = (List *)
+			preprocess_expression(root,
+								  (Node *) action->targetList,
+								  EXPRKIND_TARGET);
+		action->qual =
+			preprocess_expression(root,
+								  (Node *) action->qual,
+								  EXPRKIND_QUAL);
+	}
+
+	parse->mergeSourceTargetList = (List *)
+		preprocess_expression(root, (Node *) parse->mergeSourceTargetList,
+							  EXPRKIND_TARGET);
+
 	root->append_rel_list = (List *)
 		preprocess_expression(root, (Node *) root->append_rel_list,
 							  EXPRKIND_APPINFO);
@@ -1535,6 +1553,7 @@ inheritance_planner(PlannerInfo *root)
 									 subroot->parse->returningList);
 
 		Assert(!parse->onConflict);
+		Assert(parse->mergeActionList == NIL);
 	}
 
 	/* Result path must go into outer query's FINAL upperrel */
@@ -1593,12 +1612,15 @@ inheritance_planner(PlannerInfo *root)
 									 partitioned_rels,
 									 partColsUpdated,
 									 resultRelations,
+									 0,
 									 subpaths,
 									 subroots,
 									 withCheckOptionLists,
 									 returningLists,
 									 rowMarks,
 									 NULL,
+									 NULL,
+									 NULL,
 									 SS_assign_special_param(root)));
 }
 
@@ -2129,8 +2151,8 @@ grouping_planner(PlannerInfo *root, bool inheritance_update,
 		}
 
 		/*
-		 * If this is an INSERT/UPDATE/DELETE, and we're not being called from
-		 * inheritance_planner, add the ModifyTable node.
+		 * If this is an INSERT/UPDATE/DELETE/MERGE, and we're not being
+		 * called from inheritance_planner, add the ModifyTable node.
 		 */
 		if (parse->commandType != CMD_SELECT && !inheritance_update)
 		{
@@ -2170,12 +2192,15 @@ grouping_planner(PlannerInfo *root, bool inheritance_update,
 										NIL,
 										false,
 										list_make1_int(parse->resultRelation),
+										parse->mergeTarget_relation,
 										list_make1(path),
 										list_make1(root),
 										withCheckOptionLists,
 										returningLists,
 										rowMarks,
 										parse->onConflict,
+										parse->mergeSourceTargetList,
+										parse->mergeActionList,
 										SS_assign_special_param(root));
 		}
 

src/backend/optimizer/plan/setrefs.c

@@ -851,6 +851,60 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset)
 						fix_scan_list(root, splan->exclRelTlist, rtoffset);
 				}
 
+				/*
+				 * The MERGE produces the target rows by performing a right
+				 * join between the target relation and the source relation
+				 * (which could be a plain relation or a subquery). The INSERT
+				 * and UPDATE actions of the MERGE requires access to the
+				 * columns from the source relation. We arrange things so that
+				 * the source relation attributes are available as INNER_VAR
+				 * and the target relation attributes are available from the
+				 * scan tuple.
+				 */
+				if (splan->mergeActionList != NIL)
+				{
+					/*
+					 * mergeSourceTargetList is already setup correctly to
+					 * include all Vars coming from the source relation. So we
+					 * fix the targetList of individual action nodes by
+					 * ensuring that the source relation Vars are referenced
+					 * as INNER_VAR. Note that for this to work correctly,
+					 * during execution, the ecxt_innertuple must be set to
+					 * the tuple obtained from the source relation.
+					 *
+					 * We leave the Vars from the result relation (i.e. the
+					 * target relation) unchanged i.e. those Vars would be
+					 * picked from the scan slot. So during execution, we must
+					 * ensure that ecxt_scantuple is setup correctly to refer
+					 * to the tuple from the target relation.
+					 */
+
+					indexed_tlist *itlist;
+
+					itlist = build_tlist_index(splan->mergeSourceTargetList);
+
+					splan->mergeTargetRelation += rtoffset;
+
+					foreach(l, splan->mergeActionList)
+					{
+						MergeAction *action = (MergeAction *) lfirst(l);
+
+						/* Fix targetList of each action. */
+						action->targetList = fix_join_expr(root,
+								action->targetList,
+								NULL, itlist,
+								linitial_int(splan->resultRelations),
+								rtoffset);
+
+						/* Fix quals too. */
+						action->qual = (Node *) fix_join_expr(root,
+								(List *) action->qual,
+								NULL, itlist,
+								linitial_int(splan->resultRelations),
+								rtoffset);
+					}
+				}
+
 				splan->nominalRelation += rtoffset;
 				splan->exclRelRTI += rtoffset;
 

src/backend/optimizer/prep/preptlist.c

@@ -118,6 +118,46 @@ preprocess_targetlist(PlannerInfo *root)
 		tlist = expand_targetlist(tlist, command_type,
 								  result_relation, target_relation);
 
+	if (command_type == CMD_MERGE)
+	{
+		ListCell   *l;
+
+		/*
+		 * For MERGE, add any junk column(s) needed to allow the executor to
+		 * identify the rows to be updated or deleted, with different
+		 * handling for partitioned tables.
+		 */
+		rewriteTargetListMerge(parse, target_relation);
+
+		/*
+		 * For MERGE command, handle targetlist of each MergeAction separately.
+		 * Give the same treatment to MergeAction->targetList as we would have
+		 * given to a regular INSERT/UPDATE/DELETE.
+		 */
+		foreach(l, parse->mergeActionList)
+		{
+			MergeAction *action = (MergeAction *) lfirst(l);
+
+			switch (action->commandType)
+			{
+				case CMD_INSERT:
+				case CMD_UPDATE:
+					action->targetList = expand_targetlist(action->targetList,
+														   action->commandType,
+														   result_relation,
+														   target_relation);
+					break;
+				case CMD_DELETE:
+					break;
+				case CMD_NOTHING:
+					break;
+				default:
+					elog(ERROR, "unknown action in MERGE WHEN clause");
+
+			}
+		}
+	}
+
 	/*
 	 * Add necessary junk columns for rowmarked rels.  These values are needed
 	 * for locking of rels selected FOR UPDATE/SHARE, and to do EvalPlanQual
@@ -348,6 +388,7 @@ expand_targetlist(List *tlist, int command_type,
 													  true /* byval */ );
 					}
 					break;
+				case CMD_MERGE:
 				case CMD_UPDATE:
 					if (!att_tup->attisdropped)
 					{

src/backend/optimizer/util/pathnode.c

@@ -3284,17 +3284,21 @@ create_lockrows_path(PlannerInfo *root, RelOptInfo *rel,
  * 'rowMarks' is a list of PlanRowMarks (non-locking only)
  * 'onconflict' is the ON CONFLICT clause, or NULL
  * 'epqParam' is the ID of Param for EvalPlanQual re-eval
+ * 'mergeActionList' is a list of MERGE actions
  */
 ModifyTablePath *
 create_modifytable_path(PlannerInfo *root, RelOptInfo *rel,
 						CmdType operation, bool canSetTag,
 						Index nominalRelation, List *partitioned_rels,
 						bool partColsUpdated,
-						List *resultRelations, List *subpaths,
+						List *resultRelations,
+						Index mergeTargetRelation,
+						List *subpaths,
 						List *subroots,
 						List *withCheckOptionLists, List *returningLists,
 						List *rowMarks, OnConflictExpr *onconflict,
-						int epqParam)
+						List *mergeSourceTargetList,
+						List *mergeActionList, int epqParam)
 {
 	ModifyTablePath *pathnode = makeNode(ModifyTablePath);
 	double		total_size;
@@ -3359,6 +3363,7 @@ create_modifytable_path(PlannerInfo *root, RelOptInfo *rel,
 	pathnode->partitioned_rels = list_copy(partitioned_rels);
 	pathnode->partColsUpdated = partColsUpdated;
 	pathnode->resultRelations = resultRelations;
+	pathnode->mergeTargetRelation = mergeTargetRelation;
 	pathnode->subpaths = subpaths;
 	pathnode->subroots = subroots;
 	pathnode->withCheckOptionLists = withCheckOptionLists;
@@ -3366,6 +3371,8 @@ create_modifytable_path(PlannerInfo *root, RelOptInfo *rel,
 	pathnode->rowMarks = rowMarks;
 	pathnode->onconflict = onconflict;
 	pathnode->epqParam = epqParam;
+	pathnode->mergeSourceTargetList = mergeSourceTargetList;
+	pathnode->mergeActionList = mergeActionList;
 
 	return pathnode;
 }

src/backend/optimizer/util/plancat.c

@@ -1835,6 +1835,10 @@ has_row_triggers(PlannerInfo *root, Index rti, CmdType event)
 				 trigDesc->trig_delete_before_row))
 				result = true;
 			break;
+			/* There is no separate event for MERGE, only INSERT/UPDATE/DELETE */
+		case CMD_MERGE:
+			result = false;
+			break;
 		default:
 			elog(ERROR, "unrecognized CmdType: %d", (int) event);
 			break;