MERGE SQL Command following SQL:2016

MERGE performs actions that modify rows in the target table
using a source table or query. MERGE provides a single SQL
statement that can conditionally INSERT/UPDATE/DELETE rows
a task that would other require multiple PL statements.
e.g.

MERGE INTO target AS t
USING source AS s
ON t.tid = s.sid
WHEN MATCHED AND t.balance > s.delta THEN
  UPDATE SET balance = t.balance - s.delta
WHEN MATCHED THEN
  DELETE
WHEN NOT MATCHED AND s.delta > 0 THEN
  INSERT VALUES (s.sid, s.delta)
WHEN NOT MATCHED THEN
  DO NOTHING;

MERGE works with regular and partitioned tables, including
column and row security enforcement, as well as support for
row, statement and transition triggers.

MERGE is optimized for OLTP and is parameterizable, though
also useful for large scale ETL/ELT. MERGE is not intended
to be used in preference to existing single SQL commands
for INSERT, UPDATE or DELETE since there is some overhead.
MERGE can be used statically from PL/pgSQL.

MERGE does not yet support inheritance, write rules,
RETURNING clauses, updatable views or foreign tables.
MERGE follows SQL Standard per the most recent SQL:2016.

Includes full tests and documentation, including full
isolation tests to demonstrate the concurrent behavior.

This version written from scratch in 2017 by Simon Riggs,
using docs and tests originally written in 2009. Later work
from Pavan Deolasee has been both complex and deep, leaving
the lead author credit now in his hands.
Extensive discussion of concurrency from Peter Geoghegan,
with thanks for the time and effort contributed.

Various issues reported via sqlsmith by Andreas Seltenreich

Authors: Pavan Deolasee, Simon Riggs
Reviewer: Peter Geoghegan, Amit Langote, Tomas Vondra, Simon Riggs

Discussion:
https://postgr.es/m/CANP8+jKitBSrB7oTgT9CY2i1ObfOt36z0XMraQc+Xrz8QB0nXA@mail.gmail.com
https://postgr.es/m/CAH2-WzkJdBuxj9PO=2QaO9-3h3xGbQPZ34kJH=HukRekwM-GZg@mail.gmail.com

src/backend/parser/Makefile

@@ -14,7 +14,7 @@ override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
 
 OBJS= analyze.o gram.o scan.o parser.o \
       parse_agg.o parse_clause.o parse_coerce.o parse_collate.o parse_cte.o \
-      parse_enr.o parse_expr.o parse_func.o parse_node.o parse_oper.o \
+      parse_enr.o parse_expr.o parse_func.o parse_merge.o parse_node.o parse_oper.o \
       parse_param.o parse_relation.o parse_target.o parse_type.o \
       parse_utilcmd.o scansup.o
 

src/backend/parser/analyze.c

@@ -38,6 +38,7 @@
 #include "parser/parse_cte.h"
 #include "parser/parse_expr.h"
 #include "parser/parse_func.h"
+#include "parser/parse_merge.h"
 #include "parser/parse_oper.h"
 #include "parser/parse_param.h"
 #include "parser/parse_relation.h"
@@ -53,9 +54,6 @@ post_parse_analyze_hook_type post_parse_analyze_hook = NULL;
 static Query *transformOptionalSelectInto(ParseState *pstate, Node *parseTree);
 static Query *transformDeleteStmt(ParseState *pstate, DeleteStmt *stmt);
 static Query *transformInsertStmt(ParseState *pstate, InsertStmt *stmt);
-static List *transformInsertRow(ParseState *pstate, List *exprlist,
-				   List *stmtcols, List *icolumns, List *attrnos,
-				   bool strip_indirection);
 static OnConflictExpr *transformOnConflictClause(ParseState *pstate,
 						  OnConflictClause *onConflictClause);
 static int	count_rowexpr_columns(ParseState *pstate, Node *expr);
@@ -68,8 +66,6 @@ static void determineRecursiveColTypes(ParseState *pstate,
 						   Node *larg, List *nrtargetlist);
 static Query *transformUpdateStmt(ParseState *pstate, UpdateStmt *stmt);
 static List *transformReturningList(ParseState *pstate, List *returningList);
-static List *transformUpdateTargetList(ParseState *pstate,
-						  List *targetList);
 static Query *transformDeclareCursorStmt(ParseState *pstate,
 						   DeclareCursorStmt *stmt);
 static Query *transformExplainStmt(ParseState *pstate,
@@ -267,6 +263,7 @@ transformStmt(ParseState *pstate, Node *parseTree)
 		case T_InsertStmt:
 		case T_UpdateStmt:
 		case T_DeleteStmt:
+		case T_MergeStmt:
 			(void) test_raw_expression_coverage(parseTree, NULL);
 			break;
 		default:
@@ -291,6 +288,10 @@ transformStmt(ParseState *pstate, Node *parseTree)
 			result = transformUpdateStmt(pstate, (UpdateStmt *) parseTree);
 			break;
 
+		case T_MergeStmt:
+			result = transformMergeStmt(pstate, (MergeStmt *) parseTree);
+			break;
+
 		case T_SelectStmt:
 			{
 				SelectStmt *n = (SelectStmt *) parseTree;
@@ -366,6 +367,7 @@ analyze_requires_snapshot(RawStmt *parseTree)
 		case T_InsertStmt:
 		case T_DeleteStmt:
 		case T_UpdateStmt:
+		case T_MergeStmt:
 		case T_SelectStmt:
 			result = true;
 			break;
@@ -896,7 +898,7 @@ transformInsertStmt(ParseState *pstate, InsertStmt *stmt)
  * attrnos: integer column numbers (must be same length as icolumns)
  * strip_indirection: if true, remove any field/array assignment nodes
  */
-static List *
+List *
 transformInsertRow(ParseState *pstate, List *exprlist,
 				   List *stmtcols, List *icolumns, List *attrnos,
 				   bool strip_indirection)
@@ -2260,9 +2262,9 @@ transformUpdateStmt(ParseState *pstate, UpdateStmt *stmt)
 
 /*
  * transformUpdateTargetList -
- *	handle SET clause in UPDATE/INSERT ... ON CONFLICT UPDATE
+ *	handle SET clause in UPDATE/MERGE/INSERT ... ON CONFLICT UPDATE
  */
-static List *
+List *
 transformUpdateTargetList(ParseState *pstate, List *origTlist)
 {
 	List	   *tlist = NIL;

src/backend/parser/gram.y

@@ -282,6 +282,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 		CreateMatViewStmt RefreshMatViewStmt CreateAmStmt
 		CreatePublicationStmt AlterPublicationStmt
 		CreateSubscriptionStmt AlterSubscriptionStmt DropSubscriptionStmt
+		MergeStmt
 
 %type <node>	select_no_parens select_with_parens select_clause
 				simple_select values_clause
@@ -584,6 +585,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 %type <list>		hash_partbound partbound_datum_list range_datum_list
 %type <defelt>		hash_partbound_elem
 
+%type <node>	merge_when_clause opt_and_condition
+%type <list>	merge_when_list
+%type <node>	merge_update merge_delete merge_insert
+
 /*
  * Non-keyword token types.  These are hard-wired into the "flex" lexer.
  * They must be listed first so that their numeric codes do not depend on
@@ -651,7 +656,8 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 	LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL
 	LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED
 
-	MAPPING MATCH MATERIALIZED MAXVALUE METHOD MINUTE_P MINVALUE MODE MONTH_P MOVE
+	MAPPING MATCH MATCHED MATERIALIZED MAXVALUE MERGE METHOD
+	MINUTE_P MINVALUE MODE MONTH_P MOVE
 
 	NAME_P NAMES NATIONAL NATURAL NCHAR NEW NEXT NO NONE
 	NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF
@@ -920,6 +926,7 @@ stmt :
 			| RefreshMatViewStmt
 			| LoadStmt
 			| LockStmt
+			| MergeStmt
 			| NotifyStmt
 			| PrepareStmt
 			| ReassignOwnedStmt
@@ -10660,6 +10667,7 @@ ExplainableStmt:
 			| InsertStmt
 			| UpdateStmt
 			| DeleteStmt
+			| MergeStmt
 			| DeclareCursorStmt
 			| CreateAsStmt
 			| CreateMatViewStmt
@@ -10722,6 +10730,7 @@ PreparableStmt:
 			| InsertStmt
 			| UpdateStmt
 			| DeleteStmt					/* by default all are $$=$1 */
+			| MergeStmt
 		;
 
 /*****************************************************************************
@@ -11088,6 +11097,151 @@ set_target_list:
 		;
 
 
+/*****************************************************************************
+ *
+ *		QUERY:
+ *				MERGE STATEMENTS
+ *
+ *****************************************************************************/
+
+MergeStmt:
+			MERGE INTO relation_expr_opt_alias
+			USING table_ref
+			ON a_expr
+			merge_when_list
+				{
+					MergeStmt *m = makeNode(MergeStmt);
+
+					m->relation = $3;
+					m->source_relation = $5;
+					m->join_condition = $7;
+					m->mergeActionList = $8;
+
+					$$ = (Node *)m;
+				}
+			;
+
+
+merge_when_list:
+			merge_when_clause						{ $$ = list_make1($1); }
+			| merge_when_list merge_when_clause		{ $$ = lappend($1,$2); }
+			;
+
+merge_when_clause:
+			WHEN MATCHED opt_and_condition THEN merge_update
+				{
+					MergeAction *m = makeNode(MergeAction);
+
+					m->matched = true;
+					m->commandType = CMD_UPDATE;
+					m->condition = $3;
+					m->stmt = $5;
+
+					$$ = (Node *)m;
+				}
+			| WHEN MATCHED opt_and_condition THEN merge_delete
+				{
+					MergeAction *m = makeNode(MergeAction);
+
+					m->matched = true;
+					m->commandType = CMD_DELETE;
+					m->condition = $3;
+					m->stmt = $5;
+
+					$$ = (Node *)m;
+				}
+			| WHEN NOT MATCHED opt_and_condition THEN merge_insert
+				{
+					MergeAction *m = makeNode(MergeAction);
+
+					m->matched = false;
+					m->commandType = CMD_INSERT;
+					m->condition = $4;
+					m->stmt = $6;
+
+					$$ = (Node *)m;
+				}
+			| WHEN NOT MATCHED opt_and_condition THEN DO NOTHING
+				{
+					MergeAction *m = makeNode(MergeAction);
+
+					m->matched = false;
+					m->commandType = CMD_NOTHING;
+					m->condition = $4;
+					m->stmt = NULL;
+
+					$$ = (Node *)m;
+				}
+			;
+
+opt_and_condition:
+			AND a_expr 				{ $$ = $2; }
+			| 			 			{ $$ = NULL; }
+			;
+
+merge_delete:
+			DELETE_P
+				{
+					DeleteStmt *n = makeNode(DeleteStmt);
+					$$ = (Node *)n;
+				}
+			;
+
+merge_update:
+			UPDATE SET set_clause_list
+				{
+					UpdateStmt *n = makeNode(UpdateStmt);
+					n->targetList = $3;
+
+					$$ = (Node *)n;
+				}
+			;
+
+merge_insert:
+			INSERT values_clause
+				{
+					InsertStmt *n = makeNode(InsertStmt);
+					n->cols = NIL;
+					n->selectStmt = $2;
+
+					$$ = (Node *)n;
+				}
+			| INSERT OVERRIDING override_kind VALUE_P values_clause
+				{
+					InsertStmt *n = makeNode(InsertStmt);
+					n->cols = NIL;
+					n->override = $3;
+					n->selectStmt = $5;
+
+					$$ = (Node *)n;
+				}
+			| INSERT '(' insert_column_list ')' values_clause
+				{
+					InsertStmt *n = makeNode(InsertStmt);
+					n->cols = $3;
+					n->selectStmt = $5;
+
+					$$ = (Node *)n;
+				}
+			| INSERT '(' insert_column_list ')' OVERRIDING override_kind VALUE_P values_clause
+				{
+					InsertStmt *n = makeNode(InsertStmt);
+					n->cols = $3;
+					n->override = $6;
+					n->selectStmt = $8;
+
+					$$ = (Node *)n;
+				}
+			| INSERT DEFAULT VALUES
+				{
+					InsertStmt *n = makeNode(InsertStmt);
+					n->cols = NIL;
+					n->selectStmt = NULL;
+
+					$$ = (Node *)n;
+				}
+			;
+
 /*****************************************************************************
  *
  *		QUERY:
@@ -15088,8 +15242,10 @@ unreserved_keyword:
 			| LOGGED
 			| MAPPING
 			| MATCH
+			| MATCHED
 			| MATERIALIZED
 			| MAXVALUE
+			| MERGE
 			| METHOD
 			| MINUTE_P
 			| MINVALUE

src/backend/parser/parse_agg.c

@@ -455,6 +455,13 @@ check_agglevels_and_constraints(ParseState *pstate, Node *expr)
 		case EXPR_KIND_VALUES_SINGLE:
 			errkind = true;
 			break;
+		case EXPR_KIND_MERGE_WHEN_AND:
+			if (isAgg)
+				err = _("aggregate functions are not allowed in WHEN AND conditions");
+			else
+				err = _("grouping operations are not allowed in WHEN AND conditions");
+
+			break;
 		case EXPR_KIND_CHECK_CONSTRAINT:
 		case EXPR_KIND_DOMAIN_CHECK:
 			if (isAgg)
@@ -873,6 +880,9 @@ transformWindowFuncCall(ParseState *pstate, WindowFunc *wfunc,
 		case EXPR_KIND_VALUES_SINGLE:
 			errkind = true;
 			break;
+		case EXPR_KIND_MERGE_WHEN_AND:
+			err = _("window functions are not allowed in WHEN AND conditions");
+			break;
 		case EXPR_KIND_CHECK_CONSTRAINT:
 		case EXPR_KIND_DOMAIN_CHECK:
 			err = _("window functions are not allowed in check constraints");

src/backend/parser/parse_clause.c

@@ -76,9 +76,6 @@ static RangeTblEntry *transformRangeTableFunc(ParseState *pstate,
 						RangeTableFunc *t);
 static TableSampleClause *transformRangeTableSample(ParseState *pstate,
 						  RangeTableSample *rts);
-static Node *transformFromClauseItem(ParseState *pstate, Node *n,
-						RangeTblEntry **top_rte, int *top_rti,
-						List **namespace);
 static Node *buildMergedJoinVar(ParseState *pstate, JoinType jointype,
 				   Var *l_colvar, Var *r_colvar);
 static ParseNamespaceItem *makeNamespaceItem(RangeTblEntry *rte,
@@ -139,6 +136,7 @@ transformFromClause(ParseState *pstate, List *frmList)
 		n = transformFromClauseItem(pstate, n,
 									&rte,
 									&rtindex,
+									NULL, NULL,
 									&namespace);
 
 		checkNameSpaceConflicts(pstate, pstate->p_namespace, namespace);
@@ -1096,13 +1094,20 @@ getRTEForSpecialRelationTypes(ParseState *pstate, RangeVar *rv)
  *
  * *top_rti: receives the rangetable index of top_rte.  (Ditto.)
  *
+ * *right_rte: receives the RTE corresponding to the right side of the
+ * jointree. Only MERGE really needs to know about this and only MERGE passes a
+ * non-NULL pointer.
+ *
+ * *right_rti: receives the rangetable index of the right_rte.
+ *
  * *namespace: receives a List of ParseNamespaceItems for the RTEs exposed
  * as table/column names by this item.  (The lateral_only flags in these items
  * are indeterminate and should be explicitly set by the caller before use.)
  */
-static Node *
+Node *
 transformFromClauseItem(ParseState *pstate, Node *n,
 						RangeTblEntry **top_rte, int *top_rti,
+						RangeTblEntry **right_rte, int *right_rti,
 						List **namespace)
 {
 	if (IsA(n, RangeVar))
@@ -1194,7 +1199,7 @@ transformFromClauseItem(ParseState *pstate, Node *n,
 
 		/* Recursively transform the contained relation */
 		rel = transformFromClauseItem(pstate, rts->relation,
-									  top_rte, top_rti, namespace);
+									  top_rte, top_rti, NULL, NULL, namespace);
 		/* Currently, grammar could only return a RangeVar as contained rel */
 		rtr = castNode(RangeTblRef, rel);
 		rte = rt_fetch(rtr->rtindex, pstate->p_rtable);
@@ -1222,6 +1227,7 @@ transformFromClauseItem(ParseState *pstate, Node *n,
 		List	   *l_namespace,
 				   *r_namespace,
 				   *my_namespace,
+				   *save_namespace,
 				   *l_colnames,
 				   *r_colnames,
 				   *res_colnames,
@@ -1240,6 +1246,7 @@ transformFromClauseItem(ParseState *pstate, Node *n,
 		j->larg = transformFromClauseItem(pstate, j->larg,
 										  &l_rte,
 										  &l_rtindex,
+										  NULL, NULL,
 										  &l_namespace);
 
 		/*
@@ -1263,12 +1270,34 @@ transformFromClauseItem(ParseState *pstate, Node *n,
 		sv_namespace_length = list_length(pstate->p_namespace);
 		pstate->p_namespace = list_concat(pstate->p_namespace, l_namespace);
 
+		/*
+		 * If we are running MERGE, don't make the other RTEs visible while
+		 * parsing the source relation. It mustn't see them.
+		 *
+		 * Currently, only MERGE passes non-NULL value for right_rte, so we
+		 * can safely deduce if we're running MERGE or not by just looking at
+		 * the right_rte. If that ever changes, we should look at other means
+		 * to find that.
+		 */
+		if (right_rte)
+		{
+			save_namespace = pstate->p_namespace;
+			pstate->p_namespace = NIL;
+		}
+
 		/* And now we can process the RHS */
 		j->rarg = transformFromClauseItem(pstate, j->rarg,
 										  &r_rte,
 										  &r_rtindex,
+										  NULL, NULL,
 										  &r_namespace);
 
+		/*
+		 * And now restore the namespace again so that join-quals can see it.
+		 */
+		if (right_rte)
+			pstate->p_namespace = save_namespace;
+
 		/* Remove the left-side RTEs from the namespace list again */
 		pstate->p_namespace = list_truncate(pstate->p_namespace,
 											sv_namespace_length);
@@ -1295,6 +1324,12 @@ transformFromClauseItem(ParseState *pstate, Node *n,
 		expandRTE(r_rte, r_rtindex, 0, -1, false,
 				  &r_colnames, &r_colvars);
 
+		if (right_rte)
+			*right_rte = r_rte;
+
+		if (right_rti)
+			*right_rti = r_rtindex;
+
 		/*
 		 * Natural join does not explicitly specify columns; must generate
 		 * columns to join. Need to run through the list of columns from each

src/backend/parser/parse_collate.c

@@ -485,6 +485,7 @@ assign_collations_walker(Node *node, assign_collations_context *context)
 		case T_FromExpr:
 		case T_OnConflictExpr:
 		case T_SortGroupClause:
+		case T_MergeAction:
 			(void) expression_tree_walker(node,
 										  assign_collations_walker,
 										  (void *) &loccontext);

src/backend/parser/parse_expr.c

@@ -1818,6 +1818,7 @@ transformSubLink(ParseState *pstate, SubLink *sublink)
 		case EXPR_KIND_RETURNING:
 		case EXPR_KIND_VALUES:
 		case EXPR_KIND_VALUES_SINGLE:
+		case EXPR_KIND_MERGE_WHEN_AND:
 			/* okay */
 			break;
 		case EXPR_KIND_CHECK_CONSTRAINT:
@@ -3475,6 +3476,8 @@ ParseExprKindName(ParseExprKind exprKind)
 			return "PARTITION BY";
 		case EXPR_KIND_CALL_ARGUMENT:
 			return "CALL";
+		case EXPR_KIND_MERGE_WHEN_AND:
+			return "MERGE WHEN AND";
 
 			/*
 			 * There is intentionally no default: case here, so that the

src/backend/parser/parse_func.c

@@ -2277,6 +2277,9 @@ check_srf_call_placement(ParseState *pstate, Node *last_srf, int location)
 			/* okay, since we process this like a SELECT tlist */
 			pstate->p_hasTargetSRFs = true;
 			break;
+		case EXPR_KIND_MERGE_WHEN_AND:
+			err = _("set-returning functions are not allowed in WHEN AND conditions");
+			break;
 		case EXPR_KIND_CHECK_CONSTRAINT:
 		case EXPR_KIND_DOMAIN_CHECK:
 			err = _("set-returning functions are not allowed in check constraints");

src/backend/parser/parse_relation.c

@@ -728,6 +728,16 @@ scanRTEForColumn(ParseState *pstate, RangeTblEntry *rte, const char *colname,
 							colname),
 					 parser_errposition(pstate, location)));
 
+		/* In MERGE WHEN AND condition, no system column is allowed except tableOid or OID */
+		if (pstate->p_expr_kind == EXPR_KIND_MERGE_WHEN_AND &&
+			attnum < InvalidAttrNumber &&
+			!(attnum == TableOidAttributeNumber || attnum == ObjectIdAttributeNumber))
+			ereport(ERROR,
+					(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
+					 errmsg("system column \"%s\" reference in WHEN AND condition is invalid",
+							colname),
+					 parser_errposition(pstate, location)));
+
 		if (attnum != InvalidAttrNumber)
 		{
 			/* now check to see if column actually is defined */