Improve handling of domains over arrays.

This patch eliminates various bizarre behaviors caused by sloppy thinking
about the difference between a domain type and its underlying array type.
In particular, the operation of updating one element of such an array
has to be considered as yielding a value of the underlying array type,
*not* a value of the domain, because there's no assurance that the
domain's CHECK constraints are still satisfied.  If we're intending to
store the result back into a domain column, we have to re-cast to the
domain type so that constraints are re-checked.

For similar reasons, such a domain can't be blindly matched to an ANYARRAY
polymorphic parameter, because the polymorphic function is likely to apply
array-ish operations that could invalidate the domain constraints.  For the
moment, we just forbid such matching.  We might later wish to insert an
automatic downcast to the underlying array type, but such a change should
also change matching of domains to ANYELEMENT for consistency.

To ensure that all such logic is rechecked, this patch removes the original
hack of setting a domain's pg_type.typelem field to match its base type;
the typelem will always be zero instead.  In those places where it's really
okay to look through the domain type with no other logic changes, use the
newly added get_base_element_type function in place of get_element_type.
catversion bumped due to change in pg_type contents.

Per bug #5717 from Richard Huxton and subsequent discussion.

src/backend/parser/parse_coerce.c

@@ -1908,6 +1908,9 @@ find_coercion_pathway(Oid targetTypeId, Oid sourceTypeId,
 		 * array types.  If so, and if the element types have a suitable cast,
 		 * report that we can coerce with an ArrayCoerceExpr.
 		 *
+		 * Note that the source type can be a domain over array, but not the
+		 * target, because ArrayCoerceExpr won't check domain constraints.
+		 *
 		 * Hack: disallow coercions to oidvector and int2vector, which
 		 * otherwise tend to capture coercions that should go to "real" array
 		 * types.  We want those types to be considered "real" arrays for many
@@ -1921,7 +1924,7 @@ find_coercion_pathway(Oid targetTypeId, Oid sourceTypeId,
 			Oid			sourceElem;
 
 			if ((targetElem = get_element_type(targetTypeId)) != InvalidOid &&
-				(sourceElem = get_element_type(sourceTypeId)) != InvalidOid)
+				(sourceElem = get_base_element_type(sourceTypeId)) != InvalidOid)
 			{
 				CoercionPathType elempathtype;
 				Oid			elemfuncid;
@@ -2001,10 +2004,8 @@ find_typmod_coercion_function(Oid typeId,
 	targetType = typeidType(typeId);
 	typeForm = (Form_pg_type) GETSTRUCT(targetType);
 
-	/* Check for a varlena array type (and not a domain) */
-	if (typeForm->typelem != InvalidOid &&
-		typeForm->typlen == -1 &&
-		typeForm->typtype != TYPTYPE_DOMAIN)
+	/* Check for a varlena array type */
+	if (typeForm->typelem != InvalidOid && typeForm->typlen == -1)
 	{
 		/* Yes, switch our attention to the element type */
 		typeId = typeForm->typelem;

src/backend/parser/parse_expr.c

@@ -161,19 +161,17 @@ transformExpr(ParseState *pstate, Node *expr)
 
 					targetType = typenameTypeId(pstate, tc->typeName,
 												&targetTypmod);
+					/*
+					 * If target is a domain over array, work with the base
+					 * array type here.  transformTypeCast below will cast the
+					 * array type to the domain.  In the usual case that the
+					 * target is not a domain, transformTypeCast is a no-op.
+					 */
+					targetType = getBaseTypeAndTypmod(targetType,
+													  &targetTypmod);
 					elementType = get_element_type(targetType);
 					if (OidIsValid(elementType))
 					{
-						/*
-						 * tranformArrayExpr doesn't know how to check domain
-						 * constraints, so ask it to return the base type
-						 * instead. transformTypeCast below will cast it to
-						 * the domain. In the usual case that the target is
-						 * not a domain, transformTypeCast is a no-op.
-						 */
-						targetType = getBaseTypeAndTypmod(targetType,
-														  &targetTypmod);
-
 						tc = copyObject(tc);
 						tc->arg = transformArrayExpr(pstate,
 													 (A_ArrayExpr *) tc->arg,

src/backend/parser/parse_node.c

@@ -25,6 +25,7 @@
 #include "parser/parse_relation.h"
 #include "utils/builtins.h"
 #include "utils/int8.h"
+#include "utils/lsyscache.h"
 #include "utils/syscache.h"
 #include "utils/varbit.h"
 
@@ -198,19 +199,35 @@ make_var(ParseState *pstate, RangeTblEntry *rte, int attrno, int location)
 
 /*
  * transformArrayType()
- *		Get the element type of an array type in preparation for subscripting
+ *		Identify the types involved in a subscripting operation
+ *
+ * On entry, arrayType/arrayTypmod identify the type of the input value
+ * to be subscripted (which could be a domain type).  These are modified
+ * if necessary to identify the actual array type and typmod, and the
+ * array's element type is returned.  An error is thrown if the input isn't
+ * an array type.
  */
 Oid
-transformArrayType(Oid arrayType)
+transformArrayType(Oid *arrayType, int32 *arrayTypmod)
 {
+	Oid			origArrayType = *arrayType;
 	Oid			elementType;
 	HeapTuple	type_tuple_array;
 	Form_pg_type type_struct_array;
 
+	/*
+	 * If the input is a domain, smash to base type, and extract the actual
+	 * typmod to be applied to the base type.  Subscripting a domain is an
+	 * operation that necessarily works on the base array type, not the domain
+	 * itself.  (Note that we provide no method whereby the creator of a
+	 * domain over an array type could hide its ability to be subscripted.)
+	 */
+	*arrayType = getBaseTypeAndTypmod(*arrayType, arrayTypmod);
+
 	/* Get the type tuple for the array */
-	type_tuple_array = SearchSysCache1(TYPEOID, ObjectIdGetDatum(arrayType));
+	type_tuple_array = SearchSysCache1(TYPEOID, ObjectIdGetDatum(*arrayType));
 	if (!HeapTupleIsValid(type_tuple_array))
-		elog(ERROR, "cache lookup failed for type %u", arrayType);
+		elog(ERROR, "cache lookup failed for type %u", *arrayType);
 	type_struct_array = (Form_pg_type) GETSTRUCT(type_tuple_array);
 
 	/* needn't check typisdefined since this will fail anyway */
@@ -220,7 +237,7 @@ transformArrayType(Oid arrayType)
 		ereport(ERROR,
 				(errcode(ERRCODE_DATATYPE_MISMATCH),
 				 errmsg("cannot subscript type %s because it is not an array",
-						format_type_be(arrayType))));
+						format_type_be(origArrayType))));
 
 	ReleaseSysCache(type_tuple_array);
 
@@ -241,13 +258,17 @@ transformArrayType(Oid arrayType)
  * that array.	We produce an expression that represents the new array value
  * with the source data inserted into the right part of the array.
  *
+ * For both cases, if the source array is of a domain-over-array type,
+ * the result is of the base array type or its element type; essentially,
+ * we must fold a domain to its base type before applying subscripting.
+ *
  * pstate		Parse state
  * arrayBase	Already-transformed expression for the array as a whole
- * arrayType	OID of array's datatype (should match type of arrayBase)
+ * arrayType	OID of array's datatype (should match type of arrayBase,
+ *				or be the base type of arrayBase's domain type)
  * elementType	OID of array's element type (fetch with transformArrayType,
  *				or pass InvalidOid to do it here)
- * elementTypMod typmod to be applied to array elements (if storing) or of
- *				the source array (if fetching)
+ * arrayTypMod	typmod for the array (which is also typmod for the elements)
  * indirection	Untransformed list of subscripts (must not be NIL)
  * assignFrom	NULL for array fetch, else transformed expression for source.
  */
@@ -256,7 +277,7 @@ transformArraySubscripts(ParseState *pstate,
 						 Node *arrayBase,
 						 Oid arrayType,
 						 Oid elementType,
-						 int32 elementTypMod,
+						 int32 arrayTypMod,
 						 List *indirection,
 						 Node *assignFrom)
 {
@@ -266,9 +287,13 @@ transformArraySubscripts(ParseState *pstate,
 	ListCell   *idx;
 	ArrayRef   *aref;
 
-	/* Caller may or may not have bothered to determine elementType */
+	/*
+	 * Caller may or may not have bothered to determine elementType.  Note
+	 * that if the caller did do so, arrayType/arrayTypMod must be as
+	 * modified by transformArrayType, ie, smash domain to base type.
+	 */
 	if (!OidIsValid(elementType))
-		elementType = transformArrayType(arrayType);
+		elementType = transformArrayType(&arrayType, &arrayTypMod);
 
 	/*
 	 * A list containing only single subscripts refers to a single array
@@ -356,7 +381,7 @@ transformArraySubscripts(ParseState *pstate,
 
 		newFrom = coerce_to_target_type(pstate,
 										assignFrom, typesource,
-										typeneeded, elementTypMod,
+										typeneeded, arrayTypMod,
 										COERCION_ASSIGNMENT,
 										COERCE_IMPLICIT_CAST,
 										-1);
@@ -378,7 +403,7 @@ transformArraySubscripts(ParseState *pstate,
 	aref = makeNode(ArrayRef);
 	aref->refarraytype = arrayType;
 	aref->refelemtype = elementType;
-	aref->reftypmod = elementTypMod;
+	aref->reftypmod = arrayTypMod;
 	aref->refupperindexpr = upperIndexpr;
 	aref->reflowerindexpr = lowerIndexpr;
 	aref->refexpr = (Expr *) arrayBase;

src/backend/parser/parse_oper.c

@@ -209,7 +209,7 @@ get_sort_group_operators(Oid argtype,
 		eq_opr == ARRAY_EQ_OP ||
 		gt_opr == ARRAY_GT_OP)
 	{
-		Oid			elem_type = get_element_type(argtype);
+		Oid			elem_type = get_base_element_type(argtype);
 
 		if (OidIsValid(elem_type))
 		{
@@ -906,7 +906,7 @@ make_scalar_array_op(ParseState *pstate, List *opname,
 		rtypeId = UNKNOWNOID;
 	else
 	{
-		rtypeId = get_element_type(atypeId);
+		rtypeId = get_base_element_type(atypeId);
 		if (!OidIsValid(rtypeId))
 			ereport(ERROR,
 					(errcode(ERRCODE_WRONG_OBJECT_TYPE),

src/backend/parser/parse_target.c

@@ -43,6 +43,16 @@ static Node *transformAssignmentIndirection(ParseState *pstate,
 							   ListCell *indirection,
 							   Node *rhs,
 							   int location);
+static Node *transformAssignmentSubscripts(ParseState *pstate,
+							  Node *basenode,
+							  const char *targetName,
+							  Oid targetTypeId,
+							  int32 targetTypMod,
+							  List *subscripts,
+							  bool isSlice,
+							  ListCell *next_indirection,
+							  Node *rhs,
+							  int location);
 static List *ExpandColumnRefStar(ParseState *pstate, ColumnRef *cref,
 					bool targetlist);
 static List *ExpandAllTables(ParseState *pstate, int location);
@@ -613,27 +623,17 @@ transformAssignmentIndirection(ParseState *pstate,
 			/* process subscripts before this field selection */
 			if (subscripts)
 			{
-				Oid			elementTypeId = transformArrayType(targetTypeId);
-				Oid			typeNeeded = isSlice ? targetTypeId : elementTypeId;
-
-				/* recurse to create appropriate RHS for array assign */
-				rhs = transformAssignmentIndirection(pstate,
-													 NULL,
+				/* recurse, and then return because we're done */
+				return transformAssignmentSubscripts(pstate,
+													 basenode,
 													 targetName,
-													 true,
-													 typeNeeded,
+													 targetTypeId,
 													 targetTypMod,
+													 subscripts,
+													 isSlice,
 													 i,
 													 rhs,
 													 location);
-				/* process subscripts */
-				return (Node *) transformArraySubscripts(pstate,
-														 basenode,
-														 targetTypeId,
-														 elementTypeId,
-														 targetTypMod,
-														 subscripts,
-														 rhs);
 			}
 
 			/* No subscripts, so can process field selection here */
@@ -690,27 +690,17 @@ transformAssignmentIndirection(ParseState *pstate,
 	/* process trailing subscripts, if any */
 	if (subscripts)
 	{
-		Oid			elementTypeId = transformArrayType(targetTypeId);
-		Oid			typeNeeded = isSlice ? targetTypeId : elementTypeId;
-
-		/* recurse to create appropriate RHS for array assign */
-		rhs = transformAssignmentIndirection(pstate,
-											 NULL,
+		/* recurse, and then return because we're done */
+		return transformAssignmentSubscripts(pstate,
+											 basenode,
 											 targetName,
-											 true,
-											 typeNeeded,
+											 targetTypeId,
 											 targetTypMod,
+											 subscripts,
+											 isSlice,
 											 NULL,
 											 rhs,
 											 location);
-		/* process subscripts */
-		return (Node *) transformArraySubscripts(pstate,
-												 basenode,
-												 targetTypeId,
-												 elementTypeId,
-												 targetTypMod,
-												 subscripts,
-												 rhs);
 	}
 
 	/* base case: just coerce RHS to match target type ID */
@@ -748,6 +738,79 @@ transformAssignmentIndirection(ParseState *pstate,
 	return result;
 }
 
+/*
+ * helper for transformAssignmentIndirection: process array assignment
+ */
+static Node *
+transformAssignmentSubscripts(ParseState *pstate,
+							  Node *basenode,
+							  const char *targetName,
+							  Oid targetTypeId,
+							  int32 targetTypMod,
+							  List *subscripts,
+							  bool isSlice,
+							  ListCell *next_indirection,
+							  Node *rhs,
+							  int location)
+{
+	Node	   *result;
+	Oid			arrayType;
+	int32		arrayTypMod;
+	Oid			elementTypeId;
+	Oid			typeNeeded;
+
+	Assert(subscripts != NIL);
+
+	/* Identify the actual array type and element type involved */
+	arrayType = targetTypeId;
+	arrayTypMod = targetTypMod;
+	elementTypeId = transformArrayType(&arrayType, &arrayTypMod);
+
+	/* Identify type that RHS must provide */
+	typeNeeded = isSlice ? arrayType : elementTypeId;
+
+	/* recurse to create appropriate RHS for array assign */
+	rhs = transformAssignmentIndirection(pstate,
+										 NULL,
+										 targetName,
+										 true,
+										 typeNeeded,
+										 arrayTypMod,
+										 next_indirection,
+										 rhs,
+										 location);
+
+	/* process subscripts */
+	result = (Node *) transformArraySubscripts(pstate,
+											   basenode,
+											   arrayType,
+											   elementTypeId,
+											   arrayTypMod,
+											   subscripts,
+											   rhs);
+
+	/* If target was a domain over array, need to coerce up to the domain */
+	if (arrayType != targetTypeId)
+	{
+		result = coerce_to_target_type(pstate,
+									   result, exprType(result),
+									   targetTypeId, targetTypMod,
+									   COERCION_ASSIGNMENT,
+									   COERCE_IMPLICIT_CAST,
+									   -1);
+		/* probably shouldn't fail, but check */
+		if (result == NULL)
+			ereport(ERROR,
+					(errcode(ERRCODE_CANNOT_COERCE),
+					 errmsg("cannot cast type %s to %s",
+							format_type_be(exprType(result)),
+							format_type_be(targetTypeId)),
+					 parser_errposition(pstate, location)));
+	}
+
+	return result;
+}
+
 
 /*
  * checkInsertTargets -