Predict integer overflow to avoid buffer overruns.

Several functions, mostly type input functions, calculated an allocation
size such that the calculation wrapped to a small positive value when
arguments implied a sufficiently-large requirement.  Writes past the end
of the inadvertent small allocation followed shortly thereafter.
Coverity identified the path_in() vulnerability; code inspection led to
the rest.  In passing, add check_stack_depth() to prevent stack overflow
in related functions.

Back-patch to 8.4 (all supported versions).  The non-comment hstore
changes touch code that did not exist in 8.4, so that part stops at 9.0.

Noah Misch and Heikki Linnakangas, reviewed by Tom Lane.

Security: CVE-2014-0064

contrib/hstore/hstore.h

@@ -49,9 +49,12 @@ typedef struct
 } HStore;
 
 /*
- * it's not possible to get more than 2^28 items into an hstore,
- * so we reserve the top few bits of the size field. See hstore_compat.c
- * for one reason why.	Some bits are left for future use here.
+ * It's not possible to get more than 2^28 items into an hstore, so we reserve
+ * the top few bits of the size field.  See hstore_compat.c for one reason
+ * why.  Some bits are left for future use here.  MaxAllocSize makes the
+ * practical count limit slightly more than 2^28 / 3, or INT_MAX / 24, the
+ * limit for an hstore full of 4-byte keys and null values.  Therefore, we
+ * don't explicitly check the format-imposed limit.
  */
 #define HS_FLAG_NEWVERSION 0x80000000
 
@@ -59,6 +62,12 @@ typedef struct
 #define HS_SETCOUNT(hsp_,c_) ((hsp_)->size_ = (c_) | HS_FLAG_NEWVERSION)
 
 
+/*
+ * "x" comes from an existing HS_COUNT() (as discussed, <= INT_MAX/24) or a
+ * Pairs array length (due to MaxAllocSize, <= INT_MAX/40).  "lenstr" is no
+ * more than INT_MAX, that extreme case arising in hstore_from_arrays().
+ * Therefore, this calculation is limited to about INT_MAX / 5 + INT_MAX.
+ */
 #define HSHRDSIZE	(sizeof(HStore))
 #define CALCDATASIZE(x, lenstr) ( (x) * 2 * sizeof(HEntry) + HSHRDSIZE + (lenstr) )
 

contrib/hstore/hstore_io.c

@@ -13,6 +13,7 @@
 #include "utils/builtins.h"
 #include "utils/json.h"
 #include "utils/lsyscache.h"
+#include "utils/memutils.h"
 #include "utils/typcache.h"
 
 #include "hstore.h"
@@ -439,6 +440,11 @@ hstore_recv(PG_FUNCTION_ARGS)
 		PG_RETURN_POINTER(out);
 	}
 
+	if (pcount < 0 || pcount > MaxAllocSize / sizeof(Pairs))
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+			  errmsg("number of pairs (%d) exceeds the maximum allowed (%d)",
+					 pcount, (int) (MaxAllocSize / sizeof(Pairs)))));
 	pairs = palloc(pcount * sizeof(Pairs));
 
 	for (i = 0; i < pcount; ++i)
@@ -554,6 +560,13 @@ hstore_from_arrays(PG_FUNCTION_ARGS)
 					  TEXTOID, -1, false, 'i',
 					  &key_datums, &key_nulls, &key_count);
 
+	/* see discussion in hstoreArrayToPairs() */
+	if (key_count > MaxAllocSize / sizeof(Pairs))
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+			  errmsg("number of pairs (%d) exceeds the maximum allowed (%d)",
+					 key_count, (int) (MaxAllocSize / sizeof(Pairs)))));
+
 	/* value_array might be NULL */
 
 	if (PG_ARGISNULL(1))
@@ -676,6 +689,13 @@ hstore_from_array(PG_FUNCTION_ARGS)
 
 	count = in_count / 2;
 
+	/* see discussion in hstoreArrayToPairs() */
+	if (count > MaxAllocSize / sizeof(Pairs))
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+			  errmsg("number of pairs (%d) exceeds the maximum allowed (%d)",
+					 count, (int) (MaxAllocSize / sizeof(Pairs)))));
+
 	pairs = palloc(count * sizeof(Pairs));
 
 	for (i = 0; i < count; ++i)
@@ -807,6 +827,7 @@ hstore_from_record(PG_FUNCTION_ARGS)
 		my_extra->ncolumns = ncolumns;
 	}
 
+	Assert(ncolumns <= MaxTupleAttributeNumber);		/* thus, no overflow */
 	pairs = palloc(ncolumns * sizeof(Pairs));
 
 	if (rec)

contrib/hstore/hstore_op.c

@@ -8,6 +8,7 @@
 #include "catalog/pg_type.h"
 #include "funcapi.h"
 #include "utils/builtins.h"
+#include "utils/memutils.h"
 
 #include "hstore.h"
 
@@ -90,6 +91,19 @@ hstoreArrayToPairs(ArrayType *a, int *npairs)
 		return NULL;
 	}
 
+	/*
+	 * A text array uses at least eight bytes per element, so any overflow in
+	 * "key_count * sizeof(Pairs)" is small enough for palloc() to catch.
+	 * However, credible improvements to the array format could invalidate
+	 * that assumption.  Therefore, use an explicit check rather than relying
+	 * on palloc() to complain.
+	 */
+	if (key_count > MaxAllocSize / sizeof(Pairs))
+		ereport(ERROR,
+				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+			  errmsg("number of pairs (%d) exceeds the maximum allowed (%d)",
+					 key_count, (int) (MaxAllocSize / sizeof(Pairs)))));
+
 	key_pairs = palloc(sizeof(Pairs) * key_count);
 
 	for (i = 0, j = 0; i < key_count; i++)
@@ -648,6 +662,7 @@ hstore_slice_to_hstore(PG_FUNCTION_ARGS)
 		PG_RETURN_POINTER(out);
 	}
 
+	/* hstoreArrayToPairs() checked overflow */
 	out_pairs = palloc(sizeof(Pairs) * nkeys);
 	bufsiz = 0;