Support varlena fields with single-byte headers and unaligned storage.

This commit breaks any code that assumes that the mere act of forming a tuple
(without writing it to disk) does not "toast" any fields.  While all available
regression tests pass, I'm not totally sure that we've fixed every nook and
cranny, especially in contrib.

Greg Stark with some help from Tom Lane

src/include/access/heapam.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.121 2007/03/29 00:15:39 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.122 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -200,7 +200,8 @@ extern Size heap_compute_data_size(TupleDesc tupleDesc,
 					   Datum *values, bool *isnull);
 extern void heap_fill_tuple(TupleDesc tupleDesc,
 				Datum *values, bool *isnull,
-				char *data, uint16 *infomask, bits8 *bit);
+				char *data, Size data_size,
+				uint16 *infomask, bits8 *bit);
 extern bool heap_attisnull(HeapTuple tup, int attnum);
 extern Datum nocachegetattr(HeapTuple tup, int attnum,
 			   TupleDesc att, bool *isnull);

src/include/access/htup.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/htup.h,v 1.92 2007/02/27 23:48:09 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/htup.h,v 1.93 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -160,9 +160,8 @@ typedef HeapTupleHeaderData *HeapTupleHeader;
 #define HEAP_HASNULL			0x0001	/* has null attribute(s) */
 #define HEAP_HASVARWIDTH		0x0002	/* has variable-width attribute(s) */
 #define HEAP_HASEXTERNAL		0x0004	/* has external stored attribute(s) */
-#define HEAP_HASCOMPRESSED		0x0008	/* has compressed stored attribute(s) */
-#define HEAP_HASEXTENDED		0x000C	/* the two above combined */
-#define HEAP_HASOID				0x0010	/* has an object-id field */
+#define HEAP_HASOID				0x0008	/* has an object-id field */
+/* bit 0x0010 is available */
 #define HEAP_COMBOCID			0x0020	/* t_cid is a combo cid */
 #define HEAP_XMAX_EXCL_LOCK		0x0040	/* xmax is exclusive locker */
 #define HEAP_XMAX_SHARED_LOCK	0x0080	/* xmax is shared locker */
@@ -341,7 +340,7 @@ do { \
  * MaxAttrSize is a somewhat arbitrary upper limit on the declared size of
  * data fields of char(n) and similar types.  It need not have anything
  * directly to do with the *actual* upper limit of varlena values, which
- * is currently 1Gb (see struct varattrib in postgres.h).  I've set it
+ * is currently 1Gb (see TOAST structures in postgres.h).  I've set it
  * at 10Mb which seems like a reasonable number --- tgl 8/6/00.
  */
 #define MaxAttrSize		(10 * 1024 * 1024)
@@ -485,12 +484,6 @@ typedef HeapTupleData *HeapTuple;
 #define HeapTupleHasExternal(tuple) \
 		(((tuple)->t_data->t_infomask & HEAP_HASEXTERNAL) != 0)
 
-#define HeapTupleHasCompressed(tuple) \
-		(((tuple)->t_data->t_infomask & HEAP_HASCOMPRESSED) != 0)
-
-#define HeapTupleHasExtended(tuple) \
-		(((tuple)->t_data->t_infomask & HEAP_HASEXTENDED) != 0)
-
 #define HeapTupleGetOid(tuple) \
 		HeapTupleHeaderGetOid((tuple)->t_data)
 

src/include/access/tupmacs.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/tupmacs.h,v 1.32 2007/02/27 23:48:09 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/tupmacs.h,v 1.33 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -91,26 +91,83 @@
 #endif   /* SIZEOF_DATUM == 8 */
 
 /*
- * att_align aligns the given offset as needed for a datum of alignment
- * requirement attalign.  The cases are tested in what is hopefully something
- * like their frequency of occurrence.
+ * att_align_datum aligns the given offset as needed for a datum of alignment
+ * requirement attalign and typlen attlen.  attdatum is the Datum variable
+ * we intend to pack into a tuple (it's only accessed if we are dealing with
+ * a varlena type).  Note that this assumes the Datum will be stored as-is;
+ * callers that are intending to convert non-short varlena datums to short
+ * format have to account for that themselves.
  */
-#define att_align(cur_offset, attalign) \
+#define att_align_datum(cur_offset, attalign, attlen, attdatum) \
 ( \
-	((attalign) == 'i') ? INTALIGN(cur_offset) : \
-	 (((attalign) == 'c') ? ((long)(cur_offset)) : \
-	  (((attalign) == 'd') ? DOUBLEALIGN(cur_offset) : \
-		( \
-			AssertMacro((attalign) == 's'), \
-			SHORTALIGN(cur_offset) \
-		))) \
+	((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? (long) (cur_offset) : \
+	att_align_nominal(cur_offset, attalign) \
 )
 
 /*
- * att_addlength increments the given offset by the length of the attribute.
- * attval is only accessed if we are dealing with a variable-length attribute.
+ * att_align_pointer performs the same calculation as att_align_datum,
+ * but is used when walking a tuple.  attptr is the current actual data
+ * pointer; when accessing a varlena field we have to "peek" to see if we
+ * are looking at a pad byte or the first byte of a 1-byte-header datum.
+ * (A zero byte must be either a pad byte, or the first byte of a correctly
+ * aligned 4-byte length word; in either case we can align safely.  A non-zero
+ * byte must be either a 1-byte length word, or the first byte of a correctly
+ * aligned 4-byte length word; in either case we need not align.)
+ *
+ * Note: some callers pass a "char *" pointer for cur_offset.  This is
+ * a bit of a hack but works OK on all known platforms.  It ought to be
+ * cleaned up someday, though.
  */
-#define att_addlength(cur_offset, attlen, attval) \
+#define att_align_pointer(cur_offset, attalign, attlen, attptr) \
+( \
+	((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? (long) (cur_offset) : \
+	att_align_nominal(cur_offset, attalign) \
+)
+
+/*
+ * att_align_nominal aligns the given offset as needed for a datum of alignment
+ * requirement attalign, ignoring any consideration of packed varlena datums.
+ * There are three main use cases for using this macro directly:
+ * 	* we know that the att in question is not varlena (attlen != -1);
+ *	  in this case it is cheaper than the above macros and just as good.
+ *	* we need to estimate alignment padding cost abstractly, ie without
+ *	  reference to a real tuple.  We must assume the worst case that
+ *	  all varlenas are aligned.
+ *	* within arrays, we unconditionally align varlenas (XXX this should be
+ *	  revisited, probably).
+ *
+ * The attalign cases are tested in what is hopefully something like their
+ * frequency of occurrence.
+ */
+#define att_align_nominal(cur_offset, attalign) \
+( \
+	((attalign) == 'i') ? INTALIGN(cur_offset) : \
+	 (((attalign) == 'c') ? (long) (cur_offset) : \
+	  (((attalign) == 'd') ? DOUBLEALIGN(cur_offset) : \
+	   ( \
+			AssertMacro((attalign) == 's'), \
+			SHORTALIGN(cur_offset) \
+	   ))) \
+)
+
+/*
+ * att_addlength_datum increments the given offset by the space needed for
+ * the given Datum variable.  attdatum is only accessed if we are dealing
+ * with a variable-length attribute.
+ */
+#define att_addlength_datum(cur_offset, attlen, attdatum) \
+	att_addlength_pointer(cur_offset, attlen, DatumGetPointer(attdatum))
+
+/*
+ * att_addlength_pointer performs the same calculation as att_addlength_datum,
+ * but is used when walking a tuple --- attptr is the pointer to the field
+ * within the tuple.
+ *
+ * Note: some callers pass a "char *" pointer for cur_offset.  This is
+ * actually perfectly OK, but probably should be cleaned up along with
+ * the same practice for att_align_pointer.
+ */
+#define att_addlength_pointer(cur_offset, attlen, attptr) \
 ( \
 	((attlen) > 0) ? \
 	( \
@@ -118,12 +175,12 @@
 	) \
 	: (((attlen) == -1) ? \
 	( \
-		(cur_offset) + VARSIZE(DatumGetPointer(attval)) \
+		(cur_offset) + VARSIZE_ANY(attptr) \
 	) \
 	: \
 	( \
 		AssertMacro((attlen) == -2), \
-		(cur_offset) + (strlen(DatumGetCString(attval)) + 1) \
+		(cur_offset) + (strlen((char *) (attptr)) + 1) \
 	)) \
 )
 

src/include/access/tuptoaster.h

@@ -6,7 +6,7 @@
  *
  * Copyright (c) 2000-2007, PostgreSQL Global Development Group
  *
- * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.34 2007/04/03 04:14:26 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.35 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -111,7 +111,7 @@ extern void toast_delete(Relation rel, HeapTuple oldtup);
  *		in compressed format.
  * ----------
  */
-extern varattrib *heap_tuple_fetch_attr(varattrib *attr);
+extern struct varlena *heap_tuple_fetch_attr(struct varlena *attr);
 
 /* ----------
  * heap_tuple_untoast_attr() -
@@ -120,7 +120,7 @@ extern varattrib *heap_tuple_fetch_attr(varattrib *attr);
  *		it as needed.
  * ----------
  */
-extern varattrib *heap_tuple_untoast_attr(varattrib *attr);
+extern struct varlena *heap_tuple_untoast_attr(struct varlena *attr);
 
 /* ----------
  * heap_tuple_untoast_attr_slice() -
@@ -129,7 +129,7 @@ extern varattrib *heap_tuple_untoast_attr(varattrib *attr);
  *		(Handles all cases for attribute storage)
  * ----------
  */
-extern varattrib *heap_tuple_untoast_attr_slice(varattrib *attr,
+extern struct varlena *heap_tuple_untoast_attr_slice(struct varlena *attr,
 							  int32 sliceoffset,
 							  int32 slicelength);