database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-12 21:01:52 +03:00
Code Activity

Add an extra header byte to TOAST-pointer datums to represent their size

Browse Source 
explicitly.  This means a TOAST pointer takes 18 bytes instead of 17 --- still
smaller than in 8.2 --- which seems a good tradeoff to ensure we won't have
painted ourselves into a corner if we want to support multiple types of TOAST
pointer later on.  Per discussion with Greg Stark.
This commit is contained in:
Tom Lane
2007-09-30 19:54:58 +00:00
parent 3ff0018c2e
commit 27b8922221
4 changed files with 58 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
doc/src/sgml/storage.sgml
View File

@ -1,4 +1,4 @@
<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.19 2007/09/21 21:25:42 tgl Exp $ --> <!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.20 2007/09/30 19:54:57 tgl Exp $ -->
<chapter id="storage"> <chapter id="storage">
@ -233,8 +233,8 @@ header, and the remaining bits give the total datum size (including length
byte) in bytes. As a special case, if the remaining bits are all zero byte) in bytes. As a special case, if the remaining bits are all zero
(which would be impossible for a self-inclusive length), the value is a (which would be impossible for a self-inclusive length), the value is a
pointer to out-of-line data stored in a separate TOAST table. (The size of pointer to out-of-line data stored in a separate TOAST table. (The size of
a TOAST pointer is known a priori, so it doesn't need to be represented in a TOAST pointer is given in the second byte of the datum.)
the header.) Values with single-byte headers aren't aligned on any particular Values with single-byte headers aren't aligned on any particular
boundary, either. Lastly, when the highest-order or lowest-order bit is boundary, either. Lastly, when the highest-order or lowest-order bit is
clear but the adjacent bit is set, the content of the datum has been clear but the adjacent bit is set, the content of the datum has been
compressed and must be decompressed before use. In this case the remaining compressed and must be decompressed before use. In this case the remaining
@ -274,8 +274,8 @@ retrieval of the values. A pointer datum representing an out-of-line
<acronym>TOAST</> table in which to look and the OID of the specific value <acronym>TOAST</> table in which to look and the OID of the specific value
(its <structfield>chunk_id</>). For convenience, pointer datums also store the (its <structfield>chunk_id</>). For convenience, pointer datums also store the
logical datum size (original uncompressed data length) and actual stored size logical datum size (original uncompressed data length) and actual stored size
(different if compression was applied). Allowing for the varlena header byte, (different if compression was applied). Allowing for the varlena header bytes,
the total size of a <acronym>TOAST</> pointer datum is therefore 17 bytes the total size of a <acronym>TOAST</> pointer datum is therefore 18 bytes
regardless of the actual size of the represented value. regardless of the actual size of the represented value.
</para> </para>

45
src/backend/access/heap/tuptoaster.c
View File

@ -8,7 +8,7 @@
* *
* *
* IDENTIFICATION * IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.75 2007/09/26 23:29:10 tgl Exp $ * $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.76 2007/09/30 19:54:58 tgl Exp $
* *
* *
* INTERFACE ROUTINES * INTERFACE ROUTINES
@ -52,6 +52,21 @@
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \ #define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ) ((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ)
/*
* Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
* into a local "struct varatt_external" toast pointer. This should be
* just a memcpy, but some versions of gcc seem to produce broken code
* that assumes the datum contents are aligned. Introducing an explicit
* intermediate "varattrib_1b_e *" variable seems to fix it.
*/
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \
do { \
varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
Assert(VARSIZE_ANY_EXHDR(attre) == sizeof(toast_pointer)); \
memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
} while (0)
static void toast_delete_datum(Relation rel, Datum value); static void toast_delete_datum(Relation rel, Datum value);
static Datum toast_save_datum(Relation rel, Datum value, static Datum toast_save_datum(Relation rel, Datum value,
bool use_wal, bool use_fsm); bool use_wal, bool use_fsm);
@ -172,7 +187,7 @@ heap_tuple_untoast_attr_slice(struct varlena *attr,
{ {
struct varatt_external toast_pointer; struct varatt_external toast_pointer;
memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer)); VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
/* fast path for non-compressed external datums */ /* fast path for non-compressed external datums */
if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
@ -249,7 +264,7 @@ toast_raw_datum_size(Datum value)
/* va_rawsize is the size of the original datum -- including header */ /* va_rawsize is the size of the original datum -- including header */
struct varatt_external toast_pointer; struct varatt_external toast_pointer;
memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer)); VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
result = toast_pointer.va_rawsize; result = toast_pointer.va_rawsize;
} }
else if (VARATT_IS_COMPRESSED(attr)) else if (VARATT_IS_COMPRESSED(attr))
@ -294,7 +309,7 @@ toast_datum_size(Datum value)
*/ */
struct varatt_external toast_pointer; struct varatt_external toast_pointer;
memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer)); VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
result = toast_pointer.va_extsize; result = toast_pointer.va_extsize;
} }
else if (VARATT_IS_SHORT(attr)) else if (VARATT_IS_SHORT(attr))
@ -470,9 +485,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
VARATT_IS_EXTERNAL(old_value)) VARATT_IS_EXTERNAL(old_value))
{ {
if (toast_isnull[i] || !VARATT_IS_EXTERNAL(new_value) || if (toast_isnull[i] || !VARATT_IS_EXTERNAL(new_value) ||
memcmp(VARDATA_SHORT(old_value), memcmp((char *) old_value, (char *) new_value,
VARDATA_SHORT(new_value), VARSIZE_EXTERNAL(old_value)) != 0)
sizeof(struct varatt_external)) != 0)
{ {
/* /*
* The old external stored value isn't needed any more * The old external stored value isn't needed any more
@ -1071,7 +1085,7 @@ toast_save_datum(Relation rel, Datum value,
Datum t_values[3]; Datum t_values[3];
bool t_isnull[3]; bool t_isnull[3];
CommandId mycid = GetCurrentCommandId(); CommandId mycid = GetCurrentCommandId();
struct varlena *result; varattrib_pointer *result;
struct varatt_external toast_pointer; struct varatt_external toast_pointer;
struct struct
{ {
@ -1192,9 +1206,9 @@ toast_save_datum(Relation rel, Datum value,
/* /*
* Create the TOAST pointer value that we'll return * Create the TOAST pointer value that we'll return
*/ */
result = (struct varlena *) palloc(sizeof(varattrib_pointer)); result = (varattrib_pointer *) palloc(sizeof(varattrib_pointer));
SET_VARSIZE_EXTERNAL(result); SET_VARSIZE_EXTERNAL(result, sizeof(varattrib_pointer));
memcpy(VARDATA_SHORT(result), &toast_pointer, sizeof(toast_pointer)); memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
return PointerGetDatum(result); return PointerGetDatum(result);
} }
@ -1221,8 +1235,7 @@ toast_delete_datum(Relation rel, Datum value)
return; return;
/* Must copy to access aligned fields */ /* Must copy to access aligned fields */
memcpy(&toast_pointer, VARDATA_SHORT(attr), VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
sizeof(struct varatt_external));
/* /*
* Open the toast relation and its index * Open the toast relation and its index
@ -1289,8 +1302,7 @@ toast_fetch_datum(struct varlena *attr)
int32 chunksize; int32 chunksize;
/* Must copy to access aligned fields */ /* Must copy to access aligned fields */
memcpy(&toast_pointer, VARDATA_SHORT(attr), VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
sizeof(struct varatt_external));
ressize = toast_pointer.va_extsize; ressize = toast_pointer.va_extsize;
numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
@ -1452,8 +1464,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length)
Assert(VARATT_IS_EXTERNAL(attr)); Assert(VARATT_IS_EXTERNAL(attr));
/* Must copy to access aligned fields */ /* Must copy to access aligned fields */
memcpy(&toast_pointer, VARDATA_SHORT(attr), VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
sizeof(struct varatt_external));
/* /*
* It's nonsense to fetch slices of a compressed datum -- this isn't lo_* * It's nonsense to fetch slices of a compressed datum -- this isn't lo_*

4
src/include/catalog/catversion.h
View File

@ -37,7 +37,7 @@
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) 1994, Regents of the University of California
* *
* $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.429 2007/09/25 22:21:55 tgl Exp $ * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.430 2007/09/30 19:54:58 tgl Exp $
* *
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
*/ */
@ -53,6 +53,6 @@
*/ */
/* yyyymmddN */ /* yyyymmddN */
#define CATALOG_VERSION_NO 200709251 #define CATALOG_VERSION_NO 200709301
#endif #endif

36
src/include/postgres.h
View File

@ -10,7 +10,7 @@
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1995, Regents of the University of California * Portions Copyright (c) 1995, Regents of the University of California
* *
* $PostgreSQL: pgsql/src/include/postgres.h,v 1.83 2007/09/27 21:01:59 tgl Exp $ * $PostgreSQL: pgsql/src/include/postgres.h,v 1.84 2007/09/30 19:54:58 tgl Exp $
* *
*------------------------------------------------------------------------- *-------------------------------------------------------------------------
*/ */
@ -100,12 +100,20 @@ typedef union
typedef struct typedef struct
{ {
uint8 va_header; uint8 va_header;
char va_data[1]; /* Data or TOAST pointer */ char va_data[1]; /* Data begins here */
} varattrib_1b; } varattrib_1b;
typedef struct typedef struct
{ {
uint8 va_header; uint8 va_header; /* Always 0x80 or 0x01 */
uint8 va_len_1be; /* Physical length of datum */
char va_data[1]; /* Data (for now always a TOAST pointer) */
} varattrib_1b_e;
typedef struct
{
uint8 va_header; /* Always 0x80 or 0x01 */
uint8 va_len_1be; /* Physical length of datum */
char va_data[sizeof(struct varatt_external)]; char va_data[sizeof(struct varatt_external)];
} varattrib_pointer; } varattrib_pointer;
@ -161,9 +169,8 @@ typedef struct
(((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF) (((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \ #define VARSIZE_1B(PTR) \
(((varattrib_1b *) (PTR))->va_header & 0x7F) (((varattrib_1b *) (PTR))->va_header & 0x7F)
/* Currently there is only one size of toast pointer, but someday maybe not */
#define VARSIZE_1B_E(PTR) \ #define VARSIZE_1B_E(PTR) \
(sizeof(varattrib_pointer)) (((varattrib_1b_e *) (PTR))->va_len_1be)
#define SET_VARSIZE_4B(PTR,len) \ #define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF) (((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF)
@ -171,8 +178,9 @@ typedef struct
(((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000) (((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000)
#define SET_VARSIZE_1B(PTR,len) \ #define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (len) | 0x80) (((varattrib_1b *) (PTR))->va_header = (len) | 0x80)
#define SET_VARSIZE_1B_E(PTR) \ #define SET_VARSIZE_1B_E(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = 0x80) (((varattrib_1b_e *) (PTR))->va_header = 0x80, \
((varattrib_1b_e *) (PTR))->va_len_1be = (len))
#else /* !WORDS_BIGENDIAN */ #else /* !WORDS_BIGENDIAN */
@ -194,9 +202,8 @@ typedef struct
((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF) ((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \ #define VARSIZE_1B(PTR) \
((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F) ((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F)
/* Currently there is only one size of toast pointer, but someday maybe not */
#define VARSIZE_1B_E(PTR) \ #define VARSIZE_1B_E(PTR) \
(sizeof(varattrib_pointer)) (((varattrib_1b_e *) (PTR))->va_len_1be)
#define SET_VARSIZE_4B(PTR,len) \ #define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2)) (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2))
@ -204,8 +211,9 @@ typedef struct
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02) (((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02)
#define SET_VARSIZE_1B(PTR,len) \ #define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01) (((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01)
#define SET_VARSIZE_1B_E(PTR) \ #define SET_VARSIZE_1B_E(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = 0x01) (((varattrib_1b_e *) (PTR))->va_header = 0x01, \
((varattrib_1b_e *) (PTR))->va_len_1be = (len))
#endif /* WORDS_BIGENDIAN */ #endif /* WORDS_BIGENDIAN */
@ -220,6 +228,7 @@ typedef struct
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data) #define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data) #define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data) #define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
#define VARRAWSIZE_4B_C(PTR) \ #define VARRAWSIZE_4B_C(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_rawsize) (((varattrib_4b *) (PTR))->va_compressed.va_rawsize)
@ -249,6 +258,7 @@ typedef struct
#define VARDATA_SHORT(PTR) VARDATA_1B(PTR) #define VARDATA_SHORT(PTR) VARDATA_1B(PTR)
#define VARSIZE_EXTERNAL(PTR) VARSIZE_1B_E(PTR) #define VARSIZE_EXTERNAL(PTR) VARSIZE_1B_E(PTR)
#define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR)
#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR) #define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR)
#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR) #define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR)
@ -258,7 +268,7 @@ typedef struct
#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len) #define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len)
#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len) #define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len)
#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len) #define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len)
#define SET_VARSIZE_EXTERNAL(PTR) SET_VARSIZE_1B_E(PTR) #define SET_VARSIZE_EXTERNAL(PTR, len) SET_VARSIZE_1B_E(PTR, len)
#define VARSIZE_ANY(PTR) \ #define VARSIZE_ANY(PTR) \
(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR) : \ (VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR) : \
@ -266,7 +276,7 @@ typedef struct
VARSIZE_4B(PTR))) VARSIZE_4B(PTR)))
#define VARSIZE_ANY_EXHDR(PTR) \ #define VARSIZE_ANY_EXHDR(PTR) \
(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-1 : \ (VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-2 : \
(VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-1 : \ (VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-1 : \
VARSIZE_4B(PTR)-4)) VARSIZE_4B(PTR)-4))