diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c
index 56167ecfe1d..8c57adc5d54 100644
--- a/contrib/amcheck/verify_heapam.c
+++ b/contrib/amcheck/verify_heapam.c
@@ -74,7 +74,9 @@ typedef enum SkipPages
*/
typedef struct ToastedAttribute
{
- varatt_external_oid toast_pointer;
+ Oid8 va_valueid; /* value ID (works for both Oid and Oid8) */
+ uint32 va_extinfo; /* external size and compression method */
+ vartag_external tag; /* VARTAG_ONDISK_OID or VARTAG_ONDISK_OID8 */
BlockNumber blkno; /* block in main table */
OffsetNumber offnum; /* offset in main table */
AttrNumber attnum; /* attribute in main table */
@@ -189,7 +191,7 @@ static void check_tuple(HeapCheckContext *ctx,
XidCommitStatus *xmin_commit_status);
static void check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
ToastedAttribute *ta, int32 *expected_chunk_seq,
- uint32 extsize);
+ uint32 extsize, int32 max_chunk_size);
static bool check_tuple_attribute(HeapCheckContext *ctx);
static void check_toasted_attribute(HeapCheckContext *ctx,
@@ -1554,7 +1556,7 @@ check_tuple_visibility(HeapCheckContext *ctx, bool *xmin_commit_status_ok,
static void
check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
ToastedAttribute *ta, int32 *expected_chunk_seq,
- uint32 extsize)
+ uint32 extsize, int32 max_chunk_size)
{
int32 chunk_seq;
int32 last_chunk_seq;
@@ -1562,12 +1564,9 @@ check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
bool isnull;
int32 chunksize;
int32 expected_size;
- int32 max_chunk_size;
- Oid8 toast_valueid;
+ Oid8 toast_valueid = ta->va_valueid;
- max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
last_chunk_seq = (extsize - 1) / max_chunk_size;
- toast_valueid = ta->toast_pointer.va_valueid;
/* Sanity-check the sequence number. */
chunk_seq = DatumGetInt32(fastgetattr(toasttup, 2,
@@ -1671,8 +1670,11 @@ check_tuple_attribute(HeapCheckContext *ctx)
char *tp; /* pointer to the tuple data */
uint16 infomask;
Oid8 toast_pointer_valueid;
+ int32 va_rawsize;
+ uint32 va_extinfo;
CompactAttribute *thisatt;
- varatt_external_oid toast_pointer;
+ vartag_external va_tag_value;
+ toast_external_data toast_ext_data;
infomask = ctx->tuphdr->t_infomask;
thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum);
@@ -1731,7 +1733,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
{
uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
- if (va_tag != VARTAG_ONDISK_OID)
+ if (va_tag != VARTAG_ONDISK_OID && va_tag != VARTAG_ONDISK_OID8)
{
report_corruption(ctx,
psprintf("toasted attribute has unexpected TOAST tag %u",
@@ -1775,27 +1777,28 @@ check_tuple_attribute(HeapCheckContext *ctx)
/* It is external, and we're looking at a page on disk */
- /*
- * Must copy attr into toast_pointer for alignment considerations
- */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- toast_pointer_valueid = toast_pointer.va_valueid;
+ /* Must copy attr into a decoded pointer for alignment considerations */
+ toast_external_info_get(attr, &toast_ext_data);
+ va_tag_value = toast_ext_data.tag;
+ toast_pointer_valueid = toast_ext_data.valueid;
+ va_rawsize = toast_ext_data.rawsize;
+ va_extinfo = toast_ext_data.extinfo;
/* Toasted attributes too large to be untoasted should never be stored */
- if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT)
+ if (va_rawsize > VARLENA_SIZE_LIMIT)
report_corruption(ctx,
psprintf("toast value " OID8_FORMAT " rawsize %d exceeds limit %d",
toast_pointer_valueid,
- toast_pointer.va_rawsize,
+ va_rawsize,
VARLENA_SIZE_LIMIT));
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize))
{
ToastCompressionId cmid;
bool valid = false;
/* Compressed attributes should have a valid compression method */
- cmid = TOAST_COMPRESS_METHOD(&toast_pointer);
+ cmid = VARATT_EXTINFO_GET_COMPRESS_METHOD(toast_ext_data.extinfo);
switch (cmid)
{
/* List of all valid compression method IDs */
@@ -1849,7 +1852,10 @@ check_tuple_attribute(HeapCheckContext *ctx)
ta = palloc0_object(ToastedAttribute);
- VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr);
+ /* The pointer has already been decoded above, just reuse it */
+ ta->tag = va_tag_value;
+ ta->va_valueid = toast_pointer_valueid;
+ ta->va_extinfo = va_extinfo;
ta->blkno = ctx->blkno;
ta->offnum = ctx->offnum;
ta->attnum = ctx->attnum;
@@ -1878,18 +1884,44 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
int32 max_chunk_size;
Oid8 toast_valueid;
Oid toast_typid;
+ vartag_external expected_tag = VARTAG_ONDISK_OID;
+ toast_valueid = ta->va_valueid;
+ extsize = VARATT_EXTINFO_GET_EXTSIZE(ta->va_extinfo);
+
+ /*
+ * Take the chunk_id type from the TOAST table's own definition, not from
+ * from the vartag in the main table as that pointer is the very thing
+ * under scrutiny here. The two must agree.
+ */
toast_typid = TupleDescAttr(ctx->toast_rel->rd_att, 0)->atttypid;
- max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
+ if (toast_typid == OID8OID)
+ expected_tag = VARTAG_ONDISK_OID8;
+ else if (toast_typid == OIDOID)
+ expected_tag = VARTAG_ONDISK_OID;
+ else
+ {
+ report_toast_corruption(ctx, ta,
+ psprintf("toast value " OID8_FORMAT " stored in toast table whose chunk_id has unexpected type %u",
+ toast_valueid, toast_typid));
+ }
- extsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(ta->toast_pointer);
+ if (ta->tag != expected_tag)
+ {
+ report_toast_corruption(ctx, ta,
+ psprintf("toast value " OID8_FORMAT " has TOAST tag %u, but chunk_id of toast table has type %u",
+ toast_valueid, (uint8) ta->tag,
+ toast_typid));
+ return;
+ }
+
+ max_chunk_size = TOAST_MAX_CHUNK_SIZE(toast_typid);
last_chunk_seq = (extsize - 1) / max_chunk_size;
/*
- * Setup a scan key to find chunks in toast table with matching va_valueid
+ * Setup a scan key to find chunks in toast table with matching value ID
*/
- toast_valueid_scankey_init(&toastkey, toast_typid,
- ta->toast_pointer.va_valueid);
+ toast_valueid_scankey_init(&toastkey, toast_typid, toast_valueid);
/*
* Check if any chunks for this toasted object exist in the toast table,
@@ -1905,12 +1937,11 @@ check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
ForwardScanDirection)) != NULL)
{
found_toasttup = true;
- check_toast_tuple(toasttup, ctx, ta, &expected_chunk_seq, extsize);
+ check_toast_tuple(toasttup, ctx, ta, &expected_chunk_seq, extsize,
+ max_chunk_size);
}
systable_endscan_ordered(toastscan);
- toast_valueid = ta->toast_pointer.va_valueid;
-
if (!found_toasttup)
report_toast_corruption(ctx, ta,
psprintf("toast value " OID8_FORMAT " not found in toast table",
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 21bd67c11ea..83de016eaa5 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -417,8 +417,10 @@ described in more detail below.
Out-of-line values are divided (after compression if used) into chunks of at
-most TOAST_OID_MAX_CHUNK_SIZE bytes (by default this value is chosen
-so that four chunk rows will fit on a page, making it about 2000 bytes).
+most TOAST_OID_MAX_CHUNK_SIZE or
+TOAST_OID8_MAX_CHUNK_SIZE bytes depending on the
+chunk_id type (by default these values are chosen
+so that four chunk rows will fit on a page, making them about 2000 bytes).
Each chunk is stored as a separate row in the TOAST table
belonging to the owning table. Every
TOAST table has the columns
@@ -434,8 +436,10 @@ retrieval of the values. A pointer datum representing an out-of-line on-disk
logical datum size (original uncompressed data length), physical stored size
(different if compression was applied), and the compression method used, if
any. Allowing for the varlena header bytes,
-the total size of an on-disk TOAST pointer datum is therefore 18
-bytes regardless of the actual size of the represented value.
+the total size of an on-disk TOAST pointer datum is 18
+bytes when using an OID as chunk_id, or 22 bytes
+when using an OID8 as chunk_id, regardless of the
+actual size of the represented value.
diff --git a/src/backend/access/common/detoast.c b/src/backend/access/common/detoast.c
index a50aea7b6b1..06992de1ede 100644
--- a/src/backend/access/common/detoast.c
+++ b/src/backend/access/common/detoast.c
@@ -225,12 +225,18 @@ detoast_attr_slice(varlena *attr,
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
+ int32 extsize;
+ uint32 compress_method;
+ bool is_compressed;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get(attr, &toast_ext_data);
+ extsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo);
+ compress_method = VARATT_EXTINFO_GET_COMPRESS_METHOD(toast_ext_data.extinfo);
+ is_compressed = VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize);
/* fast path for non-compressed external datums */
- if (!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (!is_compressed)
return toast_fetch_datum_slice(attr, sliceoffset, slicelength);
/*
@@ -240,7 +246,7 @@ detoast_attr_slice(varlena *attr,
*/
if (slicelimit >= 0)
{
- int32 max_size = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ int32 max_size = extsize;
/*
* Determine maximum amount of compressed data needed for a prefix
@@ -251,8 +257,7 @@ detoast_attr_slice(varlena *attr,
* determine how much compressed data we need to be sure of being
* able to decompress the required slice.
*/
- if (VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer) ==
- TOAST_PGLZ_COMPRESSION_ID)
+ if (compress_method == TOAST_PGLZ_COMPRESSION_ID)
max_size = pglz_maximum_compressed_size(slicelimit, max_size);
/*
@@ -344,20 +349,22 @@ toast_fetch_datum(varlena *attr)
{
Relation toastrel;
varlena *result;
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
int32 attrsize;
+ Oid toastrelid;
+ Oid8 valueid;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums");
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
-
- attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ toast_external_info_get(attr, &toast_ext_data);
+ attrsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo);
+ toastrelid = toast_ext_data.toastrelid;
+ valueid = toast_ext_data.valueid;
result = (varlena *) palloc(attrsize + VARHDRSZ);
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize))
SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ);
else
SET_VARSIZE(result, attrsize + VARHDRSZ);
@@ -369,10 +376,10 @@ toast_fetch_datum(varlena *attr)
/*
* Open the toast relation and its indexes
*/
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel, valueid,
attrsize, 0, attrsize, result);
/* Close toast table */
@@ -398,23 +405,27 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
{
Relation toastrel;
varlena *result;
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
int32 attrsize;
+ Oid toastrelid;
+ Oid8 valueid;
+ bool is_compressed;
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
elog(ERROR, "toast_fetch_datum_slice shouldn't be called for non-ondisk datums");
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get(attr, &toast_ext_data);
+ attrsize = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo);
+ toastrelid = toast_ext_data.toastrelid;
+ valueid = toast_ext_data.valueid;
+ is_compressed = VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize);
/*
* It's nonsense to fetch slices of a compressed datum unless when it's a
* prefix -- this isn't lo_* we can't return a compressed datum which is
* meaningful to toast later.
*/
- Assert(!VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) || 0 == sliceoffset);
-
- attrsize = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ Assert(!is_compressed || 0 == sliceoffset);
if (sliceoffset >= attrsize)
{
@@ -427,7 +438,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
* space required by va_tcinfo, which is stored at the beginning as an
* int32 value.
*/
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer) && slicelength > 0)
+ if (is_compressed && slicelength > 0)
slicelength = slicelength + sizeof(int32);
/*
@@ -440,7 +451,7 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
result = (varlena *) palloc(slicelength + VARHDRSZ);
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (is_compressed)
SET_VARSIZE_COMPRESSED(result, slicelength + VARHDRSZ);
else
SET_VARSIZE(result, slicelength + VARHDRSZ);
@@ -449,10 +460,10 @@ toast_fetch_datum_slice(varlena *attr, int32 sliceoffset,
return result; /* Can save a lot of work at this point! */
/* Open the toast relation */
- toastrel = table_open(toast_pointer.va_toastrelid, AccessShareLock);
+ toastrel = table_open(toastrelid, AccessShareLock);
/* Fetch all chunks */
- table_relation_fetch_toast_slice(toastrel, toast_pointer.va_valueid,
+ table_relation_fetch_toast_slice(toastrel, valueid,
attrsize, sliceoffset, slicelength,
result);
@@ -550,10 +561,10 @@ toast_raw_datum_size(Datum value)
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
/* va_rawsize is the size of the original datum -- including header */
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = toast_pointer.va_rawsize;
+ toast_external_info_get(attr, &toast_ext_data);
+ result = toast_ext_data.rawsize;
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
@@ -610,10 +621,10 @@ toast_datum_size(Datum value)
* compressed or not. We do not count the size of the toast pointer
* ... should we?
*/
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
- result = VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer);
+ toast_external_info_get(attr, &toast_ext_data);
+ result = VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo);
}
else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
{
diff --git a/src/backend/access/common/toast_compression.c b/src/backend/access/common/toast_compression.c
index 3b1bd651926..09cdf880e4f 100644
--- a/src/backend/access/common/toast_compression.c
+++ b/src/backend/access/common/toast_compression.c
@@ -262,12 +262,12 @@ toast_get_compression_id(varlena *attr)
*/
if (VARATT_IS_EXTERNAL_ONDISK(attr))
{
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get(attr, &toast_ext_data);
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
- cmid = VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(toast_pointer);
+ if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize))
+ cmid = VARATT_EXTINFO_GET_COMPRESS_METHOD(toast_ext_data.extinfo);
}
else if (VARATT_IS_COMPRESSED(attr))
cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c
index 2b7c6433bb4..a4e8a096254 100644
--- a/src/backend/access/common/toast_internals.c
+++ b/src/backend/access/common/toast_internals.c
@@ -25,7 +25,6 @@
#include "utils/fmgroids.h"
#include "utils/rel.h"
#include "utils/snapmgr.h"
-#include "utils/lsyscache.h"
static bool toastrel_valueid_exists(Relation toastrel, Oid8 valueid);
static bool toastid_valueid_exists(Oid toastrelid, Oid8 valueid);
@@ -104,6 +103,56 @@ toast_compress_datum(Datum value, char cmethod)
}
}
+/* ----------
+ * toast_preserve_valueid -
+ *
+ * During a table rewrite that preserves rd_toastoid, we want to preserve
+ * toast value IDs too. If the datum previously had an external value from
+ * that same toast table, return its value ID so the caller can re-use it,
+ * otherwise return InvalidOid8.
+ *
+ * This works for both Oid and Oid8 value IDs, as the value ID is decoded
+ * independently of the pointer's vartag.
+ *
+ * There is a corner case here: the table rewrite might have to copy both
+ * live and recently-dead versions of a row, and those versions could easily
+ * reference the same toast value. When we copy the second or later version
+ * of such a row, preserving the value ID means we select one that's already
+ * in the new toast table. We detect that and set *data_todo to 0 so the
+ * caller falls through without writing the data again.
+ *
+ * While annoying and ugly-looking, this is a good thing because it ensures
+ * that we wind up with only one copy of the toast value when there is only
+ * one copy in the old toast table. Before we detected this case, we'd have
+ * made multiple copies, wasting space; and what's worse, the copies
+ * belonging to already-deleted heap tuples would not be reclaimed by VACUUM.
+ * ----------
+ */
+static Oid8
+toast_preserve_valueid(Relation toastrel, Oid toastoid,
+ varlena *oldexternal, int32 *data_todo)
+{
+ toast_external_data old;
+
+ if (!OidIsValid(toastoid) || oldexternal == NULL)
+ return InvalidOid8;
+
+ Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
+ toast_external_info_get(oldexternal, &old);
+
+ /*
+ * Only re-use the value ID if the old pointer came from this same toast
+ * table.
+ */
+ if (old.toastrelid != toastoid)
+ return InvalidOid8;
+
+ if (toastrel_valueid_exists(toastrel, old.valueid))
+ *data_todo = 0;
+
+ return old.valueid;
+}
+
/* ----------
* toast_save_datum -
*
@@ -125,14 +174,20 @@ toast_save_datum(Relation rel, Datum value,
TupleDesc toasttupDesc;
CommandId mycid = GetCurrentCommandId(true);
varlena *result;
- varatt_external_oid toast_pointer;
int32 chunk_seq = 0;
char *data_p;
int32 data_todo;
Pointer dval = DatumGetPointer(value);
int num_indexes;
int validIndex;
- Oid toast_typid = get_atttype(rel->rd_rel->reltoastrelid, 1);
+ Oid toast_typid = RelationGetToastChunkIdType(rel);
+ int32 max_chunk_size;
+
+ /* Fields that will be assembled into the TOAST pointer at the end */
+ int32 va_rawsize;
+ uint32 va_extinfo;
+ Oid8 va_valueid;
+ Oid va_toastrelid;
Assert(!VARATT_IS_EXTERNAL(dval));
@@ -164,28 +219,32 @@ toast_save_datum(Relation rel, Datum value,
{
data_p = VARDATA_SHORT(dval);
data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT;
- toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */
- toast_pointer.va_extinfo = data_todo;
+ va_rawsize = data_todo + VARHDRSZ; /* as if not short */
+ va_extinfo = data_todo;
}
else if (VARATT_IS_COMPRESSED(dval))
{
+ uint32 cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval);
+
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
/* rawsize in a compressed datum is just the size of the payload */
- toast_pointer.va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
+ va_rawsize = VARDATA_COMPRESSED_GET_EXTSIZE(dval) + VARHDRSZ;
/* set external size and compression method */
- VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, data_todo,
- VARDATA_COMPRESSED_GET_COMPRESS_METHOD(dval));
+ Assert(cmid == TOAST_PGLZ_COMPRESSION_ID ||
+ cmid == TOAST_LZ4_COMPRESSION_ID);
+ va_extinfo = data_todo | (cmid << VARLENA_EXTSIZE_BITS);
+
/* Assert that the numbers look like it's compressed */
- Assert(VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer));
+ Assert(VARATT_EXTINFO_IS_COMPRESSED(va_extinfo, va_rawsize));
}
else
{
data_p = VARDATA(dval);
data_todo = VARSIZE(dval) - VARHDRSZ;
- toast_pointer.va_rawsize = VARSIZE(dval);
- toast_pointer.va_extinfo = data_todo;
+ va_rawsize = VARSIZE(dval);
+ va_extinfo = data_todo;
}
/*
@@ -197,100 +256,44 @@ toast_save_datum(Relation rel, Datum value,
* if we have to substitute such an OID.
*/
if (OidIsValid(rel->rd_toastoid))
- toast_pointer.va_toastrelid = rel->rd_toastoid;
+ va_toastrelid = rel->rd_toastoid;
else
- toast_pointer.va_toastrelid = RelationGetRelid(toastrel);
+ va_toastrelid = RelationGetRelid(toastrel);
/*
- * Choose a new value to use as the value ID for this toast value, be it
- * for OID or OID8 TOAST relations.
- *
- * Normally we just choose an unused value within the toast table. But
- * during table-rewriting operations where we are preserving an existing
- * toast table OID, we want to preserve toast value IDs too. So, if
- * rd_toastoid is set and we had a prior external value from that same
- * toast table, re-use its value ID. If we didn't have a prior external
- * value (which is a corner case, but possible if the table's attstorage
- * options have been changed), we have to pick a value ID that doesn't
- * conflict with either new or existing toast value IDs. If the TOAST
- * table uses 8-byte value IDs, we should not really care much about that.
+ * Choose the value ID for this toast value. During a table rewrite that
+ * preserves rd_toastoid we re-use the prior value ID if we can (see
+ * toast_preserve_valueid); otherwise we pick a fresh one. For Oid, it
+ * must not conflict with old or new toast values, while for Oid8 the ID
+ * space is large enough that conflicts are not a concern.
*/
- if (!OidIsValid(rel->rd_toastoid))
+ va_valueid = toast_preserve_valueid(toastrel, rel->rd_toastoid,
+ oldexternal, &data_todo);
+ if (va_valueid == InvalidOid8)
{
- /* normal case: just choose an unused OID */
if (toast_typid == OID8OID)
- toast_pointer.va_valueid = GetNewObjectId8();
+ {
+ /* The Oid8 space is large enough that we need not check conflicts */
+ va_valueid = GetNewObjectId8();
+ }
else
- {
- toast_pointer.va_valueid =
- GetNewOidWithIndex(toastrel,
- RelationGetRelid(toastidxs[validIndex]),
- (AttrNumber) 1);
- }
- }
- else
- {
- /* rewrite case: check to see if value was in old toast table */
- toast_pointer.va_valueid = InvalidOid;
- if (oldexternal != NULL)
- {
- varatt_external_oid old_toast_pointer;
-
- Assert(VARATT_IS_EXTERNAL_ONDISK(oldexternal));
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(old_toast_pointer, oldexternal);
- if (old_toast_pointer.va_toastrelid == rel->rd_toastoid)
- {
- /* This value came from the old toast table; reuse its OID */
- toast_pointer.va_valueid = old_toast_pointer.va_valueid;
-
- /*
- * There is a corner case here: the table rewrite might have
- * to copy both live and recently-dead versions of a row, and
- * those versions could easily reference the same toast value.
- * When we copy the second or later version of such a row,
- * reusing the OID will mean we select an OID that's already
- * in the new toast table. Check for that, and if so, just
- * fall through without writing the data again.
- *
- * While annoying and ugly-looking, this is a good thing
- * because it ensures that we wind up with only one copy of
- * the toast value when there is only one copy in the old
- * toast table. Before we detected this case, we'd have made
- * multiple copies, wasting space; and what's worse, the
- * copies belonging to already-deleted heap tuples would not
- * be reclaimed by VACUUM.
- */
- if (toastrel_valueid_exists(toastrel,
- toast_pointer.va_valueid))
- {
- /* Match, so short-circuit the data storage loop below */
- data_todo = 0;
- }
- }
- }
- if (toast_pointer.va_valueid == InvalidOid)
{
/*
- * new value; must choose a value that doesn't conflict in either
- * old or new toast table.
+ * Choose an unused OID. During a rewrite we must also avoid IDs
+ * that are still present in the old toast table.
*/
- if (toast_typid == OID8OID)
- toast_pointer.va_valueid = GetNewObjectId8();
- else
- {
- do
- {
- toast_pointer.va_valueid =
- GetNewOidWithIndex(toastrel,
- RelationGetRelid(toastidxs[validIndex]),
- (AttrNumber) 1);
- } while (toastid_valueid_exists(rel->rd_toastoid,
- toast_pointer.va_valueid));
- }
+ do
+ va_valueid =
+ GetNewOidWithIndex(toastrel,
+ RelationGetRelid(toastidxs[validIndex]),
+ (AttrNumber) 1);
+ while (OidIsValid(rel->rd_toastoid) &&
+ toastid_valueid_exists(rel->rd_toastoid, va_valueid));
}
}
+ max_chunk_size = TOAST_MAX_CHUNK_SIZE(toast_typid);
+
/*
* Split up the item into chunks
*/
@@ -302,8 +305,10 @@ toast_save_datum(Relation rel, Datum value,
union
{
alignas(int32) varlena hdr;
+
/* this is to make the union big enough for a chunk: */
- char data[TOAST_OID_MAX_CHUNK_SIZE + VARHDRSZ];
+ char data[Max(TOAST_OID_MAX_CHUNK_SIZE,
+ TOAST_OID8_MAX_CHUNK_SIZE) + VARHDRSZ];
} chunk_data;
int32 chunk_size;
@@ -312,15 +317,15 @@ toast_save_datum(Relation rel, Datum value,
/*
* Calculate the size of this chunk
*/
- chunk_size = Min(TOAST_OID_MAX_CHUNK_SIZE, data_todo);
+ chunk_size = Min(max_chunk_size, data_todo);
/*
* Build a tuple and store it
*/
if (toast_typid == OID8OID)
- t_values[0] = ObjectId8GetDatum(toast_pointer.va_valueid);
+ t_values[0] = ObjectId8GetDatum(va_valueid);
else
- t_values[0] = ObjectIdGetDatum(toast_pointer.va_valueid);
+ t_values[0] = ObjectIdGetDatum((Oid) va_valueid);
t_values[1] = Int32GetDatum(chunk_seq++);
SET_VARSIZE(&chunk_data, chunk_size + VARHDRSZ);
memcpy(VARDATA(&chunk_data), data_p, chunk_size);
@@ -376,9 +381,32 @@ toast_save_datum(Relation rel, Datum value,
/*
* Create the TOAST pointer value that we'll return
*/
- result = (varlena *) palloc(TOAST_OID_POINTER_SIZE);
- SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID);
- memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ if (toast_typid == OID8OID)
+ {
+ varatt_external_oid8 toast_pointer;
+
+ toast_pointer.va_rawsize = va_rawsize;
+ toast_pointer.va_extinfo = va_extinfo;
+ VARATT_EXTERNAL_OID8_SET_VALUEID(&toast_pointer, va_valueid);
+ toast_pointer.va_toastrelid = va_toastrelid;
+
+ result = (varlena *) palloc(TOAST_OID8_POINTER_SIZE);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID8);
+ memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ toast_pointer.va_rawsize = va_rawsize;
+ toast_pointer.va_extinfo = va_extinfo;
+ toast_pointer.va_valueid = (Oid) va_valueid;
+ toast_pointer.va_toastrelid = va_toastrelid;
+
+ result = (varlena *) palloc(TOAST_OID_POINTER_SIZE);
+ SET_VARTAG_EXTERNAL(result, VARTAG_ONDISK_OID);
+ memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
+ }
return PointerGetDatum(result);
}
@@ -413,7 +441,7 @@ void
toast_delete_datum(Relation rel, Datum value, bool is_speculative)
{
varlena *attr = (varlena *) DatumGetPointer(value);
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
Relation toastrel;
Relation *toastidxs;
ScanKeyData toastkey;
@@ -425,13 +453,17 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
return;
- /* Must copy to access aligned fields */
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ /*
+ * Decode the pointer to get the toast relation OID and value ID. The
+ * vartag tells us everything we need - no TOAST table schema lookup
+ * required.
+ */
+ toast_external_info_get(attr, &toast_ext_data);
/*
* Open the toast relation and its indexes
*/
- toastrel = table_open(toast_pointer.va_toastrelid, RowExclusiveLock);
+ toastrel = table_open(toast_ext_data.toastrelid, RowExclusiveLock);
/* Fetch valid relation used for process */
validIndex = toast_open_indexes(toastrel,
@@ -444,7 +476,7 @@ toast_delete_datum(Relation rel, Datum value, bool is_speculative)
*/
toast_valueid_scankey_init(&toastkey,
TupleDescAttr(toastrel->rd_att, 0)->atttypid,
- toast_pointer.va_valueid);
+ toast_ext_data.valueid);
/*
* Find all the chunks. (We don't actually care whether we see them in
diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c
index 81d5085ca39..f8a54ff2582 100644
--- a/src/backend/access/heap/heaptoast.c
+++ b/src/backend/access/heap/heaptoast.c
@@ -649,7 +649,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid8 valueid, int32 attrsize,
&num_indexes);
toast_typid = TupleDescAttr(toastrel->rd_att, 0)->atttypid;
- max_chunk_size = TOAST_OID_MAX_CHUNK_SIZE;
+ max_chunk_size = TOAST_MAX_CHUNK_SIZE(toast_typid);
totalchunks = ((attrsize - 1) / max_chunk_size) + 1;
startchunk = sliceoffset / max_chunk_size;
diff --git a/src/backend/access/table/toast_helper.c b/src/backend/access/table/toast_helper.c
index 6f2d691cc68..2613d9dc095 100644
--- a/src/backend/access/table/toast_helper.c
+++ b/src/backend/access/table/toast_helper.c
@@ -69,12 +69,16 @@ toast_tuple_init(ToastTupleContext *ttc)
/*
* If the old value is stored on disk, check if it has changed so
* we have to delete it later.
+ *
+ * Note that TOAST pointers could have different vartags, for oid
+ * or oid8, and these can have different sizes.
*/
if (att->attlen == -1 && !ttc->ttc_oldisnull[i] &&
VARATT_IS_EXTERNAL_ONDISK(old_value))
{
if (ttc->ttc_isnull[i] ||
!VARATT_IS_EXTERNAL_ONDISK(new_value) ||
+ VARTAG_EXTERNAL(old_value) != VARTAG_EXTERNAL(new_value) ||
memcmp(old_value, new_value,
VARSIZE_EXTERNAL(old_value)) != 0)
{
@@ -171,8 +175,9 @@ toast_tuple_init(ToastTupleContext *ttc)
* The column must have attstorage EXTERNAL or EXTENDED if check_main is
* false, and must have attstorage MAIN if check_main is true.
*
- * The column must have a minimum size of MAXALIGN(TOAST_OID_POINTER_SIZE);
- * if not, no benefit is to be expected by compressing it.
+ * The column must be larger than the TOAST pointer that would replace it;
+ * if not, no benefit is to be expected by compressing it. Note that this
+ * choice depends on the TOAST value type, oid or oid8.
*
* The return value is the index of the biggest suitable column, or
* -1 if there is none.
@@ -184,10 +189,21 @@ toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
TupleDesc tupleDesc = ttc->ttc_rel->rd_att;
int numAttrs = tupleDesc->natts;
int biggest_attno = -1;
- int32 biggest_size = MAXALIGN(TOAST_OID_POINTER_SIZE);
+ int32 biggest_size;
int32 skip_colflags = TOASTCOL_IGNORE;
int i;
+ /*
+ * Size of the TOAST pointer this relation would use. A relation without
+ * a TOAST table cannot have any of its attributes moved out-of-line, but
+ * it can still have some of them compressed. Fall back to the oid size
+ * in that case.
+ */
+ if (RelationGetToastChunkIdType(ttc->ttc_rel) == OID8OID)
+ biggest_size = MAXALIGN(TOAST_OID8_POINTER_SIZE);
+ else
+ biggest_size = MAXALIGN(TOAST_OID_POINTER_SIZE);
+
if (for_compression)
skip_colflags |= TOASTCOL_INCOMPRESSIBLE;
diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c
index dd4b6e8f7f4..ede117d339e 100644
--- a/src/backend/replication/logical/reorderbuffer.c
+++ b/src/backend/replication/logical/reorderbuffer.c
@@ -5175,7 +5175,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
varlena *varlena_pointer;
/* va_rawsize is the size of the original datum -- including header */
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
varatt_indirect redirect_pointer;
varlena *new_datum = NULL;
varlena *reconstructed;
@@ -5198,11 +5198,11 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
varlena_pointer = (varlena *) DatumGetPointer(attrs[natt]);
/* no need to do anything if the tuple isn't external */
- if (!VARATT_IS_EXTERNAL(varlena_pointer))
+ if (!VARATT_IS_EXTERNAL_ONDISK(varlena_pointer))
continue;
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, varlena_pointer);
- toast_valueid = toast_pointer.va_valueid;
+ toast_external_info_get(varlena_pointer, &toast_ext_data);
+ toast_valueid = toast_ext_data.valueid;
/*
* Check whether the toast tuple changed, replace if so.
@@ -5220,7 +5220,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
free[natt] = true;
- reconstructed = palloc0(toast_pointer.va_rawsize);
+ reconstructed = palloc0(toast_ext_data.rawsize);
ent->reconstructed = reconstructed;
@@ -5245,10 +5245,10 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn,
VARSIZE(chunk) - VARHDRSZ);
data_done += VARSIZE(chunk) - VARHDRSZ;
}
- Assert(data_done == VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer));
+ Assert(data_done == VARATT_EXTINFO_GET_EXTSIZE(toast_ext_data.extinfo));
/* make sure its marked as compressed or not */
- if (VARATT_EXTERNAL_OID_IS_COMPRESSED(toast_pointer))
+ if (VARATT_EXTINFO_IS_COMPRESSED(toast_ext_data.extinfo, toast_ext_data.rawsize))
SET_VARSIZE_COMPRESSED(reconstructed, data_done + VARHDRSZ);
else
SET_VARSIZE(reconstructed, data_done + VARHDRSZ);
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 4533dac11a6..01cc61c5778 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -4261,7 +4261,7 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS)
{
int typlen;
varlena *attr;
- varatt_external_oid toast_pointer;
+ toast_external_data toast_ext_data;
/* On first call, get the input type's typlen, and save at *fn_extra */
if (fcinfo->flinfo->fn_extra == NULL)
@@ -4288,9 +4288,9 @@ pg_column_toast_chunk_id(PG_FUNCTION_ARGS)
if (!VARATT_IS_EXTERNAL_ONDISK(attr))
PG_RETURN_NULL();
- VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_external_info_get(attr, &toast_ext_data);
- PG_RETURN_OID8(toast_pointer.va_valueid);
+ PG_RETURN_OID8(toast_ext_data.valueid);
}
/*
diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c
index 541f30f0972..2555de93706 100644
--- a/src/backend/utils/cache/relcache.c
+++ b/src/backend/utils/cache/relcache.c
@@ -1205,6 +1205,9 @@ retry:
relation->rd_fkeylist = NIL;
relation->rd_fkeyvalid = false;
+ /* TOAST type data is not loaded till asked for */
+ relation->rd_toastchunkidtype = InvalidOid;
+
/* partitioning data is not loaded till asked for */
relation->rd_partkey = NULL;
relation->rd_partkeycxt = NULL;
@@ -5100,6 +5103,39 @@ RelationGetReplicaIndex(Relation relation)
return relation->rd_replidindex;
}
+/*
+ * RelationGetToastChunkIdType -- get the type of the relation's TOAST
+ * table "chunk_id" column
+ *
+ * Returns OIDOID or OID8OID, or InvalidOid if the relation has no TOAST
+ * table.
+ */
+Oid
+RelationGetToastChunkIdType(Relation relation)
+{
+ Oid toastrelid = relation->rd_rel->reltoastrelid;
+ Oid typid;
+
+ /* Quick exit if we already computed the value */
+ if (OidIsValid(relation->rd_toastchunkidtype))
+ return relation->rd_toastchunkidtype;
+
+ /* Nothing to report without a TOAST table */
+ if (!OidIsValid(toastrelid))
+ return InvalidOid;
+
+ typid = get_atttype(toastrelid, 1);
+ if (!OidIsValid(typid))
+ elog(ERROR, "cache lookup failed for TOAST relation %u",
+ toastrelid);
+ if (typid != OIDOID && typid != OID8OID)
+ elog(ERROR, "unexpected type %u for chunk_id in TOAST relation %u",
+ typid, toastrelid);
+
+ relation->rd_toastchunkidtype = typid;
+ return typid;
+}
+
/*
* RelationGetIndexExpressions -- get the index expressions for an index
*
@@ -6545,6 +6581,7 @@ load_relcache_init_file(bool shared)
rel->rd_firstRelfilelocatorSubid = InvalidSubTransactionId;
rel->rd_droppedSubid = InvalidSubTransactionId;
rel->rd_amcache = NULL;
+ rel->rd_toastchunkidtype = InvalidOid;
rel->pgstat_info = NULL;
/*
diff --git a/src/include/access/detoast.h b/src/include/access/detoast.h
index 2b1428b1e81..93b7a253760 100644
--- a/src/include/access/detoast.h
+++ b/src/include/access/detoast.h
@@ -12,6 +12,8 @@
#ifndef DETOAST_H
#define DETOAST_H
+#include "varatt.h"
+
/*
* Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
* into a local "varatt_external_oid" toast pointer. This should be
@@ -30,9 +32,55 @@ do { \
/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
#define TOAST_OID_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid))
+/* Size of an EXTERNAL datum that contains an Oid8 TOAST pointer */
+#define TOAST_OID8_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external_oid8))
+
/* Size of an EXTERNAL datum that contains an indirection pointer */
#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
+/*
+ * Decoded contents of an on-disk external TOAST pointer.
+ */
+typedef struct toast_external_data
+{
+ vartag_external tag; /* VARTAG_ONDISK_* */
+ int32 rawsize; /* original data size (includes header) */
+ uint32 extinfo; /* saved size + compression method */
+ Oid8 valueid; /* value ID (can be widened from Oid) */
+ Oid toastrelid; /* OID of the TOAST table containing it */
+} toast_external_data;
+
+/*
+ * Decode an on-disk external TOAST pointer into a toast_external_data.
+ */
+static inline void
+toast_external_info_get(const struct varlena *attr, toast_external_data *toast_ext_data)
+{
+ Assert(VARATT_IS_EXTERNAL_ONDISK(attr));
+
+ toast_ext_data->tag = VARTAG_EXTERNAL(attr);
+ if (toast_ext_data->tag == VARTAG_ONDISK_OID8)
+ {
+ varatt_external_oid8 toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_ext_data->rawsize = toast_pointer.va_rawsize;
+ toast_ext_data->extinfo = toast_pointer.va_extinfo;
+ toast_ext_data->valueid = VARATT_EXTERNAL_OID8_GET_VALUEID(toast_pointer);
+ toast_ext_data->toastrelid = toast_pointer.va_toastrelid;
+ }
+ else
+ {
+ varatt_external_oid toast_pointer;
+
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ toast_ext_data->rawsize = toast_pointer.va_rawsize;
+ toast_ext_data->extinfo = toast_pointer.va_extinfo;
+ toast_ext_data->valueid = toast_pointer.va_valueid;
+ toast_ext_data->toastrelid = toast_pointer.va_toastrelid;
+ }
+}
+
/* ----------
* detoast_external_attr() -
*
diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h
index aaae6e0fef6..4c3fda8a5f9 100644
--- a/src/include/access/heaptoast.h
+++ b/src/include/access/heaptoast.h
@@ -69,13 +69,14 @@
/*
* When we store an oversize datum externally, we divide it into chunks
- * containing at most TOAST_OID_MAX_CHUNK_SIZE data bytes. This number *must*
- * be small enough that the completed toast-table tuple (including the
- * ID and sequence fields and all overhead) will fit on a page.
+ * containing at most TOAST_OID_MAX_CHUNK_SIZE or TOAST_OID8_MAX_CHUNK_SIZE
+ * data bytes, depending on the chunk_id type of the TOAST table. These
+ * numbers *must* be small enough that the completed toast-table tuple
+ * (including the ID and sequence fields and all overhead) will fit on a page.
* The coding here sets the size on the theory that we want to fit
* EXTERN_TUPLES_PER_PAGE tuples of maximum size onto a page.
*
- * NB: Changing TOAST_OID_MAX_CHUNK_SIZE requires an initdb.
+ * NB: Changing these values requires an initdb.
*/
#define EXTERN_TUPLES_PER_PAGE 4 /* tweak only this */
@@ -88,6 +89,19 @@
sizeof(int32) - \
VARHDRSZ)
+#define TOAST_OID8_MAX_CHUNK_SIZE \
+ (EXTERN_TUPLE_MAX_SIZE - \
+ MAXALIGN(SizeofHeapTupleHeader) - \
+ sizeof(Oid8) - \
+ sizeof(int32) - \
+ VARHDRSZ)
+
+/*
+ * Select the chunk size for a TOAST table from its value type.
+ */
+#define TOAST_MAX_CHUNK_SIZE(toast_typid) \
+ ((toast_typid) == OID8OID ? TOAST_OID8_MAX_CHUNK_SIZE : TOAST_OID_MAX_CHUNK_SIZE)
+
/* ----------
* heap_toast_insert_or_update -
*
diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h
index f28b8db6e6f..c192c0a5a00 100644
--- a/src/include/catalog/catversion.h
+++ b/src/include/catalog/catversion.h
@@ -57,6 +57,6 @@
*/
/* yyyymmddN */
-#define CATALOG_VERSION_NO 202609141
+#define CATALOG_VERSION_NO 202609151
#endif
diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h
index d61432de705..9c8d17337b8 100644
--- a/src/include/utils/rel.h
+++ b/src/include/utils/rel.h
@@ -250,6 +250,15 @@ typedef struct RelationData
*/
Oid rd_toastoid; /* Real TOAST table's OID, or InvalidOid */
+ /*
+ * Type OID of the "chunk_id" column of this relation's TOAST table, i.e.
+ * OIDOID or OID8OID.
+ *
+ * This data is filled on demand by RelationGetToastChunkIdType(), not at
+ * relcache build time, so as to save in syscache lookups.
+ */
+ Oid rd_toastchunkidtype;
+
bool pgstat_enabled; /* should relation stats be counted */
/* use "struct" here to avoid needing to include pgstat.h: */
struct PgStat_RelationStatus *pgstat_info; /* statistics collection area */
diff --git a/src/include/utils/relcache.h b/src/include/utils/relcache.h
index 89c27aa1529..e17c8490833 100644
--- a/src/include/utils/relcache.h
+++ b/src/include/utils/relcache.h
@@ -57,6 +57,7 @@ extern List *RelationGetIndexList(Relation relation);
extern List *RelationGetStatExtList(Relation relation);
extern Oid RelationGetPrimaryKeyIndex(Relation relation, bool deferrable_ok);
extern Oid RelationGetReplicaIndex(Relation relation);
+extern Oid RelationGetToastChunkIdType(Relation relation);
extern List *RelationGetIndexExpressions(Relation relation);
extern List *RelationGetDummyIndexExpressions(Relation relation);
extern List *RelationGetIndexPredicate(Relation relation);
diff --git a/src/include/varatt.h b/src/include/varatt.h
index 89b47db0833..77041f7673a 100644
--- a/src/include/varatt.h
+++ b/src/include/varatt.h
@@ -42,6 +42,44 @@ StaticAssertDecl((sizeof(int32) + sizeof(uint32) + 2 * sizeof(Oid)) ==
sizeof(varatt_external_oid),
"varatt_external_oid should have no padding");
+/*
+ * varatt_external_oid8 is a "TOAST pointer" for TOAST tables that use
+ * Oid8 (64-bit) as their chunk_id type. Same layout as varatt_external_oid
+ * except for the value ID, which is 8 bytes wide. The value ID is split
+ * into two uint32 to force alignment.
+ *
+ * This struct must not contain any padding, because we sometimes compare
+ * these pointers using memcmp.
+ */
+typedef struct varatt_external_oid8
+{
+ int32 va_rawsize; /* Original data size (includes header) */
+ uint32 va_extinfo; /* External saved size (without header) and
+ * compression method */
+ uint32 va_valueid_lo; /* Low 32 bits of value ID */
+ uint32 va_valueid_hi; /* High 32 bits of value ID */
+ Oid va_toastrelid; /* RelID of TOAST table containing it */
+} varatt_external_oid8;
+
+StaticAssertDecl((sizeof(int32) + 3 * sizeof(uint32) + sizeof(Oid)) ==
+ sizeof(varatt_external_oid8),
+ "varatt_external_oid8 should have no padding");
+
+/* Get/set the 64-bit value ID from the lo/hi halves */
+static inline Oid8
+VARATT_EXTERNAL_OID8_GET_VALUEID(varatt_external_oid8 toast_pointer)
+{
+ return ((Oid8) toast_pointer.va_valueid_lo) |
+ (((Oid8) toast_pointer.va_valueid_hi) << 32);
+}
+
+static inline void
+VARATT_EXTERNAL_OID8_SET_VALUEID(varatt_external_oid8 *toast_pointer, Oid8 id)
+{
+ toast_pointer->va_valueid_lo = (uint32) id;
+ toast_pointer->va_valueid_hi = (uint32) (id >> 32);
+}
+
/*
* These macros define the "saved size" portion of va_extinfo. Its remaining
* two high-order bits identify the compression method.
@@ -91,6 +129,7 @@ typedef enum vartag_external
VARTAG_INDIRECT = 1,
VARTAG_EXPANDED_RO = 2,
VARTAG_EXPANDED_RW = 3,
+ VARTAG_ONDISK_OID8 = 4,
VARTAG_ONDISK_OID = 18
} vartag_external;
@@ -112,6 +151,8 @@ VARTAG_SIZE(vartag_external tag)
return sizeof(varatt_expanded);
else if (tag == VARTAG_ONDISK_OID)
return sizeof(varatt_external_oid);
+ else if (tag == VARTAG_ONDISK_OID8)
+ return sizeof(varatt_external_oid8);
else
{
Assert(false);
@@ -365,7 +406,12 @@ VARATT_IS_EXTERNAL(const void *PTR)
static inline bool
VARATT_IS_EXTERNAL_ONDISK(const void *PTR)
{
- return VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK_OID;
+ vartag_external tag;
+
+ if (!VARATT_IS_EXTERNAL(PTR))
+ return false;
+ tag = VARTAG_EXTERNAL(PTR);
+ return (tag == VARTAG_ONDISK_OID || tag == VARTAG_ONDISK_OID8);
}
/* Is varlena datum an indirect pointer? */
@@ -508,43 +554,31 @@ VARDATA_COMPRESSED_GET_COMPRESS_METHOD(const void *PTR)
}
/*
- * Same for external Datums; but note argument is a struct
- * varatt_external_oid.
+ * Same for external Datums, saved into an va_extinfo.
*/
static inline Size
-VARATT_EXTERNAL_OID_GET_EXTSIZE(varatt_external_oid toast_pointer)
+VARATT_EXTINFO_GET_EXTSIZE(uint32 extinfo)
{
- return toast_pointer.va_extinfo & VARLENA_EXTSIZE_MASK;
+ return extinfo & VARLENA_EXTSIZE_MASK;
}
static inline uint32
-VARATT_EXTERNAL_OID_GET_COMPRESS_METHOD(varatt_external_oid toast_pointer)
+VARATT_EXTINFO_GET_COMPRESS_METHOD(uint32 extinfo)
{
- return toast_pointer.va_extinfo >> VARLENA_EXTSIZE_BITS;
+ return extinfo >> VARLENA_EXTSIZE_BITS;
}
-/* Set size and compress method of an externally-stored varlena datum */
-/* This has to remain a macro; beware multiple evaluations! */
-#define VARATT_EXTERNAL_SET_SIZE_AND_COMPRESS_METHOD(toast_pointer, len, cm) \
- do { \
- Assert((cm) == TOAST_PGLZ_COMPRESSION_ID || \
- (cm) == TOAST_LZ4_COMPRESSION_ID); \
- ((toast_pointer).va_extinfo = \
- (len) | ((uint32) (cm) << VARLENA_EXTSIZE_BITS)); \
- } while (0)
-
/*
- * Testing whether an externally-stored value is compressed now requires
- * comparing size stored in va_extinfo (the actual length of the external data)
+ * Testing whether an externally-stored value is compressed requires comparing
+ * the saved size stored in va_extinfo (the actual length of the external data)
* to rawsize (the original uncompressed datum's size). The latter includes
* VARHDRSZ overhead, the former doesn't. We never use compression unless it
* actually saves space, so we expect either equality or less-than.
*/
static inline bool
-VARATT_EXTERNAL_OID_IS_COMPRESSED(varatt_external_oid toast_pointer)
+VARATT_EXTINFO_IS_COMPRESSED(uint32 extinfo, int32 rawsize)
{
- return VARATT_EXTERNAL_OID_GET_EXTSIZE(toast_pointer) <
- (Size) (toast_pointer.va_rawsize - VARHDRSZ);
+ return VARATT_EXTINFO_GET_EXTSIZE(extinfo) < (Size) (rawsize - VARHDRSZ);
}
#endif
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index db45ee8881e..9e4f660836f 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -4361,6 +4361,7 @@ timeout_params
timerCA
tlist_vinfo
toast_compress_header
+toast_external_data
tokenize_error_callback_arg
transferMode
transfer_thread_arg
@@ -4417,6 +4418,7 @@ vacuumingOptions
validate_string_relopt
varatt_expanded
varatt_external_oid
+varatt_external_oid8
varatt_indirect
varattrib_1b
varattrib_1b_e