database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2026-10-23 13:29:09 +03:00
Code Activity

Widen Object ID counter to 8 bytes

Browse Source 
This commit adds a new routine called GetNewObjectId8() in varsup.c,
which is able to retrieve a 8-byte OID, based on the next OID retrieved
from the control file.  GetNewObjectId() is kept compatible with its
origin, where we still check that the lower 32 bits of the counter do
not wraparound, while handling the FirstNormalObjectId case.

This change is combined with an update of the control file, with
CheckPoint.nextOid enlarged from Oid to Oid8.  pg_control_checkpoint(),
pg_resetwal and pg_upgrade are updated to adapt to this change, to make
sure that the value is not truncated due to 32-bit limitations when
used, making an upgrade to a newer version where 8-byte is supported a
transparent operation.

A couple of comments in the xlog redo logic referred to wraparound
issues, which is not the case anymore with this change in place, so
refresh these.

This change is required to be able to support external TOAST pointers
based on Oid8, under discussion, and can be used for other purposes.

Tests are added for pg_upgrade (2^32 value carried across upgrades),
pg_resetwal and pg_control_checkpoint(), hence across all the tools
that need to be covered by this change.

Bump catalog version.
Bump PG_CONTROL_VERSION.
Bump XLOG_PAGE_MAGIC, due to XLOG_NEXTOID.

Author: Michael Paquier <michael@paquier.xyz>
Reviewed-by: Greg Burd <greg@burd.me>
Reviewed-by: Bharath Rupireddy <bharath.rupireddyforpostgres@gmail.com>
Reviewed-by: Yugo Nagata <nagata@sraoss.co.jp>
Discussion: https://postgr.es/m/af19kUjwjhaoUTLn@paquier.xyz
This commit is contained in:
Michael Paquier
2026-09-10 09:39:18 +09:00
parent 1129570f65
commit cb29861646
20 changed files with 129 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
doc/src/sgml/func/func-info.sgml
View File

@@ -3433,7 +3433,7 @@ acl | {postgres=arwdDxtm/postgres,foo=r/postgres}
<row>
<entry><structfield>next_oid</structfield></entry>
<entry><type>oid</type></entry>
<entry><type>oid8</type></entry>
</row>
<row>

10
doc/src/sgml/ref/pg_resetwal.sgml
View File

@@ -286,17 +286,19 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
<term><option>-o <replaceable class="parameter">oid</replaceable></option></term>
<term><option>--next-oid=<replaceable class="parameter">oid</replaceable></option></term>
<term><option>-o <replaceable class="parameter">oid8</replaceable></option></term>
<term><option>--next-oid=<replaceable class="parameter">oid8</replaceable></option></term>
<listitem>
<para>
Manually set the next OID.
Manually set the next OID (8 bytes).
</para>
<para>
There is no comparably easy way to determine a next OID that's beyond
the largest one in the database, but fortunately it is not critical to
get the next-OID setting right.
get the next-OID setting right, as long as the counter is not moved
backwards. The counter is 8 bytes wide and never wraps around during
the life of a cluster.
</para>
</listitem>
</varlistentry>

8
src/backend/access/rmgrdesc/xlogdesc.c
View File

@@ -101,7 +101,7 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
CheckPoint *checkpoint = (CheckPoint *) rec;
appendStringInfo(buf, "redo %X/%08X; "
"tli %u; prev tli %u; fpw %s; wal_level %s; logical decoding %s; xid %u:%u; oid %u; multi %u; offset %" PRIu64 "; "
"tli %u; prev tli %u; fpw %s; wal_level %s; logical decoding %s; xid %u:%u; oid " OID8_FORMAT "; multi %u; offset %" PRIu64 "; "
"oldest xid %u in DB %u; oldest multi %u in DB %u; "
"oldest/newest commit timestamp xid: %u/%u; "
"oldest running xid %u; "
@@ -129,10 +129,10 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
}
else if (info == XLOG_NEXTOID)
{
Oid nextOid;
Oid8 nextOid;
memcpy(&nextOid, rec, sizeof(Oid));
appendStringInfo(buf, "%u", nextOid);
memcpy(&nextOid, rec, sizeof(Oid8));
appendStringInfo(buf, OID8_FORMAT, nextOid);
}
else if (info == XLOG_RESTORE_POINT)
{

59
src/backend/access/transam/varsup.c
View File

@@ -541,31 +541,51 @@ ForceTransactionIdLimitUpdate(void)
/*
* GetNewObjectId -- allocate a new OID
* GetNewObjectId -- allocate a new OID (4 bytes)
*
* OIDs are generated by a cluster-wide counter. Since they are only 32 bits
* wide, counter wraparound will occur eventually, and therefore it is unwise
* to assume they are unique unless precautions are taken to make them so.
* Hence, this routine should generally not be used directly. The only direct
* callers should be GetNewOidWithIndex() and GetNewRelFileNumber() in
* catalog/catalog.c.
* OIDs are generated by a cluster-wide counter. The callers of this routine
* expect a 32 bit-wide counter. Counter wraparound will occur eventually,
* and therefore it is unwise to assume they are unique unless precautions are
* taken to make them so. This routine should generally not be used directly.
* The only direct callers should be GetNewOidWithIndex() and
* GetNewRelFileNumber() in catalog/catalog.c.
*/
Oid
GetNewObjectId(void)
{
Oid result;
return (Oid) GetNewObjectId8();
}
/*
* GetNewObjectId8 -- allocate a new OID (8 bytes)
*
* This routine can be called directly if the consumer of the OID allocated
* stores the counter in an 8-byte space, where wraparound does not matter.
* We still need to care about the wraparound case in the low 32 bits of the
* space allocated, GetNewObjectId() expecting OIDs to never be allocated
* up to FirstNormalObjectId.
*/
Oid8
GetNewObjectId8(void)
{
Oid8 result;
Oid nextoid_lo;
uint32 nextoid_hi;
/* safety check, we should never get this far in a HS standby */
if (RecoveryInProgress())
elog(ERROR, "cannot assign OIDs during recovery");
LWLockAcquire(OidGenLock, LW_EXCLUSIVE);
nextoid_lo = (Oid) TransamVariables->nextOid;
nextoid_hi = (uint32) (TransamVariables->nextOid >> 32);
/*
* Check for wraparound of the OID counter. We *must* not return 0
* (InvalidOid), and in normal operation we mustn't return anything below
* FirstNormalObjectId since that range is reserved for initdb (see
* IsCatalogRelationOid()). Note we are relying on unsigned comparison.
* Check for wraparound of the OID counter in its lower 4 bytes. We *must*
* not return 0 (InvalidOid), and in normal operation we mustn't return
* anything below FirstNormalObjectId since that range is reserved for
* initdb (see IsCatalogRelationOid()). Note we are relying on unsigned
* comparison.
*
* During initdb, we start the OID generator at FirstGenbkiObjectId, so we
* only wrap if before that point when in bootstrap or standalone mode.
@@ -575,26 +595,29 @@ GetNewObjectId(void)
* available for automatic assignment during initdb, while ensuring they
* will never conflict with user-assigned OIDs.
*/
if (TransamVariables->nextOid < ((Oid) FirstNormalObjectId))
if (nextoid_lo < ((Oid) FirstNormalObjectId))
{
if (IsPostmasterEnvironment)
{
/* wraparound, or first post-initdb assignment, in normal mode */
TransamVariables->nextOid = FirstNormalObjectId;
nextoid_lo = FirstNormalObjectId;
TransamVariables->oidCount = 0;
}
else
{
/* we may be bootstrapping, so don't enforce the full range */
if (TransamVariables->nextOid < ((Oid) FirstGenbkiObjectId))
if (nextoid_lo < ((Oid) FirstGenbkiObjectId))
{
/* wraparound in standalone mode (unlikely but possible) */
TransamVariables->nextOid = FirstNormalObjectId;
nextoid_lo = FirstNormalObjectId;
TransamVariables->oidCount = 0;
}
}
}
/* Set next OID in its 8-byte space */
TransamVariables->nextOid = ((Oid8) nextoid_hi) << 32 | nextoid_lo;
/* If we run out of logged for use oids then we must log more */
if (TransamVariables->oidCount == 0)
{
@@ -619,7 +642,7 @@ GetNewObjectId(void)
* to the specified value.
*/
static void
SetNextObjectId(Oid nextOid)
SetNextObjectId(Oid8 nextOid)
{
/* Safety check, this is only allowable during initdb */
if (IsPostmasterEnvironment)
@@ -629,7 +652,7 @@ SetNextObjectId(Oid nextOid)
LWLockAcquire(OidGenLock, LW_EXCLUSIVE);
if (TransamVariables->nextOid > nextOid)
elog(ERROR, "too late to advance OID counter to %u, it is now %u",
elog(ERROR, "too late to advance OID counter to " OID8_FORMAT ", it is now " OID8_FORMAT,
nextOid, TransamVariables->nextOid);
TransamVariables->nextOid = nextOid;

26
src/backend/access/transam/xlog.c
View File

@@ -8604,10 +8604,10 @@ KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo)
* Write a NEXTOID log record
*/
void
XLogPutNextOid(Oid nextOid)
XLogPutNextOid(Oid8 nextOid)
{
XLogBeginInsert();
XLogRegisterData(&nextOid, sizeof(Oid));
XLogRegisterData(&nextOid, sizeof(Oid8));
(void) XLogInsert(RM_XLOG_ID, XLOG_NEXTOID);
/*
@@ -8873,16 +8873,17 @@ xlog_redo(XLogReaderState *record)
if (info == XLOG_NEXTOID)
{
Oid nextOid;
Oid8 nextOid;
/*
* We used to try to take the maximum of TransamVariables->nextOid and
* the recorded nextOid, but that fails if the OID counter wraps
* around. Since no OID allocation should be happening during replay
* anyway, better to just believe the record exactly. We still take
* OidGenLock while setting the variable, just in case.
* the recorded nextOid, but that failed back when the counter was 4
* bytes wide and could wrap around. Since no OID allocation should
* be happening during replay anyway, better to just believe the
* record exactly. We still take OidGenLock while setting the
* variable, just in case.
*/
memcpy(&nextOid, XLogRecGetData(record), sizeof(Oid));
memcpy(&nextOid, XLogRecGetData(record), sizeof(Oid8));
LWLockAcquire(OidGenLock, LW_EXCLUSIVE);
TransamVariables->nextOid = nextOid;
TransamVariables->oidCount = 0;
@@ -9010,11 +9011,10 @@ xlog_redo(XLogReaderState *record)
* to track OID assignment through XLOG_NEXTOID records. The nextOid
* counter is from the start of the checkpoint and might well be stale
* compared to later XLOG_NEXTOID records. We could try to take the
* maximum of the nextOid counter and our latest value, but since
* there's no particular guarantee about the speed with which the OID
* counter wraps around, that's a risky thing to do. In any case,
* users of the nextOid counter are required to avoid assignment of
* duplicates, so that a somewhat out-of-date value should be safe.
* maximum of the nextOid counter and our latest value, but there is
* no point in doing so: an online checkpoint records nextOid plus
* oidCount, which is never ahead of the last XLOG_NEXTOID record that
* replay has applied.
*/
/* Handle multixact */

2
src/backend/access/transam/xlogrecovery.c
View File

@@ -840,7 +840,7 @@ InitWalRecovery(ControlFileData *ControlFile, bool *wasShutdown_ptr,
LSN_FORMAT_ARGS(checkPoint.redo),
wasShutdown ? "true" : "false"));
ereport(DEBUG1,
(errmsg_internal("next transaction ID: " UINT64_FORMAT "; next OID: %u",
(errmsg_internal("next transaction ID: " UINT64_FORMAT "; next OID: " OID8_FORMAT,
U64FromFullTransactionId(checkPoint.nextXid),
checkPoint.nextOid)));
ereport(DEBUG1,

2
src/backend/utils/misc/pg_controldata.c
View File

@@ -124,7 +124,7 @@ pg_control_checkpoint(PG_FUNCTION_ARGS)
XidFromFullTransactionId(ControlFile->checkPointCopy.nextXid)));
nulls[7] = false;
values[8] = ObjectIdGetDatum(ControlFile->checkPointCopy.nextOid);
values[8] = ObjectId8GetDatum(ControlFile->checkPointCopy.nextOid);
nulls[8] = false;
values[9] = TransactionIdGetDatum(ControlFile->checkPointCopy.nextMulti);

2
src/bin/pg_controldata/pg_controldata.c
View File

@@ -269,7 +269,7 @@ main(int argc, char *argv[])
printf(_("Latest checkpoint's NextXID: %u:%u\n"),
EpochFromFullTransactionId(ControlFile->checkPointCopy.nextXid),
XidFromFullTransactionId(ControlFile->checkPointCopy.nextXid));
printf(_("Latest checkpoint's NextOID: %u\n"),
printf(_("Latest checkpoint's NextOID: " OID8_FORMAT "\n"),
ControlFile->checkPointCopy.nextOid);
printf(_("Latest checkpoint's NextMultiXactId: %u\n"),
ControlFile->checkPointCopy.nextMulti);

10
src/bin/pg_resetwal/pg_resetwal.c
View File

@@ -82,7 +82,7 @@ static TransactionId oldest_commit_ts_xid_val;
static TransactionId newest_commit_ts_xid_val;
static bool next_oid_given = false;
static Oid next_oid_val;
static Oid8 next_oid_val;
static bool mxids_given = false;
static MultiXactId next_mxid_val;
@@ -253,7 +253,7 @@ main(int argc, char *argv[])
case 'o':
errno = 0;
next_oid_val = strtouint32_strict(optarg, &endptr, 0);
next_oid_val = strtouint64_strict(optarg, &endptr, 0);
if (endptr == optarg || *endptr != '\0' || errno != 0)
{
pg_log_error("invalid argument for option %s", "-o");
@@ -775,7 +775,7 @@ PrintControlValues(bool guessed)
printf(_("Latest checkpoint's NextXID: %u:%u\n"),
EpochFromFullTransactionId(ControlFile.checkPointCopy.nextXid),
XidFromFullTransactionId(ControlFile.checkPointCopy.nextXid));
printf(_("Latest checkpoint's NextOID: %u\n"),
printf(_("Latest checkpoint's NextOID: " OID8_FORMAT "\n"),
ControlFile.checkPointCopy.nextOid);
printf(_("Latest checkpoint's NextMultiXactId: %u\n"),
ControlFile.checkPointCopy.nextMulti);
@@ -861,7 +861,7 @@ PrintNewControlValues(void)
if (next_oid_given)
{
printf(_("NextOID: %u\n"),
printf(_("NextOID: " OID8_FORMAT "\n"),
ControlFile.checkPointCopy.nextOid);
}
@@ -1236,7 +1236,7 @@ usage(void)
printf(_(" -e, --epoch=XIDEPOCH set next transaction ID epoch\n"));
printf(_(" -l, --next-wal-file=WALFILE set minimum starting location for new WAL\n"));
printf(_(" -m, --multixact-ids=MXID,MXID set next and oldest multitransaction ID\n"));
printf(_(" -o, --next-oid=OID set next OID\n"));
printf(_(" -o, --next-oid=OID8 set next OID (8 bytes)\n"));
printf(_(" -O, --multixact-offset=OFFSET set next multitransaction offset\n"));
printf(_(" -u, --oldest-transaction-id=XID set oldest transaction ID\n"));
printf(_(" -x, --next-transaction-id=XID set next transaction ID\n"));

17
src/bin/pg_resetwal/t/001_basic.pl
View File

@@ -265,4 +265,21 @@ command_like(
$node->start;
ok(1, 'server started after reset');
# Checks for 8 bytes OIDs.
$node->stop;
my $new_nextoid = "4295067296"; # 2^32 + 100,000
command_ok([ 'pg_resetwal', '--next-oid' => $new_nextoid, $node->data_dir ],
'runs with --next-oid value above 2^32');
command_like(
[ 'pg_resetwal', '--dry-run', $node->data_dir ],
qr/^Latest checkpoint's NextOID: *$new_nextoid$/m,
'8-byte --next-oid value preserved in pg_control');
$node->start;
is( $node->safe_psql(
'postgres',
"SELECT next_oid >= '$new_nextoid'::oid8 FROM pg_control_checkpoint()"
),
't',
'new 8-byte OID reported');
done_testing();

2
src/bin/pg_upgrade/controldata.c
View File

@@ -245,7 +245,7 @@ get_control_data(ClusterInfo *cluster)
pg_fatal("%d: controldata retrieval problem", __LINE__);
p++; /* remove ':' char */
cluster->controldata.chkpnt_nxtoid = str2uint(p);
cluster->controldata.chkpnt_nxtoid = str2uint64(p);
got_oid = true;
}
else if ((p = strstr(bufin, "Latest checkpoint's NextMultiXactId:")) != NULL)

2
src/bin/pg_upgrade/pg_upgrade.c
View File

@@ -196,7 +196,7 @@ main(int argc, char **argv)
*/
prep_status("Setting next OID for new cluster");
exec_prog(UTILITY_LOG_FILE, NULL, true, true,
"\"%s/pg_resetwal\" -o %u \"%s\"",
"\"%s/pg_resetwal\" -o " OID8_FORMAT " \"%s\"",
new_cluster.bindir, old_cluster.controldata.chkpnt_nxtoid,
new_cluster.pgdata);
check_ok();

2
src/bin/pg_upgrade/pg_upgrade.h
View File

@@ -216,7 +216,7 @@ typedef struct
char nextxlogfile[25];
uint32 chkpnt_nxtxid;
uint32 chkpnt_nxtepoch;
uint32 chkpnt_nxtoid;
Oid8 chkpnt_nxtoid;
uint32 chkpnt_nxtmulti;
uint64 chkpnt_nxtmxoff;
uint32 chkpnt_oldstMulti;

25
src/bin/pg_upgrade/t/002_pg_upgrade.pl
View File

@@ -588,6 +588,20 @@ $oldnode->start;
$oldnode->safe_psql('postgres', 'DROP DATABASE regression_invalid');
$oldnode->stop;
# The OID counter is 8 bytes wide, check that it is carried. Older versions
# may not support 8-byte OIDs, so skip in this case.
my $big_next_oid = '4295067296'; # 2^32 + 100000
if (!defined($ENV{oldinstall}))
{
command_ok(
[
'pg_resetwal',
'--next-oid' => $big_next_oid,
$oldnode->data_dir
],
'set an 8-byte OID counter in the old instance');
}
# --check command works here, cleans up pg_upgrade_output.d.
command_ok(
[
@@ -624,6 +638,17 @@ ok( !-d $newnode->data_dir . "/pg_upgrade_output.d",
$newnode->start;
# The 8-byte OID has been carried.
if (!defined($ENV{oldinstall}))
{
is( $newnode->safe_psql(
'postgres',
"SELECT next_oid >= '$big_next_oid'::oid8 FROM pg_control_checkpoint()"
),
't',
'8-byte OID counter is carried over by pg_upgrade');
}
# Check if there are any logs coming from pg_upgrade, that would only be
# retained on failure.
my $log_path = $newnode->data_dir . "/pg_upgrade_output.d";

8
src/include/access/transam.h
View File

@@ -186,8 +186,9 @@ FullTransactionIdAdvance(FullTransactionId *dest)
* more painful, however, since some user-defined OIDs will appear in
* on-disk data; such a change would probably break pg_upgrade.
*
* NOTE: if the OID generator wraps around, we skip over OIDs 0-16383
* and resume with 16384. This minimizes the odds of OID conflict, by not
* NOTE: the OID generator is 8 bytes wide (see GetNewObjectId8()). For
* its lower 4 bytes, we skip over OIDs 0-16383 and resume with 16384.
* This minimizes the odds of OID conflict when using GetNewObjectId(), by not
* reassigning OIDs that might have been assigned during initdb. Critically,
* it also ensures that no user-created object will be considered pinned.
* ----------
@@ -211,7 +212,7 @@ typedef struct TransamVariablesData
/*
* These fields are protected by OidGenLock.
*/
Oid nextOid; /* next OID to assign */
Oid8 nextOid; /* next OID (8 bytes) to assign */
uint32 oidCount; /* OIDs available before must do XLOG work */
/*
@@ -353,6 +354,7 @@ extern void SetTransactionIdLimit(TransactionId oldest_datfrozenxid,
extern void AdvanceOldestClogXid(TransactionId oldest_datfrozenxid);
extern bool ForceTransactionIdLimitUpdate(void);
extern Oid GetNewObjectId(void);
extern Oid8 GetNewObjectId8(void);
extern void StopGeneratingPinnedObjectIds(void);
#ifdef USE_ASSERT_CHECKING

2
src/include/access/xlog.h
View File

@@ -284,7 +284,7 @@ extern void ShutdownXLOG(int code, Datum arg);
extern bool CreateCheckPoint(int flags);
extern bool CreateRestartPoint(int flags);
extern WALAvailability GetWALAvailability(XLogRecPtr targetLSN);
extern void XLogPutNextOid(Oid nextOid);
extern void XLogPutNextOid(Oid8 nextOid);
extern XLogRecPtr XLogRestorePoint(const char *rpName);
extern XLogRecPtr XLogAssignLSN(void);
extern void UpdateFullPageWrites(void);

2
src/include/access/xlog_internal.h
View File

@@ -32,7 +32,7 @@
/*
* Each page of XLOG file has a header like this:
*/
#define XLOG_PAGE_MAGIC 0xD121 /* can be used as WAL version indicator */
#define XLOG_PAGE_MAGIC 0xD122 /* can be used as WAL version indicator */
typedef struct XLogPageHeaderData
{

2
src/include/catalog/catversion.h
View File

@@ -57,6 +57,6 @@
*/
/* yyyymmddN */
#define CATALOG_VERSION_NO 202609072
#define CATALOG_VERSION_NO 202609101
#endif

4
src/include/catalog/pg_control.h
View File

@@ -22,7 +22,7 @@
/* Version identifier for this pg_control format */
#define PG_CONTROL_VERSION 1903
#define PG_CONTROL_VERSION 2000
/* Nonce key length, see below */
#define MOCK_AUTH_NONCE_LEN 32
@@ -43,7 +43,7 @@ typedef struct CheckPoint
int wal_level; /* current wal_level */
bool logicalDecodingEnabled; /* current logical decoding status */
FullTransactionId nextXid; /* next free transaction ID */
Oid nextOid; /* next free OID */
Oid8 nextOid; /* next free OID */
MultiXactId nextMulti; /* next free MultiXactId */
MultiXactOffset nextMultiOffset; /* next free MultiXact offset */
TransactionId oldestXid; /* cluster-wide minimum datfrozenxid */

2
src/include/catalog/pg_proc.dat
View File

@@ -12429,7 +12429,7 @@
descr => 'pg_controldata checkpoint state information as a function',
proname => 'pg_control_checkpoint', provolatile => 'v',
prorettype => 'record', proargtypes => '',
proallargtypes => '{pg_lsn,pg_lsn,text,int4,int4,bool,bool,text,oid,xid,int8,xid,oid,xid,xid,oid,xid,xid,int4,timestamptz}',
proallargtypes => '{pg_lsn,pg_lsn,text,int4,int4,bool,bool,text,oid8,xid,int8,xid,oid,xid,xid,oid,xid,xid,int4,timestamptz}',
proargmodes => '{o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}',
proargnames => '{checkpoint_lsn,redo_lsn,redo_wal_file,timeline_id,prev_timeline_id,full_page_writes,logical_decoding,next_xid,next_oid,next_multixact_id,next_multi_offset,oldest_xid,oldest_xid_dbid,oldest_active_xid,oldest_multi_xid,oldest_multi_dbid,oldest_commit_ts_xid,newest_commit_ts_xid,data_page_checksum_version,checkpoint_time}',
prosrc => 'pg_control_checkpoint' },