database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2026-10-23 13:29:09 +03:00
Code Activity

Fix VM clear WAL logging by registering VM blocks

Browse Source 
Heap WAL records that clear bits on the visibility map (like inserts and
deletes) did not register the visibility map blocks they modified.
Because the WAL summarizer only records registered blocks, an
incremental backup taken over such operations would omit the changed VM
pages. On restore, the VM would retain stale all-visible/all-frozen
bits, which can cause wrong results from index-only scans and incorrect
relfrozenxid advancement due to vacuum page skipping.

Not registering the VM buffer also meant we never emitted FPIs of VM
pages when clearing bits. A torn VM page won't raise an error because
the VM is read with ZERO_ON_ERROR; with checksums on, it would be
detected and zeroed, but with checksums off, it is accepted as-is and
can lead to data corruption.

Fix this by registering the VM buffer in the WAL record when clearing VM
bits. The VM buffer must now be locked throughout the critical section
that modifies the VM and heap pages and emits the WAL record. This can
slow down operations that clear the VM, since the VM lock is held longer
and VM FPIs may be emitted, but it is required for correctness.

Note that this fix does not repair existing incremental backups.

Author: Melanie Plageman <melanieplageman@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Reviewed-by: Robert Haas <robertmhaas@gmail.com>
Reviewed-by: Andrey Borodin <x4mmm@yandex-team.ru>
Discussion: https://postgr.es/m/flat/CAAKRu_bn%2Be7F4yPFBgFbnP%2BsyJRKyNK092bjD2LKvZW7O4Svag>
Backpatch-through: 17
This commit is contained in:
Melanie Plageman
2026-07-15 17:43:38 -04:00
parent 7edec8b578
commit c0d9864f5c
6 changed files with 596 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
contrib/pg_surgery/heap_surgery.c
View File

@@ -17,6 +17,7 @@
#include "access/xloginsert.h"
#include "catalog/pg_am_d.h"
#include "catalog/pg_proc_d.h"
#include "catalog/pg_control.h"
#include "miscadmin.h"
#include "storage/bufmgr.h"
#include "utils/acl.h"
@@ -143,6 +144,7 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
{
Buffer buf;
Buffer vmbuf = InvalidBuffer;
bool unlock_vmbuf = false;
Page page;
BlockNumber blkno;
OffsetNumber curoff;
@@ -230,11 +232,15 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
}
/*
* Before entering the critical section, pin the visibility map page
* if it appears to be necessary.
* Before entering the critical section, pin and lock the visibility
* map page if it appears to be necessary.
*/
if (heap_force_opt == HEAP_FORCE_KILL && PageIsAllVisible(page))
{
visibilitymap_pin(rel, blkno, &vmbuf);
LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
unlock_vmbuf = true;
}
/* No ereport(ERROR) from here until all the changes are logged. */
START_CRIT_SECTION();
@@ -263,10 +269,11 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
*/
if (PageIsAllVisible(page))
{
if (visibilitymap_clear_locked(rel, blkno, vmbuf,
VISIBILITYMAP_VALID_BITS))
did_modify_vm = true;
PageClearAllVisible(page);
visibilitymap_clear(rel, blkno, vmbuf,
VISIBILITYMAP_VALID_BITS);
did_modify_vm = true;
}
}
else
@@ -317,18 +324,29 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt)
/* XLOG stuff */
if (RelationNeedsWAL(rel))
log_newpage_buffer(buf, true);
}
{
XLogRecPtr recptr;
/* WAL log the VM page if it was modified. */
if (did_modify_vm && RelationNeedsWAL(rel))
log_newpage_buffer(vmbuf, false);
XLogBeginInsert();
XLogRegisterBuffer(0, buf, REGBUF_STANDARD | REGBUF_FORCE_IMAGE);
/* Include the VM page if it was modified. */
if (did_modify_vm)
XLogRegisterBuffer(1, vmbuf, REGBUF_FORCE_IMAGE);
recptr = XLogInsert(RM_XLOG_ID, XLOG_FPI);
if (did_modify_vm)
PageSetLSN(BufferGetPage(vmbuf), recptr);
PageSetLSN(BufferGetPage(buf), recptr);
}
}
END_CRIT_SECTION();
UnlockReleaseBuffer(buf);
if (vmbuf != InvalidBuffer)
if (unlock_vmbuf)
LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
if (BufferIsValid(vmbuf))
ReleaseBuffer(vmbuf);
/* Update the current_start_ptr before moving to the next page. */

650
src/backend/access/heap/heapam.c
View File

File diff suppressed because it is too large Load Diff

34
src/backend/access/heap/visibilitymap.c
View File

@@ -135,9 +135,38 @@ static Buffer vm_extend(Relation rel, BlockNumber vm_nblocks);
* You must pass a buffer containing the correct map page to this function.
* Call visibilitymap_pin first to pin the right one. This function doesn't do
* any I/O. Returns true if any bits have been cleared and false otherwise.
*
* Most callers should use visibilitymap_clear_locked rather than this
* function. It is usually necessary to register the VM buffer in the WAL
* record, and this necessitates holding the lock for longer than it is held
* here. However, we retain this function for behavioral compatibility on the
* back branches (out-of-tree callers may expect it to take the lock). And,
* since we have it, we use it for in-tree callers that don't have to manage
* the VM buffer lock themselves.
*/
bool
visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
{
bool cleared = false;
LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
cleared = visibilitymap_clear_locked(rel, heapBlk, vmbuf, flags);
LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
return cleared;
}
/*
* Clear specified bits, caller holds VM lock
*
* Like visibilitymap_clear(), except the caller must already hold the VM
* buffer exclusive lock and is responsible for unlocking it.
* Returns true if any bits were actually cleared, false otherwise.
*/
bool
visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags)
{
BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk);
int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk);
@@ -157,7 +186,8 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
if (!BufferIsValid(vmbuf) || BufferGetBlockNumber(vmbuf) != mapBlock)
elog(ERROR, "wrong buffer passed to visibilitymap_clear");
LockBuffer(vmbuf, BUFFER_LOCK_EXCLUSIVE);
Assert(BufferIsLocal(vmbuf) || BufferIsExclusiveLocked(vmbuf));
map = PageGetContents(BufferGetPage(vmbuf));
if (map[mapByte] & mask)
@@ -168,8 +198,6 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf, uint8 flags
cleared = true;
}
LockBuffer(vmbuf, BUFFER_LOCK_UNLOCK);
return cleared;
}

7
src/bin/pg_walsummary/t/002_blocks.pl
View File

@@ -85,13 +85,14 @@ my $filename = sprintf "%s/pg_wal/summaries/%08s%08s%08s%08s%08s.summary",
split(m@/@, $end_lsn);
ok(-f $filename, "WAL summary file exists");
# Run pg_walsummary on it. We expect exactly two blocks to be modified,
# block 0 and one other.
# Run pg_walsummary on it. We expect exactly three blocks to be modified,
# block 0 (old tuple), another block (new tuple), and the block for the VM.
my ($stdout, $stderr) = run_command([ 'pg_walsummary', '-i', $filename ]);
note($stdout);
@lines = split(/\n/, $stdout);
like($stdout, qr/FORK main: block 0$/m, "stdout shows block 0 modified");
like($stdout, qr/FORK vm: block 0$/m, "stdout shows VM block 0 modified");
is($stderr, '', 'stderr is empty');
is(0 + @lines, 2, "UPDATE modified 2 blocks");
is(0 + @lines, 3, "UPDATE modified 3 blocks");
done_testing();

18
src/include/access/heapam_xlog.h
View File

@@ -110,6 +110,7 @@
/* This is what we need to know about delete */
#define HEAP_DELETE_BLKREF_HEAP 0
#define HEAP_DELETE_BLKREF_VM 1
typedef struct xl_heap_delete
{
@@ -159,6 +160,7 @@ typedef struct xl_heap_header
/* This is what we need to know about insert */
#define HEAP_INSERT_BLKREF_HEAP 0
#define HEAP_INSERT_BLKREF_VM 1
typedef struct xl_heap_insert
{
@@ -178,6 +180,7 @@ typedef struct xl_heap_insert
* (XLOG_HEAP_INIT_PAGE).
*/
#define HEAP_MULTI_INSERT_BLKREF_HEAP 0
#define HEAP_MULTI_INSERT_BLKREF_VM 1
/*
* In HEAP_MULTI_INSERT_BLKREF_HEAP's data portion, there is an
@@ -221,10 +224,22 @@ typedef struct xl_multi_insert_tuple
*
* HEAP_UPDATE_BLKREF_HEAP_OLD: old page, if different. (no data, just a reference
* to the block)
*
* HEAP_UPDATE_BLKREF_VM_NEW: VM page covering the new heap page. Registered
* when XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED is set and the new heap page's VM bit
* was actually cleared. Also covers the old heap page's VM bits when both heap
* pages map to the same VM page and both blocks' VM bits were actually cleared.
*
* HEAP_UPDATE_BLKREF_VM_OLD: VM page covering the old heap page. Only
* registered when XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED is set and the old heap
* page's VM bits were actually cleared. Also only registered when the old heap
* page's VM bits are on a different VM page than the new heap page's or they
* are on the same VM page and only the old block's VM bits are cleared.
*/
#define HEAP_UPDATE_BLKREF_HEAP_NEW 0
#define HEAP_UPDATE_BLKREF_HEAP_OLD 1
#define HEAP_UPDATE_BLKREF_VM_NEW 2
#define HEAP_UPDATE_BLKREF_VM_OLD 3
typedef struct xl_heap_update
{
@@ -405,6 +420,7 @@ typedef struct xlhp_prune_items
/* This is what we need to know about lock */
#define HEAP_LOCK_BLKREF_HEAP 0
#define HEAP_LOCK_BLKREF_VM 1
typedef struct xl_heap_lock
{

2
src/include/access/visibilitymap.h
View File

@@ -26,6 +26,8 @@
#define VM_ALL_FROZEN(r, b, v) \
((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0)
extern bool visibilitymap_clear_locked(Relation rel, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
extern bool visibilitymap_clear(Relation rel, BlockNumber heapBlk,
Buffer vmbuf, uint8 flags);
extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk,