diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index fecee08b0a5..3117968d125 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -788,6 +788,12 @@ PostgreSQL documentation in the pg_stat_database view. + + Checksums must be enabled on the server for the duration of the base + backup in order to be verified. If checksums are in the process of + being enabled when the base backup starts then checksum verification is + disabled for the base backup. + diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 0161d045fba..7f5d3b1417a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -560,6 +560,14 @@ typedef struct XLogCtlData uint32 data_checksum_version; slock_t info_lck; /* locks shared variables shown above */ + + /* + * lastChecksumChangeRecPtr points to the end of the last XLOG2_CHECKSUMS + * record inserted or replayed, i.e. the last change of + * data_checksum_version. InvalidXLogRecPtr if the state hasn't changed + * since the server started. + */ + pg_atomic_uint64 lastChecksumChangeRecPtr; } XLogCtlData; /* @@ -4736,6 +4744,23 @@ DataChecksumsNeedVerify(void) return (LocalDataChecksumState == PG_DATA_CHECKSUM_VERSION); } +/* + * GetLastChecksumChangeRecPtr + * Returns the location of the last data checksum state change + * + * Offline state changes by pg_checksums leave no trace here; callers must + * also inspect the current state. + * + * No barrier semantics are needed: pages reach disk under a new checksum + * state only after their writer absorbed the procsignal barrier for the + * change, which is emitted after the new location became visible. + */ +XLogRecPtr +GetLastChecksumChangeRecPtr(void) +{ + return pg_atomic_read_u64(&XLogCtl->lastChecksumChangeRecPtr); +} + /* * SetDataChecksumsOnInProgress * Sets the data checksum state to "inprogress-on" to enable checksums @@ -4846,6 +4871,8 @@ SetDataChecksumsOn(void) MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; END_CRIT_SECTION(); + INJECTION_POINT("datachecksums-on-before-checkpoint", NULL); + RequestCheckpoint(CHECKPOINT_FORCE | CHECKPOINT_WAIT | CHECKPOINT_FAST); WaitForProcSignalBarrier(barrier); } @@ -5436,6 +5463,7 @@ XLOGShmemInit(void *arg) pg_atomic_init_u64(&XLogCtl->logWriteResult, InvalidXLogRecPtr); pg_atomic_init_u64(&XLogCtl->logFlushResult, InvalidXLogRecPtr); pg_atomic_init_u64(&XLogCtl->unloggedLSN, InvalidXLogRecPtr); + pg_atomic_init_u64(&XLogCtl->lastChecksumChangeRecPtr, InvalidXLogRecPtr); } /* @@ -8743,6 +8771,7 @@ XLogChecksums(uint32 new_type) XLogRegisterData((char *) &xlrec, sizeof(xl_checksum_state)); recptr = XLogInsert(RM_XLOG2_ID, XLOG2_CHECKSUMS); + pg_atomic_write_u64(&XLogCtl->lastChecksumChangeRecPtr, recptr); XLogFlush(recptr); } @@ -9244,15 +9273,44 @@ xlog2_redo(XLogReaderState *record) if (info == XLOG2_CHECKSUMS) { xl_checksum_state state; + XLogRecPtr lsn = record->EndRecPtr; memcpy(&state, XLogRecGetData(record), sizeof(xl_checksum_state)); + /* advertise the location before the new state becomes visible */ + pg_atomic_write_u64(&XLogCtl->lastChecksumChangeRecPtr, lsn); + SpinLockAcquire(&XLogCtl->info_lck); XLogCtl->data_checksum_version = state.new_checksum_state; SpinLockRelease(&XLogCtl->info_lck); LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); ControlFile->data_checksum_version = state.new_checksum_state; + + /* + * Update minRecoveryPoint to ensure that if recovery is aborted, we + * recover back up to this point before allowing hot standby again. + * The new state is durable in pg_control while its location is only + * tracked in shared memory; a standby becoming consistent below this + * record would let base backups resume checksum verification with the + * location unknown. The local copies cannot be updated as long as + * crash recovery is happening and we expect all the WAL to be + * replayed. + */ + if (InArchiveRecovery) + { + LocalMinRecoveryPoint = ControlFile->minRecoveryPoint; + LocalMinRecoveryPointTLI = ControlFile->minRecoveryPointTLI; + } + if (XLogRecPtrIsValid(LocalMinRecoveryPoint) && LocalMinRecoveryPoint < lsn) + { + TimeLineID replayTLI; + + (void) GetCurrentReplayRecPtr(&replayTLI); + ControlFile->minRecoveryPoint = lsn; + ControlFile->minRecoveryPointTLI = replayTLI; + } + UpdateControlFile(); LWLockRelease(ControlFileLock); diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 5214e7b99c7..fe89b812bed 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -45,6 +45,7 @@ #include "storage/reinit.h" #include "utils/builtins.h" #include "utils/guc.h" +#include "utils/injection_point.h" #include "utils/ps_status.h" #include "utils/relcache.h" #include "utils/resowner.h" @@ -107,6 +108,7 @@ static off_t read_file_data_into_buffer(bbsink *sink, int *checksum_failures); static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx, size_t *bytes_done, void *data, size_t length); +static bool backup_checksums_verifiable(XLogRecPtr start_lsn); static bool verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno, uint16 *expected_checksum); @@ -325,6 +327,12 @@ perform_base_backup(basebackup_options *opt, bbsink *sink, /* notify basebackup sink about start of backup */ bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH); + /* + * Allow tests to hold the backup after the starting checkpoint but + * before any file data is sent. + */ + INJECTION_POINT("basebackup-before-send-files", NULL); + /* Send off our tablespaces one by one */ foreach(lc, state.tablespaces) { @@ -1609,13 +1617,14 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, Assert((sink->bbs_buffer_length % BLCKSZ) == 0); /* - * If we weren't told not to verify checksums, and if checksums are - * enabled for this cluster, and if this is a relation file, then verify - * the checksum. We cannot at this point check if checksums are enabled - * or disabled as that might change, thus we check at each point where we + * Verify checksums unless the client requested otherwise, but only for + * relation files, and only while checksums have been continuously enabled + * since the checkpoint this backup started from. Checksums can still be + * disabled while the backup runs, thus we check at each point where we * could be validating a checksum. */ - if (!noverify_checksums && RelFileNumberIsValid(relfilenumber)) + if (!noverify_checksums && RelFileNumberIsValid(relfilenumber) && + backup_checksums_verifiable(sink->bbs_state->startptr)) verify_checksum = true; /* @@ -1748,7 +1757,9 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, * If the amount of data we were able to read was not a multiple of * BLCKSZ, we cannot verify checksums, which are block-level. */ - if (verify_checksum && DataChecksumsNeedVerify() && (cnt % BLCKSZ != 0)) + if (verify_checksum && + backup_checksums_verifiable(sink->bbs_state->startptr) && + (cnt % BLCKSZ != 0)) { ereport(WARNING, (errmsg("could not verify checksum in file \"%s\", block " @@ -1876,7 +1887,7 @@ read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd, * The data checksum state can change at any point, so we need to * re-check before each page. */ - if (!DataChecksumsNeedVerify()) + if (!backup_checksums_verifiable(sink->bbs_state->startptr)) return cnt; page = sink->bbs_buffer + BLCKSZ * i; @@ -1905,7 +1916,7 @@ read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd, * The data checksum state may also have changed concurrently so check * again. */ - if (!DataChecksumsNeedVerify()) + if (!backup_checksums_verifiable(sink->bbs_state->startptr)) return cnt; reread_cnt = basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i, @@ -1996,6 +2007,27 @@ push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx, } } +/* + * Check whether data checksums can be verified for a backup started at + * start_lsn. + * + * Checksums are verified only while they have been continuously enabled + * since the checkpoint the backup started from: the state must be "on" and + * the last state change must predate the backup start. Such a checkpoint + * guarantees that every page flushed before it has a checksum written. Any + * later state change ends verification for the rest of the backup: pages + * written while checksums were off can lack checksums yet keep LSNs older + * than the backup start, and re-enabling completes before the rewritten + * pages are flushed, so observing the "on" state again is not enough to + * resume. + */ +static bool +backup_checksums_verifiable(XLogRecPtr start_lsn) +{ + return DataChecksumsNeedVerify() && + GetLastChecksumChangeRecPtr() <= start_lsn; +} + /* * Try to verify the checksum for the provided page, if it seems appropriate * to do so. @@ -2021,12 +2053,35 @@ verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno, if (PageIsNew(page) || PageGetLSN(page) >= start_lsn) return true; - if (!DataChecksumsNeedVerify()) + if (!backup_checksums_verifiable(start_lsn)) return true; /* Perform the actual checksum calculation. */ checksum = pg_checksum_page(page, blkno); +#ifdef USE_INJECTION_POINTS + { + /* + * Make it possible to test checksum verification failure without + * having to destroy data on disk. There is cap on how many times we + * want to cause verification failure to make tests more interesting + * and less log intensive. This makes it easy to test pg_basebackup + * with the command_checks_all test function. + */ + static int hit = 0; + + if (IS_INJECTION_POINT_ATTACHED("basebackup-fail-checksum-verification")) + { + if (hit++ < 5) + { + checksum = 0; + INJECTION_POINT_CACHED("basebackup-fail-checksum-verification", + NULL); + } + } + } +#endif + /* See whether it matches the value from the page. */ phdr = (PageHeader) page; if (phdr->pd_checksum == checksum) diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 338d68d7424..130ba929109 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -262,6 +262,7 @@ extern uint64 GetSystemIdentifier(void); extern char *GetMockAuthenticationNonce(void); extern bool DataChecksumsNeedWrite(void); extern bool DataChecksumsNeedVerify(void); +extern XLogRecPtr GetLastChecksumChangeRecPtr(void); extern bool DataChecksumsOn(void); extern bool DataChecksumsOff(void); extern bool DataChecksumsInProgressOn(void); diff --git a/src/test/modules/test_checksums/meson.build b/src/test/modules/test_checksums/meson.build index 9b1421a9b91..fb7129d796f 100644 --- a/src/test/modules/test_checksums/meson.build +++ b/src/test/modules/test_checksums/meson.build @@ -33,6 +33,8 @@ tests += { 't/007_pgbench_standby.pl', 't/008_pitr.pl', 't/009_fpi.pl', + 't/010_backup_straddle.pl', + 't/011_standby_straddle.pl', ], }, } diff --git a/src/test/modules/test_checksums/t/010_backup_straddle.pl b/src/test/modules/test_checksums/t/010_backup_straddle.pl new file mode 100644 index 00000000000..aeeb3c8ccb0 --- /dev/null +++ b/src/test/modules/test_checksums/t/010_backup_straddle.pl @@ -0,0 +1,260 @@ + +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test base backups running while the data checksum state changes. The +# transition to "on" happens before the checkpoint which flushes the rewritten +# pages, so a backup straddling it reads on-disk pages which legitimately lack +# checksums and carry LSNs older than the backup start. The same applies to +# checksums being disabled and re-enabled while a backup runs: hint bit +# updates made while checksums were off reach disk without a checksum update +# and without moving the page LSN, so once the re-enabling completes the +# backup would resume verification and misjudge those pages until the +# rewritten versions are flushed. +# +# Both scenarios hold the two sides with injection points: the backup after +# its starting checkpoint but before it sends any file data, and the enabling +# after the state changed to "on" but before the checkpoint which flushes the +# rewritten pages. The backup is thus guaranteed to read the stale on-disk +# pages after the state change, without any timing assumptions. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use File::Path qw(rmtree); +use Test::More; +use IPC::Run; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +# This test suite is potentially expensive due to it requiring an increased +# shared_buffers setting. It requires the "checksum" PG_TEST_EXTRA setting to +# not cause false positives on slow or constrained systems. +if ($ENV{PG_TEST_EXTRA}) +{ + plan skip_all => 'Expensive data checksums test disabled' + unless ($ENV{PG_TEST_EXTRA} =~ /\bchecksum(_extended)?\b/); +} +else +{ + plan skip_all => 'Expensive data checksums test disabled'; +} + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('backup_node'); +$node->init(no_data_checksums => 1, allows_streaming => 1); +# The pages rewritten while enabling must stay dirty in shared buffers until +# the final checkpoint, otherwise they reach disk with checksums on their own +# and nothing is left to misjudge. The background writer must not flush them +# behind our back, and shared_buffers must exceed four times the table size +# so that the scan below does not go through a ring buffer. Autovacuum is +# disabled so that nothing sets hint bits behind our back, and wal_log_hints +# (implied by allows_streaming) must be off so that setting them does not +# move the page LSNs past the backup start. +$node->append_conf('postgresql.conf', 'shared_buffers = 32MB'); +$node->append_conf('postgresql.conf', 'bgwriter_lru_maxpages = 0'); +$node->append_conf('postgresql.conf', 'autovacuum = off'); +$node->append_conf('postgresql.conf', 'wal_log_hints = off'); +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); + +# A body of relation pages for the backup to misjudge. The scan pulls the +# table into shared buffers so that enabling doesn't read it through a ring +# buffer, which would write the pages back out. +$node->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,100000) AS a;"); +$node->safe_psql('postgres', "SELECT count(*) FROM t;"); +test_checksum_state($node, 'off'); + +$node->safe_psql('postgres', + "SELECT injection_points_attach('basebackup-before-send-files','wait');"); +$node->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); + +my $backupdir = $node->backup_dir . '/straddle'; +my ($out, $err) = ('', ''); +my $backup = IPC::Run::start( + [ + 'pg_basebackup', '-D', + $backupdir, '--wal-method=none', + '--no-sync', '--checkpoint=fast', + '-d', $node->connstr('postgres') + ], + '>', + \$out, + '2>', + \$err, + IPC::Run::timeout(180)); + +$node->wait_for_event('walsender', 'basebackup-before-send-files'); + +# Enable checksums while the backup is held, then release the backup once the +# enabling has reached the "on" state and is held before its checkpoint. +enable_data_checksums($node); +$node->wait_for_event('datachecksums launcher', + 'datachecksums-on-before-checkpoint'); + +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('basebackup-before-send-files');"); +$node->safe_psql('postgres', + "SELECT injection_points_detach('basebackup-before-send-files');"); + +ok($backup->finish, 'backup straddling enable completion succeeds') + or diag("stderr: $err"); + +# The backup must not even mention checksums: it must skip verification +# entirely, including the warning-only path for short reads. +unlike($err, qr/checksum/, 'straddling backup does not verify checksums'); + +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('datachecksums-on-before-checkpoint');"); +$node->safe_psql('postgres', + "SELECT injection_points_detach('datachecksums-on-before-checkpoint');"); + +wait_for_checksum_state($node, 'on'); +$node->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_catalog.pg_stat_activity " + . "WHERE backend_type = 'datachecksums launcher';"); + +# A backup started once enabling has completed must verify, and pass +$node->command_ok( + [ + 'pg_basebackup', '-D', + $node->backup_dir . '/after_enable', '--wal-method=none', + '--no-sync', '--checkpoint=fast' + ], + 'backup after enable completion succeeds'); +rmtree($node->backup_dir . '/after_enable'); + +# Now test a backup which straddles checksums being disabled and re-enabled. +# Recreate the table since the earlier scan set its hint bits and the rewrite +# gave the pages checksums; the new contents are not read here, leaving the +# hint bits unset until checksums are off. +$node->safe_psql('postgres', "DROP TABLE t;"); +$node->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,100000) AS a;"); + +$node->safe_psql('postgres', + "SELECT injection_points_attach('basebackup-before-send-files','wait');"); + +$backupdir = $node->backup_dir . '/onoffon'; +($out, $err) = ('', ''); +$backup = IPC::Run::start( + [ + 'pg_basebackup', '-D', + $backupdir, '--wal-method=none', + '--no-sync', '--checkpoint=fast', + '-d', $node->connstr('postgres') + ], + '>', + \$out, + '2>', + \$err, + IPC::Run::timeout(180)); + +$node->wait_for_event('walsender', 'basebackup-before-send-files'); + +disable_data_checksums($node, wait => 1); + +# With checksums off, the scan sets hint bits without WAL logging them, and +# the checkpoint flushes the modified pages without updating their checksums. +# The on-disk pages now carry stale checksums and LSNs older than the backup +# start. +$node->safe_psql('postgres', "SELECT count(*) FROM t;"); +$node->safe_psql('postgres', "CHECKPOINT;"); + +$node->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); + +enable_data_checksums($node); +$node->wait_for_event('datachecksums launcher', + 'datachecksums-on-before-checkpoint'); + +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('basebackup-before-send-files');"); +$node->safe_psql('postgres', + "SELECT injection_points_detach('basebackup-before-send-files');"); + +ok($backup->finish, 'backup straddling disable and re-enable succeeds') + or diag("stderr: $err"); +unlike($err, qr/checksum/, + 'backup straddling disable and re-enable does not verify checksums'); + +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('datachecksums-on-before-checkpoint');"); +$node->safe_psql('postgres', + "SELECT injection_points_detach('datachecksums-on-before-checkpoint');"); + +wait_for_checksum_state($node, 'on'); +$node->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_catalog.pg_stat_activity " + . "WHERE backend_type = 'datachecksums launcher';"); +rmtree($backupdir); + +# A backup started once re-enabling has completed must verify, and pass +$node->command_ok( + [ + 'pg_basebackup', '-D', + $node->backup_dir . '/after_onoffon', '--wal-method=none', + '--no-sync', '--checkpoint=fast' + ], + 'backup after re-enable completion succeeds'); +rmtree($node->backup_dir . '/after_onoffon'); + +# Test another backup, but this time inject synthetic checksum verification +# failures into it. The regex matching the WARNING is different from the next +# test on purpose as it will have NOTICE output from the injection point as +# well. +$node->safe_psql('postgres', + "SELECT injection_points_attach('basebackup-fail-checksum-verification', 'notice');" +); +$node->command_checks_all( + [ + 'pg_basebackup', '-D', + $node->backup_dir . '/corrupt', '--wal-method=none', + '--no-sync', '--checkpoint=fast' + ], + 1, + [qr{^$}], + [qr/WARNING.*checksum verification failed/s], + 'pg_basebackup reports checksum mismatch'); +rmtree($node->backup_dir . '/corrupt'); +$node->safe_psql('postgres', + "SELECT injection_points_detach('basebackup-fail-checksum-verification');" +); + +# Now corrupt data on disk and take another backup to make sure the corruption +# is reported. +my $corrupt_table = $node->safe_psql('postgres', + q{CREATE TABLE corrupt_table AS SELECT a FROM generate_series(1,10000) AS a; ALTER TABLE corrupt_table SET (autovacuum_enabled=false); SELECT pg_relation_filepath('corrupt_table')} +); + +$node->stop; +$node->corrupt_page_checksum($corrupt_table, 0); +$node->start; + +$node->command_checks_all( + [ + 'pg_basebackup', '-D', + $node->backup_dir . '/corrupt', '--wal-method=none', + '--no-sync', '--checkpoint=fast' + ], + 1, + [qr{^$}], + [qr/^WARNING.*checksum verification failed/s], + 'pg_basebackup reports checksum mismatch'); +rmtree($node->backup_dir . '/corrupt'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_checksums/t/011_standby_straddle.pl b/src/test/modules/test_checksums/t/011_standby_straddle.pl new file mode 100644 index 00000000000..e50fb65fcaf --- /dev/null +++ b/src/test/modules/test_checksums/t/011_standby_straddle.pl @@ -0,0 +1,249 @@ + +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that a standby does not become available for base backups before it +# has re-replayed the latest data checksum state change after a restart. +# +# When a standby replays XLOG2_CHECKSUMS it writes the new state to +# pg_control, but the location of the change is only tracked in shared +# memory. If the standby restarts before a restartpoint covers the record, +# and reaches consistency below it, base backups would resume checksum +# verification with the change location unknown, while the pages rewritten +# before the change may not have reached disk. The redo routine must +# therefore advance minRecoveryPoint to the record, so that consistency (and +# with it hot standby and base backups) is withheld until the change location +# is known again. +# +# The test holds the enabling on the primary between the state change and its +# final checkpoint, waits for the standby to replay the state change, and +# crashes the standby so that the rewritten pages never reach its disk. WAL +# from the enabling onwards is removed from the standby's pg_wal and +# streaming is disabled, so that replay after the restart stalls below the +# state change record. Restarted this way, the standby must refuse +# connections; once streaming is re-enabled and the record replayed again, +# base backups must succeed without spurious checksum failures. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use File::Path qw(rmtree); +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +# This test suite is potentially expensive due to it requiring an increased +# shared_buffers setting and use custom timeouts. It requires the +# "checksum_extended" PG_TEST_EXTRA setting to not cause false positives on +# slow or constrained systems. +if ($ENV{PG_TEST_EXTRA}) +{ + plan skip_all => 'Expensive data checksums test disabled' + unless ($ENV{PG_TEST_EXTRA} =~ /\bchecksum_extended\b/); +} +else +{ + plan skip_all => 'Expensive data checksums test disabled'; +} + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node_primary = PostgreSQL::Test::Cluster->new('straddle_primary'); +$node_primary->init(no_data_checksums => 1, allows_streaming => 1); +# The pages rewritten while enabling must stay dirty in shared buffers until +# the final checkpoint, and their standby copies must stay dirty as well, so +# that the on-disk pages legitimately lack checksums. wal_log_hints (implied +# by allows_streaming) is turned off to keep the page LSNs put during setup. +$node_primary->append_conf('postgresql.conf', 'shared_buffers = 128MB'); +$node_primary->append_conf('postgresql.conf', 'autovacuum = off'); +$node_primary->append_conf('postgresql.conf', 'wal_log_hints = off'); +$node_primary->start; + +$node_primary->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); + +my $slotname = 'physical_slot'; +$node_primary->safe_psql('postgres', + "SELECT pg_create_physical_replication_slot('$slotname');"); + +# A body of relation pages whose standby copies will lack checksums +$node_primary->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,100000) AS a;"); + +my $backup_name = 'straddle_backup'; +$node_primary->backup($backup_name); + +my $node_standby = PostgreSQL::Test::Cluster->new('straddle_standby'); +$node_standby->init_from_backup($node_primary, $backup_name, + has_streaming => 1); +# The background writer must not flush the replayed page rewrites behind our +# back, or nothing is left to protect and minRecoveryPoint could move on its +# own. +$node_standby->append_conf( + 'postgresql.conf', qq[ +primary_slot_name = '$slotname' +bgwriter_lru_maxpages = 0 +]); +$node_standby->start; + +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); +test_checksum_state($node_primary, 'off'); +test_checksum_state($node_standby, 'off'); + +# Pin the restartpoint the later base backups will start from: replay a +# primary checkpoint and force a restartpoint on it. No further checkpoint +# record reaches the standby until the enabling is released, so this remains +# the standby's backup starting checkpoint throughout. +$node_primary->safe_psql('postgres', 'CHECKPOINT;'); +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); +$node_standby->safe_psql('postgres', 'CHECKPOINT;'); + +# Put everything the enabling writes into fresh WAL segments, so that the +# standby's copies of them can be removed later, and remember where the +# enabling era begins. +$node_primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); + +# Enable checksums, holding the launcher after the state change but before +# the final checkpoint, so the rewritten pages stay dirty everywhere. +$node_primary->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-enable-checksums-delay', 'wait');" +); +$node_primary->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); +enable_data_checksums($node_primary); + +$node_primary->wait_for_event('datachecksums launcher', + 'datachecksums-enable-checksums-delay'); +my $enable_start_lsn = + $node_primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();'); +my $enable_start_seg = $node_primary->safe_psql('postgres', + "SELECT pg_walfile_name('$enable_start_lsn');"); +$node_primary->safe_psql('postgres', + "SELECT injection_points_wakeup('datachecksums-enable-checksums-delay');" +); + +# Immediately start to wait for the next event and hold off on detaching the +# previous injection point till later to avoid delays and risk missing the +# wait event +$node_primary->wait_for_event('datachecksums launcher', + 'datachecksums-on-before-checkpoint'); +# Detach the injection point now that we have some more time +$node_primary->safe_psql('postgres', + "SELECT injection_points_detach('datachecksums-enable-checksums-delay');" +); + +# The standby has now replayed the state change: its pg_control says "on" +# while the rewritten pages are only dirty in its shared buffers. +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); +wait_for_checksum_state($node_standby, 'on'); + +# Crash the standby, losing the dirty rewritten pages. +$node_standby->stop('immediate'); + +# The state change record must have dragged minRecoveryPoint along with it, +# otherwise the standby can become consistent below the record after the +# restart. +my ($stdout, $stderr) = + run_command([ 'pg_controldata', $node_standby->data_dir ]); +my ($min_recovery) = + $stdout =~ /Minimum recovery ending location:\s*([0-9A-F]+\/[0-9A-F]+)/; +die "could not parse pg_controldata output" unless defined $min_recovery; + +my $result = $node_primary->safe_psql('postgres', + "SELECT '$min_recovery'::pg_lsn > '$enable_start_lsn'::pg_lsn;"); +is($result, 't', 'minRecoveryPoint advanced past the checksum state change'); + +# Remove the enabling-era WAL from the standby and cut it off from the +# primary, so that replay after the restart stalls below the state change. +# The replication slot retains the removed segments on the primary. +my $wal_dir = $node_standby->data_dir . '/pg_wal'; +opendir(my $dh, $wal_dir) or die "could not open $wal_dir: $!"; +foreach my $segment (readdir($dh)) +{ + next unless $segment =~ /^[0-9A-F]{24}$/; + next unless $segment ge $enable_start_seg; + unlink("$wal_dir/$segment") + or die "could not unlink $wal_dir/$segment: $!"; +} +closedir($dh); +$node_standby->append_conf('postgresql.conf', "primary_conninfo = ''"); + +# The standby must not reach consistency until it has re-replayed the state +# change, so startup must not complete within the timeout. Reaching hot +# standby below the record would make pg_ctl return success here. +my $started; +{ + local $ENV{PGCTLTIMEOUT} = 10; + $started = $node_standby->start(fail_ok => 1); +} +is($started, 0, + 'standby withholds consistency until the state change is replayed again'); + +my ($ret, $out, $err) = $node_standby->psql('postgres', 'SELECT 1;'); +isnt($ret, 0, 'standby refuses connections while below the state change'); + +# Reconnect the standby; streaming provides the removed WAL again, replay +# passes the state change and the standby becomes consistent. +$node_standby->enable_streaming($node_primary); +$node_standby->reload; +$node_standby->poll_query_until('postgres', 'SELECT true;'); +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); + +# The rewritten pages are again only dirty in shared buffers, so the on-disk +# pages still lack checksums. A base backup must skip verification entirely +# and pass without mentioning checksums on stderr. Standby backups always +# print a NOTICE about WAL archiving, so stderr is not empty. +$node_standby->command_checks_all( + [ + 'pg_basebackup', '-D', + $node_standby->backup_dir . '/underway', '--wal-method=none', + '--no-sync', '--checkpoint=fast' + ], + 0, + [qr{^$}], + [qr{^(?!.*checksum)}s], + 'backup from standby while enabling is underway succeeds'); +rmtree($node_standby->backup_dir . '/underway'); + +# Release the enabling; its final checkpoint flushes the rewritten pages. +$node_primary->safe_psql('postgres', + "SELECT injection_points_wakeup('datachecksums-on-before-checkpoint');"); +$node_primary->safe_psql('postgres', + "SELECT injection_points_detach('datachecksums-on-before-checkpoint');"); +wait_for_checksum_state($node_primary, 'on'); +$node_primary->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_catalog.pg_stat_activity " + . "WHERE backend_type = 'datachecksums launcher';"); + +# A restartpoint on the final checkpoint lets verification resume, and a +# backup started from it must again pass. +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); +$node_standby->safe_psql('postgres', 'CHECKPOINT;'); + +$node_standby->command_checks_all( + [ + 'pg_basebackup', '-D', + $node_standby->backup_dir . '/after_enable', '--wal-method=none', + '--no-sync', '--checkpoint=fast' + ], + 0, + [qr{^$}], + [qr{^(?!.*checksum)}s], + 'backup from standby after enable completion succeeds'); +rmtree($node_standby->backup_dir . '/after_enable'); + +$node_standby->stop; +$node_primary->stop; +done_testing();