diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index b95e3868e9d..ac3d0c9328f 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -124,6 +124,20 @@ PostgreSQL documentation on, but is enabled by default. + + The data checksum states of the source and target server must be + compatible. pg_rewind refuses to run if an + online checksum state transition was interrupted on either server, or + if data checksums are enabled on the target server, or were enabled at + the point of divergence, while the source server runs without them: the + rewound server would fail checksum verification on the blocks copied + from the source. The opposite combination is allowed with a warning; + the rewound server keeps data checksums disabled unless the source + server enabled them online after the point of divergence. See + for how to change the state + consistently across a replication setup. + + Warning: Failures While Rewinding diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 023e23b063c..6e87b00f8c2 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -167,7 +167,8 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, - XLogRecPtr *lastchkptredo, const char *restoreCommand) + XLogRecPtr *lastchkptredo, uint32 *lastchkptdatachecksums, + const char *restoreCommand) { /* Walk backwards, starting from the given record */ XLogRecord *record; @@ -255,6 +256,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, *lastchkptrec = searchptr; *lastchkpttli = checkPoint.ThisTimeLineID; *lastchkptredo = checkPoint.redo; + *lastchkptdatachecksums = checkPoint.dataChecksumState; break; } diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 0e4c2df3f4b..d2521dab333 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -145,6 +145,7 @@ main(int argc, char **argv) XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; + uint32 chkptdatachecksums; TimeLineID source_tli; TimeLineID target_tli; XLogRecPtr target_wal_endrec; @@ -472,10 +473,51 @@ main(int argc, char **argv) keepwal_init(); findLastCheckpoint(datadir_target, divergerec, lastcommontliIndex, - &chkptrec, &chkpttli, &chkptredo, restore_command); + &chkptrec, &chkpttli, &chkptredo, &chkptdatachecksums, + restore_command); pg_log_info("rewinding from last common checkpoint at %X/%08X on timeline %u", LSN_FORMAT_ARGS(chkptrec), chkpttli); + /* + * Replay on the rewound server resumes from the last common checkpoint + * and adopts the data checksum state recorded there, not the state in the + * control file installed from the source. If checksums were enabled at + * the divergence point but the source runs without them, the rewound + * server would verify checksums while the blocks copied from the source + * have none. The control files cannot reveal this: an offline disable on + * the source after the divergence leaves both of them saying "off". + * + * Only the fully enabled state needs checking. An in-progress state at + * the divergence point means a transition was still running there, and + * all of its page rewrites are logged after that point, so replay brings + * the rewound server to whatever state the copied WAL ends in. + */ + if (chkptdatachecksums == PG_DATA_CHECKSUM_VERSION && + ControlFile_source.data_checksum_version != PG_DATA_CHECKSUM_VERSION) + { + pg_log_error("data checksums were enabled at the point of divergence but are disabled on the source server"); + pg_log_error_detail("Blocks copied from the source would have no checksums, but the rewound server would resume with checksum verification enabled."); + pg_log_error_hint("Either enable data checksums on the source server or recreate the target server from a base backup."); + exit(1); + } + + /* + * The same comparison is needed against the target: every block the + * rewind does not copy keeps the target's content. The control files + * cannot reveal this case either, in the other direction: a standby's + * checkpoints are written by its upstream primary, so a standby whose + * checksums were disabled offline still has "on" checkpoints in its WAL, + * while both control files may agree. + */ + if (chkptdatachecksums == PG_DATA_CHECKSUM_VERSION && + ControlFile_target.data_checksum_version != PG_DATA_CHECKSUM_VERSION) + { + pg_log_error("data checksums were enabled at the point of divergence but are disabled on the target server"); + pg_log_error_detail("Blocks kept from the target would have no checksums, but the rewound server would resume with checksum verification enabled."); + pg_log_error_hint("Either enable data checksums on the target server with pg_checksums or recreate the target server from a base backup."); + exit(1); + } + /* Initialize the hash table to track the status of each file */ filehash_init(); @@ -799,6 +841,55 @@ sanityChecks(void) pg_fatal("target server needs to use either data checksums or \"wal_log_hints = on\""); } + /* + * The rewound target keeps its control file fields from the source, but + * every block the rewind does not copy keeps the target's content. If + * checksums are enabled on the target and disabled on the source, the + * result would claim enabled checksums while the blocks copied from the + * source have none, and the target would fail checksum verification as + * soon as it reads them. Refuse that combination, and refuse an + * interrupted online transition on either side, same as pg_checksums. + * + * The opposite mismatch is allowed: recovery under the backup label + * written by pg_rewind adopts the checksum state as of the divergence + * point, so a target whose own checkpoints carry no checksums stays + * without them, or converges through the replayed WAL if the source + * enabled them online. Only warn about it, so that an offline change on + * the source is not overlooked. + * + * That reasoning does not hold when the target is a standby, whose + * checkpoints were written by its upstream primary; see the checks after + * findLastCheckpoint(). + */ + if (ControlFile_target.data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_ON || + ControlFile_target.data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_OFF) + { + pg_log_error("an online data checksum state transition was interrupted on the target server"); + pg_log_error_hint("Start the server and shut it down cleanly to reset the state, then retry; on a standby, let replication complete the transition first."); + exit(1); + } + if (ControlFile_source.data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_ON || + ControlFile_source.data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_OFF) + { + pg_log_error("an online data checksum state transition is incomplete on the source server"); + pg_log_error_hint("Let the transition complete, or reset the state with a clean restart, then retry."); + exit(1); + } + if (ControlFile_target.data_checksum_version == PG_DATA_CHECKSUM_VERSION && + ControlFile_source.data_checksum_version == PG_DATA_CHECKSUM_OFF) + { + pg_log_error("data checksums are enabled on the target server but disabled on the source server"); + pg_log_error_detail("Blocks copied from the source would have no checksums, and the target would fail checksum verification after the rewind."); + pg_log_error_hint("Either disable data checksums on the target server or enable them on the source server, then retry."); + exit(1); + } + if (ControlFile_target.data_checksum_version == PG_DATA_CHECKSUM_OFF && + ControlFile_source.data_checksum_version == PG_DATA_CHECKSUM_VERSION) + { + pg_log_warning("data checksums are disabled on the target server but enabled on the source server"); + pg_log_warning_detail("The rewound server will keep data checksums disabled unless the source server enabled them online after the point of divergence."); + } + /* * Target cluster better not be running. This doesn't guard against * someone starting the cluster concurrently. Also, this is probably more diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h index 9a981f7f246..9187c3bf72f 100644 --- a/src/bin/pg_rewind/pg_rewind.h +++ b/src/bin/pg_rewind/pg_rewind.h @@ -39,6 +39,7 @@ extern void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, XLogRecPtr *lastchkptredo, + uint32 *lastchkptdatachecksums, const char *restoreCommand); extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, const char *restoreCommand); diff --git a/src/test/modules/test_checksums/meson.build b/src/test/modules/test_checksums/meson.build index e5d38fafb7d..7d07c757052 100644 --- a/src/test/modules/test_checksums/meson.build +++ b/src/test/modules/test_checksums/meson.build @@ -45,6 +45,8 @@ tests += { 't/019_standby_shutdown_catchup.pl', 't/020_cascade_divergence.pl', 't/021_rewind_divergent_transitions.pl', + 't/022_rewind_state.pl', + 't/023_rewind_standby_target.pl', ], }, } diff --git a/src/test/modules/test_checksums/t/022_rewind_state.pl b/src/test/modules/test_checksums/t/022_rewind_state.pl new file mode 100644 index 00000000000..e6d9c645a0b --- /dev/null +++ b/src/test/modules/test_checksums/t/022_rewind_state.pl @@ -0,0 +1,145 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# pg_rewind checks the data checksum states of source and target. +# Checksums enabled on the target, or at the point of divergence, with +# a source running without them are refused: the rewound server would +# verify checksums on blocks copied from a source that has none. The +# opposite mismatch only warns; replay keeps the target's own state. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +# This test suite is expensive to execute, require PG_TEST_EXTRA to contain +# 'checksum' to run it. +if ($ENV{PG_TEST_EXTRA}) +{ + plan skip_all => 'Expensive data checksums test disabled' + unless ($ENV{PG_TEST_EXTRA} =~ /\bchecksum(_extended)?\b/); +} +else +{ + plan skip_all => 'Expensive data checksums test disabled'; +} + +# Scenarios 1 and 2: checksums on from initdb. wal_log_hints keeps the +# target eligible for pg_rewind after checksums are disabled on it. +my $node_a = PostgreSQL::Test::Cluster->new('node_a'); +$node_a->init(allows_streaming => 1); +$node_a->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +wal_keep_size = '1GB' +wal_log_hints = on +]); +$node_a->start; +$node_a->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,10000) AS a;"); + +$node_a->backup('backup'); +my $node_b = PostgreSQL::Test::Cluster->new('node_b'); +$node_b->init_from_backup($node_a, 'backup', has_streaming => 1); +$node_b->start; +$node_a->wait_for_catchup($node_b); + +# Failover to B, and divergence on A. +$node_b->promote; +$node_b->safe_psql('postgres', "INSERT INTO t VALUES (0);"); +$node_a->safe_psql('postgres', "INSERT INTO t VALUES (-1);"); +$node_a->stop('fast'); + +# Offline disable on the source only: enabled target, disabled source. +$node_b->stop; +$node_b->checksum_disable_offline; +$node_b->start; +test_checksum_state($node_b, 'off'); + +my @rewind_cmd = ( + 'pg_rewind', + '--target-pgdata' => $node_a->data_dir, + '--source-server' => $node_b->connstr('postgres')); + +command_fails_like( + \@rewind_cmd, + qr/data checksums are enabled on the target server but disabled on the source server/, + 'refuses an enabled target with a disabled source'); + +# The refusal happens before any modification: the target still starts +# on its own timeline. +$node_a->start; +is($node_a->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10001', 'target untouched by the refused rewind'); +$node_a->stop('fast'); + +# Scenario 2: disabling the target too makes the control files match, +# but checksums were still enabled at the point of divergence, which is +# the state replay on the rewound server would resume with. +$node_a->checksum_disable_offline; +command_fails_like( + \@rewind_cmd, + qr/data checksums were enabled at the point of divergence but are disabled on the source server/, + 'refuses when checksums were enabled at the point of divergence'); + +# Scenario 3: fresh pair without checksums; offline enable on the +# source only. Allowed with a warning, and the rewound server keeps +# checksums disabled. +my $node_c = PostgreSQL::Test::Cluster->new('node_c'); +$node_c->init(allows_streaming => 1, no_data_checksums => 1); +$node_c->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +wal_keep_size = '1GB' +wal_log_hints = on +]); +$node_c->start; +$node_c->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,10000) AS a;"); + +$node_c->backup('backup'); +my $node_d = PostgreSQL::Test::Cluster->new('node_d'); +$node_d->init_from_backup($node_c, 'backup', has_streaming => 1); +$node_d->start; +$node_c->wait_for_catchup($node_d); + +$node_d->promote; +$node_d->safe_psql('postgres', "INSERT INTO t VALUES (0);"); +$node_c->safe_psql('postgres', "INSERT INTO t VALUES (-1);"); +$node_c->stop('fast'); + +$node_d->stop; +$node_d->checksum_enable_offline; +$node_d->start; +test_checksum_state($node_d, 'on'); + +my ($stdout, $stderr) = run_command( + [ + 'pg_rewind', + '--target-pgdata' => $node_c->data_dir, + '--source-server' => $node_d->connstr('postgres'), + ]); +like( + $stderr, + qr/data checksums are disabled on the target server but enabled on the source server/, + 'warns for a disabled target with an enabled source'); +like($stderr, qr/Done!/, 'rewind completed despite the warning'); + +# The rewound server follows D and keeps its own state. +$node_c->append_conf('postgresql.conf', 'port = ' . $node_c->port); +$node_c->enable_streaming($node_d); +$node_c->set_standby_mode; +$node_c->start; +$node_d->wait_for_catchup($node_c); +is($node_c->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10001', 'rewound server readable as a standby'); +test_checksum_state($node_c, 'off'); + +$node_c->stop; +$node_d->stop; +done_testing(); diff --git a/src/test/modules/test_checksums/t/023_rewind_standby_target.pl b/src/test/modules/test_checksums/t/023_rewind_standby_target.pl new file mode 100644 index 00000000000..9071114a9ae --- /dev/null +++ b/src/test/modules/test_checksums/t/023_rewind_standby_target.pl @@ -0,0 +1,153 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that pg_rewind compares the data checksum state at the point of +# divergence against the target as well as the source. +# +# A standby's checkpoints are written by its upstream primary, so a standby +# whose checksums were disabled offline still has "on" checkpoints in its +# WAL. Rewinding it onto a promoted sibling passes every control file +# check (target off + source on only warns), yet recovery after the rewind +# adopts the divergence checkpoint's state and the rewound server would +# verify checksums over the pages it wrote while it was locally "off". +# pg_rewind must refuse; after an offline enable on the target, which +# rewrites all of its pages, the rewind goes through. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +# This test suite is expensive to execute, require PG_TEST_EXTRA to contain +# 'checksum_extended' to run it. +if ($ENV{PG_TEST_EXTRA}) +{ + plan skip_all => 'Expensive data checksums test disabled' + unless ($ENV{PG_TEST_EXTRA} =~ /\bchecksum_extended\b/); +} +else +{ + plan skip_all => 'Expensive data checksums test disabled'; +} + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +wal_keep_size = '1GB' +wal_log_hints = on +]); +$primary->start; +$primary->safe_psql('postgres', + "CREATE TABLE t0 AS SELECT generate_series(1,10000) AS a;"); + +$primary->backup('backup'); + +my $lagging = PostgreSQL::Test::Cluster->new('lagging'); +$lagging->init_from_backup($primary, 'backup', has_streaming => 1); +$lagging->start; + +my $failover = PostgreSQL::Test::Cluster->new('failover'); +$failover->init_from_backup($primary, 'backup', has_streaming => 1); +$failover->start; + +$primary->wait_for_catchup($lagging); +$primary->wait_for_catchup($failover); + +test_checksum_state($primary, 'on'); +test_checksum_state($lagging, 'on'); +test_checksum_state($failover, 'on'); + +# Offline-disable checksums on the lagging standby only. This is a +# divergence 012_offline_standby.pl declares survivable: the standby keeps +# its own state, warns, and stays readable. +$lagging->stop; +system_or_bail('pg_checksums', '--disable', '--pgdata', $lagging->data_dir); +$lagging->start; +test_checksum_state($lagging, 'off'); + +# Give the lagging standby plenty of pages to write while it is locally +# "off", and force a restartpoint so they reach disk without checksums. +# The other standby replays the same WAL with checksums on. +$primary->safe_psql('postgres', + "CREATE TABLE t1 AS SELECT generate_series(1,200000) AS a;"); +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->wait_for_catchup($lagging); +$primary->wait_for_catchup($failover); +$lagging->safe_psql('postgres', "CHECKPOINT;"); + +# Take the "failover" standby out of the picture. It is promoted from +# this position later, so everything replayed from here on is the part of +# the WAL that diverges and that pg_rewind will roll back. t1 is already +# replayed everywhere, so its blocks are *not* rolled back. +$failover->stop; + +$primary->safe_psql('postgres', + "CREATE TABLE t2 AS SELECT generate_series(1,1000) AS a;"); +$primary->wait_for_catchup($lagging); + +# Failover: promote the other standby, which forks the timeline behind the +# replay position of the lagging standby. +$primary->stop('fast'); +$failover->start; +$failover->promote; +$failover->safe_psql('postgres', + "CREATE TABLE t3 AS SELECT generate_series(1,1000) AS a;"); +$failover->safe_psql('postgres', "CHECKPOINT;"); + +# Re-attaching the lagging standby with pg_rewind must be refused: the +# divergence checkpoint says "on" while the target is "off", so the rewound +# server would verify checksums over the blocks it keeps. +$lagging->stop; + +command_fails_like( + [ + 'pg_rewind', + '--target-pgdata' => $lagging->data_dir, + '--source-server' => $failover->connstr('postgres'), + ], + qr/data checksums were enabled at the point of divergence but are disabled on the target server/, + 'pg_rewind refuses a target whose divergence checkpoint has checksums'); + +# Enabling checksums offline on the target rewrites all of its pages, after +# which the rewind is safe. +system_or_bail('pg_checksums', '--enable', '--pgdata', $lagging->data_dir); + +my ($stdout, $stderr) = run_command( + [ + 'pg_rewind', + '--target-pgdata' => $lagging->data_dir, + '--source-server' => $failover->connstr('postgres'), + ]); +like($stderr, qr/Done!/, 'pg_rewind completes after the offline enable'); + +$lagging->append_conf('postgresql.conf', 'port = ' . $lagging->port); +$lagging->enable_streaming($failover); +$lagging->set_standby_mode; +$lagging->start; + +my ($rc, $out, $err) = + $lagging->psql('postgres', + "SELECT setting FROM pg_settings " . "WHERE name = 'data_checksums';"); +is($rc, 0, 'rewound standby accepts connections') or diag("stderr: $err"); +is($out, 'on', 'rewound standby resumes with checksums on'); + +($rc, $out, $err) = $lagging->psql('postgres', "SELECT count(*) FROM t1;"); +is($rc, 0, 'blocks kept from the target stay readable') + or diag("stderr: $err"); + +my $log = PostgreSQL::Test::Utils::slurp_file($lagging->logfile); +unlike( + $log, + qr/page verification failed/, + 'no checksum verification failures on the rewound standby'); + +$lagging->stop('immediate'); +$failover->stop; +done_testing();