mirror of
https://github.com/postgres/postgres.git
synced 2026-10-23 13:29:09 +03:00
Fix handling of shutdown requests during an early crash restart
A smart or fast shutdown during an early crash restart can wait forever on the checkpointer and I/O workers. FatalError is still set, so PM_STOP_BACKENDS adds them as processes to wait for, but these are relaunched by the restart and ignore the SIGTERM that a shutdown sends. AbortStartTime has been reset at that point, meaning that SIGKILL would also not fire. SIGQUIT is now sent instead of SIGTERM when FatalError is set, via HandleFatalError(), arming AbortStartTime. Going through that routine rather than signaling the children directly matters for the startup process: TerminateChildren() marks the startup process as signaled, so its exit is not later reported as a failure. Note that after sending the SIGTERM, we check for any running processes, assuming that auxiliary processes have been already SIGQUIT'd for an immediate shutdown or for a FatalError. A test case is added, able to stress the scenario of a shutdown triggered during the early phases of crash recovery with a custom restore command waiting for a shutdown by scanning the server logs. The CI is stable with this test (several runs done); the buildfarm will be the final judge regarding its stability. No backpatch is done for now, to be on the conservative side. This feels also a slightly risky change for v19, which is close to release, and letting this change bake on HEAD a bit would not hurt. This problem has been originally reported by Justin Pryzby back in 2023, without being fixed, and still existed on HEAD. Reported-by: Justin Pryzby <pryzby@telsasoft.com> Author: Ayush Tiwari <ayushtiwari.slg01@gmail.com> Co-authored-by: Michael Paquier <michael@paquier.xyz> Discussion: https://postgr.es/m/CAJTYsWXCLi00cJ6-_OkMJB4UjyV6EZjc86QwmgE9=iH_C0R-Aw@mail.gmail.com Discussion: https://postgr.es/m/ZWlrdQarrZvLsgIk@pryzbyj2023
This commit is contained in:
@@ -3041,9 +3041,19 @@ PostmasterStateMachine(void)
|
||||
*/
|
||||
ForgetUnstartedBackgroundWorkers();
|
||||
|
||||
SignalChildren(SIGTERM, targetMask);
|
||||
/*
|
||||
* While processing a crash, targetMask includes the checkpointer
|
||||
* and the io workers. These ignore SIGTERM, so upgrade to
|
||||
* SIGQUIT.
|
||||
*/
|
||||
if (FatalError)
|
||||
HandleFatalError(PMQUIT_FOR_STOP, false);
|
||||
else
|
||||
{
|
||||
SignalChildren(SIGTERM, targetMask);
|
||||
|
||||
UpdatePMState(PM_WAIT_BACKENDS);
|
||||
UpdatePMState(PM_WAIT_BACKENDS);
|
||||
}
|
||||
}
|
||||
|
||||
/* Are any of the target processes still running? */
|
||||
|
||||
@@ -66,6 +66,7 @@ tests += {
|
||||
't/055_cascade_reconnect.pl',
|
||||
't/056_standby_snapshot_export.pl',
|
||||
't/057_snapshot_commit_race.pl',
|
||||
't/058_shutdown_crash_restart.pl',
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
61
src/test/recovery/t/058_shutdown_crash_restart.pl
Normal file
61
src/test/recovery/t/058_shutdown_crash_restart.pl
Normal file
@@ -0,0 +1,61 @@
|
||||
# Copyright (c) 2026, PostgreSQL Global Development Group
|
||||
|
||||
# Test shutdown during a crash restart, before WAL redo has started. The
|
||||
# test relies on a fake restore_command that keeps the startup process at
|
||||
# some early stage, waiting for a shutdown to happen, with a checkpointer
|
||||
# spawned and running.
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use FindBin;
|
||||
use PostgreSQL::Test::Cluster;
|
||||
use PostgreSQL::Test::Utils;
|
||||
use Test::More;
|
||||
|
||||
my $node = PostgreSQL::Test::Cluster->new('node');
|
||||
$node->init(allows_streaming => 1);
|
||||
|
||||
# Make the restarted startup process wait in restore_command until shutdown.
|
||||
my $perlbin = $^X;
|
||||
$perlbin =~ s!\\!/!g if $windows_os;
|
||||
my $logfile = $node->logfile;
|
||||
$logfile =~ s!\\!/!g if $windows_os;
|
||||
my $restore_timeout = $PostgreSQL::Test::Utils::timeout_default;
|
||||
|
||||
# DEBUG2 is required for the checkpointer log entry lookup.
|
||||
$node->append_conf(
|
||||
'postgresql.conf', qq{
|
||||
restart_after_crash = on
|
||||
log_min_messages = debug2
|
||||
restore_command = '"$perlbin" "$FindBin::RealBin/wait_for_shutdown" "$logfile" $restore_timeout'
|
||||
});
|
||||
$node->start;
|
||||
|
||||
# Stop the background writer.
|
||||
$node->poll_query_until(
|
||||
'postgres',
|
||||
q{SELECT count(*) = 1 FROM pg_stat_activity
|
||||
WHERE backend_type = 'background writer'}
|
||||
) or die 'background writer did not start';
|
||||
my $pid = $node->safe_psql('postgres',
|
||||
"SELECT pid FROM pg_stat_activity WHERE backend_type = 'background writer'"
|
||||
);
|
||||
$node->set_standby_mode;
|
||||
my $log_offset = -s $node->logfile;
|
||||
is(PostgreSQL::Test::Utils::system_log('pg_ctl', 'kill', 'QUIT', $pid),
|
||||
0, "SIGQUIT sent to background writer");
|
||||
$node->wait_for_log(qr/restore_command waiting for shutdown/, $log_offset);
|
||||
|
||||
# Wait until the new checkpointer has installed its SIGTERM handler.
|
||||
$node->wait_for_log(
|
||||
qr/checkpointer updated shared memory configuration values/, $log_offset);
|
||||
|
||||
ok($node->stop('fast', fail_ok => 1),
|
||||
'fast shutdown completes during crash restart');
|
||||
|
||||
unlike(
|
||||
slurp_file($node->logfile, $log_offset),
|
||||
qr/timed out waiting for shutdown request/,
|
||||
'restore_command did not time out');
|
||||
|
||||
done_testing();
|
||||
22
src/test/recovery/t/wait_for_shutdown
Normal file
22
src/test/recovery/t/wait_for_shutdown
Normal file
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
# restore_command helper: wait until the server log shows a shutdown request.
|
||||
# This script accepts two arguments:
|
||||
# - A log file to monitor, inherited from the server spawned.
|
||||
# - A timeout value, defined by $PostgreSQL::Test::Utils::timeout_default.
|
||||
|
||||
use strict;
|
||||
use warnings FATAL => 'all';
|
||||
use Time::HiRes qw(usleep);
|
||||
|
||||
my ($logfile, $timeout) = @ARGV;
|
||||
|
||||
print STDERR "restore_command waiting for shutdown\n";
|
||||
for (my $i = 0; $i < $timeout * 10; $i++)
|
||||
{
|
||||
open my $fh, '<', $logfile or die "could not open $logfile: $!";
|
||||
exit 1 if grep { /received \w+ shutdown request/ } <$fh>;
|
||||
close $fh;
|
||||
usleep(100_000);
|
||||
}
|
||||
die "timed out waiting for shutdown request\n";
|
||||
Reference in New Issue
Block a user