1
0
mirror of https://github.com/postgres/postgres.git synced 2026-10-23 13:29:09 +03:00

Fix handling of shutdown requests during an early crash restart

A smart or fast shutdown during an early crash restart can wait forever
on the checkpointer and I/O workers.  FatalError is still set, so
PM_STOP_BACKENDS adds them as processes to wait for, but these are
relaunched by the restart and ignore the SIGTERM that a shutdown sends.
AbortStartTime has been reset at that point, meaning that SIGKILL would
also not fire.

SIGQUIT is now sent instead of SIGTERM when FatalError is set, via
HandleFatalError(), arming AbortStartTime.  Going through that routine
rather than signaling the children directly matters for the startup
process: TerminateChildren() marks the startup process as signaled, so
its exit is not later reported as a failure.  Note that after sending
the SIGTERM, we check for any running processes, assuming that auxiliary
processes have been already SIGQUIT'd for an immediate shutdown or for a
FatalError.

A test case is added, able to stress the scenario of a shutdown
triggered during the early phases of crash recovery with a custom
restore command waiting for a shutdown by scanning the server logs.  The
CI is stable with this test (several runs done); the buildfarm will be
the final judge regarding its stability.

No backpatch is done for now, to be on the conservative side.  This
feels also a slightly risky change for v19, which is close to release,
and letting this change bake on HEAD a bit would not hurt.

This problem has been originally reported by Justin Pryzby back in 2023,
without being fixed, and still existed on HEAD.

Reported-by: Justin Pryzby <pryzby@telsasoft.com>
Author: Ayush Tiwari <ayushtiwari.slg01@gmail.com>
Co-authored-by: Michael Paquier <michael@paquier.xyz>
Discussion: https://postgr.es/m/CAJTYsWXCLi00cJ6-_OkMJB4UjyV6EZjc86QwmgE9=iH_C0R-Aw@mail.gmail.com
Discussion: https://postgr.es/m/ZWlrdQarrZvLsgIk@pryzbyj2023
This commit is contained in:
Michael Paquier
2026-10-07 08:45:37 +09:00
parent a32b8dc986
commit 12f866cd5f
4 changed files with 96 additions and 2 deletions

View File

@@ -3041,9 +3041,19 @@ PostmasterStateMachine(void)
*/
ForgetUnstartedBackgroundWorkers();
SignalChildren(SIGTERM, targetMask);
/*
* While processing a crash, targetMask includes the checkpointer
* and the io workers. These ignore SIGTERM, so upgrade to
* SIGQUIT.
*/
if (FatalError)
HandleFatalError(PMQUIT_FOR_STOP, false);
else
{
SignalChildren(SIGTERM, targetMask);
UpdatePMState(PM_WAIT_BACKENDS);
UpdatePMState(PM_WAIT_BACKENDS);
}
}
/* Are any of the target processes still running? */

View File

@@ -66,6 +66,7 @@ tests += {
't/055_cascade_reconnect.pl',
't/056_standby_snapshot_export.pl',
't/057_snapshot_commit_race.pl',
't/058_shutdown_crash_restart.pl',
],
},
}

View File

@@ -0,0 +1,61 @@
# Copyright (c) 2026, PostgreSQL Global Development Group
# Test shutdown during a crash restart, before WAL redo has started. The
# test relies on a fake restore_command that keeps the startup process at
# some early stage, waiting for a shutdown to happen, with a checkpointer
# spawned and running.
use strict;
use warnings FATAL => 'all';
use FindBin;
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init(allows_streaming => 1);
# Make the restarted startup process wait in restore_command until shutdown.
my $perlbin = $^X;
$perlbin =~ s!\\!/!g if $windows_os;
my $logfile = $node->logfile;
$logfile =~ s!\\!/!g if $windows_os;
my $restore_timeout = $PostgreSQL::Test::Utils::timeout_default;
# DEBUG2 is required for the checkpointer log entry lookup.
$node->append_conf(
'postgresql.conf', qq{
restart_after_crash = on
log_min_messages = debug2
restore_command = '"$perlbin" "$FindBin::RealBin/wait_for_shutdown" "$logfile" $restore_timeout'
});
$node->start;
# Stop the background writer.
$node->poll_query_until(
'postgres',
q{SELECT count(*) = 1 FROM pg_stat_activity
WHERE backend_type = 'background writer'}
) or die 'background writer did not start';
my $pid = $node->safe_psql('postgres',
"SELECT pid FROM pg_stat_activity WHERE backend_type = 'background writer'"
);
$node->set_standby_mode;
my $log_offset = -s $node->logfile;
is(PostgreSQL::Test::Utils::system_log('pg_ctl', 'kill', 'QUIT', $pid),
0, "SIGQUIT sent to background writer");
$node->wait_for_log(qr/restore_command waiting for shutdown/, $log_offset);
# Wait until the new checkpointer has installed its SIGTERM handler.
$node->wait_for_log(
qr/checkpointer updated shared memory configuration values/, $log_offset);
ok($node->stop('fast', fail_ok => 1),
'fast shutdown completes during crash restart');
unlike(
slurp_file($node->logfile, $log_offset),
qr/timed out waiting for shutdown request/,
'restore_command did not time out');
done_testing();

View File

@@ -0,0 +1,22 @@
#!/usr/bin/perl
# restore_command helper: wait until the server log shows a shutdown request.
# This script accepts two arguments:
# - A log file to monitor, inherited from the server spawned.
# - A timeout value, defined by $PostgreSQL::Test::Utils::timeout_default.
use strict;
use warnings FATAL => 'all';
use Time::HiRes qw(usleep);
my ($logfile, $timeout) = @ARGV;
print STDERR "restore_command waiting for shutdown\n";
for (my $i = 0; $i < $timeout * 10; $i++)
{
open my $fh, '<', $logfile or die "could not open $logfile: $!";
exit 1 if grep { /received \w+ shutdown request/ } <$fh>;
close $fh;
usleep(100_000);
}
die "timed out waiting for shutdown request\n";