database/postgres
database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-05-17 06:41:24 +03:00
Code

Fix postmaster's handling of a startup-process crash.

Browse Source 
Ordinarily, a failure (unexpected exit status) of the startup subprocess
should be considered fatal, so the postmaster should just close up shop
and quit.  However, if we sent the startup process a SIGQUIT or SIGKILL
signal, the failure is hardly "unexpected", and we should attempt restart;
this is necessary for recovery from ordinary backend crashes in hot-standby
scenarios.  I attempted to implement the latter rule with a two-line patch
in commit 442231d7f71764b8c628044e7ce2225f9aa43b67, but it now emerges that
that patch was a few bricks shy of a load: it failed to distinguish the
case of a signaled startup process from the case where the new startup
process crashes before reaching database consistency.  That resulted in
infinitely respawning a new startup process only to have it crash again.

To handle this properly, we really must track whether we have sent the
*current* startup process a kill signal.  Rather than add yet another
ad-hoc boolean to the postmaster's state, I chose to unify this with the
existing RecoveryError flag into an enum tracking the startup process's
state.  That seems more consistent with the postmaster's general state
machine design.

Back-patch to 9.0, like the previous patch.
This commit is contained in:
Tom Lane 2015-07-09 13:22:23 -04:00
parent 63277305d8
commit 9c39d7ae08
1 changed files with 33 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/backend/postmaster/postmaster.c
View File

@ -275,6 +275,17 @@ static pid_t StartupPID = 0,
PgStatPID = 0,
SysLoggerPID = 0;
/* Startup process's status */
typedef enum
{
STARTUP_NOT_RUNNING,
STARTUP_RUNNING,
STARTUP_SIGNALED, /* we sent it a SIGQUIT or SIGKILL */
STARTUP_CRASHED
} StartupStatusEnum;
static StartupStatusEnum StartupStatus = STARTUP_NOT_RUNNING;
/* Startup/shutdown state */
#define NoShutdown 0
#define SmartShutdown 1
@ -283,7 +294,6 @@ static pid_t StartupPID = 0,
static int Shutdown = NoShutdown;
static bool FatalError = false; /* T if recovering from backend crash */
static bool RecoveryError = false; /* T if WAL recovery failed */
/*
* We use a simple state machine to control startup, shutdown, and
@ -326,8 +336,6 @@ static bool RecoveryError = false; /* T if WAL recovery failed */
* states, nor in PM_SHUTDOWN states (because we don't enter those states
* when trying to recover from a crash). It can be true in PM_STARTUP state,
* because we don't clear it until we've successfully started WAL redo.
* Similarly, RecoveryError means that we have crashed during recovery, and
* should not try to restart.
*/
typedef enum
{
@ -1266,6 +1274,7 @@ PostmasterMain(int argc, char *argv[])
*/
StartupPID = StartupDataBase();
Assert(StartupPID != 0);
StartupStatus = STARTUP_RUNNING;
pmState = PM_STARTUP;
/* Some workers may be scheduled to start now */
@ -2565,6 +2574,7 @@ reaper(SIGNAL_ARGS)
if (Shutdown > NoShutdown &&
(EXIT_STATUS_0(exitstatus) || EXIT_STATUS_1(exitstatus)))
{
StartupStatus = STARTUP_NOT_RUNNING;
pmState = PM_WAIT_BACKENDS;
/* PostmasterStateMachine logic does the rest */
continue;
@ -2587,16 +2597,18 @@ reaper(SIGNAL_ARGS)
/*
* After PM_STARTUP, any unexpected exit (including FATAL exit) of
* the startup process is catastrophic, so kill other children,
* and set RecoveryError so we don't try to reinitialize after
* they're gone. Exception: if FatalError is already set, that
* implies we previously sent the startup process a SIGQUIT, so
* and set StartupStatus so we don't try to reinitialize after
* they're gone. Exception: if StartupStatus is STARTUP_SIGNALED,
* then we previously sent the startup process a SIGQUIT; so
* that's probably the reason it died, and we do want to try to
* restart in that case.
*/
if (!EXIT_STATUS_0(exitstatus))
{
if (!FatalError)
RecoveryError = true;
if (StartupStatus == STARTUP_SIGNALED)
StartupStatus = STARTUP_NOT_RUNNING;
else
StartupStatus = STARTUP_CRASHED;
HandleChildCrash(pid, exitstatus,
_("startup process"));
continue;
@ -2605,6 +2617,7 @@ reaper(SIGNAL_ARGS)
/*
* Startup succeeded, commence normal operations
*/
StartupStatus = STARTUP_NOT_RUNNING;
FatalError = false;
ReachedNormalRunning = true;
pmState = PM_RUN;
@ -3115,7 +3128,10 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
/* Take care of the startup process too */
if (pid == StartupPID)
{
StartupPID = 0;
StartupStatus = STARTUP_CRASHED;
}
else if (StartupPID != 0 && !FatalError)
{
ereport(DEBUG2,
@ -3123,6 +3139,7 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
(SendStop ? "SIGSTOP" : "SIGQUIT"),
(int) StartupPID)));
signal_child(StartupPID, (SendStop ? SIGSTOP : SIGQUIT));
StartupStatus = STARTUP_SIGNALED;
}
/* Take care of the bgwriter too */
@ -3502,13 +3519,14 @@ PostmasterStateMachine(void)
}
/*
* If recovery failed, or the user does not want an automatic restart
* after backend crashes, wait for all non-syslogger children to exit, and
* then exit postmaster. We don't try to reinitialize when recovery fails,
* because more than likely it will just fail again and we will keep
* trying forever.
* If the startup process failed, or the user does not want an automatic
* restart after backend crashes, wait for all non-syslogger children to
* exit, and then exit postmaster. We don't try to reinitialize when the
* startup process fails, because more than likely it will just fail again
* and we will keep trying forever.
*/
if (pmState == PM_NO_CHILDREN && (RecoveryError || !restart_after_crash))
if (pmState == PM_NO_CHILDREN &&
(StartupStatus == STARTUP_CRASHED || !restart_after_crash))
ExitPostmaster(1);
/*
@ -3525,6 +3543,7 @@ PostmasterStateMachine(void)
StartupPID = StartupDataBase();
Assert(StartupPID != 0);
StartupStatus = STARTUP_RUNNING;
pmState = PM_STARTUP;
}
}