database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-08-28 18:48:04 +03:00
Code Activity

XLOG file archiving and point-in-time recovery. There are still some

Browse Source 
loose ends and a glaring lack of documentation, but it basically works.

Simon Riggs with some editorialization by Tom Lane.
This commit is contained in:
Tom Lane
2004-07-19 02:47:16 +00:00
parent d0c1bbdcc6
commit 66ec2db728
13 changed files with 1628 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/backend/postmaster/Makefile
View File

@@ -4,7 +4,7 @@
# Makefile for src/backend/postmaster
#
# IDENTIFICATION
# $PostgreSQL: pgsql/src/backend/postmaster/Makefile,v 1.15 2004/05/29 22:48:19 tgl Exp $
# $PostgreSQL: pgsql/src/backend/postmaster/Makefile,v 1.16 2004/07/19 02:47:08 tgl Exp $
#
#-------------------------------------------------------------------------
@@ -12,7 +12,7 @@ subdir = src/backend/postmaster
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
OBJS = postmaster.o bgwriter.o pgstat.o
OBJS = postmaster.o bgwriter.o pgstat.o pgarch.o
all: SUBSYS.o

588
src/backend/postmaster/pgarch.c Normal file
View File

@@ -0,0 +1,588 @@
/*-------------------------------------------------------------------------
*
* pgarch.c
*
* PostgreSQL WAL archiver
*
* All functions relating to archiver are included here
*
* - All functions executed by archiver process
*
* - archiver is forked from postmaster, and the two
* processes then communicate using signals. All functions
* executed by postmaster are included in this file.
*
* Initial author: Simon Riggs simon@2ndquadrant.com
*
* Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/postmaster/pgarch.c,v 1.1 2004/07/19 02:47:08 tgl Exp $
*
*-------------------------------------------------------------------------
*/
#include "postgres.h"
#include <fcntl.h>
#include <signal.h>
#include <time.h>
#include <sys/time.h>
#include <unistd.h>
#include "postmaster/pgarch.h"
#include "libpq/pqsignal.h"
#include "miscadmin.h"
#include "postmaster/postmaster.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/pg_shmem.h"
#include "storage/pmsignal.h"
#include "utils/guc.h"
#include "utils/ps_status.h"
/* ----------
* Timer definitions.
* ----------
*/
#define PGARCH_AUTOWAKE_INTERVAL 60 /* How often to force a poll of
* the archive status directory;
* in seconds. */
#define PGARCH_RESTART_INTERVAL 60 /* How often to attempt to restart
* a failed archiver; in seconds. */
/* ----------
* Archiver control info.
*
* We expect that archivable files within pg_xlog will have names between
* MIN_XFN_CHARS and MAX_XFN_CHARS in length, consisting only of characters
* appearing in VALID_XFN_CHARS. The status files in archive_status have
* corresponding names with ".ready" or ".done" appended.
* ----------
*/
#define MIN_XFN_CHARS 16
#define MAX_XFN_CHARS 16
#define VALID_XFN_CHARS "0123456789ABCDEF"
#define NUM_ARCHIVE_RETRIES 3
/* ----------
* Local data
* ----------
*/
static char XLogDir[MAXPGPATH];
static char XLogArchiveStatusDir[MAXPGPATH];
static time_t last_pgarch_start_time;
/*
* Flags set by interrupt handlers for later service in the main loop.
*/
static volatile sig_atomic_t got_SIGHUP = false;
static volatile sig_atomic_t wakened = false;
/* ----------
* Local function forward declarations
* ----------
*/
#ifdef EXEC_BACKEND
static pid_t pgarch_forkexec(void);
#endif
NON_EXEC_STATIC void PgArchiverMain(int argc, char *argv[]);
static void pgarch_exit(SIGNAL_ARGS);
static void ArchSigHupHandler(SIGNAL_ARGS);
static void pgarch_waken(SIGNAL_ARGS);
static void pgarch_MainLoop(void);
static void pgarch_ArchiverCopyLoop(void);
static bool pgarch_archiveXlog(char *xlog);
static bool pgarch_readyXlog(char *xlog);
static void pgarch_archiveDone(char *xlog);
/* ------------------------------------------------------------
* Public functions called from postmaster follow
* ------------------------------------------------------------
*/
/*
* pgarch_start
*
* Called from postmaster at startup or after an existing archiver
* died. Attempt to fire up a fresh archiver process.
*
* Returns PID of child process, or 0 if fail.
*
* Note: if fail, we will be called again from the postmaster main loop.
*/
int
pgarch_start(void)
{
time_t curtime;
pid_t pgArchPid;
/*
* Do nothing if no archiver needed
*/
if (!XLogArchivingActive())
return 0;
/*
* Do nothing if too soon since last archiver start. This is a
* safety valve to protect against continuous respawn attempts if the
* archiver is dying immediately at launch. Note that since we will
* be re-called from the postmaster main loop, we will get another
* chance later.
*/
curtime = time(NULL);
if ((unsigned int) (curtime - last_pgarch_start_time) <
(unsigned int) PGARCH_RESTART_INTERVAL)
return 0;
last_pgarch_start_time = curtime;
fflush(stdout);
fflush(stderr);
#ifdef __BEOS__
/* Specific beos actions before backend startup */
beos_before_backend_startup();
#endif
#ifdef EXEC_BACKEND
switch ((pgArchPid = pgarch_forkexec()))
#else
switch ((pgArchPid = fork()))
#endif
{
case -1:
#ifdef __BEOS__
/* Specific beos actions */
beos_backend_startup_failed();
#endif
ereport(LOG,
(errmsg("could not fork archiver: %m")));
return 0;
#ifndef EXEC_BACKEND
case 0:
/* in postmaster child ... */
#ifdef __BEOS__
/* Specific beos actions after backend startup */
beos_backend_startup();
#endif
/* Close the postmaster's sockets */
ClosePostmasterPorts();
/* Drop our connection to postmaster's shared memory, as well */
PGSharedMemoryDetach();
PgArchiverMain(0, NULL);
break;
#endif
default:
return (int) pgArchPid;
}
/* shouldn't get here */
return 0;
}
/* ------------------------------------------------------------
* Local functions called by archiver follow
* ------------------------------------------------------------
*/
#ifdef EXEC_BACKEND
/*
* pgarch_forkexec() -
*
* Format up the arglist for, then fork and exec, archive process
*/
static pid_t
pgarch_forkexec(void)
{
char *av[10];
int ac = 0;
av[ac++] = "postgres";
av[ac++] = "-forkarch";
av[ac++] = NULL; /* filled in by postmaster_forkexec */
av[ac] = NULL;
Assert(ac < lengthof(av));
return postmaster_forkexec(ac, av);
}
#endif /* EXEC_BACKEND */
/*
* PgArchiverMain
*
* The argc/argv parameters are valid only in EXEC_BACKEND case. However,
* since we don't use 'em, it hardly matters...
*/
NON_EXEC_STATIC void
PgArchiverMain(int argc, char *argv[])
{
IsUnderPostmaster = true; /* we are a postmaster subprocess now */
MyProcPid = getpid(); /* reset MyProcPid */
/* Lose the postmaster's on-exit routines */
on_exit_reset();
/*
* Ignore all signals usually bound to some action in the postmaster,
* except for SIGHUP, SIGUSR1 and SIGQUIT.
*/
pqsignal(SIGHUP, ArchSigHupHandler);
pqsignal(SIGINT, SIG_IGN);
pqsignal(SIGTERM, SIG_IGN);
pqsignal(SIGQUIT, pgarch_exit);
pqsignal(SIGALRM, SIG_IGN);
pqsignal(SIGPIPE, SIG_IGN);
pqsignal(SIGUSR1, pgarch_waken);
pqsignal(SIGUSR2, SIG_IGN);
pqsignal(SIGCHLD, SIG_DFL);
pqsignal(SIGTTIN, SIG_DFL);
pqsignal(SIGTTOU, SIG_DFL);
pqsignal(SIGCONT, SIG_DFL);
pqsignal(SIGWINCH, SIG_DFL);
PG_SETMASK(&UnBlockSig);
/*
* Identify myself via ps
*/
init_ps_display("archiver process", "", "");
set_ps_display("");
/* Init XLOG file paths */
snprintf(XLogDir, MAXPGPATH, "%s/pg_xlog", DataDir);
snprintf(XLogArchiveStatusDir, MAXPGPATH, "%s/archive_status", XLogDir);
pgarch_MainLoop();
exit(0);
}
/* SIGQUIT signal handler for archiver process */
static void
pgarch_exit(SIGNAL_ARGS)
{
/*
* For now, we just nail the doors shut and get out of town. It might
* seem cleaner to finish up any pending archive copies, but there's
* a nontrivial risk that init will kill us partway through.
*/
exit(0);
}
/* SIGHUP: set flag to re-read config file at next convenient time */
static void
ArchSigHupHandler(SIGNAL_ARGS)
{
got_SIGHUP = true;
}
/* SIGUSR1 signal handler for archiver process */
static void
pgarch_waken(SIGNAL_ARGS)
{
wakened = true;
}
/*
* pgarch_MainLoop
*
* Main loop for archiver
*/
static void
pgarch_MainLoop(void)
{
time_t last_copy_time = 0;
time_t curtime;
/*
* We run the copy loop immediately upon entry, in case there are
* unarchived files left over from a previous database run (or maybe
* the archiver died unexpectedly). After that we wait for a signal
* or timeout before doing more.
*/
wakened = true;
do {
/* Check for config update */
if (got_SIGHUP)
{
got_SIGHUP = false;
ProcessConfigFile(PGC_SIGHUP);
if (!XLogArchivingActive())
break; /* user wants us to shut down */
}
/* Do what we're here for */
if (wakened)
{
wakened = false;
pgarch_ArchiverCopyLoop();
last_copy_time = time(NULL);
}
/*
* There shouldn't be anything for the archiver to do except
* to wait for a signal, so we could use pause(3) here...
* ...however, the archiver exists to protect our data, so
* she wakes up occasionally to allow herself to be proactive.
* In particular this avoids getting stuck if a signal arrives
* just before we enter sleep().
*/
if (!wakened)
{
sleep(PGARCH_AUTOWAKE_INTERVAL);
curtime = time(NULL);
if ((unsigned int) (curtime - last_copy_time) >=
(unsigned int) PGARCH_AUTOWAKE_INTERVAL)
wakened = true;
}
} while (PostmasterIsAlive(true));
}
/*
* pgarch_ArchiverCopyLoop
*
* Archives all outstanding xlogs then returns
*/
static void
pgarch_ArchiverCopyLoop(void)
{
char xlog[MAX_XFN_CHARS + 1];
/*
* loop through all xlogs with archive_status of .ready
* and archive them...mostly we expect this to be a single
* file, though it is possible some backend will add
* files onto the list of those that need archiving while we
* are still copying earlier archives
*/
while (pgarch_readyXlog(xlog))
{
int failures = 0;
for (;;)
{
if (pgarch_archiveXlog(xlog))
{
/* successful */
pgarch_archiveDone(xlog);
break; /* out of inner retry loop */
}
else
{
if (++failures >= NUM_ARCHIVE_RETRIES)
{
ereport(WARNING,
(errmsg("transaction log file \"%s\" could not be archived",
xlog)));
return; /* give up archiving for now */
}
sleep(1); /* wait a bit before retrying */
}
}
}
}
/*
* pgarch_archiveXlog
*
* Invokes system(3) to copy one archive file to wherever it should go
*
* Returns true if successful
*/
static bool
pgarch_archiveXlog(char *xlog)
{
char xlogarchcmd[MAXPGPATH];
char pathname[MAXPGPATH];
char *dp;
char *endp;
const char *sp;
int rc;
snprintf(pathname, MAXPGPATH, "%s/%s", XLogDir, xlog);
/*
* construct the command to be executed
*/
dp = xlogarchcmd;
endp = xlogarchcmd + MAXPGPATH - 1;
*endp = '\0';
for (sp = XLogArchiveCommand; *sp; sp++)
{
if (*sp == '%')
{
switch (sp[1])
{
case 'p':
/* %p: full path of source file */
sp++;
StrNCpy(dp, pathname, endp-dp);
dp += strlen(dp);
break;
case 'f':
/* %f: filename of source file */
sp++;
StrNCpy(dp, xlog, endp-dp);
dp += strlen(dp);
break;
case '%':
/* convert %% to a single % */
sp++;
if (dp < endp)
*dp++ = *sp;
break;
default:
/* otherwise treat the % as not special */
if (dp < endp)
*dp++ = *sp;
break;
}
}
else
{
if (dp < endp)
*dp++ = *sp;
}
}
*dp = '\0';
ereport(DEBUG3,
(errmsg_internal("executing archive command \"%s\"",
xlogarchcmd)));
rc = system(xlogarchcmd);
if (rc != 0) {
ereport(LOG,
(errmsg("archive command \"%s\" failed: return code %d",
xlogarchcmd, rc)));
return false;
}
ereport(LOG,
(errmsg("archived transaction log file \"%s\"", xlog)));
return true;
}
/*
* pgarch_readyXlog
*
* Return name of the oldest xlog file that has not yet been archived.
* No notification is set that file archiving is now in progress, so
* this would need to be extended if multiple concurrent archival
* tasks were created. If a failure occurs, we will completely
* re-copy the file at the next available opportunity.
*
* It is important that we return the oldest, so that we archive xlogs
* in order that they were written, for two reasons:
* 1) to maintain the sequential chain of xlogs required for recovery
* 2) because the oldest ones will sooner become candidates for
* recycling at time of checkpoint
*/
static bool
pgarch_readyXlog(char *xlog)
{
/*
* open xlog status directory and read through list of
* xlogs that have the .ready suffix, looking for earliest file.
* It is possible to optimise this code, though only a single
* file is expected on the vast majority of calls, so....
*/
char newxlog[MAX_XFN_CHARS + 6 + 1];
DIR *rldir;
struct dirent *rlde;
bool found = false;
rldir = AllocateDir(XLogArchiveStatusDir);
if (rldir == NULL)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not open archive status directory \"%s\": %m",
XLogArchiveStatusDir)));
errno = 0;
while ((rlde = readdir(rldir)) != NULL)
{
int basenamelen = (int) strlen(rlde->d_name) - 6;
if (basenamelen >= MIN_XFN_CHARS &&
basenamelen <= MAX_XFN_CHARS &&
strspn(rlde->d_name, VALID_XFN_CHARS) >= basenamelen &&
strcmp(rlde->d_name + basenamelen, ".ready") == 0)
{
if (!found) {
strcpy(newxlog, rlde->d_name);
found = true;
} else {
if (strcmp(rlde->d_name, newxlog) < 0)
strcpy(newxlog, rlde->d_name);
}
}
errno = 0;
}
#ifdef WIN32
/* This fix is in mingw cvs (runtime/mingwex/dirent.c rev 1.4), but
not in released version */
if (GetLastError() == ERROR_NO_MORE_FILES)
errno = 0;
#endif
if (errno)
ereport(ERROR,
(errcode_for_file_access(),
errmsg("could not read archive status directory \"%s\": %m",
XLogArchiveStatusDir)));
FreeDir(rldir);
if (found)
{
/* truncate off the .ready */
newxlog[strlen(newxlog) - 6] = '\0';
strcpy(xlog, newxlog);
}
return found;
}
/*
* pgarch_archiveDone
*
* Emit notification that an xlog file has been successfully archived.
* We do this by renaming the status file from NNN.ready to NNN.done.
* Eventually, a checkpoint process will notice this and delete both the
* NNN.done file and the xlog file itself.
*/
static void
pgarch_archiveDone(char *xlog)
{
char rlogready[MAXPGPATH];
char rlogdone[MAXPGPATH];
int rc;
snprintf(rlogready, MAXPGPATH, "%s/%s.ready", XLogArchiveStatusDir, xlog);
snprintf(rlogdone, MAXPGPATH, "%s/%s.done", XLogArchiveStatusDir, xlog);
rc = rename(rlogready, rlogdone);
if (rc < 0)
ereport(WARNING,
(errcode_for_file_access(),
errmsg("could not rename \"%s\": %m",
rlogready)));
}

87
src/backend/postmaster/postmaster.c
View File

@@ -37,7 +37,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.411 2004/07/12 19:14:56 momjian Exp $
* $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.412 2004/07/19 02:47:08 tgl Exp $
*
* NOTES
*
@@ -104,6 +104,7 @@
#include "miscadmin.h"
#include "nodes/nodes.h"
#include "postmaster/postmaster.h"
#include "postmaster/pgarch.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/pg_shmem.h"
@@ -198,6 +199,7 @@ char *preload_libraries_string = NULL;
/* PIDs of special child processes; 0 when not running */
static pid_t StartupPID = 0,
BgWriterPID = 0,
PgArchPID = 0,
PgStatPID = 0;
/* Startup/shutdown state */
@@ -826,7 +828,8 @@ PostmasterMain(int argc, char *argv[])
*
* CAUTION: when changing this list, check for side-effects on the signal
* handling setup of child processes. See tcop/postgres.c,
* bootstrap/bootstrap.c, postmaster/bgwriter.c, and postmaster/pgstat.c.
* bootstrap/bootstrap.c, postmaster/bgwriter.c, postmaster/pgarch.c,
* and postmaster/pgstat.c.
*/
pqinitmask();
PG_SETMASK(&BlockSig);
@@ -1217,6 +1220,11 @@ ServerLoop(void)
kill(BgWriterPID, SIGUSR2);
}
/* If we have lost the archiver, try to start a new one */
if (XLogArchivingActive() && PgArchPID == 0 &&
StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
PgArchPID = pgarch_start();
/* If we have lost the stats collector, try to start a new one */
if (PgStatPID == 0 &&
StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
@@ -1760,6 +1768,8 @@ SIGHUP_handler(SIGNAL_ARGS)
SignalChildren(SIGHUP);
if (BgWriterPID != 0)
kill(BgWriterPID, SIGHUP);
if (PgArchPID != 0)
kill(PgArchPID, SIGHUP);
/* PgStatPID does not currently need SIGHUP */
load_hba();
load_ident();
@@ -1818,6 +1828,9 @@ pmdie(SIGNAL_ARGS)
/* And tell it to shut down */
if (BgWriterPID != 0)
kill(BgWriterPID, SIGUSR2);
/* Tell pgarch to shut down too; nothing left for it to do */
if (PgArchPID != 0)
kill(PgArchPID, SIGQUIT);
/* Tell pgstat to shut down too; nothing left for it to do */
if (PgStatPID != 0)
kill(PgStatPID, SIGQUIT);
@@ -1862,6 +1875,9 @@ pmdie(SIGNAL_ARGS)
/* And tell it to shut down */
if (BgWriterPID != 0)
kill(BgWriterPID, SIGUSR2);
/* Tell pgarch to shut down too; nothing left for it to do */
if (PgArchPID != 0)
kill(PgArchPID, SIGQUIT);
/* Tell pgstat to shut down too; nothing left for it to do */
if (PgStatPID != 0)
kill(PgStatPID, SIGQUIT);
@@ -1880,6 +1896,8 @@ pmdie(SIGNAL_ARGS)
kill(StartupPID, SIGQUIT);
if (BgWriterPID != 0)
kill(BgWriterPID, SIGQUIT);
if (PgArchPID != 0)
kill(PgArchPID, SIGQUIT);
if (PgStatPID != 0)
kill(PgStatPID, SIGQUIT);
if (DLGetHead(BackendList))
@@ -1967,12 +1985,16 @@ reaper(SIGNAL_ARGS)
/*
* Go to shutdown mode if a shutdown request was pending.
* Otherwise, try to start the stats collector too.
* Otherwise, try to start the archiver and stats collector too.
*/
if (Shutdown > NoShutdown && BgWriterPID != 0)
kill(BgWriterPID, SIGUSR2);
else if (PgStatPID == 0 && Shutdown == NoShutdown)
PgStatPID = pgstat_start();
else if (Shutdown == NoShutdown) {
if (XLogArchivingActive() && PgArchPID == 0)
PgArchPID = pgarch_start();
if (PgStatPID == 0)
PgStatPID = pgstat_start();
}
continue;
}
@@ -2004,6 +2026,23 @@ reaper(SIGNAL_ARGS)
continue;
}
/*
* Was it the archiver? If so, just try to start a new
* one; no need to force reset of the rest of the system. (If fail,
* we'll try again in future cycles of the main loop.)
*/
if (PgArchPID != 0 && pid == PgArchPID)
{
PgArchPID = 0;
if (exitstatus != 0)
LogChildExit(LOG, gettext("archiver process"),
pid, exitstatus);
if (XLogArchivingActive() &&
StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
PgArchPID = pgarch_start();
continue;
}
/*
* Was it the statistics collector? If so, just try to start a new
* one; no need to force reset of the rest of the system. (If fail,
@@ -2029,8 +2068,9 @@ reaper(SIGNAL_ARGS)
if (FatalError)
{
/*
* Wait for all children exit, then reset shmem and
* StartupDataBase.
* Wait for all important children to exit, then reset shmem and
* StartupDataBase. (We can ignore the archiver and stats processes
* here since they are not connected to shmem.)
*/
if (DLGetHead(BackendList) || StartupPID != 0 || BgWriterPID != 0)
goto reaper_done;
@@ -2191,6 +2231,17 @@ HandleChildCrash(int pid,
kill(BgWriterPID, (SendStop ? SIGSTOP : SIGQUIT));
}
/* Force a power-cycle of the pgarch process too */
/* (Shouldn't be necessary, but just for luck) */
if (PgArchPID != 0 && !FatalError)
{
ereport(DEBUG2,
(errmsg_internal("sending %s to process %d",
"SIGQUIT",
(int) PgArchPID)));
kill(PgArchPID, SIGQUIT);
}
/* Force a power-cycle of the pgstat processes too */
/* (Shouldn't be necessary, but just for luck) */
if (PgStatPID != 0 && !FatalError)
@@ -2873,6 +2924,16 @@ SubPostmasterMain(int argc, char *argv[])
BootstrapMain(argc - 2, argv + 2);
proc_exit(0);
}
if (strcmp(argv[1], "-forkarch") == 0)
{
/* Close the postmaster's sockets */
ClosePostmasterPorts();
/* Do not want to attach to shared memory */
PgArchiverMain(argc, argv);
proc_exit(0);
}
if (strcmp(argv[1], "-forkbuf") == 0)
{
/* Close the postmaster's sockets */
@@ -2951,6 +3012,18 @@ sigusr1_handler(SIGNAL_ARGS)
if (Shutdown <= SmartShutdown)
SignalChildren(SIGUSR1);
}
if (PgArchPID != 0 && Shutdown == NoShutdown)
{
if (CheckPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER))
{
/*
* Send SIGUSR1 to archiver process, to wake it up and begin
* archiving next transaction log file.
*/
kill(PgArchPID, SIGUSR1);
}
}
PG_SETMASK(&UnBlockSig);