database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-11-07 19:06:32 +03:00
Code Activity

Add a new WAL summarizer process.

Browse Source 
When active, this process writes WAL summary files to
$PGDATA/pg_wal/summaries. Each summary file contains information for a
certain range of LSNs on a certain TLI. For each relation, it stores a
"limit block" which is 0 if a relation is created or destroyed within
a certain range of WAL records, or otherwise the shortest length to
which the relation was truncated during that range of WAL records, or
otherwise InvalidBlockNumber. In addition, it stores a list of blocks
which have been modified during that range of WAL records, but
excluding blocks which were removed by truncation after they were
modified and never subsequently modified again.

In other words, it tells us which blocks need to copied in case of an
incremental backup covering that range of WAL records. But this
doesn't yet add the capability to actually perform an incremental
backup; the next patch will do that.

A new parameter summarize_wal enables or disables this new background
process.  The background process also automatically deletes summary
files that are older than wal_summarize_keep_time, if that parameter
has a non-zero value and the summarizer is configured to run.

Patch by me, with some design help from Dilip Kumar and Andres Freund.
Reviewed by Matthias van de Meent, Dilip Kumar, Jakub Wartak, Peter
Eisentraut, and Álvaro Herrera.

Discussion: http://postgr.es/m/CA+TgmoYOYZfMCyOXFyC-P+-mdrZqm5pP2N7S-r0z3_402h9rsA@mail.gmail.com
This commit is contained in:
Robert Haas
2023-12-20 08:41:09 -05:00
parent 00498b7185
commit 174c480508
30 changed files with 3743 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/backend/postmaster/postmaster.c
View File

@@ -113,6 +113,7 @@
#include "postmaster/pgarch.h"
#include "postmaster/postmaster.h"
#include "postmaster/syslogger.h"
#include "postmaster/walsummarizer.h"
#include "replication/logicallauncher.h"
#include "replication/walsender.h"
#include "storage/fd.h"
@@ -250,6 +251,7 @@ static pid_t StartupPID = 0,
CheckpointerPID = 0,
WalWriterPID = 0,
WalReceiverPID = 0,
WalSummarizerPID = 0,
AutoVacPID = 0,
PgArchPID = 0,
SysLoggerPID = 0;
@@ -441,6 +443,7 @@ static bool CreateOptsFile(int argc, char *argv[], char *fullprogname);
static pid_t StartChildProcess(AuxProcType type);
static void StartAutovacuumWorker(void);
static void MaybeStartWalReceiver(void);
static void MaybeStartWalSummarizer(void);
static void InitPostmasterDeathWatchHandle(void);
/*
@@ -564,6 +567,7 @@ static void ShmemBackendArrayRemove(Backend *bn);
#define StartCheckpointer() StartChildProcess(CheckpointerProcess)
#define StartWalWriter() StartChildProcess(WalWriterProcess)
#define StartWalReceiver() StartChildProcess(WalReceiverProcess)
#define StartWalSummarizer() StartChildProcess(WalSummarizerProcess)
/* Macros to check exit status of a child process */
#define EXIT_STATUS_0(st) ((st) == 0)
@@ -933,6 +937,9 @@ PostmasterMain(int argc, char *argv[])
if (max_wal_senders > 0 && wal_level == WAL_LEVEL_MINIMAL)
ereport(ERROR,
(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
if (summarize_wal && wal_level == WAL_LEVEL_MINIMAL)
ereport(ERROR,
(errmsg("WAL cannot be summarized when wal_level is \"minimal\"")));
/*
* Other one-time internal sanity checks can go here, if they are fast.
@@ -1835,6 +1842,9 @@ ServerLoop(void)
if (WalReceiverRequested)
MaybeStartWalReceiver();
/* If we need to start a WAL summarizer, try to do that now */
MaybeStartWalSummarizer();
/* Get other worker processes running, if needed */
if (StartWorkerNeeded || HaveCrashedWorker)
maybe_start_bgworkers();
@@ -2659,6 +2669,8 @@ process_pm_reload_request(void)
signal_child(WalWriterPID, SIGHUP);
if (WalReceiverPID != 0)
signal_child(WalReceiverPID, SIGHUP);
if (WalSummarizerPID != 0)
signal_child(WalSummarizerPID, SIGHUP);
if (AutoVacPID != 0)
signal_child(AutoVacPID, SIGHUP);
if (PgArchPID != 0)
@@ -3012,6 +3024,7 @@ process_pm_child_exit(void)
BgWriterPID = StartBackgroundWriter();
if (WalWriterPID == 0)
WalWriterPID = StartWalWriter();
MaybeStartWalSummarizer();
/*
* Likewise, start other special children as needed. In a restart
@@ -3130,6 +3143,20 @@ process_pm_child_exit(void)
continue;
}
/*
* Was it the wal summarizer? Normal exit can be ignored; we'll start
* a new one at the next iteration of the postmaster's main loop, if
* necessary. Any other exit condition is treated as a crash.
*/
if (pid == WalSummarizerPID)
{
WalSummarizerPID = 0;
if (!EXIT_STATUS_0(exitstatus))
HandleChildCrash(pid, exitstatus,
_("WAL summarizer process"));
continue;
}
/*
* Was it the autovacuum launcher? Normal exit can be ignored; we'll
* start a new one at the next iteration of the postmaster's main
@@ -3525,6 +3552,12 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
else if (WalReceiverPID != 0 && take_action)
sigquit_child(WalReceiverPID);
/* Take care of the walsummarizer too */
if (pid == WalSummarizerPID)
WalSummarizerPID = 0;
else if (WalSummarizerPID != 0 && take_action)
sigquit_child(WalSummarizerPID);
/* Take care of the autovacuum launcher too */
if (pid == AutoVacPID)
AutoVacPID = 0;
@@ -3675,6 +3708,8 @@ PostmasterStateMachine(void)
signal_child(StartupPID, SIGTERM);
if (WalReceiverPID != 0)
signal_child(WalReceiverPID, SIGTERM);
if (WalSummarizerPID != 0)
signal_child(WalSummarizerPID, SIGTERM);
/* checkpointer, archiver, stats, and syslogger may continue for now */
/* Now transition to PM_WAIT_BACKENDS state to wait for them to die */
@@ -3701,6 +3736,7 @@ PostmasterStateMachine(void)
if (CountChildren(BACKEND_TYPE_ALL - BACKEND_TYPE_WALSND) == 0 &&
StartupPID == 0 &&
WalReceiverPID == 0 &&
WalSummarizerPID == 0 &&
BgWriterPID == 0 &&
(CheckpointerPID == 0 ||
(!FatalError && Shutdown < ImmediateShutdown)) &&
@@ -3798,6 +3834,7 @@ PostmasterStateMachine(void)
/* These other guys should be dead already */
Assert(StartupPID == 0);
Assert(WalReceiverPID == 0);
Assert(WalSummarizerPID == 0);
Assert(BgWriterPID == 0);
Assert(CheckpointerPID == 0);
Assert(WalWriterPID == 0);
@@ -4019,6 +4056,8 @@ TerminateChildren(int signal)
signal_child(WalWriterPID, signal);
if (WalReceiverPID != 0)
signal_child(WalReceiverPID, signal);
if (WalSummarizerPID != 0)
signal_child(WalSummarizerPID, signal);
if (AutoVacPID != 0)
signal_child(AutoVacPID, signal);
if (PgArchPID != 0)
@@ -5326,6 +5365,10 @@ StartChildProcess(AuxProcType type)
ereport(LOG,
(errmsg("could not fork WAL receiver process: %m")));
break;
case WalSummarizerProcess:
ereport(LOG,
(errmsg("could not fork WAL summarizer process: %m")));
break;
default:
ereport(LOG,
(errmsg("could not fork process: %m")));
@@ -5462,6 +5505,19 @@ MaybeStartWalReceiver(void)
}
}
/*
* MaybeStartWalSummarizer
* Start the WAL summarizer process, if not running and our state allows.
*/
static void
MaybeStartWalSummarizer(void)
{
if (summarize_wal && WalSummarizerPID == 0 &&
(pmState == PM_RUN || pmState == PM_HOT_STANDBY) &&
Shutdown <= SmartShutdown)
WalSummarizerPID = StartWalSummarizer();
}
/*
* Create the opts file