database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-11-07 19:06:32 +03:00
Code Activity

pgstat: store statistics in shared memory.

Browse Source 
Previously the statistics collector received statistics updates via UDP and
shared statistics data by writing them out to temporary files regularly. These
files can reach tens of megabytes and are written out up to twice a
second. This has repeatedly prevented us from adding additional useful
statistics.

Now statistics are stored in shared memory. Statistics for variable-numbered
objects are stored in a dshash hashtable (backed by dynamic shared
memory). Fixed-numbered stats are stored in plain shared memory.

The header for pgstat.c contains an overview of the architecture.

The stats collector is not needed anymore, remove it.

By utilizing the transactional statistics drop infrastructure introduced in a
prior commit statistics entries cannot "leak" anymore. Previously leaked
statistics were dropped by pgstat_vacuum_stat(), called from [auto-]vacuum. On
systems with many small relations pgstat_vacuum_stat() could be quite
expensive.

Now that replicas drop statistics entries for dropped objects, it is not
necessary anymore to reset stats when starting from a cleanly shut down
replica.

Subsequent commits will perform some further code cleanup, adapt docs and add
tests.

Bumps PGSTAT_FILE_FORMAT_ID.

Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-By: Andres Freund <andres@anarazel.de>
Reviewed-By: Thomas Munro <thomas.munro@gmail.com>
Reviewed-By: Justin Pryzby <pryzby@telsasoft.com>
Reviewed-By: "David G. Johnston" <david.g.johnston@gmail.com>
Reviewed-By: Tomas Vondra <tomas.vondra@2ndquadrant.com> (in a much earlier version)
Reviewed-By: Arthur Zakirov <a.zakirov@postgrespro.ru> (in a much earlier version)
Reviewed-By: Antonin Houska <ah@cybertec.at> (in a much earlier version)
Discussion: https://postgr.es/m/20220303021600.hs34ghqcw6zcokdh@alap3.anarazel.de
Discussion: https://postgr.es/m/20220308205351.2xcn6k4x5yivcxyd@alap3.anarazel.de
Discussion: https://postgr.es/m/20210319235115.y3wz7hpnnrshdyv6@alap3.anarazel.de
This commit is contained in:
Andres Freund
2022-04-06 21:29:46 -07:00
parent be902e2651
commit 5891c7a8ed
50 changed files with 4395 additions and 5485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
src/backend/postmaster/postmaster.c
View File

@@ -255,7 +255,6 @@ static pid_t StartupPID = 0,
WalReceiverPID = 0,
AutoVacPID = 0,
PgArchPID = 0,
PgStatPID = 0,
SysLoggerPID = 0;
/* Startup process's status */
@@ -510,7 +509,6 @@ typedef struct
PGPROC *AuxiliaryProcs;
PGPROC *PreparedXactProcs;
PMSignalData *PMSignalState;
InheritableSocket pgStatSock;
pid_t PostmasterPid;
TimestampTz PgStartTime;
TimestampTz PgReloadTime;
@@ -645,9 +643,8 @@ PostmasterMain(int argc, char *argv[])
* CAUTION: when changing this list, check for side-effects on the signal
* handling setup of child processes. See tcop/postgres.c,
* bootstrap/bootstrap.c, postmaster/bgwriter.c, postmaster/walwriter.c,
* postmaster/autovacuum.c, postmaster/pgarch.c, postmaster/pgstat.c,
* postmaster/syslogger.c, postmaster/bgworker.c and
* postmaster/checkpointer.c.
* postmaster/autovacuum.c, postmaster/pgarch.c, postmaster/syslogger.c,
* postmaster/bgworker.c and postmaster/checkpointer.c.
*/
pqinitmask();
PG_SETMASK(&BlockSig);
@@ -1384,12 +1381,6 @@ PostmasterMain(int argc, char *argv[])
*/
RemovePgTempFiles();
/*
* Initialize stats collection subsystem (this does NOT start the
* collector process!)
*/
pgstat_init();
/*
* Initialize the autovacuum subsystem (again, no process start yet)
*/
@@ -1845,11 +1836,6 @@ ServerLoop(void)
start_autovac_launcher = false; /* signal processed */
}
/* If we have lost the stats collector, try to start a new one */
if (PgStatPID == 0 &&
(pmState == PM_RUN || pmState == PM_HOT_STANDBY))
PgStatPID = pgstat_start();
/* If we have lost the archiver, try to start a new one. */
if (PgArchPID == 0 && PgArchStartupAllowed())
PgArchPID = StartArchiver();
@@ -2772,8 +2758,6 @@ SIGHUP_handler(SIGNAL_ARGS)
signal_child(PgArchPID, SIGHUP);
if (SysLoggerPID != 0)
signal_child(SysLoggerPID, SIGHUP);
if (PgStatPID != 0)
signal_child(PgStatPID, SIGHUP);
/* Reload authentication config files too */
if (!load_hba())
@@ -3097,8 +3081,6 @@ reaper(SIGNAL_ARGS)
AutoVacPID = StartAutoVacLauncher();
if (PgArchStartupAllowed() && PgArchPID == 0)
PgArchPID = StartArchiver();
if (PgStatPID == 0)
PgStatPID = pgstat_start();
/* workers may be scheduled to start now */
maybe_start_bgworkers();
@@ -3165,13 +3147,6 @@ reaper(SIGNAL_ARGS)
SignalChildren(SIGUSR2);
pmState = PM_SHUTDOWN_2;
/*
* We can also shut down the stats collector now; there's
* nothing left for it to do.
*/
if (PgStatPID != 0)
signal_child(PgStatPID, SIGQUIT);
}
else
{
@@ -3250,22 +3225,6 @@ reaper(SIGNAL_ARGS)
continue;
}
/*
* Was it the statistics collector? If so, just try to start a new
* one; no need to force reset of the rest of the system. (If fail,
* we'll try again in future cycles of the main loop.)
*/
if (pid == PgStatPID)
{
PgStatPID = 0;
if (!EXIT_STATUS_0(exitstatus))
LogChildExit(LOG, _("statistics collector process"),
pid, exitstatus);
if (pmState == PM_RUN || pmState == PM_HOT_STANDBY)
PgStatPID = pgstat_start();
continue;
}
/* Was it the system logger? If so, try to start a new one */
if (pid == SysLoggerPID)
{
@@ -3707,22 +3666,6 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
signal_child(PgArchPID, (SendStop ? SIGSTOP : SIGQUIT));
}
/*
* Force a power-cycle of the pgstat process too. (This isn't absolutely
* necessary, but it seems like a good idea for robustness, and it
* simplifies the state-machine logic in the case where a shutdown request
* arrives during crash processing.)
*/
if (PgStatPID != 0 && take_action)
{
ereport(DEBUG2,
(errmsg_internal("sending %s to process %d",
"SIGQUIT",
(int) PgStatPID)));
signal_child(PgStatPID, SIGQUIT);
allow_immediate_pgstat_restart();
}
/* We do NOT restart the syslogger */
if (Shutdown != ImmediateShutdown)
@@ -3934,12 +3877,10 @@ PostmasterStateMachine(void)
FatalError = true;
pmState = PM_WAIT_DEAD_END;
/* Kill the walsenders, archiver and stats collector too */
/* Kill the walsenders and archiver too */
SignalChildren(SIGQUIT);
if (PgArchPID != 0)
signal_child(PgArchPID, SIGQUIT);
if (PgStatPID != 0)
signal_child(PgStatPID, SIGQUIT);
}
}
}
@@ -3963,8 +3904,7 @@ PostmasterStateMachine(void)
{
/*
* PM_WAIT_DEAD_END state ends when the BackendList is entirely empty
* (ie, no dead_end children remain), and the archiver and stats
* collector are gone too.
* (ie, no dead_end children remain), and the archiver is gone too.
*
* The reason we wait for those two is to protect them against a new
* postmaster starting conflicting subprocesses; this isn't an
@@ -3974,8 +3914,7 @@ PostmasterStateMachine(void)
* normal state transition leading up to PM_WAIT_DEAD_END, or during
* FatalError processing.
*/
if (dlist_is_empty(&BackendList) &&
PgArchPID == 0 && PgStatPID == 0)
if (dlist_is_empty(&BackendList) && PgArchPID == 0)
{
/* These other guys should be dead already */
Assert(StartupPID == 0);
@@ -4183,8 +4122,6 @@ TerminateChildren(int signal)
signal_child(AutoVacPID, signal);
if (PgArchPID != 0)
signal_child(PgArchPID, signal);
if (PgStatPID != 0)
signal_child(PgStatPID, signal);
}
/*
@@ -5115,12 +5052,6 @@ SubPostmasterMain(int argc, char *argv[])
StartBackgroundWorker();
}
if (strcmp(argv[1], "--forkcol") == 0)
{
/* Do not want to attach to shared memory */
PgstatCollectorMain(argc, argv); /* does not return */
}
if (strcmp(argv[1], "--forklog") == 0)
{
/* Do not want to attach to shared memory */
@@ -5224,12 +5155,6 @@ sigusr1_handler(SIGNAL_ARGS)
if (CheckPostmasterSignal(PMSIGNAL_BEGIN_HOT_STANDBY) &&
pmState == PM_RECOVERY && Shutdown == NoShutdown)
{
/*
* Likewise, start other special children as needed.
*/
Assert(PgStatPID == 0);
PgStatPID = pgstat_start();
ereport(LOG,
(errmsg("database system is ready to accept read-only connections")));
@@ -6145,7 +6070,6 @@ extern slock_t *ShmemLock;
extern slock_t *ProcStructLock;
extern PGPROC *AuxiliaryProcs;
extern PMSignalData *PMSignalState;
extern pgsocket pgStatSock;
extern pg_time_t first_syslogger_file_time;
#ifndef WIN32
@@ -6201,8 +6125,6 @@ save_backend_variables(BackendParameters *param, Port *port,
param->AuxiliaryProcs = AuxiliaryProcs;
param->PreparedXactProcs = PreparedXactProcs;
param->PMSignalState = PMSignalState;
if (!write_inheritable_socket(&param->pgStatSock, pgStatSock, childPid))
return false;
param->PostmasterPid = PostmasterPid;
param->PgStartTime = PgStartTime;
@@ -6436,7 +6358,6 @@ restore_backend_variables(BackendParameters *param, Port *port)
AuxiliaryProcs = param->AuxiliaryProcs;
PreparedXactProcs = param->PreparedXactProcs;
PMSignalState = param->PMSignalState;
read_inheritable_socket(&pgStatSock, &param->pgStatSock);
PostmasterPid = param->PostmasterPid;
PgStartTime = param->PgStartTime;
@@ -6475,8 +6396,6 @@ restore_backend_variables(BackendParameters *param, Port *port)
if (postmaster_alive_fds[1] >= 0)
ReserveExternalFD();
#endif
if (pgStatSock != PGINVALID_SOCKET)
ReserveExternalFD();
}