database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-03 20:02:46 +03:00
Code Activity

pgstat: store statistics in shared memory.

Browse Source 
Previously the statistics collector received statistics updates via UDP and
shared statistics data by writing them out to temporary files regularly. These
files can reach tens of megabytes and are written out up to twice a
second. This has repeatedly prevented us from adding additional useful
statistics.

Now statistics are stored in shared memory. Statistics for variable-numbered
objects are stored in a dshash hashtable (backed by dynamic shared
memory). Fixed-numbered stats are stored in plain shared memory.

The header for pgstat.c contains an overview of the architecture.

The stats collector is not needed anymore, remove it.

By utilizing the transactional statistics drop infrastructure introduced in a
prior commit statistics entries cannot "leak" anymore. Previously leaked
statistics were dropped by pgstat_vacuum_stat(), called from [auto-]vacuum. On
systems with many small relations pgstat_vacuum_stat() could be quite
expensive.

Now that replicas drop statistics entries for dropped objects, it is not
necessary anymore to reset stats when starting from a cleanly shut down
replica.

Subsequent commits will perform some further code cleanup, adapt docs and add
tests.

Bumps PGSTAT_FILE_FORMAT_ID.

Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-By: Andres Freund <andres@anarazel.de>
Reviewed-By: Thomas Munro <thomas.munro@gmail.com>
Reviewed-By: Justin Pryzby <pryzby@telsasoft.com>
Reviewed-By: "David G. Johnston" <david.g.johnston@gmail.com>
Reviewed-By: Tomas Vondra <tomas.vondra@2ndquadrant.com> (in a much earlier version)
Reviewed-By: Arthur Zakirov <a.zakirov@postgrespro.ru> (in a much earlier version)
Reviewed-By: Antonin Houska <ah@cybertec.at> (in a much earlier version)
Discussion: https://postgr.es/m/20220303021600.hs34ghqcw6zcokdh@alap3.anarazel.de
Discussion: https://postgr.es/m/20220308205351.2xcn6k4x5yivcxyd@alap3.anarazel.de
Discussion: https://postgr.es/m/20210319235115.y3wz7hpnnrshdyv6@alap3.anarazel.de
This commit is contained in:
Andres Freund
2022-04-06 21:29:46 -07:00
parent be902e2651
commit 5891c7a8ed
50 changed files with 4395 additions and 5485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
src/backend/utils/activity/pgstat_checkpointer.c
View File

@ -20,12 +20,7 @@
#include "utils/pgstat_internal.h"
/*
* Checkpointer global statistics counters. Stored directly in a stats
* message structure so they can be sent without needing to copy things
* around. We assume this init to zeroes.
*/
PgStat_MsgCheckpointer PendingCheckpointerStats;
PgStat_CheckpointerStats PendingCheckpointerStats = {0};
/*
@ -35,24 +30,92 @@ void
pgstat_report_checkpointer(void)
{
/* We assume this initializes to zeroes */
static const PgStat_MsgCheckpointer all_zeroes;
static const PgStat_CheckpointerStats all_zeroes;
PgStatShared_Checkpointer *stats_shmem = &pgStatLocal.shmem->checkpointer;
Assert(!pgStatLocal.shmem->is_shutdown);
pgstat_assert_is_up();
/*
* This function can be called even if nothing at all has happened. In
* this case, avoid sending a completely empty message to the stats
* collector.
* this case, avoid unnecessarily modifying the stats entry.
*/
if (memcmp(&PendingCheckpointerStats, &all_zeroes, sizeof(PgStat_MsgCheckpointer)) == 0)
if (memcmp(&PendingCheckpointerStats, &all_zeroes,
sizeof(all_zeroes)) == 0)
return;
/*
* Prepare and send the message
*/
pgstat_setheader(&PendingCheckpointerStats.m_hdr, PGSTAT_MTYPE_CHECKPOINTER);
pgstat_send(&PendingCheckpointerStats, sizeof(PendingCheckpointerStats));
pgstat_begin_changecount_write(&stats_shmem->changecount);
#define CHECKPOINTER_ACC(fld) stats_shmem->stats.fld += PendingCheckpointerStats.fld
CHECKPOINTER_ACC(timed_checkpoints);
CHECKPOINTER_ACC(requested_checkpoints);
CHECKPOINTER_ACC(checkpoint_write_time);
CHECKPOINTER_ACC(checkpoint_sync_time);
CHECKPOINTER_ACC(buf_written_checkpoints);
CHECKPOINTER_ACC(buf_written_backend);
CHECKPOINTER_ACC(buf_fsync_backend);
#undef CHECKPOINTER_ACC
pgstat_end_changecount_write(&stats_shmem->changecount);
/*
* Clear out the statistics buffer, so it can be re-used.
*/
MemSet(&PendingCheckpointerStats, 0, sizeof(PendingCheckpointerStats));
}
/*
* pgstat_fetch_stat_checkpointer() -
*
* Support function for the SQL-callable pgstat* functions. Returns
* a pointer to the checkpointer statistics struct.
*/
PgStat_CheckpointerStats *
pgstat_fetch_stat_checkpointer(void)
{
pgstat_snapshot_fixed(PGSTAT_KIND_CHECKPOINTER);
return &pgStatLocal.snapshot.checkpointer;
}
void
pgstat_checkpointer_reset_all_cb(TimestampTz ts)
{
PgStatShared_Checkpointer *stats_shmem = &pgStatLocal.shmem->checkpointer;
/* see explanation above PgStatShared_Checkpointer for the reset protocol */
LWLockAcquire(&stats_shmem->lock, LW_EXCLUSIVE);
pgstat_copy_changecounted_stats(&stats_shmem->reset_offset,
&stats_shmem->stats,
sizeof(stats_shmem->stats),
&stats_shmem->changecount);
LWLockRelease(&stats_shmem->lock);
}
void
pgstat_checkpointer_snapshot_cb(void)
{
PgStatShared_Checkpointer *stats_shmem = &pgStatLocal.shmem->checkpointer;
PgStat_CheckpointerStats *reset_offset = &stats_shmem->reset_offset;
PgStat_CheckpointerStats reset;
pgstat_copy_changecounted_stats(&pgStatLocal.snapshot.checkpointer,
&stats_shmem->stats,
sizeof(stats_shmem->stats),
&stats_shmem->changecount);
LWLockAcquire(&stats_shmem->lock, LW_SHARED);
memcpy(&reset, reset_offset, sizeof(stats_shmem->stats));
LWLockRelease(&stats_shmem->lock);
/* compensate by reset offsets */
#define CHECKPOINTER_COMP(fld) pgStatLocal.snapshot.checkpointer.fld -= reset.fld;
CHECKPOINTER_COMP(timed_checkpoints);
CHECKPOINTER_COMP(requested_checkpoints);
CHECKPOINTER_COMP(checkpoint_write_time);
CHECKPOINTER_COMP(checkpoint_sync_time);
CHECKPOINTER_COMP(buf_written_checkpoints);
CHECKPOINTER_COMP(buf_written_backend);
CHECKPOINTER_COMP(buf_fsync_backend);
#undef CHECKPOINTER_COMP
}