database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-11-19 13:42:17 +03:00
Code Activity

pgstat: store statistics in shared memory.

Browse Source 
Previously the statistics collector received statistics updates via UDP and
shared statistics data by writing them out to temporary files regularly. These
files can reach tens of megabytes and are written out up to twice a
second. This has repeatedly prevented us from adding additional useful
statistics.

Now statistics are stored in shared memory. Statistics for variable-numbered
objects are stored in a dshash hashtable (backed by dynamic shared
memory). Fixed-numbered stats are stored in plain shared memory.

The header for pgstat.c contains an overview of the architecture.

The stats collector is not needed anymore, remove it.

By utilizing the transactional statistics drop infrastructure introduced in a
prior commit statistics entries cannot "leak" anymore. Previously leaked
statistics were dropped by pgstat_vacuum_stat(), called from [auto-]vacuum. On
systems with many small relations pgstat_vacuum_stat() could be quite
expensive.

Now that replicas drop statistics entries for dropped objects, it is not
necessary anymore to reset stats when starting from a cleanly shut down
replica.

Subsequent commits will perform some further code cleanup, adapt docs and add
tests.

Bumps PGSTAT_FILE_FORMAT_ID.

Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Andres Freund <andres@anarazel.de>
Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-By: Andres Freund <andres@anarazel.de>
Reviewed-By: Thomas Munro <thomas.munro@gmail.com>
Reviewed-By: Justin Pryzby <pryzby@telsasoft.com>
Reviewed-By: "David G. Johnston" <david.g.johnston@gmail.com>
Reviewed-By: Tomas Vondra <tomas.vondra@2ndquadrant.com> (in a much earlier version)
Reviewed-By: Arthur Zakirov <a.zakirov@postgrespro.ru> (in a much earlier version)
Reviewed-By: Antonin Houska <ah@cybertec.at> (in a much earlier version)
Discussion: https://postgr.es/m/20220303021600.hs34ghqcw6zcokdh@alap3.anarazel.de
Discussion: https://postgr.es/m/20220308205351.2xcn6k4x5yivcxyd@alap3.anarazel.de
Discussion: https://postgr.es/m/20210319235115.y3wz7hpnnrshdyv6@alap3.anarazel.de
This commit is contained in:
Andres Freund
2022-04-06 21:29:46 -07:00
parent be902e2651
commit 5891c7a8ed
50 changed files with 4395 additions and 5485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
src/backend/utils/activity/pgstat_replslot.c
View File

@@ -8,6 +8,14 @@
* storage implementation and the details about individual types of
* statistics.
*
* Replication slot stats work a bit different than other other
* variable-numbered stats. Slots do not have oids (so they can be created on
* physical replicas). Use the slot index as object id while running. However,
* the slot index can change when restarting. That is addressed by using the
* name when (de-)serializing. After a restart it is possible for slots to
* have been dropped while shut down, which is addressed by not restoring
* stats for slots that cannot be found by name when starting up.
*
* Copyright (c) 2001-2022, PostgreSQL Global Development Group
*
* IDENTIFICATION
@@ -22,6 +30,9 @@
#include "utils/pgstat_internal.h"
static int get_replslot_index(const char *name);
/*
* Reset counters for a single replication slot.
*
@@ -32,18 +43,10 @@ void
pgstat_reset_replslot(const char *name)
{
ReplicationSlot *slot;
PgStat_MsgResetreplslotcounter msg;
AssertArg(name != NULL);
if (pgStatSock == PGINVALID_SOCKET)
return;
/*
* Check if the slot exists with the given name. It is possible that by
* the time this message is executed the slot is dropped but at least this
* check will ensure that the given name is for a valid slot.
*/
/* Check if the slot exits with the given name. */
slot = SearchNamedReplicationSlot(name, true);
if (!slot)
@@ -59,10 +62,9 @@ pgstat_reset_replslot(const char *name)
if (SlotIsPhysical(slot))
return;
pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_RESETREPLSLOTCOUNTER);
namestrcpy(&msg.m_slotname, name);
msg.clearall = false;
pgstat_send(&msg, sizeof(msg));
/* reset this one entry */
pgstat_reset(PGSTAT_KIND_REPLSLOT, InvalidOid,
ReplicationSlotIndex(slot));
}
/*
@@ -71,24 +73,34 @@ pgstat_reset_replslot(const char *name)
void
pgstat_report_replslot(ReplicationSlot *slot, const PgStat_StatReplSlotEntry *repSlotStat)
{
PgStat_MsgReplSlot msg;
PgStat_EntryRef *entry_ref;
PgStatShared_ReplSlot *shstatent;
PgStat_StatReplSlotEntry *statent;
entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_REPLSLOT, InvalidOid,
ReplicationSlotIndex(slot), false);
shstatent = (PgStatShared_ReplSlot *) entry_ref->shared_stats;
statent = &shstatent->stats;
/*
* Prepare and send the message
* Any mismatch should have been fixed in pgstat_create_replslot() or
* pgstat_acquire_replslot().
*/
pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_REPLSLOT);
namestrcpy(&msg.m_slotname, NameStr(repSlotStat->slotname));
msg.m_create = false;
msg.m_drop = false;
msg.m_spill_txns = repSlotStat->spill_txns;
msg.m_spill_count = repSlotStat->spill_count;
msg.m_spill_bytes = repSlotStat->spill_bytes;
msg.m_stream_txns = repSlotStat->stream_txns;
msg.m_stream_count = repSlotStat->stream_count;
msg.m_stream_bytes = repSlotStat->stream_bytes;
msg.m_total_txns = repSlotStat->total_txns;
msg.m_total_bytes = repSlotStat->total_bytes;
pgstat_send(&msg, sizeof(PgStat_MsgReplSlot));
Assert(namestrcmp(&statent->slotname, NameStr(slot->data.name)) == 0);
/* Update the replication slot statistics */
#define REPLSLOT_ACC(fld) statent->fld += repSlotStat->fld
REPLSLOT_ACC(spill_txns);
REPLSLOT_ACC(spill_count);
REPLSLOT_ACC(spill_bytes);
REPLSLOT_ACC(stream_txns);
REPLSLOT_ACC(stream_count);
REPLSLOT_ACC(stream_bytes);
REPLSLOT_ACC(total_txns);
REPLSLOT_ACC(total_bytes);
#undef REPLSLOT_ACC
pgstat_unlock_entry(entry_ref);
}
/*
@@ -100,13 +112,50 @@ pgstat_report_replslot(ReplicationSlot *slot, const PgStat_StatReplSlotEntry *re
void
pgstat_create_replslot(ReplicationSlot *slot)
{
PgStat_MsgReplSlot msg;
PgStat_EntryRef *entry_ref;
PgStatShared_ReplSlot *shstatent;
pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_REPLSLOT);
namestrcpy(&msg.m_slotname, NameStr(slot->data.name));
msg.m_create = true;
msg.m_drop = false;
pgstat_send(&msg, sizeof(PgStat_MsgReplSlot));
entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_REPLSLOT, InvalidOid,
ReplicationSlotIndex(slot), false);
shstatent = (PgStatShared_ReplSlot *) entry_ref->shared_stats;
/*
* NB: need to accept that there might be stats from an older slot, e.g.
* if we previously crashed after dropping a slot.
*/
memset(&shstatent->stats, 0, sizeof(shstatent->stats));
namestrcpy(&shstatent->stats.slotname, NameStr(slot->data.name));
pgstat_unlock_entry(entry_ref);
}
/*
* Report replication slot has been acquired.
*/
void
pgstat_acquire_replslot(ReplicationSlot *slot)
{
PgStat_EntryRef *entry_ref;
PgStatShared_ReplSlot *shstatent;
PgStat_StatReplSlotEntry *statent;
entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_REPLSLOT, InvalidOid,
ReplicationSlotIndex(slot), false);
shstatent = (PgStatShared_ReplSlot *) entry_ref->shared_stats;
statent = &shstatent->stats;
/*
* NB: need to accept that there might be stats from an older slot, e.g.
* if we previously crashed after dropping a slot.
*/
if (NameStr(statent->slotname)[0] == 0 ||
namestrcmp(&statent->slotname, NameStr(slot->data.name)) != 0)
{
memset(statent, 0, sizeof(*statent));
namestrcpy(&statent->slotname, NameStr(slot->data.name));
}
pgstat_unlock_entry(entry_ref);
}
/*
@@ -115,11 +164,65 @@ pgstat_create_replslot(ReplicationSlot *slot)
void
pgstat_drop_replslot(ReplicationSlot *slot)
{
PgStat_MsgReplSlot msg;
pgstat_setheader(&msg.m_hdr, PGSTAT_MTYPE_REPLSLOT);
namestrcpy(&msg.m_slotname, NameStr(slot->data.name));
msg.m_create = false;
msg.m_drop = true;
pgstat_send(&msg, sizeof(PgStat_MsgReplSlot));
pgstat_drop_entry(PGSTAT_KIND_REPLSLOT, InvalidOid,
ReplicationSlotIndex(slot));
}
/*
* Support function for the SQL-callable pgstat* functions. Returns
* a pointer to the replication slot statistics struct.
*/
PgStat_StatReplSlotEntry *
pgstat_fetch_replslot(NameData slotname)
{
int idx = get_replslot_index(NameStr(slotname));
if (idx == -1)
return NULL;
return (PgStat_StatReplSlotEntry *)
pgstat_fetch_entry(PGSTAT_KIND_REPLSLOT, InvalidOid, idx);
}
void
pgstat_replslot_to_serialized_name_cb(const PgStatShared_Common *header, NameData *name)
{
namestrcpy(name, NameStr(((PgStatShared_ReplSlot *) header)->stats.slotname));
}
bool
pgstat_replslot_from_serialized_name_cb(const NameData *name, PgStat_HashKey *key)
{
int idx = get_replslot_index(NameStr(*name));
/* slot might have been deleted */
if (idx == -1)
return false;
key->kind = PGSTAT_KIND_REPLSLOT;
key->dboid = InvalidOid;
key->objoid = idx;
return true;
}
void
pgstat_replslot_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts)
{
((PgStatShared_ReplSlot *) header)->stats.stat_reset_timestamp = ts;
}
static int
get_replslot_index(const char *name)
{
ReplicationSlot *slot;
AssertArg(name != NULL);
slot = SearchNamedReplicationSlot(name, true);
if (!slot)
return -1;
return ReplicationSlotIndex(slot);
}