diff --git a/CHANGELOG.md b/CHANGELOG.md index 75fb41de..38128bfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ enabling wrapping later will still create this one-time reset boundary. * [CHANGE] Wrap non-negative 64-bit counters at `2^53` to preserve single-unit precision by @gnanirahulnutakki in https://github.com/prometheus-community/postgres_exporter/pull/1351 * [CHANGE] stat_replication: add `pid` label to disambiguate replication connections that otherwise share identical labels by @sysadmind in https://github.com/prometheus-community/postgres_exporter/pull/1353 * [BUGFIX] Fix `long_running_transactions` to count only transactions older than a configurable threshold, by @ArthurSens in https://github.com/prometheus-community/postgres_exporter/pull/1379, based on the contribution by @moreinhardt in https://github.com/prometheus-community/postgres_exporter/pull/1210 +* [FEATURE] Export `pg_settings.pending_restart` as `pg_settings_pending_restart{name}` from the `settings` collector by @MarcWort in https://github.com/prometheus-community/postgres_exporter/pull/1396 ## 0.20.1 / 2026-07-07 diff --git a/collector/pg_setting.go b/collector/pg_setting.go index 48998bc3..bf5f5035 100644 --- a/collector/pg_setting.go +++ b/collector/pg_setting.go @@ -15,6 +15,7 @@ package collector import ( "context" + "errors" "fmt" "log/slog" "math" @@ -50,12 +51,27 @@ var ( // Settings intentionally ignored due to invalid format: // - `sync_commit_cancel_wait`, specific to Azure Postgres, see https://github.com/prometheus-community/postgres_exporter/issues/523 // - `google_dataplex.max_messages`, specific to Google Cloud SQL, see https://github.com/prometheus-community/postgres_exporter/issues/1240 - pgSettingsQuery = "SELECT name, setting, COALESCE(unit, ''), COALESCE(short_desc, ''), vartype FROM pg_settings WHERE vartype IN ('bool', 'integer', 'real') AND name NOT IN ('sync_commit_cancel_wait', 'google_dataplex.max_messages');" + pgSettingsQuery = "SELECT name, setting, COALESCE(unit, ''), COALESCE(short_desc, ''), vartype, pending_restart FROM pg_settings WHERE name NOT IN ('sync_commit_cancel_wait', 'google_dataplex.max_messages');" + + // pgSettingsPendingRestartDesc reports which settings have been changed + // but only take effect after a server restart, as flagged by the + // pg_settings.pending_restart column. + pgSettingsPendingRestartDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, settingsSubsystem, "pending_restart"), + "PostgreSQL settings that require a server restart to take effect.", + []string{"name"}, + nil, + ) ) +// errUnsupportedVartype distinguishes expected non-numeric settings +// from malformed numeric values. +var errUnsupportedVartype = errors.New("unsupported vartype") + // Update implements Collector and exposes PostgreSQL runtime settings. func (c PGSettingsCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { db := instance.getDB() + rows, err := db.QueryContext(ctx, pgSettingsQuery) if err != nil { return err @@ -64,12 +80,19 @@ func (c PGSettingsCollector) Update(ctx context.Context, instance *instance, ch for rows.Next() { s := &pgSetting{} - if err := rows.Scan(&s.name, &s.setting, &s.unit, &s.shortDesc, &s.vartype); err != nil { + if err := rows.Scan(&s.name, &s.setting, &s.unit, &s.shortDesc, &s.vartype, &s.pendingRestart); err != nil { return err } + + if s.pendingRestart { + ch <- prometheus.MustNewConstMetric(pgSettingsPendingRestartDesc, prometheus.GaugeValue, 1, s.name) + } + metric, err := s.metric() if err != nil { - c.log.Warn("Error normalising unit for setting", "setting", s.name, "value", s.setting, "unit", s.unit, "error", err) + if !errors.Is(err, errUnsupportedVartype) { + c.log.Warn("Error normalising unit for setting", "setting", s.name, "value", s.setting, "unit", s.unit, "error", err) + } continue } ch <- metric @@ -81,7 +104,12 @@ func (c PGSettingsCollector) Update(ctx context.Context, instance *instance, ch // pgSetting represents a PostgreSQL runtime variable as returned by the // pg_settings view. type pgSetting struct { - name, setting, unit, shortDesc, vartype string + name string + setting string + unit string + shortDesc string + vartype string + pendingRestart bool } func (s *pgSetting) metric() (prometheus.Metric, error) { @@ -108,7 +136,7 @@ func (s *pgSetting) metric() (prometheus.Metric, error) { shortDesc = fmt.Sprintf("%s [Units converted to %s.]", shortDesc, unit) } default: - return nil, fmt.Errorf("pgsetting: unsupported vartype %q", s.vartype) + return nil, fmt.Errorf("pgsetting: %w %q", errUnsupportedVartype, s.vartype) } desc := prometheus.NewDesc(prometheus.BuildFQName(namespace, settingsSubsystem, name), shortDesc, nil, nil) diff --git a/collector/pg_setting_integration_test.go b/collector/pg_setting_integration_test.go new file mode 100644 index 00000000..3f06e64f --- /dev/null +++ b/collector/pg_setting_integration_test.go @@ -0,0 +1,129 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +package collector + +import ( + "context" + "database/sql" + "fmt" + "os" + "testing" + "time" + + _ "github.com/lib/pq" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/promslog" +) + +func TestPGSettingsCollectorPendingRestart(t *testing.T) { + db, err := sql.Open("postgres", os.Getenv("DATA_SOURCE_NAME")) + if err != nil { + t.Fatalf("sql.Open() error = %v", err) + } + defer db.Close() + + version, err := queryVersion(db) + if err != nil { + t.Fatalf("queryVersion() error = %v", err) + } + + // max_connections requires a server restart to take effect. + var current int + if err := db.QueryRow(`SELECT setting::int FROM pg_settings WHERE name = 'max_connections'`).Scan(¤t); err != nil { + t.Fatalf("querying max_connections: %v", err) + } + + // ALTER SYSTEM does not support parameter placeholders. + if _, err := db.Exec(fmt.Sprintf("ALTER SYSTEM SET max_connections = %d", current+1)); err != nil { + t.Fatalf("ALTER SYSTEM SET max_connections: %v", err) + } + + // Reset the setting and reload the configuration after the test. + defer func() { + if _, err := db.Exec(`ALTER SYSTEM RESET max_connections`); err != nil { + t.Logf("ALTER SYSTEM RESET max_connections: %v", err) + } + var reloaded bool + if err := db.QueryRow(`SELECT pg_reload_conf()`).Scan(&reloaded); err != nil { + t.Logf("pg_reload_conf() after reset: %v", err) + } + }() + + // pending_restart is updated after a configuration reload. + var reloaded bool + if err := db.QueryRow(`SELECT pg_reload_conf()`).Scan(&reloaded); err != nil { + t.Fatalf("pg_reload_conf(): %v", err) + } + if !waitForPendingRestart(t, db) { + t.Fatalf("max_connections was not marked as pending a restart") + } + + inst := &instance{db: db, version: version} + collector := PGSettingsCollector{log: promslog.NewNopLogger()} + + ch := make(chan prometheus.Metric) + errCh := make(chan error, 1) + go func() { + errCh <- collector.Update(context.Background(), inst, ch) + close(ch) + }() + + foundPending, foundValue := false, false + for metric := range ch { + got := readMetric(metric) + switch metric.Desc().String() { + case pgSettingsPendingRestartDesc.String(): + if got.labels["name"] != "max_connections" { + continue + } + if got.value != 1 { + t.Fatalf("pending restart value = %v, want 1", got.value) + } + foundPending = true + case `Desc{fqName: "pg_settings_max_connections", help: "Server Parameter: max_connections", unit: "", constLabels: {}, variableLabels: {}}`: + // The value metric for the same setting must come from the very + // same query, since pending_restart is part of it. + foundValue = true + } + } + if err := <-errCh; err != nil { + t.Fatalf("Update() error = %v", err) + } + if !foundPending { + t.Errorf(`pg_settings_pending_restart{name="max_connections"} was not reported`) + } + if !foundValue { + t.Errorf("pg_settings_max_connections was not reported in the same scrape") + } +} + +// waitForPendingRestart polls pg_settings until max_connections reports a +// pending restart, returning false if it does not within the timeout. +func waitForPendingRestart(t *testing.T, db *sql.DB) bool { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + var pending bool + if err := db.QueryRow(`SELECT pending_restart FROM pg_settings WHERE name = 'max_connections'`).Scan(&pending); err != nil { + t.Fatalf("querying pending_restart: %v", err) + } + if pending { + return true + } + time.Sleep(100 * time.Millisecond) + } + return false +} diff --git a/collector/pg_setting_test.go b/collector/pg_setting_test.go index f938db0a..4c6a6710 100644 --- a/collector/pg_setting_test.go +++ b/collector/pg_setting_test.go @@ -15,6 +15,7 @@ package collector import ( "context" + "errors" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -99,6 +100,7 @@ func TestPGSettingMetric(t *testing.T) { setting pgSetting wantDesc string wantMetric float64 + wantErr string }{ { name: "integer seconds", @@ -118,11 +120,28 @@ func TestPGSettingMetric(t *testing.T) { wantDesc: `Desc{fqName: "pg_settings_rds_rds_superuser_reserved_connections", help: "Server Parameter: rds.rds-superuser-reserved-connections", unit: "", constLabels: {}, variableLabels: {}}`, wantMetric: 2, }, + { + name: "unsupported vartype", + setting: pgSetting{name: "wal_level", setting: "replica", vartype: "enum"}, + wantErr: `pgsetting: unsupported vartype "enum"`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { metric, err := tt.setting.metric() + if tt.wantErr != "" { + if err == nil { + t.Fatalf("metric() expected error %q", tt.wantErr) + } + if err.Error() != tt.wantErr { + t.Fatalf("metric() error = %q, want %q", err.Error(), tt.wantErr) + } + if !errors.Is(err, errUnsupportedVartype) { + t.Fatalf("metric() error = %v, want errUnsupportedVartype", err) + } + return + } if err != nil { t.Fatalf("metric() unexpected error: %v", err) } @@ -148,10 +167,14 @@ func TestPGSettingsCollectorUpdate(t *testing.T) { defer db.Close() inst := &instance{db: db} - rows := sqlmock.NewRows([]string{"name", "setting", "unit", "short_desc", "vartype"}). - AddRow("shared_buffers", "128", "8kB", "Sets the number of shared memory buffers used by the server.", "integer"). - AddRow("track_counts", "on", "", "Collects statistics on database activity.", "bool"). - AddRow("bad_setting", "not-a-number", "", "Bad setting.", "integer") + // shared_buffers and wal_level are pending a restart, so each gets a + // pg_settings_pending_restart metric. bad_setting has a broken value and + // wal_level no numeric vartype, so neither gets a value metric. + rows := sqlmock.NewRows([]string{"name", "setting", "unit", "short_desc", "vartype", "pending_restart"}). + AddRow("shared_buffers", "128", "8kB", "Sets the number of shared memory buffers used by the server.", "integer", true). + AddRow("track_counts", "on", "", "Collects statistics on database activity.", "bool", false). + AddRow("bad_setting", "not-a-number", "", "Bad setting.", "integer", false). + AddRow("wal_level", "replica", "", "Describe the level of write-ahead logging.", "enum", true) mock.ExpectQuery(sanitizeQuery(pgSettingsQuery)).WillReturnRows(rows) ch := make(chan prometheus.Metric) @@ -166,7 +189,9 @@ func TestPGSettingsCollectorUpdate(t *testing.T) { tests := []struct { wantDesc string wantValue float64 + wantName string }{ + {wantDesc: pgSettingsPendingRestartDesc.String(), wantValue: 1, wantName: "shared_buffers"}, { wantDesc: `Desc{fqName: "pg_settings_shared_buffers_bytes", help: "Server Parameter: shared_buffers [Units converted to bytes.]", unit: "", constLabels: {}, variableLabels: {}}`, wantValue: 1048576, @@ -175,24 +200,32 @@ func TestPGSettingsCollectorUpdate(t *testing.T) { wantDesc: `Desc{fqName: "pg_settings_track_counts", help: "Server Parameter: track_counts", unit: "", constLabels: {}, variableLabels: {}}`, wantValue: 1, }, + {wantDesc: pgSettingsPendingRestartDesc.String(), wantValue: 1, wantName: "wal_level"}, } for _, tt := range tests { metric := <-ch - got := &dto.Metric{} - if err := metric.Write(got); err != nil { - t.Fatalf("Write() unexpected error: %v", err) + got := readMetric(metric) + if got.metricType != dto.MetricType_GAUGE { + t.Fatalf("metric type = %v, want GAUGE", got.metricType) } if metric.Desc().String() != tt.wantDesc { t.Fatalf("metric desc = %q, want %q", metric.Desc().String(), tt.wantDesc) } - if got.GetGauge().GetValue() != tt.wantValue { - t.Fatalf("metric value = %v, want %v", got.GetGauge().GetValue(), tt.wantValue) + if got.value != tt.wantValue { + t.Fatalf("metric value = %v, want %v", got.value, tt.wantValue) + } + if tt.wantName != "" { + if got.labels["name"] != tt.wantName { + t.Fatalf(`metric name label = %q, want %q`, got.labels["name"], tt.wantName) + } + } else if len(got.labels) != 0 { + t.Fatalf("unexpected metric labels: %v", got.labels) } } if metric, ok := <-ch; ok { - t.Fatalf("unexpected metric emitted after bad setting was skipped: %s", metric.Desc()) + t.Fatalf("unexpected metric emitted: %s", metric.Desc()) } if err := mock.ExpectationsWereMet(); err != nil { t.Errorf("there were unfulfilled exceptions: %s", err) diff --git a/docs/configuration.md b/docs/configuration.md index 520e50e3..a7136ce7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -69,7 +69,7 @@ The per-table and per-index collectors scale with your schema — one set of ser | Collector | Default | What it tells you | |---|---|---| -| `settings` | enabled | Every boolean, integer, and real setting in `pg_settings`, exported as a metric. Makes server configuration queryable and configuration drift alertable. | +| `settings` | enabled | Every boolean, integer, and real setting in `pg_settings`, exported as a metric, plus a `pg_settings_pending_restart` gauge for settings of any type that have been changed but only take effect after a server restart. Makes server configuration queryable and configuration drift alertable. | | `postmaster` | disabled | The server process start time, as a Unix timestamp. Use it to derive uptime and to detect unplanned restarts. | | `stat_statements` | disabled | Per-query aggregates: call counts, total execution time, rows returned, and block read/write time. The most direct way to find your slowest and most frequent queries. Requires the `pg_stat_statements` extension, and it is the highest-cardinality collector here — one series set per distinct query. From `pg_stat_statements`. |