settings: export pg_settings.pending_restart (#1396)

Report settings that have been changed but only take effect after a server restart as pg_settings_pending_restart{name} from the existing settings collector. The flag is selected in the settings query itself and carried in the pgSetting struct, so it is emitted from the same loop that reports the value metrics, without an extra query. The vartype filter moves from the SQL WHERE clause into pgSetting.metric(): rows of every vartype are scanned, so pending restarts are reported for settings beyond the bool/integer/real ones exported as value metrics (e.g. wal_level is an enum); rows without a numeric vartype are skipped silently, and the existing warning is kept for broken values. Signed-off-by: MarcWort <113890636+MarcWort@users.noreply.github.com>
2026-10-23 12:08:57 +03:00 · 2026-10-01 18:34:28 +02:00
parent 1b07d8d034
commit c300bc655d
5 changed files with 207 additions and 16 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ enabling wrapping later will still create this one-time reset boundary.
 * [CHANGE] Wrap non-negative 64-bit counters at `2^53` to preserve single-unit precision by @gnanirahulnutakki in https://github.com/prometheus-community/postgres_exporter/pull/1351
 * [CHANGE] stat_replication: add `pid` label to disambiguate replication connections that otherwise share identical labels by @sysadmind in https://github.com/prometheus-community/postgres_exporter/pull/1353
 * [BUGFIX] Fix `long_running_transactions` to count only transactions older than a configurable threshold, by @ArthurSens in https://github.com/prometheus-community/postgres_exporter/pull/1379, based on the contribution by @moreinhardt in https://github.com/prometheus-community/postgres_exporter/pull/1210
+* [FEATURE] Export `pg_settings.pending_restart` as `pg_settings_pending_restart{name}` from the `settings` collector by @MarcWort in https://github.com/prometheus-community/postgres_exporter/pull/1396

 ## 0.20.1 / 2026-07-07

--- a/collector/pg_setting.go
+++ b/collector/pg_setting.go
@@ -15,6 +15,7 @@ package collector

 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"math"
@@ -50,12 +51,27 @@ var (
 	// Settings intentionally ignored due to invalid format:
 	// - `sync_commit_cancel_wait`, specific to Azure Postgres, see https://github.com/prometheus-community/postgres_exporter/issues/523
 	// - `google_dataplex.max_messages`, specific to Google Cloud SQL, see https://github.com/prometheus-community/postgres_exporter/issues/1240
-	pgSettingsQuery = "SELECT name, setting, COALESCE(unit, ''), COALESCE(short_desc, ''), vartype FROM pg_settings WHERE vartype IN ('bool', 'integer', 'real') AND name NOT IN ('sync_commit_cancel_wait', 'google_dataplex.max_messages');"
+	pgSettingsQuery = "SELECT name, setting, COALESCE(unit, ''), COALESCE(short_desc, ''), vartype, pending_restart FROM pg_settings WHERE name NOT IN ('sync_commit_cancel_wait', 'google_dataplex.max_messages');"
+
+	// pgSettingsPendingRestartDesc reports which settings have been changed
+	// but only take effect after a server restart, as flagged by the
+	// pg_settings.pending_restart column.
+	pgSettingsPendingRestartDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, settingsSubsystem, "pending_restart"),
+		"PostgreSQL settings that require a server restart to take effect.",
+		[]string{"name"},
+		nil,
+	)
 )

+// errUnsupportedVartype distinguishes expected non-numeric settings
+// from malformed numeric values.
+var errUnsupportedVartype = errors.New("unsupported vartype")
+
 // Update implements Collector and exposes PostgreSQL runtime settings.
 func (c PGSettingsCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error {
 	db := instance.getDB()
+
 	rows, err := db.QueryContext(ctx, pgSettingsQuery)
 	if err != nil {
 		return err
@@ -64,12 +80,19 @@ func (c PGSettingsCollector) Update(ctx context.Context, instance *instance, ch

 	for rows.Next() {
 		s := &pgSetting{}
-		if err := rows.Scan(&s.name, &s.setting, &s.unit, &s.shortDesc, &s.vartype); err != nil {
+		if err := rows.Scan(&s.name, &s.setting, &s.unit, &s.shortDesc, &s.vartype, &s.pendingRestart); err != nil {
 			return err
 		}
+
+		if s.pendingRestart {
+			ch <- prometheus.MustNewConstMetric(pgSettingsPendingRestartDesc, prometheus.GaugeValue, 1, s.name)
+		}
+
 		metric, err := s.metric()
 		if err != nil {
-			c.log.Warn("Error normalising unit for setting", "setting", s.name, "value", s.setting, "unit", s.unit, "error", err)
+			if !errors.Is(err, errUnsupportedVartype) {
+				c.log.Warn("Error normalising unit for setting", "setting", s.name, "value", s.setting, "unit", s.unit, "error", err)
+			}
 			continue
 		}
 		ch <- metric
@@ -81,7 +104,12 @@ func (c PGSettingsCollector) Update(ctx context.Context, instance *instance, ch
 // pgSetting represents a PostgreSQL runtime variable as returned by the
 // pg_settings view.
 type pgSetting struct {
-	name, setting, unit, shortDesc, vartype string
+	name           string
+	setting        string
+	unit           string
+	shortDesc      string
+	vartype        string
+	pendingRestart bool
 }

 func (s *pgSetting) metric() (prometheus.Metric, error) {
@@ -108,7 +136,7 @@ func (s *pgSetting) metric() (prometheus.Metric, error) {
 			shortDesc = fmt.Sprintf("%s [Units converted to %s.]", shortDesc, unit)
 		}
 	default:
-		return nil, fmt.Errorf("pgsetting: unsupported vartype %q", s.vartype)
+		return nil, fmt.Errorf("pgsetting: %w %q", errUnsupportedVartype, s.vartype)
 	}

 	desc := prometheus.NewDesc(prometheus.BuildFQName(namespace, settingsSubsystem, name), shortDesc, nil, nil)
--- a/collector/pg_setting_integration_test.go
+++ b/collector/pg_setting_integration_test.go
@@ -0,0 +1,129 @@
+// Copyright The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build integration
+
+package collector
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	_ "github.com/lib/pq"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/promslog"
+)
+
+func TestPGSettingsCollectorPendingRestart(t *testing.T) {
+	db, err := sql.Open("postgres", os.Getenv("DATA_SOURCE_NAME"))
+	if err != nil {
+		t.Fatalf("sql.Open() error = %v", err)
+	}
+	defer db.Close()
+
+	version, err := queryVersion(db)
+	if err != nil {
+		t.Fatalf("queryVersion() error = %v", err)
+	}
+
+	// max_connections requires a server restart to take effect.
+	var current int
+	if err := db.QueryRow(`SELECT setting::int FROM pg_settings WHERE name = 'max_connections'`).Scan(&current); err != nil {
+		t.Fatalf("querying max_connections: %v", err)
+	}
+
+	// ALTER SYSTEM does not support parameter placeholders.
+	if _, err := db.Exec(fmt.Sprintf("ALTER SYSTEM SET max_connections = %d", current+1)); err != nil {
+		t.Fatalf("ALTER SYSTEM SET max_connections: %v", err)
+	}
+
+	// Reset the setting and reload the configuration after the test.
+	defer func() {
+		if _, err := db.Exec(`ALTER SYSTEM RESET max_connections`); err != nil {
+			t.Logf("ALTER SYSTEM RESET max_connections: %v", err)
+		}
+		var reloaded bool
+		if err := db.QueryRow(`SELECT pg_reload_conf()`).Scan(&reloaded); err != nil {
+			t.Logf("pg_reload_conf() after reset: %v", err)
+		}
+	}()
+
+	// pending_restart is updated after a configuration reload.
+	var reloaded bool
+	if err := db.QueryRow(`SELECT pg_reload_conf()`).Scan(&reloaded); err != nil {
+		t.Fatalf("pg_reload_conf(): %v", err)
+	}
+	if !waitForPendingRestart(t, db) {
+		t.Fatalf("max_connections was not marked as pending a restart")
+	}
+
+	inst := &instance{db: db, version: version}
+	collector := PGSettingsCollector{log: promslog.NewNopLogger()}
+
+	ch := make(chan prometheus.Metric)
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- collector.Update(context.Background(), inst, ch)
+		close(ch)
+	}()
+
+	foundPending, foundValue := false, false
+	for metric := range ch {
+		got := readMetric(metric)
+		switch metric.Desc().String() {
+		case pgSettingsPendingRestartDesc.String():
+			if got.labels["name"] != "max_connections" {
+				continue
+			}
+			if got.value != 1 {
+				t.Fatalf("pending restart value = %v, want 1", got.value)
+			}
+			foundPending = true
+		case `Desc{fqName: "pg_settings_max_connections", help: "Server Parameter: max_connections", unit: "", constLabels: {}, variableLabels: {}}`:
+			// The value metric for the same setting must come from the very
+			// same query, since pending_restart is part of it.
+			foundValue = true
+		}
+	}
+	if err := <-errCh; err != nil {
+		t.Fatalf("Update() error = %v", err)
+	}
+	if !foundPending {
+		t.Errorf(`pg_settings_pending_restart{name="max_connections"} was not reported`)
+	}
+	if !foundValue {
+		t.Errorf("pg_settings_max_connections was not reported in the same scrape")
+	}
+}
+
+// waitForPendingRestart polls pg_settings until max_connections reports a
+// pending restart, returning false if it does not within the timeout.
+func waitForPendingRestart(t *testing.T, db *sql.DB) bool {
+	t.Helper()
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		var pending bool
+		if err := db.QueryRow(`SELECT pending_restart FROM pg_settings WHERE name = 'max_connections'`).Scan(&pending); err != nil {
+			t.Fatalf("querying pending_restart: %v", err)
+		}
+		if pending {
+			return true
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return false
+}
--- a/collector/pg_setting_test.go
+++ b/collector/pg_setting_test.go
@@ -15,6 +15,7 @@ package collector

 import (
 	"context"
+	"errors"
 	"testing"

 	"github.com/DATA-DOG/go-sqlmock"
@@ -99,6 +100,7 @@ func TestPGSettingMetric(t *testing.T) {
 		setting    pgSetting
 		wantDesc   string
 		wantMetric float64
+		wantErr    string
 	}{
 		{
 			name:       "integer seconds",
@@ -118,11 +120,28 @@ func TestPGSettingMetric(t *testing.T) {
 			wantDesc:   `Desc{fqName: "pg_settings_rds_rds_superuser_reserved_connections", help: "Server Parameter: rds.rds-superuser-reserved-connections", unit: "", constLabels: {}, variableLabels: {}}`,
 			wantMetric: 2,
 		},
+		{
+			name:    "unsupported vartype",
+			setting: pgSetting{name: "wal_level", setting: "replica", vartype: "enum"},
+			wantErr: `pgsetting: unsupported vartype "enum"`,
+		},
 	}

 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			metric, err := tt.setting.metric()
+			if tt.wantErr != "" {
+				if err == nil {
+					t.Fatalf("metric() expected error %q", tt.wantErr)
+				}
+				if err.Error() != tt.wantErr {
+					t.Fatalf("metric() error = %q, want %q", err.Error(), tt.wantErr)
+				}
+				if !errors.Is(err, errUnsupportedVartype) {
+					t.Fatalf("metric() error = %v, want errUnsupportedVartype", err)
+				}
+				return
+			}
 			if err != nil {
 				t.Fatalf("metric() unexpected error: %v", err)
 			}
@@ -148,10 +167,14 @@ func TestPGSettingsCollectorUpdate(t *testing.T) {
 	defer db.Close()

 	inst := &instance{db: db}
-	rows := sqlmock.NewRows([]string{"name", "setting", "unit", "short_desc", "vartype"}).
-		AddRow("shared_buffers", "128", "8kB", "Sets the number of shared memory buffers used by the server.", "integer").
-		AddRow("track_counts", "on", "", "Collects statistics on database activity.", "bool").
-		AddRow("bad_setting", "not-a-number", "", "Bad setting.", "integer")
+	// shared_buffers and wal_level are pending a restart, so each gets a
+	// pg_settings_pending_restart metric. bad_setting has a broken value and
+	// wal_level no numeric vartype, so neither gets a value metric.
+	rows := sqlmock.NewRows([]string{"name", "setting", "unit", "short_desc", "vartype", "pending_restart"}).
+		AddRow("shared_buffers", "128", "8kB", "Sets the number of shared memory buffers used by the server.", "integer", true).
+		AddRow("track_counts", "on", "", "Collects statistics on database activity.", "bool", false).
+		AddRow("bad_setting", "not-a-number", "", "Bad setting.", "integer", false).
+		AddRow("wal_level", "replica", "", "Describe the level of write-ahead logging.", "enum", true)
 	mock.ExpectQuery(sanitizeQuery(pgSettingsQuery)).WillReturnRows(rows)

 	ch := make(chan prometheus.Metric)
@@ -166,7 +189,9 @@ func TestPGSettingsCollectorUpdate(t *testing.T) {
 	tests := []struct {
 		wantDesc  string
 		wantValue float64
+		wantName  string
 	}{
+		{wantDesc: pgSettingsPendingRestartDesc.String(), wantValue: 1, wantName: "shared_buffers"},
 		{
 			wantDesc:  `Desc{fqName: "pg_settings_shared_buffers_bytes", help: "Server Parameter: shared_buffers [Units converted to bytes.]", unit: "", constLabels: {}, variableLabels: {}}`,
 			wantValue: 1048576,
@@ -175,24 +200,32 @@ func TestPGSettingsCollectorUpdate(t *testing.T) {
 			wantDesc:  `Desc{fqName: "pg_settings_track_counts", help: "Server Parameter: track_counts", unit: "", constLabels: {}, variableLabels: {}}`,
 			wantValue: 1,
 		},
+		{wantDesc: pgSettingsPendingRestartDesc.String(), wantValue: 1, wantName: "wal_level"},
 	}

 	for _, tt := range tests {
 		metric := <-ch
-		got := &dto.Metric{}
-		if err := metric.Write(got); err != nil {
-			t.Fatalf("Write() unexpected error: %v", err)
+		got := readMetric(metric)
+		if got.metricType != dto.MetricType_GAUGE {
+			t.Fatalf("metric type = %v, want GAUGE", got.metricType)
 		}
 		if metric.Desc().String() != tt.wantDesc {
 			t.Fatalf("metric desc = %q, want %q", metric.Desc().String(), tt.wantDesc)
 		}
-		if got.GetGauge().GetValue() != tt.wantValue {
-			t.Fatalf("metric value = %v, want %v", got.GetGauge().GetValue(), tt.wantValue)
+		if got.value != tt.wantValue {
+			t.Fatalf("metric value = %v, want %v", got.value, tt.wantValue)
+		}
+		if tt.wantName != "" {
+			if got.labels["name"] != tt.wantName {
+				t.Fatalf(`metric name label = %q, want %q`, got.labels["name"], tt.wantName)
+			}
+		} else if len(got.labels) != 0 {
+			t.Fatalf("unexpected metric labels: %v", got.labels)
 		}
 	}

 	if metric, ok := <-ch; ok {
-		t.Fatalf("unexpected metric emitted after bad setting was skipped: %s", metric.Desc())
+		t.Fatalf("unexpected metric emitted: %s", metric.Desc())
 	}
 	if err := mock.ExpectationsWereMet(); err != nil {
 		t.Errorf("there were unfulfilled exceptions: %s", err)
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -69,7 +69,7 @@ The per-table and per-index collectors scale with your schema — one set of ser

 | Collector | Default | What it tells you |
 |---|---|---|
-| `settings` | enabled | Every boolean, integer, and real setting in `pg_settings`, exported as a metric. Makes server configuration queryable and configuration drift alertable. |
+| `settings` | enabled | Every boolean, integer, and real setting in `pg_settings`, exported as a metric, plus a `pg_settings_pending_restart` gauge for settings of any type that have been changed but only take effect after a server restart. Makes server configuration queryable and configuration drift alertable. |
 | `postmaster` | disabled | The server process start time, as a Unix timestamp. Use it to derive uptime and to detect unplanned restarts. |
 | `stat_statements` | disabled | Per-query aggregates: call counts, total execution time, rows returned, and block read/write time. The most direct way to find your slowest and most frequent queries. Requires the `pg_stat_statements` extension, and it is the highest-cardinality collector here — one series set per distinct query. From `pg_stat_statements`. |