feat(mixin): Add grafana mixins support for postgres exporter

Signed-off-by: Kaviraj <kavirajkanagaraj@gmail.com>

postgres_mixin/.gitignore

@@ -0,0 +1,3 @@
+/alerts.yaml
+/rules.yaml
+dashboards_out

postgres_mixin/Makefile

@@ -0,0 +1,23 @@
+JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 2 --string-style s --comment-style s
+
+default: build
+
+all: fmt lint build clean
+
+fmt:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		xargs -n 1 -- $(JSONNET_FMT) -i
+
+lint:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		while read f; do \
+			$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
+		done
+
+	mixtool lint mixin.libsonnet
+
+build:
+	mixtool generate all mixin.libsonnet
+
+clean:
+	rm -rf dashboards_out alerts.yaml rules.yaml

postgres_mixin/README.md

@@ -0,0 +1,26 @@
+# Postgres Mixin
+
+_This is a work in progress. We aim for it to become a good role model for alerts
+and dashboards eventually, but it is not quite there yet._
+
+The Postgres Mixin is a set of configurable, reusable, and extensible alerts and
+dashboards based on the metrics exported by the Postgres Exporter. The mixin creates
+recording and alerting rules for Prometheus and suitable dashboard descriptions
+for Grafana.
+
+To use them, you need to have `mixtool` and `jsonnetfmt` installed. If you
+have a working Go development environment, it's easiest to run the following:
+```bash
+$ go get github.com/monitoring-mixins/mixtool/cmd/mixtool
+$ go get github.com/google/go-jsonnet/cmd/jsonnetfmt
+```
+
+You can then build the Prometheus rules files `alerts.yaml` and
+`rules.yaml` and a directory `dashboard_out` with the JSON dashboard files
+for Grafana:
+```bash
+$ make build
+```
+
+For more advanced uses of mixins, see
+https://github.com/monitoring-mixins/docs.

postgres_mixin/alerts/alerts.yaml

@@ -0,0 +1,57 @@
+---
+groups:
+  - name: PostgreSQL
+    rules:
+    - alert: PostgreSQLMaxConnectionsReached
+      expr: sum(pg_stat_activity_count) by (instance) >= sum(pg_settings_max_connections) by (instance) - sum(pg_settings_superuser_reserved_connections) by (instance)
+      for: 1m
+      labels:
+        severity: email
+      annotations:
+        summary: "{{ $labels.instance }} has maxed out Postgres connections."
+        description: "{{ $labels.instance }} is exceeding the currently configured maximum Postgres connection limit (current value: {{ $value }}s). Services may be degraded - please take immediate action (you probably need to increase max_connections in the Docker image and re-deploy."
+
+    - alert: PostgreSQLHighConnections
+      expr: sum(pg_stat_activity_count) by (instance) > (sum(pg_settings_max_connections) by (instance) - sum(pg_settings_superuser_reserved_connections) by (instance)) * 0.8
+      for: 10m
+      labels:
+        severity: email
+      annotations:
+        summary: "{{ $labels.instance }} is over 80% of max Postgres connections."
+        description: "{{ $labels.instance }} is exceeding 80% of the currently configured maximum Postgres connection limit (current value: {{ $value }}s). Please check utilization graphs and confirm if this is normal service growth, abuse or an otherwise temporary condition or if new resources need to be provisioned (or the limits increased, which is mostly likely)."
+
+    - alert: PostgreSQLDown
+      expr: pg_up != 1
+      for: 1m
+      labels:
+        severity: email
+      annotations:
+        summary: "PostgreSQL is not processing queries: {{ $labels.instance }}"
+        description: "{{ $labels.instance }} is rejecting query requests from the exporter, and thus probably not allowing DNS requests to work either. User services should not be effected provided at least 1 node is still alive."
+
+    - alert: PostgreSQLSlowQueries
+      expr: avg(rate(pg_stat_activity_max_tx_duration{datname!~"template.*"}[2m])) by (datname) > 2 * 60
+      for: 2m
+      labels:
+        severity: email
+      annotations:
+        summary: "PostgreSQL high number of slow on {{ $labels.cluster }} for database {{ $labels.datname }} "
+        description: "PostgreSQL high number of slow queries {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }} "
+
+    - alert: PostgreSQLQPS
+      expr: avg(irate(pg_stat_database_xact_commit{datname!~"template.*"}[5m]) + irate(pg_stat_database_xact_rollback{datname!~"template.*"}[5m])) by (datname) > 10000
+      for: 5m
+      labels:
+        severity: email
+      annotations:
+        summary: "PostgreSQL high number of queries per second {{ $labels.cluster }} for database {{ $labels.datname }}"
+        description: "PostgreSQL high number of queries per second on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}"
+
+    - alert: PostgreSQLCacheHitRatio
+      expr: avg(rate(pg_stat_database_blks_hit{datname!~"template.*"}[5m]) / (rate(pg_stat_database_blks_hit{datname!~"template.*"}[5m]) + rate(pg_stat_database_blks_read{datname!~"template.*"}[5m]))) by (datname) < 0.98
+      for: 5m
+      labels:
+        severity: email
+      annotations:
+        summary: "PostgreSQL low cache hit rate on {{ $labels.cluster }} for database {{ $labels.datname }}"
+        description: "PostgreSQL low on cache hit rate on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}"

postgres_mixin/go.mod

@@ -0,0 +1,3 @@
+module github.com/wrouesnel/postgres_exporter/postgres_mixin
+
+go 1.15

postgres_mixin/mixin.libsonnet

@@ -0,0 +1,13 @@
+{
+  grafanaDashboards: {
+    'postgres-overview.json': (import 'dashboards/postgres-overview.json'),
+  },
+
+  // Helper function to ensure that we don't override other rules, by forcing
+  // the patching of the groups list, and not the overall rules object.
+  local importRules(rules) = {
+    groups+: std.native('parseYaml')(rules)[0].groups,
+  },
+
+  prometheusAlerts+: importRules(importstr 'alerts/alerts.yaml'),
+}