database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-27 12:41:57 +03:00
Code Activity

pgstat: add pg_stat_force_next_flush(), use it to simplify tests.

Browse Source 
In the stats collector days it was hard to write tests for the stats system,
because fundamentally delivery of stats messages over UDP was not
synchronous (nor guaranteed). Now we easily can force pending stats updates to
be flushed synchronously.

This moves stats.sql into a parallel group, there isn't a reason for it to run
in isolation anymore. And it may shake out some bugs.

Bumps catversion.

Author: Andres Freund <andres@anarazel.de>
Discussion: https://postgr.es/m/20220303021600.hs34ghqcw6zcokdh@alap3.anarazel.de
This commit is contained in:
Andres Freund
2022-04-06 23:35:56 -07:00
parent 5e07d3d6bd
commit 0f96965c65
11 changed files with 103 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
contrib/test_decoding/expected/stats.out
View File

@ -7,50 +7,6 @@ SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot_stats', '
(1 row)
CREATE TABLE stats_test(data text);
-- function to wait for counters to advance
CREATE FUNCTION wait_for_decode_stats(check_reset bool, check_spill_txns bool) RETURNS void AS $$
DECLARE
start_time timestamptz := clock_timestamp();
updated bool;
BEGIN
-- we don't want to wait forever; loop will exit after 30 seconds
FOR i IN 1 .. 300 LOOP
IF check_spill_txns THEN
-- check to see if all updates have been reset/updated
SELECT CASE WHEN check_reset THEN (spill_txns = 0)
ELSE (spill_txns > 0)
END
INTO updated
FROM pg_stat_replication_slots WHERE slot_name='regression_slot_stats';
ELSE
-- check to see if all updates have been reset/updated
SELECT CASE WHEN check_reset THEN (total_txns = 0)
ELSE (total_txns > 0)
END
INTO updated
FROM pg_stat_replication_slots WHERE slot_name='regression_slot_stats';
END IF;
exit WHEN updated;
-- wait a little
perform pg_sleep_for('100 milliseconds');
-- reset stats snapshot so we can test again
perform pg_stat_clear_snapshot();
END LOOP;
-- report time waited in postmaster log (where it won't change test output)
RAISE LOG 'wait_for_decode_stats delayed % seconds',
extract(epoch from clock_timestamp() - start_time);
END
$$ LANGUAGE plpgsql;
-- non-spilled xact
SET logical_decoding_work_mem to '64MB';
INSERT INTO stats_test values(1);
@ -60,9 +16,9 @@ SELECT count(*) FROM pg_logical_slot_get_changes('regression_slot_stats', NULL,
3
(1 row)
SELECT wait_for_decode_stats(false, false);
wait_for_decode_stats
-----------------------
SELECT pg_stat_force_next_flush();
pg_stat_force_next_flush
--------------------------
(1 row)
@ -73,19 +29,13 @@ SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count,
(1 row)
RESET logical_decoding_work_mem;
-- reset the slot stats, and wait for stats collector's total txn to reset
-- reset the slot stats
SELECT pg_stat_reset_replication_slot('regression_slot_stats');
pg_stat_reset_replication_slot
--------------------------------
(1 row)
SELECT wait_for_decode_stats(true, false);
wait_for_decode_stats
-----------------------
(1 row)
SELECT slot_name, spill_txns, spill_count, total_txns, total_bytes FROM pg_stat_replication_slots;
slot_name | spill_txns | spill_count | total_txns | total_bytes
-----------------------+------------+-------------+------------+-------------
@ -102,12 +52,12 @@ SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot_stats', NULL,
5002
(1 row)
-- Check stats, wait for the stats collector to update. We can't test the
-- exact stats count as that can vary if any background transaction (say by
-- autovacuum) happens in parallel to the main transaction.
SELECT wait_for_decode_stats(false, true);
wait_for_decode_stats
-----------------------
-- Check stats. We can't test the exact stats count as that can vary if any
-- background transaction (say by autovacuum) happens in parallel to the main
-- transaction.
SELECT pg_stat_force_next_flush();
pg_stat_force_next_flush
--------------------------
(1 row)
@ -133,7 +83,6 @@ SELECT slot_name FROM pg_stat_replication_slots;
(1 row)
COMMIT;
DROP FUNCTION wait_for_decode_stats(bool, bool);
DROP TABLE stats_test;
SELECT pg_drop_replication_slot('regression_slot_stats');
pg_drop_replication_slot

59
contrib/test_decoding/sql/stats.sql
View File

@ -5,62 +5,16 @@ SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot_stats', '
CREATE TABLE stats_test(data text);
-- function to wait for counters to advance
CREATE FUNCTION wait_for_decode_stats(check_reset bool, check_spill_txns bool) RETURNS void AS $$
DECLARE
start_time timestamptz := clock_timestamp();
updated bool;
BEGIN
-- we don't want to wait forever; loop will exit after 30 seconds
FOR i IN 1 .. 300 LOOP
IF check_spill_txns THEN
-- check to see if all updates have been reset/updated
SELECT CASE WHEN check_reset THEN (spill_txns = 0)
ELSE (spill_txns > 0)
END
INTO updated
FROM pg_stat_replication_slots WHERE slot_name='regression_slot_stats';
ELSE
-- check to see if all updates have been reset/updated
SELECT CASE WHEN check_reset THEN (total_txns = 0)
ELSE (total_txns > 0)
END
INTO updated
FROM pg_stat_replication_slots WHERE slot_name='regression_slot_stats';
END IF;
exit WHEN updated;
-- wait a little
perform pg_sleep_for('100 milliseconds');
-- reset stats snapshot so we can test again
perform pg_stat_clear_snapshot();
END LOOP;
-- report time waited in postmaster log (where it won't change test output)
RAISE LOG 'wait_for_decode_stats delayed % seconds',
extract(epoch from clock_timestamp() - start_time);
END
$$ LANGUAGE plpgsql;
-- non-spilled xact
SET logical_decoding_work_mem to '64MB';
INSERT INTO stats_test values(1);
SELECT count(*) FROM pg_logical_slot_get_changes('regression_slot_stats', NULL, NULL, 'skip-empty-xacts', '1');
SELECT wait_for_decode_stats(false, false);
SELECT pg_stat_force_next_flush();
SELECT slot_name, spill_txns = 0 AS spill_txns, spill_count = 0 AS spill_count, total_txns > 0 AS total_txns, total_bytes > 0 AS total_bytes FROM pg_stat_replication_slots;
RESET logical_decoding_work_mem;
-- reset the slot stats, and wait for stats collector's total txn to reset
-- reset the slot stats
SELECT pg_stat_reset_replication_slot('regression_slot_stats');
SELECT wait_for_decode_stats(true, false);
SELECT slot_name, spill_txns, spill_count, total_txns, total_bytes FROM pg_stat_replication_slots;
-- spilling the xact
@ -69,10 +23,10 @@ INSERT INTO stats_test SELECT 'serialize-topbig--1:'||g.i FROM generate_series(1
COMMIT;
SELECT count(*) FROM pg_logical_slot_peek_changes('regression_slot_stats', NULL, NULL, 'skip-empty-xacts', '1');
-- Check stats, wait for the stats collector to update. We can't test the
-- exact stats count as that can vary if any background transaction (say by
-- autovacuum) happens in parallel to the main transaction.
SELECT wait_for_decode_stats(false, true);
-- Check stats. We can't test the exact stats count as that can vary if any
-- background transaction (say by autovacuum) happens in parallel to the main
-- transaction.
SELECT pg_stat_force_next_flush();
SELECT slot_name, spill_txns > 0 AS spill_txns, spill_count > 0 AS spill_count FROM pg_stat_replication_slots;
-- Ensure stats can be repeatedly accessed using the same stats snapshot. See
@ -82,6 +36,5 @@ SELECT slot_name FROM pg_stat_replication_slots;
SELECT slot_name FROM pg_stat_replication_slots;
COMMIT;
DROP FUNCTION wait_for_decode_stats(bool, bool);
DROP TABLE stats_test;
SELECT pg_drop_replication_slot('regression_slot_stats');