1
0
mirror of https://github.com/postgres/postgres.git synced 2025-04-24 10:47:04 +03:00

Improve test coverage for changes to inplace-updated catalogs.

This covers both regular and inplace changes, since bugs arise at their
intersection.  Where marked, these witness extant bugs.  Back-patch to
v12 (all supported versions).

Reviewed (in an earlier version) by Robert Haas.

Discussion: https://postgr.es/m/20240512232923.aa.nmisch@google.com
This commit is contained in:
Noah Misch 2024-06-27 19:21:05 -07:00
parent 22a4b104ba
commit 0844b39689
16 changed files with 708 additions and 1 deletions

View File

@ -68,6 +68,34 @@ $node->pgbench(
"CREATE TYPE pg_temp.e AS ENUM ($labels); DROP TYPE pg_temp.e;"
});
# Test inplace updates from VACUUM concurrent with heap_update from GRANT.
# The PROC_IN_VACUUM environment can't finish MVCC table scans consistently,
# so this fails rarely. To reproduce consistently, add a sleep after
# GetCatalogSnapshot(non-catalog-rel).
Test::More->builder->todo_start('PROC_IN_VACUUM scan breakage');
$node->safe_psql('postgres', 'CREATE TABLE ddl_target ()');
$node->pgbench(
'--no-vacuum --client=5 --protocol=prepared --transactions=50',
0,
[qr{processed: 250/250}],
[qr{^$}],
'concurrent GRANT/VACUUM',
{
'001_pgbench_grant@9' => q(
DO $$
BEGIN
PERFORM pg_advisory_xact_lock(42);
FOR i IN 1 .. 10 LOOP
GRANT SELECT ON ddl_target TO PUBLIC;
REVOKE SELECT ON ddl_target FROM PUBLIC;
END LOOP;
END
$$;
),
'001_pgbench_vacuum_ddl_target@1' => "VACUUM ddl_target;",
});
Test::More->builder->todo_end;
# Trigger various connection errors
$node->pgbench(
'no-such-database',

View File

@ -1337,3 +1337,29 @@ a|b|c| d
2|2|2|1004
(2 rows)
starting permutation: sys1 sysupd2 c1 c2
step sys1:
UPDATE pg_class SET reltuples = 123 WHERE oid = 'accounts'::regclass;
step sysupd2:
UPDATE pg_class SET reltuples = reltuples * 2
WHERE oid = 'accounts'::regclass;
<waiting ...>
step c1: COMMIT;
step sysupd2: <... completed>
step c2: COMMIT;
starting permutation: sys1 sysmerge2 c1 c2
step sys1:
UPDATE pg_class SET reltuples = 123 WHERE oid = 'accounts'::regclass;
step sysmerge2:
MERGE INTO pg_class
USING (SELECT 'accounts'::regclass AS o) j
ON o = oid
WHEN MATCHED THEN UPDATE SET reltuples = reltuples * 2;
<waiting ...>
step c1: COMMIT;
step sysmerge2: <... completed>
step c2: COMMIT;

View File

@ -0,0 +1,32 @@
Parsed test spec with 3 sessions
starting permutation: cachefill3 cir1 cic2 ddl3 read1
step cachefill3: TABLE newly_indexed;
c
-
(0 rows)
step cir1: BEGIN; CREATE INDEX i1 ON newly_indexed (c); ROLLBACK;
step cic2: CREATE INDEX i2 ON newly_indexed (c);
step ddl3: ALTER TABLE newly_indexed ADD extra int;
step read1:
SELECT relhasindex FROM pg_class WHERE oid = 'newly_indexed'::regclass;
relhasindex
-----------
f
(1 row)
starting permutation: cir1 cic2 ddl3 read1
step cir1: BEGIN; CREATE INDEX i1 ON newly_indexed (c); ROLLBACK;
step cic2: CREATE INDEX i2 ON newly_indexed (c);
step ddl3: ALTER TABLE newly_indexed ADD extra int;
step read1:
SELECT relhasindex FROM pg_class WHERE oid = 'newly_indexed'::regclass;
relhasindex
-----------
t
(1 row)

View File

@ -0,0 +1,28 @@
Parsed test spec with 3 sessions
starting permutation: snap3 b1 grant1 vac2 snap3 c1 cmp3
step snap3:
INSERT INTO frozen_witness
SELECT datfrozenxid FROM pg_database WHERE datname = current_catalog;
step b1: BEGIN;
step grant1:
GRANT TEMP ON DATABASE isolation_regression TO regress_temp_grantee;
step vac2: VACUUM (FREEZE);
step snap3:
INSERT INTO frozen_witness
SELECT datfrozenxid FROM pg_database WHERE datname = current_catalog;
step c1: COMMIT;
step cmp3:
SELECT 'datfrozenxid retreated'
FROM pg_database
WHERE datname = current_catalog
AND age(datfrozenxid) > (SELECT min(age(x)) FROM frozen_witness);
?column?
----------------------
datfrozenxid retreated
(1 row)

View File

@ -0,0 +1,225 @@
Parsed test spec with 5 sessions
starting permutation: b1 grant1 read2 addk2 c1 read2
step b1: BEGIN;
step grant1:
GRANT SELECT ON intra_grant_inplace TO PUBLIC;
step read2:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass;
relhasindex
-----------
f
(1 row)
step addk2: ALTER TABLE intra_grant_inplace ADD PRIMARY KEY (c);
step c1: COMMIT;
step read2:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass;
relhasindex
-----------
f
(1 row)
starting permutation: keyshr5 addk2
step keyshr5:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR KEY SHARE;
relhasindex
-----------
f
(1 row)
step addk2: ALTER TABLE intra_grant_inplace ADD PRIMARY KEY (c);
starting permutation: keyshr5 b3 sfnku3 addk2 r3
step keyshr5:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR KEY SHARE;
relhasindex
-----------
f
(1 row)
step b3: BEGIN ISOLATION LEVEL READ COMMITTED;
step sfnku3:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR NO KEY UPDATE;
relhasindex
-----------
f
(1 row)
step addk2: ALTER TABLE intra_grant_inplace ADD PRIMARY KEY (c);
step r3: ROLLBACK;
starting permutation: b2 sfnku2 addk2 c2
step b2: BEGIN;
step sfnku2:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR NO KEY UPDATE;
relhasindex
-----------
f
(1 row)
step addk2: ALTER TABLE intra_grant_inplace ADD PRIMARY KEY (c);
step c2: COMMIT;
starting permutation: keyshr5 b2 sfnku2 addk2 c2
step keyshr5:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR KEY SHARE;
relhasindex
-----------
f
(1 row)
step b2: BEGIN;
step sfnku2:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR NO KEY UPDATE;
relhasindex
-----------
f
(1 row)
step addk2: ALTER TABLE intra_grant_inplace ADD PRIMARY KEY (c);
step c2: COMMIT;
starting permutation: b3 sfu3 b1 grant1 read2 addk2 r3 c1 read2
step b3: BEGIN ISOLATION LEVEL READ COMMITTED;
step sfu3:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR UPDATE;
relhasindex
-----------
f
(1 row)
step b1: BEGIN;
step grant1:
GRANT SELECT ON intra_grant_inplace TO PUBLIC;
<waiting ...>
step read2:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass;
relhasindex
-----------
f
(1 row)
step addk2: ALTER TABLE intra_grant_inplace ADD PRIMARY KEY (c);
step r3: ROLLBACK;
step grant1: <... completed>
step c1: COMMIT;
step read2:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass;
relhasindex
-----------
f
(1 row)
starting permutation: b2 sfnku2 b1 grant1 addk2 c2 c1 read2
step b2: BEGIN;
step sfnku2:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR NO KEY UPDATE;
relhasindex
-----------
f
(1 row)
step b1: BEGIN;
step grant1:
GRANT SELECT ON intra_grant_inplace TO PUBLIC;
<waiting ...>
step addk2: ALTER TABLE intra_grant_inplace ADD PRIMARY KEY (c);
step c2: COMMIT;
step grant1: <... completed>
step c1: COMMIT;
step read2:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass;
relhasindex
-----------
f
(1 row)
starting permutation: b1 grant1 b3 sfu3 revoke4 c1 r3
step b1: BEGIN;
step grant1:
GRANT SELECT ON intra_grant_inplace TO PUBLIC;
step b3: BEGIN ISOLATION LEVEL READ COMMITTED;
step sfu3:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR UPDATE;
<waiting ...>
step revoke4:
DO $$
BEGIN
REVOKE SELECT ON intra_grant_inplace FROM PUBLIC;
EXCEPTION WHEN others THEN
RAISE WARNING 'got: %', regexp_replace(sqlerrm, '[0-9]+', 'REDACTED');
END
$$;
<waiting ...>
step c1: COMMIT;
step sfu3: <... completed>
relhasindex
-----------
f
(1 row)
s4: WARNING: got: tuple concurrently updated
step revoke4: <... completed>
step r3: ROLLBACK;
starting permutation: b1 drop1 b3 sfu3 revoke4 c1 r3
step b1: BEGIN;
step drop1:
DROP TABLE intra_grant_inplace;
step b3: BEGIN ISOLATION LEVEL READ COMMITTED;
step sfu3:
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR UPDATE;
<waiting ...>
step revoke4:
DO $$
BEGIN
REVOKE SELECT ON intra_grant_inplace FROM PUBLIC;
EXCEPTION WHEN others THEN
RAISE WARNING 'got: %', regexp_replace(sqlerrm, '[0-9]+', 'REDACTED');
END
$$;
<waiting ...>
step c1: COMMIT;
step sfu3: <... completed>
relhasindex
-----------
(0 rows)
s4: WARNING: got: tuple concurrently deleted
step revoke4: <... completed>
step r3: ROLLBACK;

View File

@ -37,6 +37,9 @@ test: fk-snapshot
test: subxid-overflow
test: eval-plan-qual
test: eval-plan-qual-trigger
test: inplace-inval
test: intra-grant-inplace
test: intra-grant-inplace-db
test: lock-update-delete
test: lock-update-traversal
test: inherit-temp

View File

@ -194,6 +194,12 @@ step simplepartupdate_noroute {
update parttbl set b = 2 where c = 1 returning *;
}
# test system class updates
step sys1 {
UPDATE pg_class SET reltuples = 123 WHERE oid = 'accounts'::regclass;
}
session s2
setup { BEGIN ISOLATION LEVEL READ COMMITTED; }
@ -282,6 +288,18 @@ step wnested2 {
);
}
step sysupd2 {
UPDATE pg_class SET reltuples = reltuples * 2
WHERE oid = 'accounts'::regclass;
}
step sysmerge2 {
MERGE INTO pg_class
USING (SELECT 'accounts'::regclass AS o) j
ON o = oid
WHEN MATCHED THEN UPDATE SET reltuples = reltuples * 2;
}
step c2 { COMMIT; }
step r2 { ROLLBACK; }
@ -380,3 +398,6 @@ permutation simplepartupdate complexpartupdate c1 c2 read_part
permutation simplepartupdate_route1to2 complexpartupdate_route_err1 c1 c2 read_part
permutation simplepartupdate_noroute complexpartupdate_route c1 c2 read_part
permutation simplepartupdate_noroute complexpartupdate_doesnt_route c1 c2 read_part
permutation sys1 sysupd2 c1 c2
permutation sys1 sysmerge2 c1 c2

View File

@ -0,0 +1,38 @@
# If a heap_update() caller retrieves its oldtup from a cache, it's possible
# for that cache entry to predate an inplace update, causing loss of that
# inplace update. This arises because the transaction may abort before
# sending the inplace invalidation message to the shared queue.
setup
{
CREATE TABLE newly_indexed (c int);
}
teardown
{
DROP TABLE newly_indexed;
}
session s1
step cir1 { BEGIN; CREATE INDEX i1 ON newly_indexed (c); ROLLBACK; }
step read1 {
SELECT relhasindex FROM pg_class WHERE oid = 'newly_indexed'::regclass;
}
session s2
step cic2 { CREATE INDEX i2 ON newly_indexed (c); }
session s3
step cachefill3 { TABLE newly_indexed; }
step ddl3 { ALTER TABLE newly_indexed ADD extra int; }
permutation
cachefill3 # populates the pg_class row in the catcache
cir1 # sets relhasindex=true; rollback discards cache inval
cic2 # sees relhasindex=true, skips changing it (so no inval)
ddl3 # cached row as the oldtup of an update, losing relhasindex
read1 # observe damage XXX is an extant bug
# without cachefill3, no bug
permutation cir1 cic2 ddl3 read1

View File

@ -0,0 +1,46 @@
# GRANT's lock is the catalog tuple xmax. GRANT doesn't acquire a heavyweight
# lock on the object undergoing an ACL change. In-place updates, namely
# datfrozenxid, need special code to cope.
setup
{
CREATE ROLE regress_temp_grantee;
}
teardown
{
REVOKE ALL ON DATABASE isolation_regression FROM regress_temp_grantee;
DROP ROLE regress_temp_grantee;
}
# heap_update(pg_database)
session s1
step b1 { BEGIN; }
step grant1 {
GRANT TEMP ON DATABASE isolation_regression TO regress_temp_grantee;
}
step c1 { COMMIT; }
# inplace update
session s2
step vac2 { VACUUM (FREEZE); }
# observe datfrozenxid
session s3
setup {
CREATE TEMP TABLE frozen_witness (x xid);
}
step snap3 {
INSERT INTO frozen_witness
SELECT datfrozenxid FROM pg_database WHERE datname = current_catalog;
}
step cmp3 {
SELECT 'datfrozenxid retreated'
FROM pg_database
WHERE datname = current_catalog
AND age(datfrozenxid) > (SELECT min(age(x)) FROM frozen_witness);
}
# XXX extant bug
permutation snap3 b1 grant1 vac2(c1) snap3 c1 cmp3

View File

@ -0,0 +1,153 @@
# GRANT's lock is the catalog tuple xmax. GRANT doesn't acquire a heavyweight
# lock on the object undergoing an ACL change. Inplace updates, such as
# relhasindex=true, need special code to cope.
setup
{
CREATE TABLE intra_grant_inplace (c int);
}
teardown
{
DROP TABLE IF EXISTS intra_grant_inplace;
}
# heap_update()
session s1
step b1 { BEGIN; }
step grant1 {
GRANT SELECT ON intra_grant_inplace TO PUBLIC;
}
step drop1 {
DROP TABLE intra_grant_inplace;
}
step c1 { COMMIT; }
# inplace update
session s2
step read2 {
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass;
}
step b2 { BEGIN; }
step addk2 { ALTER TABLE intra_grant_inplace ADD PRIMARY KEY (c); }
step sfnku2 {
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR NO KEY UPDATE;
}
step c2 { COMMIT; }
# rowmarks
session s3
step b3 { BEGIN ISOLATION LEVEL READ COMMITTED; }
step sfnku3 {
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR NO KEY UPDATE;
}
step sfu3 {
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR UPDATE;
}
step r3 { ROLLBACK; }
# Additional heap_update()
session s4
# swallow error message to keep any OID value out of expected output
step revoke4 {
DO $$
BEGIN
REVOKE SELECT ON intra_grant_inplace FROM PUBLIC;
EXCEPTION WHEN others THEN
RAISE WARNING 'got: %', regexp_replace(sqlerrm, '[0-9]+', 'REDACTED');
END
$$;
}
# Additional rowmarks
session s5
setup { BEGIN; }
step keyshr5 {
SELECT relhasindex FROM pg_class
WHERE oid = 'intra_grant_inplace'::regclass FOR KEY SHARE;
}
teardown { ROLLBACK; }
# XXX extant bugs: permutation comments refer to planned post-bugfix behavior
permutation
b1
grant1
read2
addk2(c1) # inplace waits
c1
read2
# inplace thru KEY SHARE
permutation
keyshr5
addk2
# inplace wait NO KEY UPDATE w/ KEY SHARE
permutation
keyshr5
b3
sfnku3
addk2(r3)
r3
# same-xact rowmark
permutation
b2
sfnku2
addk2
c2
# same-xact rowmark in multixact
permutation
keyshr5
b2
sfnku2
addk2
c2
permutation
b3
sfu3
b1
grant1(r3) # acquire LockTuple(), await sfu3 xmax
read2
addk2(c1) # block in LockTuple() behind grant1
r3 # unblock grant1; addk2 now awaits grant1 xmax
c1
read2
permutation
b2
sfnku2
b1
grant1(c2) # acquire LockTuple(), await sfnku2 xmax
addk2 # block in LockTuple() behind grant1 = deadlock
c2
c1
read2
# SearchSysCacheLocked1() calling LockRelease()
permutation
b1
grant1
b3
sfu3(c1) # acquire LockTuple(), await grant1 xmax
revoke4(sfu3) # block in LockTuple() behind sfu3
c1
r3 # revoke4 unlocks old tuple and finds new
# SearchSysCacheLocked1() finding a tuple, then no tuple
permutation
b1
drop1
b3
sfu3(c1) # acquire LockTuple(), await drop1 xmax
revoke4(sfu3) # block in LockTuple() behind sfu3
c1 # sfu3 locks none; revoke4 unlocks old and finds none
r3

View File

@ -120,6 +120,40 @@ command_ok(
[ 'diff', $outputdir . '/primary.dump', $outputdir . '/standby.dump' ],
'compare primary and standby dumps');
# Likewise for the catalogs of the regression database, after disabling
# autovacuum to make fields like relpages stop changing.
$node_primary->append_conf('postgresql.conf', 'autovacuum = off');
$node_primary->restart;
$node_primary->wait_for_replay_catchup($node_standby_1);
command_ok(
[
'pg_dump',
('--schema', 'pg_catalog'),
('-f', $outputdir . '/catalogs_primary.dump'),
'--no-sync',
('-p', $node_primary->port),
'--no-unlogged-table-data',
'regression'
],
'dump catalogs of primary server');
command_ok(
[
'pg_dump',
('--schema', 'pg_catalog'),
('-f', $outputdir . '/catalogs_standby.dump'),
'--no-sync',
('-p', $node_standby_1->port),
'regression'
],
'dump catalogs of standby server');
command_ok(
[
'diff',
$outputdir . '/catalogs_primary.dump',
$outputdir . '/catalogs_standby.dump'
],
'compare primary and standby catalog dumps');
# Check some data from pg_stat_statements.
$node_primary->safe_psql('postgres', 'CREATE EXTENSION pg_stat_statements');
# This gathers data based on the first characters for some common query types,

View File

@ -0,0 +1,15 @@
CREATE DATABASE regression_tbd
ENCODING utf8 LC_COLLATE "C" LC_CTYPE "C" TEMPLATE template0;
ALTER DATABASE regression_tbd RENAME TO regression_utf8;
ALTER DATABASE regression_utf8 SET TABLESPACE regress_tblspace;
ALTER DATABASE regression_utf8 RESET TABLESPACE;
ALTER DATABASE regression_utf8 CONNECTION_LIMIT 123;
-- Test PgDatabaseToastTable. Doing this with GRANT would be slow.
BEGIN;
UPDATE pg_database
SET datacl = array_fill(makeaclitem(10, 10, 'USAGE', false), ARRAY[5e5::int])
WHERE datname = 'regression_utf8';
-- load catcache entry, if nothing else does
ALTER DATABASE regression_utf8 RESET TABLESPACE;
ROLLBACK;
DROP DATABASE regression_utf8;

View File

@ -2691,6 +2691,30 @@ drop cascades to table measurement_y2007m01
DROP FUNCTION measurement_insert_trigger();
-- prepare
RESET SESSION AUTHORIZATION;
-- try a system catalog
MERGE INTO pg_class c
USING (SELECT 'pg_depend'::regclass AS oid) AS j
ON j.oid = c.oid
WHEN MATCHED THEN
UPDATE SET reltuples = reltuples + 1
RETURNING j.oid;
oid
-----------
pg_depend
(1 row)
CREATE VIEW classv AS SELECT * FROM pg_class;
MERGE INTO classv c
USING pg_namespace n
ON n.oid = c.relnamespace
WHEN MATCHED AND c.oid = 'pg_depend'::regclass THEN
UPDATE SET reltuples = reltuples - 1
RETURNING c.oid;
oid
------
2608
(1 row)
DROP TABLE target, target2;
DROP TABLE source, source2;
DROP FUNCTION merge_trigfunc();

View File

@ -28,7 +28,7 @@ test: strings md5 numerology point lseg line box path polygon circle date time t
# geometry depends on point, lseg, line, box, path, polygon, circle
# horology depends on date, time, timetz, timestamp, timestamptz, interval
# ----------
test: geometry horology tstypes regex type_sanity opr_sanity misc_sanity comments expressions unicode xid mvcc
test: geometry horology tstypes regex type_sanity opr_sanity misc_sanity comments expressions unicode xid mvcc database
# ----------
# Load huge amounts of data

View File

@ -0,0 +1,17 @@
CREATE DATABASE regression_tbd
ENCODING utf8 LC_COLLATE "C" LC_CTYPE "C" TEMPLATE template0;
ALTER DATABASE regression_tbd RENAME TO regression_utf8;
ALTER DATABASE regression_utf8 SET TABLESPACE regress_tblspace;
ALTER DATABASE regression_utf8 RESET TABLESPACE;
ALTER DATABASE regression_utf8 CONNECTION_LIMIT 123;
-- Test PgDatabaseToastTable. Doing this with GRANT would be slow.
BEGIN;
UPDATE pg_database
SET datacl = array_fill(makeaclitem(10, 10, 'USAGE', false), ARRAY[5e5::int])
WHERE datname = 'regression_utf8';
-- load catcache entry, if nothing else does
ALTER DATABASE regression_utf8 RESET TABLESPACE;
ROLLBACK;
DROP DATABASE regression_utf8;

View File

@ -1713,6 +1713,23 @@ DROP FUNCTION measurement_insert_trigger();
-- prepare
RESET SESSION AUTHORIZATION;
-- try a system catalog
MERGE INTO pg_class c
USING (SELECT 'pg_depend'::regclass AS oid) AS j
ON j.oid = c.oid
WHEN MATCHED THEN
UPDATE SET reltuples = reltuples + 1
RETURNING j.oid;
CREATE VIEW classv AS SELECT * FROM pg_class;
MERGE INTO classv c
USING pg_namespace n
ON n.oid = c.relnamespace
WHEN MATCHED AND c.oid = 'pg_depend'::regclass THEN
UPDATE SET reltuples = reltuples - 1
RETURNING c.oid;
DROP TABLE target, target2;
DROP TABLE source, source2;
DROP FUNCTION merge_trigfunc();