mirror of
https://github.com/postgres/postgres.git
synced 2025-10-31 10:30:33 +03:00
e9edc1ba0b
Repeatedly rewriting a mapped catalog table with VACUUM FULL or CLUSTER could cause logical decoding to fail with: ERROR, "could not map filenode \"%s\" to relation OID" To trigger the problem the rewritten catalog had to have live tuples with toasted columns. The problem was triggered as during catalog table rewrites the heap_insert() check that prevents logical decoding information to be emitted for system catalogs, failed to treat the new heap's toast table as a system catalog (because the new heap is not recognized as a catalog table via RelationIsLogicallyLogged()). The relmapper, in contrast to the normal catalog contents, does not contain historical information. After a single rewrite of a mapped table the new relation is known to the relmapper, but if the table is rewritten twice before logical decoding occurs, the relfilenode cannot be mapped to a relation anymore. Which then leads us to error out. This only happens for toast tables, because the main table contents aren't re-inserted with heap_insert(). The fix is simple, add a new heap_insert() flag that prevents logical decoding information from being emitted, and accept during decoding that there might not be tuple data for toast tables. Unfortunately that does not fix pre-existing logical decoding errors. Doing so would require not throwing an error when a filenode cannot be mapped to a relation during decoding, and that seems too likely to hide bugs. If it's crucial to fix decoding for an existing slot, temporarily changing the ERROR in ReorderBufferCommit() to a WARNING appears to be the best fix. Author: Andres Freund Discussion: https://postgr.es/m/20180914021046.oi7dm4ra3ot2g2kt@alap3.anarazel.de Backpatch: 9.4-, where logical decoding was introduced
161 lines
6.7 KiB
Plaintext
161 lines
6.7 KiB
Plaintext
-- predictability
|
|
SET synchronous_commit = on;
|
|
DROP TABLE IF EXISTS replication_example;
|
|
-- Ensure there's tables with toast datums. To do so, we dynamically
|
|
-- create a function returning a large textblob. We want tables of
|
|
-- different kinds: mapped catalog table, unmapped catalog table,
|
|
-- shared catalog table and usertable.
|
|
CREATE FUNCTION exec(text) returns void language plpgsql volatile
|
|
AS $f$
|
|
BEGIN
|
|
EXECUTE $1;
|
|
END;
|
|
$f$;
|
|
CREATE ROLE justforcomments NOLOGIN;
|
|
SELECT exec(
|
|
format($outer$CREATE FUNCTION iamalongfunction() RETURNS TEXT IMMUTABLE LANGUAGE SQL AS $f$SELECT text %L$f$$outer$,
|
|
(SELECT repeat(string_agg(to_char(g.i, 'FM0000'), ''), 50) FROM generate_series(1, 500) g(i))));
|
|
exec
|
|
------
|
|
|
|
(1 row)
|
|
|
|
SELECT exec(
|
|
format($outer$COMMENT ON FUNCTION iamalongfunction() IS %L$outer$,
|
|
iamalongfunction()));
|
|
exec
|
|
------
|
|
|
|
(1 row)
|
|
|
|
SELECT exec(
|
|
format($outer$COMMENT ON ROLE JUSTFORCOMMENTS IS %L$outer$,
|
|
iamalongfunction()));
|
|
exec
|
|
------
|
|
|
|
(1 row)
|
|
|
|
CREATE TABLE iamalargetable AS SELECT iamalongfunction() longfunctionoutput;
|
|
-- verify toast usage
|
|
SELECT pg_relation_size((SELECT reltoastrelid FROM pg_class WHERE oid = 'pg_proc'::regclass)) > 0;
|
|
?column?
|
|
----------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT pg_relation_size((SELECT reltoastrelid FROM pg_class WHERE oid = 'pg_description'::regclass)) > 0;
|
|
?column?
|
|
----------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT pg_relation_size((SELECT reltoastrelid FROM pg_class WHERE oid = 'pg_shdescription'::regclass)) > 0;
|
|
?column?
|
|
----------
|
|
t
|
|
(1 row)
|
|
|
|
SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding');
|
|
?column?
|
|
----------
|
|
init
|
|
(1 row)
|
|
|
|
CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120));
|
|
INSERT INTO replication_example(somedata) VALUES (1);
|
|
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
|
|
data
|
|
----------------------------------------------------------------------------------------------------------
|
|
BEGIN
|
|
table public.replication_example: INSERT: id[integer]:1 somedata[integer]:1 text[character varying]:null
|
|
COMMIT
|
|
(3 rows)
|
|
|
|
BEGIN;
|
|
INSERT INTO replication_example(somedata) VALUES (2);
|
|
ALTER TABLE replication_example ADD COLUMN testcolumn1 int;
|
|
INSERT INTO replication_example(somedata, testcolumn1) VALUES (3, 1);
|
|
COMMIT;
|
|
BEGIN;
|
|
INSERT INTO replication_example(somedata) VALUES (3);
|
|
ALTER TABLE replication_example ADD COLUMN testcolumn2 int;
|
|
INSERT INTO replication_example(somedata, testcolumn1, testcolumn2) VALUES (4, 2, 1);
|
|
COMMIT;
|
|
VACUUM FULL pg_am;
|
|
VACUUM FULL pg_amop;
|
|
VACUUM FULL pg_proc;
|
|
VACUUM FULL pg_opclass;
|
|
VACUUM FULL pg_type;
|
|
VACUUM FULL pg_index;
|
|
VACUUM FULL pg_database;
|
|
-- repeated rewrites that fail
|
|
BEGIN;
|
|
CLUSTER pg_class USING pg_class_oid_index;
|
|
CLUSTER pg_class USING pg_class_oid_index;
|
|
ROLLBACK;
|
|
-- repeated rewrites that succeed
|
|
BEGIN;
|
|
CLUSTER pg_class USING pg_class_oid_index;
|
|
CLUSTER pg_class USING pg_class_oid_index;
|
|
CLUSTER pg_class USING pg_class_oid_index;
|
|
COMMIT;
|
|
-- repeated rewrites in different transactions
|
|
VACUUM FULL pg_class;
|
|
VACUUM FULL pg_class;
|
|
INSERT INTO replication_example(somedata, testcolumn1) VALUES (5, 3);
|
|
BEGIN;
|
|
INSERT INTO replication_example(somedata, testcolumn1) VALUES (6, 4);
|
|
ALTER TABLE replication_example ADD COLUMN testcolumn3 int;
|
|
INSERT INTO replication_example(somedata, testcolumn1, testcolumn3) VALUES (7, 5, 1);
|
|
COMMIT;
|
|
-- make old files go away
|
|
CHECKPOINT;
|
|
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
|
|
data
|
|
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
BEGIN
|
|
table public.replication_example: INSERT: id[integer]:2 somedata[integer]:2 text[character varying]:null
|
|
table public.replication_example: INSERT: id[integer]:3 somedata[integer]:3 text[character varying]:null testcolumn1[integer]:1
|
|
COMMIT
|
|
BEGIN
|
|
table public.replication_example: INSERT: id[integer]:4 somedata[integer]:3 text[character varying]:null testcolumn1[integer]:null
|
|
table public.replication_example: INSERT: id[integer]:5 somedata[integer]:4 text[character varying]:null testcolumn1[integer]:2 testcolumn2[integer]:1
|
|
COMMIT
|
|
BEGIN
|
|
table public.replication_example: INSERT: id[integer]:6 somedata[integer]:5 text[character varying]:null testcolumn1[integer]:3 testcolumn2[integer]:null
|
|
COMMIT
|
|
BEGIN
|
|
table public.replication_example: INSERT: id[integer]:7 somedata[integer]:6 text[character varying]:null testcolumn1[integer]:4 testcolumn2[integer]:null
|
|
table public.replication_example: INSERT: id[integer]:8 somedata[integer]:7 text[character varying]:null testcolumn1[integer]:5 testcolumn2[integer]:null testcolumn3[integer]:1
|
|
COMMIT
|
|
(15 rows)
|
|
|
|
-- trigger repeated rewrites of a system catalog with a toast table,
|
|
-- that previously was buggy: 20180914021046.oi7dm4ra3ot2g2kt@alap3.anarazel.de
|
|
VACUUM FULL pg_proc; VACUUM FULL pg_description; VACUUM FULL pg_shdescription; VACUUM FULL iamalargetable;
|
|
INSERT INTO replication_example(somedata, testcolumn1, testcolumn3) VALUES (8, 6, 1);
|
|
VACUUM FULL pg_proc; VACUUM FULL pg_description; VACUUM FULL pg_shdescription; VACUUM FULL iamalargetable;
|
|
INSERT INTO replication_example(somedata, testcolumn1, testcolumn3) VALUES (9, 7, 1);
|
|
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
|
|
data
|
|
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
BEGIN
|
|
table public.replication_example: INSERT: id[integer]:9 somedata[integer]:8 text[character varying]:null testcolumn1[integer]:6 testcolumn2[integer]:null testcolumn3[integer]:1
|
|
COMMIT
|
|
BEGIN
|
|
table public.replication_example: INSERT: id[integer]:10 somedata[integer]:9 text[character varying]:null testcolumn1[integer]:7 testcolumn2[integer]:null testcolumn3[integer]:1
|
|
COMMIT
|
|
(6 rows)
|
|
|
|
SELECT pg_drop_replication_slot('regression_slot');
|
|
pg_drop_replication_slot
|
|
--------------------------
|
|
|
|
(1 row)
|
|
|
|
DROP TABLE IF EXISTS replication_example;
|
|
DROP FUNCTION iamalongfunction();
|
|
DROP FUNCTION exec(text);
|
|
DROP ROLE justforcomments;
|