database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-06-11 20:28:21 +03:00
Code Activity

Fix logical decoding error when system table w/ toast is repeatedly rewritten.

Browse Source 
Repeatedly rewriting a mapped catalog table with VACUUM FULL or
CLUSTER could cause logical decoding to fail with:
ERROR, "could not map filenode \"%s\" to relation OID"

To trigger the problem the rewritten catalog had to have live tuples
with toasted columns.

The problem was triggered as during catalog table rewrites the
heap_insert() check that prevents logical decoding information to be
emitted for system catalogs, failed to treat the new heap's toast table
as a system catalog (because the new heap is not recognized as a
catalog table via RelationIsLogicallyLogged()). The relmapper, in
contrast to the normal catalog contents, does not contain historical
information. After a single rewrite of a mapped table the new relation
is known to the relmapper, but if the table is rewritten twice before
logical decoding occurs, the relfilenode cannot be mapped to a
relation anymore.  Which then leads us to error out.   This only
happens for toast tables, because the main table contents aren't
re-inserted with heap_insert().

The fix is simple, add a new heap_insert() flag that prevents logical
decoding information from being emitted, and accept during decoding
that there might not be tuple data for toast tables.

Unfortunately that does not fix pre-existing logical decoding
errors. Doing so would require not throwing an error when a filenode
cannot be mapped to a relation during decoding, and that seems too
likely to hide bugs.  If it's crucial to fix decoding for an existing
slot, temporarily changing the ERROR in ReorderBufferCommit() to a
WARNING appears to be the best fix.

Author: Andres Freund
Discussion: https://postgr.es/m/20180914021046.oi7dm4ra3ot2g2kt@alap3.anarazel.de
Backpatch: 9.4-, where logical decoding was introduced
This commit is contained in:
Andres Freund
2018-10-10 13:53:02 -07:00
parent a653569c14
commit a88482dd24
6 changed files with 163 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
contrib/test_decoding/sql/rewrite.sql
View File

@ -3,6 +3,35 @@ SET synchronous_commit = on;
DROP TABLE IF EXISTS replication_example;
-- Ensure there's tables with toast datums. To do so, we dynamically
-- create a function returning a large textblob. We want tables of
-- different kinds: mapped catalog table, unmapped catalog table,
-- shared catalog table and usertable.
CREATE FUNCTION exec(text) returns void language plpgsql volatile
AS $f$
BEGIN
EXECUTE $1;
END;
$f$;
CREATE ROLE justforcomments NOLOGIN;
SELECT exec(
format($outer$CREATE FUNCTION iamalongfunction() RETURNS TEXT IMMUTABLE LANGUAGE SQL AS $f$SELECT text %L$f$$outer$,
(SELECT repeat(string_agg(to_char(g.i, 'FM0000'), ''), 50) FROM generate_series(1, 500) g(i))));
SELECT exec(
format($outer$COMMENT ON FUNCTION iamalongfunction() IS %L$outer$,
iamalongfunction()));
SELECT exec(
format($outer$COMMENT ON ROLE JUSTFORCOMMENTS IS %L$outer$,
iamalongfunction()));
CREATE TABLE iamalargetable AS SELECT iamalongfunction() longfunctionoutput;
-- verify toast usage
SELECT pg_relation_size((SELECT reltoastrelid FROM pg_class WHERE oid = 'pg_proc'::regclass)) > 0;
SELECT pg_relation_size((SELECT reltoastrelid FROM pg_class WHERE oid = 'pg_description'::regclass)) > 0;
SELECT pg_relation_size((SELECT reltoastrelid FROM pg_class WHERE oid = 'pg_shdescription'::regclass)) > 0;
SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'test_decoding');
CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120));
INSERT INTO replication_example(somedata) VALUES (1);
@ -57,6 +86,17 @@ COMMIT;
CHECKPOINT;
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
SELECT pg_drop_replication_slot('regression_slot');
-- trigger repeated rewrites of a system catalog with a toast table,
-- that previously was buggy: 20180914021046.oi7dm4ra3ot2g2kt@alap3.anarazel.de
VACUUM FULL pg_proc; VACUUM FULL pg_description; VACUUM FULL pg_shdescription; VACUUM FULL iamalargetable;
INSERT INTO replication_example(somedata, testcolumn1, testcolumn3) VALUES (8, 6, 1);
VACUUM FULL pg_proc; VACUUM FULL pg_description; VACUUM FULL pg_shdescription; VACUUM FULL iamalargetable;
INSERT INTO replication_example(somedata, testcolumn1, testcolumn3) VALUES (9, 7, 1);
SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
SELECT pg_drop_replication_slot('regression_slot');
DROP TABLE IF EXISTS replication_example;
DROP FUNCTION iamalongfunction();
DROP FUNCTION exec(text);
DROP ROLE justforcomments;