database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-30 11:03:19 +03:00
Code Activity

Extend amcheck to check heap pages.

Browse Source 
Mark Dilger, reviewed by Peter Geoghegan, Andres Freund, Álvaro Herrera,
Michael Paquier, Amul Sul, and by me. Some last-minute cosmetic
revisions by me.

Discussion: http://postgr.es/m/12ED3DA8-25F0-4B68-937D-D907CFBF08E7@enterprisedb.com
This commit is contained in:
Robert Haas
2020-10-22 08:44:18 -04:00
parent f8721bd752
commit 866e24d47d
12 changed files with 2299 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
contrib/amcheck/Makefile
View File

@ -3,13 +3,16 @@
MODULE_big = amcheck
OBJS = \
$(WIN32RES) \
verify_heapam.o \
verify_nbtree.o
EXTENSION = amcheck
DATA = amcheck--1.1--1.2.sql amcheck--1.0--1.1.sql amcheck--1.0.sql
DATA = amcheck--1.2--1.3.sql amcheck--1.1--1.2.sql amcheck--1.0--1.1.sql amcheck--1.0.sql
PGFILEDESC = "amcheck - function for verifying relation integrity"
REGRESS = check check_btree
REGRESS = check check_btree check_heap
TAP_TESTS = 1
ifdef USE_PGXS
PG_CONFIG = pg_config

30
contrib/amcheck/amcheck--1.2--1.3.sql Normal file
View File

@ -0,0 +1,30 @@
/* contrib/amcheck/amcheck--1.2--1.3.sql */
-- complain if script is sourced in psql, rather than via CREATE EXTENSION
\echo Use "ALTER EXTENSION amcheck UPDATE TO '1.3'" to load this file. \quit
--
-- verify_heapam()
--
CREATE FUNCTION verify_heapam(relation regclass,
on_error_stop boolean default false,
check_toast boolean default false,
skip text default 'none',
startblock bigint default null,
endblock bigint default null,
blkno OUT bigint,
offnum OUT integer,
attnum OUT integer,
msg OUT text)
RETURNS SETOF record
AS 'MODULE_PATHNAME', 'verify_heapam'
LANGUAGE C;
-- Don't want this to be available to public
REVOKE ALL ON FUNCTION verify_heapam(regclass,
boolean,
boolean,
text,
bigint,
bigint)
FROM PUBLIC;

2
contrib/amcheck/amcheck.control
View File

@ -1,5 +1,5 @@
# amcheck extension
comment = 'functions for verifying relation integrity'
default_version = '1.2'
default_version = '1.3'
module_pathname = '$libdir/amcheck'
relocatable = true

194
contrib/amcheck/expected/check_heap.out Normal file
View File

@ -0,0 +1,194 @@
CREATE TABLE heaptest (a integer, b text);
REVOKE ALL ON heaptest FROM PUBLIC;
-- Check that invalid skip option is rejected
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'rope');
ERROR: invalid skip option
HINT: Valid skip options are "all-visible", "all-frozen", and "none".
-- Check specifying invalid block ranges when verifying an empty table
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0);
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 5, endblock := 8);
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
-- Check that valid options are not rejected nor corruption reported
-- for an empty table, and that skip enum-like parameter is case-insensitive
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'None');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'All-Frozen');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'All-Visible');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'NONE');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-FROZEN');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-VISIBLE');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
-- Add some data so subsequent tests are not entirely trivial
INSERT INTO heaptest (a, b)
(SELECT gs, repeat('x', gs)
FROM generate_series(1,50) gs);
-- Check that valid options are not rejected nor corruption reported
-- for a non-empty table
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0);
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
CREATE ROLE regress_heaptest_role;
-- verify permissions are checked (error due to function not callable)
SET ROLE regress_heaptest_role;
SELECT * FROM verify_heapam(relation := 'heaptest');
ERROR: permission denied for function verify_heapam
RESET ROLE;
GRANT EXECUTE ON FUNCTION verify_heapam(regclass, boolean, boolean, text, bigint, bigint) TO regress_heaptest_role;
-- verify permissions are now sufficient
SET ROLE regress_heaptest_role;
SELECT * FROM verify_heapam(relation := 'heaptest');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
RESET ROLE;
-- Check specifying invalid block ranges when verifying a non-empty table.
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 10000);
ERROR: ending block number must be between 0 and 0
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 10000, endblock := 11000);
ERROR: starting block number must be between 0 and 0
-- Vacuum freeze to change the xids encountered in subsequent tests
VACUUM FREEZE heaptest;
-- Check that valid options are not rejected nor corruption reported
-- for a non-empty frozen table
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible');
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0);
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
-- Check that partitioned tables (the parent ones) which don't have visibility
-- maps are rejected
CREATE TABLE test_partitioned (a int, b text default repeat('x', 5000))
PARTITION BY list (a);
SELECT * FROM verify_heapam('test_partitioned',
startblock := NULL,
endblock := NULL);
ERROR: "test_partitioned" is not a table, materialized view, or TOAST table
-- Check that valid options are not rejected nor corruption reported
-- for an empty partition table (the child one)
CREATE TABLE test_partition partition OF test_partitioned FOR VALUES IN (1);
SELECT * FROM verify_heapam('test_partition',
startblock := NULL,
endblock := NULL);
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
-- Check that valid options are not rejected nor corruption reported
-- for a non-empty partition table (the child one)
INSERT INTO test_partitioned (a) (SELECT 1 FROM generate_series(1,1000) gs);
SELECT * FROM verify_heapam('test_partition',
startblock := NULL,
endblock := NULL);
blkno | offnum | attnum | msg
-------+--------+--------+-----
(0 rows)
-- Check that indexes are rejected
CREATE INDEX test_index ON test_partition (a);
SELECT * FROM verify_heapam('test_index',
startblock := NULL,
endblock := NULL);
ERROR: "test_index" is not a table, materialized view, or TOAST table
-- Check that views are rejected
CREATE VIEW test_view AS SELECT 1;
SELECT * FROM verify_heapam('test_view',
startblock := NULL,
endblock := NULL);
ERROR: "test_view" is not a table, materialized view, or TOAST table
-- Check that sequences are rejected
CREATE SEQUENCE test_sequence;
SELECT * FROM verify_heapam('test_sequence',
startblock := NULL,
endblock := NULL);
ERROR: "test_sequence" is not a table, materialized view, or TOAST table
-- Check that foreign tables are rejected
CREATE FOREIGN DATA WRAPPER dummy;
CREATE SERVER dummy_server FOREIGN DATA WRAPPER dummy;
CREATE FOREIGN TABLE test_foreign_table () SERVER dummy_server;
SELECT * FROM verify_heapam('test_foreign_table',
startblock := NULL,
endblock := NULL);
ERROR: "test_foreign_table" is not a table, materialized view, or TOAST table
-- cleanup
DROP TABLE heaptest;
DROP TABLE test_partition;
DROP TABLE test_partitioned;
DROP OWNED BY regress_heaptest_role; -- permissions
DROP ROLE regress_heaptest_role;

116
contrib/amcheck/sql/check_heap.sql Normal file
View File

@ -0,0 +1,116 @@
CREATE TABLE heaptest (a integer, b text);
REVOKE ALL ON heaptest FROM PUBLIC;
-- Check that invalid skip option is rejected
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'rope');
-- Check specifying invalid block ranges when verifying an empty table
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0);
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 5, endblock := 8);
-- Check that valid options are not rejected nor corruption reported
-- for an empty table, and that skip enum-like parameter is case-insensitive
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'None');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'All-Frozen');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'All-Visible');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'NONE');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-FROZEN');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'ALL-VISIBLE');
-- Add some data so subsequent tests are not entirely trivial
INSERT INTO heaptest (a, b)
(SELECT gs, repeat('x', gs)
FROM generate_series(1,50) gs);
-- Check that valid options are not rejected nor corruption reported
-- for a non-empty table
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible');
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0);
CREATE ROLE regress_heaptest_role;
-- verify permissions are checked (error due to function not callable)
SET ROLE regress_heaptest_role;
SELECT * FROM verify_heapam(relation := 'heaptest');
RESET ROLE;
GRANT EXECUTE ON FUNCTION verify_heapam(regclass, boolean, boolean, text, bigint, bigint) TO regress_heaptest_role;
-- verify permissions are now sufficient
SET ROLE regress_heaptest_role;
SELECT * FROM verify_heapam(relation := 'heaptest');
RESET ROLE;
-- Check specifying invalid block ranges when verifying a non-empty table.
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 10000);
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 10000, endblock := 11000);
-- Vacuum freeze to change the xids encountered in subsequent tests
VACUUM FREEZE heaptest;
-- Check that valid options are not rejected nor corruption reported
-- for a non-empty frozen table
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'none');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-frozen');
SELECT * FROM verify_heapam(relation := 'heaptest', skip := 'all-visible');
SELECT * FROM verify_heapam(relation := 'heaptest', startblock := 0, endblock := 0);
-- Check that partitioned tables (the parent ones) which don't have visibility
-- maps are rejected
CREATE TABLE test_partitioned (a int, b text default repeat('x', 5000))
PARTITION BY list (a);
SELECT * FROM verify_heapam('test_partitioned',
startblock := NULL,
endblock := NULL);
-- Check that valid options are not rejected nor corruption reported
-- for an empty partition table (the child one)
CREATE TABLE test_partition partition OF test_partitioned FOR VALUES IN (1);
SELECT * FROM verify_heapam('test_partition',
startblock := NULL,
endblock := NULL);
-- Check that valid options are not rejected nor corruption reported
-- for a non-empty partition table (the child one)
INSERT INTO test_partitioned (a) (SELECT 1 FROM generate_series(1,1000) gs);
SELECT * FROM verify_heapam('test_partition',
startblock := NULL,
endblock := NULL);
-- Check that indexes are rejected
CREATE INDEX test_index ON test_partition (a);
SELECT * FROM verify_heapam('test_index',
startblock := NULL,
endblock := NULL);
-- Check that views are rejected
CREATE VIEW test_view AS SELECT 1;
SELECT * FROM verify_heapam('test_view',
startblock := NULL,
endblock := NULL);
-- Check that sequences are rejected
CREATE SEQUENCE test_sequence;
SELECT * FROM verify_heapam('test_sequence',
startblock := NULL,
endblock := NULL);
-- Check that foreign tables are rejected
CREATE FOREIGN DATA WRAPPER dummy;
CREATE SERVER dummy_server FOREIGN DATA WRAPPER dummy;
CREATE FOREIGN TABLE test_foreign_table () SERVER dummy_server;
SELECT * FROM verify_heapam('test_foreign_table',
startblock := NULL,
endblock := NULL);
-- cleanup
DROP TABLE heaptest;
DROP TABLE test_partition;
DROP TABLE test_partitioned;
DROP OWNED BY regress_heaptest_role; -- permissions
DROP ROLE regress_heaptest_role;

242
contrib/amcheck/t/001_verify_heapam.pl Normal file
View File

@ -0,0 +1,242 @@
use strict;
use warnings;
use PostgresNode;
use TestLib;
use Test::More tests => 65;
my ($node, $result);
#
# Test set-up
#
$node = get_new_node('test');
$node->init;
$node->append_conf('postgresql.conf', 'autovacuum=off');
$node->start;
$node->safe_psql('postgres', q(CREATE EXTENSION amcheck));
#
# Check a table with data loaded but no corruption, freezing, etc.
#
fresh_test_table('test');
check_all_options_uncorrupted('test', 'plain');
#
# Check a corrupt table
#
fresh_test_table('test');
corrupt_first_page('test');
detects_corruption(
"verify_heapam('test')",
"plain corrupted table");
detects_corruption(
"verify_heapam('test', skip := 'all-visible')",
"plain corrupted table skipping all-visible");
detects_corruption(
"verify_heapam('test', skip := 'all-frozen')",
"plain corrupted table skipping all-frozen");
detects_corruption(
"verify_heapam('test', check_toast := false)",
"plain corrupted table skipping toast");
detects_corruption(
"verify_heapam('test', startblock := 0, endblock := 0)",
"plain corrupted table checking only block zero");
#
# Check a corrupt table with all-frozen data
#
fresh_test_table('test');
$node->safe_psql('postgres', q(VACUUM FREEZE test));
corrupt_first_page('test');
detects_corruption(
"verify_heapam('test')",
"all-frozen corrupted table");
detects_no_corruption(
"verify_heapam('test', skip := 'all-frozen')",
"all-frozen corrupted table skipping all-frozen");
#
# Check a corrupt table with corrupt page header
#
fresh_test_table('test');
corrupt_first_page_and_header('test');
detects_corruption(
"verify_heapam('test')",
"corrupted test table with bad page header");
#
# Check an uncorrupted table with corrupt toast page header
#
fresh_test_table('test');
my $toast = get_toast_for('test');
corrupt_first_page_and_header($toast);
detects_corruption(
"verify_heapam('test', check_toast := true)",
"table with corrupted toast page header checking toast");
detects_no_corruption(
"verify_heapam('test', check_toast := false)",
"table with corrupted toast page header skipping toast");
detects_corruption(
"verify_heapam('$toast')",
"corrupted toast page header");
#
# Check an uncorrupted table with corrupt toast
#
fresh_test_table('test');
$toast = get_toast_for('test');
corrupt_first_page($toast);
detects_corruption(
"verify_heapam('test', check_toast := true)",
"table with corrupted toast checking toast");
detects_no_corruption(
"verify_heapam('test', check_toast := false)",
"table with corrupted toast skipping toast");
detects_corruption(
"verify_heapam('$toast')",
"corrupted toast table");
#
# Check an uncorrupted all-frozen table with corrupt toast
#
fresh_test_table('test');
$node->safe_psql('postgres', q(VACUUM FREEZE test));
$toast = get_toast_for('test');
corrupt_first_page($toast);
detects_corruption(
"verify_heapam('test', check_toast := true)",
"all-frozen table with corrupted toast checking toast");
detects_no_corruption(
"verify_heapam('test', check_toast := false)",
"all-frozen table with corrupted toast skipping toast");
detects_corruption(
"verify_heapam('$toast')",
"corrupted toast table of all-frozen table");
# Returns the filesystem path for the named relation.
sub relation_filepath
{
my ($relname) = @_;
my $pgdata = $node->data_dir;
my $rel = $node->safe_psql('postgres',
qq(SELECT pg_relation_filepath('$relname')));
die "path not found for relation $relname" unless defined $rel;
return "$pgdata/$rel";
}
# Returns the fully qualified name of the toast table for the named relation
sub get_toast_for
{
my ($relname) = @_;
$node->safe_psql('postgres', qq(
SELECT 'pg_toast.' || t.relname
FROM pg_catalog.pg_class c, pg_catalog.pg_class t
WHERE c.relname = '$relname'
AND c.reltoastrelid = t.oid));
}
# (Re)create and populate a test table of the given name.
sub fresh_test_table
{
my ($relname) = @_;
$node->safe_psql('postgres', qq(
DROP TABLE IF EXISTS $relname CASCADE;
CREATE TABLE $relname (a integer, b text);
ALTER TABLE $relname SET (autovacuum_enabled=false);
ALTER TABLE $relname ALTER b SET STORAGE external;
INSERT INTO $relname (a, b)
(SELECT gs, repeat('b',gs*10) FROM generate_series(1,1000) gs);
));
}
# Stops the test node, corrupts the first page of the named relation, and
# restarts the node.
sub corrupt_first_page_internal
{
my ($relname, $corrupt_header) = @_;
my $relpath = relation_filepath($relname);
$node->stop;
my $fh;
open($fh, '+<', $relpath);
binmode $fh;
# If we corrupt the header, postgres won't allow the page into the buffer.
syswrite($fh, '\xFF\xFF\xFF\xFF', 8) if ($corrupt_header);
# Corrupt at least the line pointers. Exactly what this corrupts will
# depend on the page, as it may run past the line pointers into the user
# data. We stop short of writing 2048 bytes (2k), the smallest supported
# page size, as we don't want to corrupt the next page.
seek($fh, 32, 0);
syswrite($fh, '\x77\x77\x77\x77', 500);
close($fh);
$node->start;
}
sub corrupt_first_page
{
corrupt_first_page_internal($_[0], undef);
}
sub corrupt_first_page_and_header
{
corrupt_first_page_internal($_[0], 1);
}
sub detects_corruption
{
my ($function, $testname) = @_;
my $result = $node->safe_psql('postgres',
qq(SELECT COUNT(*) > 0 FROM $function));
is($result, 't', $testname);
}
sub detects_no_corruption
{
my ($function, $testname) = @_;
my $result = $node->safe_psql('postgres',
qq(SELECT COUNT(*) = 0 FROM $function));
is($result, 't', $testname);
}
# Check various options are stable (don't abort) and do not report corruption
# when running verify_heapam on an uncorrupted test table.
#
# The relname *must* be an uncorrupted table, or this will fail.
#
# The prefix is used to identify the test, along with the options,
# and should be unique.
sub check_all_options_uncorrupted
{
my ($relname, $prefix) = @_;
for my $stop (qw(true false))
{
for my $check_toast (qw(true false))
{
for my $skip ("'none'", "'all-frozen'", "'all-visible'")
{
for my $startblock (qw(NULL 0))
{
for my $endblock (qw(NULL 0))
{
my $opts = "on_error_stop := $stop, " .
"check_toast := $check_toast, " .
"skip := $skip, " .
"startblock := $startblock, " .
"endblock := $endblock";
detects_no_corruption(
"verify_heapam('$relname', $opts)",
"$prefix: $opts");
}
}
}
}
}
}

1447
contrib/amcheck/verify_heapam.c Normal file
View File

File diff suppressed because it is too large Load Diff