database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-09-03 15:22:11 +03:00
Code Activity

Fix replay of create database records on standby

Browse Source 
Crash recovery on standby may encounter missing directories
when replaying database-creation WAL records.  Prior to this
patch, the standby would fail to recover in such a case;
however, the directories could be legitimately missing.
Consider the following sequence of commands:

    CREATE DATABASE
    DROP DATABASE
    DROP TABLESPACE

If, after replaying the last WAL record and removing the
tablespace directory, the standby crashes and has to replay the
create database record again, crash recovery must be able to continue.

A fix for this problem was already attempted in 49d9cfc68b, but it
was reverted because of design issues.  This new version is based
on Robert Haas' proposal: any missing tablespaces are created
during recovery before reaching consistency.  Tablespaces
are created as real directories, and should be deleted
by later replay.  CheckRecoveryConsistency ensures
they have disappeared.

The problems detected by this new code are reported as PANIC,
except when allow_in_place_tablespaces is set to ON, in which
case they are WARNING.  Apart from making tests possible, this
gives users an escape hatch in case things don't go as planned.

Author: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Author: Asim R Praveen <apraveen@pivotal.io>
Author: Paul Guo <paulguo@gmail.com>
Reviewed-by: Anastasia Lubennikova <lubennikovaav@gmail.com> (older versions)
Reviewed-by: Fujii Masao <masao.fujii@oss.nttdata.com> (older versions)
Reviewed-by: Michaël Paquier <michael@paquier.xyz>
Diagnosed-by: Paul Guo <paulguo@gmail.com>
Discussion: https://postgr.es/m/CAEET0ZGx9AvioViLf7nbR_8tH9-=27DN5xWJ2P9-ROH16e4JUA@mail.gmail.com
This commit is contained in:
Alvaro Herrera
2022-07-28 08:26:05 +02:00
parent 5ad478c9d9
commit a3aacb7cbf
4 changed files with 290 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
src/test/recovery/t/033_replay_tsp_drops.pl Normal file
View File

@@ -0,0 +1,163 @@
# Copyright (c) 2021-2022, PostgreSQL Global Development Group
# Test replay of tablespace/database creation/drop
use strict;
use warnings;
use PostgreSQL::Test::Cluster;
use PostgreSQL::Test::Utils;
use Test::More;
sub test_tablespace
{
my $node_primary = PostgreSQL::Test::Cluster->new("primary1");
$node_primary->init(allows_streaming => 1);
$node_primary->start;
$node_primary->psql(
'postgres',
qq[
SET allow_in_place_tablespaces=on;
CREATE TABLESPACE dropme_ts1 LOCATION '';
CREATE TABLESPACE dropme_ts2 LOCATION '';
CREATE TABLESPACE source_ts LOCATION '';
CREATE TABLESPACE target_ts LOCATION '';
CREATE DATABASE template_db IS_TEMPLATE = true;
]);
my $backup_name = 'my_backup';
$node_primary->backup($backup_name);
my $node_standby = PostgreSQL::Test::Cluster->new("standby2");
$node_standby->init_from_backup($node_primary, $backup_name,
has_streaming => 1);
$node_standby->append_conf('postgresql.conf',
"allow_in_place_tablespaces = on");
$node_standby->start;
# Make sure connection is made
$node_primary->poll_query_until('postgres',
'SELECT count(*) = 1 FROM pg_stat_replication');
$node_standby->safe_psql('postgres', 'CHECKPOINT');
# Do immediate shutdown just after a sequence of CREAT DATABASE / DROP
# DATABASE / DROP TABLESPACE. This causes CREATE DATABASE WAL records
# to be applied to already-removed directories.
my $query = q[
CREATE DATABASE dropme_db1 WITH TABLESPACE dropme_ts1;
CREATE TABLE t (a int) TABLESPACE dropme_ts2;
CREATE DATABASE dropme_db2 WITH TABLESPACE dropme_ts2;
CREATE DATABASE moveme_db TABLESPACE source_ts;
ALTER DATABASE moveme_db SET TABLESPACE target_ts;
CREATE DATABASE newdb TEMPLATE template_db;
ALTER DATABASE template_db IS_TEMPLATE = false;
DROP DATABASE dropme_db1;
DROP TABLE t;
DROP DATABASE dropme_db2; DROP TABLESPACE dropme_ts2;
DROP TABLESPACE source_ts;
DROP DATABASE template_db;
];
$node_primary->safe_psql('postgres', $query);
$node_primary->wait_for_catchup($node_standby, 'replay',
$node_primary->lsn('write'));
# show "create missing directory" log message
$node_standby->safe_psql('postgres',
"ALTER SYSTEM SET log_min_messages TO debug1;");
$node_standby->stop('immediate');
# Should restart ignoring directory creation error.
is($node_standby->start(fail_ok => 1), 1, "standby node started");
$node_standby->stop('immediate');
}
test_tablespace();
# Ensure that a missing tablespace directory during create database
# replay immediately causes panic if the standby has already reached
# consistent state (archive recovery is in progress).
my $node_primary = PostgreSQL::Test::Cluster->new('primary2');
$node_primary->init(allows_streaming => 1);
$node_primary->start;
# Create tablespace
$node_primary->safe_psql(
'postgres', q[
SET allow_in_place_tablespaces=on;
CREATE TABLESPACE ts1 LOCATION ''
]);
$node_primary->safe_psql('postgres',
"CREATE DATABASE db1 WITH TABLESPACE ts1");
# Take backup
my $backup_name = 'my_backup';
$node_primary->backup($backup_name);
my $node_standby = PostgreSQL::Test::Cluster->new('standby3');
$node_standby->init_from_backup($node_primary, $backup_name,
has_streaming => 1);
$node_standby->append_conf('postgresql.conf',
"allow_in_place_tablespaces = on");
$node_standby->start;
# Make sure standby reached consistency and starts accepting connections
$node_standby->poll_query_until('postgres', 'SELECT 1', '1');
# Remove standby tablespace directory so it will be missing when
# replay resumes.
my $tspoid = $node_standby->safe_psql('postgres',
"SELECT oid FROM pg_tablespace WHERE spcname = 'ts1';");
my $tspdir = $node_standby->data_dir . "/pg_tblspc/$tspoid";
File::Path::rmtree($tspdir);
my $logstart = get_log_size($node_standby);
# Create a database in the tablespace and a table in default tablespace
$node_primary->safe_psql(
'postgres',
q[
CREATE TABLE should_not_replay_insertion(a int);
CREATE DATABASE db2 WITH TABLESPACE ts1;
INSERT INTO should_not_replay_insertion VALUES (1);
]);
# Standby should fail and should not silently skip replaying the wal
# In this test, PANIC turns into WARNING by allow_in_place_tablespaces.
# Check the log messages instead of confirming standby failure.
my $max_attempts = $PostgreSQL::Test::Utils::timeout_default;
while ($max_attempts-- >= 0)
{
last
if (
find_in_log(
$node_standby, "WARNING: creating missing directory: pg_tblspc/",
$logstart));
sleep 1;
}
ok($max_attempts > 0, "invalid directory creation is detected");
done_testing();
# return the size of logfile of $node in bytes
sub get_log_size
{
my ($node) = @_;
return (stat $node->logfile)[7];
}
# find $pat in logfile of $node after $off-th byte
sub find_in_log
{
my ($node, $pat, $off) = @_;
$off = 0 unless defined $off;
my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
return 0 if (length($log) <= $off);
$log = substr($log, $off);
return $log =~ m/$pat/;
}