1
0
mirror of https://github.com/MariaDB/server.git synced 2026-10-23 18:49:29 +03:00

MDEV-38976 Add MTR test for Galera write loss after crash and partition

Add galera_3nodes.MDEV-38976, a reduced version of the Jepsen scenario
from the bug report:

1. Nodes 1,2,3 form a primary component.
2. Node 3 is partitioned away (gmcast.isolate=1) and becomes
   non-primary, keeping the old primary component in its saved view.
3. Nodes 1,2 stay primary and commit more rows.
4. Nodes 1,2 are killed together. Their committed positions are
   recovered with --wsrep-recover. The partition then heals and nodes
   1,2 are restarted with --wsrep-start-position and without bootstrap.

Nodes 1,2 are ahead of node 3, so the group reports "Reversing history"
and nodes 1,2 abort. That outcome is accepted, because the cluster was
restarted in the wrong order. What the test checks is that the abort
does not destroy the saved state of nodes 1,2:

- grastate.dat of nodes 1,2 must keep its UUID. It must not be reset to
  00000000-0000-0000-0000-000000000000:-1, which sends the node through
  a full SST from the stale node and loses the committed writes.
- After the cluster is restarted in the correct order (node 1
  bootstraps, node 2 joins, node 3 is resynced by SST), all three nodes
  must have all 10 rows.

The test requires Galera 26.4.29 or later, which no longer invalidates
the saved state in async_recv() on the "Reversing history"
inconsistency. It is skipped with an older library. With the older
library nodes 1,2 come back through SST from node 3 and the rows are
lost.

The test waits for the abort by searching the error logs after the
offset recorded before the restart. MTR does not truncate the logs
between --repeat iterations, so a search of the whole log would match
output from earlier runs.

Assisted-by Claude AI
This commit is contained in:
Pekka Lampio
2026-10-19 14:54:45 +03:00
parent b2bc6c2f70
commit 2c769b2a84
3 changed files with 392 additions and 0 deletions

View File

@@ -0,0 +1,91 @@
connection node_2;
connection node_1;
# Correct Galera library found
connection node_1;
connection node_2;
connection node_3;
connection node_1;
connection node_2;
connection node_3;
connection node_1;
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1), (2), (3);
connection node_3;
SET SESSION wsrep_sync_wait = 0;
SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
connection node_1;
SET SESSION wsrep_sync_wait = 0;
INSERT INTO t1 VALUES (4), (5), (6), (7), (8), (9), (10);
connection node_2;
SELECT COUNT(*) AS node_2_rows_before_kill FROM t1;
node_2_rows_before_kill
10
connection node_2;
Killing server ...
connection node_1;
Killing server ...
Performing --wsrep-recover ...
Performing --wsrep-recover ...
connection node_3;
SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
# grastate.dat of nodes 1,2 kept its UUID after the abort
connection node_3;
connection node_1;
connection node_2;
connection node_3;
connection node_1;
connection node_2;
connection node_3;
connection node_1;
SELECT COUNT(*) AS node_1_rows, MAX(f1) AS node_1_max FROM t1;
node_1_rows node_1_max
10 10
connection node_2;
SELECT COUNT(*) AS node_2_rows, MAX(f1) AS node_2_max FROM t1;
node_2_rows node_2_max
10 10
connection node_3;
SELECT COUNT(*) AS node_3_rows, MAX(f1) AS node_3_max FROM t1;
node_3_rows node_3_max
10 10
connection node_1;
DROP TABLE t1;
connection node_1;
CALL mtr.add_suppression("WSREP: .*Reversing history");
CALL mtr.add_suppression("WSREP: gcs/src/gcs_group\\.cpp:group_post_state_exchange\\(\\)");
CALL mtr.add_suppression("WSREP: Node is ahead of the primary component");
CALL mtr.add_suppression("WSREP: node uuid: .* is inconsistent to restored view");
CALL mtr.add_suppression("WSREP: Wsrep startup was interrupted");
CALL mtr.add_suppression("WSREP: .*Calling destroy\\(\\) before close");
CALL mtr.add_suppression("WSREP: .*GCS core destructor failed");
CALL mtr.add_suppression("WSREP: server: .* unallowed state transition");
CALL mtr.add_suppression("Aborting");
CALL mtr.add_suppression("Assertion .0. failed");
CALL mtr.add_suppression("got signal 6");
CALL mtr.add_suppression("Attempting backtrace");
connection node_2;
CALL mtr.add_suppression("WSREP: .*Reversing history");
CALL mtr.add_suppression("WSREP: gcs/src/gcs_group\\.cpp:group_post_state_exchange\\(\\)");
CALL mtr.add_suppression("WSREP: Node is ahead of the primary component");
CALL mtr.add_suppression("WSREP: node uuid: .* is inconsistent to restored view");
CALL mtr.add_suppression("WSREP: Wsrep startup was interrupted");
CALL mtr.add_suppression("WSREP: .*Calling destroy\\(\\) before close");
CALL mtr.add_suppression("WSREP: .*GCS core destructor failed");
CALL mtr.add_suppression("WSREP: server: .* unallowed state transition");
CALL mtr.add_suppression("Aborting");
CALL mtr.add_suppression("Assertion .0. failed");
CALL mtr.add_suppression("got signal 6");
CALL mtr.add_suppression("Attempting backtrace");
connection node_3;
CALL mtr.add_suppression("WSREP: .*Reversing history");
CALL mtr.add_suppression("WSREP: gcs/src/gcs_group\\.cpp:group_post_state_exchange\\(\\)");
CALL mtr.add_suppression("WSREP: Node is ahead of the primary component");
CALL mtr.add_suppression("WSREP: node uuid: .* is inconsistent to restored view");
CALL mtr.add_suppression("WSREP: Wsrep startup was interrupted");
CALL mtr.add_suppression("WSREP: .*Calling destroy\\(\\) before close");
CALL mtr.add_suppression("WSREP: .*GCS core destructor failed");
CALL mtr.add_suppression("WSREP: server: .* unallowed state transition");
CALL mtr.add_suppression("Aborting");
CALL mtr.add_suppression("Assertion .0. failed");
CALL mtr.add_suppression("got signal 6");
CALL mtr.add_suppression("Attempting backtrace");

View File

@@ -0,0 +1,4 @@
# Dedicated cluster: the test kills nodes 1 and 2 and leaves node 3 with a
# stale primary component.
!include ../galera_3nodes.cnf

View File

@@ -0,0 +1,297 @@
#
# MDEV-38976: Galera write loss with process crashes and network partitions
#
# Scenario (reduced from a Jepsen run):
#
# 1. Nodes 1,2,3 form a primary component.
# 2. Node 3 is partitioned away and becomes non-primary. It keeps the old
# primary component (nodes 1,2,3) in its saved view.
# 3. Nodes 1,2 stay primary and commit more writes.
# 4. Nodes 1,2 are killed at the same time. The partition towards node 3 then
# heals, and nodes 1,2 are restarted without bootstrapping.
#
# Nodes 1,2 are ahead of node 3, so the cluster refuses to continue and
# nodes 1,2 abort ("Reversing history"). That is acceptable: the cluster was
# restarted incorrectly. What must not happen is that nodes 1,2 lose their
# state when they abort (grastate.dat reset to 0000...:-1), so that they come
# back through a full SST from the stale node 3 and lose the writes made in
# step 3. The test checks that the saved state of nodes 1,2 survives the abort
# and that, after restarting the cluster in the correct order (nodes 1,2 first,
# node 3 last), all committed rows are still there.
#
--source include/big_test.inc
--source include/galera_cluster.inc
--source include/have_innodb.inc
# Galera 26.4.29 keeps the saved state (grastate.dat) of a node that aborts
# because it is ahead of the primary component. Earlier versions invalidate it.
--let $galera_version=26.4.29
source ../wsrep/include/check_galera_version.inc;
--source include/force_restart.inc
--let $galera_connection_name = node_3
--let $galera_server_number = 3
--source include/galera_connect.inc
--let $node_1 = node_1
--let $node_2 = node_2
--let $node_3 = node_3
--source ../galera/include/auto_increment_offset_save.inc
# Galera ports, needed for the restart addresses
--connection node_1
--source ../galera/include/galera_base_port.inc
--let $NODE_GALERAPORT_1 = $_NODE_GALERAPORT
--connection node_2
--source ../galera/include/galera_base_port.inc
--let $NODE_GALERAPORT_2 = $_NODE_GALERAPORT
--connection node_3
--source ../galera/include/galera_base_port.inc
--let $NODE_GALERAPORT_3 = $_NODE_GALERAPORT
--connection node_1
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1), (2), (3);
--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
--source include/wait_condition.inc
#
# Step 2: partition node 3 away
#
--connection node_3
SET SESSION wsrep_sync_wait = 0;
SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
--let $wait_condition = SELECT VARIABLE_VALUE = 'non-Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'
--source include/wait_condition.inc
--connection node_1
SET SESSION wsrep_sync_wait = 0;
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
--source include/wait_condition.inc
--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'
--source include/wait_condition.inc
#
# Step 3: nodes 1,2 advance while node 3 is away
#
INSERT INTO t1 VALUES (4), (5), (6), (7), (8), (9), (10);
--connection node_2
SELECT COUNT(*) AS node_2_rows_before_kill FROM t1;
#
# Step 4: kill nodes 1,2 at the same time
#
--connection node_2
--source include/kill_galera.inc
--connection node_1
--source include/kill_galera.inc
# Recover the committed positions of the killed nodes, like the service
# scripts do before restarting a crashed node.
--let $galera_wsrep_recover_server_id = 1
--source ../galera/include/galera_wsrep_recover.inc
--let $node_1_start_position = $galera_wsrep_start_position
--let $galera_wsrep_recover_server_id = 2
--source ../galera/include/galera_wsrep_recover.inc
--let $node_2_start_position = $galera_wsrep_start_position
# The error logs are not truncated between test runs (e.g. with --repeat), so
# remember where they end now and only look at what is written after this point.
--let MDEV38976_LOGDIR = $MYSQLTEST_VARDIR/log
--let MDEV38976_TMPDIR = $MYSQLTEST_VARDIR/tmp
--perl
use strict;
for my $n (1, 2)
{
my $size = -s "$ENV{MDEV38976_LOGDIR}/mysqld.$n.err" || 0;
my $file = "$ENV{MDEV38976_TMPDIR}/mdev38976_$n.offset";
open(my $fh, '>', $file) or die "Cannot write $file: $!\n";
print $fh "$size\n";
close($fh);
}
EOF
# Heal the partition: node 3 is reachable again, still non-primary
--connection node_3
SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
# Restart nodes 1,2 simultaneously, neither of them bootstraps.
--write_line "restart: --wsrep_cluster_address=gcomm://127.0.0.1:$NODE_GALERAPORT_2,127.0.0.1:$NODE_GALERAPORT_3 --wsrep-start-position=$node_1_start_position" $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--write_line "restart: --wsrep_cluster_address=gcomm://127.0.0.1:$NODE_GALERAPORT_1,127.0.0.1:$NODE_GALERAPORT_3 --wsrep-start-position=$node_2_start_position" $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
#
# Nodes 1,2 are ahead of node 3, so they must abort. Wait for that.
#
--perl
use strict;
use Time::HiRes qw(sleep);
for my $n (1, 2)
{
# mysql-test-run.pl removes the expect file when it restarts the server.
# Then mark the coming abort as expected, so that it is not reported as
# an unexpected server exit.
my $expect = "$ENV{MDEV38976_TMPDIR}/mysqld.$n.expect";
for (my $i = 0; $i < 600 && -e $expect; $i++) { sleep(0.1); }
die "mysqld.$n was not restarted\n" if -e $expect;
open(my $ef, '>', $expect) or die "Cannot write $expect: $!\n";
print $ef "wait\n";
close($ef);
}
for my $n (1, 2)
{
my $log = "$ENV{MDEV38976_LOGDIR}/mysqld.$n.err";
my $ofile = "$ENV{MDEV38976_TMPDIR}/mdev38976_$n.offset";
open(my $of, '<', $ofile) or die "Cannot read $ofile: $!\n";
my $offset = <$of> + 0;
close($of);
unlink($ofile);
my $found = 0;
for (my $i = 0; $i < 1200 && !$found; $i++)
{
if (open(my $fh, '<', $log))
{
seek($fh, $offset, 0);
while (<$fh>) { if (/Reversing history/) { $found = 1; last; } }
close($fh);
}
sleep(0.1) unless $found;
}
die "Node $n did not report 'Reversing history'\n" unless $found;
# Wait until the aborting server process has exited
my $pidfile = "$ENV{MYSQLTEST_VARDIR}/run/mysqld.$n.pid";
if (open(my $pf, '<', $pidfile))
{
my $pid = <$pf>;
close($pf);
chomp($pid);
if ($pid =~ /^\d+$/)
{
my $i;
for ($i = 0; $i < 1200 && kill(0, $pid); $i++) { sleep(0.1); }
die "Node $n did not exit after the abort\n" if kill(0, $pid);
}
}
}
EOF
#
# The saved state of nodes 1,2 must have survived the abort: it must not have
# been reset to the undefined UUID.
#
--error 1
--exec grep -q '^uuid: *00000000-0000-0000-0000-000000000000' $MYSQLTEST_VARDIR/mysqld.1/data/grastate.dat
--error 1
--exec grep -q '^uuid: *00000000-0000-0000-0000-000000000000' $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat
--echo # grastate.dat of nodes 1,2 kept its UUID after the abort
#
# Restart the cluster in the correct order: nodes 1,2 hold the newest state.
# Node 3 is stale, so it has to rejoin last.
#
--connection node_3
--source include/shutdown_mysqld.inc
# Node 3 is stale: it was dragged into a primary component without the newest
# writes. Force a full state transfer for it, as an operator would.
--remove_file $MYSQLTEST_VARDIR/mysqld.3/data/grastate.dat
# Node 1 is the bootstrap node, mark it safe to bootstrap like an operator does
--exec sed -i 's/^safe_to_bootstrap: .*/safe_to_bootstrap: 1/' $MYSQLTEST_VARDIR/mysqld.1/data/grastate.dat
--write_line "restart: --wsrep_new_cluster --wsrep_cluster_address=gcomm:// --wsrep-start-position=$node_1_start_position" $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--connection node_1
--enable_reconnect
--source include/wait_until_connected_again.inc
--source include/galera_wait_ready.inc
--write_line "restart: --wsrep_cluster_address=gcomm://127.0.0.1:$NODE_GALERAPORT_1 --wsrep-start-position=$node_2_start_position" $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
--connection node_2
--enable_reconnect
--source include/wait_until_connected_again.inc
--source include/galera_wait_ready.inc
--connection node_3
--let $restart_noprint = 2
--let $restart_parameters = --wsrep_cluster_address=gcomm://127.0.0.1:$NODE_GALERAPORT_1
--source include/start_mysqld.inc
--source include/galera_wait_ready.inc
#
# All nodes must be back in a single primary component
#
--connection node_1
--let $wait_condition = SELECT VARIABLE_VALUE = 3 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
--source include/wait_condition.inc
--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
--source include/wait_condition.inc
--connection node_2
--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
--source include/wait_condition.inc
--connection node_3
--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'
--source include/wait_condition.inc
#
# The writes committed in step 3 must not be lost, on any node
#
--connection node_1
SELECT COUNT(*) AS node_1_rows, MAX(f1) AS node_1_max FROM t1;
--connection node_2
SELECT COUNT(*) AS node_2_rows, MAX(f1) AS node_2_max FROM t1;
--connection node_3
SELECT COUNT(*) AS node_3_rows, MAX(f1) AS node_3_max FROM t1;
--connection node_1
DROP TABLE t1;
--source ../galera/include/auto_increment_offset_restore.inc
--connection node_1
CALL mtr.add_suppression("WSREP: .*Reversing history");
CALL mtr.add_suppression("WSREP: gcs/src/gcs_group\\.cpp:group_post_state_exchange\\(\\)");
CALL mtr.add_suppression("WSREP: Node is ahead of the primary component");
CALL mtr.add_suppression("WSREP: node uuid: .* is inconsistent to restored view");
CALL mtr.add_suppression("WSREP: Wsrep startup was interrupted");
CALL mtr.add_suppression("WSREP: .*Calling destroy\\(\\) before close");
CALL mtr.add_suppression("WSREP: .*GCS core destructor failed");
CALL mtr.add_suppression("WSREP: server: .* unallowed state transition");
CALL mtr.add_suppression("Aborting");
CALL mtr.add_suppression("Assertion .0. failed");
CALL mtr.add_suppression("got signal 6");
CALL mtr.add_suppression("Attempting backtrace");
--connection node_2
CALL mtr.add_suppression("WSREP: .*Reversing history");
CALL mtr.add_suppression("WSREP: gcs/src/gcs_group\\.cpp:group_post_state_exchange\\(\\)");
CALL mtr.add_suppression("WSREP: Node is ahead of the primary component");
CALL mtr.add_suppression("WSREP: node uuid: .* is inconsistent to restored view");
CALL mtr.add_suppression("WSREP: Wsrep startup was interrupted");
CALL mtr.add_suppression("WSREP: .*Calling destroy\\(\\) before close");
CALL mtr.add_suppression("WSREP: .*GCS core destructor failed");
CALL mtr.add_suppression("WSREP: server: .* unallowed state transition");
CALL mtr.add_suppression("Aborting");
CALL mtr.add_suppression("Assertion .0. failed");
CALL mtr.add_suppression("got signal 6");
CALL mtr.add_suppression("Attempting backtrace");
--connection node_3
CALL mtr.add_suppression("WSREP: .*Reversing history");
CALL mtr.add_suppression("WSREP: gcs/src/gcs_group\\.cpp:group_post_state_exchange\\(\\)");
CALL mtr.add_suppression("WSREP: Node is ahead of the primary component");
CALL mtr.add_suppression("WSREP: node uuid: .* is inconsistent to restored view");
CALL mtr.add_suppression("WSREP: Wsrep startup was interrupted");
CALL mtr.add_suppression("WSREP: .*Calling destroy\\(\\) before close");
CALL mtr.add_suppression("WSREP: .*GCS core destructor failed");
CALL mtr.add_suppression("WSREP: server: .* unallowed state transition");
CALL mtr.add_suppression("Aborting");
CALL mtr.add_suppression("Assertion .0. failed");
CALL mtr.add_suppression("got signal 6");
CALL mtr.add_suppression("Attempting backtrace");