1
0
mirror of https://github.com/MariaDB/server.git synced 2026-10-23 18:49:29 +03:00

MDEV-38999: Galera - stale reads in explicit transactions

wsrep_sync_wait is a per-statement-type bitmask (READ, UPDATE/DELETE,
INSERT/REPLACE, SHOW) that forces a causal wait - block until this
node has applied every write already certified cluster-wide - before
executing a matching statement. wsrep_must_sync_wait() unconditionally
skipped this wait whenever the session was already inside a
multi-statement transaction, regardless of which bit was configured
for the statement at hand. Once BEGIN (or, via autocommit=0, the
first statement) had opened a transaction, every later statement in
it was exempt from its own causal wait - so writes could run against
a view no fresher than whatever BEGIN itself happened to check, which
is only the READ bit, regardless of the actual configured mask.

Fix: only skip the wait once this transaction has generated its first
write (wsrep_has_changes(thd)). Every statement up to and including
that first write is still safe to wait for - nothing of this
transaction's is locked yet when the pre-statement check runs - which
closes the gap above. Nothing after the first write re-waits, matching
the original behaviour for the rest of the transaction: once this
session may be holding row locks, a fresh causal wait risks a
self-deadlock against its own local applier (which can be blocked
needing one of those same locks to apply a write-set this wait is
waiting for, with nothing to resolve it but the wait's own timeout).

A second, independent defect surfaced once in-transaction waits could
time out at all: wsrep::client_state::after_command_after_result()
only clears a pending wsrep error once the transaction is no longer
active - correct for a genuinely doomed BF-aborted transaction, but a
causal-wait timeout is a pre-check failure that touches nothing, so
this left every later statement in the same transaction, ROLLBACK
included, failing with the same stale error. wsrep_sync_wait() now
resets the error immediately after reporting it for the one statement
that actually hit it, scoped to this failure path only - genuine
BF-abort/deadlock errors elsewhere keep their correct sticky-until-
rollback behaviour.

Tests: galera_MDEV-38999.test covers the original bitmask gap across
both explicit BEGIN and autocommit=0 transactions, and a repeated-read
scenario proving a satisfied bit isn't cached across statements.
galera_MDEV-38999_locks.test and galera_MDEV-38999_autocommit.test
exercise the self-deadlock risk directly: a transaction already
holding a row lock the paused applier needs must still return promptly
once the applier is released, not block for the full
repl.causal_read_timeout.

Assisted-by Claude AI
This commit is contained in:
Pekka Lampio
2026-10-05 14:59:08 +03:00
parent f59e460200
commit 5a887ba65d
7 changed files with 521 additions and 5 deletions

View File

@@ -0,0 +1,97 @@
connection node_2;
connection node_1;
connection node_1;
CREATE TABLE t1 (id INT PRIMARY KEY, f1 INT) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1, 100);
connection node_2;
SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT1S";
SET SESSION wsrep_sync_wait = 2;
connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
#
# Control case: standalone (autocommit) UPDATE correctly waits.
#
connection node_2;
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
connection node_1;
INSERT INTO t1 VALUES (2, 200);
connection node_2a;
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_2;
UPDATE t1 SET f1 = f1 + 1 WHERE id = 1;
ERROR HY000: Lock wait timeout exceeded; try restarting transaction
connection node_2a;
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
connection node_2;
#
# Fixed case: the SAME UPDATE, issued inside an explicit BEGIN, must
# still wait - bit 2 (UPDATE/DELETE) hasn't been satisfied yet in this
# transaction (BEGIN only ever checks bit 1).
#
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
connection node_1;
INSERT INTO t1 VALUES (3, 300);
connection node_2a;
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_2;
BEGIN;
UPDATE t1 SET f1 = f1 + 1 WHERE id = 1;
ERROR HY000: Lock wait timeout exceeded; try restarting transaction
ROLLBACK;
connection node_2a;
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
connection node_2;
#
# Same scenario again, but via an implicit (autocommit=0) transaction
# instead of an explicit BEGIN.
#
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
connection node_1;
INSERT INTO t1 VALUES (4, 400);
connection node_2a;
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_2;
SET SESSION autocommit = 0;
SELECT COUNT(*) FROM t1 WHERE id = 1;
COUNT(*)
1
UPDATE t1 SET f1 = f1 + 1 WHERE id = 1;
ERROR HY000: Lock wait timeout exceeded; try restarting transaction
ROLLBACK;
SET SESSION autocommit = 1;
connection node_2a;
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
connection node_2;
#
# wsrep_sync_wait=1 (READ only): two reads in a row inside one explicit
# transaction must each independently wait - the second one is not
# exempted just because the first already waited.
#
SET SESSION wsrep_sync_wait = 1;
connection node_2;
BEGIN;
SELECT COUNT(*) FROM t1;
COUNT(*)
4
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
connection node_1;
INSERT INTO t1 VALUES (5, 500);
connection node_2a;
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_2;
SELECT COUNT(*) FROM t1;
ERROR HY000: Lock wait timeout exceeded; try restarting transaction
ROLLBACK;
connection node_2a;
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
connection node_2;
connection node_1;
DROP TABLE t1;
disconnect node_2a;

View File

@@ -0,0 +1,31 @@
connection node_2;
connection node_1;
connection node_1;
CREATE TABLE t1 (id INT PRIMARY KEY, f1 INT) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1, 100), (2, 200);
connection node_2;
SET SESSION wsrep_sync_wait = 6;
connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
connection node_2;
SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT1S";
SET autocommit=0;
UPDATE t1 SET f1=f1+1 WHERE id=1;
COMMIT;
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
connection node_1;
INSERT INTO t1 VALUES (3, 300);
connection node_2a;
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_2;
UPDATE t1 SET f1=f1+1 WHERE id=1;
ERROR HY000: Lock wait timeout exceeded; try restarting transaction
ROLLBACK;
connection node_2a;
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
connection node_2;
connection node_2;
connection node_1;
DROP TABLE t1;
disconnect node_2a;

View File

@@ -0,0 +1,33 @@
connection node_2;
connection node_1;
connection node_1;
CREATE TABLE t1 (id INT PRIMARY KEY, f1 INT) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1, 100), (2, 200);
connection node_2;
SET SESSION wsrep_sync_wait = 6;
connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
connection node_2;
SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT20S";
BEGIN;
UPDATE t1 SET f1=f1+1 WHERE id=2;
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
connection node_1;
UPDATE t1 SET f1=0 WHERE id=2;
connection node_2a;
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_2;
INSERT INTO t1 VALUES (6, 600);
connection node_2a;
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
connection node_2;
connection node_2a;
returned_promptly
1
connection node_2;
COMMIT;
connection node_2;
connection node_1;
DROP TABLE t1;
disconnect node_2a;

View File

@@ -0,0 +1,198 @@
#
# MDEV-38999: Galera - stale reads
#
# wsrep_sync_wait is a bitmask (1=READ/BEGIN, 2=UPDATE/DELETE,
# 4=INSERT/REPLACE, 8=SHOW) that is supposed to force a causal wait before
# executing a statement of the matching type. wsrep_must_sync_wait() used to
# unconditionally skip the wait whenever thd->in_active_multi_stmt_transaction()
# was true, regardless of the configured bitmask - so once a transaction was
# open (explicit or, via autocommit=0, implicit), every later statement in
# it was exempt from its own causal wait, even if its bit was configured.
#
# Fix: wsrep_must_sync_wait() now skips the wait only once this transaction
# has generated its first write (wsrep_has_changes(thd)), since only then
# might it already hold row locks that a wait could self-deadlock against
# (the local applier blocked needing one of those same locks, with nothing
# to resolve it but a timeout). Every statement up to and including the
# transaction's first write is still safe to wait for - nothing of its own
# is locked yet when the pre-statement check runs - which is exactly what
# was missing before.
#
# A second, independent defect surfaced once in-transaction waits could
# time out at all: wsrep::client_state::after_command_after_result() only
# clears a pending wsrep error once the transaction is no longer active -
# correct for a genuinely doomed BF-aborted transaction, but it meant a
# single recoverable causal-wait timeout poisoned every later statement in
# the same transaction, ROLLBACK included, with the same stale error.
# wsrep_sync_wait() now resets the error immediately after reporting it for
# the one statement that actually hit it.
#
# This test proves both fixes, and exercises the two transaction-start
# paths (explicit BEGIN and implicit autocommit=0) plus two bitmask values:
#
# 1. wsrep_sync_wait=2 (UPDATE/DELETE only): a standalone autocommit UPDATE
# correctly blocks on the causal wait and times out while the applier on
# node_2 is paused (control case). The same UPDATE, issued inside an
# explicit BEGIN, must now time out the same way - before the fix it ran
# immediately against a stale view instead - and the following ROLLBACK
# must succeed cleanly, not inherit the same error.
# 2. The same UPDATE-after-BEGIN scenario again, but with the transaction
# started implicitly via autocommit=0 instead of BEGIN.
# 3. wsrep_sync_wait=1 (READ only): two SELECTs in a row inside one explicit
# transaction, with the applier paused in between. Since neither read
# writes anything, both must independently wait and the second one must
# time out too - nothing should be considered "already satisfied" just
# because an earlier read in the same transaction already waited.
#
--source include/galera_cluster.inc
--source include/have_innodb.inc
--source include/have_debug.inc
--source include/have_debug_sync.inc
--source include/galera_have_debug_sync.inc
--connection node_1
CREATE TABLE t1 (id INT PRIMARY KEY, f1 INT) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1, 100);
--connection node_2
--let $wsrep_provider_options_orig = `SELECT @@wsrep_provider_options`
SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT1S";
# wsrep_sync_wait is SESSION-scoped: the galera suite's default .cnf already
# sets it (non-zero) for connections opened before this point, and SET GLOBAL
# would only affect *new* connections, not this one - so set it with SESSION
# directly on the connection that will run the statements below.
SET SESSION wsrep_sync_wait = 2;
--let $wait_condition = SELECT COUNT(*) = 1 FROM t1
--source include/wait_condition.inc
# Control connection used to synchronize with the paused applier.
--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
--echo #
--echo # Control case: standalone (autocommit) UPDATE correctly waits.
--echo #
--connection node_2
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
--connection node_1
INSERT INTO t1 VALUES (2, 200);
--connection node_2a
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
--connection node_2
--error ER_LOCK_WAIT_TIMEOUT
UPDATE t1 SET f1 = f1 + 1 WHERE id = 1;
--connection node_2a
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
--connection node_2
--let $wait_condition = SELECT COUNT(*) = 2 FROM t1
--source include/wait_condition.inc
--echo #
--echo # Fixed case: the SAME UPDATE, issued inside an explicit BEGIN, must
--echo # still wait - bit 2 (UPDATE/DELETE) hasn't been satisfied yet in this
--echo # transaction (BEGIN only ever checks bit 1).
--echo #
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
--connection node_1
INSERT INTO t1 VALUES (3, 300);
--connection node_2a
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
--connection node_2
BEGIN;
--error ER_LOCK_WAIT_TIMEOUT
UPDATE t1 SET f1 = f1 + 1 WHERE id = 1;
# The failed UPDATE never touched anything (it failed at the pre-check,
# before taking any lock), so this ROLLBACK must succeed cleanly - it must
# NOT inherit the UPDATE's now-stale error.
ROLLBACK;
--connection node_2a
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
--connection node_2
--let $wait_condition = SELECT COUNT(*) = 3 FROM t1
--source include/wait_condition.inc
--echo #
--echo # Same scenario again, but via an implicit (autocommit=0) transaction
--echo # instead of an explicit BEGIN.
--echo #
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
--connection node_1
INSERT INTO t1 VALUES (4, 400);
--connection node_2a
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
--connection node_2
SET SESSION autocommit = 0;
SELECT COUNT(*) FROM t1 WHERE id = 1;
--error ER_LOCK_WAIT_TIMEOUT
UPDATE t1 SET f1 = f1 + 1 WHERE id = 1;
ROLLBACK;
SET SESSION autocommit = 1;
--connection node_2a
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
--connection node_2
--let $wait_condition = SELECT COUNT(*) = 4 FROM t1
--source include/wait_condition.inc
--echo #
--echo # wsrep_sync_wait=1 (READ only): two reads in a row inside one explicit
--echo # transaction must each independently wait - the second one is not
--echo # exempted just because the first already waited.
--echo #
SET SESSION wsrep_sync_wait = 1;
--connection node_2
BEGIN;
SELECT COUNT(*) FROM t1;
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
--connection node_1
INSERT INTO t1 VALUES (5, 500);
--connection node_2a
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
--connection node_2
--error ER_LOCK_WAIT_TIMEOUT
SELECT COUNT(*) FROM t1;
ROLLBACK;
--connection node_2a
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
--connection node_2
--let $wait_condition = SELECT COUNT(*) = 5 FROM t1
--source include/wait_condition.inc
--disable_query_log
--eval SET GLOBAL wsrep_provider_options = "$wsrep_provider_options_orig"
SET SESSION wsrep_sync_wait = 0;
--enable_query_log
--connection node_1
DROP TABLE t1;
--disconnect node_2a

View File

@@ -0,0 +1,61 @@
#
# MDEV-38999: the set of satisfied wsrep_sync_wait bits must not survive
# COMMIT. With autocommit=0, the first UPDATE after COMMIT starts a new
# transaction and must wait for the applier again.
#
--source include/galera_cluster.inc
--source include/have_innodb.inc
--source include/have_debug.inc
--source include/have_debug_sync.inc
--source include/galera_have_debug_sync.inc
--connection node_1
CREATE TABLE t1 (id INT PRIMARY KEY, f1 INT) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1, 100), (2, 200);
--connection node_2
--let $wsrep_provider_options_orig = `SELECT @@wsrep_provider_options`
SET SESSION wsrep_sync_wait = 6;
--let $wait_condition = SELECT COUNT(*) = 2 FROM t1
--source include/wait_condition.inc
--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
--connection node_2
SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT1S";
SET autocommit=0;
UPDATE t1 SET f1=f1+1 WHERE id=1;
COMMIT;
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
--connection node_1
INSERT INTO t1 VALUES (3, 300);
--connection node_2a
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
--connection node_2
--error ER_LOCK_WAIT_TIMEOUT
UPDATE t1 SET f1=f1+1 WHERE id=1;
--error 0,ER_LOCK_WAIT_TIMEOUT
ROLLBACK;
--connection node_2a
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
--connection node_2
--let $wait_condition = SELECT COUNT(*) = 3 FROM t1
--source include/wait_condition.inc
--connection node_2
--disable_query_log
--eval SET GLOBAL wsrep_provider_options = "$wsrep_provider_options_orig"
SET SESSION wsrep_sync_wait = 0;
--enable_query_log
--connection node_1
DROP TABLE t1;
--disconnect node_2a

View File

@@ -0,0 +1,75 @@
#
# MDEV-38999: a statement in a transaction that holds a row lock waits for
# the applier when its wsrep_sync_wait bit is not yet satisfied. The applier
# needs that row lock and BF-aborts the transaction, but the aborted thread
# is still inside the causal wait. The wait ends only at
# repl.causal_read_timeout, although the applier is released after one
# second. Expected: the INSERT returns soon after the applier is released.
#
--source include/galera_cluster.inc
--source include/have_innodb.inc
--source include/have_debug.inc
--source include/have_debug_sync.inc
--source include/galera_have_debug_sync.inc
--connection node_1
CREATE TABLE t1 (id INT PRIMARY KEY, f1 INT) ENGINE=InnoDB;
INSERT INTO t1 VALUES (1, 100), (2, 200);
--connection node_2
--let $wsrep_provider_options_orig = `SELECT @@wsrep_provider_options`
SET SESSION wsrep_sync_wait = 6;
--let $wait_condition = SELECT COUNT(*) = 2 FROM t1
--source include/wait_condition.inc
--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
--connection node_2
SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT20S";
BEGIN;
UPDATE t1 SET f1=f1+1 WHERE id=2;
SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
--connection node_1
UPDATE t1 SET f1=0 WHERE id=2;
--connection node_2a
SET DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
--connection node_2
--let $t0 = `SELECT UNIX_TIMESTAMP()`
--send INSERT INTO t1 VALUES (6, 600)
--connection node_2a
--sleep 1
SET GLOBAL debug_dbug = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET DEBUG_SYNC = "RESET";
--connection node_2
--error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK
--reap
# A genuine BF-abort/deadlock error is (correctly, unlike a causal-wait
# timeout) sticky until the transaction is explicitly ended, so node_2's
# connection won't take plain queries cleanly yet - measure elapsed time via
# node_2a instead, which was never part of the aborted transaction.
--connection node_2a
--let $dt = `SELECT UNIX_TIMESTAMP() - $t0`
--disable_query_log
--eval SELECT $dt < 10 AS returned_promptly
--enable_query_log
--connection node_2
--error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK
COMMIT;
--connection node_2
--disable_query_log
--eval SET GLOBAL wsrep_provider_options = "$wsrep_provider_options_orig"
SET SESSION wsrep_sync_wait = 0;
--enable_query_log
--connection node_1
DROP TABLE t1;
--disconnect node_2a

View File

@@ -1624,12 +1624,25 @@ bool wsrep_must_sync_wait (THD* thd, uint mask)
if (thd->variables.wsrep_on)
{
mysql_mutex_lock(&thd->LOCK_thd_data);
/*
Once this transaction has generated its first write (appended
certification keys / written to the binlog cache), it may already be
holding row locks. Waiting on the causal wait at that point risks a
self-deadlock: the local applier thread can be blocked trying to take
one of those same locks to apply a write-set this wait is waiting for,
and nothing resolves that except the wait eventually timing out. So,
per the transaction's lifetime, only wait before the first write -
every statement up to and including that first write is still safe
(nothing of ours is locked yet when the check runs, since it runs
before the statement executes), but nothing after it re-waits,
matching the original behaviour for the remainder of the transaction.
*/
ret= (thd->variables.wsrep_sync_wait & mask) &&
thd->wsrep_client_thread &&
WSREP_ON &&
!(thd->variables.wsrep_dirty_reads &&
!is_update_query(thd->lex->sql_command)) &&
!thd->in_active_multi_stmt_transaction() &&
!wsrep_has_changes(thd) &&
thd->wsrep_trx().state() !=
wsrep::transaction::s_replaying &&
thd->wsrep_cs().sync_wait_gtid().is_undefined();
@@ -1646,14 +1659,22 @@ bool wsrep_sync_wait (THD* thd, uint mask)
"mask= %u, thd->variables.wsrep_on= %d",
thd->variables.wsrep_sync_wait, mask,
thd->variables.wsrep_on);
/*
This allows autocommit SELECTs and a first SELECT after SET AUTOCOMMIT=0
TODO: modify to check if thd has locked any rows.
*/
if (thd->wsrep_cs().sync_wait(-1))
{
wsrep_override_error(thd, thd->wsrep_cs().current_error(),
thd->wsrep_cs().current_error_status());
/*
Unlike a BF-abort/deadlock, a causal-wait timeout here is a
pre-check failure: nothing of this transaction's has executed or
been touched by it. wsrep::client_state::after_command_after_result()
only clears current_error() once the transaction is no longer
active, which is the right policy for a genuinely doomed (BF-aborted)
transaction, but would otherwise leave every later statement in this
still-perfectly-usable transaction - including ROLLBACK - failing
with this same stale error. Clear it immediately: it has already
been reported to the caller of this one statement above.
*/
thd->wsrep_cs().reset_error();
return true;
}
}