1
0
mirror of https://github.com/MariaDB/server.git synced 2026-10-23 18:49:29 +03:00

MDEV-38218 : Galera test failure on galera_bf_abort_flush_for_export

Problem was in wsrep_handle_mdl_conflict function was comparing
thd->lex->sql_command variable for granted MDL-lock.

There is two possible schedules:

    (1) FLUSH TABLES ... FOR EXPORT that will take MDL-lock (granted_thd).
        INSERT from other node is conflicting operation (request_thd)
        and sees MDL-conflict. Because granted_thd has not executed anything
        else thd->lex->sql_command == SQLCOM_FLUSH and this case was
        correctly handled in wsrep_handle_mdl_conflict i.e. INSERT needs
        to wait.

    (2) FLUSH TABLES ... FOR EXPORT that will take MDL-lock (granted_thd).
        SET SESSION wsrep_sync_wait=0; (granted_thd)
        INSERT from other node is conflicting operation (request_thd)

        However, thd->lex->sql_command is not stored to taken MDL-lock. Now
        as granted_thd is executing SET thd->lex->sql_command != SQLCOM_FLUSH
        and INSERT that is BF will abort it and that means also FTFE is
        killed and MDL-lock relesed. This is incorrect as FTFE has written
        file on filesystem and it can't be really killed.

In this fix wsrep_handle_mdl_conflict is refactored not to use
thd->lex->sql_command as a variable used for decisions. Instead
connection state can be determined also via THD members. E.g.:

    * wsrep_thd_is_toi() || wsrep_thd_is_applying - ongoing TOI or applier
    * wsrep_thd_is_BF - thread is brute force
    * wsrep_thd_is_SR - thread is streaming replication thread
    * thd->current_backup_stage != BACKUP_FINISHED - there's ongoing BACKUP
    * thd->global_read_lock.is_acquired() - ongoing FTWRL
    * thd->locked_tables_mode == LTM_LOCK_TABLES - ongoing FTFE or LOCK TABLES
This commit is contained in:
Jan Lindström
2025-12-18 09:58:38 +02:00
committed by Jan Lindström
parent f92337ac17
commit e40277d29b
17 changed files with 151 additions and 91 deletions

View File

@@ -9,7 +9,7 @@ connection node_1;
SET DEBUG_SYNC = 'now WAIT_FOR may_run';
TRUNCATE TABLE t1;
connection con1;
ERROR 70100: Query execution was interrupted
Got one of the listed errors
SET DEBUG_SYNC = 'RESET';
--- CTAS with non-empty result set ---
INSERT INTO t1 VALUES (10), (20), (30);
@@ -19,7 +19,7 @@ connection node_1;
SET DEBUG_SYNC = 'now WAIT_FOR may_run';
TRUNCATE TABLE t1;
connection con1;
ERROR 70100: Query execution was interrupted
Got one of the listed errors
SET DEBUG_SYNC = 'RESET';
DROP TABLE t1;
disconnect con1;

View File

@@ -3,16 +3,15 @@ connection node_1;
connect con1,127.0.0.1,root,,test,$NODE_MYPORT_1;
call mtr.add_suppression("WSREP: ALTER TABLE isolation failure");
CREATE TABLE t1(c1 INT PRIMARY KEY, c2 INT) ENGINE=InnoDB;
SET DEBUG_SYNC = 'wsrep_append_fk_toi_keys_before_close_tables SIGNAL may_alter WAIT_FOR bf_abort';
SET DEBUG_SYNC = 'wsrep_append_fk_toi_keys_before_close_tables SIGNAL may_alter';
ALTER TABLE t1 DROP COLUMN c2;
connection node_1;
SET DEBUG_SYNC = 'now WAIT_FOR may_alter';
ALTER TABLE t1 ADD COLUMN c3 INT;
connection con1;
ERROR 70100: Query execution was interrupted
INSERT INTO t1 (c1, c2, c3) VALUES (1, 0, 0);
INSERT INTO t1 (c1, c3) VALUES (1, 0);
connection node_2;
INSERT INTO t1 (c1, c2, c3) VALUES (2, 0, 0);
INSERT INTO t1 (c1, c3) VALUES (2, 0);
connection node_1;
SET DEBUG_SYNC = 'RESET';
DROP TABLE t1;

View File

@@ -24,8 +24,11 @@ connection node_3;
DROP TABLE source;
DROP TABLE target;
connection node_2;
call mtr.add_suppression("WSREP: Inconsistency detected: Inconsistent by consensus on");
call mtr.add_suppression("WSREP: Failed to apply write set: gtid:");
connection node_1;
STOP SLAVE;
RESET SLAVE ALL;
call mtr.add_suppression("WSREP: unexpected async replication event: 0");
connection node_3;
RESET MASTER;

View File

@@ -12,6 +12,9 @@ connection node_2a;
connection node_2;
INSERT INTO t1 VALUES (2, 'node_2');
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
COMMIT;
wsrep_local_aborts_increment
1
DROP TABLE t1;
disconnect node_2;
disconnect node_1;

View File

@@ -88,7 +88,7 @@ COUNT(*) = 5
1
CREATE TABLE t1 AS SELECT * FROM t2;
connection node_1;
ERROR 70100: Query execution was interrupted
Got one of the listed errors
SET GLOBAL DEBUG_DBUG = '';
SET DEBUG_SYNC = 'RESET';
DROP TABLE t1, t2;

View File

@@ -68,6 +68,7 @@ push @::global_suppressions,
qr|WSREP: Trying to continue unpaused monitor|,
qr|WSREP: Wait for gtid returned error 3 while waiting for prior transactions to commit before setting position|,
qr|WSREP: Failed to report last committed|,
qr|bytes freed by|,
);
sub which($) { return `sh -c "command -v $_[0]"` }

View File

@@ -31,7 +31,7 @@ TRUNCATE TABLE t1;
--connection con1
# CTAS gets BF aborted.
--error ER_QUERY_INTERRUPTED
--error ER_QUERY_INTERRUPTED, ER_LOCK_DEADLOCK
--reap
# Cleanup
@@ -56,7 +56,7 @@ TRUNCATE TABLE t1;
--connection con1
# CTAS gets BF aborted.
--error ER_QUERY_INTERRUPTED
--error ER_QUERY_INTERRUPTED, ER_LOCK_DEADLOCK
--reap
# Cleanup

View File

@@ -14,7 +14,9 @@
# - after entering TOI mode, it would later find it'd been killed, and complete with an error
# - at the same time, the command would successfully apply on every other node except the initiator.
#
# Fixed by checking killed state on THD before entering TOI.
#
# NOTE : After MDL-lock conflict resolution rework this test behaviour has
# changed.
#
--source include/galera_cluster.inc
@@ -28,26 +30,24 @@ call mtr.add_suppression("WSREP: ALTER TABLE isolation failure");
CREATE TABLE t1(c1 INT PRIMARY KEY, c2 INT) ENGINE=InnoDB;
# Run ALTER DROP COLUMN and hang before closing tables on adding FK keys and before entering TOI.
# Wait until it gets BF-aborted.
SET DEBUG_SYNC = 'wsrep_append_fk_toi_keys_before_close_tables SIGNAL may_alter WAIT_FOR bf_abort';
SET DEBUG_SYNC = 'wsrep_append_fk_toi_keys_before_close_tables SIGNAL may_alter';
--send
ALTER TABLE t1 DROP COLUMN c2;
--connection node_1
# Run ALTER ADD COLUMN and BF-abort the previous ALTER DROP COLUMN.
# Run ALTER ADD COLUMN
SET DEBUG_SYNC = 'now WAIT_FOR may_alter';
ALTER TABLE t1 ADD COLUMN c3 INT;
--connection con1
# ALTER DROP COLUMN gets BF aborted.
--error ER_QUERY_INTERRUPTED
# ALTER DROP COLUMN waits
--reap
INSERT INTO t1 (c1, c2, c3) VALUES (1, 0, 0);
INSERT INTO t1 (c1, c3) VALUES (1, 0);
--connection node_2
# ALTER DROP COLUMN must not be replicated.
INSERT INTO t1 (c1, c2, c3) VALUES (2, 0, 0);
# ALTER DROP COLUMN must be replicated.
INSERT INTO t1 (c1, c3) VALUES (2, 0);
# Cleanup.
--connection node_1

View File

@@ -66,10 +66,13 @@ DROP TABLE target;
--connection node_2
--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'target';
--source include/wait_condition.inc
call mtr.add_suppression("WSREP: Inconsistency detected: Inconsistent by consensus on");
call mtr.add_suppression("WSREP: Failed to apply write set: gtid:");
--connection node_1
STOP SLAVE;
RESET SLAVE ALL;
call mtr.add_suppression("WSREP: unexpected async replication event: 0");
--connection node_3
RESET MASTER;

View File

@@ -24,6 +24,7 @@ INSERT INTO t1 VALUES (1,'node_1');
--connection node_2
--error ER_LOCK_DEADLOCK
INSERT INTO t1 VALUES (2, 'node_2');
COMMIT;
--let $wsrep_local_bf_aborts_after = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_bf_aborts'`
@@ -32,3 +33,4 @@ INSERT INTO t1 VALUES (2, 'node_2');
--enable_query_log
DROP TABLE t1;
--source include/galera_end.inc

View File

@@ -121,7 +121,7 @@ SELECT COUNT(*) = 5 FROM t2;
CREATE TABLE t1 AS SELECT * FROM t2;
--connection node_1
--error ER_QUERY_INTERRUPTED
--error ER_QUERY_INTERRUPTED,ER_LOCK_DEADLOCK
--reap
SET GLOBAL DEBUG_DBUG = '';

View File

@@ -226,6 +226,11 @@ extern "C" my_bool wsrep_thd_bf_abort(THD *bf_thd, THD *victim_thd,
*/
if ((ret || !wsrep_on(victim_thd)) && signal)
{
WSREP_DEBUG("wsrep_thd_bf_abort thread %ld sending "
"KILL_QUERY_HARD to %ld ret=%d signal %d",
bf_thd->thread_id,
victim_thd->thread_id,
ret, signal);
victim_thd->wsrep_aborter= bf_thd->thread_id;
victim_thd->awake_no_mutex(KILL_QUERY_HARD);
} else {

View File

@@ -2182,6 +2182,10 @@ int THD::killed_errno()
DBUG_RETURN(ER_CONNECTION_KILLED);
case KILL_QUERY:
case KILL_QUERY_HARD:
#ifdef WITH_WSREP
if (WSREP(this))
wsrep_report_query_interrupted(this, __FILE__, __LINE__);
#endif /* WITH_WSREP */
DBUG_RETURN(ER_QUERY_INTERRUPTED);
case KILL_TIMEOUT:
case KILL_TIMEOUT_HARD:
@@ -8352,6 +8356,10 @@ wait_for_commit::wait_for_prior_commit2(THD *thd, bool allow_kill)
wakeup_error= ER_QUERY_INTERRUPTED;
my_message(wakeup_error, ER_THD(thd, wakeup_error), MYF(0));
thd->EXIT_COND(&old_stage);
#ifdef WITH_WSREP
if (WSREP(thd))
wsrep_report_query_interrupted(thd, __FILE__, __LINE__);
#endif /* WITH_WSREP */
/*
Must do the DEBUG_SYNC() _after_ exit_cond(), as DEBUG_SYNC is not safe to
use within enter_cond/exit_cond.

View File

@@ -3227,18 +3227,21 @@ void wsrep_to_isolation_end(THD *thd)
if (wsrep_emulate_bin_log) wsrep_thd_binlog_trx_reset(thd);
}
#define WSREP_MDL_LOG(severity, msg, schema, schema_len, req, gra) \
WSREP_##severity( \
"%s\n" \
"schema: %.*s\n" \
"request: (%llu \tseqno %lld \twsrep (%s, %s, %s) cmd %d %d \t%s)\n" \
"granted: (%llu \tseqno %lld \twsrep (%s, %s, %s) cmd %d %d \t%s)", \
msg, schema_len, schema, \
req->thread_id, (long long)wsrep_thd_trx_seqno(req), \
wsrep_thd_client_mode_str(req), wsrep_thd_client_state_str(req), wsrep_thd_transaction_state_str(req), \
req->get_command(), req->lex->sql_command, req->query(), \
gra->thread_id, (long long)wsrep_thd_trx_seqno(gra), \
wsrep_thd_client_mode_str(gra), wsrep_thd_client_state_str(gra), wsrep_thd_transaction_state_str(gra), \
#define WSREP_MDL_LOG(severity, msg, schema, schema_len, name, name_len, req, gra) \
WSREP_##severity( \
"%s\n" \
"schema.name: %.*s.%.*s\n" \
"request: (%llu \tseqno %lld \twsrep (%s, %s, %s, BF%d) cmd %d %d \t%s)\n" \
"granted: (%llu \tseqno %lld \twsrep (%s, %s, %s, BF%d) cmd %d %d \t%s)", \
msg, schema_len, schema, \
name_len, name, \
req->thread_id, wsrep_thd_trx_seqno(req), \
wsrep_thd_client_mode_str(req), wsrep_thd_client_state_str(req), \
wsrep_thd_transaction_state_str(req), wsrep_thd_is_BF(req, false), \
req->get_command(), req->lex->sql_command, req->query(), \
gra->thread_id, wsrep_thd_trx_seqno(gra), \
wsrep_thd_client_mode_str(gra), wsrep_thd_client_state_str(gra), \
wsrep_thd_transaction_state_str(gra), wsrep_thd_is_BF(gra, false), \
gra->get_command(), gra->lex->sql_command, gra->query());
/**
@@ -3260,6 +3263,8 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx,
mysql_mutex_lock(&request_thd->LOCK_thd_data);
// If requestor is not BF it has to wait for
// release of MDL-lock.
if (wsrep_thd_is_toi(request_thd) ||
wsrep_thd_is_applying(request_thd))
{
@@ -3267,13 +3272,13 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx,
const char* schema= key->db_name();
int schema_len= key->db_name_length();
const char* name= key->name();
int name_len= key->name_length();
WSREP_DEBUG("wsrep_handle_mdl_conflict request TOI/APPLY for %s",
wsrep_thd_query(request_thd));
THD_STAGE_INFO(request_thd, stage_waiting_isolation);
mysql_mutex_unlock(&request_thd->LOCK_thd_data);
WSREP_MDL_LOG(DEBUG, "MDL conflict ", schema, schema_len,
request_thd, granted_thd);
name, name_len, request_thd, granted_thd);
ticket->wsrep_report(wsrep_debug);
DEBUG_SYNC(request_thd, "before_wsrep_thd_abort");
@@ -3293,109 +3298,103 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx,
if (granted_thd->wsrep_aborter != 0)
{
// Granted thread has being already selected as a victim for
// BF kill, we can wait until it releases MDL-lock
DBUG_ASSERT(granted_thd->wsrep_aborter == request_thd->thread_id);
WSREP_DEBUG("BF thread waiting for a victim to release locks");
}
else if (wsrep_thd_is_toi(granted_thd) ||
wsrep_thd_is_applying(granted_thd))
{
// Here both request thread and granted thread are either TOI
// or applying.
if (wsrep_thd_is_aborting(granted_thd))
{
WSREP_DEBUG("BF thread waiting for SR in aborting state for %s",
wsrep_thd_query(request_thd));
THD_STAGE_INFO(request_thd, stage_waiting_isolation);
// Granted thread is aborting, we wait it to release MDL-locs
WSREP_DEBUG("BF thread for %s waiting for thread in aborting state for %s",
wsrep_thd_query(request_thd),
wsrep_thd_query(granted_thd));
ticket->wsrep_report(wsrep_debug);
}
else if (wsrep_thd_is_SR(granted_thd) && !wsrep_thd_is_SR(request_thd))
{
// Granted thread is executing streaming replication and request is DDL,
// abort granted
WSREP_MDL_LOG(INFO, "MDL conflict, DDL vs SR",
schema, schema_len, request_thd, granted_thd);
WSREP_DEBUG("wsrep_handle_mdl_conflict DDL vs SR for %s",
wsrep_thd_query(request_thd));
THD_STAGE_INFO(request_thd, stage_waiting_isolation);
schema, schema_len, name, name_len,
request_thd, granted_thd);
wsrep_abort_thd(request_thd, granted_thd, 1);
}
else
{
DBUG_ASSERT(0);
// In this case request thread waits
WSREP_MDL_LOG(INFO, "MDL BF-BF conflict", schema, schema_len,
request_thd, granted_thd);
name, name_len, request_thd, granted_thd);
ticket->wsrep_report(true);
mysql_mutex_unlock(&granted_thd->LOCK_thd_data);
mysql_mutex_unlock(&granted_thd->LOCK_thd_kill);
unireg_abort(1);
}
}
else if (granted_thd->lex->sql_command == SQLCOM_FLUSH ||
/* System transactions with explicit locks are BACKUP. */
(granted_thd->system_thread != NON_SYSTEM_THREAD &&
granted_thd->mdl_context.has_explicit_locks()))
/* If granted thread has
ongoing BACKUP OR
ongoing FLUSH TABLES WITH READ LOCK OR
ongoing FLUSH TABLES FOR EXPORT OR LOCK TABLES
*/
else if (granted_thd->current_backup_stage != BACKUP_FINISHED ||
granted_thd->global_read_lock.is_acquired() ||
granted_thd->locked_tables_mode == LTM_LOCK_TABLES)
{
WSREP_DEBUG("BF thread waiting for %s",
granted_thd->lex->sql_command == SQLCOM_FLUSH ? "FLUSH" : "BACKUP");
(granted_thd->current_backup_stage != BACKUP_FINISHED ? "BACKUP" :
(granted_thd->global_read_lock.is_acquired() ? "FTWRL" : "LOCK TABLES")));
ticket->wsrep_report(wsrep_debug);
if (granted_thd->current_backup_stage != BACKUP_FINISHED &&
wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP))
wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP))
{
wsrep_abort_thd(request_thd, granted_thd, 1);
// User has allowed killing mariabackup
wsrep_abort_thd(request_thd, granted_thd, 1);
}
}
else if (granted_thd->lex->sql_command == SQLCOM_LOCK_TABLES)
{
WSREP_DEBUG("BF thread waiting for LOCK TABLES");
ticket->wsrep_report(wsrep_debug);
}
else if (request_thd->lex->sql_command == SQLCOM_DROP_TABLE)
{
WSREP_DEBUG("DROP caused BF abort, conf %s for %s",
wsrep_thd_transaction_state_str(granted_thd),
wsrep_thd_query(request_thd));
THD_STAGE_INFO(request_thd, stage_waiting_isolation);
ticket->wsrep_report(wsrep_debug);
wsrep_abort_thd(request_thd, granted_thd, 1);
}
else
{
WSREP_MDL_LOG(DEBUG, "MDL conflict-> BF abort", schema, schema_len,
request_thd, granted_thd);
name, name_len, request_thd, granted_thd);
WSREP_DEBUG("wsrep_handle_mdl_conflict -> BF abort for %s",
wsrep_thd_query(request_thd));
THD_STAGE_INFO(request_thd, stage_waiting_isolation);
wsrep_thd_query(granted_thd));
ticket->wsrep_report(wsrep_debug);
if (granted_thd->wsrep_trx().active())
{
// Granted thread has active wsrep transaction
wsrep_abort_thd(request_thd, granted_thd, 1);
}
else if (!WSREP(granted_thd) ||
wsrep_thd_is_BF(granted_thd, false))
{
// Granted thread is not wsrep transaction or it is not BF kill it
WSREP_MDL_LOG(DEBUG, "MDL conflict-> kill", schema, schema_len,
name, name_len, request_thd, granted_thd);
granted_thd->awake_no_mutex(KILL_QUERY_HARD);
ha_abort_transaction(request_thd, granted_thd, TRUE);
}
else
{
/*
Granted_thd is likely executing with wsrep_on=0. If the requesting
thd is BF, BF abort and wait.
*/
if (wsrep_thd_is_BF(request_thd, FALSE))
{
granted_thd->awake_no_mutex(KILL_QUERY_HARD);
ha_abort_transaction(request_thd, granted_thd, TRUE);
}
else
{
WSREP_MDL_LOG(INFO, "MDL unknown BF-BF conflict",
schema, schema_len,
request_thd, granted_thd);
ticket->wsrep_report(true);
mysql_mutex_unlock(&granted_thd->LOCK_thd_data);
mysql_mutex_unlock(&granted_thd->LOCK_thd_kill);
unireg_abort(1);
}
WSREP_DEBUG("MDL conflig-> wait request BF:%ld:%s granted %s:%ld query %s",
request_thd->thread_id,
wsrep_thd_query(request_thd),
(wsrep_thd_is_BF(granted_thd, FALSE) ? "LOCAL" : "BF"),
granted_thd->thread_id,
wsrep_thd_query(granted_thd));
}
}
mysql_mutex_unlock(&granted_thd->LOCK_thd_data);
mysql_mutex_unlock(&granted_thd->LOCK_thd_kill);
DEBUG_SYNC(request_thd, "after_wsrep_thd_abort");
}
else
{
// Request thread is normal local transaction it has to wait
mysql_mutex_unlock(&request_thd->LOCK_thd_data);
}
}
@@ -4154,3 +4153,25 @@ bool wsrep_foreign_key_append(THD *thd, FOREIGN_KEY_INFO *fk)
return false;
}
void wsrep_report_query_interrupted(const THD *thd, const char *file, const int line)
{
const wsrep::client_error ce= wsrep_current_error(thd);
const enum wsrep::provider::status status= wsrep_current_error_status(thd);
WSREP_DEBUG("Query %s interrupted client_error %s provider_status %s in %s:%d",
wsrep_thd_query(thd),
wsrep::to_string(ce).c_str(),
wsrep::provider::to_string(status).c_str(),
file, line);
if (ce != wsrep::e_success)
{
push_warning_printf((THD *)thd, Sql_condition::WARN_LEVEL_WARN,
ER_QUERY_INTERRUPTED,
"Query interrupted or killed client_error: %s provider_status: %s killed: %d",
wsrep::to_string(ce).c_str(),
wsrep::provider::to_string(status).c_str(),
thd->killed);
}
}

View File

@@ -1,4 +1,4 @@
/* Copyright 2008-2023 Codership Oy <http://www.codership.com>
/* Copyright 2008-2025 Codership Oy <http://www.codership.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -610,6 +610,8 @@ bool wsrep_table_list_has_non_temp_tables(THD *thd, TABLE_LIST *tables);
*/
bool wsrep_foreign_key_append(THD *thd, FOREIGN_KEY_INFO *fk);
void wsrep_report_query_interrupted(const THD *thd, const char* file, const int line);
#else /* !WITH_WSREP */
/* These macros are needed to compile MariaDB without WSREP support

View File

@@ -1,4 +1,4 @@
/* Copyright (C) 2013-2023 Codership Oy <info@codership.com>
/* Copyright (C) 2013-2025 Codership Oy <info@codership.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by

View File

@@ -1,4 +1,4 @@
/* Copyright (C) 2013-2023 Codership Oy <info@codership.com>
/* Copyright (C) 2013-2025 Codership Oy <info@codership.com>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -198,6 +198,18 @@ void wsrep_store_threadvars(THD *);
*/
void wsrep_reset_threadvars(THD *);
static inline enum wsrep::client_error wsrep_current_error(const THD* thd)
{
return thd->wsrep_cs().current_error();
}
static inline enum wsrep::provider::status
wsrep_current_error_status(const THD* thd)
{
return thd->wsrep_cs().current_error_status();
}
/**
Helper functions to override error status
@@ -267,6 +279,7 @@ static inline void wsrep_override_error(THD* thd,
}
break;
case wsrep::e_interrupted_error:
wsrep_report_query_interrupted(thd, __FILE__, __LINE__);
wsrep_override_error(thd, ER_QUERY_INTERRUPTED);
break;
case wsrep::e_size_exceeded_error: