mirror of
https://github.com/MariaDB/server.git
synced 2025-08-08 11:22:35 +03:00
MDEV-30619: Parallel Slave SQL Thread Can Update Seconds_Behind_Master with Active Workers
MDEV-31749 sporadic assert in MDEV-30619 new test If the workers of a parallel replica are busy (potentially with long queues), but the SQL thread has no events left to distribute (so it goes idle), then the next event that comes from the primary will update mi->last_master_timestamp with its timestamp, even if the workers have not yet finished. This patch changes the parallel replica logic which updates last_master_timestamp after idling from using solely sql_thread_caught_up (added in MDEV-29639) to using the latter with rli queued/dequeued event counters. That is, if the queued count is equal to the dequeued count, it means all events have been processed and the replica is considered idle when the driver thread has also distributed all events. Low level details of the commit include - to make a more generalized test for Seconds_Behind_Master on the parallel replica, rpl_delayed_parallel_slave_sbm.test is renamed to rpl_parallel_sbm.test for this purpose. - pause_sql_thread_on_next_event usage was removed with the MDEV-30619 fixes. Rather than remove it, we adapt it to the needs of this test case - added test case to cover SBM spike of relay log read and LMT update that was fixed by MDEV-29639 - rpl_seconds_behind_master_spike.test is made to use the negate_clock_diff_with_master debug eval. Reviewed By: ============ Andrei Elkin <andrei.elkin@mariadb.com>
This commit is contained in:
Brandon Nesterenko
committed by
Andrei
Andrei
parent
734583b0d7
commit
063f4ac25e
199
mysql-test/suite/rpl/t/rpl_parallel_sbm.test
Normal file
199
mysql-test/suite/rpl/t/rpl_parallel_sbm.test
Normal file
@@ -0,0 +1,199 @@
|
||||
#
|
||||
# Ensure that Seconds_Behind_Master works correctly on the parallel replica.
|
||||
#
|
||||
--source include/master-slave.inc
|
||||
--source include/have_log_bin.inc
|
||||
--source include/have_debug.inc
|
||||
|
||||
--echo #
|
||||
--echo # MDEV-29639: Seconds_Behind_Master is incorrect for Delayed, Parallel Replicas
|
||||
--echo #
|
||||
|
||||
# This test ensures that after a delayed parallel slave has idled, i.e.
|
||||
# executed everything in its relay log, the next event group that the SQL
|
||||
# thread reads from the relay log will immediately be used in the
|
||||
# Seconds_Behind_Master. In particular, it ensures that the calculation for
|
||||
# Seconds_Behind_Master is based on the timestamp of the new transaction,
|
||||
# rather than the last committed transaction.
|
||||
#
|
||||
|
||||
--connection slave
|
||||
--source include/stop_slave.inc
|
||||
--let $save_dbug= `SELECT @@GLOBAL.debug_dbug`
|
||||
--let $save_parallel_mode= `SELECT @@GLOBAL.slave_parallel_mode`
|
||||
set @@GLOBAL.debug_dbug= "d,negate_clock_diff_with_master";
|
||||
set @@GLOBAL.slave_parallel_mode= CONSERVATIVE;
|
||||
--let $master_delay= 3
|
||||
--eval change master to master_delay=$master_delay, master_use_gtid=Slave_Pos
|
||||
--source include/start_slave.inc
|
||||
|
||||
--connection master
|
||||
--let insert_ctr= 0
|
||||
create table t1 (a int);
|
||||
create table t2 (a int);
|
||||
--source include/sync_slave_sql_with_master.inc
|
||||
|
||||
--echo #
|
||||
--echo # Pt 1) Ensure SBM is updated immediately upon arrival of the next event
|
||||
|
||||
--echo # Lock t1 on slave so the first received transaction does not complete/commit
|
||||
--connection slave
|
||||
LOCK TABLES t1 WRITE;
|
||||
|
||||
--connection master
|
||||
--echo # Sleep 2 to allow a buffer between events for SBM check
|
||||
sleep 2;
|
||||
|
||||
--let $ts_trx_before_ins= `SELECT UNIX_TIMESTAMP()`
|
||||
--eval insert into t1 values ($insert_ctr)
|
||||
--inc $insert_ctr
|
||||
--source include/save_master_gtid.inc
|
||||
|
||||
--connection slave
|
||||
|
||||
--echo # Waiting for transaction to arrive on slave and begin SQL Delay..
|
||||
--let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting until MASTER_DELAY seconds after master executed event';
|
||||
--source include/wait_condition.inc
|
||||
|
||||
--echo # Validating SBM is updated on event arrival..
|
||||
--let $sbm_trx1_arrive= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1)
|
||||
--let $seconds_since_idling= `SELECT UNIX_TIMESTAMP() - $ts_trx_before_ins`
|
||||
if (`SELECT $sbm_trx1_arrive > ($seconds_since_idling + 1)`)
|
||||
{
|
||||
--echo # SBM was $sbm_trx1_arrive yet shouldn't have been larger than $seconds_since_idling + 1 (for possible negative clock_diff_with_master)
|
||||
--die Seconds_Behind_Master should reset after idling
|
||||
}
|
||||
--echo # ..done
|
||||
|
||||
--connection slave
|
||||
UNLOCK TABLES;
|
||||
--source include/sync_with_master_gtid.inc
|
||||
|
||||
--echo #
|
||||
--echo # Pt 2) If the worker threads have not entered an idle state, ensure
|
||||
--echo # following events do not update SBM
|
||||
|
||||
--connection slave
|
||||
LOCK TABLES t1 WRITE;
|
||||
|
||||
--connection master
|
||||
--echo # Sleep 2 to allow a buffer between events for SBM check
|
||||
sleep 2;
|
||||
--let $ts_trxpt2_before_ins= `SELECT UNIX_TIMESTAMP()`
|
||||
--eval insert into t1 values ($insert_ctr)
|
||||
--inc $insert_ctr
|
||||
--echo # Sleep 3 to create gap between events
|
||||
sleep 3;
|
||||
--eval insert into t1 values ($insert_ctr)
|
||||
--inc $insert_ctr
|
||||
--let $ts_trx_after_ins= `SELECT UNIX_TIMESTAMP()`
|
||||
--source include/save_master_pos.inc
|
||||
|
||||
--connection slave
|
||||
--echo # Wait for first transaction to complete SQL delay and begin execution..
|
||||
--let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for table metadata lock%' AND command LIKE 'Slave_Worker';
|
||||
--source include/wait_condition.inc
|
||||
|
||||
--echo # Validate SBM calculation doesn't use the second transaction because worker threads shouldn't have gone idle..
|
||||
--let $sbm_after_trx_no_idle= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1)
|
||||
--let $timestamp_trxpt2_arrive= `SELECT UNIX_TIMESTAMP()`
|
||||
if (`SELECT $sbm_after_trx_no_idle < $timestamp_trxpt2_arrive - $ts_trx_after_ins - 1`)
|
||||
{
|
||||
--let $cmpv= `SELECT $timestamp_trxpt2_arrive - $ts_trx_after_ins`
|
||||
--echo # SBM $sbm_after_trx_no_idle was more recent than time since last transaction ($cmpv seconds)
|
||||
--die Seconds_Behind_Master should not have used second transaction timestamp
|
||||
}
|
||||
--let $seconds_since_idling= `SELECT ($timestamp_trxpt2_arrive - $ts_trxpt2_before_ins)`
|
||||
--echo # ..and that SBM wasn't calculated using prior committed transactions
|
||||
if (`SELECT $sbm_after_trx_no_idle > ($seconds_since_idling + 1)`)
|
||||
{
|
||||
--echo # SBM was $sbm_after_trx_no_idle yet shouldn't have been larger than $seconds_since_idling + 1 (for possible negative clock_diff_with_master)
|
||||
--die Seconds_Behind_Master calculation should not have used prior committed transaction
|
||||
}
|
||||
--echo # ..done
|
||||
|
||||
--connection slave
|
||||
UNLOCK TABLES;
|
||||
--source include/sync_with_master.inc
|
||||
|
||||
--echo # Cleanup
|
||||
--source include/stop_slave.inc
|
||||
--eval CHANGE MASTER TO master_delay=0
|
||||
--source include/start_slave.inc
|
||||
|
||||
|
||||
--echo #
|
||||
--echo # MDEV-30619: Parallel Slave SQL Thread Can Update Seconds_Behind_Master with Active Workers
|
||||
--echo #
|
||||
|
||||
# This test ensures that a parallel slave will not update
|
||||
# Seconds_Behind_Master after the SQL Thread has idled if the worker threads
|
||||
# are still executing events. To test this, two events are executed on the
|
||||
# primary with $sleep seconds in-between them. Once the second event begins
|
||||
# execution on the replica, Seconds_Behind_Master is queried to ensure it
|
||||
# reflects the value of the first transaction, rather than the second.
|
||||
|
||||
--connection slave
|
||||
--echo # Ensure the replica is fully idle before starting transactions
|
||||
--let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Slave has read all relay log%';
|
||||
--source include/wait_condition.inc
|
||||
--let $wait_condition= SELECT count(*)=2 FROM information_schema.processlist WHERE state LIKE 'Waiting for work from SQL thread';
|
||||
--source include/wait_condition.inc
|
||||
|
||||
--echo # Lock t1 on slave so the first received transaction does not complete/commit
|
||||
LOCK TABLES t1 WRITE;
|
||||
|
||||
--connection master
|
||||
--let $ts_t1_before_master_ins= `SELECT UNIX_TIMESTAMP()`
|
||||
--eval insert into t1 values ($insert_ctr)
|
||||
--inc $insert_ctr
|
||||
--source include/save_master_gtid.inc
|
||||
|
||||
--connection slave
|
||||
--echo # Waiting for first transaction to begin..
|
||||
--let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for table metadata lock';
|
||||
--source include/wait_condition.inc
|
||||
|
||||
--let $sbm_1= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1)
|
||||
|
||||
--connection master
|
||||
--let $sleep = 2
|
||||
--echo # Sleep $sleep sec to create a gap between events
|
||||
sleep $sleep;
|
||||
INSERT INTO t2 VALUES (1);
|
||||
--source include/save_master_gtid.inc
|
||||
|
||||
--connection slave
|
||||
--echo # Waiting for second transaction to begin..
|
||||
--let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for prior transaction to start commit%';
|
||||
--source include/wait_condition.inc
|
||||
|
||||
--let $sbm_2= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1)
|
||||
|
||||
if (`SELECT $sbm_1 + $sleep > $sbm_2`)
|
||||
{
|
||||
--echo # Seconds_Behind_Masters: $sbm_1 $sbm_2_0
|
||||
--die Two successive Seconds_Behind_Master timestamps must be separated by the sleep parameter value or greater
|
||||
}
|
||||
|
||||
--connection slave
|
||||
UNLOCK TABLES;
|
||||
--source include/sync_with_master_gtid.inc
|
||||
|
||||
|
||||
--echo #
|
||||
--echo # Cleanup
|
||||
|
||||
--connection master
|
||||
DROP TABLE t1, t2;
|
||||
--source include/save_master_gtid.inc
|
||||
|
||||
--connection slave
|
||||
--source include/sync_with_master_gtid.inc
|
||||
--source include/stop_slave.inc
|
||||
--eval set @@GLOBAL.debug_dbug= "$save_dbug"
|
||||
--evalp set @@GLOBAL.slave_parallel_mode= "$save_parallel_mode"
|
||||
--source include/start_slave.inc
|
||||
|
||||
--source include/rpl_end.inc
|
||||
--echo # End of rpl_parallel_sbm.test
|
||||
Reference in New Issue
Block a user