database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-07-30 16:24:05 +03:00
Code Activity

Bug#17283409 4-WAY DEADLOCK: ZOMBIES, PURGING BINLOGS,

Browse Source 
SHOW PROCESSLIST, SHOW BINLOGS

Problem:  A deadlock was occurring when 4 threads were
involved in acquiring locks in the following way
Thread 1: Dump thread ( Slave is reconnecting, so on
              Master, a new dump thread is trying kill
              zombie dump threads. It acquired thread's
              LOCK_thd_data and it is about to acquire
              mysys_var->current_mutex ( which LOCK_log)
Thread 2: Application thread is executing show binlogs and
               acquired LOCK_log and it is about to acquire
               LOCK_index.
Thread 3: Application thread is executing Purge binary logs
               and acquired LOCK_index and it is about to
               acquire LOCK_thread_count.
Thread 4: Application thread is executing show processlist
               and acquired LOCK_thread_count and it is
               about to acquire zombie dump thread's
               LOCK_thd_data.
Deadlock Cycle:
     Thread 1 -> Thread 2 -> Thread 3-> Thread 4 ->Thread 1

The same above deadlock was observed even when thread 4 is
executing 'SELECT * FROM information_schema.processlist' command and
acquired LOCK_thread_count and it is about to acquire zombie
dump thread's LOCK_thd_data.

Analysis:
There are four locks involved in the deadlock.  LOCK_log,
LOCK_thread_count, LOCK_index and LOCK_thd_data.
LOCK_log, LOCK_thread_count, LOCK_index are global mutexes
where as LOCK_thd_data is local to a thread.
We can divide these four locks in two groups.
Group 1 consists of LOCK_log and LOCK_index and the order
should be LOCK_log followed by LOCK_index.
Group 2 consists of other two mutexes
LOCK_thread_count, LOCK_thd_data and the order should
be LOCK_thread_count followed by LOCK_thd_data.
Unfortunately, there is no specific predefined lock order defined
to follow in the MySQL system when it comes to locks across these
two groups. In the above problematic example,
there is no problem in the way we are acquiring the locks
if you see each thread individually.
But If you combine all 4 threads, they end up in a deadlock.

Fix: 
Since everything seems to be fine in the way threads are taking locks,
In this patch We are changing the duration of the locks in Thread 4
to break the deadlock. i.e., before the patch, Thread 4
('show processlist' command) mysqld_list_processes()
function acquires LOCK_thread_count for the complete duration
of the function and it also acquires/releases
each thread's LOCK_thd_data.

LOCK_thread_count is used to protect addition and
deletion of threads in global threads list. While show
process list is looping through all the existing threads,
it will be a problem if a thread is exited but there is no problem
if a new thread is added to the system. Hence a new mutex is
introduced "LOCK_thd_remove" which will protect deletion
of a thread from global threads list. All threads which are
getting exited should acquire LOCK_thd_remove
followed by LOCK_thread_count. (It should take LOCK_thread_count
also because other places of the code still thinks that exit thread
is protected with LOCK_thread_count. In this fix, we are changing
only 'show process list' query logic )
(Eg: unlink_thd logic will be protected with
LOCK_thd_remove).

Logic of mysqld_list_processes(or file_schema_processlist)
will now be protected with 'LOCK_thd_remove' instead of
'LOCK_thread_count'.

Now the new locking order after this patch is:
LOCK_thd_remove -> LOCK_thd_data -> LOCK_log ->
LOCK_index -> LOCK_thread_count
This commit is contained in:
Venkatesh Duggirala
2014-05-08 18:13:01 +05:30
parent 263d47d3a1
commit 33f15dc7ac
15 changed files with 377 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
sql/mysqld.cc
View File

@ -642,7 +642,7 @@ SHOW_COMP_OPTION have_profiling;
pthread_key(MEM_ROOT**,THR_MALLOC);
pthread_key(THD*, THR_THD);
mysql_mutex_t LOCK_thread_created;
mysql_mutex_t LOCK_thread_count;
mysql_mutex_t LOCK_thread_count, LOCK_thd_remove;
mysql_mutex_t
LOCK_status, LOCK_error_log, LOCK_uuid_generator,
LOCK_delayed_insert, LOCK_delayed_status, LOCK_delayed_create,
@ -1607,6 +1607,7 @@ static void clean_up_mutexes()
mysql_mutex_destroy(&LOCK_prepared_stmt_count);
mysql_mutex_destroy(&LOCK_error_messages);
mysql_cond_destroy(&COND_thread_count);
mysql_mutex_destroy(&LOCK_thd_remove);
mysql_cond_destroy(&COND_thread_cache);
mysql_cond_destroy(&COND_flush_thread_cache);
mysql_cond_destroy(&COND_manager);
@ -2160,6 +2161,8 @@ void dec_connection_count()
void delete_thd(THD *thd)
{
mysql_mutex_assert_owner(&LOCK_thread_count);
mysql_mutex_assert_owner(&LOCK_thd_remove);
thread_count--;
delete thd;
}
@ -2173,7 +2176,7 @@ void delete_thd(THD *thd)
thd Thread handler
NOTES
LOCK_thread_count is locked and left locked
LOCK_thread_count, LOCK_thd_remove are locked and left locked
*/
void unlink_thd(THD *thd)
@ -2183,6 +2186,7 @@ void unlink_thd(THD *thd)
thd_cleanup(thd);
dec_connection_count();
mysql_mutex_lock(&LOCK_thd_remove);
mysql_mutex_lock(&LOCK_thread_count);
/*
Used by binlog_reset_master. It would be cleaner to use
@ -2295,6 +2299,7 @@ bool one_thread_per_connection_end(THD *thd, bool put_in_cache)
{
DBUG_ENTER("one_thread_per_connection_end");
unlink_thd(thd);
mysql_mutex_unlock(&LOCK_thd_remove);
if (put_in_cache)
put_in_cache= cache_thread();
mysql_mutex_unlock(&LOCK_thread_count);
@ -3589,6 +3594,8 @@ static int init_thread_environment()
mysql_mutex_init(key_LOCK_thread_created, &LOCK_thread_created, MY_MUTEX_INIT_FAST);
mysql_mutex_init(key_LOCK_thread_count, &LOCK_thread_count, MY_MUTEX_INIT_FAST);
mysql_mutex_init(key_LOCK_status, &LOCK_status, MY_MUTEX_INIT_FAST);
mysql_mutex_init(key_LOCK_thd_remove,
&LOCK_thd_remove, MY_MUTEX_INIT_FAST);
mysql_mutex_init(key_LOCK_delayed_insert,
&LOCK_delayed_insert, MY_MUTEX_INIT_FAST);
mysql_mutex_init(key_LOCK_delayed_status,
@ -5080,9 +5087,11 @@ void create_thread_to_handle_connection(THD *thd)
ER_THD(thd, ER_CANT_CREATE_THREAD), error);
net_send_error(thd, ER_CANT_CREATE_THREAD, error_message_buff, NULL);
close_connection(thd);
mysql_mutex_lock(&LOCK_thd_remove);
mysql_mutex_lock(&LOCK_thread_count);
delete thd;
mysql_mutex_unlock(&LOCK_thread_count);
mysql_mutex_unlock(&LOCK_thd_remove);
return;
/* purecov: end */
}
@ -7873,6 +7882,7 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_prep_xids,
key_structure_guard_mutex, key_TABLE_SHARE_LOCK_ha_data,
key_LOCK_error_messages, key_LOG_INFO_lock, key_LOCK_thread_count,
key_PARTITION_LOCK_auto_inc;
PSI_mutex_key key_LOCK_thd_remove;
PSI_mutex_key key_RELAYLOG_LOCK_index;
PSI_mutex_key key_LOCK_thread_created;
@ -7928,7 +7938,8 @@ static PSI_mutex_info all_server_mutexes[]=
{ &key_LOG_INFO_lock, "LOG_INFO::lock", 0},
{ &key_LOCK_thread_count, "LOCK_thread_count", PSI_FLAG_GLOBAL},
{ &key_PARTITION_LOCK_auto_inc, "HA_DATA_PARTITION::LOCK_auto_inc", 0},
{ &key_LOCK_thread_created, "LOCK_thread_created", PSI_FLAG_GLOBAL }
{ &key_LOCK_thread_created, "LOCK_thread_created", PSI_FLAG_GLOBAL },
{ &key_LOCK_thd_remove, "LOCK_thd_remove", PSI_FLAG_GLOBAL}
};
PSI_rwlock_key key_rwlock_LOCK_grant, key_rwlock_LOCK_logger,