database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-07-29 05:21:33 +03:00
Code Activity

MDEV-4506: Parallel replication

Browse Source 
MDEV-5217: SQL thread hangs during stop if error occurs in the middle of an event group

Normally, when we stop the slave SQL thread in parallel replication, we want
the worker threads to continue processing events until the end of the current
event group. But if we stop due to an error that prevents further events from
being queued, such as an error reading the relay log, no more events can be
queued for the workers, so they have to abort even if they are in the middle
of an event group. There was a bug that we would deadlock, the workers
waiting for more events to be queued for the event group, the SQL thread
stopped and waiting for the workers to complete their current event group
before exiting.

Fixed by now signalling from the SQL thread to all workers when it is about
to exit, and cleaning up in all workers when so signalled.

This patch fixes one of multiple problems reported in MDEV-5217.
This commit is contained in:
unknown
2013-11-05 12:01:26 +01:00
parent bf603250b0
commit c834242ad4
2 changed files with 96 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
sql/rpl_parallel.cc
View File

@ -120,6 +120,46 @@ sql_worker_killed(THD *thd, rpl_group_info *rgi, bool in_event_group)
}
static void
finish_event_group(THD *thd, int err, uint64 sub_id,
rpl_parallel_entry *entry, wait_for_commit *wfc)
{
/*
Remove any left-over registration to wait for a prior commit to
complete. Normally, such wait would already have been removed at
this point by wait_for_prior_commit(), but eg. in error case we
might have skipped waiting, so we would need to remove it explicitly.
*/
wfc->unregister_wait_for_prior_commit();
thd->wait_for_commit_ptr= NULL;
/*
Record that this event group has finished (eg. transaction is
committed, if transactional), so other event groups will no longer
attempt to wait for us to commit. Once we have increased
entry->last_committed_sub_id, no other threads will execute
register_wait_for_prior_commit() against us. Thus, by doing one
extra (usually redundant) wakeup_subsequent_commits() we can ensure
that no register_wait_for_prior_commit() can ever happen without a
subsequent wakeup_subsequent_commits() to wake it up.
We can race here with the next transactions, but that is fine, as
long as we check that we do not decrease last_committed_sub_id. If
this commit is done, then any prior commits will also have been
done and also no longer need waiting for.
*/
mysql_mutex_lock(&entry->LOCK_parallel_entry);
if (entry->last_committed_sub_id < sub_id)
{
entry->last_committed_sub_id= sub_id;
mysql_cond_broadcast(&entry->COND_parallel_entry);
}
mysql_mutex_unlock(&entry->LOCK_parallel_entry);
wfc->wakeup_subsequent_commits(err);
}
pthread_handler_t
handle_rpl_parallel_thread(void *arg)
{
@ -128,6 +168,7 @@ handle_rpl_parallel_thread(void *arg)
struct rpl_parallel_thread::queued_event *events;
bool group_standalone= true;
bool in_event_group= false;
rpl_group_info *group_rgi= NULL;
uint64 event_gtid_sub_id= 0;
int err;
@ -174,7 +215,8 @@ handle_rpl_parallel_thread(void *arg)
old_msg= thd->proc_info;
thd->enter_cond(&rpt->COND_rpl_thread, &rpt->LOCK_rpl_thread,
"Waiting for work from SQL thread");
while (!(events= rpt->event_queue) && !rpt->stop && !thd->killed)
while (!(events= rpt->event_queue) && !rpt->stop && !thd->killed &&
!(rpt->current_entry && rpt->current_entry->force_abort))
mysql_cond_wait(&rpt->COND_rpl_thread, &rpt->LOCK_rpl_thread);
rpt->dequeue(events);
thd->exit_cond(old_msg);
@ -200,6 +242,7 @@ handle_rpl_parallel_thread(void *arg)
}
err= 0;
group_rgi= rgi;
/* Handle a new event group, which will be initiated by a GTID event. */
if ((event_type= events->ev->get_type_code()) == GTID_EVENT)
{
@ -295,41 +338,10 @@ handle_rpl_parallel_thread(void *arg)
if (end_of_group)
{
in_event_group= false;
/*
Remove any left-over registration to wait for a prior commit to
complete. Normally, such wait would already have been removed at
this point by wait_for_prior_commit(), but eg. in error case we
might have skipped waiting, so we would need to remove it explicitly.
*/
rgi->commit_orderer.unregister_wait_for_prior_commit();
thd->wait_for_commit_ptr= NULL;
/*
Record that this event group has finished (eg. transaction is
committed, if transactional), so other event groups will no longer
attempt to wait for us to commit. Once we have increased
entry->last_committed_sub_id, no other threads will execute
register_wait_for_prior_commit() against us. Thus, by doing one
extra (usually redundant) wakeup_subsequent_commits() we can ensure
that no register_wait_for_prior_commit() can ever happen without a
subsequent wakeup_subsequent_commits() to wake it up.
We can race here with the next transactions, but that is fine, as
long as we check that we do not decrease last_committed_sub_id. If
this commit is done, then any prior commits will also have been
done and also no longer need waiting for.
*/
mysql_mutex_lock(&entry->LOCK_parallel_entry);
if (entry->last_committed_sub_id < event_gtid_sub_id)
{
entry->last_committed_sub_id= event_gtid_sub_id;
mysql_cond_broadcast(&entry->COND_parallel_entry);
}
mysql_mutex_unlock(&entry->LOCK_parallel_entry);
rgi->commit_orderer.wakeup_subsequent_commits(err);
finish_event_group(thd, err, event_gtid_sub_id, entry,
&rgi->commit_orderer);
delete rgi;
group_rgi= rgi= NULL;
}
events= next;
@ -349,6 +361,27 @@ handle_rpl_parallel_thread(void *arg)
goto more_events;
}
if (in_event_group && group_rgi->parallel_entry->force_abort)
{
/*
We are asked to abort, without getting the remaining events in the
current event group.
We have to rollback the current transaction and update the last
sub_id value so that SQL thread will know we are done with the
half-processed event group.
*/
mysql_mutex_unlock(&rpt->LOCK_rpl_thread);
group_rgi->is_error= true;
finish_event_group(thd, 1, group_rgi->gtid_sub_id,
group_rgi->parallel_entry, &group_rgi->commit_orderer);
group_rgi->cleanup_context(thd, true);
group_rgi->rli->abort_slave= true;
in_event_group= false;
delete group_rgi;
group_rgi= NULL;
mysql_mutex_lock(&rpt->LOCK_rpl_thread);
}
if (!in_event_group)
{
rpt->current_entry= NULL;
@ -646,6 +679,8 @@ rpl_parallel::find(uint32 domain_id)
MY_MUTEX_INIT_FAST);
mysql_cond_init(key_COND_parallel_entry, &e->COND_parallel_entry, NULL);
}
else
e->force_abort= false;
return e;
}
@ -657,6 +692,25 @@ rpl_parallel::wait_for_done()
struct rpl_parallel_entry *e;
uint32 i;
/*
First signal all workers that they must force quit; no more events will
be queued to complete any partial event groups executed.
*/
for (i= 0; i < domain_hash.records; ++i)
{
rpl_parallel_thread *rpt;
e= (struct rpl_parallel_entry *)my_hash_element(&domain_hash, i);
e->force_abort= true;
if ((rpt= e->rpl_thread))
{
mysql_mutex_lock(&rpt->LOCK_rpl_thread);
if (rpt->current_entry == e)
mysql_cond_signal(&rpt->COND_rpl_thread);
mysql_mutex_unlock(&rpt->LOCK_rpl_thread);
}
}
for (i= 0; i < domain_hash.records; ++i)
{
e= (struct rpl_parallel_entry *)my_hash_element(&domain_hash, i);

7
sql/rpl_parallel.h
View File

@ -76,6 +76,13 @@ struct rpl_parallel_entry {
uint64 last_seq_no;
uint64 last_commit_id;
bool active;
/*
Set when SQL thread is shutting down, and no more events can be processed,
so worker threads must force abort any current transactions without
waiting for event groups to complete.
*/
bool force_abort;
rpl_parallel_thread *rpl_thread;
/*
The sub_id of the last transaction to commit within this domain_id.