From 6559ba71a564aecad487dee1fbb52fc42c768c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 13 Dec 2017 16:18:08 +0200 Subject: [PATCH] MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup while rolling back recovered incomplete transactions trx_rollback_resurrected(): If shutdown was initiated, fake all remaining active transactions to XA PREPARE state, so that shutdown can proceed. Also, make the parameter "all" an output that will be assigned to FALSE in this case. trx_rollback_or_clean_recovered(): Remove the shutdown check (it was moved to trx_rollback_resurrected()). trx_undo_free_prepared(): Relax assertions. --- .../suite/innodb/r/recovery_shutdown.result | 51 +++++++++++++++++++ .../suite/innodb/t/recovery_shutdown.test | 51 +++++++++++++++++++ storage/innobase/trx/trx0roll.cc | 37 +++++++------- storage/innobase/trx/trx0undo.cc | 16 ++++-- storage/xtradb/trx/trx0roll.cc | 37 +++++++------- storage/xtradb/trx/trx0undo.cc | 16 ++++-- 6 files changed, 166 insertions(+), 42 deletions(-) create mode 100644 mysql-test/suite/innodb/r/recovery_shutdown.result create mode 100644 mysql-test/suite/innodb/t/recovery_shutdown.test diff --git a/mysql-test/suite/innodb/r/recovery_shutdown.result b/mysql-test/suite/innodb/r/recovery_shutdown.result new file mode 100644 index 00000000000..028a0bd6239 --- /dev/null +++ b/mysql-test/suite/innodb/r/recovery_shutdown.result @@ -0,0 +1,51 @@ +# +# MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup +# while rolling back recovered incomplete transactions +# +CREATE TABLE t (a INT) ENGINE=InnoDB; +BEGIN; +COMMIT; +CREATE TABLE t8 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t8 (a) SELECT NULL FROM t; +UPDATE t8 SET a=a+100, b=a; +DELETE FROM t8; +CREATE TABLE t7 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t7 (a) SELECT NULL FROM t; +UPDATE t7 SET a=a+100, b=a; +DELETE FROM t7; +CREATE TABLE t6 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t6 (a) SELECT NULL FROM t; +UPDATE t6 SET a=a+100, b=a; +DELETE FROM t6; +CREATE TABLE t5 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t5 (a) SELECT NULL FROM t; +UPDATE t5 SET a=a+100, b=a; +DELETE FROM t5; +CREATE TABLE t4 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t4 (a) SELECT NULL FROM t; +UPDATE t4 SET a=a+100, b=a; +DELETE FROM t4; +CREATE TABLE t3 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t3 (a) SELECT NULL FROM t; +UPDATE t3 SET a=a+100, b=a; +DELETE FROM t3; +CREATE TABLE t2 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t2 (a) SELECT NULL FROM t; +UPDATE t2 SET a=a+100, b=a; +DELETE FROM t2; +CREATE TABLE t1 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +INSERT INTO t1 (a) SELECT NULL FROM t; +UPDATE t1 SET a=a+100, b=a; +DELETE FROM t1; +SET GLOBAL innodb_flush_log_at_trx_commit=1; +CREATE TABLE u(a SERIAL) ENGINE=INNODB; +# Kill and restart +DROP TABLE t,u; diff --git a/mysql-test/suite/innodb/t/recovery_shutdown.test b/mysql-test/suite/innodb/t/recovery_shutdown.test new file mode 100644 index 00000000000..ea38bd19a9f --- /dev/null +++ b/mysql-test/suite/innodb/t/recovery_shutdown.test @@ -0,0 +1,51 @@ +--source include/have_innodb.inc +--source include/not_embedded.inc + +--echo # +--echo # MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup +--echo # while rolling back recovered incomplete transactions +--echo # + +CREATE TABLE t (a INT) ENGINE=InnoDB; +let $size = 100; +let $trx = 8; +let $c = $size; +BEGIN; +--disable_query_log +while ($c) { +INSERT INTO t VALUES(); +dec $c; +} +--enable_query_log +COMMIT; + +let $c = $trx; +while ($c) +{ +connect (con$c,localhost,root,,); +eval CREATE TABLE t$c (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB; +BEGIN; +eval INSERT INTO t$c (a) SELECT NULL FROM t; +eval UPDATE t$c SET a=a+$size, b=a; +eval DELETE FROM t$c; +dec $c; +} + +--connection default +SET GLOBAL innodb_flush_log_at_trx_commit=1; +CREATE TABLE u(a SERIAL) ENGINE=INNODB; + +--source include/kill_and_restart_mysqld.inc +--source include/restart_mysqld.inc + +--disable_query_log +let $c = $trx; +while ($c) +{ +disconnect con$c; +eval DROP TABLE t$c; +dec $c; +} +--enable_query_log + +DROP TABLE t,u; diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index ae5e792a4bb..dbb902258c6 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -672,7 +672,7 @@ ibool trx_rollback_resurrected( /*=====================*/ trx_t* trx, /*!< in: transaction to rollback or clean */ - ibool all) /*!< in: FALSE=roll back dictionary transactions; + ibool* all) /*!< in/out: FALSE=roll back dictionary transactions; TRUE=roll back all non-PREPARED transactions */ { ut_ad(mutex_own(&trx_sys->mutex)); @@ -683,16 +683,15 @@ trx_rollback_resurrected( to accidentally clean up a non-recovered transaction here. */ trx_mutex_enter(trx); - bool is_recovered = trx->is_recovered; - trx_state_t state = trx->state; - trx_mutex_exit(trx); - - if (!is_recovered) { + if (!trx->is_recovered) { +func_exit: + trx_mutex_exit(trx); return(FALSE); } - switch (state) { + switch (trx->state) { case TRX_STATE_COMMITTED_IN_MEMORY: + trx_mutex_exit(trx); mutex_exit(&trx_sys->mutex); fprintf(stderr, "InnoDB: Cleaning up trx with id " TRX_ID_FMT "\n", @@ -701,7 +700,17 @@ trx_rollback_resurrected( trx_free_for_background(trx); return(TRUE); case TRX_STATE_ACTIVE: - if (all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { + if (srv_shutdown_state != SRV_SHUTDOWN_NONE + && srv_fast_shutdown) { + trx->state = TRX_STATE_PREPARED; + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; + *all = FALSE; + goto func_exit; + } + trx_mutex_exit(trx); + + if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { mutex_exit(&trx_sys->mutex); trx_rollback_active(trx); trx_free_for_background(trx); @@ -709,13 +718,13 @@ trx_rollback_resurrected( } return(FALSE); case TRX_STATE_PREPARED: - return(FALSE); + goto func_exit; case TRX_STATE_NOT_STARTED: break; } ut_error; - return(FALSE); + goto func_exit; } /*******************************************************************//** @@ -762,17 +771,11 @@ trx_rollback_or_clean_recovered( assert_trx_in_rw_list(trx); - if (srv_shutdown_state != SRV_SHUTDOWN_NONE - && srv_fast_shutdown != 0) { - all = FALSE; - break; - } - /* If this function does a cleanup or rollback then it will release the trx_sys->mutex, therefore we need to reacquire it before retrying the loop. */ - if (trx_rollback_resurrected(trx, all)) { + if (trx_rollback_resurrected(trx, &all)) { mutex_enter(&trx_sys->mutex); diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 1836d282cd4..5638ccd17a6 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -2023,9 +2023,13 @@ trx_undo_free_prepared( /* fall through */ case TRX_UNDO_ACTIVE: /* lock_trx_release_locks() assigns - trx->is_recovered=false */ + trx->is_recovered=false and + trx->state = TRX_STATE_COMMITTED_IN_MEMORY, + also for transactions that we faked + to TRX_STATE_PREPARED in trx_rollback_resurrected(). */ ut_a(srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || srv_fast_shutdown); break; default: ut_error; @@ -2047,9 +2051,13 @@ trx_undo_free_prepared( /* fall through */ case TRX_UNDO_ACTIVE: /* lock_trx_release_locks() assigns - trx->is_recovered=false */ + trx->is_recovered=false and + trx->state = TRX_STATE_COMMITTED_IN_MEMORY, + also for transactions that we faked + to TRX_STATE_PREPARED in trx_rollback_resurrected(). */ ut_a(srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || srv_fast_shutdown); break; default: ut_error; diff --git a/storage/xtradb/trx/trx0roll.cc b/storage/xtradb/trx/trx0roll.cc index ae5e792a4bb..dbb902258c6 100644 --- a/storage/xtradb/trx/trx0roll.cc +++ b/storage/xtradb/trx/trx0roll.cc @@ -672,7 +672,7 @@ ibool trx_rollback_resurrected( /*=====================*/ trx_t* trx, /*!< in: transaction to rollback or clean */ - ibool all) /*!< in: FALSE=roll back dictionary transactions; + ibool* all) /*!< in/out: FALSE=roll back dictionary transactions; TRUE=roll back all non-PREPARED transactions */ { ut_ad(mutex_own(&trx_sys->mutex)); @@ -683,16 +683,15 @@ trx_rollback_resurrected( to accidentally clean up a non-recovered transaction here. */ trx_mutex_enter(trx); - bool is_recovered = trx->is_recovered; - trx_state_t state = trx->state; - trx_mutex_exit(trx); - - if (!is_recovered) { + if (!trx->is_recovered) { +func_exit: + trx_mutex_exit(trx); return(FALSE); } - switch (state) { + switch (trx->state) { case TRX_STATE_COMMITTED_IN_MEMORY: + trx_mutex_exit(trx); mutex_exit(&trx_sys->mutex); fprintf(stderr, "InnoDB: Cleaning up trx with id " TRX_ID_FMT "\n", @@ -701,7 +700,17 @@ trx_rollback_resurrected( trx_free_for_background(trx); return(TRUE); case TRX_STATE_ACTIVE: - if (all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { + if (srv_shutdown_state != SRV_SHUTDOWN_NONE + && srv_fast_shutdown) { + trx->state = TRX_STATE_PREPARED; + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; + *all = FALSE; + goto func_exit; + } + trx_mutex_exit(trx); + + if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) { mutex_exit(&trx_sys->mutex); trx_rollback_active(trx); trx_free_for_background(trx); @@ -709,13 +718,13 @@ trx_rollback_resurrected( } return(FALSE); case TRX_STATE_PREPARED: - return(FALSE); + goto func_exit; case TRX_STATE_NOT_STARTED: break; } ut_error; - return(FALSE); + goto func_exit; } /*******************************************************************//** @@ -762,17 +771,11 @@ trx_rollback_or_clean_recovered( assert_trx_in_rw_list(trx); - if (srv_shutdown_state != SRV_SHUTDOWN_NONE - && srv_fast_shutdown != 0) { - all = FALSE; - break; - } - /* If this function does a cleanup or rollback then it will release the trx_sys->mutex, therefore we need to reacquire it before retrying the loop. */ - if (trx_rollback_resurrected(trx, all)) { + if (trx_rollback_resurrected(trx, &all)) { mutex_enter(&trx_sys->mutex); diff --git a/storage/xtradb/trx/trx0undo.cc b/storage/xtradb/trx/trx0undo.cc index 1836d282cd4..5638ccd17a6 100644 --- a/storage/xtradb/trx/trx0undo.cc +++ b/storage/xtradb/trx/trx0undo.cc @@ -2023,9 +2023,13 @@ trx_undo_free_prepared( /* fall through */ case TRX_UNDO_ACTIVE: /* lock_trx_release_locks() assigns - trx->is_recovered=false */ + trx->is_recovered=false and + trx->state = TRX_STATE_COMMITTED_IN_MEMORY, + also for transactions that we faked + to TRX_STATE_PREPARED in trx_rollback_resurrected(). */ ut_a(srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || srv_fast_shutdown); break; default: ut_error; @@ -2047,9 +2051,13 @@ trx_undo_free_prepared( /* fall through */ case TRX_UNDO_ACTIVE: /* lock_trx_release_locks() assigns - trx->is_recovered=false */ + trx->is_recovered=false and + trx->state = TRX_STATE_COMMITTED_IN_MEMORY, + also for transactions that we faked + to TRX_STATE_PREPARED in trx_rollback_resurrected(). */ ut_a(srv_read_only_mode - || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO + || srv_fast_shutdown); break; default: ut_error;