From 62dca454e725b47eb8d7b4ee383a2ff9ef28255f Mon Sep 17 00:00:00 2001 From: Aditya A Date: Wed, 4 Jan 2017 14:34:38 +0530 Subject: [PATCH] Bug #23070734 CONCURRENT TRUNCATE TABLES CAUSE STALLS PROBLEM When truncating single tablespace tables, we need to scan the entire buffer pool to remove the pages of the table from the buffer pool. During this scan and removal dict_sys->mutex is being held ,causing stalls in other DDL operations. FIX Release the dict_sys->mutex during the scan and reacquire it after the scan. Make sure that purge thread doesn't purge the records of the table being truncated and background stats collection thread skips the updation of stats for the table being truncated. [#rb 14564 Approved by Jimmy and satya ] --- .../innodb/r/innodb-truncate-debug.result | 124 +++++++++++ .../suite/innodb/t/innodb-truncate-debug.test | 194 ++++++++++++++++++ storage/innobase/dict/dict0stats_bg.cc | 11 +- storage/innobase/fil/fil0fil.cc | 34 ++- storage/innobase/include/fil0fil.h | 18 +- storage/innobase/row/row0purge.cc | 17 +- storage/innobase/row/row0trunc.cc | 4 +- storage/innobase/row/row0uins.cc | 7 +- storage/innobase/row/row0umod.cc | 8 +- 9 files changed, 388 insertions(+), 29 deletions(-) create mode 100644 mysql-test/suite/innodb/r/innodb-truncate-debug.result create mode 100644 mysql-test/suite/innodb/t/innodb-truncate-debug.test diff --git a/mysql-test/suite/innodb/r/innodb-truncate-debug.result b/mysql-test/suite/innodb/r/innodb-truncate-debug.result new file mode 100644 index 00000000000..3012462d373 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-truncate-debug.result @@ -0,0 +1,124 @@ +# +# Bug #23070734 CONCURRENT TRUNCATE TABLES CAUSE STALLS +# +Test_1 :- Check if DDL operations are possible on +table being truncated. Also check if +DDL operations on other tables succeed. +create table t1 (f1 int,f2 int,key(f2),f3 int) engine=innodb; +create index idx1 on t1(f3); +create table t2 (f1 int,f2 int,key(f2),f3 int) engine=innodb; +create table t3 (f1 int,f2 int,key(f2)) engine=innodb; +insert into t1 values (10,20,30),(30,40,50),(50,60,70); +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t2 values (10,20,30),(30,40,50),(50,60,70); +insert into t2 select * from t2; +insert into t2 select * from t2; +insert into t2 select * from t2; +insert into t3 values (10,20),(30,40),(50,50); +insert into t3 select * from t3; +insert into t3 select * from t3; +SET session lock_wait_timeout = 1; +show variables like 'lock_wait_timeout%'; +Variable_name Value +lock_wait_timeout 1 +connection 1 +SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan'; +truncate table t1;; +connection default +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +Check Analyze table. Gives lock time out error. +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze Error Lock wait timeout exceeded; try restarting transaction +test.t1 analyze status Operation failed +Check if we can turn off auto recalculation. +alter table t1 STATS_AUTO_RECALC=0; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +Check if we can turn off persistent stats on the table +alter table t1 STATS_PERSISTENT=0; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +Check if DML is possible on table being truncated +insert into t1 values (10,89,99); +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +Check if DDL is possible on table being truncated +alter table t1 add column f4 int; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +check if table can be created with the same name +create table t1 (bd int) engine=innodb; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +check if index can be created on table being truncated +create index idx1 on t1(f1); +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +Check if DROP of table is possible during truncate +drop table t1; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +Check if select is possible during truncate +select * from t1; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +select * from t2 where t2.f1=t1.f1; +ERROR 42S22: Unknown column 't1.f1' in 'where clause' +Check concurrent truncate operation on table; +truncate table t1; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +Check concurrent selects and inserts on the other table +insert into t3 values (10,20),(30,40),(50,50); +select * from t3 where f1=40; +f1 f2 +Concurrent truncate on other tables +truncate table t2; +Concurrent alters on the other tables +alter table t2 add column f4 int; +check if index can be created on the other table +create index idx1 on t2(f3); +Check if we can turn off persistent stats off entire instance +SET GLOBAL innodb_stats_persistent=off; +connection con2 +set global innodb_adaptive_hash_index=off;; +connection default +SET DEBUG_SYNC= 'now SIGNAL finish_scan'; +connection con1 +connection con2 +connection default +SET session lock_wait_timeout=default; +set global innodb_adaptive_hash_index=default; +drop table t1,t2,t3; +SET @@global.innodb_stats_persistent=default; +Test_2 :- Check if purge thread ignores the undo +log records of the table being truncated. +create table t1 (f1 int ,f2 int,key(f2)) engine=innodb; +insert into t1 values (10,45),(20,78),(30,88),(40,23),(50,78),(60,11),(70,56),(80,90); +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +select count(*) from t1; +count(*) +131072 +Stop the purge thread +set global innodb_purge_stop_now = 1; +delete from t1; +connection con1 +SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan'; +truncate table t1;; +Connection default +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +set global innodb_purge_run_now=1; +SET DEBUG_SYNC= 'now SIGNAL finish_scan'; +connection con1 +connection default +drop table t1; +Pattern "InnoDB: Record with space id \d+ belongs to table which is being truncated therefore skipping this undo record." found +# restart server +# restart: diff --git a/mysql-test/suite/innodb/t/innodb-truncate-debug.test b/mysql-test/suite/innodb/t/innodb-truncate-debug.test new file mode 100644 index 00000000000..9705190332c --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-truncate-debug.test @@ -0,0 +1,194 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc + +--echo # +--echo # Bug #23070734 CONCURRENT TRUNCATE TABLES CAUSE STALLS +--echo # + +--echo Test_1 :- Check if DDL operations are possible on +--echo table being truncated. Also check if +--echo DDL operations on other tables succeed. + +create table t1 (f1 int,f2 int,key(f2),f3 int) engine=innodb; +create index idx1 on t1(f3); +create table t2 (f1 int,f2 int,key(f2),f3 int) engine=innodb; +create table t3 (f1 int,f2 int,key(f2)) engine=innodb; + +insert into t1 values (10,20,30),(30,40,50),(50,60,70); +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t2 values (10,20,30),(30,40,50),(50,60,70); + +insert into t2 select * from t2; +insert into t2 select * from t2; +insert into t2 select * from t2; + +insert into t3 values (10,20),(30,40),(50,50); +insert into t3 select * from t3; +insert into t3 select * from t3; + +SET session lock_wait_timeout = 1; +show variables like 'lock_wait_timeout%'; + +connect (con1,localhost,root,,); +connect (con2,localhost,root,,); + +--echo connection 1 +connection con1; +SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan'; +--send truncate table t1; + +--echo connection default +connection default; +SET DEBUG_SYNC= 'now WAIT_FOR opened'; + +--echo Check Analyze table. Gives lock time out error. +analyze table t1; + +--echo Check if we can turn off auto recalculation. +--error ER_LOCK_WAIT_TIMEOUT +alter table t1 STATS_AUTO_RECALC=0; + +--echo Check if we can turn off persistent stats on the table +--error ER_LOCK_WAIT_TIMEOUT +alter table t1 STATS_PERSISTENT=0; + +--echo Check if DML is possible on table being truncated +--error ER_LOCK_WAIT_TIMEOUT +insert into t1 values (10,89,99); + +--echo Check if DDL is possible on table being truncated +--error ER_LOCK_WAIT_TIMEOUT +alter table t1 add column f4 int; + +--echo check if table can be created with the same name +--error ER_LOCK_WAIT_TIMEOUT +create table t1 (bd int) engine=innodb; + +--echo check if index can be created on table being truncated +--error ER_LOCK_WAIT_TIMEOUT +create index idx1 on t1(f1); + +--echo Check if DROP of table is possible during truncate +--error ER_LOCK_WAIT_TIMEOUT +drop table t1; + +--echo Check if select is possible during truncate +--error ER_LOCK_WAIT_TIMEOUT +select * from t1; + +--error ER_BAD_FIELD_ERROR +select * from t2 where t2.f1=t1.f1; + +--echo Check concurrent truncate operation on table; +--error ER_LOCK_WAIT_TIMEOUT +truncate table t1; + +--echo Check concurrent selects and inserts on the other table +insert into t3 values (10,20),(30,40),(50,50); +select * from t3 where f1=40; + +--echo Concurrent truncate on other tables +truncate table t2; + +--echo Concurrent alters on the other tables +alter table t2 add column f4 int; + +--echo check if index can be created on the other table +create index idx1 on t2(f3); + + +--echo Check if we can turn off persistent stats off entire instance +SET GLOBAL innodb_stats_persistent=off; + +--echo connection con2 +connection con2; +--send set global innodb_adaptive_hash_index=off; + +--echo connection default +connection default; +SET DEBUG_SYNC= 'now SIGNAL finish_scan'; + +--echo connection con1 +connection con1; +reap; + +--echo connection con2 +connection con2; +reap; + +--echo connection default +connection default; +disconnect con1; +disconnect con2; + +SET session lock_wait_timeout=default; +set global innodb_adaptive_hash_index=default; + +drop table t1,t2,t3; +SET @@global.innodb_stats_persistent=default; + + +--echo Test_2 :- Check if purge thread ignores the undo +--echo log records of the table being truncated. + +let $restart_parameters = restart: --innodb_purge_threads=1 --innodb_purge_batch_size=1 --innodb_stats_persistent=OFF +--source include/restart_mysqld.inc; + +create table t1 (f1 int ,f2 int,key(f2)) engine=innodb; +insert into t1 values (10,45),(20,78),(30,88),(40,23),(50,78),(60,11),(70,56),(80,90); +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +select count(*) from t1; + +--echo Stop the purge thread +set global innodb_purge_stop_now = 1; +delete from t1; + +connect (con1,localhost,root,,); + +--echo connection con1 +connection con1; +SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan'; +--send truncate table t1; + +--echo Connection default +connection default; + +SET DEBUG_SYNC= 'now WAIT_FOR opened'; +set global innodb_purge_run_now=1; +--sleep 60 +SET DEBUG_SYNC= 'now SIGNAL finish_scan'; + +--echo connection con1 +connection con1; +reap; + +--echo connection default +connection default; +disconnect con1; + +drop table t1; + +let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN = InnoDB: Record with space id \d+ belongs to table which is being truncated therefore skipping this undo record.; +--source include/search_pattern.inc + +#cleanup +--echo # restart server +let $restart_parameters = restart:; +--source include/restart_mysqld.inc diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 4ffee160c9f..986b6a2613f 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -1,7 +1,7 @@ /***************************************************************************** -Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2012, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -31,6 +31,7 @@ Created Apr 25, 2012 Vasil Dimov #include "row0mysql.h" #include "srv0start.h" #include "ut0new.h" +#include "fil0fil.h" #include @@ -318,6 +319,12 @@ dict_stats_process_entry_from_recalc_pool() return; } + if (fil_space_is_being_truncated(table->space)) { + dict_table_close(table, TRUE, FALSE); + mutex_exit(&dict_sys->mutex); + return; + } + /* Check whether table is corrupted */ if (table->corrupted) { dict_table_close(table, TRUE, FALSE); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index d91ab3530a1..7fd41a8fcde 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -30,6 +30,7 @@ Created 10/25/1995 Heikki Tuuri #include "fil0crypt.h" #include "btr0btr.h" +#include "btr0sea.h" #include "buf0buf.h" #include "dict0boot.h" #include "dict0dict.h" @@ -55,9 +56,6 @@ Created 10/25/1995 Heikki Tuuri #include "os0event.h" #include "sync0sync.h" #include "buf0flu.h" -#include "srv0start.h" -#include "trx0purge.h" -#include "ut0new.h" #include "os0api.h" /** Tries to close a file in the LRU list. The caller must hold the fil_sys @@ -3266,20 +3264,34 @@ fil_prepare_for_truncate( return(err); } -/**********************************************************************//** -Reinitialize the original tablespace header with the same space id -for single tablespace */ +/** Reinitialize the original tablespace header with the same space id +for single tablespace +@param[in] id space id of the tablespace +@param[in] size size in blocks +@param[in] trx Transaction covering truncate */ void fil_reinit_space_header( -/*====================*/ - ulint id, /*!< in: space id */ - ulint size) /*!< in: size in blocks */ + ulint id, + ulint size, + trx_t* trx) { ut_a(!is_system_tablespace(id)); /* Invalidate in the buffer pool all pages belonging - to the tablespace */ + to the tablespace. The buffer pool scan may take long + time to complete, therefore we release dict_sys->mutex + and the dict operation lock during the scan and aquire + it again after the buffer pool scan.*/ + + row_mysql_unlock_data_dictionary(trx); + + /* Lock the search latch in shared mode to prevent user + from disabling AHI during the scan */ + btr_search_s_lock_all(); + DEBUG_SYNC_C("simulate_buffer_pool_scan"); buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_ALL_NO_WRITE, 0); + btr_search_s_unlock_all(); + row_mysql_lock_data_dictionary(trx); /* Remove all insert buffer entries for the tablespace */ ibuf_delete_for_discarded_space(id); diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 666c42a9628..9c6d4f920ff 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -914,14 +914,18 @@ dberr_t fil_prepare_for_truncate( /*=====================*/ ulint id); /*!< in: space id */ -/**********************************************************************//** -Reinitialize the original tablespace header with the same space id -for single tablespace */ + +/** Reinitialize the original tablespace header with the same space id +for single tablespace +@param[in] id space id of the tablespace +@param[in] size size in blocks +@param[in] trx Transaction covering truncate */ void fil_reinit_space_header( -/*====================*/ - ulint id, /*!< in: space id */ - ulint size); /*!< in: size in blocks */ + ulint id, + ulint size, + trx_t* trx); + /*******************************************************************//** Closes a single-table tablespace. The tablespace must be cached in the memory cache. Free all pages used by the tablespace. diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 2a13203b747..5202fb15af8 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -43,6 +43,7 @@ Created 3/14/1997 Heikki Tuuri #include "srv0start.h" #include "handler.h" #include "ha_innodb.h" +#include "fil0fil.h" /************************************************************************* IMPORTANT NOTE: Any operation that generates redo MUST check that there @@ -854,6 +855,20 @@ try_again: } ut_ad(!dict_table_is_temporary(node->table)); + if (fil_space_is_being_truncated(node->table->space)) { + +#if UNIV_DEBUG + ib::info() << "Record with space id " + << node->table->space + << " belongs to table which is being truncated" + << " therefore skipping this undo record."; +#endif + ut_ad(dict_table_is_file_per_table(node->table)); + dict_table_close(node->table, FALSE, FALSE); + node->table = NULL; + goto err_exit; + } + if (node->table->n_v_cols && !node->table->vc_templ && dict_table_has_indexed_v_cols(node->table)) { /* Need server fully up for virtual column computation */ diff --git a/storage/innobase/row/row0trunc.cc b/storage/innobase/row/row0trunc.cc index 8529cfec9fd..1fc30e714f4 100644 --- a/storage/innobase/row/row0trunc.cc +++ b/storage/innobase/row/row0trunc.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -2015,7 +2015,7 @@ row_truncate_table_for_mysql( space_size -= ib_vector_size(table->fts->indexes); } - fil_reinit_space_header(table->space, space_size); + fil_reinit_space_header(table->space, space_size, trx); } DBUG_EXECUTE_IF("ib_trunc_crash_with_intermediate_log_checkpoint", diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc index 25504e32087..e0b9508caed 100644 --- a/storage/innobase/row/row0uins.cc +++ b/storage/innobase/row/row0uins.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,6 +41,7 @@ Created 2/25/1997 Heikki Tuuri #include "que0que.h" #include "ibuf0ibuf.h" #include "log0log.h" +#include "fil0fil.h" /************************************************************************* IMPORTANT NOTE: Any operation that generates redo MUST check that there @@ -347,6 +348,10 @@ row_undo_ins_parse_undo_rec( if (UNIV_UNLIKELY(node->table == NULL)) { } else if (UNIV_UNLIKELY(node->table->ibd_file_missing)) { close_table: + dict_table_close(node->table, dict_locked, FALSE); + node->table = NULL; + } else if (fil_space_is_being_truncated(node->table->space)) { + dict_table_close(node->table, dict_locked, FALSE); node->table = NULL; } else { diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index 378cad00b93..c583a438988 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1139,12 +1139,10 @@ row_undo_mod_parse_undo_rec( return; } - if (node->table->ibd_file_missing) { + if (node->table->ibd_file_missing || + fil_space_is_being_truncated(node->table->space) ) { dict_table_close(node->table, dict_locked, FALSE); - - /* We skip undo operations to missing .ibd files */ node->table = NULL; - return; }