From d03abc71a490f4ee6d173d07c95a2adb9c60457c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 8 Jun 2017 10:34:10 +0300 Subject: [PATCH 1/9] MDEV-12609: Allow suppression of InnoDB log messages about reserving extents Removed this output. --- storage/innobase/fsp/fsp0fsp.cc | 11 ----------- storage/xtradb/fsp/fsp0fsp.cc | 11 ----------- 2 files changed, 22 deletions(-) diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 1732a02aeac..d04cd7168ed 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -2729,7 +2729,6 @@ fsp_reserve_free_extents( ibool success; ulint n_pages_added; size_t total_reserved = 0; - ulint rounds = 0; ut_ad(mtr); *n_reserved = n_ext; @@ -2809,17 +2808,7 @@ try_to_extend: space_header, mtr); if (success && n_pages_added > 0) { - - rounds++; total_reserved += n_pages_added; - - if (rounds > 50) { - ib_logf(IB_LOG_LEVEL_INFO, - "Space id %lu trying to reserve %lu extents actually reserved %lu " - " reserve %lu free %lu size %lu rounds %lu total_reserved %llu", - space, n_ext, n_pages_added, reserve, n_free, size, rounds, (ullint) total_reserved); - } - goto try_again; } diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index 9292f84ab02..cf6fa4211ca 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -2738,7 +2738,6 @@ fsp_reserve_free_extents( ibool success; ulint n_pages_added; size_t total_reserved = 0; - ulint rounds = 0; ut_ad(mtr); *n_reserved = n_ext; @@ -2817,17 +2816,7 @@ try_to_extend: success = fsp_try_extend_data_file(&n_pages_added, space, space_header, mtr); if (success && n_pages_added > 0) { - - rounds++; total_reserved += n_pages_added; - - if (rounds > 50) { - ib_logf(IB_LOG_LEVEL_INFO, - "Space id %lu trying to reserve %lu extents actually reserved %lu " - " reserve %lu free %lu size %lu rounds %lu total_reserved %llu", - space, n_ext, n_pages_added, reserve, n_free, size, rounds, (ullint) total_reserved); - } - goto try_again; } From 58c56dd7f8e787c48b6f09a4d03cddd7c922588b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 8 Jun 2017 15:40:25 +0300 Subject: [PATCH 2/9] MDEV-12610: MariaDB start is slow Problem appears to be that the function fsp_flags_try_adjust() is being unconditionally invoked on every .ibd file on startup. Based on performance investigation also the top function fsp_header_get_crypt_offset() needs to addressed. Ported implementation of fsp_header_get_encryption_offset() function from 10.2 to fsp_header_get_crypt_offset(). Introduced a new function fil_crypt_read_crypt_data() to read page 0 if it is not yet read. fil_crypt_find_space_to_rotate(): Now that page 0 for every .ibd file is not read on startup we need to check has page 0 read from space that we investigate for key rotation, if it is not read we read it. fil_space_crypt_get_status(): Now that page 0 for every .ibd file is not read on startup here also we need to read page 0 if it is not yet read it. This is needed as tests use IS query to wait until background encryption or decryption has finished and this function is used to produce results. fil_crypt_thread(): Add is_stopping condition for tablespace so that we do not rotate pages if usage of tablespace should be stopped. This was needed for failure seen on regression testing. fil_space_create: Remove page_0_crypt_read and extra unnecessary info output. fil_open_single_table_tablespace(): We call fsp_flags_try_adjust only when when no errors has happened and server was not started on read only mode and tablespace validation was requested or flags contain other table options except low order bits to FSP_FLAGS_POS_PAGE_SSIZE position. fil_space_t::page_0_crypt_read removed. Added test case innodb-first-page-read to test startup when encryption is on and when encryption is off to check that not for all tables page 0 is read on startup. --- .../r/innodb-first-page-read.result | 89 +++++++++++++++++ .../encryption/r/innodb_lotoftables.result | 28 +++--- .../encryption/t/innodb-first-page-read.opt | 5 + .../encryption/t/innodb-first-page-read.test | 97 +++++++++++++++++++ storage/innobase/buf/buf0buf.cc | 1 + storage/innobase/fil/fil0crypt.cc | 61 +++++++++++- storage/innobase/fil/fil0fil.cc | 41 ++++---- storage/innobase/fsp/fsp0fsp.cc | 16 +-- storage/innobase/include/fil0fil.h | 3 - storage/xtradb/buf/buf0buf.cc | 1 + storage/xtradb/fil/fil0crypt.cc | 61 +++++++++++- storage/xtradb/fil/fil0fil.cc | 41 ++++---- storage/xtradb/fsp/fsp0fsp.cc | 16 +-- storage/xtradb/include/fil0fil.h | 3 - 14 files changed, 359 insertions(+), 104 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innodb-first-page-read.result create mode 100644 mysql-test/suite/encryption/t/innodb-first-page-read.opt create mode 100644 mysql-test/suite/encryption/t/innodb-first-page-read.test diff --git a/mysql-test/suite/encryption/r/innodb-first-page-read.result b/mysql-test/suite/encryption/r/innodb-first-page-read.result new file mode 100644 index 00000000000..6c9eea80fa9 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-first-page-read.result @@ -0,0 +1,89 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +create database innodb_test; +use innodb_test; +create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic; +create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed; +create table innodb_compressed1(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=1; +create table innodb_compressed2(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=2; +create table innodb_compressed4(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=4; +create table innodb_compressed8(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=8; +create table innodb_compressed16(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=16; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant; +create table innodb_pagecomp(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes; +create table innodb_pagecomp1(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=1; +create table innodb_pagecomp2(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=2; +create table innodb_pagecomp3(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=3; +create table innodb_pagecomp4(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=4; +create table innodb_pagecomp5(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=5; +create table innodb_pagecomp6(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=6; +create table innodb_pagecomp7(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=7; +create table innodb_pagecomp8(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=8; +create table innodb_pagecomp9(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=9; +create table innodb_datadir1(c1 bigint not null, b char(200)) engine=innodb DATA DIRECTORY='MYSQL_TMP_DIR'; +create table innodb_datadir2(c1 bigint not null, b char(200)) engine=innodb row_format=compressed DATA DIRECTORY='MYSQL_TMP_DIR'; +create table innodb_datadir3(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes DATA DIRECTORY='MYSQL_TMP_DIR'; +begin; +insert into innodb_normal values (1,'secret'); +insert into innodb_compact select * from innodb_normal; +insert into innodb_dynamic select * from innodb_normal; +insert into innodb_compressed select * from innodb_normal; +insert into innodb_compressed1 select * from innodb_normal; +insert into innodb_compressed2 select * from innodb_normal; +insert into innodb_compressed4 select * from innodb_normal; +insert into innodb_compressed8 select * from innodb_normal; +insert into innodb_compressed16 select * from innodb_normal; +insert into innodb_redundant select * from innodb_normal; +insert into innodb_pagecomp select * from innodb_normal; +insert into innodb_pagecomp1 select * from innodb_normal; +insert into innodb_pagecomp2 select * from innodb_normal; +insert into innodb_pagecomp3 select * from innodb_normal; +insert into innodb_pagecomp4 select * from innodb_normal; +insert into innodb_pagecomp5 select * from innodb_normal; +insert into innodb_pagecomp6 select * from innodb_normal; +insert into innodb_pagecomp7 select * from innodb_normal; +insert into innodb_pagecomp8 select * from innodb_normal; +insert into innodb_pagecomp9 select * from innodb_normal; +insert into innodb_datadir1 select * from innodb_normal; +insert into innodb_datadir2 select * from innodb_normal; +insert into innodb_datadir3 select * from innodb_normal; +commit; +# Restart server and see how many page 0's are read +# result should be less than actual number of tables +# i.e. < 23 + 3 = 26 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 17 +use innodb_test; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 17 +use test; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 17 +set global innodb_encrypt_tables=OFF; +# wait until tables are decrypted +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 29 +use innodb_test; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 29 +use test; +# restart and see number read page 0 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 17 +use innodb_test; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 17 +use test; +drop database innodb_test; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 29 diff --git a/mysql-test/suite/encryption/r/innodb_lotoftables.result b/mysql-test/suite/encryption/r/innodb_lotoftables.result index 418ab175a01..b7cfdd2db9d 100644 --- a/mysql-test/suite/encryption/r/innodb_lotoftables.result +++ b/mysql-test/suite/encryption/r/innodb_lotoftables.result @@ -12,13 +12,13 @@ create database innodb_encrypted_1; use innodb_encrypted_1; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 set autocommit=0; set autocommit=1; commit work; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 # should be 100 SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE NAME LIKE 'innodb_encrypted%'; COUNT(*) @@ -88,47 +88,47 @@ Innodb_pages0_read 3 # Restart Success! show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 1 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 1 use test; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 1 use innodb_encrypted_1; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 1 use innodb_encrypted_2; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 1 use innodb_encrypted_3; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 1 use innodb_encrypted_1; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 1 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 101 use innodb_encrypted_2; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 101 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 201 use innodb_encrypted_3; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 201 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 301 SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; COUNT(*) 100 diff --git a/mysql-test/suite/encryption/t/innodb-first-page-read.opt b/mysql-test/suite/encryption/t/innodb-first-page-read.opt new file mode 100644 index 00000000000..38d69691ed6 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-first-page-read.opt @@ -0,0 +1,5 @@ +--innodb-encrypt-tables=ON +--innodb-encrypt-log=ON +--innodb-encryption-rotate-key-age=15 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb-first-page-read.test b/mysql-test/suite/encryption/t/innodb-first-page-read.test new file mode 100644 index 00000000000..1fc07159e05 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-first-page-read.test @@ -0,0 +1,97 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +-- source include/not_embedded.inc + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + +create database innodb_test; +use innodb_test; +create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; +create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact; +create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic; +create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed; +create table innodb_compressed1(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=1; +create table innodb_compressed2(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=2; +create table innodb_compressed4(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=4; +create table innodb_compressed8(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=8; +create table innodb_compressed16(c1 bigint not null, b char(200)) engine=innodb row_format=compressed key_block_size=16; +create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant; +create table innodb_pagecomp(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes; +create table innodb_pagecomp1(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=1; +create table innodb_pagecomp2(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=2; +create table innodb_pagecomp3(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=3; +create table innodb_pagecomp4(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=4; +create table innodb_pagecomp5(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=5; +create table innodb_pagecomp6(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=6; +create table innodb_pagecomp7(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=7; +create table innodb_pagecomp8(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=8; +create table innodb_pagecomp9(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes page_compression_level=9; + +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +eval create table innodb_datadir1(c1 bigint not null, b char(200)) engine=innodb DATA DIRECTORY='$MYSQL_TMP_DIR'; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +eval create table innodb_datadir2(c1 bigint not null, b char(200)) engine=innodb row_format=compressed DATA DIRECTORY='$MYSQL_TMP_DIR'; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +eval create table innodb_datadir3(c1 bigint not null, b char(200)) engine=innodb page_compressed=yes DATA DIRECTORY='$MYSQL_TMP_DIR'; + +begin; +insert into innodb_normal values (1,'secret'); +insert into innodb_compact select * from innodb_normal; +insert into innodb_dynamic select * from innodb_normal; +insert into innodb_compressed select * from innodb_normal; +insert into innodb_compressed1 select * from innodb_normal; +insert into innodb_compressed2 select * from innodb_normal; +insert into innodb_compressed4 select * from innodb_normal; +insert into innodb_compressed8 select * from innodb_normal; +insert into innodb_compressed16 select * from innodb_normal; +insert into innodb_redundant select * from innodb_normal; +insert into innodb_pagecomp select * from innodb_normal; +insert into innodb_pagecomp1 select * from innodb_normal; +insert into innodb_pagecomp2 select * from innodb_normal; +insert into innodb_pagecomp3 select * from innodb_normal; +insert into innodb_pagecomp4 select * from innodb_normal; +insert into innodb_pagecomp5 select * from innodb_normal; +insert into innodb_pagecomp6 select * from innodb_normal; +insert into innodb_pagecomp7 select * from innodb_normal; +insert into innodb_pagecomp8 select * from innodb_normal; +insert into innodb_pagecomp9 select * from innodb_normal; +insert into innodb_datadir1 select * from innodb_normal; +insert into innodb_datadir2 select * from innodb_normal; +insert into innodb_datadir3 select * from innodb_normal; +commit; + +--echo # Restart server and see how many page 0's are read +--source include/restart_mysqld.inc + +--echo # result should be less than actual number of tables +--echo # i.e. < 23 + 3 = 26 +show status like 'innodb_pages0_read%'; +use innodb_test; +show status like 'innodb_pages0_read%'; +use test; +show status like 'innodb_pages0_read%'; + +set global innodb_encrypt_tables=OFF; + +--echo # wait until tables are decrypted +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 +--source include/wait_condition.inc + +show status like 'innodb_pages0_read%'; +use innodb_test; +show status like 'innodb_pages0_read%'; +use test; + +--echo # restart and see number read page 0 +-- source include/restart_mysqld.inc + +show status like 'innodb_pages0_read%'; +use innodb_test; +show status like 'innodb_pages0_read%'; +use test; + +drop database innodb_test; +show status like 'innodb_pages0_read%'; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index b4ee3ebc067..334b2fbddb5 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4549,6 +4549,7 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) !bpage->encrypted && fil_space_verify_crypt_checksum(dst_frame, zip_size, space, bpage->offset)); + if (!still_encrypted) { /* If traditional checksums match, we assume that page is not anymore encrypted. */ diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 2131a936656..70d8558ede2 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -1115,6 +1115,43 @@ fil_crypt_needs_rotation( return false; } +/** Read page 0 and possible crypt data from there. +@param[in] space Tablespace */ +static inline +void +fil_crypt_read_crypt_data(fil_space_t* space) +{ + mutex_enter(&fil_system->mutex); + + /* If space does not contain crypt data and space size is 0 + we have not yet read first page of tablespace. We need to + read it to find out tablespace current encryption status. */ + if (!space->crypt_data && space->size == 0) { + mtr_t mtr; + mtr_start(&mtr); + ulint zip_size = fsp_flags_get_zip_size(space->flags); + ulint offset = fsp_header_get_crypt_offset(zip_size); + mutex_exit(&fil_system->mutex); + if (buf_block_t* block = buf_page_get(space->id, zip_size, 0, + RW_X_LATCH, &mtr)) { + byte* frame = buf_block_get_frame(block); + + mutex_enter(&fil_system->mutex); + + if (!space->crypt_data) { + space->crypt_data = fil_space_read_crypt_data(space->id, + frame, offset); + } + + mutex_exit(&fil_system->mutex); + } + + mtr_commit(&mtr); + } else { + mutex_exit(&fil_system->mutex); + } +} + /*********************************************************************** Start encrypting a space @param[in,out] space Tablespace @@ -1125,6 +1162,7 @@ fil_crypt_start_encrypting_space( fil_space_t* space) { bool recheck = false; + mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t *crypt_data = space->crypt_data; @@ -1191,8 +1229,6 @@ fil_crypt_start_encrypting_space( byte* frame = buf_block_get_frame(block); crypt_data->type = CRYPT_SCHEME_1; crypt_data->write_page0(frame, &mtr); - - mtr_commit(&mtr); /* record lsn of update */ @@ -1620,6 +1656,13 @@ fil_crypt_find_space_to_rotate( } while (!state->should_shutdown() && state->space) { + /* If there is no crypt data and we have not yet read + page 0 for this tablespace, we need to read it before + we can continue. */ + if (!state->space->crypt_data) { + fil_crypt_read_crypt_data(state->space); + } + if (fil_crypt_space_needs_rotation(state, key_state, recheck)) { ut_ad(key_state->key_id); /* init state->min_key_version_found before @@ -2314,8 +2357,10 @@ DECLARE_THREAD(fil_crypt_thread)( while (!thr.should_shutdown() && fil_crypt_find_page_to_rotate(&new_state, &thr)) { - /* rotate a (set) of pages */ - fil_crypt_rotate_pages(&new_state, &thr); + if (!thr.space->is_stopping()) { + /* rotate a (set) of pages */ + fil_crypt_rotate_pages(&new_state, &thr); + } /* If space is marked as stopping, release space and stop rotation. */ @@ -2544,6 +2589,14 @@ fil_space_crypt_get_status( memset(status, 0, sizeof(*status)); ut_ad(space->n_pending_ops > 0); + + /* If there is no crypt data and we have not yet read + page 0 for this tablespace, we need to read it before + we can continue. */ + if (!space->crypt_data) { + fil_crypt_read_crypt_data(const_cast(space)); + } + fil_space_crypt_t* crypt_data = space->crypt_data; status->space = space->id; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index f88bb2add59..ffb01312fdb 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -653,12 +653,10 @@ fil_node_open_file( /* Try to read crypt_data from page 0 if it is not yet read. */ - if (!node->space->page_0_crypt_read) { - ulint offset = fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(flags)); - ut_ad(node->space->crypt_data == NULL); + if (!node->space->crypt_data) { + const ulint offset = fsp_header_get_crypt_offset( + fsp_flags_get_zip_size(flags)); node->space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); - node->space->page_0_crypt_read = true; } ut_free(buf2); @@ -1557,22 +1555,6 @@ fil_space_create( space->magic_n = FIL_SPACE_MAGIC_N; space->crypt_data = crypt_data; - /* In create table we write page 0 so we have already - "read" it and for system tablespaces we have read - crypt data at startup. */ - if (create_table || crypt_data != NULL) { - space->page_0_crypt_read = true; - } - -#ifdef UNIV_DEBUG - ib_logf(IB_LOG_LEVEL_INFO, - "Created tablespace for space %lu name %s key_id %u encryption %d.", - space->id, - space->name, - space->crypt_data ? space->crypt_data->key_id : 0, - space->crypt_data ? space->crypt_data->encryption : 0); -#endif - rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); HASH_INSERT(fil_space_t, hash, fil_system->spaces, id, space); @@ -2401,8 +2383,8 @@ fil_read_first_page( /* Possible encryption crypt data is also stored only to first page of the first datafile. */ - ulint offset = fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(*flags)); + const ulint offset = fsp_header_get_crypt_offset( + fsp_flags_get_zip_size(*flags)); cdata = fil_space_read_crypt_data(*space_id, page, offset); @@ -4018,6 +4000,7 @@ fsp_flags_try_adjust(ulint space_id, ulint flags) flags, MLOG_4BYTES, &mtr); } } + mtr_commit(&mtr); } @@ -4437,7 +4420,17 @@ cleanup_and_exit: mem_free(def.filepath); - if (err == DB_SUCCESS && !srv_read_only_mode) { + /* We need to check fsp flags when no errors has happened and + server was not started on read only mode and tablespace validation + was requested or flags contain other table options except + low order bits to FSP_FLAGS_POS_PAGE_SSIZE position. + Note that flag comparison is pessimistic. Adjust is required + only when flags contain buggy MariaDB 10.1.0 - + MariaDB 10.1.20 flags. */ + if (err == DB_SUCCESS + && !srv_read_only_mode + && (validate + || flags >= (1U << FSP_FLAGS_POS_PAGE_SSIZE))) { fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK); } diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 4f05549bc1c..878b8d824c7 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -4124,20 +4124,8 @@ ulint fsp_header_get_crypt_offset( const ulint zip_size) { - ulint pageno = 0; - /* compute first page_no that will have xdes stored on page != 0*/ - for (ulint i = 0; - (pageno = xdes_calc_descriptor_page(zip_size, i)) == 0; ) - i++; - - /* use pageno prior to this...i.e last page on page 0 */ - ut_ad(pageno > 0); - pageno--; - - ulint iv_offset = XDES_ARR_OFFSET + - XDES_SIZE * (1 + xdes_calc_descriptor_index(zip_size, pageno)); - - return FSP_HEADER_OFFSET + iv_offset; + return (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE * + (zip_size ? zip_size : UNIV_PAGE_SIZE) / FSP_EXTENT_SIZE)); } /**********************************************************************//** diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index ba440cb2a1c..e16c7cb102e 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -351,9 +351,6 @@ struct fil_space_t { compression failure */ fil_space_crypt_t* crypt_data; /*!< tablespace crypt data or NULL */ - bool page_0_crypt_read; - /*!< tablespace crypt data has been - read */ ulint file_block_size; /*!< file system block size */ diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index c72900bd082..54f3ac311be 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -4636,6 +4636,7 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) !bpage->encrypted && fil_space_verify_crypt_checksum(dst_frame, zip_size, space, bpage->offset)); + if (!still_encrypted) { /* If traditional checksums match, we assume that page is not anymore encrypted. */ diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 21c1e3b730e..e24278dd102 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -1115,6 +1115,43 @@ fil_crypt_needs_rotation( return false; } +/** Read page 0 and possible crypt data from there. +@param[in] space Tablespace */ +static inline +void +fil_crypt_read_crypt_data(fil_space_t* space) +{ + mutex_enter(&fil_system->mutex); + + /* If space does not contain crypt data and space size is 0 + we have not yet read first page of tablespace. We need to + read it to find out tablespace current encryption status. */ + if (!space->crypt_data && space->size == 0) { + mtr_t mtr; + mtr_start(&mtr); + ulint zip_size = fsp_flags_get_zip_size(space->flags); + ulint offset = fsp_header_get_crypt_offset(zip_size); + mutex_exit(&fil_system->mutex); + if (buf_block_t* block = buf_page_get(space->id, zip_size, 0, + RW_X_LATCH, &mtr)) { + byte* frame = buf_block_get_frame(block); + + mutex_enter(&fil_system->mutex); + + if (!space->crypt_data) { + space->crypt_data = fil_space_read_crypt_data(space->id, + frame, offset); + } + + mutex_exit(&fil_system->mutex); + } + + mtr_commit(&mtr); + } else { + mutex_exit(&fil_system->mutex); + } +} + /*********************************************************************** Start encrypting a space @param[in,out] space Tablespace @@ -1125,6 +1162,7 @@ fil_crypt_start_encrypting_space( fil_space_t* space) { bool recheck = false; + mutex_enter(&fil_crypt_threads_mutex); fil_space_crypt_t *crypt_data = space->crypt_data; @@ -1191,8 +1229,6 @@ fil_crypt_start_encrypting_space( byte* frame = buf_block_get_frame(block); crypt_data->type = CRYPT_SCHEME_1; crypt_data->write_page0(frame, &mtr); - - mtr_commit(&mtr); /* record lsn of update */ @@ -1620,6 +1656,13 @@ fil_crypt_find_space_to_rotate( } while (!state->should_shutdown() && state->space) { + /* If there is no crypt data and we have not yet read + page 0 for this tablespace, we need to read it before + we can continue. */ + if (!state->space->crypt_data) { + fil_crypt_read_crypt_data(state->space); + } + if (fil_crypt_space_needs_rotation(state, key_state, recheck)) { ut_ad(key_state->key_id); /* init state->min_key_version_found before @@ -2314,8 +2357,10 @@ DECLARE_THREAD(fil_crypt_thread)( while (!thr.should_shutdown() && fil_crypt_find_page_to_rotate(&new_state, &thr)) { - /* rotate a (set) of pages */ - fil_crypt_rotate_pages(&new_state, &thr); + if (!thr.space->is_stopping()) { + /* rotate a (set) of pages */ + fil_crypt_rotate_pages(&new_state, &thr); + } /* If space is marked as stopping, release space and stop rotation. */ @@ -2545,6 +2590,14 @@ fil_space_crypt_get_status( memset(status, 0, sizeof(*status)); ut_ad(space->n_pending_ops > 0); + + /* If there is no crypt data and we have not yet read + page 0 for this tablespace, we need to read it before + we can continue. */ + if (!space->crypt_data) { + fil_crypt_read_crypt_data(const_cast(space)); + } + fil_space_crypt_t* crypt_data = space->crypt_data; status->space = space->id; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index d979c05c9a6..12048bc479f 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -661,12 +661,10 @@ fil_node_open_file( /* Try to read crypt_data from page 0 if it is not yet read. */ - if (!node->space->page_0_crypt_read) { - ulint offset = fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(flags)); - ut_ad(node->space->crypt_data == NULL); + if (!node->space->crypt_data) { + const ulint offset = fsp_header_get_crypt_offset( + fsp_flags_get_zip_size(flags)); node->space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); - node->space->page_0_crypt_read = true; } ut_free(buf2); @@ -1600,22 +1598,6 @@ fil_space_create( space->magic_n = FIL_SPACE_MAGIC_N; space->crypt_data = crypt_data; - /* In create table we write page 0 so we have already - "read" it and for system tablespaces we have read - crypt data at startup. */ - if (create_table || crypt_data != NULL) { - space->page_0_crypt_read = true; - } - -#ifdef UNIV_DEBUG - ib_logf(IB_LOG_LEVEL_INFO, - "Created tablespace for space %lu name %s key_id %u encryption %d.", - space->id, - space->name, - space->crypt_data ? space->crypt_data->key_id : 0, - space->crypt_data ? space->crypt_data->encryption : 0); -#endif - rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); HASH_INSERT(fil_space_t, hash, fil_system->spaces, id, space); @@ -2463,8 +2445,8 @@ fil_read_first_page( /* Possible encryption crypt data is also stored only to first page of the first datafile. */ - ulint offset = fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(*flags)); + const ulint offset = fsp_header_get_crypt_offset( + fsp_flags_get_zip_size(*flags)); cdata = fil_space_read_crypt_data(*space_id, page, offset); @@ -4211,6 +4193,7 @@ fsp_flags_try_adjust(ulint space_id, ulint flags) flags, MLOG_4BYTES, &mtr); } } + mtr_commit(&mtr); } @@ -4631,7 +4614,17 @@ cleanup_and_exit: mem_free(def.filepath); - if (err == DB_SUCCESS && !srv_read_only_mode) { + /* We need to check fsp flags when no errors has happened and + server was not started on read only mode and tablespace validation + was requested or flags contain other table options except + low order bits to FSP_FLAGS_POS_PAGE_SSIZE position. + Note that flag comparison is pessimistic. Adjust is required + only when flags contain buggy MariaDB 10.1.0 - + MariaDB 10.1.20 flags. */ + if (err == DB_SUCCESS + && !srv_read_only_mode + && (validate + || flags >= (1U << FSP_FLAGS_POS_PAGE_SSIZE))) { fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK); } diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index bd87b88f58d..40a9faa6914 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -4150,20 +4150,8 @@ ulint fsp_header_get_crypt_offset( const ulint zip_size) { - ulint pageno = 0; - /* compute first page_no that will have xdes stored on page != 0*/ - for (ulint i = 0; - (pageno = xdes_calc_descriptor_page(zip_size, i)) == 0; ) - i++; - - /* use pageno prior to this...i.e last page on page 0 */ - ut_ad(pageno > 0); - pageno--; - - ulint iv_offset = XDES_ARR_OFFSET + - XDES_SIZE * (1 + xdes_calc_descriptor_index(zip_size, pageno)); - - return FSP_HEADER_OFFSET + iv_offset; + return (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE * + (zip_size ? zip_size : UNIV_PAGE_SIZE) / FSP_EXTENT_SIZE)); } /**********************************************************************//** diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index b861225f562..a09833c3a73 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -350,9 +350,6 @@ struct fil_space_t { compression failure */ fil_space_crypt_t* crypt_data; /*!< tablespace crypt data or NULL */ - bool page_0_crypt_read; - /*!< tablespace crypt data has been - read */ ulint file_block_size; /*!< file system block size */ From a9117c90083d3f26e43c460073e06f9562b66f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 9 Jun 2017 13:44:04 +0300 Subject: [PATCH 3/9] Correct a merge error of MDEV-11626 --- storage/xtradb/buf/buf0buf.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 26b95926e86..1b031f5eabc 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -3201,6 +3201,7 @@ got_block: } if (buf_flush_page_try(buf_pool, fix_block)) { + guess = fix_block; goto loop; } From 417434f12dba3ccf5b79ca4e7afe72fe27807fd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 8 Jun 2017 15:43:06 +0300 Subject: [PATCH 4/9] MDEV-13039 innodb_fast_shutdown=0 may fail to purge all undo log When a slow shutdown is performed soon after spawning some work for background threads that can create or commit transactions, it is possible that new transactions are started or committed after the purge has finished. This is violating the specification of innodb_fast_shutdown=0, namely that the purge must be completed. (None of the history of the recent transactions would be purged.) Also, it is possible that the purge threads would exit in slow shutdown while there exist active transactions, such as recovered incomplete transactions that are being rolled back. Thus, the slow shutdown could fail to purge some undo log that becomes purgeable after the transaction commit or rollback. srv_undo_sources: A flag that indicates if undo log can be generated or the persistent, whether by background threads or by user SQL. Even when this flag is clear, active transactions that already exist in the system may be committed or rolled back. innodb_shutdown(): Renamed from innobase_shutdown_for_mysql(). Do not return an error code; the operation never fails. Clear the srv_undo_sources flag, and also ensure that the background DROP TABLE queue is empty. srv_purge_should_exit(): Do not allow the purge to exit if srv_undo_sources are active or the background DROP TABLE queue is not empty, or in slow shutdown, if any active transactions exist (and are being rolled back). srv_purge_coordinator_thread(): Remove some previous workarounds for this bug. innobase_start_or_create_for_mysql(): Set buf_page_cleaner_is_active and srv_dict_stats_thread_active directly. Set srv_undo_sources before starting the purge subsystem, to prevent immediate shutdown of the purge. Create dict_stats_thread and fts_optimize_thread immediately after setting srv_undo_sources, so that shutdown can use this flag to determine if these subsystems were started. dict_stats_shutdown(): Shut down dict_stats_thread. Backported from 10.2. srv_shutdown_table_bg_threads(): Remove (unused). --- .../innodb/r/row_format_redundant.result | 48 +++++++ .../suite/innodb/t/row_format_redundant.test | 65 +++++++++ storage/innobase/buf/buf0flu.cc | 8 +- storage/innobase/dict/dict0stats_bg.cc | 35 +++-- storage/innobase/handler/ha_innodb.cc | 29 ++-- storage/innobase/include/buf0flu.h | 2 +- storage/innobase/include/dict0stats_bg.h | 6 +- storage/innobase/include/srv0srv.h | 2 +- storage/innobase/include/srv0start.h | 20 +-- storage/innobase/srv/srv0srv.cc | 71 +++------- storage/innobase/srv/srv0start.cc | 128 ++++-------------- storage/innobase/trx/trx0purge.cc | 13 ++ storage/xtradb/buf/buf0flu.cc | 8 +- storage/xtradb/dict/dict0stats_bg.cc | 35 +++-- storage/xtradb/handler/ha_innodb.cc | 29 ++-- storage/xtradb/include/buf0flu.h | 2 +- storage/xtradb/include/dict0stats_bg.h | 6 +- storage/xtradb/include/srv0srv.h | 2 +- storage/xtradb/include/srv0start.h | 20 +-- storage/xtradb/srv/srv0srv.cc | 71 +++------- storage/xtradb/srv/srv0start.cc | 128 ++++-------------- storage/xtradb/trx/trx0purge.cc | 13 ++ 22 files changed, 339 insertions(+), 402 deletions(-) create mode 100644 mysql-test/suite/innodb/r/row_format_redundant.result create mode 100644 mysql-test/suite/innodb/t/row_format_redundant.test diff --git a/mysql-test/suite/innodb/r/row_format_redundant.result b/mysql-test/suite/innodb/r/row_format_redundant.result new file mode 100644 index 00000000000..db31c32559f --- /dev/null +++ b/mysql-test/suite/innodb/r/row_format_redundant.result @@ -0,0 +1,48 @@ +create table t1 (a int not null, d varchar(15) not null, b +varchar(198) not null, c char(156), +fulltext ftsic(c)) engine=InnoDB +row_format=redundant; +insert into t1 values(123, 'abcdef', 'jghikl', 'mnop'); +insert into t1 values(456, 'abcdef', 'jghikl', 'mnop'); +insert into t1 values(789, 'abcdef', 'jghikl', 'mnop'); +insert into t1 values(134, 'kasdfsdsadf', 'adfjlasdkfjasd', 'adfsadflkasdasdfljasdf'); +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +SET GLOBAL innodb_file_per_table=OFF; +create table t2 (a int not null, d varchar(15) not null, b +varchar(198) not null, c char(156), fulltext ftsic(c)) engine=InnoDB +row_format=redundant; +insert into t2 select * from t1; +create table t3 (a int not null, d varchar(15) not null, b varchar(198), +c varchar(150), index k1(c(99), b(56)), index k2(b(5), c(10))) engine=InnoDB +row_format=redundant; +insert into t3 values(444, 'dddd', 'bbbbb', 'aaaaa'); +insert into t3 values(555, 'eeee', 'ccccc', 'aaaaa'); +SET GLOBAL innodb_fast_shutdown=0; +SELECT COUNT(*) FROM t1; +COUNT(*) +4096 +SELECT COUNT(*) FROM t2; +COUNT(*) +4096 +SELECT COUNT(*) FROM t3; +COUNT(*) +2 +TRUNCATE TABLE t1; +ERROR HY000: Table 't1' is read only +TRUNCATE TABLE t2; +ERROR HY000: Table 't2' is read only +TRUNCATE TABLE t3; +ERROR HY000: Table 't3' is read only +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +TRUNCATE TABLE t3; +DROP TABLE t1,t2,t3; diff --git a/mysql-test/suite/innodb/t/row_format_redundant.test b/mysql-test/suite/innodb/t/row_format_redundant.test new file mode 100644 index 00000000000..b17b365651a --- /dev/null +++ b/mysql-test/suite/innodb/t/row_format_redundant.test @@ -0,0 +1,65 @@ +--source include/innodb_page_size.inc +# Embedded mode doesn't allow restarting +--source include/not_embedded.inc + +create table t1 (a int not null, d varchar(15) not null, b +varchar(198) not null, c char(156), +fulltext ftsic(c)) engine=InnoDB +row_format=redundant; + +insert into t1 values(123, 'abcdef', 'jghikl', 'mnop'); +insert into t1 values(456, 'abcdef', 'jghikl', 'mnop'); +insert into t1 values(789, 'abcdef', 'jghikl', 'mnop'); +insert into t1 values(134, 'kasdfsdsadf', 'adfjlasdkfjasd', 'adfsadflkasdasdfljasdf'); +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; +insert into t1 select * from t1; + +SET GLOBAL innodb_file_per_table=OFF; +create table t2 (a int not null, d varchar(15) not null, b +varchar(198) not null, c char(156), fulltext ftsic(c)) engine=InnoDB +row_format=redundant; + +insert into t2 select * from t1; + +create table t3 (a int not null, d varchar(15) not null, b varchar(198), +c varchar(150), index k1(c(99), b(56)), index k2(b(5), c(10))) engine=InnoDB +row_format=redundant; + +insert into t3 values(444, 'dddd', 'bbbbb', 'aaaaa'); +insert into t3 values(555, 'eeee', 'ccccc', 'aaaaa'); + +# read-only restart requires the change buffer to be empty; therefore we +# do a slow shutdown. +SET GLOBAL innodb_fast_shutdown=0; +--let $restart_parameters = --innodb-read-only +--source include/restart_mysqld.inc + +SELECT COUNT(*) FROM t1; +SELECT COUNT(*) FROM t2; +SELECT COUNT(*) FROM t3; + +--error ER_OPEN_AS_READONLY +TRUNCATE TABLE t1; +--error ER_OPEN_AS_READONLY +TRUNCATE TABLE t2; +--error ER_OPEN_AS_READONLY +TRUNCATE TABLE t3; + +--let $restart_parameters = +--source include/restart_mysqld.inc + +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +TRUNCATE TABLE t3; + +# TODO: Shutdown, corrupt the SYS_TABLES.TYPE of the tables, restart + +DROP TABLE t1,t2,t3; diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 486b7b4db0c..643c5287e17 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -58,7 +58,7 @@ is set to TRUE by the page_cleaner thread when it is spawned and is set back to FALSE at shutdown by the page_cleaner as well. Therefore no need to protect it by a mutex. It is only ever read by the thread doing the shutdown */ -UNIV_INTERN ibool buf_page_cleaner_is_active = FALSE; +UNIV_INTERN bool buf_page_cleaner_is_active; /** LRU flush batch is further divided into this chunk size to reduce the wait time for the threads waiting for a clean block */ @@ -2422,8 +2422,6 @@ DECLARE_THREAD(buf_flush_page_cleaner_thread)( os_thread_pf(os_thread_get_curr_id())); #endif /* UNIV_DEBUG_THREAD_CREATION */ - buf_page_cleaner_is_active = TRUE; - while (srv_shutdown_state == SRV_SHUTDOWN_NONE) { page_cleaner_sleep_if_needed(next_loop_time); @@ -2517,7 +2515,7 @@ DECLARE_THREAD(buf_flush_page_cleaner_thread)( /* We have lived our life. Time to die. */ thread_exit: - buf_page_cleaner_is_active = FALSE; + buf_page_cleaner_is_active = false; my_thread_end(); /* We count the number of threads in os_thread_exit(). A created diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 40a2a1488f8..75214a1b2e9 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -38,12 +38,18 @@ Created Apr 25, 2012 Vasil Dimov /** Minimum time interval between stats recalc for a given table */ #define MIN_RECALC_INTERVAL 10 /* seconds */ -#define SHUTTING_DOWN() (srv_shutdown_state != SRV_SHUTDOWN_NONE) - /** Event to wake up dict_stats_thread on dict_stats_recalc_pool_add() or shutdown. Not protected by any mutex. */ UNIV_INTERN os_event_t dict_stats_event; +/** Variable to initiate shutdown the dict stats thread. Note we don't +use 'srv_shutdown_state' because we want to shutdown dict stats thread +before purge thread. */ +static bool dict_stats_start_shutdown; + +/** Event to wait for shutdown of the dict stats thread */ +static os_event_t dict_stats_shutdown_event; + /** This mutex protects the "recalc_pool" variable. */ static ib_mutex_t recalc_pool_mutex; #ifdef HAVE_PSI_INTERFACE @@ -217,11 +223,11 @@ Must be called before dict_stats_thread() is started. */ UNIV_INTERN void dict_stats_thread_init() -/*====================*/ { ut_a(!srv_read_only_mode); dict_stats_event = os_event_create(); + dict_stats_shutdown_event = os_event_create(); /* The recalc_pool_mutex is acquired from: 1) the background stats gathering thread before any other latch @@ -260,6 +266,9 @@ dict_stats_thread_deinit() os_event_free(dict_stats_event); dict_stats_event = NULL; + os_event_free(dict_stats_shutdown_event); + dict_stats_shutdown_event = NULL; + dict_stats_start_shutdown = false; } /*****************************************************************//** @@ -349,9 +358,7 @@ DECLARE_THREAD(dict_stats_thread)( my_thread_init(); ut_a(!srv_read_only_mode); - srv_dict_stats_thread_active = TRUE; - - while (!SHUTTING_DOWN()) { + while (!dict_stats_start_shutdown) { /* Wake up periodically even if not signaled. This is because we may lose an event - if the below call to @@ -361,7 +368,7 @@ DECLARE_THREAD(dict_stats_thread)( os_event_wait_time( dict_stats_event, MIN_RECALC_INTERVAL * 1000000); - if (SHUTTING_DOWN()) { + if (dict_stats_start_shutdown) { break; } @@ -370,8 +377,9 @@ DECLARE_THREAD(dict_stats_thread)( os_event_reset(dict_stats_event); } - srv_dict_stats_thread_active = FALSE; + srv_dict_stats_thread_active = false; + os_event_set(dict_stats_shutdown_event); my_thread_end(); /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit instead of return(). */ @@ -379,3 +387,12 @@ DECLARE_THREAD(dict_stats_thread)( OS_THREAD_DUMMY_RETURN; } + +/** Shut down the dict_stats_thread. */ +void +dict_stats_shutdown() +{ + dict_stats_start_shutdown = true; + os_event_set(dict_stats_event); + os_event_wait(dict_stats_shutdown_event); +} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 091f660dcee..e77c9bb4caf 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1142,14 +1142,11 @@ innobase_drop_database( the path is used as the database name: for example, in 'mysql/data/test' the database name is 'test' */ -/*******************************************************************//** -Closes an InnoDB database. */ +/** Shut down the InnoDB storage engine. +@return 0 */ static int -innobase_end( -/*=========*/ - handlerton* hton, /* in: Innodb handlerton */ - ha_panic_function type); +innobase_end(handlerton*, ha_panic_function); /*****************************************************************//** Creates an InnoDB transaction struct for the thd if it does not yet have one. @@ -3651,21 +3648,13 @@ error: DBUG_RETURN(TRUE); } -/*******************************************************************//** -Closes an InnoDB database. -@return TRUE if error */ +/** Shut down the InnoDB storage engine. +@return 0 */ static int -innobase_end( -/*=========*/ - handlerton* hton, /*!< in/out: InnoDB handlerton */ - ha_panic_function type MY_ATTRIBUTE((unused))) - /*!< in: ha_panic() parameter */ +innobase_end(handlerton*, ha_panic_function) { - int err= 0; - DBUG_ENTER("innobase_end"); - DBUG_ASSERT(hton == innodb_hton_ptr); if (innodb_inited) { @@ -3682,9 +3671,7 @@ innobase_end( innodb_inited = 0; hash_table_free(innobase_open_tables); innobase_open_tables = NULL; - if (innobase_shutdown_for_mysql() != DB_SUCCESS) { - err = 1; - } + innodb_shutdown(); srv_free_paths_and_sizes(); my_free(internal_innobase_data_file_path); mysql_mutex_destroy(&innobase_share_mutex); @@ -3693,7 +3680,7 @@ innobase_end( mysql_mutex_destroy(&pending_checkpoint_mutex); } - DBUG_RETURN(err); + DBUG_RETURN(0); } /****************************************************************//** diff --git a/storage/innobase/include/buf0flu.h b/storage/innobase/include/buf0flu.h index 0c5812f802c..4a0781ffeda 100644 --- a/storage/innobase/include/buf0flu.h +++ b/storage/innobase/include/buf0flu.h @@ -34,7 +34,7 @@ Created 11/5/1995 Heikki Tuuri #include "buf0types.h" /** Flag indicating if the page_cleaner is in active state. */ -extern ibool buf_page_cleaner_is_active; +extern bool buf_page_cleaner_is_active; /********************************************************************//** Remove a block from the flush list of modified blocks. */ diff --git a/storage/innobase/include/dict0stats_bg.h b/storage/innobase/include/dict0stats_bg.h index 1973380d197..2bd441d5f5e 100644 --- a/storage/innobase/include/dict0stats_bg.h +++ b/storage/innobase/include/dict0stats_bg.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -122,6 +122,10 @@ DECLARE_THREAD(dict_stats_thread)( void* arg); /*!< in: a dummy parameter required by os_thread_create */ +/** Shut down the dict_stats_thread. */ +void +dict_stats_shutdown(); + # ifndef UNIV_NONINL # include "dict0stats_bg.ic" # endif diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 696d3892db2..190397595ee 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -398,7 +398,7 @@ extern ibool srv_error_monitor_active; extern ibool srv_buf_dump_thread_active; /* TRUE during the lifetime of the stats thread */ -extern ibool srv_dict_stats_thread_active; +extern bool srv_dict_stats_thread_active; extern ulong srv_n_spin_wait_rounds; extern ulong srv_n_free_tickets_to_enter; diff --git a/storage/innobase/include/srv0start.h b/storage/innobase/include/srv0start.h index a60776a4665..b055a9d834f 100644 --- a/storage/innobase/include/srv0start.h +++ b/storage/innobase/include/srv0start.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -75,22 +76,12 @@ are not found and the user wants. @return DB_SUCCESS or error code */ UNIV_INTERN dberr_t -innobase_start_or_create_for_mysql(void); -/*====================================*/ -/****************************************************************//** -Shuts down the Innobase database. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -innobase_shutdown_for_mysql(void); +innobase_start_or_create_for_mysql(); -/******************************************************************** -Signal all per-table background threads to shutdown, and wait for them to do -so. */ +/** Shut down InnoDB. */ UNIV_INTERN void -srv_shutdown_table_bg_threads(void); -/*=============================*/ +innodb_shutdown(); /*************************************************************//** Copy the file path component of the physical file to parameter. It will @@ -158,6 +149,9 @@ enum srv_shutdown_state { SRV_SHUTDOWN_EXIT_THREADS/*!< Exit all threads */ }; +/** Whether any undo log records can be generated */ +extern bool srv_undo_sources; + /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ extern enum srv_shutdown_state srv_shutdown_state; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index bad1579070c..10baf546e5b 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -84,7 +84,7 @@ UNIV_INTERN ibool srv_error_monitor_active = FALSE; UNIV_INTERN ibool srv_buf_dump_thread_active = FALSE; -UNIV_INTERN ibool srv_dict_stats_thread_active = FALSE; +UNIV_INTERN bool srv_dict_stats_thread_active; UNIV_INTERN const char* srv_main_thread_op_info = ""; @@ -2424,31 +2424,29 @@ suspend_thread: goto loop; } -/*********************************************************************//** -Check if purge should stop. -@return true if it should shutdown. */ +/** Check if purge should stop. +@param[in] n_purged pages purged in the last batch +@return whether purge should exit */ static bool -srv_purge_should_exit( -/*==============*/ - ulint n_purged) /*!< in: pages purged in last batch */ +srv_purge_should_exit(ulint n_purged) { - switch (srv_shutdown_state) { - case SRV_SHUTDOWN_NONE: - /* Normal operation. */ - break; + ut_ad(srv_shutdown_state == SRV_SHUTDOWN_NONE + || srv_shutdown_state == SRV_SHUTDOWN_CLEANUP); - case SRV_SHUTDOWN_CLEANUP: - case SRV_SHUTDOWN_EXIT_THREADS: - /* Exit unless slow shutdown requested or all done. */ - return(srv_fast_shutdown != 0 || n_purged == 0); - - case SRV_SHUTDOWN_LAST_PHASE: - case SRV_SHUTDOWN_FLUSH_PHASE: - ut_error; + if (srv_undo_sources) { + return(false); } - - return(false); + if (srv_fast_shutdown) { + return(true); + } + /* Slow shutdown was requested. */ + if (n_purged) { + /* The previous round still did some work. */ + return(false); + } + /* Exit if there are no active transactions to roll back. */ + return(trx_sys_any_active_transactions() == 0); } /*********************************************************************//** @@ -2709,7 +2707,7 @@ srv_purge_coordinator_suspend( } rw_lock_x_unlock(&purge_sys->latch); - } while (stop); + } while (stop && srv_undo_sources); srv_resume_thread(slot, 0, false); } @@ -2760,6 +2758,7 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( purge didn't purge any records then wait for activity. */ if (srv_shutdown_state == SRV_SHUTDOWN_NONE + && srv_undo_sources && (purge_sys->state == PURGE_STATE_STOP || n_total_purged == 0)) { @@ -2776,36 +2775,8 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( rseg_history_len = srv_do_purge( srv_n_purge_threads, &n_total_purged); - } while (!srv_purge_should_exit(n_total_purged)); - /* Ensure that we don't jump out of the loop unless the - exit condition is satisfied. */ - - ut_a(srv_purge_should_exit(n_total_purged)); - - ulint n_pages_purged = ULINT_MAX; - - /* Ensure that all records are purged if it is not a fast shutdown. - This covers the case where a record can be added after we exit the - loop above. */ - while (srv_fast_shutdown == 0 && n_pages_purged > 0) { - n_pages_purged = trx_purge(1, srv_purge_batch_size, false); - } - - /* This trx_purge is called to remove any undo records (added by - background threads) after completion of the above loop. When - srv_fast_shutdown != 0, a large batch size can cause significant - delay in shutdown ,so reducing the batch size to magic number 20 - (which was default in 5.5), which we hope will be sufficient to - remove all the undo records */ - const uint temp_batch_size = 20; - - n_pages_purged = trx_purge(1, srv_purge_batch_size <= temp_batch_size - ? srv_purge_batch_size : temp_batch_size, - true); - ut_a(n_pages_purged == 0 || srv_fast_shutdown != 0); - /* The task queue should always be empty, independent of fast shutdown state. */ ut_a(srv_get_task_queue_length() == 0); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index c70260ff1a3..789fe50d337 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -121,7 +121,10 @@ UNIV_INTERN ibool srv_is_being_started = FALSE; /** TRUE if the server was successfully started */ UNIV_INTERN ibool srv_was_started = FALSE; /** TRUE if innobase_start_or_create_for_mysql() has been called */ -static ibool srv_start_has_been_called = FALSE; +static ibool srv_start_has_been_called; + +/** Whether any undo log records can be generated */ +UNIV_INTERN bool srv_undo_sources; /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ @@ -1565,8 +1568,7 @@ are not found and the user wants. @return DB_SUCCESS or error code */ UNIV_INTERN dberr_t -innobase_start_or_create_for_mysql(void) -/*====================================*/ +innobase_start_or_create_for_mysql() { ibool create_new_db; lsn_t min_flushed_lsn; @@ -2705,8 +2707,8 @@ files_checked: } } - srv_startup_is_before_trx_rollback_phase = FALSE; recv_recovery_rollback_active(); + srv_startup_is_before_trx_rollback_phase = FALSE; /* It is possible that file_format tag has never been set. In this case we initialize it to minimum @@ -2827,6 +2829,16 @@ files_checked: srv_master_thread, NULL, thread_ids + (1 + SRV_MAX_N_IO_THREADS)); thread_started[1 + SRV_MAX_N_IO_THREADS] = true; + + srv_undo_sources = true; + /* Create the dict stats gathering thread */ + srv_dict_stats_thread_active = true; + dict_stats_thread_handle = os_thread_create( + dict_stats_thread, NULL, NULL); + dict_stats_thread_started = true; + + /* Create the thread that will optimize the FTS sub-system. */ + fts_optimize_init(); } if (!srv_read_only_mode @@ -2856,6 +2868,7 @@ files_checked: } if (!srv_read_only_mode) { + buf_page_cleaner_is_active = true; buf_flush_page_cleaner_thread_handle = os_thread_create(buf_flush_page_cleaner_thread, NULL, NULL); buf_flush_page_cleaner_thread_started = true; } @@ -2885,13 +2898,6 @@ files_checked: /* Create the buffer pool dump/load thread */ buf_dump_thread_handle = os_thread_create(buf_dump_thread, NULL, NULL); buf_dump_thread_started = true; - - /* Create the dict stats gathering thread */ - dict_stats_thread_handle = os_thread_create(dict_stats_thread, NULL, NULL); - dict_stats_thread_started = true; - - /* Create the thread that will optimize the FTS sub-system. */ - fts_optimize_init(); } srv_was_started = TRUE; @@ -2929,13 +2935,10 @@ srv_fts_close(void) } #endif -/****************************************************************//** -Shuts down the InnoDB database. -@return DB_SUCCESS or error code */ +/** Shut down InnoDB. */ UNIV_INTERN -dberr_t -innobase_shutdown_for_mysql(void) -/*=============================*/ +void +innodb_shutdown() { ulint i; @@ -2945,15 +2948,20 @@ innobase_shutdown_for_mysql(void) "Shutting down an improperly started, " "or created database!"); } - - return(DB_SUCCESS); } - if (!srv_read_only_mode) { + if (srv_undo_sources) { + ut_ad(!srv_read_only_mode); /* Shutdown the FTS optimize sub system. */ fts_optimize_start_shutdown(); fts_optimize_end(); + dict_stats_shutdown(); + while (row_get_background_drop_list_len_low()) { + srv_wake_master_thread(); + os_thread_yield(); + } + srv_undo_sources = false; } /* 1. Flush the buffer pool to disk, write the current lsn to @@ -3156,89 +3164,9 @@ innobase_shutdown_for_mysql(void) srv_was_started = FALSE; srv_start_has_been_called = FALSE; - - return(DB_SUCCESS); } #endif /* !UNIV_HOTBACKUP */ - -/******************************************************************** -Signal all per-table background threads to shutdown, and wait for them to do -so. */ -UNIV_INTERN -void -srv_shutdown_table_bg_threads(void) -/*===============================*/ -{ - dict_table_t* table; - dict_table_t* first; - dict_table_t* last = NULL; - - mutex_enter(&dict_sys->mutex); - - /* Signal all threads that they should stop. */ - table = UT_LIST_GET_FIRST(dict_sys->table_LRU); - first = table; - while (table) { - dict_table_t* next; - fts_t* fts = table->fts; - - if (fts != NULL) { - fts_start_shutdown(table, fts); - } - - next = UT_LIST_GET_NEXT(table_LRU, table); - - if (!next) { - last = table; - } - - table = next; - } - - /* We must release dict_sys->mutex here; if we hold on to it in the - loop below, we will deadlock if any of the background threads try to - acquire it (for example, the FTS thread by calling que_eval_sql). - - Releasing it here and going through dict_sys->table_LRU without - holding it is safe because: - - a) MySQL only starts the shutdown procedure after all client - threads have been disconnected and no new ones are accepted, so no - new tables are added or old ones dropped. - - b) Despite its name, the list is not LRU, and the order stays - fixed. - - To safeguard against the above assumptions ever changing, we store - the first and last items in the list above, and then check that - they've stayed the same below. */ - - mutex_exit(&dict_sys->mutex); - - /* Wait for the threads of each table to stop. This is not inside - the above loop, because by signaling all the threads first we can - overlap their shutting down delays. */ - table = UT_LIST_GET_FIRST(dict_sys->table_LRU); - ut_a(first == table); - while (table) { - dict_table_t* next; - fts_t* fts = table->fts; - - if (fts != NULL) { - fts_shutdown(table, fts); - } - - next = UT_LIST_GET_NEXT(table_LRU, table); - - if (table == last) { - ut_a(!next); - } - - table = next; - } -} - /*****************************************************************//** Get the meta-data filename from the table name. */ UNIV_INTERN diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 8aae2969b39..cfe6309c36b 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -243,6 +243,19 @@ trx_purge_add_update_undo_to_history( hist_size + undo->size, MLOG_4BYTES, mtr); } + /* Before any transaction-generating background threads or the + purge have been started, recv_recovery_rollback_active() can + start transactions in row_merge_drop_temp_indexes() and + fts_drop_orphaned_tables(), and roll back recovered transactions. + After the purge thread has been given permission to exit, + in fast shutdown, we may roll back transactions (trx->undo_no==0) + in THD::cleanup() invoked from unlink_thd(). */ + ut_ad(srv_undo_sources + || ((srv_startup_is_before_trx_rollback_phase + || trx_rollback_or_clean_is_active) + && purge_sys->state == PURGE_STATE_INIT) + || (trx->undo_no == 0 && srv_fast_shutdown)); + /* Add the log as the first in the history list */ flst_add_first(rseg_header + TRX_RSEG_HISTORY, undo_header + TRX_UNDO_HISTORY_NODE, mtr); diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 4781b874032..955be5a6161 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -58,7 +58,7 @@ is set to TRUE by the page_cleaner thread when it is spawned and is set back to FALSE at shutdown by the page_cleaner as well. Therefore no need to protect it by a mutex. It is only ever read by the thread doing the shutdown */ -UNIV_INTERN ibool buf_page_cleaner_is_active = FALSE; +UNIV_INTERN bool buf_page_cleaner_is_active; /** Flag indicating if the lru_manager is in active state. */ UNIV_INTERN bool buf_lru_manager_is_active = false; @@ -2718,8 +2718,6 @@ DECLARE_THREAD(buf_flush_page_cleaner_thread)( os_thread_pf(os_thread_get_curr_id())); #endif /* UNIV_DEBUG_THREAD_CREATION */ - buf_page_cleaner_is_active = TRUE; - while (srv_shutdown_state == SRV_SHUTDOWN_NONE) { ulint page_cleaner_sleep_time; @@ -2829,7 +2827,7 @@ DECLARE_THREAD(buf_flush_page_cleaner_thread)( /* We have lived our life. Time to die. */ thread_exit: - buf_page_cleaner_is_active = FALSE; + buf_page_cleaner_is_active = false; my_thread_end(); /* We count the number of threads in os_thread_exit(). A created diff --git a/storage/xtradb/dict/dict0stats_bg.cc b/storage/xtradb/dict/dict0stats_bg.cc index 8cb419c6c37..4d646ed3081 100644 --- a/storage/xtradb/dict/dict0stats_bg.cc +++ b/storage/xtradb/dict/dict0stats_bg.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -38,12 +38,18 @@ Created Apr 25, 2012 Vasil Dimov /** Minimum time interval between stats recalc for a given table */ #define MIN_RECALC_INTERVAL 10 /* seconds */ -#define SHUTTING_DOWN() (srv_shutdown_state != SRV_SHUTDOWN_NONE) - /** Event to wake up dict_stats_thread on dict_stats_recalc_pool_add() or shutdown. Not protected by any mutex. */ UNIV_INTERN os_event_t dict_stats_event; +/** Variable to initiate shutdown the dict stats thread. Note we don't +use 'srv_shutdown_state' because we want to shutdown dict stats thread +before purge thread. */ +static bool dict_stats_start_shutdown; + +/** Event to wait for shutdown of the dict stats thread */ +static os_event_t dict_stats_shutdown_event; + /** This mutex protects the "recalc_pool" variable. */ static ib_mutex_t recalc_pool_mutex; #ifdef HAVE_PSI_INTERFACE @@ -207,11 +213,11 @@ Must be called before dict_stats_thread() is started. */ UNIV_INTERN void dict_stats_thread_init() -/*====================*/ { ut_a(!srv_read_only_mode); dict_stats_event = os_event_create(); + dict_stats_shutdown_event = os_event_create(); /* The recalc_pool_mutex is acquired from: 1) the background stats gathering thread before any other latch @@ -250,6 +256,9 @@ dict_stats_thread_deinit() os_event_free(dict_stats_event); dict_stats_event = NULL; + os_event_free(dict_stats_shutdown_event); + dict_stats_shutdown_event = NULL; + dict_stats_start_shutdown = false; } /*****************************************************************//** @@ -339,9 +348,7 @@ DECLARE_THREAD(dict_stats_thread)( my_thread_init(); ut_a(!srv_read_only_mode); - srv_dict_stats_thread_active = TRUE; - - while (!SHUTTING_DOWN()) { + while (!dict_stats_start_shutdown) { /* Wake up periodically even if not signaled. This is because we may lose an event - if the below call to @@ -351,7 +358,7 @@ DECLARE_THREAD(dict_stats_thread)( os_event_wait_time( dict_stats_event, MIN_RECALC_INTERVAL * 1000000); - if (SHUTTING_DOWN()) { + if (dict_stats_start_shutdown) { break; } @@ -360,8 +367,9 @@ DECLARE_THREAD(dict_stats_thread)( os_event_reset(dict_stats_event); } - srv_dict_stats_thread_active = FALSE; + srv_dict_stats_thread_active = false; + os_event_set(dict_stats_shutdown_event); my_thread_end(); /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit instead of return(). */ @@ -369,3 +377,12 @@ DECLARE_THREAD(dict_stats_thread)( OS_THREAD_DUMMY_RETURN; } + +/** Shut down the dict_stats_thread. */ +void +dict_stats_shutdown() +{ + dict_stats_start_shutdown = true; + os_event_set(dict_stats_event); + os_event_wait(dict_stats_shutdown_event); +} diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 7d1ed3da5fd..9717136e030 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1366,14 +1366,11 @@ innobase_drop_database( the path is used as the database name: for example, in 'mysql/data/test' the database name is 'test' */ -/*******************************************************************//** -Closes an InnoDB database. */ +/** Shut down the InnoDB storage engine. +@return 0 */ static int -innobase_end( -/*=========*/ - handlerton* hton, /* in: Innodb handlerton */ - ha_panic_function type); +innobase_end(handlerton*, ha_panic_function); #if NOT_USED /*****************************************************************//** @@ -4140,21 +4137,13 @@ error: DBUG_RETURN(TRUE); } -/*******************************************************************//** -Closes an InnoDB database. -@return TRUE if error */ +/** Shut down the InnoDB storage engine. +@return 0 */ static int -innobase_end( -/*=========*/ - handlerton* hton, /*!< in/out: InnoDB handlerton */ - ha_panic_function type MY_ATTRIBUTE((unused))) - /*!< in: ha_panic() parameter */ +innobase_end(handlerton*, ha_panic_function) { - int err= 0; - DBUG_ENTER("innobase_end"); - DBUG_ASSERT(hton == innodb_hton_ptr); if (innodb_inited) { @@ -4171,9 +4160,7 @@ innobase_end( innodb_inited = 0; hash_table_free(innobase_open_tables); innobase_open_tables = NULL; - if (innobase_shutdown_for_mysql() != DB_SUCCESS) { - err = 1; - } + innodb_shutdown(); srv_free_paths_and_sizes(); my_free(internal_innobase_data_file_path); mysql_mutex_destroy(&innobase_share_mutex); @@ -4182,7 +4169,7 @@ innobase_end( mysql_mutex_destroy(&pending_checkpoint_mutex); } - DBUG_RETURN(err); + DBUG_RETURN(0); } /****************************************************************//** diff --git a/storage/xtradb/include/buf0flu.h b/storage/xtradb/include/buf0flu.h index 96035452e6e..384014939ae 100644 --- a/storage/xtradb/include/buf0flu.h +++ b/storage/xtradb/include/buf0flu.h @@ -34,7 +34,7 @@ Created 11/5/1995 Heikki Tuuri #include "buf0types.h" /** Flag indicating if the page_cleaner is in active state. */ -extern ibool buf_page_cleaner_is_active; +extern bool buf_page_cleaner_is_active; /** Flag indicating if the lru_manager is in active state. */ extern bool buf_lru_manager_is_active; diff --git a/storage/xtradb/include/dict0stats_bg.h b/storage/xtradb/include/dict0stats_bg.h index 1973380d197..2bd441d5f5e 100644 --- a/storage/xtradb/include/dict0stats_bg.h +++ b/storage/xtradb/include/dict0stats_bg.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -122,6 +122,10 @@ DECLARE_THREAD(dict_stats_thread)( void* arg); /*!< in: a dummy parameter required by os_thread_create */ +/** Shut down the dict_stats_thread. */ +void +dict_stats_shutdown(); + # ifndef UNIV_NONINL # include "dict0stats_bg.ic" # endif diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index cfd961d9c46..f27ed9857ec 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -499,7 +499,7 @@ extern ibool srv_error_monitor_active; extern ibool srv_buf_dump_thread_active; /* TRUE during the lifetime of the stats thread */ -extern ibool srv_dict_stats_thread_active; +extern bool srv_dict_stats_thread_active; extern ulong srv_n_spin_wait_rounds; extern ulong srv_n_free_tickets_to_enter; diff --git a/storage/xtradb/include/srv0start.h b/storage/xtradb/include/srv0start.h index a60776a4665..b055a9d834f 100644 --- a/storage/xtradb/include/srv0start.h +++ b/storage/xtradb/include/srv0start.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -75,22 +76,12 @@ are not found and the user wants. @return DB_SUCCESS or error code */ UNIV_INTERN dberr_t -innobase_start_or_create_for_mysql(void); -/*====================================*/ -/****************************************************************//** -Shuts down the Innobase database. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -innobase_shutdown_for_mysql(void); +innobase_start_or_create_for_mysql(); -/******************************************************************** -Signal all per-table background threads to shutdown, and wait for them to do -so. */ +/** Shut down InnoDB. */ UNIV_INTERN void -srv_shutdown_table_bg_threads(void); -/*=============================*/ +innodb_shutdown(); /*************************************************************//** Copy the file path component of the physical file to parameter. It will @@ -158,6 +149,9 @@ enum srv_shutdown_state { SRV_SHUTDOWN_EXIT_THREADS/*!< Exit all threads */ }; +/** Whether any undo log records can be generated */ +extern bool srv_undo_sources; + /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ extern enum srv_shutdown_state srv_shutdown_state; diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 2fa3be014a5..e11697081c6 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -92,7 +92,7 @@ UNIV_INTERN ibool srv_error_monitor_active = FALSE; UNIV_INTERN ibool srv_buf_dump_thread_active = FALSE; -UNIV_INTERN ibool srv_dict_stats_thread_active = FALSE; +UNIV_INTERN bool srv_dict_stats_thread_active; UNIV_INTERN const char* srv_main_thread_op_info = ""; @@ -3097,31 +3097,29 @@ suspend_thread: goto loop; } -/*********************************************************************//** -Check if purge should stop. -@return true if it should shutdown. */ +/** Check if purge should stop. +@param[in] n_purged pages purged in the last batch +@return whether purge should exit */ static bool -srv_purge_should_exit( -/*==============*/ - ulint n_purged) /*!< in: pages purged in last batch */ +srv_purge_should_exit(ulint n_purged) { - switch (srv_shutdown_state) { - case SRV_SHUTDOWN_NONE: - /* Normal operation. */ - break; + ut_ad(srv_shutdown_state == SRV_SHUTDOWN_NONE + || srv_shutdown_state == SRV_SHUTDOWN_CLEANUP); - case SRV_SHUTDOWN_CLEANUP: - case SRV_SHUTDOWN_EXIT_THREADS: - /* Exit unless slow shutdown requested or all done. */ - return(srv_fast_shutdown != 0 || n_purged == 0); - - case SRV_SHUTDOWN_LAST_PHASE: - case SRV_SHUTDOWN_FLUSH_PHASE: - ut_error; + if (srv_undo_sources) { + return(false); } - - return(false); + if (srv_fast_shutdown) { + return(true); + } + /* Slow shutdown was requested. */ + if (n_purged) { + /* The previous round still did some work. */ + return(false); + } + /* Exit if there are no active transactions to roll back. */ + return(trx_sys_any_active_transactions() == 0); } /*********************************************************************//** @@ -3396,7 +3394,7 @@ srv_purge_coordinator_suspend( } rw_lock_x_unlock(&purge_sys->latch); - } while (stop); + } while (stop && srv_undo_sources); srv_resume_thread(slot, 0, false); } @@ -3450,6 +3448,7 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( purge didn't purge any records then wait for activity. */ if (srv_shutdown_state == SRV_SHUTDOWN_NONE + && srv_undo_sources && (purge_sys->state == PURGE_STATE_STOP || n_total_purged == 0)) { @@ -3470,36 +3469,8 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( srv_n_purge_threads, &n_total_purged); srv_inc_activity_count(); - } while (!srv_purge_should_exit(n_total_purged)); - /* Ensure that we don't jump out of the loop unless the - exit condition is satisfied. */ - - ut_a(srv_purge_should_exit(n_total_purged)); - - ulint n_pages_purged = ULINT_MAX; - - /* Ensure that all records are purged if it is not a fast shutdown. - This covers the case where a record can be added after we exit the - loop above. */ - while (srv_fast_shutdown == 0 && n_pages_purged > 0) { - n_pages_purged = trx_purge(1, srv_purge_batch_size, false); - } - - /* This trx_purge is called to remove any undo records (added by - background threads) after completion of the above loop. When - srv_fast_shutdown != 0, a large batch size can cause significant - delay in shutdown ,so reducing the batch size to magic number 20 - (which was default in 5.5), which we hope will be sufficient to - remove all the undo records */ - const uint temp_batch_size = 20; - - n_pages_purged = trx_purge(1, srv_purge_batch_size <= temp_batch_size - ? srv_purge_batch_size : temp_batch_size, - true); - ut_a(n_pages_purged == 0 || srv_fast_shutdown != 0); - /* The task queue should always be empty, independent of fast shutdown state. */ ut_a(srv_get_task_queue_length() == 0); diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index e5b06a4919c..525c985f71b 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -123,7 +123,10 @@ UNIV_INTERN ibool srv_is_being_started = FALSE; /** TRUE if the server was successfully started */ UNIV_INTERN ibool srv_was_started = FALSE; /** TRUE if innobase_start_or_create_for_mysql() has been called */ -static ibool srv_start_has_been_called = FALSE; +static ibool srv_start_has_been_called; + +/** Whether any undo log records can be generated */ +UNIV_INTERN bool srv_undo_sources; /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ @@ -1623,8 +1626,7 @@ are not found and the user wants. @return DB_SUCCESS or error code */ UNIV_INTERN dberr_t -innobase_start_or_create_for_mysql(void) -/*====================================*/ +innobase_start_or_create_for_mysql() { ibool create_new_db; lsn_t min_flushed_lsn; @@ -2777,8 +2779,8 @@ files_checked: } } - srv_startup_is_before_trx_rollback_phase = FALSE; recv_recovery_rollback_active(); + srv_startup_is_before_trx_rollback_phase = FALSE; /* It is possible that file_format tag has never been set. In this case we initialize it to minimum @@ -2905,6 +2907,16 @@ files_checked: srv_master_thread, NULL, thread_ids + (1 + SRV_MAX_N_IO_THREADS)); thread_started[1 + SRV_MAX_N_IO_THREADS] = true; + + srv_undo_sources = true; + /* Create the dict stats gathering thread */ + srv_dict_stats_thread_active = true; + dict_stats_thread_handle = os_thread_create( + dict_stats_thread, NULL, NULL); + dict_stats_thread_started = true; + + /* Create the thread that will optimize the FTS sub-system. */ + fts_optimize_init(); } if (!srv_read_only_mode @@ -2934,6 +2946,7 @@ files_checked: } if (!srv_read_only_mode) { + buf_page_cleaner_is_active = true; buf_flush_page_cleaner_thread_handle = os_thread_create(buf_flush_page_cleaner_thread, NULL, NULL); buf_flush_page_cleaner_thread_started = true; } @@ -2973,13 +2986,6 @@ files_checked: /* Create the buffer pool dump/load thread */ buf_dump_thread_handle = os_thread_create(buf_dump_thread, NULL, NULL); buf_dump_thread_started = true; - - /* Create the dict stats gathering thread */ - dict_stats_thread_handle = os_thread_create(dict_stats_thread, NULL, NULL); - dict_stats_thread_started = true; - - /* Create the thread that will optimize the FTS sub-system. */ - fts_optimize_init(); } srv_was_started = TRUE; @@ -3017,13 +3023,10 @@ srv_fts_close(void) } #endif -/****************************************************************//** -Shuts down the InnoDB database. -@return DB_SUCCESS or error code */ +/** Shut down InnoDB. */ UNIV_INTERN -dberr_t -innobase_shutdown_for_mysql(void) -/*=============================*/ +void +innodb_shutdown() { ulint i; @@ -3033,15 +3036,20 @@ innobase_shutdown_for_mysql(void) "Shutting down an improperly started, " "or created database!"); } - - return(DB_SUCCESS); } - if (!srv_read_only_mode) { + if (srv_undo_sources) { + ut_ad(!srv_read_only_mode); /* Shutdown the FTS optimize sub system. */ fts_optimize_start_shutdown(); fts_optimize_end(); + dict_stats_shutdown(); + while (row_get_background_drop_list_len_low()) { + srv_wake_master_thread(); + os_thread_yield(); + } + srv_undo_sources = false; } /* 1. Flush the buffer pool to disk, write the current lsn to @@ -3245,89 +3253,9 @@ innobase_shutdown_for_mysql(void) srv_was_started = FALSE; srv_start_has_been_called = FALSE; - - return(DB_SUCCESS); } #endif /* !UNIV_HOTBACKUP */ - -/******************************************************************** -Signal all per-table background threads to shutdown, and wait for them to do -so. */ -UNIV_INTERN -void -srv_shutdown_table_bg_threads(void) -/*===============================*/ -{ - dict_table_t* table; - dict_table_t* first; - dict_table_t* last = NULL; - - mutex_enter(&dict_sys->mutex); - - /* Signal all threads that they should stop. */ - table = UT_LIST_GET_FIRST(dict_sys->table_LRU); - first = table; - while (table) { - dict_table_t* next; - fts_t* fts = table->fts; - - if (fts != NULL) { - fts_start_shutdown(table, fts); - } - - next = UT_LIST_GET_NEXT(table_LRU, table); - - if (!next) { - last = table; - } - - table = next; - } - - /* We must release dict_sys->mutex here; if we hold on to it in the - loop below, we will deadlock if any of the background threads try to - acquire it (for example, the FTS thread by calling que_eval_sql). - - Releasing it here and going through dict_sys->table_LRU without - holding it is safe because: - - a) MySQL only starts the shutdown procedure after all client - threads have been disconnected and no new ones are accepted, so no - new tables are added or old ones dropped. - - b) Despite its name, the list is not LRU, and the order stays - fixed. - - To safeguard against the above assumptions ever changing, we store - the first and last items in the list above, and then check that - they've stayed the same below. */ - - mutex_exit(&dict_sys->mutex); - - /* Wait for the threads of each table to stop. This is not inside - the above loop, because by signaling all the threads first we can - overlap their shutting down delays. */ - table = UT_LIST_GET_FIRST(dict_sys->table_LRU); - ut_a(first == table); - while (table) { - dict_table_t* next; - fts_t* fts = table->fts; - - if (fts != NULL) { - fts_shutdown(table, fts); - } - - next = UT_LIST_GET_NEXT(table_LRU, table); - - if (table == last) { - ut_a(!next); - } - - table = next; - } -} - /*****************************************************************//** Get the meta-data filename from the table name. */ UNIV_INTERN diff --git a/storage/xtradb/trx/trx0purge.cc b/storage/xtradb/trx/trx0purge.cc index 474ca077ce5..a4463744a69 100644 --- a/storage/xtradb/trx/trx0purge.cc +++ b/storage/xtradb/trx/trx0purge.cc @@ -247,6 +247,19 @@ trx_purge_add_update_undo_to_history( hist_size + undo->size, MLOG_4BYTES, mtr); } + /* Before any transaction-generating background threads or the + purge have been started, recv_recovery_rollback_active() can + start transactions in row_merge_drop_temp_indexes() and + fts_drop_orphaned_tables(), and roll back recovered transactions. + After the purge thread has been given permission to exit, + in fast shutdown, we may roll back transactions (trx->undo_no==0) + in THD::cleanup() invoked from unlink_thd(). */ + ut_ad(srv_undo_sources + || ((srv_startup_is_before_trx_rollback_phase + || trx_rollback_or_clean_is_active) + && purge_sys->state == PURGE_STATE_INIT) + || (trx->undo_no == 0 && srv_fast_shutdown)); + /* Add the log as the first in the history list */ flst_add_first(rseg_header + TRX_RSEG_HISTORY, undo_header + TRX_UNDO_HISTORY_NODE, mtr); From 4325041df6cf13ed1e62749fdb43fb8f138d1e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 12 Jun 2017 11:08:06 +0300 Subject: [PATCH 5/9] MDEV-13057 innodb_read_only=1 should avoid creating buf_flush_page_cleaner_thread When the server is started in innodb_read_only mode, there cannot be any writes to persistent InnoDB/XtraDB files. Just like the creation of buf_flush_page_cleaner_thread is skipped in this case, also the creation of the XtraDB-specific buf_flush_lru_manager_thread should be skipped. --- .../r/innodb_sched_priority_cleaner_basic.result | 2 +- .../t/innodb_sched_priority_cleaner_basic.test | 10 +++++++++- storage/xtradb/buf/buf0flu.cc | 4 +--- storage/xtradb/handler/ha_innodb.cc | 8 ++++---- storage/xtradb/srv/srv0start.cc | 11 +++++++---- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/innodb_sched_priority_cleaner_basic.result b/mysql-test/suite/sys_vars/r/innodb_sched_priority_cleaner_basic.result index 1183fb27732..f2bfaf2ed61 100644 --- a/mysql-test/suite/sys_vars/r/innodb_sched_priority_cleaner_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_sched_priority_cleaner_basic.result @@ -1,4 +1,4 @@ -SET @start_value = @@GLOBAL.innodb_sched_priority_cleaner; +SET GLOBAL innodb_sched_priority_cleaner=39; SELECT @@GLOBAL.innodb_sched_priority_cleaner; @@GLOBAL.innodb_sched_priority_cleaner 19 diff --git a/mysql-test/suite/sys_vars/t/innodb_sched_priority_cleaner_basic.test b/mysql-test/suite/sys_vars/t/innodb_sched_priority_cleaner_basic.test index b2382fd7844..2c2037f167f 100644 --- a/mysql-test/suite/sys_vars/t/innodb_sched_priority_cleaner_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_sched_priority_cleaner_basic.test @@ -4,7 +4,15 @@ # A dynamic, global variable -SET @start_value = @@GLOBAL.innodb_sched_priority_cleaner; +# Test in read-only mode +--let $restart_parameters= --innodb-read-only +--source include/restart_mysqld.inc +--let $restart_parameters= + +# This has no actual effect in innodb_read_only mode +SET GLOBAL innodb_sched_priority_cleaner=39; + +--source include/restart_mysqld.inc # Default value SELECT @@GLOBAL.innodb_sched_priority_cleaner; diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 955be5a6161..17bfb92580e 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -61,7 +61,7 @@ doing the shutdown */ UNIV_INTERN bool buf_page_cleaner_is_active; /** Flag indicating if the lru_manager is in active state. */ -UNIV_INTERN bool buf_lru_manager_is_active = false; +UNIV_INTERN bool buf_lru_manager_is_active; #ifdef UNIV_PFS_THREAD UNIV_INTERN mysql_pfs_key_t buf_page_cleaner_thread_key; @@ -2868,8 +2868,6 @@ DECLARE_THREAD(buf_flush_lru_manager_thread)( os_thread_pf(os_thread_get_curr_id())); #endif /* UNIV_DEBUG_THREAD_CREATION */ - buf_lru_manager_is_active = true; - /* On server shutdown, the LRU manager thread runs through cleanup phase to provide free pages for the master and purge threads. */ while (srv_shutdown_state == SRV_SHUTDOWN_NONE diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 9717136e030..e4238af7747 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -17032,6 +17032,10 @@ innodb_sched_priority_cleaner_update( const void* save) /*!< in: immediate result from check function */ { + if (srv_read_only_mode) { + return; + } + ulint priority = *static_cast(save); ulint actual_priority; ulint nice = 0; @@ -17058,10 +17062,6 @@ innodb_sched_priority_cleaner_update( } /* Set the priority for the page cleaner thread */ - if (srv_read_only_mode) { - - return; - } ut_ad(buf_page_cleaner_is_active); nice = os_thread_get_priority(srv_cleaner_tid); diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 525c985f71b..f12f895488d 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -2947,12 +2947,15 @@ files_checked: if (!srv_read_only_mode) { buf_page_cleaner_is_active = true; - buf_flush_page_cleaner_thread_handle = os_thread_create(buf_flush_page_cleaner_thread, NULL, NULL); + buf_flush_page_cleaner_thread_handle = os_thread_create( + buf_flush_page_cleaner_thread, NULL, NULL); buf_flush_page_cleaner_thread_started = true; - } - buf_flush_lru_manager_thread_handle = os_thread_create(buf_flush_lru_manager_thread, NULL, NULL); - buf_flush_lru_manager_thread_started = true; + buf_lru_manager_is_active = true; + buf_flush_lru_manager_thread_handle = os_thread_create( + buf_flush_lru_manager_thread, NULL, NULL); + buf_flush_lru_manager_thread_started = true; + } if (!srv_file_per_table && srv_pass_corrupt_table) { fprintf(stderr, "InnoDB: Warning:" From 75b35a3b6897a0fcef28d73a480d6d9d65c7ac9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 12 Jun 2017 14:10:39 +0300 Subject: [PATCH 6/9] Partially disable a test affected by MDEV-13059 --- mysql-test/suite/innodb/t/row_format_redundant.test | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mysql-test/suite/innodb/t/row_format_redundant.test b/mysql-test/suite/innodb/t/row_format_redundant.test index b17b365651a..974588d11eb 100644 --- a/mysql-test/suite/innodb/t/row_format_redundant.test +++ b/mysql-test/suite/innodb/t/row_format_redundant.test @@ -2,6 +2,16 @@ # Embedded mode doesn't allow restarting --source include/not_embedded.inc +# MDEV-13059 XtraDB hangs on Windows due to failing to release +# block->lock X-latch in innodb_read_only mode +if (`SELECT count(*) FROM information_schema.plugins WHERE + plugin_name = 'innodb' AND plugin_status = 'active' AND + plugin_description LIKE '%xtradb%'`){ + if (`SELECT @@version_compile_os IN ('Win32','Win64','Windows')`) { + skip MDEV-13059 XtraDB hangs on Windows in innodb_read_only mode; + } +} + create table t1 (a int not null, d varchar(15) not null, b varchar(198) not null, c char(156), fulltext ftsic(c)) engine=InnoDB From 757339efd0894be2c486cfeb79f61182831a050f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 12 Jun 2017 17:09:44 +0300 Subject: [PATCH 7/9] Adjust a test result after merge --- .../suite/encryption/r/innodb-first-page-read.result | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb-first-page-read.result b/mysql-test/suite/encryption/r/innodb-first-page-read.result index 6c9eea80fa9..4a55612af27 100644 --- a/mysql-test/suite/encryption/r/innodb-first-page-read.result +++ b/mysql-test/suite/encryption/r/innodb-first-page-read.result @@ -55,15 +55,15 @@ commit; # i.e. < 23 + 3 = 26 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 17 +Innodb_pages0_read 19 use innodb_test; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 17 +Innodb_pages0_read 19 use test; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 17 +Innodb_pages0_read 19 set global innodb_encrypt_tables=OFF; # wait until tables are decrypted show status like 'innodb_pages0_read%'; @@ -77,11 +77,11 @@ use test; # restart and see number read page 0 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 17 +Innodb_pages0_read 19 use innodb_test; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 17 +Innodb_pages0_read 19 use test; drop database innodb_test; show status like 'innodb_pages0_read%'; From 3005cebc96ef65dc54bf6caaa5e089d52d52e7ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 12 Jun 2017 17:10:56 +0300 Subject: [PATCH 8/9] Post-push fix for MDEV-12610 MariaDB start is slow fil_crypt_read_crypt_data(): Remove an unnecessary acquisition of fil_system->mutex. Remove a duplicated condition from the callers. --- storage/innobase/fil/fil0crypt.cc | 64 +++++++++++-------------------- storage/xtradb/fil/fil0crypt.cc | 64 +++++++++++-------------------- 2 files changed, 44 insertions(+), 84 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 70d8558ede2..df5c250df90 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -1116,40 +1116,33 @@ fil_crypt_needs_rotation( } /** Read page 0 and possible crypt data from there. -@param[in] space Tablespace */ +@param[in,out] space Tablespace */ static inline void fil_crypt_read_crypt_data(fil_space_t* space) { - mutex_enter(&fil_system->mutex); + if (space->crypt_data || space->size) { + /* The encryption metadata has already been read, or + the tablespace is not encrypted and the file has been + opened already. */ + return; + } - /* If space does not contain crypt data and space size is 0 - we have not yet read first page of tablespace. We need to - read it to find out tablespace current encryption status. */ - if (!space->crypt_data && space->size == 0) { - mtr_t mtr; - mtr_start(&mtr); - ulint zip_size = fsp_flags_get_zip_size(space->flags); - ulint offset = fsp_header_get_crypt_offset(zip_size); - mutex_exit(&fil_system->mutex); - if (buf_block_t* block = buf_page_get(space->id, zip_size, 0, - RW_X_LATCH, &mtr)) { - byte* frame = buf_block_get_frame(block); - - mutex_enter(&fil_system->mutex); - - if (!space->crypt_data) { - space->crypt_data = fil_space_read_crypt_data(space->id, - frame, offset); - } - - mutex_exit(&fil_system->mutex); + mtr_t mtr; + mtr_start(&mtr); + ulint zip_size = fsp_flags_get_zip_size(space->flags); + ulint offset = fsp_header_get_crypt_offset(zip_size); + if (buf_block_t* block = buf_page_get(space->id, zip_size, 0, + RW_S_LATCH, &mtr)) { + mutex_enter(&fil_system->mutex); + if (!space->crypt_data) { + space->crypt_data = fil_space_read_crypt_data( + space->id, block->frame, offset); } - - mtr_commit(&mtr); - } else { mutex_exit(&fil_system->mutex); } + + mtr_commit(&mtr); } /*********************************************************************** @@ -1656,12 +1649,7 @@ fil_crypt_find_space_to_rotate( } while (!state->should_shutdown() && state->space) { - /* If there is no crypt data and we have not yet read - page 0 for this tablespace, we need to read it before - we can continue. */ - if (!state->space->crypt_data) { - fil_crypt_read_crypt_data(state->space); - } + fil_crypt_read_crypt_data(state->space); if (fil_crypt_space_needs_rotation(state, key_state, recheck)) { ut_ad(key_state->key_id); @@ -2589,18 +2577,10 @@ fil_space_crypt_get_status( memset(status, 0, sizeof(*status)); ut_ad(space->n_pending_ops > 0); - - /* If there is no crypt data and we have not yet read - page 0 for this tablespace, we need to read it before - we can continue. */ - if (!space->crypt_data) { - fil_crypt_read_crypt_data(const_cast(space)); - } - - fil_space_crypt_t* crypt_data = space->crypt_data; + fil_crypt_read_crypt_data(const_cast(space)); status->space = space->id; - if (crypt_data != NULL) { + if (fil_space_crypt_t* crypt_data = space->crypt_data) { mutex_enter(&crypt_data->mutex); status->scheme = crypt_data->type; status->keyserver_requests = crypt_data->keyserver_requests; diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index e24278dd102..6d3c2e98010 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -1116,40 +1116,33 @@ fil_crypt_needs_rotation( } /** Read page 0 and possible crypt data from there. -@param[in] space Tablespace */ +@param[in,out] space Tablespace */ static inline void fil_crypt_read_crypt_data(fil_space_t* space) { - mutex_enter(&fil_system->mutex); + if (space->crypt_data || space->size) { + /* The encryption metadata has already been read, or + the tablespace is not encrypted and the file has been + opened already. */ + return; + } - /* If space does not contain crypt data and space size is 0 - we have not yet read first page of tablespace. We need to - read it to find out tablespace current encryption status. */ - if (!space->crypt_data && space->size == 0) { - mtr_t mtr; - mtr_start(&mtr); - ulint zip_size = fsp_flags_get_zip_size(space->flags); - ulint offset = fsp_header_get_crypt_offset(zip_size); - mutex_exit(&fil_system->mutex); - if (buf_block_t* block = buf_page_get(space->id, zip_size, 0, - RW_X_LATCH, &mtr)) { - byte* frame = buf_block_get_frame(block); - - mutex_enter(&fil_system->mutex); - - if (!space->crypt_data) { - space->crypt_data = fil_space_read_crypt_data(space->id, - frame, offset); - } - - mutex_exit(&fil_system->mutex); + mtr_t mtr; + mtr_start(&mtr); + ulint zip_size = fsp_flags_get_zip_size(space->flags); + ulint offset = fsp_header_get_crypt_offset(zip_size); + if (buf_block_t* block = buf_page_get(space->id, zip_size, 0, + RW_S_LATCH, &mtr)) { + mutex_enter(&fil_system->mutex); + if (!space->crypt_data) { + space->crypt_data = fil_space_read_crypt_data( + space->id, block->frame, offset); } - - mtr_commit(&mtr); - } else { mutex_exit(&fil_system->mutex); } + + mtr_commit(&mtr); } /*********************************************************************** @@ -1656,12 +1649,7 @@ fil_crypt_find_space_to_rotate( } while (!state->should_shutdown() && state->space) { - /* If there is no crypt data and we have not yet read - page 0 for this tablespace, we need to read it before - we can continue. */ - if (!state->space->crypt_data) { - fil_crypt_read_crypt_data(state->space); - } + fil_crypt_read_crypt_data(state->space); if (fil_crypt_space_needs_rotation(state, key_state, recheck)) { ut_ad(key_state->key_id); @@ -2590,18 +2578,10 @@ fil_space_crypt_get_status( memset(status, 0, sizeof(*status)); ut_ad(space->n_pending_ops > 0); - - /* If there is no crypt data and we have not yet read - page 0 for this tablespace, we need to read it before - we can continue. */ - if (!space->crypt_data) { - fil_crypt_read_crypt_data(const_cast(space)); - } - - fil_space_crypt_t* crypt_data = space->crypt_data; + fil_crypt_read_crypt_data(const_cast(space)); status->space = space->id; - if (crypt_data != NULL) { + if (fil_space_crypt_t* crypt_data = space->crypt_data) { mutex_enter(&crypt_data->mutex); status->scheme = crypt_data->type; status->keyserver_requests = crypt_data->keyserver_requests; From 3976ec1e83d7ae1079b4afa4c12b4ec533d665f6 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Fri, 9 Jun 2017 17:22:59 +0300 Subject: [PATCH 9/9] MDEV-13043 Skipped tests ignore warning suppressions --- mysql-test/lib/mtr_report.pm | 4 ++++ mysql-test/mysql-test-run.pl | 11 ++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/mysql-test/lib/mtr_report.pm b/mysql-test/lib/mtr_report.pm index 97ace54f0fb..d93d8adf34c 100644 --- a/mysql-test/lib/mtr_report.pm +++ b/mysql-test/lib/mtr_report.pm @@ -208,6 +208,10 @@ sub mtr_report_test ($) { { mtr_report("[ skipped ]"); } + if ( $tinfo->{'warnings'} ) + { + mtr_report($tinfo->{'warnings'}); + } } elsif ($result eq 'MTR_RES_PASSED') { diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index b5c43344797..64483fa50f1 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -4025,12 +4025,13 @@ sub run_testcase ($$) { { my $res= $test->exit_status(); - if ($res == 0 and $opt_warnings and check_warnings($tinfo) ) + if (($res == 0 or $res == 62) and $opt_warnings and check_warnings($tinfo) ) { - # Test case suceeded, but it has produced unexpected - # warnings, continue in $res == 1 - $res= 1; - resfile_output($tinfo->{'warnings'}) if $opt_resfile; + # If test case suceeded, but it has produced unexpected + # warnings, continue with $res == 1; + # but if the test was skipped, it should remain skipped + $res= 1 if $res == 0; + resfile_output($tinfo->{'warnings'}) if $opt_resfile; } if ( $res == 0 )