From 3266216f2c8f90c866b371fbd4a8bf6b0c628996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 1 Oct 2015 08:16:14 +0300 Subject: [PATCH] MDEV-8727: Server/InnoDB hangs on shutdown after trying to read an encrypted table with a wrong key Analysis: When a page is read from encrypted table and page can't be decrypted because of bad key (or incorrect encryption algorithm or method) page was incorrectly left on buffer pool. Fix: Remove page from buffer pool and from pending IO. --- .../encryption/r/innodb-bad-key-change.result | 1 - .../r/innodb-bad-key-shutdown.result | 20 ++++++ .../encryption/t/innodb-bad-key-shutdown.test | 68 +++++++++++++++++++ storage/innobase/buf/buf0buf.cc | 30 +++++--- storage/innobase/dict/dict0stats.cc | 5 ++ storage/xtradb/buf/buf0buf.cc | 30 +++++--- storage/xtradb/dict/dict0stats.cc | 5 ++ 7 files changed, 141 insertions(+), 18 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result create mode 100644 mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index bc9cbac889f..cf9791887cc 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -50,7 +50,6 @@ ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be SHOW WARNINGS; Level Code Message Warning 192 is encrypted but encryption service or used key_id is not available. Can't continue reading table. -Warning 192 is encrypted but encryption service or used key_id is not available. Can't continue reading table. Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SELECT * FROM t2 where id = 1; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result b/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result new file mode 100644 index 00000000000..630fba146bf --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result @@ -0,0 +1,20 @@ +call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); +call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); +call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); +call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); +call mtr.add_suppression("Couldn't load plugins from 'file_key_management*"); +# +# Restart the server with key 4 in the key file +# +CREATE TABLE t1 (i INT, KEY(i)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +INSERT INTO t1 VALUES (1); +# +# Restart the server with a different value for key 4 in the key file +# +SELECT * FROM t1; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +SELECT * FROM t1; +i +1 +DROP TABLE t1; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test b/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test new file mode 100644 index 00000000000..d27402edaa8 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test @@ -0,0 +1,68 @@ +# +# MDEV-8727: Server/InnoDB hangs on shutdown after trying to read an encrypted table with a wrong key +# + +call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); +call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); +call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); +call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); +# Suppression for builds where file_key_management plugin is linked statically +call mtr.add_suppression("Couldn't load plugins from 'file_key_management*"); + +--echo # +--echo # Restart the server with key 4 in the key file +--echo # + +--source include/have_innodb.inc +--source include/not_embedded.inc + +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--shutdown_server +--source include/wait_until_disconnected.inc + +--write_file $MYSQLTEST_VARDIR/keys1.txt +1;770A8A65DA156D24EE2A093277530142 +4;18420B5CBA31CCDFFE9716E91EB61374D05914F3ADE23E03 +EOF + +--exec echo "restart:--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc + +CREATE TABLE t1 (i INT, KEY(i)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +INSERT INTO t1 VALUES (1); + +--echo # +--echo # Restart the server with a different value for key 4 in the key file +--echo # + +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--shutdown_server +--source include/wait_until_disconnected.inc + +--write_file $MYSQLTEST_VARDIR/keys2.txt +1;770A8A65DA156D24EE2A093277530142 +4;22222222222222222222222222222222 +EOF + +--exec echo "restart:--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc + +--error 1296 +SELECT * FROM t1; + +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--shutdown_server +--source include/wait_until_disconnected.inc + +--exec echo "restart:--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc + +SELECT * FROM t1; +DROP TABLE t1; + +--remove_file $MYSQLTEST_VARDIR/keys2.txt +--remove_file $MYSQLTEST_VARDIR/keys1.txt diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index f2986f05afe..0ba3593b944 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2868,18 +2868,24 @@ loop: /* Do not try again for encrypted pages */ if (!corrupted) { ib_mutex_t* pmutex = buf_page_get_mutex(bpage); + + buf_pool = buf_pool_from_bpage(bpage); buf_pool_mutex_enter(buf_pool); mutex_enter(pmutex); - buf_block_t* block = buf_page_get_block(bpage); + + ut_ad(buf_pool->n_pend_reads > 0); + os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); buf_page_set_io_fix(bpage, BUF_IO_NONE); - buf_block_set_state(block, BUF_BLOCK_NOT_USED); - buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - buf_pool_mutex_exit(buf_pool); mutex_exit(pmutex); + buf_LRU_free_page(bpage, true); + buf_pool_mutex_exit(buf_pool); + rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, + BUF_IO_READ); if (err) { *err = DB_DECRYPTION_FAILED; } + return (NULL); } @@ -2914,18 +2920,24 @@ loop: ut_error; } else { ib_mutex_t* pmutex = buf_page_get_mutex(bpage); + + buf_pool = buf_pool_from_bpage(bpage); buf_pool_mutex_enter(buf_pool); mutex_enter(pmutex); - buf_block_t* block = buf_page_get_block(bpage); - buf_page_set_io_fix(bpage, BUF_IO_NONE); - buf_block_set_state(block, BUF_BLOCK_NOT_USED); - buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - buf_pool_mutex_exit(buf_pool); + + ut_ad(buf_pool->n_pend_reads > 0); + os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); + buf_page_set_io_fix(bpage, BUF_IO_NONE); mutex_exit(pmutex); + buf_LRU_free_page(bpage, true); + buf_pool_mutex_exit(buf_pool); + rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, + BUF_IO_READ); if (err) { *err = DB_DECRYPTION_FAILED; } + return (NULL); } } diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 001623a49bc..12ead09d829 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -962,6 +962,11 @@ dict_stats_update_transient( continue; } + /* Do not continue if table decryption has failed. */ + if (index->table->is_encrypted) { + break; + } + dict_stats_update_transient_for_index(index); sum_of_index_sizes += index->stat_index_size; diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index ed147683817..2476c8a31f9 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -2911,16 +2911,23 @@ loop: ib_mutex_t* pmutex = buf_page_get_mutex(bpage); mutex_enter(&buf_pool->LRU_list_mutex); mutex_enter(pmutex); - buf_block_t* block = buf_page_get_block(bpage); + + ut_ad(buf_pool->n_pend_reads > 0); + os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); buf_page_set_io_fix(bpage, BUF_IO_NONE); - buf_block_set_state(block, BUF_BLOCK_NOT_USED); - buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - mutex_exit(&buf_pool->LRU_list_mutex); + + if (!buf_LRU_free_page(bpage, true)) { + mutex_exit(&buf_pool->LRU_list_mutex); + } + mutex_exit(pmutex); + rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, + BUF_IO_READ); if (err) { *err = DB_DECRYPTION_FAILED; } + return (NULL); } @@ -2957,16 +2964,23 @@ loop: ib_mutex_t* pmutex = buf_page_get_mutex(bpage); mutex_enter(&buf_pool->LRU_list_mutex); mutex_enter(pmutex); - buf_block_t* block = buf_page_get_block(bpage); + + ut_ad(buf_pool->n_pend_reads > 0); + os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); buf_page_set_io_fix(bpage, BUF_IO_NONE); - buf_block_set_state(block, BUF_BLOCK_NOT_USED); - buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - mutex_exit(&buf_pool->LRU_list_mutex); + + if (!buf_LRU_free_page(bpage, true)) { + mutex_exit(&buf_pool->LRU_list_mutex); + } + mutex_exit(pmutex); + rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, + BUF_IO_READ); if (err) { *err = DB_DECRYPTION_FAILED; } + return (NULL); } } diff --git a/storage/xtradb/dict/dict0stats.cc b/storage/xtradb/dict/dict0stats.cc index 001623a49bc..12ead09d829 100644 --- a/storage/xtradb/dict/dict0stats.cc +++ b/storage/xtradb/dict/dict0stats.cc @@ -962,6 +962,11 @@ dict_stats_update_transient( continue; } + /* Do not continue if table decryption has failed. */ + if (index->table->is_encrypted) { + break; + } + dict_stats_update_transient_for_index(index); sum_of_index_sizes += index->stat_index_size;