From fa2701c6f7b028782cf231565f578b2fc0f10d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 14 Sep 2017 09:23:20 +0300 Subject: [PATCH 1/9] =?UTF-8?q?MDEV-12634:=20Uninitialised=20ROW=5FMERGE?= =?UTF-8?q?=5FRESERVE=5FSIZE=20bytes=20written=20to=20tem=E2=80=A6=20?= =?UTF-8?q?=E2=80=A6porary=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed by removing writing key version to start of every block that was encrypted. Instead we will use single key version from log_sys crypt info. After this MDEV also blocks writen to row log are encrypted and blocks read from row log aren decrypted if encryption is configured for the table. innodb_status_variables[], struct srv_stats_t Added status variables for merge block and row log block encryption and decryption amounts. Removed ROW_MERGE_RESERVE_SIZE define. row_merge_fts_doc_tokenize Remove ROW_MERGE_RESERVE_SIZE row_log_t Add index, crypt_tail, crypt_head to be used in case of encryption. row_log_online_op, row_log_table_close_func Before writing a block encrypt it if encryption is enabled row_log_table_apply_ops, row_log_apply_ops After reading a block decrypt it if encryption is enabled row_log_allocate Allocate temporary buffers crypt_head and crypt_tail if needed. row_log_free Free temporary buffers crypt_head and crypt_tail if they exist. row_merge_encrypt_buf, row_merge_decrypt_buf Removed. row_merge_buf_create, row_merge_buf_write Remove ROW_MERGE_RESERVE_SIZE row_merge_build_indexes Allocate temporary buffer used in decryption and encryption if needed. log_tmp_blocks_crypt, log_tmp_block_encrypt, log_temp_block_decrypt New functions used in block encryption and decryption log_tmp_is_encrypted New function to check is encryption enabled. Added test case innodb-rowlog to force creating a row log and verify that operations are done using introduced status variables. --- .../suite/encryption/r/innodb-rowlog.result | 30 ++ .../suite/encryption/t/innodb-rowlog.opt | 7 + .../suite/encryption/t/innodb-rowlog.test | 90 ++++++ storage/innobase/handler/ha_innodb.cc | 12 + storage/innobase/include/log0crypt.h | 44 ++- storage/innobase/include/row0ftsort.h | 3 +- storage/innobase/include/row0merge.h | 26 +- storage/innobase/include/srv0srv.h | 17 ++ storage/innobase/log/log0crypt.cc | 125 +++++++- storage/innobase/row/row0ftsort.cc | 33 +-- storage/innobase/row/row0log.cc | 114 ++++++- storage/innobase/row/row0merge.cc | 271 ++++++----------- storage/innobase/srv/srv0srv.cc | 4 + storage/xtradb/handler/ha_innodb.cc | 12 + storage/xtradb/include/log0crypt.h | 44 ++- storage/xtradb/include/row0ftsort.h | 3 +- storage/xtradb/include/row0merge.h | 30 +- storage/xtradb/include/srv0srv.h | 17 ++ storage/xtradb/log/log0crypt.cc | 125 +++++++- storage/xtradb/row/row0ftsort.cc | 33 +-- storage/xtradb/row/row0log.cc | 114 ++++++- storage/xtradb/row/row0merge.cc | 280 ++++++------------ storage/xtradb/srv/srv0srv.cc | 4 + 23 files changed, 971 insertions(+), 467 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innodb-rowlog.result create mode 100644 mysql-test/suite/encryption/t/innodb-rowlog.opt create mode 100644 mysql-test/suite/encryption/t/innodb-rowlog.test diff --git a/mysql-test/suite/encryption/r/innodb-rowlog.result b/mysql-test/suite/encryption/r/innodb-rowlog.result new file mode 100644 index 00000000000..873041ab421 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-rowlog.result @@ -0,0 +1,30 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; +SET GLOBAL innodb_encryption_rotate_key_age = 1; +create table t2(id int) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +drop table t2; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; +variable_value > 0 +1 +drop table t1; +SET GLOBAL innodb_file_format=Antelope; +SET GLOBAL innodb_encryption_rotate_key_age=15; diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.opt b/mysql-test/suite/encryption/t/innodb-rowlog.opt new file mode 100644 index 00000000000..6338ddbde35 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-rowlog.opt @@ -0,0 +1,7 @@ +--aria-encrypt-tables +--encrypt-tmp-disk-tables +--innodb-encrypt-tables +--innodb-encrypt-log +--innodb-encryption-rotate-key-age=15 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.test b/mysql-test/suite/encryption/t/innodb-rowlog.test new file mode 100644 index 00000000000..135293ef4b9 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-rowlog.test @@ -0,0 +1,90 @@ +-- source include/have_innodb.inc +-- source include/have_example_key_management_plugin.inc +# needs dbug_dbug +-- source include/have_debug.inc + +--disable_warnings +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $key_age = `SELECT @@innodb_encryption_rotate_key_age`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +--enable_warnings + +let $MYSQLD_DATADIR = `SELECT @@datadir`; +let $MYSQLD_TMPDIR = `SELECT @@tmpdir`; + +# +# Create a table that will be encrypted and put some sensitive data to it (credit card numbers) +# + +create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; + +let $rows = 15000; +--disable_query_log +begin; +while ($rows) +{ + eval insert into t1 values(NULL, '0000-0000-0000-0000','private_data'); + dec $rows; +} +commit; +--enable_query_log + +--let $wait_timeout= 600 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--source include/wait_condition.inc + +# +# Now we create secondary index for credit_card column in parallel we create new rows +# forcing alter table to wait so that row log is used. +# +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +send alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; +# +# Force key rotation and create second index for same table +# +connect (con2,localhost,root,,); +SET GLOBAL innodb_encryption_rotate_key_age = 1; +create table t2(id int) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +send alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; +# +# Create new rows to row log +# +connect (con1,localhost,root,,); +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; + +connection default; +reap; +disconnect con1; + +connection con2; +reap; +drop table t2; + +connection default; +disconnect con2; + +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; +--source include/wait_condition.inc + +# +# Verify that both merge blocks and row log blocks are encryted and decrypted +# + +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; + +drop table t1; + +--disable_warnings +eval SET GLOBAL innodb_file_format=$innodb_file_format_orig; +eval SET GLOBAL innodb_encryption_rotate_key_age=$key_age; +--enable_warnings diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4733c33a83f..afc320de517 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1087,6 +1087,18 @@ static SHOW_VAR innodb_status_variables[]= { {"encryption_key_rotation_list_length", (char*)&export_vars.innodb_key_rotation_list_length, SHOW_LONGLONG}, + {"encryption_n_merge_blocks_encrypted", + (char*)&export_vars.innodb_n_merge_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_merge_blocks_decrypted", + (char*)&export_vars.innodb_n_merge_blocks_decrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_encrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_decrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_decrypted, + SHOW_LONGLONG}, /* scrubing */ {"scrub_background_page_reorganizations", diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index 6b164e90d6e..0ad7e7d7037 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +29,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "univ.i" #include "ut0byte.h" #include "my_crypt.h" +#include "os0file.h" typedef int Crypt_result; @@ -125,4 +126,45 @@ log_crypt_print_checkpoint_keys( /*============================*/ const byte* log_block); +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() MY_ATTRIBUTE((warn_unused_result)); #endif // log0crypt.h diff --git a/storage/innobase/include/row0ftsort.h b/storage/innobase/include/row0ftsort.h index 00bd3317de3..e784fae78b9 100644 --- a/storage/innobase/include/row0ftsort.h +++ b/storage/innobase/include/row0ftsort.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -72,7 +72,6 @@ struct fts_psort_common_t { store Doc ID during sort, if Doc ID will not be big enough to use 8 bytes value */ - fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */ }; struct fts_psort_t { diff --git a/storage/innobase/include/row0merge.h b/storage/innobase/include/row0merge.h index 04d4010ad48..fea6bd99799 100644 --- a/storage/innobase/include/row0merge.h +++ b/storage/innobase/include/row0merge.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +41,6 @@ Created 13/06/2005 Jan Lindstrom #include "lock0types.h" #include "srv0srv.h" -/* Reserve free space from every block for key_version */ -#define ROW_MERGE_RESERVE_SIZE 4 - /* Cluster index read task is mandatory */ #define COST_READ_CLUSTERED_INDEX 1.0 @@ -352,17 +349,16 @@ row_merge_buf_sort( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ - + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Empty a sort buffer. @return sort buffer */ @@ -400,10 +396,10 @@ row_merge_sort( const bool update_progress, /*!< in: update progress status variable or not */ const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,5))); + MY_ATTRIBUTE((warn_unused_result)); + /*********************************************************************//** Allocate a sort buffer. @return own: sort buffer */ @@ -433,7 +429,7 @@ row_merge_file_destroy( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -441,9 +437,9 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Read a merge record. @@ -462,8 +458,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,6,7,8), warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); + #endif /* row0merge.h */ diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 79cf9cdd2ab..09af0b2cdd2 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -137,6 +137,14 @@ struct srv_stats_t { ulint_ctr_64_t pages_encrypted; /* Number of pages decrypted */ ulint_ctr_64_t pages_decrypted; + /* Number of merge blocks encrypted */ + ulint_ctr_64_t n_merge_blocks_encrypted; + /* Number of merge blocks decrypted */ + ulint_ctr_64_t n_merge_blocks_decrypted; + /* Number of row log blocks encrypted */ + ulint_ctr_64_t n_rowlog_blocks_encrypted; + /* Number of row log blocks decrypted */ + ulint_ctr_64_t n_rowlog_blocks_decrypted; /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1033,6 +1041,15 @@ struct export_var_t{ ib_int64_t innodb_pages_decrypted; /*!< Number of pages decrypted */ + /*!< Number of merge blocks encrypted */ + ib_int64_t innodb_n_merge_blocks_encrypted; + /*!< Number of merge blocks decrypted */ + ib_int64_t innodb_n_merge_blocks_decrypted; + /*!< Number of row log blocks encrypted */ + ib_int64_t innodb_n_rowlog_blocks_encrypted; + /*!< Number of row log blocks decrypted */ + ib_int64_t innodb_n_rowlog_blocks_decrypted; + ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ ulint innodb_sec_rec_cluster_reads_avoided;/*!< srv_sec_rec_cluster_reads_avoided */ diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index af9b2349187..fe1b41202ea 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -237,6 +237,129 @@ next: return rc; } +/** Encrypt/decrypt temporary log blocks. + +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] what ENCRYPTION_FLAG_ENCRYPT or + ENCRYPTION_FLAG_DECRYPT +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +static +bool +log_tmp_blocks_crypt( + const byte* src_block, + ulint size, + byte* dst_block, + int what, + os_offset_t offs, + ulint space_id) +{ + Crypt_result rc = MY_AES_OK; + uint dst_len; + byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; + byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; + const crypt_info_t* info = static_cast(&crypt_info[0]); + + // AES_CTR_COUNTER = space_id + offs + + bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); + mach_write_to_8(aes_ctr_counter, space_id); + mach_write_to_8(aes_ctr_counter + 8, offs); + + rc = encryption_crypt(src_block, size, + dst_block, &dst_len, + (unsigned char*)(info->crypt_key), 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + what | ENCRYPTION_FLAG_NOPAD, + LOG_DEFAULT_ENCRYPTION_KEY, + info->key_version); + + if (rc != MY_AES_OK) { + ib_logf(IB_LOG_LEVEL_ERROR, + "%s failed for temporary log file with rc = %d", + is_encrypt ? "Encryption" : "Decryption", + rc); + return false; + } + + return true; +} + +/** Get crypt info +@return pointer to log crypt info or NULL +*/ +inline +const crypt_info_t* +get_crypt_info() +{ + mutex_enter(&log_sys->mutex); + const crypt_info_t* info = get_crypt_info(log_sys->next_checkpoint_no); + mutex_exit(&log_sys->mutex); + + return info; +} + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() +{ + const crypt_info_t* info = get_crypt_info(); + + if (info == NULL || info->key_version == UNENCRYPTED_KEY_VER) { + return false; + } + + return true; +} + +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_ENCRYPT, offs, space_id)); +} + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_DECRYPT, offs, space_id)); +} + /*********************************************************************//** Generate crypt key from crypt msg. @return true if successfull, false if not. */ diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 62271e766cf..40ddb3a6a87 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -39,7 +39,7 @@ Created 10/13/2010 Jimmy Yang b[N] = row_merge_read_rec( \ block[N], buf[N], b[N], index, \ fd[N], &foffs[N], &mrec[N], offsets[N], \ - crypt_data, crypt_block[N], space); \ + crypt_block[N], space); \ if (UNIV_UNLIKELY(!b[N])) { \ if (mrec[N]) { \ goto exit; \ @@ -191,7 +191,6 @@ row_fts_psort_info_init( fts_psort_t* merge_info = NULL; ulint block_size; ibool ret = TRUE; - fil_space_crypt_t* crypt_data = NULL; bool encrypted = false; block_size = 3 * srv_sort_buf_size; @@ -222,21 +221,8 @@ row_fts_psort_info_init( common_info->merge_event = os_event_create(); common_info->opt_doc_id_size = opt_doc_id_size; - /* Theoretically the tablespace can be dropped straight away. - In practice, the DDL completion will wait for this thread to - finish. */ - if (fil_space_t* space = fil_space_acquire(new_table->space)) { - crypt_data = space->crypt_data; - fil_space_release(space); - } - - if (crypt_data && crypt_data->should_encrypt()) { - common_info->crypt_data = crypt_data; + if (log_tmp_is_encrypted()) { encrypted = true; - } else { - /* Not needed */ - common_info->crypt_data = NULL; - crypt_data = NULL; } ut_ad(trx->mysql_thd != NULL); @@ -574,11 +560,9 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* Reserve one byte for the end marker of row_merge_block_t - and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for - encryption key_version in the beginning of the buffer. */ + /* Reserve one byte for the end marker of row_merge_block_t */ if (buf->total_size + data_size[idx] + cur_len - >= (srv_sort_buf_size - 1 - ROW_MERGE_RESERVE_SIZE)) { + >= (srv_sort_buf_size - 1)) { buf_full = TRUE; break; @@ -672,7 +656,6 @@ fts_parallel_tokenization( fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; - fil_space_crypt_t* crypt_data = NULL; ut_ad(psort_info->psort_common->trx->mysql_thd != NULL); @@ -693,7 +676,6 @@ fts_parallel_tokenization( block = psort_info->merge_block; crypt_block = psort_info->crypt_block; - crypt_data = psort_info->psort_common->crypt_data; zip_size = dict_table_zip_size(table); row_merge_fts_get_next_doc_item(psort_info, &doc_item); @@ -788,7 +770,6 @@ loop: if (!row_merge_write(merge_file[t_ctx.buf_used]->fd, merge_file[t_ctx.buf_used]->offset++, block[t_ctx.buf_used], - crypt_data, crypt_block[t_ctx.buf_used], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -884,7 +865,6 @@ exit: if (!row_merge_write(merge_file[i]->fd, merge_file[i]->offset++, block[i], - crypt_data, crypt_block[i], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -924,7 +904,7 @@ exit: psort_info->psort_common->dup, merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */, - crypt_data, crypt_block[i], table->space); + crypt_block[i], table->space); if (error != DB_SUCCESS) { close(tmpfd[i]); @@ -1436,7 +1416,6 @@ row_fts_merge_insert( ulint start; fts_psort_insert_t ins_ctx; ulint count_diag = 0; - fil_space_crypt_t* crypt_data = NULL; ulint space; ut_ad(index); @@ -1450,7 +1429,6 @@ row_fts_merge_insert( ins_ctx.trx->op_info = "inserting index entries"; ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size; - crypt_data = psort_info[0].psort_common->crypt_data; heap = mem_heap_create(500 + sizeof(mrec_buf_t)); @@ -1544,7 +1522,6 @@ row_fts_merge_insert( && (!row_merge_read( fd[i], foffs[i], (row_merge_block_t*) block[i], - crypt_data, (row_merge_block_t*) crypt_block[i], space))) { error = DB_CORRUPTION; diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 88398c20342..ec6b4a60680 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -182,6 +182,7 @@ struct row_log_t { dict_table_t* table; /*!< table that is being rebuilt, or NULL when this is a secondary index that is being created online */ + dict_index_t* index; /*!< index to be build */ bool same_pk;/*!< whether the definition of the PRIMARY KEY has remained the same */ const dtuple_t* add_cols; @@ -197,8 +198,14 @@ struct row_log_t { row_log_buf_t tail; /*!< writer context; protected by mutex and index->lock S-latch, or by index->lock X-latch only */ + byte* crypt_tail; /*!< writer context; + temporary buffer used in encryption, + decryption or NULL*/ row_log_buf_t head; /*!< reader context; protected by MDL only; modifiable by row_log_apply_ops() */ + byte* crypt_head; /*!< reader context; + temporary buffer used in encryption, + decryption or NULL */ const char* path; /*!< where to create temporary file during log operation */ }; @@ -349,6 +356,7 @@ row_log_online_op( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -368,11 +376,29 @@ row_log_online_op( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: /* We set the flag directly instead of invoking @@ -380,7 +406,9 @@ write_failed: because the index is not "public" yet. */ index->type |= DICT_CORRUPT; } + UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); + memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); log->tail.bytes = mrec_size - avail_size; @@ -465,6 +493,7 @@ row_log_table_close_func( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -484,11 +513,29 @@ row_log_table_close_func( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + log->index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: log->error = DB_ONLINE_LOG_TOO_BIG; @@ -2622,10 +2669,29 @@ all_done: goto func_exit; } + byte * buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for table %s\n", index->table_name); @@ -2932,9 +2998,22 @@ row_log_allocate( log->head.blocks = log->head.bytes = 0; log->head.total = 0; log->path = path; + log->crypt_tail = log->crypt_head = NULL; + log->index = index; dict_index_set_online_status(index, ONLINE_INDEX_CREATION); index->online_log = log; + if (log_tmp_is_encrypted()) { + ulint size = srv_sort_buf_size; + log->crypt_head = static_cast(os_mem_alloc_large(&size)); + log->crypt_tail = static_cast(os_mem_alloc_large(&size)); + + if (!log->crypt_head || !log->crypt_tail) { + row_log_free(log); + DBUG_RETURN(false); + } + } + /* While we might be holding an exclusive data dictionary lock here, in row_log_abort_sec() we will not always be holding it. Use atomic operations in both cases. */ @@ -2957,6 +3036,15 @@ row_log_free( row_log_block_free(log->tail); row_log_block_free(log->head); row_merge_file_destroy_low(log->fd); + + if (log->crypt_head) { + os_mem_free_large(log->crypt_head, srv_sort_buf_size); + } + + if (log->crypt_tail) { + os_mem_free_large(log->crypt_tail, srv_sort_buf_size); + } + mutex_free(&log->mutex); ut_free(log); log = 0; @@ -3451,11 +3539,29 @@ all_done: goto func_exit; } + byte* buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for index %s\n", index->name + 1); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 65569eabf57..a91c0f67218 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -79,88 +79,6 @@ UNIV_INTERN char srv_disable_sort_file_cache; /* Maximum pending doc memory limit in bytes for a fts tokenization thread */ #define FTS_PENDING_DOC_MEMORY_LIMIT 1000000 - -/******************************************************//** -Encrypt a merge block. */ -static -void -row_merge_encrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - key_version = encryption_key_get_latest_version(crypt_data->key_id); - - /* Store key_version at the begining of the input buffer */ - mach_write_to_4((byte *)crypted_buf, key_version); - - int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } -} - -/******************************************************//** -Decrypt a merge block. */ -static -bool -row_merge_decrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - /* Read key_version from begining of the buffer */ - key_version = mach_read_from_4((byte *)input_buf); - - if (key_version == 0) { - /* block not encrypted */ - return false; - } - - int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } - - return true; -} - #ifdef UNIV_DEBUG /******************************************************//** Display a merge tuple. */ @@ -279,7 +197,7 @@ row_merge_buf_create( ulint buf_size; mem_heap_t* heap; - max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE) + max_tuples = (srv_sort_buf_size) / ut_max(1, dict_index_get_min_size(index)); buf_size = (sizeof *buf); @@ -695,7 +613,7 @@ row_merge_buf_add( ut_ad(data_size < srv_sort_buf_size); /* Reserve bytes for the end marker of row_merge_block_t. */ - if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) { + if (buf->total_size + data_size >= srv_sort_buf_size) { DBUG_RETURN(0); } @@ -816,7 +734,7 @@ UT_SORT_FUNCTION_BODY(). /**********************************************************************//** Merge sort the tuple buffer in main memory. */ -static MY_ATTRIBUTE((nonnull(4,5))) +static void row_merge_tuple_sort( /*=================*/ @@ -867,7 +785,7 @@ row_merge_buf_write( { const dict_index_t* index = buf->index; ulint n_fields= dict_index_get_n_fields(index); - byte* b = &block[ROW_MERGE_RESERVE_SIZE]; + byte* b = &block[0]; for (ulint i = 0; i < buf->n_tuples; i++) { const mtuple_t* entry = &buf->tuples[i]; @@ -886,7 +804,7 @@ row_merge_buf_write( /* Write an "end-of-chunk" marker. */ ut_a(b < &block[srv_sort_buf_size]); - ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE); + ut_a(b == &block[0] + buf->total_size); *b++ = 0; #ifdef UNIV_DEBUG_VALGRIND /* The rest of the block is uninitialized. Initialize it @@ -936,7 +854,7 @@ row_merge_heap_create( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -944,12 +862,11 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size; - ibool success; + bool success; DBUG_EXECUTE_IF("row_merge_read_failure", return(FALSE);); @@ -964,10 +881,14 @@ row_merge_read( ofs, srv_sort_buf_size); /* For encrypted tables, decrypt data after reading and copy data */ - if (crypt_data && crypt_buf) { - if (row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) { - memcpy(buf, crypt_buf, srv_sort_buf_size); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, srv_sort_buf_size, + crypt_buf, ofs, space)) { + return (FALSE); } + + srv_stats.n_merge_blocks_decrypted.inc(); + memcpy(buf, crypt_buf, srv_sort_buf_size); } #ifdef POSIX_FADV_DONTNEED @@ -989,31 +910,32 @@ row_merge_read( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { size_t buf_len = srv_sort_buf_size; os_offset_t ofs = buf_len * (os_offset_t) offset; - ibool ret; + bool ret; void* out_buf = (void *)buf; DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE);); /* For encrypted tables, encrypt data before writing */ - if (crypt_data && crypt_buf) { - row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt((const byte *)buf, buf_len, + (byte *)crypt_buf, ofs, space)) { + return (FALSE); + } + + srv_stats.n_merge_blocks_encrypted.inc(); out_buf = crypt_buf; - } else { - /* Mark block unencrypted */ - mach_write_to_4((byte *)out_buf, 0); } ret = os_file_write_int_fd("(merge)", fd, out_buf, ofs, buf_len); @@ -1051,9 +973,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint data_size; @@ -1065,10 +986,6 @@ row_merge_read_rec( ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - extra_size = *b++; if (UNIV_UNLIKELY(!extra_size)) { @@ -1090,7 +1007,8 @@ row_merge_read_rec( if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) { if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { err_exit: /* Signal I/O error. */ *mrec = b; @@ -1098,7 +1016,7 @@ err_exit: } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; } extra_size = (extra_size & 0x7f) << 8; @@ -1120,13 +1038,14 @@ err_exit: memcpy(*buf, b, avail_size); if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the record. */ memcpy(*buf + avail_size, b, extra_size - avail_size); @@ -1182,13 +1101,14 @@ err_exit: #endif /* UNIV_DEBUG */ if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the rest of the record. */ memcpy(*buf + avail_size, b, extra_size + data_size - avail_size); @@ -1265,7 +1185,6 @@ row_merge_write_rec( ulint* foffs, /*!< in/out: file offset */ const mrec_t* mrec, /*!< in: record to write */ const ulint* offsets,/*!< in: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1288,10 +1207,6 @@ row_merge_write_rec( size = extra_size + (extra_size >= 0x80) + rec_offs_data_size(offsets); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) { /* The record spans two blocks. Copy it to the temporary buffer first. */ @@ -1307,14 +1222,15 @@ row_merge_write_rec( memcpy(b, buf[0], avail_size); if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); /* Copy the rest. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; memcpy(b, buf[0] + avail_size, size - avail_size); b += size - avail_size; } else { @@ -1337,7 +1253,6 @@ row_merge_write_eof( byte* b, /*!< in: pointer to end of block */ int fd, /*!< in: file descriptor */ ulint* foffs, /*!< in/out: file offset */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1352,10 +1267,6 @@ row_merge_write_eof( } #endif /* UNIV_DEBUG */ - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - *b++ = 0; UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]); UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size); @@ -1367,7 +1278,8 @@ row_merge_write_eof( #endif /* UNIV_DEBUG_VALGRIND */ if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } @@ -1445,10 +1357,9 @@ containing the index entries for the indexes to be built. @param[in,out] block file buffer @param[in,out] tmpfd temporary file handle @param[in] pct_cost percent of task weight out of total alter job -@param[in] crypt_data crypt data or NULL @param[in,out] crypt_block crypted file buffer -return DB_SUCCESS or error */ -static MY_ATTRIBUTE((nonnull(1,2,3,4,6,9,10,16), warn_unused_result)) +@return DB_SUCCESS or error */ +static MY_ATTRIBUTE((warn_unused_result)) dberr_t row_merge_read_clustered_index( trx_t* trx, @@ -1469,7 +1380,6 @@ row_merge_read_clustered_index( row_merge_block_t* block, int* tmpfd, float pct_cost, - fil_space_crypt_t* crypt_data, row_merge_block_t* crypt_block) { dict_index_t* clust_index; /* Clustered index */ @@ -2004,8 +1914,8 @@ write_buffers: row_merge_buf_write(buf, file, block); if (!row_merge_write(file->fd, file->offset++, - block, crypt_data, crypt_block, - new_table->space)) { + block, crypt_block, + new_table->space)) { err = DB_TEMP_FILE_WRITE_FAILURE; trx->error_key_num = i; break; @@ -2054,7 +1964,7 @@ write_buffers: if(read_rows % 1000 == 0) { /* Update progress for each 1000 rows */ curr_progress = (read_rows >= table_total_rows) ? - pct_cost : + pct_cost : ((pct_cost * read_rows) / table_total_rows); /* presenting 10.12% as 1012 integer */ onlineddl_pct_progress = curr_progress * 100; @@ -2183,9 +2093,8 @@ wait_again: &buf[2], b2, \ of->fd, &of->offset, \ mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL , \ - space); \ + space); \ if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \ goto corrupt; \ } \ @@ -2193,7 +2102,6 @@ wait_again: &buf[N], b##N, INDEX, \ file->fd, foffs##N, \ &mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[N * srv_sort_buf_size] : NULL, \ space); \ \ @@ -2208,7 +2116,7 @@ wait_again: /*************************************************************//** Merge two blocks of records on disk and write a bigger block. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_blocks( /*=============*/ @@ -2222,7 +2130,6 @@ row_merge_blocks( ulint* foffs1, /*!< in/out: offset of second source list in the file */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ row_merge_block_t* crypt_block,/*!< in: in/out: crypted file buffer */ ulint space) /*!< in: space id */ @@ -2258,9 +2165,11 @@ row_merge_blocks( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space) - || !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space) || + !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space)) { corrupt: mem_heap_free(heap); return(DB_CORRUPTION); @@ -2273,13 +2182,15 @@ corrupt: b0 = row_merge_read_rec( &block[0], &buf[0], b0, dup->index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); b1 = row_merge_read_rec( &block[srv_sort_buf_size], &buf[srv_sort_buf_size], b1, dup->index, file->fd, foffs1, &mrec1, offsets1, - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0) || UNIV_UNLIKELY(!b1 && mrec1)) { @@ -2325,7 +2236,8 @@ done1: b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space); return(b2 ? DB_SUCCESS : DB_CORRUPTION); } @@ -2333,8 +2245,8 @@ done1: /*************************************************************//** Copy a block of index entries. @return TRUE on success, FALSE on failure */ -static __attribute__((nonnull(1,2,3,4,5), warn_unused_result)) -ibool +static __attribute__((warn_unused_result)) +bool row_merge_blocks_copy( /*==================*/ const dict_index_t* index, /*!< in: index being created */ @@ -2342,9 +2254,8 @@ row_merge_blocks_copy( row_merge_block_t* block, /*!< in/out: 3 buffers */ ulint* foffs0, /*!< in/out: input file offset */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */ @@ -2372,7 +2283,8 @@ row_merge_blocks_copy( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space)) { corrupt: mem_heap_free(heap); return(FALSE); @@ -2384,7 +2296,8 @@ corrupt: b0 = row_merge_read_rec(&block[0], &buf[0], b0, index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0)) { @@ -2407,15 +2320,15 @@ done0: return(row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, - crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space) + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space) != NULL); } /*************************************************************//** Merge disk files. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6,7))) +static dberr_t row_merge( /*======*/ @@ -2431,9 +2344,8 @@ row_merge( ulint* run_offset, /*!< in/out: Array contains the first offset number for each merge run */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint foffs0; /*!< first input offset */ ulint foffs1; /*!< second input offset */ @@ -2481,7 +2393,8 @@ row_merge( error = row_merge_blocks(dup, file, block, &foffs0, &foffs1, &of, - crypt_data, crypt_block, space); + crypt_block, + space); if (error != DB_SUCCESS) { return(error); @@ -2502,7 +2415,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs0, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2520,7 +2434,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs1, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2576,9 +2491,8 @@ row_merge_sort( /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const ulint half = file->offset / 2; ulint num_runs; @@ -2646,7 +2560,8 @@ row_merge_sort( error = row_merge(trx, dup, file, block, tmpfd, &num_runs, run_offset, - crypt_data, crypt_block, space); + crypt_block, + space); if(update_progress) { merge_count++; @@ -2723,7 +2638,7 @@ row_merge_copy_blobs( Read sorted file containing index data tuples and insert these data tuples to the index @return DB_SUCCESS or error number */ -static __attribute__((nonnull(2,3,5), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_insert_index_tuples( /*==========================*/ @@ -2736,9 +2651,8 @@ row_merge_insert_index_tuples( const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const byte* b; mem_heap_t* heap; @@ -2772,7 +2686,8 @@ row_merge_insert_index_tuples( b = &block[0]; if (!row_merge_read(fd, foffs, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { error = DB_CORRUPTION; } else { buf = static_cast( @@ -2789,7 +2704,9 @@ row_merge_insert_index_tuples( b = row_merge_read_rec(block, buf, b, index, fd, &foffs, &mrec, offsets, - crypt_data, crypt_block, space); + crypt_block, + space); + if (UNIV_UNLIKELY(!b)) { /* End of list, or I/O error */ if (mrec) { @@ -3974,7 +3891,6 @@ row_merge_build_indexes( fts_psort_t* merge_info = NULL; ib_int64_t sig_count = 0; bool fts_psort_initiated = false; - fil_space_crypt_t * crypt_data = NULL; float total_static_cost = 0; float total_dynamic_cost = 0; @@ -4003,25 +3919,20 @@ row_merge_build_indexes( from concurrent DDL (e.g. drop table) by MDL-locks. */ fil_space_t* space = fil_space_acquire(new_table->space); - if (space) { - crypt_data = space->crypt_data; - } else { + if (!space) { DBUG_RETURN(DB_TABLESPACE_NOT_FOUND); } /* If tablespace is encrypted, allocate additional buffer for encryption/decryption. */ - if (crypt_data && crypt_data->should_encrypt()) { + if (log_tmp_is_encrypted()) { crypt_block = static_cast( - os_mem_alloc_large(&block_size)); + os_mem_alloc_large(&block_size)); if (crypt_block == NULL) { fil_space_release(space); DBUG_RETURN(DB_OUT_OF_MEMORY); } - } else { - /* Not needed */ - crypt_data = NULL; } trx_start_if_not_started_xa(trx); @@ -4103,7 +4014,7 @@ row_merge_build_indexes( fts_sort_idx, psort_info, merge_files, key_numbers, n_indexes, add_cols, col_map, add_autoinc, sequence, block, &tmpfd, - pct_cost, crypt_data, crypt_block); + pct_cost, crypt_block); pct_progress += pct_cost; @@ -4240,7 +4151,8 @@ wait_again: trx, &dup, &merge_files[i], block, &tmpfd, true, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, + new_table->space); pct_progress += pct_cost; @@ -4278,7 +4190,8 @@ wait_again: trx->id, sort_idx, old_table, merge_files[i].fd, block, merge_files[i].n_rec, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, new_table->space); + pct_progress += pct_cost; if (global_system_variables.log_warnings > 2) { diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 7ddd934a9a1..1431211279b 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1619,6 +1619,10 @@ srv_export_innodb_status(void) export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; export_vars.innodb_pages_encrypted = srv_stats.pages_encrypted; + export_vars.innodb_n_merge_blocks_encrypted = srv_stats.n_merge_blocks_encrypted; + export_vars.innodb_n_merge_blocks_decrypted = srv_stats.n_merge_blocks_decrypted; + export_vars.innodb_n_rowlog_blocks_encrypted = srv_stats.n_rowlog_blocks_encrypted; + export_vars.innodb_n_rowlog_blocks_decrypted = srv_stats.n_rowlog_blocks_decrypted; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index caf7a3ef412..4af523b1018 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1309,6 +1309,18 @@ static SHOW_VAR innodb_status_variables[]= { {"encryption_key_rotation_list_length", (char*)&export_vars.innodb_key_rotation_list_length, SHOW_LONGLONG}, + {"encryption_n_merge_blocks_encrypted", + (char*)&export_vars.innodb_n_merge_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_merge_blocks_decrypted", + (char*)&export_vars.innodb_n_merge_blocks_decrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_encrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_decrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_decrypted, + SHOW_LONGLONG}, /* Scrubing feature */ {"scrub_background_page_reorganizations", diff --git a/storage/xtradb/include/log0crypt.h b/storage/xtradb/include/log0crypt.h index 6b164e90d6e..0ad7e7d7037 100644 --- a/storage/xtradb/include/log0crypt.h +++ b/storage/xtradb/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +29,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "univ.i" #include "ut0byte.h" #include "my_crypt.h" +#include "os0file.h" typedef int Crypt_result; @@ -125,4 +126,45 @@ log_crypt_print_checkpoint_keys( /*============================*/ const byte* log_block); +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() MY_ATTRIBUTE((warn_unused_result)); #endif // log0crypt.h diff --git a/storage/xtradb/include/row0ftsort.h b/storage/xtradb/include/row0ftsort.h index 7c9ed23645c..b2dd90e7e3b 100644 --- a/storage/xtradb/include/row0ftsort.h +++ b/storage/xtradb/include/row0ftsort.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -72,7 +72,6 @@ struct fts_psort_common_t { store Doc ID during sort, if Doc ID will not be big enough to use 8 bytes value */ - fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */ }; struct fts_psort_t { diff --git a/storage/xtradb/include/row0merge.h b/storage/xtradb/include/row0merge.h index 04d4010ad48..152a51dafc6 100644 --- a/storage/xtradb/include/row0merge.h +++ b/storage/xtradb/include/row0merge.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +41,6 @@ Created 13/06/2005 Jan Lindstrom #include "lock0types.h" #include "srv0srv.h" -/* Reserve free space from every block for key_version */ -#define ROW_MERGE_RESERVE_SIZE 4 - /* Cluster index read task is mandatory */ #define COST_READ_CLUSTERED_INDEX 1.0 @@ -352,17 +349,16 @@ row_merge_buf_sort( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ - + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Empty a sort buffer. @return sort buffer */ @@ -400,10 +396,9 @@ row_merge_sort( const bool update_progress, /*!< in: update progress status variable or not */ const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,5))); + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /*********************************************************************//** Allocate a sort buffer. @return own: sort buffer */ @@ -433,7 +428,7 @@ row_merge_file_destroy( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -441,10 +436,9 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ - + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Read a merge record. @return pointer to next record, or NULL on I/O error or end of list */ @@ -462,8 +456,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,6,7,8), warn_unused_result)); + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); + #endif /* row0merge.h */ diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 30c125ea269..e7f6350987e 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -137,6 +137,14 @@ struct srv_stats_t { ulint_ctr_64_t pages_encrypted; /* Number of pages decrypted */ ulint_ctr_64_t pages_decrypted; + /* Number of merge blocks encrypted */ + ulint_ctr_64_t n_merge_blocks_encrypted; + /* Number of merge blocks decrypted */ + ulint_ctr_64_t n_merge_blocks_decrypted; + /* Number of row log blocks encrypted */ + ulint_ctr_64_t n_rowlog_blocks_encrypted; + /* Number of row log blocks decrypted */ + ulint_ctr_64_t n_rowlog_blocks_decrypted; /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1276,6 +1284,15 @@ struct export_var_t{ ib_int64_t innodb_pages_decrypted; /*!< Number of pages decrypted */ + /*!< Number of merge blocks encrypted */ + ib_int64_t innodb_n_merge_blocks_encrypted; + /*!< Number of merge blocks decrypted */ + ib_int64_t innodb_n_merge_blocks_decrypted; + /*!< Number of row log blocks encrypted */ + ib_int64_t innodb_n_rowlog_blocks_encrypted; + /*!< Number of row log blocks decrypted */ + ib_int64_t innodb_n_rowlog_blocks_decrypted; + ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ ulint innodb_sec_rec_cluster_reads_avoided;/*!< srv_sec_rec_cluster_reads_avoided */ diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index f6c1416d81a..9509797dde3 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -236,6 +236,129 @@ next: return rc; } +/** Encrypt/decrypt temporary log blocks. + +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] what ENCRYPTION_FLAG_ENCRYPT or + ENCRYPTION_FLAG_DECRYPT +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +static +bool +log_tmp_blocks_crypt( + const byte* src_block, + ulint size, + byte* dst_block, + int what, + os_offset_t offs, + ulint space_id) +{ + Crypt_result rc = MY_AES_OK; + uint dst_len; + byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; + byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; + const crypt_info_t* info = static_cast(&crypt_info[0]); + + // AES_CTR_COUNTER = space_id + offs + + bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); + mach_write_to_8(aes_ctr_counter, space_id); + mach_write_to_8(aes_ctr_counter + 8, offs); + + rc = encryption_crypt(src_block, size, + dst_block, &dst_len, + (unsigned char*)(info->crypt_key), 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + what | ENCRYPTION_FLAG_NOPAD, + LOG_DEFAULT_ENCRYPTION_KEY, + info->key_version); + + if (rc != MY_AES_OK) { + ib_logf(IB_LOG_LEVEL_ERROR, + "%s failed for temporary log file with rc = %d", + is_encrypt ? "Encryption" : "Decryption", + rc); + return false; + } + + return true; +} + +/** Get crypt info +@return pointer to log crypt info or NULL +*/ +inline +const crypt_info_t* +get_crypt_info() +{ + mutex_enter(&log_sys->mutex); + const crypt_info_t* info = get_crypt_info(log_sys->next_checkpoint_no); + mutex_exit(&log_sys->mutex); + + return info; +} + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() +{ + const crypt_info_t* info = get_crypt_info(); + + if (info == NULL || info->key_version == UNENCRYPTED_KEY_VER) { + return false; + } + + return true; +} + +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_ENCRYPT, offs, space_id)); +} + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_DECRYPT, offs, space_id)); +} + /*********************************************************************//** Generate crypt key from crypt msg. @return true if successfull, false if not. */ diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc index 7ffcc59dc5f..0f9c15f6a69 100644 --- a/storage/xtradb/row/row0ftsort.cc +++ b/storage/xtradb/row/row0ftsort.cc @@ -40,7 +40,7 @@ Created 10/13/2010 Jimmy Yang b[N] = row_merge_read_rec( \ block[N], buf[N], b[N], index, \ fd[N], &foffs[N], &mrec[N], offsets[N], \ - crypt_data, crypt_block[N], space); \ + crypt_block[N], space); \ if (UNIV_UNLIKELY(!b[N])) { \ if (mrec[N]) { \ goto exit; \ @@ -194,7 +194,6 @@ row_fts_psort_info_init( fts_psort_t* merge_info = NULL; ulint block_size; ibool ret = TRUE; - fil_space_crypt_t* crypt_data = NULL; bool encrypted = false; block_size = 3 * srv_sort_buf_size; @@ -225,21 +224,8 @@ row_fts_psort_info_init( common_info->merge_event = os_event_create(); common_info->opt_doc_id_size = opt_doc_id_size; - /* Theoretically the tablespace can be dropped straight away. - In practice, the DDL completion will wait for this thread to - finish. */ - if (fil_space_t* space = fil_space_acquire(new_table->space)) { - crypt_data = space->crypt_data; - fil_space_release(space); - } - - if (crypt_data && crypt_data->should_encrypt()) { - common_info->crypt_data = crypt_data; + if (log_tmp_is_encrypted()) { encrypted = true; - } else { - /* Not needed */ - common_info->crypt_data = NULL; - crypt_data = NULL; } ut_ad(trx->mysql_thd != NULL); @@ -577,11 +563,9 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* Reserve one byte for the end marker of row_merge_block_t - and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for - encryption key_version in the beginning of the buffer. */ + /* Reserve one byte for the end marker of row_merge_block_t */ if (buf->total_size + data_size[idx] + cur_len - >= (srv_sort_buf_size - 1 - ROW_MERGE_RESERVE_SIZE)) { + >= (srv_sort_buf_size - 1)) { buf_full = TRUE; break; @@ -675,7 +659,6 @@ fts_parallel_tokenization( fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; - fil_space_crypt_t* crypt_data = NULL; ut_ad(psort_info->psort_common->trx->mysql_thd != NULL); @@ -696,7 +679,6 @@ fts_parallel_tokenization( block = psort_info->merge_block; crypt_block = psort_info->crypt_block; - crypt_data = psort_info->psort_common->crypt_data; zip_size = dict_table_zip_size(table); row_merge_fts_get_next_doc_item(psort_info, &doc_item); @@ -791,7 +773,6 @@ loop: if (!row_merge_write(merge_file[t_ctx.buf_used]->fd, merge_file[t_ctx.buf_used]->offset++, block[t_ctx.buf_used], - crypt_data, crypt_block[t_ctx.buf_used], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -887,7 +868,6 @@ exit: if (!row_merge_write(merge_file[i]->fd, merge_file[i]->offset++, block[i], - crypt_data, crypt_block[i], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -927,7 +907,7 @@ exit: psort_info->psort_common->dup, merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */, - crypt_data, crypt_block[i], table->space); + crypt_block[i], table->space); if (error != DB_SUCCESS) { close(tmpfd[i]); @@ -1439,7 +1419,6 @@ row_fts_merge_insert( ulint start; fts_psort_insert_t ins_ctx; ulint count_diag = 0; - fil_space_crypt_t* crypt_data = NULL; ulint space; ut_ad(index); @@ -1453,7 +1432,6 @@ row_fts_merge_insert( ins_ctx.trx->op_info = "inserting index entries"; ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size; - crypt_data = psort_info[0].psort_common->crypt_data; heap = mem_heap_create(500 + sizeof(mrec_buf_t)); @@ -1547,7 +1525,6 @@ row_fts_merge_insert( && (!row_merge_read( fd[i], foffs[i], (row_merge_block_t*) block[i], - crypt_data, (row_merge_block_t*) crypt_block[i], space))) { error = DB_CORRUPTION; diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc index 2cd663fd600..f49ff2f1530 100644 --- a/storage/xtradb/row/row0log.cc +++ b/storage/xtradb/row/row0log.cc @@ -182,6 +182,7 @@ struct row_log_t { dict_table_t* table; /*!< table that is being rebuilt, or NULL when this is a secondary index that is being created online */ + dict_index_t* index; /*!< index to be build */ bool same_pk;/*!< whether the definition of the PRIMARY KEY has remained the same */ const dtuple_t* add_cols; @@ -197,8 +198,14 @@ struct row_log_t { row_log_buf_t tail; /*!< writer context; protected by mutex and index->lock S-latch, or by index->lock X-latch only */ + byte* crypt_tail; /*!< writer context; + temporary buffer used in encryption, + decryption or NULL*/ row_log_buf_t head; /*!< reader context; protected by MDL only; modifiable by row_log_apply_ops() */ + byte* crypt_head; /*!< reader context; + temporary buffer used in encryption, + decryption or NULL */ const char* path; /*!< where to create temporary file during log operation */ }; @@ -349,6 +356,7 @@ row_log_online_op( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -368,11 +376,29 @@ row_log_online_op( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: /* We set the flag directly instead of invoking @@ -380,7 +406,9 @@ write_failed: because the index is not "public" yet. */ index->type |= DICT_CORRUPT; } + UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); + memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); log->tail.bytes = mrec_size - avail_size; @@ -465,6 +493,7 @@ row_log_table_close_func( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -484,11 +513,29 @@ row_log_table_close_func( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + log->index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: log->error = DB_ONLINE_LOG_TOO_BIG; @@ -2619,10 +2666,29 @@ all_done: goto func_exit; } + byte * buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for table %s\n", index->table_name); @@ -2929,9 +2995,22 @@ row_log_allocate( log->head.blocks = log->head.bytes = 0; log->head.total = 0; log->path = path; + log->crypt_tail = log->crypt_head = NULL; + log->index = index; dict_index_set_online_status(index, ONLINE_INDEX_CREATION); index->online_log = log; + if (log_tmp_is_encrypted()) { + ulint size = srv_sort_buf_size; + log->crypt_head = static_cast(os_mem_alloc_large(&size)); + log->crypt_tail = static_cast(os_mem_alloc_large(&size)); + + if (!log->crypt_head || !log->crypt_tail) { + row_log_free(log); + DBUG_RETURN(false); + } + } + /* While we might be holding an exclusive data dictionary lock here, in row_log_abort_sec() we will not always be holding it. Use atomic operations in both cases. */ @@ -2954,6 +3033,15 @@ row_log_free( row_log_block_free(log->tail); row_log_block_free(log->head); row_merge_file_destroy_low(log->fd); + + if (log->crypt_head) { + os_mem_free_large(log->crypt_head, srv_sort_buf_size); + } + + if (log->crypt_tail) { + os_mem_free_large(log->crypt_tail, srv_sort_buf_size); + } + mutex_free(&log->mutex); ut_free(log); log = 0; @@ -3445,11 +3533,29 @@ all_done: goto func_exit; } + byte* buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for index %s\n", index->name + 1); diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 408e94b3997..28b43d6abef 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -79,88 +79,6 @@ UNIV_INTERN char srv_disable_sort_file_cache; /* Maximum pending doc memory limit in bytes for a fts tokenization thread */ #define FTS_PENDING_DOC_MEMORY_LIMIT 1000000 - -/******************************************************//** -Encrypt a merge block. */ -static -void -row_merge_encrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - key_version = encryption_key_get_latest_version(crypt_data->key_id); - - /* Store key_version at the begining of the input buffer */ - mach_write_to_4((byte *)crypted_buf, key_version); - - int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } -} - -/******************************************************//** -Decrypt a merge block. */ -static -bool -row_merge_decrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - /* Read key_version from begining of the buffer */ - key_version = mach_read_from_4((byte *)input_buf); - - if (key_version == 0) { - /* block not encrypted */ - return false; - } - - int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } - - return (true); -} - #ifdef UNIV_DEBUG /******************************************************//** Display a merge tuple. */ @@ -279,7 +197,7 @@ row_merge_buf_create( ulint buf_size; mem_heap_t* heap; - max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE) + max_tuples = (srv_sort_buf_size) / ut_max(1, dict_index_get_min_size(index)); buf_size = (sizeof *buf); @@ -702,7 +620,7 @@ row_merge_buf_add( ut_ad(data_size < srv_sort_buf_size); /* Reserve bytes for the end marker of row_merge_block_t. */ - if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) { + if (buf->total_size + data_size >= srv_sort_buf_size) { DBUG_RETURN(0); } @@ -823,7 +741,7 @@ UT_SORT_FUNCTION_BODY(). /**********************************************************************//** Merge sort the tuple buffer in main memory. */ -static MY_ATTRIBUTE((nonnull(4,5))) +static void row_merge_tuple_sort( /*=================*/ @@ -874,7 +792,7 @@ row_merge_buf_write( { const dict_index_t* index = buf->index; ulint n_fields= dict_index_get_n_fields(index); - byte* b = &block[ROW_MERGE_RESERVE_SIZE]; + byte* b = &block[0]; for (ulint i = 0; i < buf->n_tuples; i++) { const mtuple_t* entry = &buf->tuples[i]; @@ -893,7 +811,7 @@ row_merge_buf_write( /* Write an "end-of-chunk" marker. */ ut_a(b < &block[srv_sort_buf_size]); - ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE); + ut_a(b == &block[0] + buf->total_size); *b++ = 0; #ifdef UNIV_DEBUG_VALGRIND /* The rest of the block is uninitialized. Initialize it @@ -943,7 +861,7 @@ row_merge_heap_create( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -951,12 +869,11 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size; - ibool success; + bool success; DBUG_EXECUTE_IF("row_merge_read_failure", return(FALSE);); @@ -971,10 +888,14 @@ row_merge_read( ofs, srv_sort_buf_size); /* For encrypted tables, decrypt data after reading and copy data */ - if (crypt_data && crypt_buf) { - if( row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) { - memcpy(buf, crypt_buf, srv_sort_buf_size); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, srv_sort_buf_size, + crypt_buf, ofs, space)) { + return (FALSE); } + + srv_stats.n_merge_blocks_decrypted.inc(); + memcpy(buf, crypt_buf, srv_sort_buf_size); } #ifdef POSIX_FADV_DONTNEED @@ -996,31 +917,32 @@ row_merge_read( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { size_t buf_len = srv_sort_buf_size; os_offset_t ofs = buf_len * (os_offset_t) offset; - ibool ret; + bool ret; void* out_buf = (void *)buf; DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE);); /* For encrypted tables, encrypt data before writing */ - if (crypt_data && crypt_buf) { - row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt((const byte *)buf, buf_len, + (byte *)crypt_buf, ofs, space)) { + return (FALSE); + } + + srv_stats.n_merge_blocks_encrypted.inc(); out_buf = crypt_buf; - } else { - /* Mark block unencrypted */ - mach_write_to_4((byte *)out_buf, 0); } ret = os_file_write_int_fd("(merge)", fd, out_buf, ofs, buf_len); @@ -1058,9 +980,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint data_size; @@ -1072,10 +993,6 @@ row_merge_read_rec( ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - extra_size = *b++; if (UNIV_UNLIKELY(!extra_size)) { @@ -1097,7 +1014,8 @@ row_merge_read_rec( if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) { if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { err_exit: /* Signal I/O error. */ *mrec = b; @@ -1105,7 +1023,7 @@ err_exit: } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; } extra_size = (extra_size & 0x7f) << 8; @@ -1127,13 +1045,14 @@ err_exit: memcpy(*buf, b, avail_size); if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the record. */ memcpy(*buf + avail_size, b, extra_size - avail_size); @@ -1189,13 +1108,14 @@ err_exit: #endif /* UNIV_DEBUG */ if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the rest of the record. */ memcpy(*buf + avail_size, b, extra_size + data_size - avail_size); @@ -1272,9 +1192,8 @@ row_merge_write_rec( ulint* foffs, /*!< in/out: file offset */ const mrec_t* mrec, /*!< in: record to write */ const ulint* offsets,/*!< in: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint size; @@ -1295,10 +1214,6 @@ row_merge_write_rec( size = extra_size + (extra_size >= 0x80) + rec_offs_data_size(offsets); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) { /* The record spans two blocks. Copy it to the temporary buffer first. */ @@ -1314,14 +1229,15 @@ row_merge_write_rec( memcpy(b, buf[0], avail_size); if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); /* Copy the rest. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; memcpy(b, buf[0] + avail_size, size - avail_size); b += size - avail_size; } else { @@ -1344,7 +1260,6 @@ row_merge_write_eof( byte* b, /*!< in: pointer to end of block */ int fd, /*!< in: file descriptor */ ulint* foffs, /*!< in/out: file offset */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1359,10 +1274,6 @@ row_merge_write_eof( } #endif /* UNIV_DEBUG */ - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - *b++ = 0; UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]); UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size); @@ -1374,7 +1285,8 @@ row_merge_write_eof( #endif /* UNIV_DEBUG_VALGRIND */ if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } @@ -1451,8 +1363,10 @@ containing the index entries for the indexes to be built. @param[in,out] sequence autoinc sequence @param[in,out] block file buffer @param[in,out] tmpfd temporary file handle -return DB_SUCCESS or error */ -static MY_ATTRIBUTE((nonnull(1,2,3,4,6,9,10,16), warn_unused_result)) +@param[in] pct_cost percent of task weight out of total alter job +@param[in,out] crypt_block crypted file buffer +@return DB_SUCCESS or error */ +static MY_ATTRIBUTE((warn_unused_result)) dberr_t row_merge_read_clustered_index( trx_t* trx, @@ -1472,11 +1386,8 @@ row_merge_read_clustered_index( ib_sequence_t& sequence, row_merge_block_t* block, int* tmpfd, - float pct_cost, /*!< in: percent of task weight - out of total alter job */ - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ - row_merge_block_t* crypt_block)/*!< in: in/out: crypted file - buffer */ + float pct_cost, + row_merge_block_t* crypt_block) { dict_index_t* clust_index; /* Clustered index */ mem_heap_t* row_heap; /* Heap memory to create @@ -2014,7 +1925,7 @@ write_buffers: row_merge_buf_write(buf, file, block); if (!row_merge_write(file->fd, file->offset++, - block, crypt_data, crypt_block, + block, crypt_block, new_table->space)) { err = DB_TEMP_FILE_WRITE_FAILURE; trx->error_key_num = i; @@ -2066,7 +1977,7 @@ write_buffers: if(read_rows % 1000 == 0) { /* Update progress for each 1000 rows */ curr_progress = (read_rows >= table_total_rows) ? - pct_cost : + pct_cost : ((pct_cost * read_rows) / table_total_rows); /* presenting 10.12% as 1012 integer */ onlineddl_pct_progress = curr_progress * 100; @@ -2195,9 +2106,8 @@ wait_again: &buf[2], b2, \ of->fd, &of->offset, \ mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL , \ - space); \ + space); \ if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \ goto corrupt; \ } \ @@ -2205,7 +2115,6 @@ wait_again: &buf[N], b##N, INDEX, \ file->fd, foffs##N, \ &mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[N * srv_sort_buf_size] : NULL, \ space); \ \ @@ -2220,7 +2129,7 @@ wait_again: /*************************************************************//** Merge two blocks of records on disk and write a bigger block. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_blocks( /*=============*/ @@ -2234,7 +2143,6 @@ row_merge_blocks( ulint* foffs1, /*!< in/out: offset of second source list in the file */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ row_merge_block_t* crypt_block,/*!< in: in/out: crypted file buffer */ ulint space) /*!< in: space id */ @@ -2270,9 +2178,11 @@ row_merge_blocks( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space) - || !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space) || + !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space)) { corrupt: mem_heap_free(heap); return(DB_CORRUPTION); @@ -2285,13 +2195,15 @@ corrupt: b0 = row_merge_read_rec( &block[0], &buf[0], b0, dup->index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); b1 = row_merge_read_rec( &block[srv_sort_buf_size], &buf[srv_sort_buf_size], b1, dup->index, file->fd, foffs1, &mrec1, offsets1, - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0) || UNIV_UNLIKELY(!b1 && mrec1)) { @@ -2337,7 +2249,8 @@ done1: b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space); return(b2 ? DB_SUCCESS : DB_CORRUPTION); } @@ -2345,8 +2258,8 @@ done1: /*************************************************************//** Copy a block of index entries. @return TRUE on success, FALSE on failure */ -static __attribute__((nonnull(1,2,3,4,5), warn_unused_result)) -ibool +static __attribute__((warn_unused_result)) +bool row_merge_blocks_copy( /*==================*/ const dict_index_t* index, /*!< in: index being created */ @@ -2354,9 +2267,8 @@ row_merge_blocks_copy( row_merge_block_t* block, /*!< in/out: 3 buffers */ ulint* foffs0, /*!< in/out: input file offset */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */ @@ -2384,7 +2296,8 @@ row_merge_blocks_copy( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space)) { corrupt: mem_heap_free(heap); return(FALSE); @@ -2396,7 +2309,8 @@ corrupt: b0 = row_merge_read_rec(&block[0], &buf[0], b0, index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0)) { @@ -2419,15 +2333,15 @@ done0: return(row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, - crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space) + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space) != NULL); } /*************************************************************//** Merge disk files. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6,7))) +static dberr_t row_merge( /*======*/ @@ -2443,9 +2357,8 @@ row_merge( ulint* run_offset, /*!< in/out: Array contains the first offset number for each merge run */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint foffs0; /*!< first input offset */ ulint foffs1; /*!< second input offset */ @@ -2493,7 +2406,8 @@ row_merge( error = row_merge_blocks(dup, file, block, &foffs0, &foffs1, &of, - crypt_data, crypt_block, space); + crypt_block, + space); if (error != DB_SUCCESS) { return(error); @@ -2514,7 +2428,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs0, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2532,7 +2447,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs1, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2588,9 +2504,8 @@ row_merge_sort( /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const ulint half = file->offset / 2; ulint num_runs; @@ -2650,7 +2565,8 @@ row_merge_sort( error = row_merge(trx, dup, file, block, tmpfd, &num_runs, run_offset, - crypt_data, crypt_block, space); + crypt_block, + space); if(update_progress) { merge_count++; @@ -2725,7 +2641,7 @@ row_merge_copy_blobs( Read sorted file containing index data tuples and insert these data tuples to the index @return DB_SUCCESS or error number */ -static __attribute__((nonnull(2,3,5), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_insert_index_tuples( /*==========================*/ @@ -2738,9 +2654,8 @@ row_merge_insert_index_tuples( const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const byte* b; mem_heap_t* heap; @@ -2774,7 +2689,8 @@ row_merge_insert_index_tuples( b = &block[0]; if (!row_merge_read(fd, foffs, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { error = DB_CORRUPTION; } else { buf = static_cast( @@ -2791,7 +2707,9 @@ row_merge_insert_index_tuples( b = row_merge_read_rec(block, buf, b, index, fd, &foffs, &mrec, offsets, - crypt_data, crypt_block, space); + crypt_block, + space); + if (UNIV_UNLIKELY(!b)) { /* End of list, or I/O error */ if (mrec) { @@ -3976,7 +3894,6 @@ row_merge_build_indexes( fts_psort_t* merge_info = NULL; ib_int64_t sig_count = 0; bool fts_psort_initiated = false; - fil_space_crypt_t * crypt_data = NULL; float total_static_cost = 0; float total_dynamic_cost = 0; @@ -4005,25 +3922,20 @@ row_merge_build_indexes( from concurrent DDL (e.g. drop table) by MDL-locks. */ fil_space_t* space = fil_space_acquire(new_table->space); - if (space) { - crypt_data = space->crypt_data; - } else { + if (!space) { DBUG_RETURN(DB_TABLESPACE_NOT_FOUND); } - /* If tablespace is encrypted, allocate additional buffer for + /* If temporal log file is encrypted allocate memory for encryption/decryption. */ - if (crypt_data && crypt_data->should_encrypt()) { + if (log_tmp_is_encrypted()) { crypt_block = static_cast( - os_mem_alloc_large(&block_size)); + os_mem_alloc_large(&block_size)); if (crypt_block == NULL) { fil_space_release(space); DBUG_RETURN(DB_OUT_OF_MEMORY); } - } else { - /* Not needed */ - crypt_data = NULL; } trx_start_if_not_started_xa(trx); @@ -4105,7 +4017,7 @@ row_merge_build_indexes( fts_sort_idx, psort_info, merge_files, key_numbers, n_indexes, add_cols, col_map, add_autoinc, sequence, block, &tmpfd, pct_cost, - crypt_data, crypt_block); + crypt_block); pct_progress += pct_cost; @@ -4242,7 +4154,8 @@ wait_again: trx, &dup, &merge_files[i], block, &tmpfd, true, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, + new_table->space); pct_progress += pct_cost; @@ -4280,7 +4193,8 @@ wait_again: trx->id, sort_idx, old_table, merge_files[i].fd, block, merge_files[i].n_rec, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, new_table->space); + pct_progress += pct_cost; if (global_system_variables.log_warnings > 2) { diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 1f1fef6bea6..d6070107432 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -2060,6 +2060,10 @@ srv_export_innodb_status(void) export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; export_vars.innodb_pages_encrypted = srv_stats.pages_encrypted; + export_vars.innodb_n_merge_blocks_encrypted = srv_stats.n_merge_blocks_encrypted; + export_vars.innodb_n_merge_blocks_decrypted = srv_stats.n_merge_blocks_decrypted; + export_vars.innodb_n_rowlog_blocks_encrypted = srv_stats.n_rowlog_blocks_encrypted; + export_vars.innodb_n_rowlog_blocks_decrypted = srv_stats.n_rowlog_blocks_decrypted; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; From 65d26d1f31070b5b9c05eba4eb8f4432e56055e0 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 15 Sep 2017 20:59:04 +0400 Subject: [PATCH 2/9] MDEV-10191 non convertible chars convert() resulted in Null instead "?" on Windows The problem was introduced by the patch for MDEV-7661, which (in addition to the fix itself) included an attempt to make CONVERT/CAST work in the same way with fields (i.e. return NULL in strict mode if a non-convertable character found). It appeared to be a bad idea and some users were affected by this behavior change. Changing CONVERT/CAST not depend on sql_mode (restoring pre-10.1.4 behavior). --- mysql-test/r/ctype_utf8.result | 29 ++++++++++++++++++++++-- mysql-test/r/ctype_utf8mb4.result | 4 ++-- mysql-test/r/ctype_utf8mb4_heap.result | 4 ++-- mysql-test/r/ctype_utf8mb4_innodb.result | 4 ++-- mysql-test/r/ctype_utf8mb4_myisam.result | 4 ++-- mysql-test/t/ctype_utf8.test | 12 ++++++++++ sql/item.cc | 4 ++-- sql/item.h | 3 +-- sql/item_timefunc.cc | 2 +- 9 files changed, 51 insertions(+), 15 deletions(-) diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result index 2960b648b84..2b53f43cfc9 100644 --- a/mysql-test/r/ctype_utf8.result +++ b/mysql-test/r/ctype_utf8.result @@ -1945,7 +1945,7 @@ Warnings: Warning 1300 Invalid utf8 character string: 'FD' select convert(char(0xff,0x8f) using utf8); convert(char(0xff,0x8f) using utf8) -NULL +?? Warnings: Warning 1300 Invalid utf8 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8)); @@ -2114,7 +2114,7 @@ Warnings: Warning 1300 Invalid utf8 character string: 'FF' select hex(convert(0xFF using utf8)); hex(convert(0xFF using utf8)) -NULL +3F Warnings: Warning 1300 Invalid utf8 character string: '\xFF' select hex(_utf8 0x616263FF); @@ -10473,5 +10473,30 @@ END DROP PROCEDURE p1; SET @@SQL_MODE=default; # +# MDEV-10191 non convertible chars convert() resulted in Null instead "?" on Windows +# +SET sql_mode='STRICT_TRANS_TABLES'; +SELECT CONVERT(_utf8 0xC499 USING latin1); +CONVERT(_utf8 0xC499 USING latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); +CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +SET sql_mode=default; +SELECT CONVERT(_utf8 0xC499 USING latin1); +CONVERT(_utf8 0xC499 USING latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); +CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +# # End of 10.1 tests # diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result index dd1c5249bc7..bac0fc34f80 100644 --- a/mysql-test/r/ctype_utf8mb4.result +++ b/mysql-test/r/ctype_utf8mb4.result @@ -1970,7 +1970,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -2139,7 +2139,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/r/ctype_utf8mb4_heap.result b/mysql-test/r/ctype_utf8mb4_heap.result index d70e009228e..46b0cc3789f 100644 --- a/mysql-test/r/ctype_utf8mb4_heap.result +++ b/mysql-test/r/ctype_utf8mb4_heap.result @@ -1802,7 +1802,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -1971,7 +1971,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/r/ctype_utf8mb4_innodb.result b/mysql-test/r/ctype_utf8mb4_innodb.result index 7d193f397ac..cfd9bf969ad 100644 --- a/mysql-test/r/ctype_utf8mb4_innodb.result +++ b/mysql-test/r/ctype_utf8mb4_innodb.result @@ -1930,7 +1930,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -2099,7 +2099,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/r/ctype_utf8mb4_myisam.result b/mysql-test/r/ctype_utf8mb4_myisam.result index 28cf36c7492..53ae410046f 100644 --- a/mysql-test/r/ctype_utf8mb4_myisam.result +++ b/mysql-test/r/ctype_utf8mb4_myisam.result @@ -1930,7 +1930,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -2099,7 +2099,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test index 5f1de609df1..e6792922b95 100644 --- a/mysql-test/t/ctype_utf8.test +++ b/mysql-test/t/ctype_utf8.test @@ -1983,6 +1983,18 @@ SET @@SQL_MODE=default; #DROP FUNCTION f1; #SET NAMES utf8; +--echo # +--echo # MDEV-10191 non convertible chars convert() resulted in Null instead "?" on Windows +--echo # + +SET sql_mode='STRICT_TRANS_TABLES'; +SELECT CONVERT(_utf8 0xC499 USING latin1); +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); + +SET sql_mode=default; +SELECT CONVERT(_utf8 0xC499 USING latin1); +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); + --echo # --echo # End of 10.1 tests diff --git a/sql/item.cc b/sql/item.cc index ae77e90fa85..d4f3525c27c 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -5628,7 +5628,7 @@ String_copier_for_item::copy_with_warn(CHARSET_INFO *dstcs, String *dst, srccs == &my_charset_bin ? dstcs->csname : srccs->csname, err.ptr()); - return m_thd->is_strict_mode(); + return false; } if (const char *pos= cannot_convert_error_pos()) { @@ -5641,7 +5641,7 @@ String_copier_for_item::copy_with_warn(CHARSET_INFO *dstcs, String *dst, ER_CANNOT_CONVERT_CHARACTER, ER_THD(m_thd, ER_CANNOT_CONVERT_CHARACTER), srccs->csname, buf, dstcs->csname); - return m_thd->is_strict_mode(); + return false; } return false; } diff --git a/sql/item.h b/sql/item.h index 8ee76a626d4..c338b14e340 100644 --- a/sql/item.h +++ b/sql/item.h @@ -548,8 +548,7 @@ class Item_func_not; class Item_splocal; /** - String_copier that honors the current sql_mode (strict vs non strict) - and can send warnings. + String_copier that sends Item specific warnings. */ class String_copier_for_item: public String_copier { diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index fa74a1903da..d8a37b37315 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -2396,7 +2396,7 @@ String *Item_char_typecast::copy(String *str, CHARSET_INFO *strcs) if (copier.copy_with_warn(cast_cs, &tmp_value, strcs, str->ptr(), str->length(), cast_length)) { - null_value= 1; // In strict mode: malformed data or could not convert + null_value= 1; // EOM return 0; } check_truncation_with_warn(str, copier.source_end_pos() - str->ptr()); From 8c4df595b8bdd186ee68aee8a7953958fa503d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 16 Sep 2017 09:53:29 +0300 Subject: [PATCH 3/9] MDEV-13807 mariabackup --apply-log-only does generate redo log by performing rollback and possibly other tasks Skip rollback and other redo-log-generating tasks if srv_apply_log_only is set. Instead of assigning the debug variable recv_no_log_write = FALSE, assign it to srv_apply_log_only, so that any unwanted writes are caught. --- .../mariabackup/apply-log-only-incr.result | 29 +++++++++ .../mariabackup/apply-log-only-incr.test | 62 +++++++++++++++++++ .../suite/mariabackup/apply-log-only.result | 10 +++ .../suite/mariabackup/apply-log-only.test | 25 ++++++++ storage/xtradb/log/log0recv.cc | 5 +- storage/xtradb/srv/srv0srv.cc | 2 +- storage/xtradb/srv/srv0start.cc | 10 ++- storage/xtradb/trx/trx0rseg.cc | 2 + storage/xtradb/trx/trx0sys.cc | 7 +-- storage/xtradb/trx/trx0trx.cc | 1 + storage/xtradb/trx/trx0undo.cc | 1 + 11 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 mysql-test/suite/mariabackup/apply-log-only-incr.result create mode 100644 mysql-test/suite/mariabackup/apply-log-only-incr.test create mode 100644 mysql-test/suite/mariabackup/apply-log-only.result create mode 100644 mysql-test/suite/mariabackup/apply-log-only.test diff --git a/mysql-test/suite/mariabackup/apply-log-only-incr.result b/mysql-test/suite/mariabackup/apply-log-only-incr.result new file mode 100644 index 00000000000..2baed8c1db9 --- /dev/null +++ b/mysql-test/suite/mariabackup/apply-log-only-incr.result @@ -0,0 +1,29 @@ +call mtr.add_suppression("InnoDB: New log files created"); +CREATE TABLE t(a INT UNSIGNED PRIMARY KEY) ENGINE INNODB; +INSERT INTO t VALUES(0); +COMMIT; +start transaction; +NOT FOUND /Rollback of trx with id/ in current_test +# expect NOT FOUND +NOT FOUND /Rollback of trx with id/ in current_test +# expect NOT FOUND +commit; +SELECT count(*) FROM t; +count(*) +201 +# Restore and check results +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT COUNT(*) FROM t; +COUNT(*) +1 +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SELECT COUNT(*) FROM t; +COUNT(*) +201 +SELECT * FROM t; +a +0 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/apply-log-only-incr.test b/mysql-test/suite/mariabackup/apply-log-only-incr.test new file mode 100644 index 00000000000..a580d0ffe04 --- /dev/null +++ b/mysql-test/suite/mariabackup/apply-log-only-incr.test @@ -0,0 +1,62 @@ +--source include/have_innodb.inc + +call mtr.add_suppression("InnoDB: New log files created"); + +let $basedir=$MYSQLTEST_VARDIR/tmp/backup; +let $incremental_dir=$MYSQLTEST_VARDIR/tmp/backup_inc1; + +CREATE TABLE t(a INT UNSIGNED PRIMARY KEY) ENGINE INNODB; +INSERT INTO t VALUES(0); +COMMIT; + +start transaction; +--disable_query_log +let $n=100; +while ($n) { +eval INSERT t VALUES(101-$n); +dec $n; +} + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log +let $n=100; +while ($n) { +eval INSERT t VALUES(201-$n); +dec $n; +} +--enable_query_log + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --ftwrl-wait-timeout=5 --ftwrl-wait-threshold=300 --ftwrl-wait-query-type=all --target-dir=$incremental_dir --incremental-basedir=$basedir ; +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir ; +--enable_result_log + +let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test; +--let SEARCH_PATTERN= Rollback of trx with id +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ; + +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +commit; +SELECT count(*) FROM t; +echo # Restore and check results; +--let $targetdir=$basedir +--let $restart_parameters= --innodb-force-recovery=3 +--source include/restart_and_restore.inc + +rmdir $basedir; + +SELECT COUNT(*) FROM t; +SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SELECT COUNT(*) FROM t; + +--let $restart_parameters= +--source include/restart_mysqld.inc + +SELECT * FROM t; +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/apply-log-only.result b/mysql-test/suite/mariabackup/apply-log-only.result new file mode 100644 index 00000000000..04b9c0d8ee2 --- /dev/null +++ b/mysql-test/suite/mariabackup/apply-log-only.result @@ -0,0 +1,10 @@ +call mtr.add_suppression("InnoDB: New log files created"); +CREATE TABLE t(a varchar(60)) ENGINE INNODB; +start transaction; +INSERT INTO t VALUES(1); +NOT FOUND /Rollback of trx with id/ in current_test +# expect NOT FOUND +SELECT count(*) FROM t; +count(*) +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/apply-log-only.test b/mysql-test/suite/mariabackup/apply-log-only.test new file mode 100644 index 00000000000..96a251fc03b --- /dev/null +++ b/mysql-test/suite/mariabackup/apply-log-only.test @@ -0,0 +1,25 @@ +--source include/have_innodb.inc +call mtr.add_suppression("InnoDB: New log files created"); + +let $basedir=$MYSQLTEST_VARDIR/tmp/backup; + +CREATE TABLE t(a varchar(60)) ENGINE INNODB; +start transaction; +INSERT INTO t VALUES(1); + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$basedir; +--enable_result_log + +exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir ; + +let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test; +--let SEARCH_PATTERN= Rollback of trx with id +--source include/search_pattern_in_file.inc +--echo # expect NOT FOUND + +SELECT count(*) FROM t; +DROP TABLE t; + +# Cleanup +rmdir $basedir; diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 75fc7713c5b..1943fb51f37 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -1980,7 +1980,7 @@ recv_apply_hashed_log_recs(bool last_batch) mutex_enter(&(log_sys->mutex)); mutex_enter(&(recv_sys->mutex)); - ut_d(recv_no_log_write = FALSE); + ut_d(recv_no_log_write = srv_apply_log_only); recv_no_ibuf_operations = FALSE; } @@ -3461,7 +3461,8 @@ recv_recovery_from_checkpoint_finish(void) that the data dictionary tables will be free of any locks. The data dictionary latch should guarantee that there is at most one data dictionary transaction active at a time. */ - if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO) { + if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO + && !srv_apply_log_only) { trx_rollback_or_clean_recovered(FALSE); } } diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index d6070107432..cbfe54b1105 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -2470,7 +2470,7 @@ srv_get_active_thread_type(void) { srv_thread_type ret = SRV_NONE; - if (srv_read_only_mode) { + if (srv_read_only_mode || srv_apply_log_only) { return(SRV_NONE); } diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 94e680d0e80..2aa2426ca9c 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -767,7 +767,7 @@ create_log_files( /* Create a log checkpoint. */ mutex_enter(&log_sys->mutex); - ut_d(recv_no_log_write = FALSE); + ut_d(recv_no_log_write = srv_apply_log_only); recv_reset_logs( #ifdef UNIV_LOG_ARCHIVE UT_LIST_GET_FIRST(log_sys->log_groups)->archived_file_no, @@ -2748,6 +2748,11 @@ files_checked: recv_recovery_from_checkpoint_finish(); + if (srv_apply_log_only) { + ut_ad(IS_XTRABACKUP()); + goto skip_processes; + } + if (srv_force_recovery < SRV_FORCE_NO_IBUF_MERGE) { /* The following call is necessary for the insert buffer to work with multiple tablespaces. We must @@ -3156,6 +3161,7 @@ files_checked: os_thread_create(btr_defragment_thread, NULL, NULL); } +skip_processes: srv_was_started = TRUE; return(DB_SUCCESS); @@ -3318,7 +3324,7 @@ innodb_shutdown() srv_misc_tmpfile = 0; } - if (!srv_read_only_mode) { + if (!srv_read_only_mode && !srv_apply_log_only) { dict_stats_thread_deinit(); fil_crypt_threads_cleanup(); btr_scrub_cleanup(); diff --git a/storage/xtradb/trx/trx0rseg.cc b/storage/xtradb/trx/trx0rseg.cc index 16fa334872b..38719a1c771 100644 --- a/storage/xtradb/trx/trx0rseg.cc +++ b/storage/xtradb/trx/trx0rseg.cc @@ -122,9 +122,11 @@ trx_rseg_mem_free( mutex_free(&rseg->mutex); + if (!srv_apply_log_only) { /* There can't be any active transactions. */ ut_a(UT_LIST_GET_LEN(rseg->update_undo_list) == 0); ut_a(UT_LIST_GET_LEN(rseg->insert_undo_list) == 0); + } for (undo = UT_LIST_GET_FIRST(rseg->update_undo_cached); undo != NULL; diff --git a/storage/xtradb/trx/trx0sys.cc b/storage/xtradb/trx/trx0sys.cc index d2b1a8f9f0d..83bb28b3c07 100644 --- a/storage/xtradb/trx/trx0sys.cc +++ b/storage/xtradb/trx/trx0sys.cc @@ -1346,8 +1346,7 @@ trx_sys_close(void) ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx || srv_read_only_mode || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO - || (IS_XTRABACKUP() && srv_apply_log_only)); - + || srv_apply_log_only); while ((trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list)) != NULL) { trx_free_prepared(trx); @@ -1378,7 +1377,7 @@ trx_sys_close(void) UT_LIST_REMOVE(view_list, trx_sys->view_list, prev_view); } - if (!IS_XTRABACKUP() || !srv_apply_log_only) { + if (!srv_apply_log_only) { ut_a(UT_LIST_GET_LEN(trx_sys->view_list) == 0); ut_a(UT_LIST_GET_LEN(trx_sys->ro_trx_list) == 0); ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == 0); @@ -1430,7 +1429,7 @@ ulint trx_sys_any_active_transactions(void) /*=================================*/ { - if (IS_XTRABACKUP() && srv_apply_log_only) { + if (srv_apply_log_only) { return(0); } mutex_enter(&trx_sys->mutex); diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index cfc1f83053a..ef6e7d01cc4 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -483,6 +483,7 @@ trx_free_prepared( || (trx_state_eq(trx, TRX_STATE_ACTIVE) && trx->is_recovered && (srv_read_only_mode + || srv_apply_log_only || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO))); ut_a(trx->magic_n == TRX_MAGIC_N); diff --git a/storage/xtradb/trx/trx0undo.cc b/storage/xtradb/trx/trx0undo.cc index 220589dd9ff..cb064e3d485 100644 --- a/storage/xtradb/trx/trx0undo.cc +++ b/storage/xtradb/trx/trx0undo.cc @@ -2037,6 +2037,7 @@ trx_undo_free_prepared( /* lock_trx_release_locks() assigns trx->is_recovered=false */ ut_a(srv_read_only_mode + || srv_apply_log_only || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); break; default: From 93087d5fe74b0078e0b6db4233710648c79043cc Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 15 Sep 2017 15:10:36 +0000 Subject: [PATCH 4/9] Fix some warnings --- storage/innobase/include/ut0timer.ic | 2 +- storage/xtradb/btr/btr0cur.cc | 2 +- storage/xtradb/include/page0zip.ic | 2 +- storage/xtradb/include/ut0timer.ic | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/innobase/include/ut0timer.ic b/storage/innobase/include/ut0timer.ic index 027e89c6279..46dcd0cb718 100644 --- a/storage/innobase/include/ut0timer.ic +++ b/storage/innobase/include/ut0timer.ic @@ -106,7 +106,7 @@ ut_microseconds_to_timer( /*=====================*/ ulonglong when) /*!< in: time where to calculate */ { - double ret = when; + double ret = (double)when; ret *= (double)(ut_timer.frequency); ret /= 1000000.0; return (ulonglong)ret; diff --git a/storage/xtradb/btr/btr0cur.cc b/storage/xtradb/btr/btr0cur.cc index 1a27fd0d6ea..59470b95c35 100644 --- a/storage/xtradb/btr/btr0cur.cc +++ b/storage/xtradb/btr/btr0cur.cc @@ -4379,7 +4379,7 @@ btr_estimate_number_of_different_key_vals( if (index->stat_index_size > 1) { n_sample_pages = (srv_stats_transient_sample_pages < index->stat_index_size) ? ut_min(index->stat_index_size, - log2(index->stat_index_size)*srv_stats_transient_sample_pages) + ulint(log2(index->stat_index_size)*srv_stats_transient_sample_pages)) : index->stat_index_size; } diff --git a/storage/xtradb/include/page0zip.ic b/storage/xtradb/include/page0zip.ic index bee35a48cc2..c6b155b73a0 100644 --- a/storage/xtradb/include/page0zip.ic +++ b/storage/xtradb/include/page0zip.ic @@ -174,7 +174,7 @@ page_zip_rec_needs_ext( ulint zip_size) /*!< in: compressed page size in bytes, or 0 */ { ut_ad(rec_size - > (comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES)); + > ulint((comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES))); ut_ad(ut_is_2pow(zip_size)); ut_ad(comp || !zip_size); diff --git a/storage/xtradb/include/ut0timer.ic b/storage/xtradb/include/ut0timer.ic index 62e17a10fb1..815726e9d0a 100644 --- a/storage/xtradb/include/ut0timer.ic +++ b/storage/xtradb/include/ut0timer.ic @@ -106,7 +106,7 @@ ut_microseconds_to_timer( /*=====================*/ ulonglong when) /*!< in: time where to calculate */ { - double ret = when; + double ret = (double)when; ret *= (double)(ut_timer.frequency); ret /= 1000000.0; return (ulonglong)ret; From ad17e8e518c85a156d05f091874dd5c2ddfafa32 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 16 Sep 2017 09:36:21 +0000 Subject: [PATCH 5/9] MDEV-13821 : mariabackup sometimes could lose ib_logf(FATAL) messages, The messages are getting lost because they are written with fprintf() but without fflush(), so abort() would lose buffered text. Applied fix from 10.2, which makes in_logf() use sql_print_information(), which in turn does proper flush after each message --- extra/mariabackup/xtrabackup.cc | 6 +++++ storage/xtradb/handler/ha_innodb.cc | 36 ++++++++++++----------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index c746b469329..c42a282e203 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -95,6 +95,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include #include +#include /* TODO: replace with appropriate macros used in InnoDB 5.6 */ #define PAGE_ZIP_MIN_SIZE_SHIFT 10 @@ -6483,6 +6484,11 @@ int main(int argc, char **argv) system_charset_info = &my_charset_utf8_general_ci; key_map_full.set_all(); + logger.init_base(); + logger.set_handlers(LOG_FILE, LOG_NONE, LOG_NONE); + mysql_mutex_init(key_LOCK_error_log, &LOCK_error_log, + MY_MUTEX_INIT_FAST); + handle_options(argc, argv, &client_defaults, &server_defaults); int argc_server; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 4af523b1018..9fb73f98275 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -21949,27 +21949,21 @@ ib_logf( str = static_cast(malloc(BUFSIZ)); my_vsnprintf(str, BUFSIZ, format, args); #endif /* __WIN__ */ - if (!IS_XTRABACKUP()) { - switch (level) { - case IB_LOG_LEVEL_INFO: - sql_print_information("InnoDB: %s", str); - break; - case IB_LOG_LEVEL_WARN: - sql_print_warning("InnoDB: %s", str); - break; - case IB_LOG_LEVEL_ERROR: - sql_print_error("InnoDB: %s", str); - sd_notifyf(0, "STATUS=InnoDB: Error: %s", str); - break; - case IB_LOG_LEVEL_FATAL: - sql_print_error("InnoDB: %s", str); - sd_notifyf(0, "STATUS=InnoDB: Fatal: %s", str); - break; - } - } - else { - /* Don't use server logger for XtraBackup, just print to stderr. */ - fprintf(stderr, "InnoDB: %s\n", str); + switch (level) { + case IB_LOG_LEVEL_INFO: + sql_print_information("InnoDB: %s", str); + break; + case IB_LOG_LEVEL_WARN: + sql_print_warning("InnoDB: %s", str); + break; + case IB_LOG_LEVEL_ERROR: + sql_print_error("InnoDB: %s", str); + sd_notifyf(0, "STATUS=InnoDB: Error: %s", str); + break; + case IB_LOG_LEVEL_FATAL: + sql_print_error("InnoDB: %s", str); + sd_notifyf(0, "STATUS=InnoDB: Fatal: %s", str); + break; } va_end(args); From 1d7bc3b58263153d18198a6bde03b2b2b6519a77 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 16 Sep 2017 09:44:52 +0000 Subject: [PATCH 6/9] Innodb : do not call fflush() in os_get_last_error_low(), if no error message was written. --- storage/innobase/os/os0file.cc | 3 +-- storage/xtradb/os/os0file.cc | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 5a945aae7bc..44093935bc7 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -569,10 +569,9 @@ os_file_get_last_error_low( REFMAN "operating-system-error-codes.html\n"); } + fflush(stderr); } - fflush(stderr); - if (err == ERROR_FILE_NOT_FOUND) { return(OS_FILE_NOT_FOUND); } else if (err == ERROR_DISK_FULL) { diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 32536306320..f3c72576501 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -676,10 +676,9 @@ os_file_get_last_error_low( REFMAN "operating-system-error-codes.html\n"); } + fflush(stderr); } - fflush(stderr); - if (err == ERROR_FILE_NOT_FOUND) { return(OS_FILE_NOT_FOUND); } else if (err == ERROR_DISK_FULL) { From 75dd3bcb4c815202ddb147a1ccffb2b691f2c0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 16 Sep 2017 15:05:22 +0300 Subject: [PATCH 7/9] Clean up after commit 93087d5fe74b0078e0b6db4233710648c79043cc Apply the same changes to both InnoDB and XtraDB. --- storage/innobase/btr/btr0cur.cc | 2 +- storage/innobase/include/page0zip.ic | 2 +- storage/xtradb/include/page0zip.ic | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index d848ee538c6..5597a085525 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -4106,7 +4106,7 @@ btr_estimate_number_of_different_key_vals( if (index->stat_index_size > 1) { n_sample_pages = (srv_stats_transient_sample_pages < index->stat_index_size) ? ut_min(index->stat_index_size, - log2(index->stat_index_size)*srv_stats_transient_sample_pages) + ulint(log2(index->stat_index_size)*srv_stats_transient_sample_pages)) : index->stat_index_size; } diff --git a/storage/innobase/include/page0zip.ic b/storage/innobase/include/page0zip.ic index bee35a48cc2..3fa8a08c4f9 100644 --- a/storage/innobase/include/page0zip.ic +++ b/storage/innobase/include/page0zip.ic @@ -174,7 +174,7 @@ page_zip_rec_needs_ext( ulint zip_size) /*!< in: compressed page size in bytes, or 0 */ { ut_ad(rec_size - > (comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES)); + > ulint(comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES)); ut_ad(ut_is_2pow(zip_size)); ut_ad(comp || !zip_size); diff --git a/storage/xtradb/include/page0zip.ic b/storage/xtradb/include/page0zip.ic index c6b155b73a0..3fa8a08c4f9 100644 --- a/storage/xtradb/include/page0zip.ic +++ b/storage/xtradb/include/page0zip.ic @@ -174,7 +174,7 @@ page_zip_rec_needs_ext( ulint zip_size) /*!< in: compressed page size in bytes, or 0 */ { ut_ad(rec_size - > ulint((comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES))); + > ulint(comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES)); ut_ad(ut_is_2pow(zip_size)); ut_ad(comp || !zip_size); From f24d36ae1e91cb98462139dfae41211b9e9a3a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 16 Sep 2017 21:13:47 +0300 Subject: [PATCH 8/9] Clean up a directory to avoid a failure of another test --- mysql-test/suite/mariabackup/apply-log-only-incr.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/mariabackup/apply-log-only-incr.test b/mysql-test/suite/mariabackup/apply-log-only-incr.test index a580d0ffe04..aa5110cc51e 100644 --- a/mysql-test/suite/mariabackup/apply-log-only-incr.test +++ b/mysql-test/suite/mariabackup/apply-log-only-incr.test @@ -50,6 +50,7 @@ echo # Restore and check results; --source include/restart_and_restore.inc rmdir $basedir; +rmdir $incremental_dir; SELECT COUNT(*) FROM t; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; From 836d4e74d9416a1af839f8068c5a07bee6f4b71e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 16 Sep 2017 16:55:16 +0300 Subject: [PATCH 9/9] Write proper tests for MDEV-12634: Uninitialised ROW_MERGE_RESERVE_SIZE bytes Introduce innodb_encrypt_log.combinations and prove that the encryption and decryption take place during both online ADD INDEX (WL#5266) and online table-rebuilding ALTER (WL#6625). --- .../include/innodb_encrypt_log.combinations | 7 ++ mysql-test/include/innodb_encrypt_log.inc | 4 + .../suite/encryption/r/innodb-rowlog.result | 30 ------ .../suite/encryption/t/innodb-rowlog.opt | 7 -- .../suite/encryption/t/innodb-rowlog.test | 90 ------------------ .../innodb/r/innodb-index-online,crypt.rdiff | 20 ++++ .../suite/innodb/r/innodb-index-online.result | 87 ++++++++++++++++++ .../innodb/r/innodb-table-online,crypt.rdiff | 20 ++++ .../suite/innodb/r/innodb-table-online.result | 91 ++++++++++++++++--- .../suite/innodb/t/innodb-index-online.opt | 2 +- .../suite/innodb/t/innodb-index-online.test | 77 +++++++++++++++- .../innodb/t/innodb-table-online-master.opt | 2 +- .../suite/innodb/t/innodb-table-online.test | 80 ++++++++++++++-- storage/innobase/row/row0merge.cc | 4 - storage/xtradb/row/row0merge.cc | 4 - 15 files changed, 365 insertions(+), 160 deletions(-) create mode 100644 mysql-test/include/innodb_encrypt_log.combinations create mode 100644 mysql-test/include/innodb_encrypt_log.inc delete mode 100644 mysql-test/suite/encryption/r/innodb-rowlog.result delete mode 100644 mysql-test/suite/encryption/t/innodb-rowlog.opt delete mode 100644 mysql-test/suite/encryption/t/innodb-rowlog.test create mode 100644 mysql-test/suite/innodb/r/innodb-index-online,crypt.rdiff create mode 100644 mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff diff --git a/mysql-test/include/innodb_encrypt_log.combinations b/mysql-test/include/innodb_encrypt_log.combinations new file mode 100644 index 00000000000..fd21a57c3c2 --- /dev/null +++ b/mysql-test/include/innodb_encrypt_log.combinations @@ -0,0 +1,7 @@ +[crypt] +innodb_encrypt_log=ON +innodb_encryption_rotate_key_age=1 +plugin_load_add=$DEBUG_KEY_MANAGEMENT_SO + +[clear] +skip_innodb_encrypt_log diff --git a/mysql-test/include/innodb_encrypt_log.inc b/mysql-test/include/innodb_encrypt_log.inc new file mode 100644 index 00000000000..5beebeae81f --- /dev/null +++ b/mysql-test/include/innodb_encrypt_log.inc @@ -0,0 +1,4 @@ +# The goal of including this file is to enable innodb_encrypt_log combinations +# (see include/innodb_encrypt_log.combinations) + +--source include/have_innodb.inc diff --git a/mysql-test/suite/encryption/r/innodb-rowlog.result b/mysql-test/suite/encryption/r/innodb-rowlog.result deleted file mode 100644 index 873041ab421..00000000000 --- a/mysql-test/suite/encryption/r/innodb-rowlog.result +++ /dev/null @@ -1,30 +0,0 @@ -SET GLOBAL innodb_file_format = `Barracuda`; -create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; -SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; -alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; -SET GLOBAL innodb_encryption_rotate_key_age = 1; -create table t2(id int) engine=innodb; -SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; -alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; -insert into t1(credit_card) select credit_card from t1; -insert into t1(credit_card) select credit_card from t1; -insert into t1(credit_card) select credit_card from t1; -drop table t2; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value > 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; -variable_value > 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; -variable_value > 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; -variable_value > 0 -1 -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; -variable_value > 0 -1 -drop table t1; -SET GLOBAL innodb_file_format=Antelope; -SET GLOBAL innodb_encryption_rotate_key_age=15; diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.opt b/mysql-test/suite/encryption/t/innodb-rowlog.opt deleted file mode 100644 index 6338ddbde35..00000000000 --- a/mysql-test/suite/encryption/t/innodb-rowlog.opt +++ /dev/null @@ -1,7 +0,0 @@ ---aria-encrypt-tables ---encrypt-tmp-disk-tables ---innodb-encrypt-tables ---innodb-encrypt-log ---innodb-encryption-rotate-key-age=15 ---innodb-encryption-threads=4 ---innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.test b/mysql-test/suite/encryption/t/innodb-rowlog.test deleted file mode 100644 index 135293ef4b9..00000000000 --- a/mysql-test/suite/encryption/t/innodb-rowlog.test +++ /dev/null @@ -1,90 +0,0 @@ --- source include/have_innodb.inc --- source include/have_example_key_management_plugin.inc -# needs dbug_dbug --- source include/have_debug.inc - ---disable_warnings ---disable_query_log -let $innodb_file_format_orig = `SELECT @@innodb_file_format`; -let $key_age = `SELECT @@innodb_encryption_rotate_key_age`; ---enable_query_log - -SET GLOBAL innodb_file_format = `Barracuda`; ---enable_warnings - -let $MYSQLD_DATADIR = `SELECT @@datadir`; -let $MYSQLD_TMPDIR = `SELECT @@tmpdir`; - -# -# Create a table that will be encrypted and put some sensitive data to it (credit card numbers) -# - -create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; - -let $rows = 15000; ---disable_query_log -begin; -while ($rows) -{ - eval insert into t1 values(NULL, '0000-0000-0000-0000','private_data'); - dec $rows; -} -commit; ---enable_query_log - ---let $wait_timeout= 600 ---let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 ---source include/wait_condition.inc - -# -# Now we create secondary index for credit_card column in parallel we create new rows -# forcing alter table to wait so that row log is used. -# -SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; -send alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; -# -# Force key rotation and create second index for same table -# -connect (con2,localhost,root,,); -SET GLOBAL innodb_encryption_rotate_key_age = 1; -create table t2(id int) engine=innodb; -SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; -send alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; -# -# Create new rows to row log -# -connect (con1,localhost,root,,); -insert into t1(credit_card) select credit_card from t1; -insert into t1(credit_card) select credit_card from t1; -insert into t1(credit_card) select credit_card from t1; - -connection default; -reap; -disconnect con1; - -connection con2; -reap; -drop table t2; - -connection default; -disconnect con2; - -let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; ---source include/wait_condition.inc - -# -# Verify that both merge blocks and row log blocks are encryted and decrypted -# - -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; -SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; - -drop table t1; - ---disable_warnings -eval SET GLOBAL innodb_file_format=$innodb_file_format_orig; -eval SET GLOBAL innodb_encryption_rotate_key_age=$key_age; ---enable_warnings diff --git a/mysql-test/suite/innodb/r/innodb-index-online,crypt.rdiff b/mysql-test/suite/innodb/r/innodb-index-online,crypt.rdiff new file mode 100644 index 00000000000..767e36f3bfb --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-index-online,crypt.rdiff @@ -0,0 +1,20 @@ +--- innodb-index-online.result ++++ innodb-index-online,crypt.reject +@@ -260,7 +260,7 @@ + @merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, + @rowlog_encrypt_1>@rowlog_encrypt_0; + sort_balance @merge_encrypt_1>@merge_encrypt_0 @merge_decrypt_1>@merge_decrypt_0 @rowlog_encrypt_1>@rowlog_encrypt_0 +-0 0 0 0 ++0 1 1 1 + SET DEBUG_SYNC = 'now SIGNAL dml2_done'; + ERROR HY000: Creating index 'c2e' required more than 'innodb_online_alter_log_max_size' bytes of modification log. Please try again. + SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; +@@ -364,7 +364,7 @@ + @rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, + @rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; + sort_encrypted sort_decrypted log_encrypted log_decrypted +-0 0 0 0 ++1 1 1 1 + SELECT COUNT(c22f) FROM t1; + COUNT(c22f) + 320 diff --git a/mysql-test/suite/innodb/r/innodb-index-online.result b/mysql-test/suite/innodb/r/innodb-index-online.result index 77b7bc365aa..d6941770cd3 100644 --- a/mysql-test/suite/innodb/r/innodb-index-online.result +++ b/mysql-test/suite/innodb/r/innodb-index-online.result @@ -180,6 +180,15 @@ t1 CREATE TABLE `t1` ( PRIMARY KEY (`c1`), KEY `c2d` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=1 +SET @merge_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); SET DEBUG_SYNC = 'row_log_apply_before SIGNAL c2e_created WAIT_FOR dml2_done'; SET lock_wait_timeout = 10; ALTER TABLE t1 DROP INDEX c2d, ADD INDEX c2e(c2), @@ -209,6 +218,22 @@ BEGIN; UPDATE t1 SET c2 = c2 + 1; DELETE FROM t1; ROLLBACK; +BEGIN; +DELETE FROM t1; +ROLLBACK; +UPDATE t1 SET c2 = c2 + 1; +BEGIN; +UPDATE t1 SET c2 = c2 + 1; +DELETE FROM t1; +ROLLBACK; +BEGIN; +DELETE FROM t1; +ROLLBACK; +UPDATE t1 SET c2 = c2 + 1; +BEGIN; +UPDATE t1 SET c2 = c2 + 1; +DELETE FROM t1; +ROLLBACK; SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; name count ddl_background_drop_indexes 0 @@ -220,6 +245,22 @@ INNER JOIN INFORMATION_SCHEMA.INNODB_SYS_FIELDS sf ON si.index_id = sf.index_id WHERE si.name = '?c2e'; name pos c2 0 +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SELECT +(@merge_encrypt_1-@merge_encrypt_0)- +(@merge_decrypt_1-@merge_decrypt_0) as sort_balance, +@merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, +@rowlog_encrypt_1>@rowlog_encrypt_0; +sort_balance @merge_encrypt_1>@merge_encrypt_0 @merge_decrypt_1>@merge_decrypt_0 @rowlog_encrypt_1>@rowlog_encrypt_0 +0 0 0 0 SET DEBUG_SYNC = 'now SIGNAL dml2_done'; ERROR HY000: Creating index 'c2e' required more than 'innodb_online_alter_log_max_size' bytes of modification log. Please try again. SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; @@ -245,6 +286,18 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 0 ddl_pending_alter_table 0 +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); SET DEBUG_SYNC = 'row_log_apply_before SIGNAL c2f_created WAIT_FOR dml3_done'; ALTER TABLE t1 ADD INDEX c2f(c2); SET DEBUG_SYNC = 'now WAIT_FOR c2f_created'; @@ -262,6 +315,14 @@ BEGIN; UPDATE t1 SET c2 = c2 + 1; DELETE FROM t1; ROLLBACK; +BEGIN; +INSERT INTO t1 SELECT 320 + c1, c2, c3 FROM t1 WHERE c1 > 160; +DELETE FROM t1 WHERE c1 > 320; +ROLLBACK; +BEGIN; +UPDATE t1 SET c2 = c2 + 1; +DELETE FROM t1; +ROLLBACK; SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; name count ddl_background_drop_indexes 0 @@ -278,6 +339,32 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 0 ddl_pending_alter_table 0 +SET @merge_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); +SELECT +(@merge_encrypt_2-@merge_encrypt_1)- +(@merge_decrypt_2-@merge_decrypt_1) as sort_balance, +(@rowlog_encrypt_2-@rowlog_encrypt_1)- +(@rowlog_decrypt_2-@rowlog_decrypt_1) as log_balance; +sort_balance log_balance +0 0 +SELECT +@merge_encrypt_2-@merge_encrypt_1>0 as sort_encrypted, +@merge_decrypt_2-@merge_decrypt_1>0 as sort_decrypted, +@rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, +@rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; +sort_encrypted sort_decrypted log_encrypted log_decrypted +0 0 0 0 SELECT COUNT(c22f) FROM t1; COUNT(c22f) 320 diff --git a/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff b/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff new file mode 100644 index 00000000000..fa36666ba48 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-table-online,crypt.rdiff @@ -0,0 +1,20 @@ +--- innodb-table-online.result ++++ innodb-table-online,crypt.reject +@@ -254,7 +254,7 @@ + @merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, + @rowlog_encrypt_1>@rowlog_encrypt_0; + sort_balance @merge_encrypt_1>@merge_encrypt_0 @merge_decrypt_1>@merge_decrypt_0 @rowlog_encrypt_1>@rowlog_encrypt_0 +-0 0 0 0 ++0 1 1 1 + SET DEBUG_SYNC = 'now SIGNAL dml2_done'; + # session con1 + ERROR HY000: Creating index 'PRIMARY' required more than 'innodb_online_alter_log_max_size' bytes of modification log. Please try again. +@@ -345,7 +345,7 @@ + @rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, + @rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; + sort_encrypted sort_decrypted log_encrypted log_decrypted +-0 0 0 0 ++1 1 1 1 + ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)); + ERROR 23000: Duplicate entry '' for key 'PRIMARY' + UPDATE t1 SET c3 = NULL WHERE c3 = ''; diff --git a/mysql-test/suite/innodb/r/innodb-table-online.result b/mysql-test/suite/innodb/r/innodb-table-online.result index a352f0c2a90..b8cd99659fa 100644 --- a/mysql-test/suite/innodb/r/innodb-table-online.result +++ b/mysql-test/suite/innodb/r/innodb-table-online.result @@ -3,7 +3,7 @@ call mtr.add_suppression("InnoDB: Error: table 'test/t1'"); call mtr.add_suppression("MySQL is trying to open a table handle but the .ibd file for"); SET @global_innodb_file_per_table_orig = @@global.innodb_file_per_table; SET GLOBAL innodb_file_per_table = on; -CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 TEXT NOT NULL) +CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 CHAR(255) NOT NULL) ENGINE = InnoDB; INSERT INTO t1 VALUES (1,1,''), (2,2,''), (3,3,''), (4,4,''), (5,5,''); SET GLOBAL innodb_monitor_enable = module_ddl; @@ -53,7 +53,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, PRIMARY KEY (`c1`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT BEGIN; @@ -81,7 +81,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, UNIQUE KEY `c2` (`c2`), UNIQUE KEY `c2_2` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT @@ -92,7 +92,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, UNIQUE KEY `c2` (`c2`), UNIQUE KEY `c2_2` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT @@ -118,7 +118,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, UNIQUE KEY `c2` (`c2`), UNIQUE KEY `c2_2` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT @@ -173,17 +173,26 @@ INSERT INTO t1 SELECT 20 + c1, c2, c3 FROM t1; INSERT INTO t1 SELECT 40 + c1, c2, c3 FROM t1; EXPLAIN SELECT COUNT(*) FROM t1 WHERE c2 > 3; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ALL NULL NULL NULL NULL 80 Using where +1 SIMPLE t1 ALL NULL NULL NULL NULL ROWS Using where ANALYZE TABLE t1; Table Op Msg_type Msg_text test.t1 analyze status OK +SET @merge_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); # session con1 SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( `c1` int(11) NOT NULL, `c2` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, PRIMARY KEY (`c1`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL rebuilt2 WAIT_FOR dml2_done'; @@ -230,6 +239,22 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 1 ddl_pending_alter_table 1 +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SELECT +(@merge_encrypt_1-@merge_encrypt_0)- +(@merge_decrypt_1-@merge_decrypt_0) as sort_balance, +@merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, +@rowlog_encrypt_1>@rowlog_encrypt_0; +sort_balance @merge_encrypt_1>@merge_encrypt_0 @merge_decrypt_1>@merge_decrypt_0 @rowlog_encrypt_1>@rowlog_encrypt_0 +0 0 0 0 SET DEBUG_SYNC = 'now SIGNAL dml2_done'; # session con1 ERROR HY000: Creating index 'PRIMARY' required more than 'innodb_online_alter_log_max_size' bytes of modification log. Please try again. @@ -239,14 +264,26 @@ ddl_background_drop_indexes 0 ddl_background_drop_tables 0 ddl_online_create_index 0 ddl_pending_alter_table 0 +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL rebuilt3 WAIT_FOR dml3_done'; ALTER TABLE t1 ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; ERROR 42000: Multiple primary key defined ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; ERROR 23000: Duplicate entry '5' for key 'PRIMARY' ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c1,c22f,c4(5)), -CHANGE c2 c22f INT, CHANGE c3 c3 TEXT NULL, CHANGE c1 c1 INT AFTER c22f, -ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online'; +CHANGE c2 c22f INT, CHANGE c3 c3 CHAR(255) NULL, CHANGE c1 c1 INT AFTER c22f, +ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online', LOCK=NONE; # session default SET DEBUG_SYNC = 'now WAIT_FOR rebuilt3'; SELECT name, count FROM INFORMATION_SCHEMA.INNODB_METRICS WHERE subsystem = 'ddl'; @@ -283,6 +320,32 @@ COUNT(c22f) CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK +SET @merge_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); +SELECT +(@merge_encrypt_2-@merge_encrypt_1)- +(@merge_decrypt_2-@merge_decrypt_1) as sort_balance, +(@rowlog_encrypt_2-@rowlog_encrypt_1)- +(@rowlog_decrypt_2-@rowlog_decrypt_1) as log_balance; +sort_balance log_balance +0 0 +SELECT +@merge_encrypt_2-@merge_encrypt_1>0 as sort_encrypted, +@merge_decrypt_2-@merge_decrypt_1>0 as sort_decrypted, +@rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, +@rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; +sort_encrypted sort_decrypted log_encrypted log_decrypted +0 0 0 0 ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)); ERROR 23000: Duplicate entry '' for key 'PRIMARY' UPDATE t1 SET c3 = NULL WHERE c3 = ''; @@ -294,13 +357,13 @@ SET @@sql_mode = 'STRICT_TRANS_TABLES'; ALTER TABLE t1 DROP COLUMN c22f, DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)), ALGORITHM = INPLACE; ERROR 22004: Invalid use of NULL value -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL; +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL; ERROR 22004: Invalid use of NULL value SET @@sql_mode = @old_sql_mode; -UPDATE t1 SET c3=CONCAT(c1,REPEAT('foo',c1)) WHERE c3 IS NULL; +UPDATE t1 SET c3=LEFT(CONCAT(c1,REPEAT('foo',c1)),255) WHERE c3 IS NULL; SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL c3p5_created0 WAIT_FOR ins_done0'; SET @@sql_mode = 'STRICT_TRANS_TABLES'; -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL, DROP COLUMN c22f, +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL, DROP COLUMN c22f, ADD COLUMN c5 CHAR(5) DEFAULT 'tired' FIRST; # session default SET DEBUG_SYNC = 'now WAIT_FOR c3p5_created0'; @@ -314,7 +377,7 @@ SET @@sql_mode = @old_sql_mode; # session default ROLLBACK; # session con1 -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL; +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL; SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL c3p5_created WAIT_FOR ins_done'; ALTER TABLE t1 DROP PRIMARY KEY, DROP COLUMN c22f, ADD COLUMN c6 VARCHAR(1000) DEFAULT @@ -361,7 +424,7 @@ Table Create Table t1 CREATE TABLE `t1` ( `c22f` int(11) NOT NULL, `c1` int(11) NOT NULL, - `c3` text NOT NULL, + `c3` char(255) NOT NULL, `c4` varchar(6) NOT NULL DEFAULT 'Online', PRIMARY KEY (`c1`,`c22f`,`c4`(5)) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT diff --git a/mysql-test/suite/innodb/t/innodb-index-online.opt b/mysql-test/suite/innodb/t/innodb-index-online.opt index 3f3c063b38f..820778b0004 100644 --- a/mysql-test/suite/innodb/t/innodb-index-online.opt +++ b/mysql-test/suite/innodb/t/innodb-index-online.opt @@ -1,5 +1,5 @@ --innodb-sort-buffer-size=64k ---innodb-online-alter-log-max-size=64k +--innodb-online-alter-log-max-size=128k --innodb-buffer-pool-size=5M --innodb-log-buffer-size=256k --innodb-sys-indexes diff --git a/mysql-test/suite/innodb/t/innodb-index-online.test b/mysql-test/suite/innodb/t/innodb-index-online.test index 24e3cd1ba22..efd700498c6 100644 --- a/mysql-test/suite/innodb/t/innodb-index-online.test +++ b/mysql-test/suite/innodb/t/innodb-index-online.test @@ -1,4 +1,5 @@ --source include/innodb_page_size_small.inc +--source include/innodb_encrypt_log.inc --source include/have_debug_sync.inc let $innodb_metrics_select= @@ -200,6 +201,18 @@ EXPLAIN SELECT COUNT(*) FROM t1 WHERE c2 > 3; SHOW CREATE TABLE t1; +connection default; +SET @merge_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +connection con1; + # Exceed the configured innodb_online_alter_log_max_size. # The actual limit is a multiple of innodb_sort_buf_size, # because that is the size of the in-memory log buffers. @@ -223,7 +236,7 @@ SET DEBUG_SYNC = 'now WAIT_FOR c2e_created'; # At this point, the clustered index scan must have completed, # but the modification log keeps accumulating due to the DEBUG_SYNC. eval $innodb_metrics_select; -let $c= 2; +let $c= 4; while ($c) { BEGIN; @@ -243,6 +256,22 @@ SELECT sf.name, sf.pos FROM INFORMATION_SCHEMA.INNODB_SYS_INDEXES si INNER JOIN INFORMATION_SCHEMA.INNODB_SYS_FIELDS sf ON si.index_id = sf.index_id WHERE si.name = '?c2e'; +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); + +SELECT +(@merge_encrypt_1-@merge_encrypt_0)- +(@merge_decrypt_1-@merge_decrypt_0) as sort_balance, +@merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, +@rowlog_encrypt_1>@rowlog_encrypt_0; + # Release con1. SET DEBUG_SYNC = 'now SIGNAL dml2_done'; @@ -270,6 +299,19 @@ ALTER TABLE t1 COMMENT 'testing if c2e will be dropped'; # Check that the 'zombie' index c2e was dropped. eval $innodb_metrics_select; +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); + connection con1; # Accumulate and apply some modification log. SET DEBUG_SYNC = 'row_log_apply_before SIGNAL c2f_created WAIT_FOR dml3_done'; @@ -282,6 +324,9 @@ connection default; SET DEBUG_SYNC = 'now WAIT_FOR c2f_created'; # Generate some log (delete-mark, delete-unmark, insert etc.) eval $innodb_metrics_select; +let $c= 2; +while ($c) +{ BEGIN; INSERT INTO t1 SELECT 320 + c1, c2, c3 FROM t1 WHERE c1 > 160; DELETE FROM t1 WHERE c1 > 320; @@ -290,6 +335,8 @@ BEGIN; UPDATE t1 SET c2 = c2 + 1; DELETE FROM t1; ROLLBACK; +dec $c; +} eval $innodb_metrics_select; # Release con1. SET DEBUG_SYNC = 'now SIGNAL dml3_done'; @@ -300,6 +347,34 @@ reap; ALTER TABLE t1 CHANGE c2 c22f INT; eval $innodb_metrics_select; + +connection default; + +SET @merge_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); + +SELECT +(@merge_encrypt_2-@merge_encrypt_1)- +(@merge_decrypt_2-@merge_decrypt_1) as sort_balance, +(@rowlog_encrypt_2-@rowlog_encrypt_1)- +(@rowlog_decrypt_2-@rowlog_decrypt_1) as log_balance; +SELECT +@merge_encrypt_2-@merge_encrypt_1>0 as sort_encrypted, +@merge_decrypt_2-@merge_decrypt_1>0 as sort_decrypted, +@rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, +@rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; + +connection con1; SELECT COUNT(c22f) FROM t1; CHECK TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb-table-online-master.opt b/mysql-test/suite/innodb/t/innodb-table-online-master.opt index 9ac01ffb5ad..92eea2b0d2e 100644 --- a/mysql-test/suite/innodb/t/innodb-table-online-master.opt +++ b/mysql-test/suite/innodb/t/innodb-table-online-master.opt @@ -1 +1 @@ ---innodb-sort-buffer-size=64k --innodb-online-alter-log-max-size=64k --innodb-buffer-pool-size=5M --innodb-log-buffer-size=256k +--innodb-sort-buffer-size=64k --innodb-online-alter-log-max-size=512k --innodb-buffer-pool-size=5M --innodb-log-buffer-size=256k diff --git a/mysql-test/suite/innodb/t/innodb-table-online.test b/mysql-test/suite/innodb/t/innodb-table-online.test index ff5c55d9cd9..938fd176b1e 100644 --- a/mysql-test/suite/innodb/t/innodb-table-online.test +++ b/mysql-test/suite/innodb/t/innodb-table-online.test @@ -1,4 +1,5 @@ --source include/innodb_page_size_small.inc +--source include/innodb_encrypt_log.inc --source include/have_debug.inc --source include/have_debug_sync.inc @@ -17,7 +18,7 @@ SET GLOBAL innodb_file_per_table = on; # Save the initial number of concurrent sessions. --source include/count_sessions.inc -CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 TEXT NOT NULL) +CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 INT NOT NULL, c3 CHAR(255) NOT NULL) ENGINE = InnoDB; INSERT INTO t1 VALUES (1,1,''), (2,2,''), (3,3,''), (4,4,''), (5,5,''); @@ -183,10 +184,20 @@ INSERT INTO t1 SELECT 10 + c1, c2, c3 FROM t1; INSERT INTO t1 SELECT 20 + c1, c2, c3 FROM t1; INSERT INTO t1 SELECT 40 + c1, c2, c3 FROM t1; # Purge may or may not have cleaned up the DELETE FROM t1 WHERE c1 = 7; ---replace_result 81 80 +--replace_column 9 ROWS EXPLAIN SELECT COUNT(*) FROM t1 WHERE c2 > 3; ANALYZE TABLE t1; +SET @merge_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_0= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); + --echo # session con1 connection con1; SHOW CREATE TABLE t1; @@ -228,6 +239,22 @@ while ($c) # Temporary table should exist until the DDL thread notices the overflow. eval $innodb_metrics_select; +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); + +SELECT +(@merge_encrypt_1-@merge_encrypt_0)- +(@merge_decrypt_1-@merge_decrypt_0) as sort_balance, +@merge_encrypt_1>@merge_encrypt_0, @merge_decrypt_1>@merge_decrypt_0, +@rowlog_encrypt_1>@rowlog_encrypt_0; + # Release con1. SET DEBUG_SYNC = 'now SIGNAL dml2_done'; @@ -241,6 +268,19 @@ reap; # when the above error was noticed. eval $innodb_metrics_select; +SET @merge_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_1= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); + # Accumulate and apply some modification log. SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL rebuilt3 WAIT_FOR dml3_done'; --error ER_MULTIPLE_PRI_KEY @@ -249,8 +289,8 @@ ALTER TABLE t1 ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c22f), CHANGE c2 c22f INT; --send ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY(c1,c22f,c4(5)), -CHANGE c2 c22f INT, CHANGE c3 c3 TEXT NULL, CHANGE c1 c1 INT AFTER c22f, -ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online'; +CHANGE c2 c22f INT, CHANGE c3 c3 CHAR(255) NULL, CHANGE c1 c1 INT AFTER c22f, +ADD COLUMN c4 VARCHAR(6) DEFAULT 'Online', LOCK=NONE; --echo # session default connection default; @@ -276,6 +316,30 @@ eval $innodb_metrics_select; SELECT COUNT(c22f) FROM t1; CHECK TABLE t1; +SET @merge_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'); +SET @merge_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'); +SET @rowlog_encrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'); +SET @rowlog_decrypt_2= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'); + +SELECT +(@merge_encrypt_2-@merge_encrypt_1)- +(@merge_decrypt_2-@merge_decrypt_1) as sort_balance, +(@rowlog_encrypt_2-@rowlog_encrypt_1)- +(@rowlog_decrypt_2-@rowlog_decrypt_1) as log_balance; +SELECT +@merge_encrypt_2-@merge_encrypt_1>0 as sort_encrypted, +@merge_decrypt_2-@merge_decrypt_1>0 as sort_decrypted, +@rowlog_encrypt_2-@rowlog_encrypt_1>0 as log_encrypted, +@rowlog_decrypt_2-@rowlog_decrypt_1>0 as log_decrypted; + # Create a column prefix index. --error ER_DUP_ENTRY ALTER TABLE t1 DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)); @@ -292,15 +356,15 @@ ALTER TABLE t1 DROP COLUMN c22f, DROP PRIMARY KEY, ADD PRIMARY KEY c3p5(c3(5)), ALGORITHM = INPLACE; --error ER_INVALID_USE_OF_NULL -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL; +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL; SET @@sql_mode = @old_sql_mode; -UPDATE t1 SET c3=CONCAT(c1,REPEAT('foo',c1)) WHERE c3 IS NULL; +UPDATE t1 SET c3=LEFT(CONCAT(c1,REPEAT('foo',c1)),255) WHERE c3 IS NULL; SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL c3p5_created0 WAIT_FOR ins_done0'; # NULL -> NOT NULL only allowed INPLACE if strict sql_mode is on. SET @@sql_mode = 'STRICT_TRANS_TABLES'; --send -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL, DROP COLUMN c22f, +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL, DROP COLUMN c22f, ADD COLUMN c5 CHAR(5) DEFAULT 'tired' FIRST; --echo # session default @@ -324,7 +388,7 @@ ROLLBACK; --echo # session con1 connection con1; -ALTER TABLE t1 MODIFY c3 TEXT NOT NULL; +ALTER TABLE t1 MODIFY c3 CHAR(255) NOT NULL; SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL c3p5_created WAIT_FOR ins_done'; --send ALTER TABLE t1 DROP PRIMARY KEY, DROP COLUMN c22f, diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index a91c0f67218..62dc98a3e30 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -4037,10 +4037,6 @@ row_merge_build_indexes( /* Now we have files containing index entries ready for sorting and inserting. */ - DBUG_EXECUTE_IF( - "ib_merge_wait_after_read", - os_thread_sleep(20000000);); /* 20 sec */ - for (i = 0; i < n_indexes; i++) { dict_index_t* sort_idx = indexes[i]; diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 28b43d6abef..1b0566dd2d3 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -4040,10 +4040,6 @@ row_merge_build_indexes( /* Now we have files containing index entries ready for sorting and inserting. */ - DBUG_EXECUTE_IF( - "ib_merge_wait_after_read", - os_thread_sleep(20000000);); /* 20 sec */ - for (i = 0; i < n_indexes; i++) { dict_index_t* sort_idx = indexes[i];