From d50cf42bc05b1faa5d39c766389ac345e119037e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 28 Dec 2016 15:54:24 +0200 Subject: [PATCH 1/3] MDEV-9282 Debian: the Lintian complains about "shlib-calls-exit" in ha_innodb.so Replace all exit() calls in InnoDB with abort() [possibly via ut_a()]. Calling exit() in a multi-threaded program is problematic also for the reason that other threads could see corrupted data structures while some data structures are being cleaned up by atexit() handlers or similar. In the long term, all these calls should be replaced with something that returns an error all the way up the call stack. --- storage/innobase/api/api0misc.cc | 4 ---- storage/innobase/buf/buf0dblwr.cc | 15 +++------------ storage/innobase/fil/fil0fil.cc | 2 +- storage/innobase/include/fts0ast.h | 3 +++ storage/innobase/log/log0log.cc | 20 +++----------------- storage/innobase/os/os0file.cc | 2 +- storage/innobase/os/os0thread.cc | 6 +----- storage/innobase/row/row0mysql.cc | 4 +--- storage/innobase/row/row0undo.cc | 3 +-- storage/innobase/srv/srv0start.cc | 13 ++----------- storage/xtradb/api/api0misc.cc | 4 ---- storage/xtradb/buf/buf0dblwr.cc | 15 +++------------ storage/xtradb/fil/fil0fil.cc | 2 +- storage/xtradb/include/fts0ast.h | 3 +++ storage/xtradb/log/log0log.cc | 30 +++++------------------------- storage/xtradb/log/log0online.cc | 19 ++++++------------- storage/xtradb/os/os0file.cc | 2 +- storage/xtradb/os/os0thread.cc | 6 +----- storage/xtradb/row/row0mysql.cc | 4 +--- storage/xtradb/row/row0undo.cc | 3 +-- storage/xtradb/srv/srv0start.cc | 13 ++----------- 21 files changed, 40 insertions(+), 133 deletions(-) diff --git a/storage/innobase/api/api0misc.cc b/storage/innobase/api/api0misc.cc index a980d32c33f..5daee5de4c9 100644 --- a/storage/innobase/api/api0misc.cc +++ b/storage/innobase/api/api0misc.cc @@ -184,10 +184,6 @@ handle_new_error: trx_rollback_for_mysql(trx); break; - case DB_MUST_GET_MORE_FILE_SPACE: - - exit(1); - case DB_CORRUPTION: case DB_FOREIGN_EXCEED_MAX_CASCADE: break; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 16877818ba9..cc5fe52f80a 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -224,12 +224,10 @@ start_again: + FSP_EXTENT_SIZE / 2 + 100) * UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your buffer pool size. Cannot continue " "operation."); - - exit(EXIT_FAILURE); } block2 = fseg_create(TRX_SYS_SPACE, TRX_SYS_PAGE_NO, @@ -242,15 +240,10 @@ start_again: buf_block_dbg_add_level(block2, SYNC_NO_ORDER_CHECK); if (block2 == NULL) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your tablespace size. " "Cannot continue operation."); - - /* We exit without committing the mtr to prevent - its modifications to the database getting to disk */ - - exit(EXIT_FAILURE); } fseg_header = doublewrite + TRX_SYS_DOUBLEWRITE_FSEG; @@ -261,12 +254,10 @@ start_again: new_block = fseg_alloc_free_page( fseg_header, prev_page_no + 1, FSP_UP, &mtr); if (new_block == NULL) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your tablespace size. " "Cannot continue operation."); - - exit(EXIT_FAILURE); } /* We read the allocated pages to the buffer pool; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 03f6b9148ba..22352d94332 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -4599,7 +4599,7 @@ will_not_choose: return; } - exit(1); + abort(); } if (def.success && remote.success) { diff --git a/storage/innobase/include/fts0ast.h b/storage/innobase/include/fts0ast.h index 50f62063893..6229869e8d0 100644 --- a/storage/innobase/include/fts0ast.h +++ b/storage/innobase/include/fts0ast.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +30,8 @@ Created 2007/03/16/03 Sunny Bains #include "mem0mem.h" #include "ha_prototypes.h" +#define exit(x) abort() + /* The type of AST Node */ enum fts_ast_type_t { FTS_AST_OPER, /*!< Operator */ diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 89b616aba01..a8326571366 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -815,24 +815,10 @@ failure: mutex_exit(&(log_sys->mutex)); if (!success) { - fprintf(stderr, - "InnoDB: Error: ib_logfiles are too small" - " for innodb_thread_concurrency %lu.\n" - "InnoDB: The combined size of ib_logfiles" + ib_logf(IB_LOG_LEVEL_FATAL, + "The combined size of ib_logfiles" " should be bigger than\n" - "InnoDB: 200 kB * innodb_thread_concurrency.\n" - "InnoDB: To get mysqld to start up, set" - " innodb_thread_concurrency in my.cnf\n" - "InnoDB: to a lower value, for example, to 8." - " After an ERROR-FREE shutdown\n" - "InnoDB: of mysqld you can adjust the size of" - " ib_logfiles, as explained in\n" - "InnoDB: " REFMAN "adding-and-removing.html\n" - "InnoDB: Cannot continue operation." - " Calling exit(1).\n", - (ulong) srv_thread_concurrency); - - exit(1); + "InnoDB: 200 kB * innodb_thread_concurrency."); } return(success); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 2db53c25b04..46518ef8d35 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -814,7 +814,7 @@ os_file_handle_error_cond_exit( } if (should_exit) { - exit(1); + abort(); } } diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index a5b0f7de6ae..88f4292d6fe 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -171,11 +171,7 @@ os_thread_create_func( #else ret = pthread_create(&pthread, &attr, func, arg); #endif - if (ret) { - fprintf(stderr, - "InnoDB: Error: pthread_create returned %d\n", ret); - exit(1); - } + ut_a(ret == 0); #ifndef UNIV_HPUX10 pthread_attr_destroy(&attr); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 35011247105..1c02590933f 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -660,9 +660,7 @@ handle_new_error: "InnoDB: lack of space. You must add" " a new data file to\n" "InnoDB: my.cnf and restart the database.\n", stderr); - - ut_ad(0); - exit(1); + abort(); case DB_CORRUPTION: fputs("InnoDB: We detected index corruption" diff --git a/storage/innobase/row/row0undo.cc b/storage/innobase/row/row0undo.cc index 149dc671930..82b1ab049fa 100644 --- a/storage/innobase/row/row0undo.cc +++ b/storage/innobase/row/row0undo.cc @@ -363,8 +363,7 @@ row_undo_step( "InnoDB: Out of tablespace.\n" "InnoDB: Consider increasing" " your tablespace.\n"); - - exit(1); + abort(); } ut_error; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 1f5d6eef0e8..bbb9dc0205e 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -600,7 +600,7 @@ create_log_file( fprintf(stderr, "innodb_force_recovery_crash=%lu\n", \ srv_force_recovery_crash); \ fflush(stderr); \ - exit(3); \ + abort(); \ } \ } while (0) #endif @@ -2912,16 +2912,7 @@ files_checked: /* Check that os_fast_mutexes work as expected */ os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); - if (0 != os_fast_mutex_trylock(&srv_os_test_mutex)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: pthread_mutex_trylock returns" - " an unexpected value on\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: success! Cannot continue.\n"); - exit(1); - } + ut_a(0 == os_fast_mutex_trylock(&srv_os_test_mutex)); os_fast_mutex_unlock(&srv_os_test_mutex); diff --git a/storage/xtradb/api/api0misc.cc b/storage/xtradb/api/api0misc.cc index a980d32c33f..5daee5de4c9 100644 --- a/storage/xtradb/api/api0misc.cc +++ b/storage/xtradb/api/api0misc.cc @@ -184,10 +184,6 @@ handle_new_error: trx_rollback_for_mysql(trx); break; - case DB_MUST_GET_MORE_FILE_SPACE: - - exit(1); - case DB_CORRUPTION: case DB_FOREIGN_EXCEED_MAX_CASCADE: break; diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index d8d85c25289..62ed17296f5 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -224,12 +224,10 @@ start_again: + FSP_EXTENT_SIZE / 2 + 100) * UNIV_PAGE_SIZE)) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your buffer pool size. Cannot continue " "operation."); - - exit(EXIT_FAILURE); } block2 = fseg_create(TRX_SYS_SPACE, TRX_SYS_PAGE_NO, @@ -242,15 +240,10 @@ start_again: buf_block_dbg_add_level(block2, SYNC_NO_ORDER_CHECK); if (block2 == NULL) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your tablespace size. " "Cannot continue operation."); - - /* We exit without committing the mtr to prevent - its modifications to the database getting to disk */ - - exit(EXIT_FAILURE); } fseg_header = doublewrite + TRX_SYS_DOUBLEWRITE_FSEG; @@ -261,12 +254,10 @@ start_again: new_block = fseg_alloc_free_page( fseg_header, prev_page_no + 1, FSP_UP, &mtr); if (new_block == NULL) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "Cannot create doublewrite buffer: you must " "increase your tablespace size. " "Cannot continue operation."); - - exit(EXIT_FAILURE); } /* We read the allocated pages to the buffer pool; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 28f262b50c7..5900bdb2140 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -4618,7 +4618,7 @@ will_not_choose: return; } - exit(1); + abort(); } if (def.success && remote.success) { diff --git a/storage/xtradb/include/fts0ast.h b/storage/xtradb/include/fts0ast.h index 50f62063893..6229869e8d0 100644 --- a/storage/xtradb/include/fts0ast.h +++ b/storage/xtradb/include/fts0ast.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +30,8 @@ Created 2007/03/16/03 Sunny Bains #include "mem0mem.h" #include "ha_prototypes.h" +#define exit(x) abort() + /* The type of AST Node */ enum fts_ast_type_t { FTS_AST_OPER, /*!< Operator */ diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index a01a2ed9570..f3a3486017c 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -920,24 +920,10 @@ failure: mutex_exit(&(log_sys->mutex)); if (!success) { - fprintf(stderr, - "InnoDB: Error: ib_logfiles are too small" - " for innodb_thread_concurrency %lu.\n" - "InnoDB: The combined size of ib_logfiles" + ib_logf(IB_LOG_LEVEL_FATAL, + "The combined size of ib_logfiles" " should be bigger than\n" - "InnoDB: 200 kB * innodb_thread_concurrency.\n" - "InnoDB: To get mysqld to start up, set" - " innodb_thread_concurrency in my.cnf\n" - "InnoDB: to a lower value, for example, to 8." - " After an ERROR-FREE shutdown\n" - "InnoDB: of mysqld you can adjust the size of" - " ib_logfiles, as explained in\n" - "InnoDB: " REFMAN "adding-and-removing.html\n" - "InnoDB: Cannot continue operation." - " Calling exit(1).\n", - (ulong) srv_thread_concurrency); - - exit(1); + "InnoDB: 200 kB * innodb_thread_concurrency."); } return(success); @@ -2861,15 +2847,9 @@ loop: } if (!ret) { - fprintf(stderr, + ib_logf(IB_LOG_LEVEL_FATAL, "InnoDB: Cannot create or open" - " archive log file %s.\n" - "InnoDB: Cannot continue operation.\n" - "InnoDB: Check that the log archive" - " directory exists,\n" - "InnoDB: you have access rights to it, and\n" - "InnoDB: there is space available.\n", name); - exit(1); + " archive log file %s.\n", name); } #ifdef UNIV_DEBUG diff --git a/storage/xtradb/log/log0online.cc b/storage/xtradb/log/log0online.cc index 2a1ac63dc5b..4e6ad65a906 100644 --- a/storage/xtradb/log/log0online.cc +++ b/storage/xtradb/log/log0online.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011-2012 Percona Inc. All Rights Reserved. +Copyright (C) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -405,12 +406,11 @@ log_online_can_track_missing( last_tracked_lsn = ut_max(last_tracked_lsn, MIN_TRACKED_LSN); if (last_tracked_lsn > tracking_start_lsn) { - ib_logf(IB_LOG_LEVEL_ERROR, + ib_logf(IB_LOG_LEVEL_FATAL, "last tracked LSN " LSN_PF " is ahead of tracking " "start LSN " LSN_PF ". This can be caused by " "mismatched bitmap files.", last_tracked_lsn, tracking_start_lsn); - exit(1); } return (last_tracked_lsn == tracking_start_lsn) @@ -450,9 +450,7 @@ log_online_track_missing_on_startup( log_bmp_sys->start_lsn = ut_max(last_tracked_lsn, MIN_TRACKED_LSN); log_set_tracked_lsn(log_bmp_sys->start_lsn); - if (!log_online_follow_redo_log()) { - exit(1); - } + ut_a(log_online_follow_redo_log()); ut_ad(log_bmp_sys->end_lsn >= tracking_start_lsn); ib_logf(IB_LOG_LEVEL_INFO, @@ -677,9 +675,8 @@ log_online_read_init(void) if (os_file_closedir(bitmap_dir)) { os_file_get_last_error(TRUE); - ib_logf(IB_LOG_LEVEL_ERROR, "cannot close \'%s\'", + ib_logf(IB_LOG_LEVEL_FATAL, "cannot close \'%s\'", log_bmp_sys->bmp_file_home); - exit(1); } if (!log_bmp_sys->out_seq_num) { @@ -699,9 +696,7 @@ log_online_read_init(void) if (!success) { /* New file, tracking from scratch */ - if (!log_online_start_bitmap_file()) { - exit(1); - } + ut_a(log_online_start_bitmap_file()); } else { @@ -738,9 +733,7 @@ log_online_read_init(void) } else { file_start_lsn = tracking_start_lsn; } - if (!log_online_rotate_bitmap_file(file_start_lsn)) { - exit(1); - } + ut_a(log_online_rotate_bitmap_file(file_start_lsn)); if (last_tracked_lsn < tracking_start_lsn) { diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 007b100285d..5e107dac0eb 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -920,7 +920,7 @@ os_file_handle_error_cond_exit( } if (should_exit) { - exit(1); + abort(); } } diff --git a/storage/xtradb/os/os0thread.cc b/storage/xtradb/os/os0thread.cc index af826027efc..5ddc40b0eeb 100644 --- a/storage/xtradb/os/os0thread.cc +++ b/storage/xtradb/os/os0thread.cc @@ -192,11 +192,7 @@ os_thread_create_func( #else ret = pthread_create(&pthread, &attr, func, arg); #endif - if (ret) { - fprintf(stderr, - "InnoDB: Error: pthread_create returned %d\n", ret); - exit(1); - } + ut_a(ret == 0); #ifndef UNIV_HPUX10 pthread_attr_destroy(&attr); diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 0bdee1282f8..e53a0ea9586 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -1196,9 +1196,7 @@ handle_new_error: "InnoDB: lack of space. You must add" " a new data file to\n" "InnoDB: my.cnf and restart the database.\n", stderr); - - ut_ad(0); - exit(1); + abort(); case DB_CORRUPTION: fputs("InnoDB: We detected index corruption" diff --git a/storage/xtradb/row/row0undo.cc b/storage/xtradb/row/row0undo.cc index 149dc671930..82b1ab049fa 100644 --- a/storage/xtradb/row/row0undo.cc +++ b/storage/xtradb/row/row0undo.cc @@ -363,8 +363,7 @@ row_undo_step( "InnoDB: Out of tablespace.\n" "InnoDB: Consider increasing" " your tablespace.\n"); - - exit(1); + abort(); } ut_error; diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index ada91c64524..2dd0285d03f 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -630,7 +630,7 @@ create_log_file( fprintf(stderr, "innodb_force_recovery_crash=%lu\n", \ srv_force_recovery_crash); \ fflush(stderr); \ - exit(3); \ + abort(); \ } \ } while (0) #endif @@ -2979,16 +2979,7 @@ files_checked: /* Check that os_fast_mutexes work as expected */ os_fast_mutex_init(PFS_NOT_INSTRUMENTED, &srv_os_test_mutex); - if (0 != os_fast_mutex_trylock(&srv_os_test_mutex)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: pthread_mutex_trylock returns" - " an unexpected value on\n"); - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: success! Cannot continue.\n"); - exit(1); - } + ut_a(0 == os_fast_mutex_trylock(&srv_os_test_mutex)); os_fast_mutex_unlock(&srv_os_test_mutex); From 283e9cf4cbb34e1325699707068ab72ec3accfff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 28 Dec 2016 16:14:28 +0200 Subject: [PATCH 2/3] MDEV-11656: 'Data structure corruption' IMPORT TABLESPACE doesn't work for encrypted InnoDB tables if space_id changed Problem was that for encryption we use temporary scratch area for reading and writing tablespace pages. But if page was not really decrypted the correct updated page was not moved to scratch area that was then written. This can happen e.g. for page 0 as it is newer encrypted even if encryption is enabled and as we write the contents of old page 0 to tablespace it contained naturally incorrect space_id that is then later noted and error message was written. Updated page with correct space_id was lost. If tablespace is encrypted we use additional temporary scratch area where pages are read for decrypting readptr == crypt_io_buffer != io_buffer. Destination for decryption is a buffer pool block block->frame == dst == io_buffer that is updated. Pages that did not require decryption even when tablespace is marked as encrypted are not copied instead block->frame is set to src == readptr. If tablespace was encrypted we copy updated page to writeptr != io_buffer. This fixes above bug. For encryption we again use temporary scratch area writeptr != io_buffer == dst that is then written to the tablespace (1) For normal tables src == dst == writeptr ut_ad(!encrypted && !page_compressed ? src == dst && dst == writeptr + (i * size):1); (2) For page compressed tables src == dst == writeptr ut_ad(page_compressed && !encrypted ? src == dst && dst == writeptr + (i * size):1); (3) For encrypted tables src != dst != writeptr ut_ad(encrypted ? src != dst && dst != writeptr + (i * size):1); --- .../r/innodb-discard-import-change.result | 105 ++++++++++++++ .../t/innodb-discard-import-change.test | 131 ++++++++++++++++++ storage/innobase/fil/fil0fil.cc | 61 ++++++-- storage/xtradb/fil/fil0fil.cc | 61 ++++++-- 4 files changed, 342 insertions(+), 16 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innodb-discard-import-change.result create mode 100644 mysql-test/suite/encryption/t/innodb-discard-import-change.test diff --git a/mysql-test/suite/encryption/r/innodb-discard-import-change.result b/mysql-test/suite/encryption/r/innodb-discard-import-change.result new file mode 100644 index 00000000000..7071f9eaf20 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-discard-import-change.result @@ -0,0 +1,105 @@ +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t2(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; +insert into t1 values (NULL, 'verysecretmessage'); +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +insert into t5 select * from t1; +FLUSH TABLE t1,t2,t3,t4,t5 FOR EXPORT; +backup: t1 +backup: t2 +backup: t3 +backup: t4 +backup: t5 +t1.cfg +t1.frm +t1.ibd +t2.cfg +t2.frm +t2.ibd +t3.cfg +t3.frm +t3.ibd +t4.cfg +t4.frm +t4.ibd +t5.cfg +t5.frm +t5.ibd +UNLOCK TABLES; +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t2 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; +DROP TABLE t1; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; +create table t6(a int) engine=innodb; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; +restore: t1 .ibd and .cfg files +restore: t2 .ibd and .cfg files +restore: t3 .ibd and .cfg files +restore: t4 .ibd and .cfg files +restore: t5 .ibd and .cfg files +ALTER TABLE t1 IMPORT TABLESPACE; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` bigint(20) NOT NULL AUTO_INCREMENT, + `b` char(200) DEFAULT NULL, + PRIMARY KEY (`c1`) +) ENGINE=InnoDB AUTO_INCREMENT=377 DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=4 +SELECT COUNT(*) FROM t1; +COUNT(*) +256 +ALTER TABLE t2 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t2; +COUNT(*) +256 +ALTER TABLE t3 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t3; +COUNT(*) +256 +ALTER TABLE t4 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t4; +COUNT(*) +256 +ALTER TABLE t5 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t5; +COUNT(*) +256 +# t1 encrypted expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t1.ibd +# t2 encrypted expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t2.ibd +# t3 page compressed expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t3.ibd +# t4 page compressed and encrypted expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t4.ibd +# t5 normal expecting FOUND +FOUND /verysecretmessage/ in t5.ibd +DROP TABLE t1,t2,t3,t4,t5,t6; diff --git a/mysql-test/suite/encryption/t/innodb-discard-import-change.test b/mysql-test/suite/encryption/t/innodb-discard-import-change.test new file mode 100644 index 00000000000..a278a8fba29 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-discard-import-change.test @@ -0,0 +1,131 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +# +# MDEV-11656: 'Data structure corruption' IMPORT TABLESPACE doesn't work for encrypted InnoDB tables if space_id changed +# + +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); + +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`; +--enable_query_log + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; +--enable_warnings + +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t2(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; + +insert into t1 values (NULL, 'verysecretmessage'); +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +insert into t5 select * from t1; + +let MYSQLD_DATADIR =`SELECT @@datadir`; +FLUSH TABLE t1,t2,t3,t4,t5 FOR EXPORT; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_backup_tablespaces("test", "t1","t2","t3","t4","t5"); +EOF +--list_files $MYSQLD_DATADIR/test +UNLOCK TABLES; + +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t2 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; + +# +# Now intentionally change space_id for t1,t3,t4,t5 +# +DROP TABLE t1; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; + +create table t6(a int) engine=innodb; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; + +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; + +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_discard_tablespaces("test", "t1","t2","t3","t4","t5"); +ib_restore_tablespaces("test", "t1","t2","t3","t4","t5"); +EOF + +ALTER TABLE t1 IMPORT TABLESPACE; +SHOW CREATE TABLE t1; +SELECT COUNT(*) FROM t1; +ALTER TABLE t2 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t2; +ALTER TABLE t3 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t3; +ALTER TABLE t4 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t4; +ALTER TABLE t5 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t5; + +# +# Verify +# +--let $MYSQLD_TMPDIR = `SELECT @@tmpdir` +--let $MYSQLD_DATADIR = `SELECT @@datadir` +--let SEARCH_RANGE = 10000000 +--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd +--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd +--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd +--let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd +--let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd +--let SEARCH_PATTERN=verysecretmessage +--echo # t1 encrypted expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # t2 encrypted expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--echo # t3 page compressed expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--echo # t4 page compressed and encrypted expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc +--echo # t5 normal expecting FOUND +-- let SEARCH_FILE=$t5_IBD +-- source include/search_pattern_in_file.inc + +DROP TABLE t1,t2,t3,t4,t5,t6; + +# reset system +--disable_warnings +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo; +--enable_query_log +--enable_warnings + diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 22352d94332..ce5c62a8c8b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -6533,6 +6533,7 @@ fil_iterate( for (offset = iter.start; offset < iter.end; offset += n_bytes) { byte* io_buffer = iter.io_buffer; + bool row_compressed = false; block->frame = io_buffer; @@ -6545,6 +6546,7 @@ fil_iterate( /* Zip IO is done in the compressed page buffer. */ io_buffer = block->page.zip.data; + row_compressed = true; } else { io_buffer = iter.io_buffer; } @@ -6585,8 +6587,9 @@ fil_iterate( for (ulint i = 0; i < n_pages_read; ++i) { ulint size = iter.page_size; dberr_t err = DB_SUCCESS; - byte* src = (readptr + (i * size)); - byte* dst = (io_buffer + (i * size)); + byte* src = readptr + (i * size); + byte* dst = io_buffer + (i * size); + bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -6610,8 +6613,12 @@ fil_iterate( if (decrypted) { updated = true; } else { - /* TODO: remove unnecessary memcpy's */ - memcpy(dst, src, size); + if (!page_compressed && !row_compressed) { + block->frame = src; + frame_changed = true; + } else { + memcpy(dst, src, size); + } } } @@ -6636,7 +6643,45 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - src = (io_buffer + (i * size)); + /* If tablespace is encrypted we use additional + temporary scratch area where pages are read + for decrypting readptr == crypt_io_buffer != io_buffer. + + Destination for decryption is a buffer pool block + block->frame == dst == io_buffer that is updated. + Pages that did not require decryption even when + tablespace is marked as encrypted are not copied + instead block->frame is set to src == readptr. + + For encryption we again use temporary scratch area + writeptr != io_buffer == dst + that is then written to the tablespace + + (1) For normal tables io_buffer == dst == writeptr + (2) For only page compressed tables + io_buffer == dst == writeptr + (3) For encrypted (and page compressed) + readptr != io_buffer == dst != writeptr + */ + + ut_ad(!encrypted && !page_compressed ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(page_compressed && !encrypted ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(encrypted ? + src != dst && dst != writeptr + (i * size):1); + + if (encrypted) { + memcpy(writeptr + (i * size), + row_compressed ? block->page.zip.data : + block->frame, size); + } + + if (frame_changed) { + block->frame = dst; + } + + src = io_buffer + (i * size); if (page_compressed) { ulint len = 0; @@ -6657,7 +6702,7 @@ fil_iterate( write it back. Note that we should not encrypt the buffer that is in buffer pool. */ if (decrypted && encrypted) { - unsigned char *dest = (writeptr + (i * size)); + byte *dest = writeptr + (i * size); ulint space = mach_read_from_4( src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); @@ -6839,9 +6884,9 @@ fil_tablespace_iterate( void* crypt_io_buffer = NULL; if (iter.crypt_data != NULL) { crypt_io_buffer = mem_alloc( - iter.n_io_buffers * UNIV_PAGE_SIZE); + (2 + iter.n_io_buffers) * UNIV_PAGE_SIZE); iter.crypt_io_buffer = static_cast( - crypt_io_buffer); + ut_align(crypt_io_buffer, UNIV_PAGE_SIZE)); } err = fil_iterate(iter, &block, callback); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 5900bdb2140..e7da4569f0d 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -6594,6 +6594,7 @@ fil_iterate( for (offset = iter.start; offset < iter.end; offset += n_bytes) { byte* io_buffer = iter.io_buffer; + bool row_compressed = false; block->frame = io_buffer; @@ -6606,6 +6607,7 @@ fil_iterate( /* Zip IO is done in the compressed page buffer. */ io_buffer = block->page.zip.data; + row_compressed = true; } else { io_buffer = iter.io_buffer; } @@ -6646,8 +6648,9 @@ fil_iterate( for (ulint i = 0; i < n_pages_read; ++i) { ulint size = iter.page_size; dberr_t err = DB_SUCCESS; - byte* src = (readptr + (i * size)); - byte* dst = (io_buffer + (i * size)); + byte* src = readptr + (i * size); + byte* dst = io_buffer + (i * size); + bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -6671,8 +6674,12 @@ fil_iterate( if (decrypted) { updated = true; } else { - /* TODO: remove unnecessary memcpy's */ - memcpy(dst, src, size); + if (!page_compressed && !row_compressed) { + block->frame = src; + frame_changed = true; + } else { + memcpy(dst, src, size); + } } } @@ -6697,7 +6704,45 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - src = (io_buffer + (i * size)); + /* If tablespace is encrypted we use additional + temporary scratch area where pages are read + for decrypting readptr == crypt_io_buffer != io_buffer. + + Destination for decryption is a buffer pool block + block->frame == dst == io_buffer that is updated. + Pages that did not require decryption even when + tablespace is marked as encrypted are not copied + instead block->frame is set to src == readptr. + + For encryption we again use temporary scratch area + writeptr != io_buffer == dst + that is then written to the tablespace + + (1) For normal tables io_buffer == dst == writeptr + (2) For only page compressed tables + io_buffer == dst == writeptr + (3) For encrypted (and page compressed) + readptr != io_buffer == dst != writeptr + */ + + ut_ad(!encrypted && !page_compressed ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(page_compressed && !encrypted ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(encrypted ? + src != dst && dst != writeptr + (i * size):1); + + if (encrypted) { + memcpy(writeptr + (i * size), + row_compressed ? block->page.zip.data : + block->frame, size); + } + + if (frame_changed) { + block->frame = dst; + } + + src = io_buffer + (i * size); if (page_compressed) { ulint len = 0; @@ -6718,7 +6763,7 @@ fil_iterate( write it back. Note that we should not encrypt the buffer that is in buffer pool. */ if (decrypted && encrypted) { - unsigned char *dest = (writeptr + (i * size)); + byte *dest = writeptr + (i * size); ulint space = mach_read_from_4( src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); @@ -6900,9 +6945,9 @@ fil_tablespace_iterate( void* crypt_io_buffer = NULL; if (iter.crypt_data != NULL) { crypt_io_buffer = mem_alloc( - iter.n_io_buffers * UNIV_PAGE_SIZE); + (2 + iter.n_io_buffers) * UNIV_PAGE_SIZE); iter.crypt_io_buffer = static_cast( - crypt_io_buffer); + ut_align(crypt_io_buffer, UNIV_PAGE_SIZE)); } err = fil_iterate(iter, &block, callback); From 23cc1be270c7304963643947d8e5ab88f4e312ee Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 21 Dec 2016 20:11:14 +0100 Subject: [PATCH 3/3] MDEV-11584: GRANT inside an SP does not work well on 2nd execution Allocate password hash in statment memory --- mysql-test/r/sp.result | 10 ++++++++++ mysql-test/t/sp.test | 14 ++++++++++++++ sql/sql_acl.cc | 5 +++++ 3 files changed, 29 insertions(+) diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index 820701f844c..4142fc0b796 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -8032,3 +8032,13 @@ return 1; end | ERROR 0A000: Not allowed to return a result set from a function drop table t1,t2; +# +# MDEV-11584: GRANT inside an SP does not work well on 2nd execution +# +CREATE PROCEDURE sp1() +GRANT ALL PRIVILEGES ON *.* TO 'foo'@'%' IDENTIFIED BY 'pass'; +CALL sp1(); +CALL sp1(); +drop user 'foo'@'%'; +drop procedure sp1; +#End of 10.1 tests diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index bcddbd6f97e..3034f34d763 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9504,3 +9504,17 @@ end | --delimiter ; drop table t1,t2; + +--echo # +--echo # MDEV-11584: GRANT inside an SP does not work well on 2nd execution +--echo # + +CREATE PROCEDURE sp1() + GRANT ALL PRIVILEGES ON *.* TO 'foo'@'%' IDENTIFIED BY 'pass'; +CALL sp1(); +CALL sp1(); +drop user 'foo'@'%'; +drop procedure sp1; + + +--echo #End of 10.1 tests diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 3f26bc2e08a..2accb3abc91 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -1067,7 +1067,12 @@ static bool fix_lex_user(THD *thd, LEX_USER *user) make_scramble= my_make_scrambled_password; } + Query_arena *arena, backup; + arena= thd->activate_stmt_arena_if_needed(&backup); char *buff= (char *) thd->alloc(scramble_length + 1); + if (arena) + thd->restore_active_arena(arena, &backup); + if (buff == NULL) return true; make_scramble(buff, user->pwtext.str, user->pwtext.length);