From 73b3ace8b26ef809e47b264e7d3630d7b887bff6 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 11 Apr 2018 09:14:35 +0200 Subject: [PATCH 01/12] MDEV-15808 Fix and re-enable test galera.galera_gra_log Test would occasionally fail as follows: ``` mysqltest: At line 20: query 'SELECT COUNT(*) = 0 FROM t1' failed: 1317: Query execution was interrupted ``` This was due to a `CREATE TABLE` applied concurrently that would occasionally cause the `SELECT` to be BF aborted, due to MDL conflict. The `SELECT` is executed while `wsrep_on=OFF`. The change makes sure it is executed only after wsrep is enabled again. --- mysql-test/suite/galera/disabled.def | 1 - mysql-test/suite/galera/r/galera_gra_log.result | 2 +- mysql-test/suite/galera/t/galera_gra_log.test | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 4a9edd1bf32..9c1ddb3af17 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -31,6 +31,5 @@ galera_var_retry_autocommit : MDEV-15794 Test failure on galera.galera_var_retry galera_var_auto_inc_control_on : MDEV-15803 Test failure on galera.galera_var_auto_inc_control_on pxc-421 : MDEV-15804 Test failure on galera.pxc-421 query_cache : MDEV-15805 Test failure on galera.query_cache -galera.galera_gra_log : MDEV-15808 Test failure on galera.galera_gra_log galera.MW-44 : MDEV-15809 Test failure on galera.MW-44 galera.galera_pc_ignore_sb : MDEV-15811 Test failure on galera_pc_ignore_sb diff --git a/mysql-test/suite/galera/r/galera_gra_log.result b/mysql-test/suite/galera/r/galera_gra_log.result index 6b816d0127f..a5b87c0ef0c 100644 --- a/mysql-test/suite/galera/r/galera_gra_log.result +++ b/mysql-test/suite/galera/r/galera_gra_log.result @@ -1,6 +1,7 @@ SET SESSION wsrep_on=OFF; CREATE TABLE t1 (f1 INTEGER); CREATE TABLE t1 (f1 INTEGER); +SET SESSION wsrep_on=ON; SELECT COUNT(*) = 0 FROM t1; COUNT(*) = 0 1 @@ -27,6 +28,5 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; -SET SESSION wsrep_on=ON; CALL mtr.add_suppression("Slave SQL: Error 'Table 't1' already exists' on query"); DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_gra_log.test b/mysql-test/suite/galera/t/galera_gra_log.test index adb37de9188..8b5aaaae5bd 100644 --- a/mysql-test/suite/galera/t/galera_gra_log.test +++ b/mysql-test/suite/galera/t/galera_gra_log.test @@ -17,6 +17,7 @@ CREATE TABLE t1 (f1 INTEGER); CREATE TABLE t1 (f1 INTEGER); --connection node_2 +SET SESSION wsrep_on=ON; SELECT COUNT(*) = 0 FROM t1; # Make sure the GRA file produced is readable and contains the failure @@ -24,8 +25,6 @@ SELECT COUNT(*) = 0 FROM t1; --replace_regex /SET TIMESTAMP=[0-9]+/SET TIMESTAMP=/ /pseudo_thread_id=[0-9]+/pseudo_thread_id=/ --exec $MYSQL_BINLOG --short-form $MYSQLTEST_VARDIR/mysqld.2/data/GRA_*.log -SET SESSION wsrep_on=ON; - CALL mtr.add_suppression("Slave SQL: Error 'Table 't1' already exists' on query"); DROP TABLE t1; From 4dc60dc3f10dd7522236fbc80dc184678352bdc9 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 11 Apr 2018 14:24:39 +0200 Subject: [PATCH 02/12] MDEV-15804 Fix and re-enable MTR test galera.pxc-421 This patch makes two changes: * Remove unnecessary `set SESSION wsrep_sync_wait=0`. Disabling `wsrep_sync_wait` caused the test to fail occasionally, due to subsequent `SELECT`s observing stale values. * Remove redundant `--source include/galera_wait_ready.inc`, `galera_wait_ready.inc` is already included in the above `wait_until_connected_again.inc` --- mysql-test/suite/galera/disabled.def | 1 - mysql-test/suite/galera/r/pxc-421.result | 1 - mysql-test/suite/galera/t/pxc-421.test | 2 -- 3 files changed, 4 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 4a9edd1bf32..f705368a5e2 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -29,7 +29,6 @@ galera.MW-329 : wsrep_local_replays not stable galera.MW-328A : have_deadlocks test not stable galera_var_retry_autocommit : MDEV-15794 Test failure on galera.galera_var_retry_autocommit galera_var_auto_inc_control_on : MDEV-15803 Test failure on galera.galera_var_auto_inc_control_on -pxc-421 : MDEV-15804 Test failure on galera.pxc-421 query_cache : MDEV-15805 Test failure on galera.query_cache galera.galera_gra_log : MDEV-15808 Test failure on galera.galera_gra_log galera.MW-44 : MDEV-15809 Test failure on galera.MW-44 diff --git a/mysql-test/suite/galera/r/pxc-421.result b/mysql-test/suite/galera/r/pxc-421.result index ba1f3d66720..1822201f338 100644 --- a/mysql-test/suite/galera/r/pxc-421.result +++ b/mysql-test/suite/galera/r/pxc-421.result @@ -6,7 +6,6 @@ set GLOBAL wsrep_slave_threads=16; SET GLOBAL wsrep_provider='none'; INSERT INTO t1 VALUES (2); INSERT INTO t1 VALUES (3); -set SESSION wsrep_sync_wait=0; INSERT INTO t1 VALUES (4); set GLOBAL wsrep_slave_threads=5; SELECT COUNT(*) = 5 FROM t1; diff --git a/mysql-test/suite/galera/t/pxc-421.test b/mysql-test/suite/galera/t/pxc-421.test index 7bb2354853b..33a2b157f18 100644 --- a/mysql-test/suite/galera/t/pxc-421.test +++ b/mysql-test/suite/galera/t/pxc-421.test @@ -36,9 +36,7 @@ INSERT INTO t1 VALUES (3); --eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig'; --enable_query_log -set SESSION wsrep_sync_wait=0; --source include/wait_until_connected_again.inc ---source include/galera_wait_ready.inc INSERT INTO t1 VALUES (4); set GLOBAL wsrep_slave_threads=5; From d98514690fc647682ac06e7b09502b4274fa3e20 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 11 Apr 2018 15:44:31 +0100 Subject: [PATCH 03/12] Make mroonga respect -DWITHOUT_DYNAMIC_PLUGINS. Return early if plugin is not built. --- storage/mroonga/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/mroonga/CMakeLists.txt b/storage/mroonga/CMakeLists.txt index 5d8e8c1eeb8..5a7d4699386 100644 --- a/storage/mroonga/CMakeLists.txt +++ b/storage/mroonga/CMakeLists.txt @@ -315,6 +315,9 @@ if(MRN_BUNDLED) ${MRN_ALL_SOURCES} STORAGE_ENGINE MODULE_ONLY LINK_LIBRARIES ${MRN_LIBRARIES}) + if(NOT TARGET mroonga) + return() + endif() else() add_library(mroonga MODULE ${MRN_ALL_SOURCES}) From 9518ddd1fb4f811df369a0ac729d3fcacbf1aee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 11 Apr 2018 14:06:29 +0300 Subject: [PATCH 04/12] MDEV-12903: encryption.innodb_encryption_discard_import fails in buildbot with FOUND vs NOT FOUND Wait until rotation has ended and shutdown before grep to make sure that dirty pages are on datafiles. --- .../r/innodb_encryption_discard_import.result | 2 ++ .../t/innodb_encryption_discard_import.test | 27 +++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result index d08a6e943f4..a2a4f25b026 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result +++ b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result @@ -73,6 +73,8 @@ NOT FOUND /foobar/ in t1.ibd NOT FOUND /temp/ in t2.ibd # t3 ... on expecting NOT FOUND NOT FOUND /barfoo/ in t3.ibd +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; ALTER TABLE t1 ENGINE InnoDB; SHOW CREATE TABLE t1; Table Create Table diff --git a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test index 192233a535f..29c762a5d0f 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test @@ -50,10 +50,12 @@ set autocommit=1; --echo # Wait max 10 min for key encryption threads to encrypt all spaces --let $wait_timeout= 600 ---let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND ROTATING_OR_FLUSHING <> 0 --source include/wait_condition.inc ---sleep 5 +# shutdown so that grep is safe +--source include/shutdown_mysqld.inc + --echo # tablespaces should be now encrypted --let SEARCH_PATTERN=foobar --echo # t1 yes on expecting NOT FOUND @@ -68,7 +70,9 @@ set autocommit=1; -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc +--source include/start_mysqld.inc let MYSQLD_DATADIR =`SELECT @@datadir`; + --list_files $MYSQLD_DATADIR/test FLUSH TABLES t1, t2, t3 FOR EXPORT; perl; @@ -95,7 +99,9 @@ SELECT COUNT(1) FROM t2; ALTER TABLE t3 IMPORT TABLESPACE; SELECT COUNT(1) FROM t3; ---sleep 5 +# shutdown so that grep is safe +--source include/shutdown_mysqld.inc + --echo # tablespaces should remain encrypted after import --let SEARCH_PATTERN=foobar --echo # t1 yes on expecting NOT FOUND @@ -110,6 +116,13 @@ SELECT COUNT(1) FROM t3; -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc +--source include/start_mysqld.inc + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + ALTER TABLE t1 ENGINE InnoDB; SHOW CREATE TABLE t1; ALTER TABLE t2 ENGINE InnoDB; @@ -126,7 +139,9 @@ SELECT COUNT(1) FROM t1; SELECT COUNT(1) FROM t2; SELECT COUNT(1) FROM t3; ---sleep 5 +# shutdown so that grep is safe +--source include/shutdown_mysqld.inc + --echo # Tablespaces should be encrypted after restart --let SEARCH_PATTERN=foobar --echo # t1 yes on expecting NOT FOUND @@ -141,9 +156,11 @@ SELECT COUNT(1) FROM t3; -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc +--source include/start_mysqld.inc + --echo # Wait max 10 min for key encryption threads to encrypt all spaces --let $wait_timeout= 600 ---let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND ROTATING_OR_FLUSHING <> 0 --source include/wait_condition.inc --echo # Success! From 0ae13b51d780d4c0e2ef6fc251a8e03140663d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 12 Apr 2018 11:00:48 +0300 Subject: [PATCH 05/12] MDEV-12632: Source and destination overlap in memcpy, encryption.innodb-discard-import-change fails in buildbot with valgrind Problem was that if tablespace was encrypted we try to copy also page 0 from read buffer to write buffer that are in that case the same memory area. fil_iterate When tablespace is encrypted or compressed its first page (i.e. page 0) is not encrypted or compressed and there is no need to copy buffer. --- storage/innobase/row/row0import.cc | 14 ++++++++++---- storage/xtradb/row/row0import.cc | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index c00eb57f91d..b7da200f10a 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -3434,13 +3434,13 @@ fil_iterate( bool updated = false; os_offset_t page_off = offset; ulint n_pages_read = (ulint) n_bytes / iter.page_size; - bool decrypted = false; const ulint size = iter.page_size; block->page.offset = page_off / size; for (ulint i = 0; i < n_pages_read; ++i, page_off += size, block->frame += size, block->page.offset++) { + bool decrypted = false; dberr_t err = DB_SUCCESS; byte* src = readptr + (i * size); byte* dst = io_buffer + (i * size); @@ -3487,6 +3487,7 @@ fil_iterate( block->frame = src; frame_changed = true; } else { + ut_ad(dst != src); memcpy(dst, src, size); } } @@ -3547,9 +3548,13 @@ page_corrupted: ut_ad(encrypted ? src != dst && dst != writeptr + (i * size):1); - if (encrypted) { - memcpy(writeptr + (i * size), - callback.get_frame(block), size); + /* When tablespace is encrypted or compressed its + first page (i.e. page 0) is not encrypted or + compressed and there is no need to copy frame. */ + if (encrypted && i != 0) { + byte *local_frame = callback.get_frame(block); + ut_ad((writeptr + (i * size)) != local_frame); + memcpy((writeptr + (i * size)), local_frame, size); } if (frame_changed) { @@ -3597,6 +3602,7 @@ page_corrupted: if (tmp == src) { /* TODO: remove unnecessary memcpy's */ + ut_ad(dest != src); memcpy(dest, src, size); } diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index c00eb57f91d..b7da200f10a 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -3434,13 +3434,13 @@ fil_iterate( bool updated = false; os_offset_t page_off = offset; ulint n_pages_read = (ulint) n_bytes / iter.page_size; - bool decrypted = false; const ulint size = iter.page_size; block->page.offset = page_off / size; for (ulint i = 0; i < n_pages_read; ++i, page_off += size, block->frame += size, block->page.offset++) { + bool decrypted = false; dberr_t err = DB_SUCCESS; byte* src = readptr + (i * size); byte* dst = io_buffer + (i * size); @@ -3487,6 +3487,7 @@ fil_iterate( block->frame = src; frame_changed = true; } else { + ut_ad(dst != src); memcpy(dst, src, size); } } @@ -3547,9 +3548,13 @@ page_corrupted: ut_ad(encrypted ? src != dst && dst != writeptr + (i * size):1); - if (encrypted) { - memcpy(writeptr + (i * size), - callback.get_frame(block), size); + /* When tablespace is encrypted or compressed its + first page (i.e. page 0) is not encrypted or + compressed and there is no need to copy frame. */ + if (encrypted && i != 0) { + byte *local_frame = callback.get_frame(block); + ut_ad((writeptr + (i * size)) != local_frame); + memcpy((writeptr + (i * size)), local_frame, size); } if (frame_changed) { @@ -3597,6 +3602,7 @@ page_corrupted: if (tmp == src) { /* TODO: remove unnecessary memcpy's */ + ut_ad(dest != src); memcpy(dest, src, size); } From 15071226a04125976e19a552103821eeb7227d75 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 11 Apr 2018 23:07:23 +0100 Subject: [PATCH 06/12] MDEV-15780 : Mariabackup with absolute paths in innodb_data_file_path System tablespace can be specified with absolute path, if innodb_data_home_dir is empty. This was not handled well by mariabackup 1. In backup phase, empty innodb_data_home_dir variable from server was not recognized. full paths were stored in backup-my.ini, even if it stored all files locally. thus prepare phase would not find the system tablespace files. 2. In copy-back phase, copy would not be done to the absolute destination path, as path would be stripped with trim_dotslash This patch fixes the above defects. --- extra/mariabackup/backup_copy.cc | 6 ++- extra/mariabackup/backup_mysql.cc | 42 ++++++++++++++++++- .../mariabackup/absolute_ibdata_paths.opt | 1 + .../mariabackup/absolute_ibdata_paths.result | 10 +++++ .../mariabackup/absolute_ibdata_paths.test | 31 ++++++++++++++ 5 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 mysql-test/suite/mariabackup/absolute_ibdata_paths.opt create mode 100644 mysql-test/suite/mariabackup/absolute_ibdata_paths.result create mode 100644 mysql-test/suite/mariabackup/absolute_ibdata_paths.test diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index c019209faad..d3253348d72 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -975,6 +975,9 @@ copy_file(ds_ctxt_t *datasink, datafile_cur_t cursor; xb_fil_cur_result_t res; const char *action; + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back)? + dst_file_path : trim_dotslash(dst_file_path); if (!datafile_open(src_file_path, &cursor, thread_n)) { goto error_close; @@ -982,8 +985,7 @@ copy_file(ds_ctxt_t *datasink, strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); - dstfile = ds_open(datasink, trim_dotslash(dst_file_path), - &cursor.statinfo); + dstfile = ds_open(datasink, dst_path, &cursor.statinfo); if (dstfile == NULL) { msg("[%02u] error: " "cannot open the destination stream for %s\n", diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 6f425e9419f..661992a1696 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -477,7 +477,7 @@ get_mysql_vars(MYSQL *connection) innodb_data_file_path_var, MYF(MY_FAE)); } - if (innodb_data_home_dir_var && *innodb_data_home_dir_var) { + if (innodb_data_home_dir_var) { innobase_data_home_dir = my_strdup( innodb_data_home_dir_var, MYF(MY_FAE)); } @@ -1521,6 +1521,44 @@ cleanup: extern const char *innodb_checksum_algorithm_names[]; +#ifdef _WIN32 +#include +#endif + +static std::string make_local_paths(const char *data_file_path) +{ + if (strchr(data_file_path, '/') == 0 +#ifdef _WIN32 + && strchr(data_file_path, '\\') == 0 +#endif + ){ + return std::string(data_file_path); + } + + std::ostringstream buf; + + char *dup = strdup(innobase_data_file_path); + ut_a(dup); + char *p; + char * token = strtok_r(dup, ";", &p); + while (token) { + if (buf.tellp()) + buf << ";"; + + char *fname = strrchr(token, '/'); +#ifdef _WIN32 + fname = std::max(fname,strrchr(token, '\\')); +#endif + if (fname) + buf << fname + 1; + else + buf << token; + token = strtok_r(NULL, ";", &p); + } + free(dup); + return buf.str(); +} + bool write_backup_config_file() { int rc= backup_file_printf("backup-my.cnf", @@ -1541,7 +1579,7 @@ bool write_backup_config_file() "%s\n", innodb_checksum_algorithm_names[srv_checksum_algorithm], innodb_checksum_algorithm_names[srv_log_checksum_algorithm], - innobase_data_file_path, + make_local_paths(innobase_data_file_path).c_str(), srv_n_log_files, innobase_log_file_size, srv_page_size, diff --git a/mysql-test/suite/mariabackup/absolute_ibdata_paths.opt b/mysql-test/suite/mariabackup/absolute_ibdata_paths.opt new file mode 100644 index 00000000000..52b6b743ac8 --- /dev/null +++ b/mysql-test/suite/mariabackup/absolute_ibdata_paths.opt @@ -0,0 +1 @@ +--innodb --innodb-data-home-dir= --innodb-data-file-path=$MYSQLTEST_VARDIR/tmp/absolute_path_ibdata1:3M;ibdata_second:1M:autoextend \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/absolute_ibdata_paths.result b/mysql-test/suite/mariabackup/absolute_ibdata_paths.result new file mode 100644 index 00000000000..fe211e71f2f --- /dev/null +++ b/mysql-test/suite/mariabackup/absolute_ibdata_paths.result @@ -0,0 +1,10 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup +# remove datadir +# xtrabackup copy back +# restart server +SELECT * from t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/absolute_ibdata_paths.test b/mysql-test/suite/mariabackup/absolute_ibdata_paths.test new file mode 100644 index 00000000000..6717f16d199 --- /dev/null +++ b/mysql-test/suite/mariabackup/absolute_ibdata_paths.test @@ -0,0 +1,31 @@ +# This test just backs up and restores empty database +# Innodb system tablespace is specified with absolute path in the .opt file +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +echo # xtrabackup backup; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +let $_innodb_data_file_path=`select @@innodb_data_file_path`; +let $_innodb_data_home_dir=`select @@innodb_data_home_dir`; +let $_datadir= `SELECT @@datadir`; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; + +--source include/shutdown_mysqld.inc +echo # remove datadir; +rmdir $_datadir; +#remove out-of-datadir ibdata1 +remove_file $MYSQLTEST_VARDIR/tmp/absolute_path_ibdata1; +echo # xtrabackup copy back; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$_datadir --target-dir=$targetdir "--innodb_data_file_path=$_innodb_data_file_path" --innodb_data_home_dir=$_innodb_data_home_dir; +echo # restart server; +--source include/start_mysqld.inc +--enable_result_log + +SELECT * from t; +DROP TABLE t; +rmdir $targetdir; + From c2dc72c0c3952a43e29b374de83ecd4bdfa58e74 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 12 Apr 2018 12:09:32 +0100 Subject: [PATCH 07/12] MDEV-15779 - mariabackup incremental prepare fails on CIFS mount. CIFS does not like O_DIRECT flag (it is set successfully, but pread would fail). The fix is not to use O_DIRECT, there is not need for it. posix_fadvise() was used already that should prevent buffer cache pollution on Linux. As recommended by documentation of posix_fadvise(), we'll also fsync() tablespaces after a batch of writes. --- extra/mariabackup/xtrabackup.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 02432dbb5b1..c4d385fef5e 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4966,8 +4966,6 @@ xtrabackup_apply_delta( posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL); - os_file_set_nocache(src_file, src_path, "OPEN"); - dst_file = xb_delta_open_matching_space( dbname, space_name, info.space_id, info.zip_size, dst_path, sizeof(dst_path), &success); @@ -4978,8 +4976,6 @@ xtrabackup_apply_delta( posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED); - os_file_set_nocache(dst_file, dst_path, "OPEN"); - /* allocate buffer for incremental backup (4096 pages) */ incremental_buffer_base = static_cast (ut_malloc((page_size / 4 + 1) * @@ -5079,6 +5075,13 @@ xtrabackup_apply_delta( } } + /* Free file system buffer cache after the batch was written. */ +#ifdef __linux__ + os_file_flush_func(dst_file); +#endif + posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED); + + incremental_buffers++; } From 82e968197adcd97177f992cdf76a1d0119bfc648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 12 Apr 2018 16:02:25 +0300 Subject: [PATCH 08/12] MDEV-15580: Assertion `!lex->explain' failed in lex_start(THD*): Problem was that we did not delete explain information when Galera must replay a query. Could not find easily repeatable test case that would not cause other problems. --- sql/sql_parse.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index cb432eb7ccd..352a1704ab7 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -7201,6 +7201,8 @@ static void wsrep_mysql_parse(THD *thd, char *rawbuf, uint length, if (thd->wsrep_conflict_state == MUST_REPLAY) { + if (thd->lex->explain) + delete_explain_query(thd->lex); wsrep_replay_transaction(thd); } @@ -7251,8 +7253,12 @@ static void wsrep_mysql_parse(THD *thd, char *rawbuf, uint length, } /* If retry is requested clean up explain structure */ - if (thd->wsrep_conflict_state == RETRY_AUTOCOMMIT && thd->lex->explain) + if ((thd->wsrep_conflict_state == RETRY_AUTOCOMMIT || + thd->wsrep_conflict_state == MUST_REPLAY ) + && thd->lex->explain) + { delete_explain_query(thd->lex); + } } while (thd->wsrep_conflict_state== RETRY_AUTOCOMMIT); From 3855943fe09ff7d3dec45815139cb5f606b121e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 13 Apr 2018 09:25:52 +0300 Subject: [PATCH 09/12] MDEV-12632: Source and destination overlap in memcpy, encryption.innodb-discard-import-change fails in buildbot with valgrind Use block->page.offset for checking page number. --- storage/innobase/row/row0import.cc | 2 +- storage/xtradb/row/row0import.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index b7da200f10a..3e60287a2e4 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -3551,7 +3551,7 @@ page_corrupted: /* When tablespace is encrypted or compressed its first page (i.e. page 0) is not encrypted or compressed and there is no need to copy frame. */ - if (encrypted && i != 0) { + if (encrypted && block->page.offset != 0) { byte *local_frame = callback.get_frame(block); ut_ad((writeptr + (i * size)) != local_frame); memcpy((writeptr + (i * size)), local_frame, size); diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index b7da200f10a..3e60287a2e4 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -3551,7 +3551,7 @@ page_corrupted: /* When tablespace is encrypted or compressed its first page (i.e. page 0) is not encrypted or compressed and there is no need to copy frame. */ - if (encrypted && i != 0) { + if (encrypted && block->page.offset != 0) { byte *local_frame = callback.get_frame(block); ut_ad((writeptr + (i * size)) != local_frame); memcpy((writeptr + (i * size)), local_frame, size); From 2bf355fa812e30a25427c10b13c7db51fd1a1f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 13 Apr 2018 12:50:03 +0300 Subject: [PATCH 10/12] MDEV-15672: encryption.innodb_encryption_filekeys - typo in I_S column name: ENCRYPTION_SHCEME Fixed typo on select that is executed only when something unexpected happens. --- .../suite/encryption/t/innodb_encryption_filekeys.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test b/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test index f73e78aa5bf..60afb6033ac 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test @@ -56,7 +56,7 @@ while ($cnt) } if (!$success) { - SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SELECT NAME,ENCRYPTION_SCHEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; SHOW STATUS LIKE 'innodb_encryption%'; -- die Timeout waiting for encryption threads } @@ -86,7 +86,7 @@ while ($cnt) } if (!$success) { - SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SELECT NAME,ENCRYPTION_SCHEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; SHOW STATUS LIKE 'innodb_encryption%'; -- die Timeout waiting for encryption threads } @@ -115,7 +115,7 @@ while ($cnt) } if (!$success) { - SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SELECT NAME,ENCRYPTION_SCHEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; SHOW STATUS LIKE 'innodb_encryption%'; -- die Timeout waiting for encryption threads } From 3e12e39fb3dd0346578d50fe98215f1c6bce354c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 13 Apr 2018 22:18:36 +0200 Subject: [PATCH 11/12] skip innodb-table-online,crypt tests if no encryption plugin --- mysql-test/suite.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm index cb9ab0c375d..5ac3bd3e138 100644 --- a/mysql-test/suite.pm +++ b/mysql-test/suite.pm @@ -20,6 +20,9 @@ sub skip_combinations { my %skip = ( 'include/have_innodb.combinations' => [ @combinations ], 'include/have_xtradb.combinations' => [ @combinations ]); + $skip{'include/innodb_encrypt_log.combinations'} = [ 'crypt' ] + unless $ENV{DEBUG_KEY_MANAGEMENT_SO}; + # don't run tests for the wrong platform $skip{'include/platform.combinations'} = [ (IS_WINDOWS) ? 'unix' : 'win' ]; From 3d1ad2a5e85284b5ec4bb28f744b93b0b73e3934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 16 Apr 2018 15:06:41 +0300 Subject: [PATCH 12/12] MDEV-13516: encryption.create_or_replace test fails in buildbot with InnoDB assertion failure fil_crypt_rotate_pages If tablespace is marked as stopping stop also page rotation fil_crypt_flush_space If tablespace is marked as stopping do not try to read page 0 and write it back. --- storage/innobase/fil/fil0crypt.cc | 24 ++++++++++++++++-------- storage/xtradb/fil/fil0crypt.cc | 24 ++++++++++++++++-------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index cb499121c86..2d149f07433 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2019,6 +2019,12 @@ fil_crypt_rotate_pages( continue; } + /* If space is marked as stopping, stop rotating + pages. */ + if (state->space->is_stopping()) { + break; + } + fil_crypt_rotate_page(key_state, state); } } @@ -2067,20 +2073,22 @@ fil_crypt_flush_space( crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; } - /* update page 0 */ - mtr_t mtr; - mtr_start(&mtr); + if (!space->is_stopping()) { + /* update page 0 */ + mtr_t mtr; + mtr_start(&mtr); - const uint zip_size = fsp_flags_get_zip_size(state->space->flags); + const uint zip_size = fsp_flags_get_zip_size(state->space->flags); - buf_block_t* block = buf_page_get_gen(space->id, zip_size, 0, + buf_block_t* block = buf_page_get_gen(space->id, zip_size, 0, RW_X_LATCH, NULL, BUF_GET, __FILE__, __LINE__, &mtr); - byte* frame = buf_block_get_frame(block); + byte* frame = buf_block_get_frame(block); - crypt_data->write_page0(frame, &mtr); + crypt_data->write_page0(frame, &mtr); - mtr_commit(&mtr); + mtr_commit(&mtr); + } } /*********************************************************************** diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index cb499121c86..2d149f07433 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -2019,6 +2019,12 @@ fil_crypt_rotate_pages( continue; } + /* If space is marked as stopping, stop rotating + pages. */ + if (state->space->is_stopping()) { + break; + } + fil_crypt_rotate_page(key_state, state); } } @@ -2067,20 +2073,22 @@ fil_crypt_flush_space( crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; } - /* update page 0 */ - mtr_t mtr; - mtr_start(&mtr); + if (!space->is_stopping()) { + /* update page 0 */ + mtr_t mtr; + mtr_start(&mtr); - const uint zip_size = fsp_flags_get_zip_size(state->space->flags); + const uint zip_size = fsp_flags_get_zip_size(state->space->flags); - buf_block_t* block = buf_page_get_gen(space->id, zip_size, 0, + buf_block_t* block = buf_page_get_gen(space->id, zip_size, 0, RW_X_LATCH, NULL, BUF_GET, __FILE__, __LINE__, &mtr); - byte* frame = buf_block_get_frame(block); + byte* frame = buf_block_get_frame(block); - crypt_data->write_page0(frame, &mtr); + crypt_data->write_page0(frame, &mtr); - mtr_commit(&mtr); + mtr_commit(&mtr); + } } /***********************************************************************