From 73b3ace8b26ef809e47b264e7d3630d7b887bff6 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 11 Apr 2018 09:14:35 +0200 Subject: [PATCH 01/17] MDEV-15808 Fix and re-enable test galera.galera_gra_log Test would occasionally fail as follows: ``` mysqltest: At line 20: query 'SELECT COUNT(*) = 0 FROM t1' failed: 1317: Query execution was interrupted ``` This was due to a `CREATE TABLE` applied concurrently that would occasionally cause the `SELECT` to be BF aborted, due to MDL conflict. The `SELECT` is executed while `wsrep_on=OFF`. The change makes sure it is executed only after wsrep is enabled again. --- mysql-test/suite/galera/disabled.def | 1 - mysql-test/suite/galera/r/galera_gra_log.result | 2 +- mysql-test/suite/galera/t/galera_gra_log.test | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 4a9edd1bf32..9c1ddb3af17 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -31,6 +31,5 @@ galera_var_retry_autocommit : MDEV-15794 Test failure on galera.galera_var_retry galera_var_auto_inc_control_on : MDEV-15803 Test failure on galera.galera_var_auto_inc_control_on pxc-421 : MDEV-15804 Test failure on galera.pxc-421 query_cache : MDEV-15805 Test failure on galera.query_cache -galera.galera_gra_log : MDEV-15808 Test failure on galera.galera_gra_log galera.MW-44 : MDEV-15809 Test failure on galera.MW-44 galera.galera_pc_ignore_sb : MDEV-15811 Test failure on galera_pc_ignore_sb diff --git a/mysql-test/suite/galera/r/galera_gra_log.result b/mysql-test/suite/galera/r/galera_gra_log.result index 6b816d0127f..a5b87c0ef0c 100644 --- a/mysql-test/suite/galera/r/galera_gra_log.result +++ b/mysql-test/suite/galera/r/galera_gra_log.result @@ -1,6 +1,7 @@ SET SESSION wsrep_on=OFF; CREATE TABLE t1 (f1 INTEGER); CREATE TABLE t1 (f1 INTEGER); +SET SESSION wsrep_on=ON; SELECT COUNT(*) = 0 FROM t1; COUNT(*) = 0 1 @@ -27,6 +28,5 @@ DELIMITER ; ROLLBACK /* added by mysqlbinlog */; /*!50003 SET COMPLETION_TYPE=@OLD_COMPLETION_TYPE*/; /*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/; -SET SESSION wsrep_on=ON; CALL mtr.add_suppression("Slave SQL: Error 'Table 't1' already exists' on query"); DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_gra_log.test b/mysql-test/suite/galera/t/galera_gra_log.test index adb37de9188..8b5aaaae5bd 100644 --- a/mysql-test/suite/galera/t/galera_gra_log.test +++ b/mysql-test/suite/galera/t/galera_gra_log.test @@ -17,6 +17,7 @@ CREATE TABLE t1 (f1 INTEGER); CREATE TABLE t1 (f1 INTEGER); --connection node_2 +SET SESSION wsrep_on=ON; SELECT COUNT(*) = 0 FROM t1; # Make sure the GRA file produced is readable and contains the failure @@ -24,8 +25,6 @@ SELECT COUNT(*) = 0 FROM t1; --replace_regex /SET TIMESTAMP=[0-9]+/SET TIMESTAMP=/ /pseudo_thread_id=[0-9]+/pseudo_thread_id=/ --exec $MYSQL_BINLOG --short-form $MYSQLTEST_VARDIR/mysqld.2/data/GRA_*.log -SET SESSION wsrep_on=ON; - CALL mtr.add_suppression("Slave SQL: Error 'Table 't1' already exists' on query"); DROP TABLE t1; From 4dc60dc3f10dd7522236fbc80dc184678352bdc9 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 11 Apr 2018 14:24:39 +0200 Subject: [PATCH 02/17] MDEV-15804 Fix and re-enable MTR test galera.pxc-421 This patch makes two changes: * Remove unnecessary `set SESSION wsrep_sync_wait=0`. Disabling `wsrep_sync_wait` caused the test to fail occasionally, due to subsequent `SELECT`s observing stale values. * Remove redundant `--source include/galera_wait_ready.inc`, `galera_wait_ready.inc` is already included in the above `wait_until_connected_again.inc` --- mysql-test/suite/galera/disabled.def | 1 - mysql-test/suite/galera/r/pxc-421.result | 1 - mysql-test/suite/galera/t/pxc-421.test | 2 -- 3 files changed, 4 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 4a9edd1bf32..f705368a5e2 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -29,7 +29,6 @@ galera.MW-329 : wsrep_local_replays not stable galera.MW-328A : have_deadlocks test not stable galera_var_retry_autocommit : MDEV-15794 Test failure on galera.galera_var_retry_autocommit galera_var_auto_inc_control_on : MDEV-15803 Test failure on galera.galera_var_auto_inc_control_on -pxc-421 : MDEV-15804 Test failure on galera.pxc-421 query_cache : MDEV-15805 Test failure on galera.query_cache galera.galera_gra_log : MDEV-15808 Test failure on galera.galera_gra_log galera.MW-44 : MDEV-15809 Test failure on galera.MW-44 diff --git a/mysql-test/suite/galera/r/pxc-421.result b/mysql-test/suite/galera/r/pxc-421.result index ba1f3d66720..1822201f338 100644 --- a/mysql-test/suite/galera/r/pxc-421.result +++ b/mysql-test/suite/galera/r/pxc-421.result @@ -6,7 +6,6 @@ set GLOBAL wsrep_slave_threads=16; SET GLOBAL wsrep_provider='none'; INSERT INTO t1 VALUES (2); INSERT INTO t1 VALUES (3); -set SESSION wsrep_sync_wait=0; INSERT INTO t1 VALUES (4); set GLOBAL wsrep_slave_threads=5; SELECT COUNT(*) = 5 FROM t1; diff --git a/mysql-test/suite/galera/t/pxc-421.test b/mysql-test/suite/galera/t/pxc-421.test index 7bb2354853b..33a2b157f18 100644 --- a/mysql-test/suite/galera/t/pxc-421.test +++ b/mysql-test/suite/galera/t/pxc-421.test @@ -36,9 +36,7 @@ INSERT INTO t1 VALUES (3); --eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig'; --enable_query_log -set SESSION wsrep_sync_wait=0; --source include/wait_until_connected_again.inc ---source include/galera_wait_ready.inc INSERT INTO t1 VALUES (4); set GLOBAL wsrep_slave_threads=5; From d98514690fc647682ac06e7b09502b4274fa3e20 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 11 Apr 2018 15:44:31 +0100 Subject: [PATCH 03/17] Make mroonga respect -DWITHOUT_DYNAMIC_PLUGINS. Return early if plugin is not built. --- storage/mroonga/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage/mroonga/CMakeLists.txt b/storage/mroonga/CMakeLists.txt index 5d8e8c1eeb8..5a7d4699386 100644 --- a/storage/mroonga/CMakeLists.txt +++ b/storage/mroonga/CMakeLists.txt @@ -315,6 +315,9 @@ if(MRN_BUNDLED) ${MRN_ALL_SOURCES} STORAGE_ENGINE MODULE_ONLY LINK_LIBRARIES ${MRN_LIBRARIES}) + if(NOT TARGET mroonga) + return() + endif() else() add_library(mroonga MODULE ${MRN_ALL_SOURCES}) From 9518ddd1fb4f811df369a0ac729d3fcacbf1aee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 11 Apr 2018 14:06:29 +0300 Subject: [PATCH 04/17] MDEV-12903: encryption.innodb_encryption_discard_import fails in buildbot with FOUND vs NOT FOUND Wait until rotation has ended and shutdown before grep to make sure that dirty pages are on datafiles. --- .../r/innodb_encryption_discard_import.result | 2 ++ .../t/innodb_encryption_discard_import.test | 27 +++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result index d08a6e943f4..a2a4f25b026 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result +++ b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result @@ -73,6 +73,8 @@ NOT FOUND /foobar/ in t1.ibd NOT FOUND /temp/ in t2.ibd # t3 ... on expecting NOT FOUND NOT FOUND /barfoo/ in t3.ibd +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; ALTER TABLE t1 ENGINE InnoDB; SHOW CREATE TABLE t1; Table Create Table diff --git a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test index 192233a535f..29c762a5d0f 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test @@ -50,10 +50,12 @@ set autocommit=1; --echo # Wait max 10 min for key encryption threads to encrypt all spaces --let $wait_timeout= 600 ---let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND ROTATING_OR_FLUSHING <> 0 --source include/wait_condition.inc ---sleep 5 +# shutdown so that grep is safe +--source include/shutdown_mysqld.inc + --echo # tablespaces should be now encrypted --let SEARCH_PATTERN=foobar --echo # t1 yes on expecting NOT FOUND @@ -68,7 +70,9 @@ set autocommit=1; -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc +--source include/start_mysqld.inc let MYSQLD_DATADIR =`SELECT @@datadir`; + --list_files $MYSQLD_DATADIR/test FLUSH TABLES t1, t2, t3 FOR EXPORT; perl; @@ -95,7 +99,9 @@ SELECT COUNT(1) FROM t2; ALTER TABLE t3 IMPORT TABLESPACE; SELECT COUNT(1) FROM t3; ---sleep 5 +# shutdown so that grep is safe +--source include/shutdown_mysqld.inc + --echo # tablespaces should remain encrypted after import --let SEARCH_PATTERN=foobar --echo # t1 yes on expecting NOT FOUND @@ -110,6 +116,13 @@ SELECT COUNT(1) FROM t3; -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc +--source include/start_mysqld.inc + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + ALTER TABLE t1 ENGINE InnoDB; SHOW CREATE TABLE t1; ALTER TABLE t2 ENGINE InnoDB; @@ -126,7 +139,9 @@ SELECT COUNT(1) FROM t1; SELECT COUNT(1) FROM t2; SELECT COUNT(1) FROM t3; ---sleep 5 +# shutdown so that grep is safe +--source include/shutdown_mysqld.inc + --echo # Tablespaces should be encrypted after restart --let SEARCH_PATTERN=foobar --echo # t1 yes on expecting NOT FOUND @@ -141,9 +156,11 @@ SELECT COUNT(1) FROM t3; -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc +--source include/start_mysqld.inc + --echo # Wait max 10 min for key encryption threads to encrypt all spaces --let $wait_timeout= 600 ---let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND ROTATING_OR_FLUSHING <> 0 --source include/wait_condition.inc --echo # Success! From 0ae13b51d780d4c0e2ef6fc251a8e03140663d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 12 Apr 2018 11:00:48 +0300 Subject: [PATCH 05/17] MDEV-12632: Source and destination overlap in memcpy, encryption.innodb-discard-import-change fails in buildbot with valgrind Problem was that if tablespace was encrypted we try to copy also page 0 from read buffer to write buffer that are in that case the same memory area. fil_iterate When tablespace is encrypted or compressed its first page (i.e. page 0) is not encrypted or compressed and there is no need to copy buffer. --- storage/innobase/row/row0import.cc | 14 ++++++++++---- storage/xtradb/row/row0import.cc | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index c00eb57f91d..b7da200f10a 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -3434,13 +3434,13 @@ fil_iterate( bool updated = false; os_offset_t page_off = offset; ulint n_pages_read = (ulint) n_bytes / iter.page_size; - bool decrypted = false; const ulint size = iter.page_size; block->page.offset = page_off / size; for (ulint i = 0; i < n_pages_read; ++i, page_off += size, block->frame += size, block->page.offset++) { + bool decrypted = false; dberr_t err = DB_SUCCESS; byte* src = readptr + (i * size); byte* dst = io_buffer + (i * size); @@ -3487,6 +3487,7 @@ fil_iterate( block->frame = src; frame_changed = true; } else { + ut_ad(dst != src); memcpy(dst, src, size); } } @@ -3547,9 +3548,13 @@ page_corrupted: ut_ad(encrypted ? src != dst && dst != writeptr + (i * size):1); - if (encrypted) { - memcpy(writeptr + (i * size), - callback.get_frame(block), size); + /* When tablespace is encrypted or compressed its + first page (i.e. page 0) is not encrypted or + compressed and there is no need to copy frame. */ + if (encrypted && i != 0) { + byte *local_frame = callback.get_frame(block); + ut_ad((writeptr + (i * size)) != local_frame); + memcpy((writeptr + (i * size)), local_frame, size); } if (frame_changed) { @@ -3597,6 +3602,7 @@ page_corrupted: if (tmp == src) { /* TODO: remove unnecessary memcpy's */ + ut_ad(dest != src); memcpy(dest, src, size); } diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index c00eb57f91d..b7da200f10a 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -3434,13 +3434,13 @@ fil_iterate( bool updated = false; os_offset_t page_off = offset; ulint n_pages_read = (ulint) n_bytes / iter.page_size; - bool decrypted = false; const ulint size = iter.page_size; block->page.offset = page_off / size; for (ulint i = 0; i < n_pages_read; ++i, page_off += size, block->frame += size, block->page.offset++) { + bool decrypted = false; dberr_t err = DB_SUCCESS; byte* src = readptr + (i * size); byte* dst = io_buffer + (i * size); @@ -3487,6 +3487,7 @@ fil_iterate( block->frame = src; frame_changed = true; } else { + ut_ad(dst != src); memcpy(dst, src, size); } } @@ -3547,9 +3548,13 @@ page_corrupted: ut_ad(encrypted ? src != dst && dst != writeptr + (i * size):1); - if (encrypted) { - memcpy(writeptr + (i * size), - callback.get_frame(block), size); + /* When tablespace is encrypted or compressed its + first page (i.e. page 0) is not encrypted or + compressed and there is no need to copy frame. */ + if (encrypted && i != 0) { + byte *local_frame = callback.get_frame(block); + ut_ad((writeptr + (i * size)) != local_frame); + memcpy((writeptr + (i * size)), local_frame, size); } if (frame_changed) { @@ -3597,6 +3602,7 @@ page_corrupted: if (tmp == src) { /* TODO: remove unnecessary memcpy's */ + ut_ad(dest != src); memcpy(dest, src, size); } From 15071226a04125976e19a552103821eeb7227d75 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 11 Apr 2018 23:07:23 +0100 Subject: [PATCH 06/17] MDEV-15780 : Mariabackup with absolute paths in innodb_data_file_path System tablespace can be specified with absolute path, if innodb_data_home_dir is empty. This was not handled well by mariabackup 1. In backup phase, empty innodb_data_home_dir variable from server was not recognized. full paths were stored in backup-my.ini, even if it stored all files locally. thus prepare phase would not find the system tablespace files. 2. In copy-back phase, copy would not be done to the absolute destination path, as path would be stripped with trim_dotslash This patch fixes the above defects. --- extra/mariabackup/backup_copy.cc | 6 ++- extra/mariabackup/backup_mysql.cc | 42 ++++++++++++++++++- .../mariabackup/absolute_ibdata_paths.opt | 1 + .../mariabackup/absolute_ibdata_paths.result | 10 +++++ .../mariabackup/absolute_ibdata_paths.test | 31 ++++++++++++++ 5 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 mysql-test/suite/mariabackup/absolute_ibdata_paths.opt create mode 100644 mysql-test/suite/mariabackup/absolute_ibdata_paths.result create mode 100644 mysql-test/suite/mariabackup/absolute_ibdata_paths.test diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index c019209faad..d3253348d72 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -975,6 +975,9 @@ copy_file(ds_ctxt_t *datasink, datafile_cur_t cursor; xb_fil_cur_result_t res; const char *action; + const char *dst_path = + (xtrabackup_copy_back || xtrabackup_move_back)? + dst_file_path : trim_dotslash(dst_file_path); if (!datafile_open(src_file_path, &cursor, thread_n)) { goto error_close; @@ -982,8 +985,7 @@ copy_file(ds_ctxt_t *datasink, strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); - dstfile = ds_open(datasink, trim_dotslash(dst_file_path), - &cursor.statinfo); + dstfile = ds_open(datasink, dst_path, &cursor.statinfo); if (dstfile == NULL) { msg("[%02u] error: " "cannot open the destination stream for %s\n", diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 6f425e9419f..661992a1696 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -477,7 +477,7 @@ get_mysql_vars(MYSQL *connection) innodb_data_file_path_var, MYF(MY_FAE)); } - if (innodb_data_home_dir_var && *innodb_data_home_dir_var) { + if (innodb_data_home_dir_var) { innobase_data_home_dir = my_strdup( innodb_data_home_dir_var, MYF(MY_FAE)); } @@ -1521,6 +1521,44 @@ cleanup: extern const char *innodb_checksum_algorithm_names[]; +#ifdef _WIN32 +#include +#endif + +static std::string make_local_paths(const char *data_file_path) +{ + if (strchr(data_file_path, '/') == 0 +#ifdef _WIN32 + && strchr(data_file_path, '\\') == 0 +#endif + ){ + return std::string(data_file_path); + } + + std::ostringstream buf; + + char *dup = strdup(innobase_data_file_path); + ut_a(dup); + char *p; + char * token = strtok_r(dup, ";", &p); + while (token) { + if (buf.tellp()) + buf << ";"; + + char *fname = strrchr(token, '/'); +#ifdef _WIN32 + fname = std::max(fname,strrchr(token, '\\')); +#endif + if (fname) + buf << fname + 1; + else + buf << token; + token = strtok_r(NULL, ";", &p); + } + free(dup); + return buf.str(); +} + bool write_backup_config_file() { int rc= backup_file_printf("backup-my.cnf", @@ -1541,7 +1579,7 @@ bool write_backup_config_file() "%s\n", innodb_checksum_algorithm_names[srv_checksum_algorithm], innodb_checksum_algorithm_names[srv_log_checksum_algorithm], - innobase_data_file_path, + make_local_paths(innobase_data_file_path).c_str(), srv_n_log_files, innobase_log_file_size, srv_page_size, diff --git a/mysql-test/suite/mariabackup/absolute_ibdata_paths.opt b/mysql-test/suite/mariabackup/absolute_ibdata_paths.opt new file mode 100644 index 00000000000..52b6b743ac8 --- /dev/null +++ b/mysql-test/suite/mariabackup/absolute_ibdata_paths.opt @@ -0,0 +1 @@ +--innodb --innodb-data-home-dir= --innodb-data-file-path=$MYSQLTEST_VARDIR/tmp/absolute_path_ibdata1:3M;ibdata_second:1M:autoextend \ No newline at end of file diff --git a/mysql-test/suite/mariabackup/absolute_ibdata_paths.result b/mysql-test/suite/mariabackup/absolute_ibdata_paths.result new file mode 100644 index 00000000000..fe211e71f2f --- /dev/null +++ b/mysql-test/suite/mariabackup/absolute_ibdata_paths.result @@ -0,0 +1,10 @@ +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup +# remove datadir +# xtrabackup copy back +# restart server +SELECT * from t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/absolute_ibdata_paths.test b/mysql-test/suite/mariabackup/absolute_ibdata_paths.test new file mode 100644 index 00000000000..6717f16d199 --- /dev/null +++ b/mysql-test/suite/mariabackup/absolute_ibdata_paths.test @@ -0,0 +1,31 @@ +# This test just backs up and restores empty database +# Innodb system tablespace is specified with absolute path in the .opt file +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +echo # xtrabackup backup; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +let $_innodb_data_file_path=`select @@innodb_data_file_path`; +let $_innodb_data_home_dir=`select @@innodb_data_home_dir`; +let $_datadir= `SELECT @@datadir`; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; + +--source include/shutdown_mysqld.inc +echo # remove datadir; +rmdir $_datadir; +#remove out-of-datadir ibdata1 +remove_file $MYSQLTEST_VARDIR/tmp/absolute_path_ibdata1; +echo # xtrabackup copy back; +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$_datadir --target-dir=$targetdir "--innodb_data_file_path=$_innodb_data_file_path" --innodb_data_home_dir=$_innodb_data_home_dir; +echo # restart server; +--source include/start_mysqld.inc +--enable_result_log + +SELECT * from t; +DROP TABLE t; +rmdir $targetdir; + From c2dc72c0c3952a43e29b374de83ecd4bdfa58e74 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 12 Apr 2018 12:09:32 +0100 Subject: [PATCH 07/17] MDEV-15779 - mariabackup incremental prepare fails on CIFS mount. CIFS does not like O_DIRECT flag (it is set successfully, but pread would fail). The fix is not to use O_DIRECT, there is not need for it. posix_fadvise() was used already that should prevent buffer cache pollution on Linux. As recommended by documentation of posix_fadvise(), we'll also fsync() tablespaces after a batch of writes. --- extra/mariabackup/xtrabackup.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 02432dbb5b1..c4d385fef5e 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -4966,8 +4966,6 @@ xtrabackup_apply_delta( posix_fadvise(src_file, 0, 0, POSIX_FADV_SEQUENTIAL); - os_file_set_nocache(src_file, src_path, "OPEN"); - dst_file = xb_delta_open_matching_space( dbname, space_name, info.space_id, info.zip_size, dst_path, sizeof(dst_path), &success); @@ -4978,8 +4976,6 @@ xtrabackup_apply_delta( posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED); - os_file_set_nocache(dst_file, dst_path, "OPEN"); - /* allocate buffer for incremental backup (4096 pages) */ incremental_buffer_base = static_cast (ut_malloc((page_size / 4 + 1) * @@ -5079,6 +5075,13 @@ xtrabackup_apply_delta( } } + /* Free file system buffer cache after the batch was written. */ +#ifdef __linux__ + os_file_flush_func(dst_file); +#endif + posix_fadvise(dst_file, 0, 0, POSIX_FADV_DONTNEED); + + incremental_buffers++; } From 82e968197adcd97177f992cdf76a1d0119bfc648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 12 Apr 2018 16:02:25 +0300 Subject: [PATCH 08/17] MDEV-15580: Assertion `!lex->explain' failed in lex_start(THD*): Problem was that we did not delete explain information when Galera must replay a query. Could not find easily repeatable test case that would not cause other problems. --- sql/sql_parse.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index cb432eb7ccd..352a1704ab7 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -7201,6 +7201,8 @@ static void wsrep_mysql_parse(THD *thd, char *rawbuf, uint length, if (thd->wsrep_conflict_state == MUST_REPLAY) { + if (thd->lex->explain) + delete_explain_query(thd->lex); wsrep_replay_transaction(thd); } @@ -7251,8 +7253,12 @@ static void wsrep_mysql_parse(THD *thd, char *rawbuf, uint length, } /* If retry is requested clean up explain structure */ - if (thd->wsrep_conflict_state == RETRY_AUTOCOMMIT && thd->lex->explain) + if ((thd->wsrep_conflict_state == RETRY_AUTOCOMMIT || + thd->wsrep_conflict_state == MUST_REPLAY ) + && thd->lex->explain) + { delete_explain_query(thd->lex); + } } while (thd->wsrep_conflict_state== RETRY_AUTOCOMMIT); From 3855943fe09ff7d3dec45815139cb5f606b121e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 13 Apr 2018 09:25:52 +0300 Subject: [PATCH 09/17] MDEV-12632: Source and destination overlap in memcpy, encryption.innodb-discard-import-change fails in buildbot with valgrind Use block->page.offset for checking page number. --- storage/innobase/row/row0import.cc | 2 +- storage/xtradb/row/row0import.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index b7da200f10a..3e60287a2e4 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -3551,7 +3551,7 @@ page_corrupted: /* When tablespace is encrypted or compressed its first page (i.e. page 0) is not encrypted or compressed and there is no need to copy frame. */ - if (encrypted && i != 0) { + if (encrypted && block->page.offset != 0) { byte *local_frame = callback.get_frame(block); ut_ad((writeptr + (i * size)) != local_frame); memcpy((writeptr + (i * size)), local_frame, size); diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index b7da200f10a..3e60287a2e4 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -3551,7 +3551,7 @@ page_corrupted: /* When tablespace is encrypted or compressed its first page (i.e. page 0) is not encrypted or compressed and there is no need to copy frame. */ - if (encrypted && i != 0) { + if (encrypted && block->page.offset != 0) { byte *local_frame = callback.get_frame(block); ut_ad((writeptr + (i * size)) != local_frame); memcpy((writeptr + (i * size)), local_frame, size); From 2bf355fa812e30a25427c10b13c7db51fd1a1f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Fri, 13 Apr 2018 12:50:03 +0300 Subject: [PATCH 10/17] MDEV-15672: encryption.innodb_encryption_filekeys - typo in I_S column name: ENCRYPTION_SHCEME Fixed typo on select that is executed only when something unexpected happens. --- .../suite/encryption/t/innodb_encryption_filekeys.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test b/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test index f73e78aa5bf..60afb6033ac 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test @@ -56,7 +56,7 @@ while ($cnt) } if (!$success) { - SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SELECT NAME,ENCRYPTION_SCHEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; SHOW STATUS LIKE 'innodb_encryption%'; -- die Timeout waiting for encryption threads } @@ -86,7 +86,7 @@ while ($cnt) } if (!$success) { - SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SELECT NAME,ENCRYPTION_SCHEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; SHOW STATUS LIKE 'innodb_encryption%'; -- die Timeout waiting for encryption threads } @@ -115,7 +115,7 @@ while ($cnt) } if (!$success) { - SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; + SELECT NAME,ENCRYPTION_SCHEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION; SHOW STATUS LIKE 'innodb_encryption%'; -- die Timeout waiting for encryption threads } From 3e12e39fb3dd0346578d50fe98215f1c6bce354c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 13 Apr 2018 22:18:36 +0200 Subject: [PATCH 11/17] skip innodb-table-online,crypt tests if no encryption plugin --- mysql-test/suite.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm index cb9ab0c375d..5ac3bd3e138 100644 --- a/mysql-test/suite.pm +++ b/mysql-test/suite.pm @@ -20,6 +20,9 @@ sub skip_combinations { my %skip = ( 'include/have_innodb.combinations' => [ @combinations ], 'include/have_xtradb.combinations' => [ @combinations ]); + $skip{'include/innodb_encrypt_log.combinations'} = [ 'crypt' ] + unless $ENV{DEBUG_KEY_MANAGEMENT_SO}; + # don't run tests for the wrong platform $skip{'include/platform.combinations'} = [ (IS_WINDOWS) ? 'unix' : 'win' ]; From 3d1ad2a5e85284b5ec4bb28f744b93b0b73e3934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 16 Apr 2018 15:06:41 +0300 Subject: [PATCH 12/17] MDEV-13516: encryption.create_or_replace test fails in buildbot with InnoDB assertion failure fil_crypt_rotate_pages If tablespace is marked as stopping stop also page rotation fil_crypt_flush_space If tablespace is marked as stopping do not try to read page 0 and write it back. --- storage/innobase/fil/fil0crypt.cc | 24 ++++++++++++++++-------- storage/xtradb/fil/fil0crypt.cc | 24 ++++++++++++++++-------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index cb499121c86..2d149f07433 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2019,6 +2019,12 @@ fil_crypt_rotate_pages( continue; } + /* If space is marked as stopping, stop rotating + pages. */ + if (state->space->is_stopping()) { + break; + } + fil_crypt_rotate_page(key_state, state); } } @@ -2067,20 +2073,22 @@ fil_crypt_flush_space( crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; } - /* update page 0 */ - mtr_t mtr; - mtr_start(&mtr); + if (!space->is_stopping()) { + /* update page 0 */ + mtr_t mtr; + mtr_start(&mtr); - const uint zip_size = fsp_flags_get_zip_size(state->space->flags); + const uint zip_size = fsp_flags_get_zip_size(state->space->flags); - buf_block_t* block = buf_page_get_gen(space->id, zip_size, 0, + buf_block_t* block = buf_page_get_gen(space->id, zip_size, 0, RW_X_LATCH, NULL, BUF_GET, __FILE__, __LINE__, &mtr); - byte* frame = buf_block_get_frame(block); + byte* frame = buf_block_get_frame(block); - crypt_data->write_page0(frame, &mtr); + crypt_data->write_page0(frame, &mtr); - mtr_commit(&mtr); + mtr_commit(&mtr); + } } /*********************************************************************** diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index cb499121c86..2d149f07433 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -2019,6 +2019,12 @@ fil_crypt_rotate_pages( continue; } + /* If space is marked as stopping, stop rotating + pages. */ + if (state->space->is_stopping()) { + break; + } + fil_crypt_rotate_page(key_state, state); } } @@ -2067,20 +2073,22 @@ fil_crypt_flush_space( crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; } - /* update page 0 */ - mtr_t mtr; - mtr_start(&mtr); + if (!space->is_stopping()) { + /* update page 0 */ + mtr_t mtr; + mtr_start(&mtr); - const uint zip_size = fsp_flags_get_zip_size(state->space->flags); + const uint zip_size = fsp_flags_get_zip_size(state->space->flags); - buf_block_t* block = buf_page_get_gen(space->id, zip_size, 0, + buf_block_t* block = buf_page_get_gen(space->id, zip_size, 0, RW_X_LATCH, NULL, BUF_GET, __FILE__, __LINE__, &mtr); - byte* frame = buf_block_get_frame(block); + byte* frame = buf_block_get_frame(block); - crypt_data->write_page0(frame, &mtr); + crypt_data->write_page0(frame, &mtr); - mtr_commit(&mtr); + mtr_commit(&mtr); + } } /*********************************************************************** From efae12680e7a245a6bdc919a67c2ec17ed6ddfda Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Sun, 8 Apr 2018 22:25:37 +0530 Subject: [PATCH 13/17] MDEV-15611 Due to the failure of foreign key detection, Galera... slave node killed himself. Problem:- If we try to delete table with foreign key and table whom it is referring with wsrep_slave_threads>1 then galera tries to execute both Delete_rows_log-event in parallel, which should not happen. Solution:- This is happening because we do not have foreign key info in write set. Upto version 10.2.7 it used to work fine. Actually it happening because of issue in commit 2f342c4. wsrep_must_process_fk has changed to make it similar to original condition. --- .../suite/galera/r/galera_mdev_15611.result | 16 ++++++++++ .../suite/galera/t/galera_mdev_15611.cnf | 5 ++++ .../suite/galera/t/galera_mdev_15611.test | 30 +++++++++++++++++++ storage/innobase/row/row0upd.cc | 14 ++++----- 4 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_mdev_15611.result create mode 100644 mysql-test/suite/galera/t/galera_mdev_15611.cnf create mode 100644 mysql-test/suite/galera/t/galera_mdev_15611.test diff --git a/mysql-test/suite/galera/r/galera_mdev_15611.result b/mysql-test/suite/galera/r/galera_mdev_15611.result new file mode 100644 index 00000000000..9ea1684494a --- /dev/null +++ b/mysql-test/suite/galera/r/galera_mdev_15611.result @@ -0,0 +1,16 @@ +connection node_1; +CREATE TABLE t1 ( +id int primary key +); +CREATE TABLE t2 ( +id int primary key , +f_id int DEFAULT NULL, FOREIGN KEY(f_id) REFERENCES t1 (id) +); +insert into t1 select 1; +#Running 200 insert in t2 table +select count(*) from t2; +count(*) +200 +delete from t2; +delete from t1; +drop table t2,t1; diff --git a/mysql-test/suite/galera/t/galera_mdev_15611.cnf b/mysql-test/suite/galera/t/galera_mdev_15611.cnf new file mode 100644 index 00000000000..b6f601c56b1 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_mdev_15611.cnf @@ -0,0 +1,5 @@ +!include ../galera_2nodes.cnf +[mysqld.1] + +[mysqld.2] +wsrep_slave_threads=6 diff --git a/mysql-test/suite/galera/t/galera_mdev_15611.test b/mysql-test/suite/galera/t/galera_mdev_15611.test new file mode 100644 index 00000000000..d32d7e75262 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_mdev_15611.test @@ -0,0 +1,30 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +--connection node_1 +CREATE TABLE t1 ( + id int primary key +); + +CREATE TABLE t2 ( + id int primary key , + f_id int DEFAULT NULL, FOREIGN KEY(f_id) REFERENCES t1 (id) +); + +insert into t1 select 1; + +--disable_query_log +--let $count=200 +--echo #Running 200 insert in t2 table +while($count) +{ + #Repeatedly execute the following SQL until you generate thousands of data + --eval insert into t2 values ($count, 1); + --dec $count +} +--enable_query_log + +select count(*) from t2; +delete from t2; +delete from t1; +drop table t2,t1; diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 5009ac02408..946bbae9534 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -466,17 +466,15 @@ func_exit: @param[in] node query node @param[in] trx transaction @return whether the node cannot be ignored */ -inline -bool -wsrep_must_process_fk(const upd_node_t* node, const trx_t* trx) + +inline bool wsrep_must_process_fk(const upd_node_t* node, const trx_t* trx) { - if (que_node_get_type(node->common.parent) != QUE_NODE_UPDATE - || !wsrep_on_trx(trx)) { + if (!wsrep_on_trx(trx)) { return false; } - - return static_cast(node->common.parent)->cascade_node - == node; + return que_node_get_type(node->common.parent) != QUE_NODE_UPDATE + || static_cast(node->common.parent)->cascade_node + != node; } #endif /* WITH_WSREP */ From 0c02c91bc144a79c77fc5d5f0d28184c90f34933 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Thu, 19 Apr 2018 14:11:53 +0300 Subject: [PATCH 14/17] MyRocks: MDEV-15911: Reduce debug logging on default levels in error log MyRocks internally will print non-critical messages to sql_print_verbose_info() which will do what InnoDB does in similar cases: check if (global_system_variables.log_warnings > 2). --- sql/log.cc | 12 ++++++++---- sql/log.h | 1 + storage/rocksdb/ha_rocksdb.cc | 34 +++++++++++++++++++++++----------- storage/rocksdb/ha_rocksdb.h | 4 ++++ storage/rocksdb/rdb_datadic.cc | 8 ++++---- 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index f1a0834217a..7241aa68fec 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -8747,16 +8747,20 @@ void sql_print_information(const char *format, ...) va_list args; DBUG_ENTER("sql_print_information"); - if (disable_log_notes) - DBUG_VOID_RETURN; // Skip notes during start/shutdown - va_start(args, format); - error_log_print(INFORMATION_LEVEL, format, args); + sql_print_information_v(format, args); va_end(args); DBUG_VOID_RETURN; } +void sql_print_information_v(const char *format, va_list ap) +{ + if (disable_log_notes) + return; // Skip notes during start/shutdown + + error_log_print(INFORMATION_LEVEL, format, ap); +} void TC_LOG::run_prepare_ordered(THD *thd, bool all) diff --git a/sql/log.h b/sql/log.h index c09ae14e02b..2a3912a52d3 100644 --- a/sql/log.h +++ b/sql/log.h @@ -1075,6 +1075,7 @@ int vprint_msg_to_log(enum loglevel level, const char *format, va_list args); void sql_print_error(const char *format, ...); void sql_print_warning(const char *format, ...); void sql_print_information(const char *format, ...); +void sql_print_information_v(const char *format, va_list ap); typedef void (*sql_print_message_func)(const char *format, ...); extern sql_print_message_func sql_print_message_handlers[]; diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 18d5c6ee9a2..03086db3da9 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -220,8 +220,8 @@ static int rocksdb_compact_column_family(THD *const thd, if (const char *const cf = value->val_str(value, buff, &len)) { auto cfh = cf_manager.get_cf(cf); if (cfh != nullptr && rdb != nullptr) { - sql_print_information("RocksDB: Manual compaction of column family: %s\n", - cf); + sql_print_verbose_info("RocksDB: Manual compaction of column family: %s\n", + cf); rdb->CompactRange(getCompactRangeOptions(), cfh, nullptr, nullptr); } } @@ -7776,8 +7776,8 @@ int ha_rocksdb::check(THD *const thd, HA_CHECK_OPT *const check_opt) { /* For each secondary index, check that we can get a PK value from it */ // NO_LINT_DEBUG - sql_print_information("CHECKTABLE %s: Checking table %s", table_name, - table_name); + sql_print_verbose_info("CHECKTABLE %s: Checking table %s", table_name, + table_name); ha_rows UNINIT_VAR(row_checksums_at_start); // set/used iff first_index==true ha_rows row_checksums = ha_rows(-1); bool first_index = true; @@ -7792,8 +7792,8 @@ int ha_rocksdb::check(THD *const thd, HA_CHECK_OPT *const check_opt) { row_checksums_at_start = m_row_checksums_checked; int res; // NO_LINT_DEBUG - sql_print_information("CHECKTABLE %s: Checking index %s", table_name, - table->key_info[keyno].name); + sql_print_verbose_info("CHECKTABLE %s: Checking index %s", table_name, + table->key_info[keyno].name); while (1) { if (!rows) res = index_first(table->record[0]); @@ -7881,9 +7881,9 @@ int ha_rocksdb::check(THD *const thd, HA_CHECK_OPT *const check_opt) { } } // NO_LINT_DEBUG - sql_print_information("CHECKTABLE %s: ... %lld index entries checked " - "(%lld had checksums)", - table_name, rows, checksums); + sql_print_verbose_info("CHECKTABLE %s: ... %lld index entries checked " + "(%lld had checksums)", + table_name, rows, checksums); if (first_index) { row_checksums = m_row_checksums_checked - row_checksums_at_start; @@ -7894,8 +7894,8 @@ int ha_rocksdb::check(THD *const thd, HA_CHECK_OPT *const check_opt) { } if (row_checksums != ha_rows(-1)) { // NO_LINT_DEBUG - sql_print_information("CHECKTABLE %s: %lld table records had checksums", - table_name, row_checksums); + sql_print_verbose_info("CHECKTABLE %s: %lld table records had checksums", + table_name, row_checksums); } extra(HA_EXTRA_NO_KEYREAD); @@ -12655,8 +12655,20 @@ double ha_rocksdb::read_time(uint index, uint ranges, ha_rows rows) { DBUG_RETURN((rows / 20.0) + 1); } +void sql_print_verbose_info(const char *format, ...) +{ + va_list args; + + if (global_system_variables.log_warnings > 2) { + va_start(args, format); + sql_print_information_v(format, args); + va_end(args); + } +} + } // namespace myrocks + /** Construct and emit duplicate key error message using information from table's record buffer. diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index 31adef85507..d929ca15093 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -1415,4 +1415,8 @@ private: const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_GAMMA; extern bool prevent_myrocks_loading; + +void sql_print_verbose_info(const char *format, ...); + } // namespace myrocks + diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index 68ad8e091b2..01dc2d6b176 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -4906,8 +4906,8 @@ void Rdb_dict_manager::add_create_index( rocksdb::WriteBatch *const batch) const { for (const auto &gl_index_id : gl_index_ids) { // NO_LINT_DEBUG - sql_print_information("RocksDB: Begin index creation (%u,%u)", - gl_index_id.cf_id, gl_index_id.index_id); + sql_print_verbose_info("RocksDB: Begin index creation (%u,%u)", + gl_index_id.cf_id, gl_index_id.index_id); start_create_index(batch, gl_index_id); } } @@ -4986,8 +4986,8 @@ void Rdb_dict_manager::rollback_ongoing_index_creation() const { for (const auto &gl_index_id : gl_index_ids) { // NO_LINT_DEBUG - sql_print_information("RocksDB: Removing incomplete create index (%u,%u)", - gl_index_id.cf_id, gl_index_id.index_id); + sql_print_verbose_info("RocksDB: Removing incomplete create index (%u,%u)", + gl_index_id.cf_id, gl_index_id.index_id); start_drop_index(batch, gl_index_id); } From 469a4b02ceb39ff405077414d9220f0a5ffbb360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 21 Apr 2018 20:59:26 +0300 Subject: [PATCH 15/17] Fix an intermittent test failure --- mysql-test/suite/innodb_gis/r/types.result | 1 + mysql-test/suite/innodb_gis/t/types.test | 3 +++ 2 files changed, 4 insertions(+) diff --git a/mysql-test/suite/innodb_gis/r/types.result b/mysql-test/suite/innodb_gis/r/types.result index 12e1eb44cd7..dc46710c365 100644 --- a/mysql-test/suite/innodb_gis/r/types.result +++ b/mysql-test/suite/innodb_gis/r/types.result @@ -47,6 +47,7 @@ SELECT ST_AsText(g) FROM t_wl6455 WHERE g = POINT(10,10); ST_AsText(g) INSERT INTO t_wl6455 VALUES(10, POINT(10,10)); COMMIT; +FLUSH TABLES; INSERT INTO t_wl6455 VALUES(11, POINT(11,11)); BEGIN; INSERT INTO t_wl6455 VALUES(1, POINT(1,1)); diff --git a/mysql-test/suite/innodb_gis/t/types.test b/mysql-test/suite/innodb_gis/t/types.test index fab0db5f755..0db836b94ba 100644 --- a/mysql-test/suite/innodb_gis/t/types.test +++ b/mysql-test/suite/innodb_gis/t/types.test @@ -62,6 +62,9 @@ SELECT ST_AsText(g) FROM t_wl6455 WHERE g = POINT(10,10); INSERT INTO t_wl6455 VALUES(10, POINT(10,10)); COMMIT; +# Avoid corrupting non-crash-safe system tables on the kill below. +FLUSH TABLES; + INSERT INTO t_wl6455 VALUES(11, POINT(11,11)); BEGIN; INSERT INTO t_wl6455 VALUES(1, POINT(1,1)); From 211842dd86f28f4610bc87ea221730c82c0e2494 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 23 Apr 2018 11:22:58 +0530 Subject: [PATCH 16/17] MDEV-15374 Server hangs and aborts with long semaphore wait or assertion `len < ((ulint) srv_page_size)' fails in trx_undo_rec_copy upon ROLLBACK on temporary table Problem: ======= InnoDB cleans all temporary undo logs during commit. During rollback of secondary index entry, InnoDB tries to build the previous version of clustered index. It leads to access of freed undo page during previous transaction commit and it leads to undo log corruption. Solution: ========= During rollback, temporary undo logs should not try to build the previous version of the record. --- storage/innobase/row/row0umod.cc | 15 ++++++++++----- storage/innobase/trx/trx0rec.cc | 19 ++++++------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index 76ad0815119..b30194f3562 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -428,7 +428,7 @@ row_undo_mod_del_mark_or_remove_sec_low( btr_pcur_t pcur; btr_cur_t* btr_cur; ibool success; - ibool old_has; + ibool old_has = FALSE; dberr_t err = DB_SUCCESS; mtr_t mtr; mtr_t mtr_vers; @@ -504,10 +504,15 @@ row_undo_mod_del_mark_or_remove_sec_low( &mtr_vers); ut_a(success); - old_has = row_vers_old_has_index_entry(FALSE, - btr_pcur_get_rec(&(node->pcur)), - &mtr_vers, index, entry, - 0, 0); + /* For temporary table, we can skip to check older version of + clustered index entry. Because the purge won't process + any no-redo rollback segment undo logs. */ + if (!dict_table_is_temporary(node->table)) { + old_has = row_vers_old_has_index_entry( + FALSE, btr_pcur_get_rec(&(node->pcur)), + &mtr_vers, index, entry, 0, 0); + } + if (old_has) { err = btr_cur_del_mark_set_sec_rec(BTR_NO_LOCKING_FLAG, btr_cur, TRUE, thr, &mtr); diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index 358bd458912..1851f30339b 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -2087,13 +2087,11 @@ err_exit: /** Copy an undo record to heap. @param[in] roll_ptr roll pointer to a record that exists -@param[in] is_temp whether this is a temporary table @param[in,out] heap memory heap where copied */ static trx_undo_rec_t* trx_undo_get_undo_rec_low( roll_ptr_t roll_ptr, - bool is_temp, mem_heap_t* heap) { trx_undo_rec_t* undo_rec; @@ -2109,10 +2107,7 @@ trx_undo_get_undo_rec_low( &offset); ut_ad(page_no > FSP_FIRST_INODE_PAGE_NO); ut_ad(offset >= TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_HDR_SIZE); - rseg = is_temp - ? trx_sys->temp_rsegs[rseg_id] - : trx_sys->rseg_array[rseg_id]; - ut_ad(is_temp == !rseg->is_persistent()); + rseg = trx_sys->rseg_array[rseg_id]; mtr_start(&mtr); @@ -2128,7 +2123,6 @@ trx_undo_get_undo_rec_low( /** Copy an undo record to heap. @param[in] roll_ptr roll pointer to record -@param[in] is_temp whether this is a temporary table @param[in,out] heap memory heap where copied @param[in] trx_id id of the trx that generated the roll pointer: it points to an @@ -2143,7 +2137,6 @@ static MY_ATTRIBUTE((warn_unused_result)) bool trx_undo_get_undo_rec( roll_ptr_t roll_ptr, - bool is_temp, mem_heap_t* heap, trx_id_t trx_id, const table_name_t& name, @@ -2155,7 +2148,7 @@ trx_undo_get_undo_rec( missing_history = purge_sys->view.changes_visible(trx_id, name); if (!missing_history) { - *undo_rec = trx_undo_get_undo_rec_low(roll_ptr, is_temp, heap); + *undo_rec = trx_undo_get_undo_rec_low(roll_ptr, heap); } rw_lock_s_unlock(&purge_sys->latch); @@ -2236,19 +2229,19 @@ trx_undo_prev_version_build( return(true); } - const bool is_temp = dict_table_is_temporary(index->table); + ut_ad(!dict_table_is_temporary(index->table)); + rec_trx_id = row_get_rec_trx_id(rec, index, offsets); ut_ad(!index->table->skip_alter_undo); if (trx_undo_get_undo_rec( - roll_ptr, is_temp, heap, rec_trx_id, index->table->name, + roll_ptr, heap, rec_trx_id, index->table->name, &undo_rec)) { if (v_status & TRX_UNDO_PREV_IN_PURGE) { /* We are fetching the record being purged */ - ut_ad(!is_temp); undo_rec = trx_undo_get_undo_rec_low( - roll_ptr, is_temp, heap); + roll_ptr, heap); } else { /* The undo record may already have been purged, during purge or semi-consistent read. */ From 619dc2b24f26aea29345dc3f3289bed406738025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 23 Apr 2018 09:29:32 +0300 Subject: [PATCH 17/17] Fix test results after merge from 10.1 --- mysql-test/suite/galera/r/galera_ist_rsync.result | 2 ++ .../galera/r/galera_toi_ddl_fk_insert.result | 10 ++++++++++ mysql-test/suite/galera/r/mysql-wsrep#33.result | 15 +++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_ist_rsync.result b/mysql-test/suite/galera/r/galera_ist_rsync.result index e4eb9d821fa..9c0d78d96e9 100644 --- a/mysql-test/suite/galera/r/galera_ist_rsync.result +++ b/mysql-test/suite/galera/r/galera_ist_rsync.result @@ -1,3 +1,5 @@ +connection node_1; +connection node_2; Performing State Transfer on a server that has been temporarily disconnected connection node_1; CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; diff --git a/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result b/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result index 1726bb6445f..0dbc89978d4 100644 --- a/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result +++ b/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result @@ -10,16 +10,26 @@ id INT PRIMARY KEY AUTO_INCREMENT, parent_id INT ) ENGINE=InnoDB; INSERT INTO parent VALUES (1, 0); +connection node_2; INSERT INTO child (parent_id) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;; +connection node_1a; INSERT INTO parent (f2) SELECT 1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;; +connection node_2a; INSERT INTO parent (f2) SELECT 2 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;; +connection node_1b; ALTER TABLE child ADD FOREIGN KEY (parent_id) REFERENCES parent(id);; +connection node_1a; +connection node_1b; +connection node_2; +connection node_2a; +connection node_1; SELECT COUNT(*) = 20001 FROM parent; COUNT(*) = 20001 1 SELECT COUNT(*) = 10000 FROM child; COUNT(*) = 10000 1 +connection node_2; SELECT COUNT(*) = 20001 FROM parent; COUNT(*) = 20001 1 diff --git a/mysql-test/suite/galera/r/mysql-wsrep#33.result b/mysql-test/suite/galera/r/mysql-wsrep#33.result index fc3db249c25..6a5251204b9 100644 --- a/mysql-test/suite/galera/r/mysql-wsrep#33.result +++ b/mysql-test/suite/galera/r/mysql-wsrep#33.result @@ -1,11 +1,16 @@ +connection node_1; +connection node_2; Setting SST method to mysqldump ... call mtr.add_suppression("WSREP: wsrep_sst_method is set to 'mysqldump' yet mysqld bind_address is set to '127.0.0.1'"); call mtr.add_suppression("Failed to load slave replication state from table mysql.gtid_slave_pos"); +connection node_1; CREATE USER 'sst'; GRANT ALL PRIVILEGES ON *.* TO 'sst'; SET GLOBAL wsrep_sst_auth = 'sst:'; +connection node_2; SET GLOBAL wsrep_sst_method = 'mysqldump'; Performing State Transfer on a server that has been temporarily disconnected +connection node_1; CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -15,6 +20,7 @@ INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); INSERT INTO t1 VALUES ('node1_committed_before'); COMMIT; +connection node_2; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node2_committed_before'); @@ -25,6 +31,7 @@ INSERT INTO t1 VALUES ('node2_committed_before'); COMMIT; Unloading wsrep provider ... SET GLOBAL wsrep_provider = 'none'; +connection node_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_committed_during'); @@ -39,6 +46,7 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); +connect node_1a_galera_st_disconnect_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1; SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -46,6 +54,7 @@ INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); +connection node_2; Loading wsrep provider ... SET AUTOCOMMIT=OFF; START TRANSACTION; @@ -55,6 +64,7 @@ INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); INSERT INTO t1 VALUES ('node2_committed_after'); COMMIT; +connection node_1; INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); INSERT INTO t1 VALUES ('node1_to_be_committed_after'); @@ -69,6 +79,7 @@ INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); INSERT INTO t1 VALUES ('node1_committed_after'); COMMIT; +connection node_1a_galera_st_disconnect_slave; INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after'); @@ -83,6 +94,7 @@ COUNT(*) = 0 1 COMMIT; SET AUTOCOMMIT=ON; +connection node_1; SELECT COUNT(*) = 35 FROM t1; COUNT(*) = 35 1 @@ -92,12 +104,15 @@ COUNT(*) = 0 DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +connection node_1; CALL mtr.add_suppression("Slave SQL: Error 'The MySQL server is running with the --skip-grant-tables option so it cannot execute this statement' on query"); DROP USER sst; +connection node_2; CALL mtr.add_suppression("Slave SQL: Error 'The MySQL server is running with the --skip-grant-tables option so it cannot execute this statement' on query"); CALL mtr.add_suppression("InnoDB: Error: Table \"mysql\"\\.\"innodb_index_stats\" not found"); CALL mtr.add_suppression("Can't open and lock time zone table"); CALL mtr.add_suppression("Can't open and lock privilege tables"); CALL mtr.add_suppression("Info table is not ready to be used"); CALL mtr.add_suppression("Native table .* has the wrong structure"); +connection node_2; Restarting server ...