From de13fccfc6b16915190cd740f7fe1e3445dff0e7 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Mon, 11 May 2020 12:50:03 -0400 Subject: [PATCH 01/13] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5048bc29198..bccc9d350b1 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=5 MYSQL_VERSION_MINOR=5 -MYSQL_VERSION_PATCH=68 +MYSQL_VERSION_PATCH=69 MYSQL_VERSION_EXTRA= From 76f4ae8295f63e4f3f7b8b72c2120d5da59b302c Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Tue, 26 May 2020 01:50:46 +0530 Subject: [PATCH 02/13] MDEV-21495: Conditional jump or move depends on uninitialised value in sel_arg_range_seq_next Initialize the parameter PARAM::max_key_part when we iterate over the ranges to get estimates from EITS. --- mysql-test/r/selectivity.result | 15 +++++++++++++++ mysql-test/r/selectivity_innodb.result | 15 +++++++++++++++ mysql-test/t/selectivity.test | 12 ++++++++++++ sql/opt_range_mrr.cc | 1 + 4 files changed, 43 insertions(+) diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index dc7e9494946..9bb197ad056 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -1866,4 +1866,19 @@ id select_type table type possible_keys key key_len ref rows Extra set optimizer_switch= @save_optimizer_switch; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; +# +# MDEV-21495: Conditional jump or move depends on uninitialised value in sel_arg_range_seq_next +# +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 SELECT seq, seq from seq_1_to_100; +set optimizer_use_condition_selectivity=4; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * from t1 WHERE a = 5 and b = 5; +a b +5 5 +set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +drop table t1; # End of 10.1 tests diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index f0cbf1662aa..d0e31374784 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -1876,6 +1876,21 @@ id select_type table type possible_keys key key_len ref rows Extra set optimizer_switch= @save_optimizer_switch; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; +# +# MDEV-21495: Conditional jump or move depends on uninitialised value in sel_arg_range_seq_next +# +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 SELECT seq, seq from seq_1_to_100; +set optimizer_use_condition_selectivity=4; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * from t1 WHERE a = 5 and b = 5; +a b +5 5 +set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +drop table t1; # End of 10.1 tests set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index 7df326edb44..1d0256b1b7e 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -1266,5 +1266,17 @@ set optimizer_switch= @save_optimizer_switch; set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; drop table t1,t2; +--echo # +--echo # MDEV-21495: Conditional jump or move depends on uninitialised value in sel_arg_range_seq_next +--echo # + +CREATE TABLE t1(a INT, b INT); +INSERT INTO t1 SELECT seq, seq from seq_1_to_100; +set optimizer_use_condition_selectivity=4; +ANALYZE TABLE t1 PERSISTENT FOR ALL; +SELECT * from t1 WHERE a = 5 and b = 5; +set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +drop table t1; + --echo # End of 10.1 tests diff --git a/sql/opt_range_mrr.cc b/sql/opt_range_mrr.cc index 5d62ceda7d4..b1894f4b6c4 100644 --- a/sql/opt_range_mrr.cc +++ b/sql/opt_range_mrr.cc @@ -73,6 +73,7 @@ range_seq_t sel_arg_range_seq_init(void *init_param, uint n_ranges, uint flags) { SEL_ARG_RANGE_SEQ *seq= (SEL_ARG_RANGE_SEQ*)init_param; seq->at_start= TRUE; + seq->param->max_key_part= 0; seq->stack[0].key_tree= NULL; seq->stack[0].min_key= seq->param->min_key; seq->stack[0].min_key_flag= 0; From fbcfbb0e1ce16db6c7debe77119de1141a8c6391 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Tue, 26 May 2020 11:43:43 +0300 Subject: [PATCH 03/13] MDEV-19751 Wrong partitioning by KEY() after primary key dropped Default (empty) field list in partitioning by KEY() clause is assigned from primary key. If primary key is changed the partitioning field list is changed as well, so repartitioning required. Not applicable to any non-primary keys as default field list may be taken only from primary key. --- mysql-test/r/partition_alter.result | 43 +++++++++++++++++++++++++++++ mysql-test/t/partition_alter.test | 39 ++++++++++++++++++++++++++ sql/sql_partition.cc | 22 +++++++++++++++ 3 files changed, 104 insertions(+) diff --git a/mysql-test/r/partition_alter.result b/mysql-test/r/partition_alter.result index 1dced95b17a..8de3d4b45ad 100644 --- a/mysql-test/r/partition_alter.result +++ b/mysql-test/r/partition_alter.result @@ -122,3 +122,46 @@ t1 CREATE TABLE `t1` ( PARTITION `p02` ENGINE = MyISAM, PARTITION `p03` ENGINE = MyISAM) drop table t1; +# +# MDEV-19751 Wrong partitioning by KEY() after key dropped +# +create or replace table t1 (pk int, x timestamp(6), primary key (pk, x)) engine innodb +partition by key() partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +# Inplace for DROP PRIMARY KEY when partitioned by default field list is denied +alter table t1 drop primary key, drop column x, add primary key (pk), algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY +alter table t1 drop primary key, drop column x, add primary key (pk); +select * from t1 partition (p0); +pk +1 +drop table t1; +create or replace table t1 (pk int not null, x timestamp(6), unique u(pk, x)) engine innodb +partition by key() partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +# Same for NOT NULL UNIQUE KEY as this is actually primary key +alter table t1 drop key u, drop column x, add unique (pk), algorithm=inplace; +ERROR 0A000: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY +alter table t1 drop key u, drop column x, add unique (pk); +select * from t1 partition (p0); +pk +1 +drop table t1; +create or replace table t1 (pk int, x timestamp(6), primary key (pk)) engine innodb +partition by key(pk) partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +# Inplace for DROP PRIMARY KEY when partitioned by explicit field list is allowed +alter table t1 drop primary key, add primary key (pk, x), algorithm=inplace; +select * from t1 partition (p0); +pk x +1 2000-01-01 00:00:00.000000 +drop table t1; +create or replace table t1 (k int, x timestamp(6), unique key u (x, k)) engine innodb +partition by key(k) partitions 2; +insert into t1 (k, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +# Inplace for DROP KEY is allowed +alter table t1 drop key u, algorithm=inplace; +select * from t1 partition (p0); +k x +1 2000-01-01 00:00:00.000000 +drop table t1; diff --git a/mysql-test/t/partition_alter.test b/mysql-test/t/partition_alter.test index 1235e5b8dc2..3c783ebe375 100644 --- a/mysql-test/t/partition_alter.test +++ b/mysql-test/t/partition_alter.test @@ -112,3 +112,42 @@ insert into t1 values(0, 1, 1, NULL, now(), now()); alter online table t1 delay_key_write=1; show create table t1; drop table t1; + +--echo # +--echo # MDEV-19751 Wrong partitioning by KEY() after key dropped +--echo # +create or replace table t1 (pk int, x timestamp(6), primary key (pk, x)) engine innodb +partition by key() partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +--echo # Inplace for DROP PRIMARY KEY when partitioned by default field list is denied +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t1 drop primary key, drop column x, add primary key (pk), algorithm=inplace; +alter table t1 drop primary key, drop column x, add primary key (pk); +select * from t1 partition (p0); +drop table t1; + +create or replace table t1 (pk int not null, x timestamp(6), unique u(pk, x)) engine innodb +partition by key() partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +--echo # Same for NOT NULL UNIQUE KEY as this is actually primary key +--error ER_ALTER_OPERATION_NOT_SUPPORTED +alter table t1 drop key u, drop column x, add unique (pk), algorithm=inplace; +alter table t1 drop key u, drop column x, add unique (pk); +select * from t1 partition (p0); +drop table t1; + +create or replace table t1 (pk int, x timestamp(6), primary key (pk)) engine innodb +partition by key(pk) partitions 2; +insert into t1 (pk, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +--echo # Inplace for DROP PRIMARY KEY when partitioned by explicit field list is allowed +alter table t1 drop primary key, add primary key (pk, x), algorithm=inplace; +select * from t1 partition (p0); +drop table t1; + +create or replace table t1 (k int, x timestamp(6), unique key u (x, k)) engine innodb +partition by key(k) partitions 2; +insert into t1 (k, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); +--echo # Inplace for DROP KEY is allowed +alter table t1 drop key u, algorithm=inplace; +select * from t1 partition (p0); +drop table t1; diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index 071c9c05129..2334286b039 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -5473,6 +5473,28 @@ the generated partition syntax in a correct manner. *partition_changed= TRUE; } } + /* + Prohibit inplace when partitioned by primary key and the primary key is dropped. + */ + if (!*partition_changed && + tab_part_info->part_field_array && + !tab_part_info->part_field_list.elements && + table->s->primary_key != MAX_KEY) + { + KEY *primary_key= table->key_info + table->s->primary_key; + List_iterator_fast drop_it(alter_info->drop_list); + const char *primary_name= primary_key->name; + const Alter_drop *drop; + drop_it.rewind(); + while ((drop= drop_it++)) + { + if (drop->type == Alter_drop::KEY && + 0 == my_strcasecmp(system_charset_info, primary_name, drop->name)) + break; + } + if (drop) + *partition_changed= TRUE; + } } if (thd->work_part_info) { From 9bbd685e8d325c5868a93083addc7a930c1c90de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 26 May 2020 12:23:20 +0300 Subject: [PATCH 04/13] MDEV-22513 main.processlist_notembedded Timeout in wait_until_count_sessions Do not blindly disconnect the connection that is in WAIT_FOR because it could happen that neither the disconnect nor the SIGNAL would be processed before RESET would discard the signal. --- mysql-test/r/processlist_notembedded.result | 1 + mysql-test/t/processlist_notembedded.test | 3 +++ 2 files changed, 4 insertions(+) diff --git a/mysql-test/r/processlist_notembedded.result b/mysql-test/r/processlist_notembedded.result index ab1ced79e25..ed8509ca13c 100644 --- a/mysql-test/r/processlist_notembedded.result +++ b/mysql-test/r/processlist_notembedded.result @@ -4,5 +4,6 @@ SET DEBUG_SYNC= 'before_join_optimize SIGNAL in_sync WAIT_FOR go'; SET DEBUG_SYNC= 'now WAIT_FOR in_sync'; SET DEBUG_SYNC= 'now SIGNAL go'; +user SET DEBUG_SYNC = 'RESET'; End of 5.5 tests diff --git a/mysql-test/t/processlist_notembedded.test b/mysql-test/t/processlist_notembedded.test index b07ac00a44a..90750b40e62 100644 --- a/mysql-test/t/processlist_notembedded.test +++ b/mysql-test/t/processlist_notembedded.test @@ -30,7 +30,10 @@ remove_file $MYSQLTEST_VARDIR/tmp//MDEV-20466.text; SET DEBUG_SYNC= 'now SIGNAL go'; +connection con1; +reap; disconnect con1; +connection default; SET DEBUG_SYNC = 'RESET'; From f1f14c209205b3a7dc375abf0cad19107221d499 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Tue, 26 May 2020 13:14:47 +0300 Subject: [PATCH 05/13] MDEV-20015 Assertion `!in_use->is_error()' failed in TABLE::update_virtual_field update_virtual_field() is called as part of index rebuild in ha_myisam::repair() (MDEV-5800) which is done on bulk INSERT finish. Assertion in update_virtual_field() was put as part of MDEV-16222 because update_virtual_field() returns in_use->is_error(). The idea: wrongly mixed semantics of error status before update_virtual_field() and the status returned by update_virtual_field(). The former can falsely influence the latter. --- mysql-test/suite/vcol/r/vcol_keys_myisam.result | 9 +++++++++ mysql-test/suite/vcol/t/vcol_keys_myisam.test | 11 +++++++++++ sql/table.cc | 8 +++++--- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/vcol/r/vcol_keys_myisam.result b/mysql-test/suite/vcol/r/vcol_keys_myisam.result index b7086600ab1..dd6e4c414f0 100644 --- a/mysql-test/suite/vcol/r/vcol_keys_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_keys_myisam.result @@ -408,3 +408,12 @@ SELECT * FROM t1 WHERE d2 < d1; i d1 d2 t 1 2023-03-16 2023-03-15 1 DROP TABLE t1; +# +# MDEV-20015 Assertion `!in_use->is_error()' failed in TABLE::update_virtual_field +# +create or replace table t1 (a int); +insert into t1 (a) values (1), (1); +create or replace table t2 (pk int, b int, c int as (b) virtual, primary key (pk), key(c)); +insert into t2 (pk) select a from t1; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +drop tables t1, t2; diff --git a/mysql-test/suite/vcol/t/vcol_keys_myisam.test b/mysql-test/suite/vcol/t/vcol_keys_myisam.test index 99b1c9a444b..24612f4d55f 100644 --- a/mysql-test/suite/vcol/t/vcol_keys_myisam.test +++ b/mysql-test/suite/vcol/t/vcol_keys_myisam.test @@ -300,3 +300,14 @@ DROP TABLE t1; # Cleanup --let $datadir= `SELECT @@datadir` --remove_file $datadir/test/load_t1 + + +--echo # +--echo # MDEV-20015 Assertion `!in_use->is_error()' failed in TABLE::update_virtual_field +--echo # +create or replace table t1 (a int); +insert into t1 (a) values (1), (1); +create or replace table t2 (pk int, b int, c int as (b) virtual, primary key (pk), key(c)); +--error ER_DUP_ENTRY +insert into t2 (pk) select a from t1; +drop tables t1, t2; diff --git a/sql/table.cc b/sql/table.cc index 3bb321d3306..d2a45032dbb 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -7778,15 +7778,17 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) int TABLE::update_virtual_field(Field *vf) { - DBUG_ASSERT(!in_use->is_error()); - Query_arena backup_arena; DBUG_ENTER("TABLE::update_virtual_field"); + Query_arena backup_arena; + Counting_error_handler count_errors; + in_use->push_internal_handler(&count_errors); in_use->set_n_backup_active_arena(expr_arena, &backup_arena); bitmap_clear_all(&tmp_set); vf->vcol_info->expr->walk(&Item::update_vcol_processor, 0, &tmp_set); vf->vcol_info->expr->save_in_field(vf, 0); in_use->restore_active_arena(expr_arena, &backup_arena); - DBUG_RETURN(in_use->is_error()); + in_use->pop_internal_handler(); + DBUG_RETURN(count_errors.errors); } From dbe447a78908214614db53061dccbc6bde52764e Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Tue, 5 May 2020 20:32:32 +0300 Subject: [PATCH 06/13] MDEV-15152 Optimistic parallel slave doesnt cope well with START SLAVE UNTIL The immediate bug was caused by a failure to recognize a correct position to stop the slave applier run in optimistic parallel mode. There were the following set of issues that the analysis unveil. 1 incorrect estimate for the event binlog position passed to is_until_satisfied 2 wait for workers to complete by the driver thread did not account non-group events that could be left unprocessed and thus to mix up the last executed binlog group's file and position: the file remained old and the position related to the new rotated file 3 incorrect 'slave reached file:pos' by the parallel slave report in the error log 4 relay log UNTIL missed out the parallel slave branch in is_until_satisfied. The patch addresses all of them to simplify logics of log change notification in either the master and relay-log until case. P.1 is addressed with passing the event into is_until_satisfied() for proper analisis by the function. P.2 is fixed by changes in handle_queued_pos_update(). P.4 required removing relay-log change notification by workers. Instead the driver thread updates the notion of the current relay-log fully itself with aid of introduced bool Relay_log_info::until_relay_log_names_defer. An extra print out of the requested until file:pos is arranged with --log-warning=3. --- .../r/rpl_parallel_optimistic_until.result | 238 +++++++++ .../rpl/t/rpl_parallel_optimistic_until.test | 466 ++++++++++++++++++ sql/rpl_parallel.cc | 6 +- sql/rpl_rli.cc | 65 ++- sql/rpl_rli.h | 19 +- sql/slave.cc | 31 +- 6 files changed, 806 insertions(+), 19 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result create mode 100644 mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test diff --git a/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result b/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result new file mode 100644 index 00000000000..d3d1a52d5db --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result @@ -0,0 +1,238 @@ +include/master-slave.inc +[connection master] +include/stop_slave.inc +RESET MASTER; +RESET SLAVE; +RESET MASTER; +CREATE TABLE t1 (a int primary key, b text) ENGINE=InnoDB; +INSERT INTO t1 SET a=25, b='trx0'; +include/start_slave.inc +include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=2; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; +SET @old_max_relay_log_size = @@global.max_relay_log_size; +SET @@global.max_relay_log_size=4096; +BEGIN; +INSERT INTO t1 SET a=1, b='trx1'; +INSERT INTO t1 SET a=2, b='trx1'; +INSERT INTO t1 SET a=3, b='trx1'; +INSERT INTO t1 SET a=4, b='trx1'; +INSERT INTO t1 SET a=5, b='trx1'; +INSERT INTO t1 SET a=6, b='trx1'; +INSERT INTO t1 SET a=7, b='trx1'; +INSERT INTO t1 SET a=8, b='trx1'; +INSERT INTO t1 SET a=9, b='trx1'; +INSERT INTO t1 SET a=10, b='trx1'; +INSERT INTO t1 SET a=11, b='trx1'; +INSERT INTO t1 SET a=12, b='trx1'; +INSERT INTO t1 SET a=13, b='trx1'; +INSERT INTO t1 SET a=14, b='trx1'; +INSERT INTO t1 SET a=15, b='trx1'; +INSERT INTO t1 SET a=16, b='trx1'; +INSERT INTO t1 SET a=17, b='trx1'; +INSERT INTO t1 SET a=18, b='trx1'; +INSERT INTO t1 SET a=19, b='trx1'; +INSERT INTO t1 SET a=20, b='trx1'; +INSERT INTO t1 SET a=21, b='trx1'; +INSERT INTO t1 SET a=22, b='trx1'; +INSERT INTO t1 SET a=23, b='trx1'; +INSERT INTO t1 SET a=24, b='trx1'; +COMMIT; +FLUSH LOGS; +BEGIN; +UPDATE t1 SET b='trx2_0' WHERE a = 25; +UPDATE t1 SET b='trx2' WHERE a = 25; +COMMIT; +INSERT INTO t1 SET a=26,b='trx3'; +*** case 1 UNTIL inside trx2 +BEGIN; +INSERT INTO t1 SET a= 1; +SELECT <= AND < as "pos_until < trx0 and is within trx2"; +pos_until < trx0 and is within trx2 +1 +CHANGE MASTER TO MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +ROLLBACK; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2'; +trx2 is committed +1 +SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3'; +trx3 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +*** case 2 UNTIL inside trx2 +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +BEGIN; +INSERT INTO t1 SET a= 1; +include/stop_slave.inc +SELECT <= AND < as "pos_until >= trx0 and is within trx2"; +pos_until >= trx0 and is within trx2 +1 +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +ROLLBACK; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2'; +trx2 is committed +1 +SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3'; +trx3 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +*** case 3 UNTIL inside trx1 +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +BEGIN; +INSERT INTO t1 SET a= 1; # block trx1; +include/stop_slave.inc +SELECT < as "pos_until before trx2 start position"; +pos_until before trx2 start position +1 +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +ROLLBACK; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 25-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1'; +trx1 is committed +1 +SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2'; +trx2 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +*** case 4 Relay-log UNTIL inside trx1 +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +BEGIN; +INSERT INTO t1 SET a= 1; # block trx1; +include/stop_slave.inc +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +ROLLBACK; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 25-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1'; +trx1 is committed +1 +SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2'; +trx2 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +*** case 5 Relay-log UNTIL inside a "big" trx that spawns few relay logs +CREATE TABLE t2 (a TEXT) ENGINE=InnoDB; +FLUSH LOGS; +include/stop_slave.inc +BEGIN; +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +COMMIT; +INSERT INTO t2 SET a='a'; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +Proof 2: Resume works out +include/start_slave.inc +include/diff_tables.inc [master:t2,slave:t2] +*** case 6 Relay-log UNTIL inside a small trx inside a sequence of relay logs +include/stop_slave.inc +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +COMMIT; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +include/sync_slave_io_with_master.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +Proof 1: Correct stop +include/wait_for_slave_sql_to_stop.inc +Proof 2: Resume works out +include/start_slave.inc +include/diff_tables.inc [master:t2,slave:t2] +include/stop_slave.inc +SET GLOBAL max_relay_log_size=@old_max_relay_log_size; +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +include/start_slave.inc +DROP TABLE t1, t2; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test b/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test new file mode 100644 index 00000000000..9e42fae8464 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test @@ -0,0 +1,466 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/master-slave.inc +# Format is restricted because the test expects a specific result of +# relay-logging that splits a transaction into two different files. +--source include/have_binlog_format_row.inc + +# +# MDEV-15152 Optimistic parallel slave doesn't cope well with START SLAVE UNTIL +# +--connection slave +--source include/stop_slave.inc +RESET MASTER; +RESET SLAVE; + +--connection master +RESET MASTER; +CREATE TABLE t1 (a int primary key, b text) ENGINE=InnoDB; +--let $a0 = 25 +--eval INSERT INTO t1 SET a=$a0, b='trx0' +# Memorize the position for replication restart from it +--let $pos_trx0 = query_get_value(SHOW MASTER STATUS, Position, 1) + +--connection slave +--source include/start_slave.inc + +--connection master +# --connection slave +--sync_slave_with_master +--source include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=2; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; + +# Run the slave in the following modes one by one. +# +# 1. the until position is set in the middle of trx2 +# below $pos_trx0 of the last exec position in the first file +# 2. and above $pos_trx0 +# In either case trx2 must commit before slave stops. +# 3. the until postion is inside trx1 +# 4. RELAY log until inside trx1 +# 5. RELAY log until inside a "big" trx +# 6. RELAY log until inside a trx within a sequence of relay logs +# +# Execution flaw for Until_Master_Pos cases follows as: +# create the transaction trx1, trx2 +# logged at the beginning of two subsequent binlog files. +# Set the until position to at the middle of the 2rd transaction. +# Engage the optimistic scheduler while having trx1 execution blocked. +# Lift the block after trx2 has reached waiting its order to commit. +# *Proof 1* +# Observe that the slave applier stops at a correct position. +# In the bug condition it would stop prematurely having the stop position +# in the first file, therefore trx2 not committed. +# Specifically, an internal transaction position until makes the applier to run +# beyond it to commit commit the current transaction. +# *Proof 2* +# Observe the following START SLAVE resumes OK. +# +# Auxiliary third trx3 on master is just for triggering the actual stop +# (whihc is a legacy UNTIL's property). +# trx0 is to produce a specific value of the last executed binlog file:pos +# to emulate the bug condition. +# +# Intermediate checks via SELECT are supposed to succeed +# with putting out value 1. +# +# NOTE: Relay log until tests have to use explicit log names and position +# which may require to adjust with future changes to event formats etc. +# + +--connection slave +SET @old_max_relay_log_size = @@global.max_relay_log_size; +SET @@global.max_relay_log_size=4096; + +--connection master +# trx1 +--let $a=1 +BEGIN; +while (`SELECT $a < $a0`) +{ + --eval INSERT INTO t1 SET a=$a, b='trx1' +--inc $a +} +COMMIT; +--let $fil_1 = query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx1 = query_get_value(SHOW MASTER STATUS, Position, 1) + +FLUSH LOGS; + +# $pos_0 the offset of the first event of trx2 in new file +--let $pos_0=query_get_value(SHOW MASTER STATUS, Position, 1) +# trx2 +--let $a=$a0 +BEGIN; +--eval UPDATE t1 SET b='trx2_0' WHERE a = $a +--eval UPDATE t1 SET b='trx2' WHERE a = $a +COMMIT; +--let $fil_2=query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx2=query_get_value(SHOW MASTER STATUS, Position, 1) + +# trx3 +--let $a=$a0 +--inc $a +--eval INSERT INTO t1 SET a=$a,b='trx3' +--let $pos_trx3=query_get_value(SHOW MASTER STATUS, Position, 1) +--let $a= + + +--echo *** case 1 UNTIL inside trx2 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1 +--connection slave +--let $pos_until=`SELECT $pos_trx0 - 1` +--replace_result $pos_0 $pos_until $pos_trx2 +--eval SELECT $pos_0 <= $pos_until AND $pos_until < $pos_trx2 as "pos_until < trx0 and is within trx2" +CHANGE MASTER TO MASTER_USE_GTID=no; +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--let $wait_condition= SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2' +--eval SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 2 UNTIL inside trx2 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1 + +--connection slave +--source include/stop_slave.inc + +--let $pos_until=`SELECT $pos_trx2 - 1` +--replace_result $pos_trx0 $pos_until $pos_trx2 +--eval SELECT $pos_trx0 <= $pos_until AND $pos_until < $pos_trx2 as "pos_until >= trx0 and is within trx2" +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--let $wait_condition= SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2' +--eval SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 3 UNTIL inside trx1 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1; # block trx1 + +--connection slave +--source include/stop_slave.inc + +--let $pos_until=`SELECT $pos_0 - 1` +--replace_result $pos_0 $pos_until $pos_trx2 +--eval SELECT $pos_until < $pos_0 as "pos_until before trx2 start position" +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = $a0-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1' +--eval SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 4 Relay-log UNTIL inside trx1 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1; # block trx1 + +--connection slave +--source include/stop_slave.inc +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +# The following test sets the stop coordinate is set to inside the first event +# of a relay log that holds events of a transaction started in an earlier log. +# Peek the stop position in the middle of trx1 (Xid event is here) +# whose last event follow it +--let $pos_until=2568 +--let $file_rl=slave-relay-bin.000002 +--let $binlog_file=$file_rl +--let $info= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_until LIMIT 1, Info, 1) + +if (`SELECT "$info" NOT LIKE "COMMIT /* xid=% */"`) +{ + --echo *** Unexpected offset. Refine it to point to the correct XID event! + --die +} + +# Complicate the test to screw the coordinate away from event boundary. +--dec $pos_until +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +if (`SELECT strcmp("$file_rl","$file_stop") > -1`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $file_rl:$pos_until. + --die +} + +--eval SELECT count(*) = $a0-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1' +--eval SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + + +--echo *** case 5 Relay-log UNTIL inside a "big" trx that spawns few relay logs + +--connection master +CREATE TABLE t2 (a TEXT) ENGINE=InnoDB; +FLUSH LOGS; + +--sync_slave_with_master +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +--let $records=`SELECT floor(4*@@global.max_relay_log_size / 1024) + 1` + +--connection slave +--source include/stop_slave.inc + +--connection master +# trx4 +BEGIN; +--let $i=$records +while ($i) +{ + INSERT INTO t2 SET a=repeat('a',1024); + +--dec $i +} +COMMIT; + +# slave will stop there: +--let $file_trx4 = query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx4 = query_get_value(SHOW MASTER STATUS, Position, 1) + +# trx5 +INSERT INTO t2 SET a='a'; +--let $pos_trx5 = query_get_value(SHOW MASTER STATUS, Position, 1) + +--connection slave +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +# Set position inside the transaction though the value +# specified is beyond that relay log file. +# The coordianate may point to in a different event in future changes +# but should not move away from inside this big group of events. +# So we don't test which event in the transaction it points to. +--let $pos_until= 4500 +--let $file_rl= slave-relay-bin.000010 + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +# It's showed the actual stop occurred before trx5 +if (`SELECT strcmp("$file_trx4", "$file_stop") <> 0 OR $pos_stop >= $pos_trx5 OR count(*) <> $records FROM t2`) +{ + --echo *** ERROR: Slave stopped at *binlog* $file_stop:$pos_stop which is not $file_trx4:$pos_trx4. + --die +} + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + +--let $diff_tables=master:t2,slave:t2 +--source include/diff_tables.inc + + + +--echo *** case 6 Relay-log UNTIL inside a small trx inside a sequence of relay logs + +--connection slave +--source include/stop_slave.inc + +--connection master +# trx6 +--let $records=`SELECT count(*) FROM t2` +while ($records) +{ + BEGIN; + DELETE FROM t2 LIMIT 1; + COMMIT; +--dec $records +} +COMMIT; + +--connection slave +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +--connection master +--source include/sync_slave_io_with_master.inc + +--connection slave +# The relay-log coordinate is not at an event boundary and +# also may change across the server version. +# The test makes its best to check its coherance. +--let $pos_until= 4100 +--let $file_rl= slave-relay-bin.000015 + +--let $pos_gtid = 4010 +--let $info= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_gtid LIMIT 1, Info, 1) + +if (`SELECT "$info" != "BEGIN GTID 0-1-23"`) +{ + --echo *** Unexpected offset. Refine it to point to the correct GTID! + --die +} +--let $pos_event = 4090 +--let $type= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_event LIMIT 1, Event_type, 1) +if (`SELECT "$type" != "Delete_rows_v1"`) +{ + --echo *** Unexpected offset. Refine it to point to the expected event! + --die +} + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +if (`SELECT strcmp("$file_stop", "$file_rl") = -1 OR + strcmp("$file_stop", "$file_rl") = 0 AND $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at *relay* $file_stop:$pos_stop which is not $file_rl:$pos_until. + --die +} + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + +--let $diff_tables=master:t2,slave:t2 +--source include/diff_tables.inc + +# +# Clean up. +# +--connection slave +--source include/stop_slave.inc +SET GLOBAL max_relay_log_size=@old_max_relay_log_size; +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +--source include/start_slave.inc + +--connection master +DROP TABLE t1, t2; + +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 574730b96c1..fb6f23af295 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -75,18 +75,18 @@ handle_queued_pos_update(THD *thd, rpl_parallel_thread::queued_event *qev) /* Do not update position if an earlier event group caused an error abort. */ DBUG_ASSERT(qev->typ == rpl_parallel_thread::queued_event::QUEUED_POS_UPDATE); + rli= qev->rgi->rli; e= qev->entry_for_queued; - if (e->stop_on_error_sub_id < (uint64)ULONGLONG_MAX || e->force_abort) + if (e->stop_on_error_sub_id < (uint64)ULONGLONG_MAX || + (e->force_abort && !rli->stop_for_until)) return; - rli= qev->rgi->rli; mysql_mutex_lock(&rli->data_lock); cmp= strcmp(rli->group_relay_log_name, qev->event_relay_log_name); if (cmp < 0) { rli->group_relay_log_pos= qev->future_event_relay_log_pos; strmake_buf(rli->group_relay_log_name, qev->event_relay_log_name); - rli->notify_group_relay_log_name_update(); } else if (cmp == 0 && rli->group_relay_log_pos < qev->future_event_relay_log_pos) rli->group_relay_log_pos= qev->future_event_relay_log_pos; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index f3514253874..905462ecfbd 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -65,6 +65,7 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) gtid_skip_flag(GTID_SKIP_NOT), inited(0), abort_slave(0), stop_for_until(0), slave_running(MYSQL_SLAVE_NOT_RUN), until_condition(UNTIL_NONE), until_log_pos(0), retried_trans(0), executed_entries(0), + until_relay_log_names_defer(false), m_flags(0) { DBUG_ENTER("Relay_log_info::Relay_log_info"); @@ -512,6 +513,8 @@ void Relay_log_info::clear_until_condition() until_condition= Relay_log_info::UNTIL_NONE; until_log_name[0]= 0; until_log_pos= 0; + until_relay_log_names_defer= false; + DBUG_VOID_RETURN; } @@ -997,7 +1000,6 @@ void Relay_log_info::inc_group_relay_log_pos(ulonglong log_pos, { group_relay_log_pos= rgi->future_event_relay_log_pos; strmake_buf(group_relay_log_name, rgi->event_relay_log_name); - notify_group_relay_log_name_update(); } else if (cmp == 0 && group_relay_log_pos < rgi->future_event_relay_log_pos) group_relay_log_pos= rgi->future_event_relay_log_pos; @@ -1258,29 +1260,78 @@ err: autoincrement or if we have transactions). Should be called ONLY if until_condition != UNTIL_NONE ! + + In the parallel execution mode and UNTIL_MASTER_POS the file name is + presented by future_event_master_log_name which may be ahead of + group_master_log_name. Log_event::log_pos does relate to it nevertheless + so the pair comprises a correct binlog coordinate. + Internal group events and events that have zero log_pos also + produce the zero for the local log_pos which may not lead to the + function falsely return true. + In UNTIL_RELAY_POS the original caching and notification are simplified + to straightforward files comparison when the current event can't be + a part of an event group. + RETURN VALUE true - condition met or error happened (condition seems to have bad log file name) false - condition not met */ -bool Relay_log_info::is_until_satisfied(my_off_t master_beg_pos) +bool Relay_log_info::is_until_satisfied(Log_event *ev) { const char *log_name; ulonglong log_pos; + /* Prevents stopping within transaction; needed solely for Relay UNTIL. */ + bool in_trans= false; + DBUG_ENTER("Relay_log_info::is_until_satisfied"); if (until_condition == UNTIL_MASTER_POS) { log_name= (mi->using_parallel() ? future_event_master_log_name : group_master_log_name); - log_pos= master_beg_pos; + log_pos= (get_flag(Relay_log_info::IN_TRANSACTION) || !ev || !ev->log_pos) ? + (mi->using_parallel() ? 0 : group_master_log_pos) : + ev->log_pos - ev->data_written; } else { DBUG_ASSERT(until_condition == UNTIL_RELAY_POS); - log_name= group_relay_log_name; - log_pos= group_relay_log_pos; + if (!mi->using_parallel()) + { + log_name= group_relay_log_name; + log_pos= group_relay_log_pos; + } + else + { + log_name= event_relay_log_name; + log_pos= event_relay_log_pos; + in_trans= get_flag(Relay_log_info::IN_TRANSACTION); + /* + until_log_names_cmp_result is set to UNKNOWN either + - by a non-group event *and* only when it is in the middle of a group + - or by a group event when the preceding group made the above + non-group event to defer the resetting. + */ + if ((ev && !Log_event::is_group_event(ev->get_type_code()))) + { + if (in_trans) + { + until_relay_log_names_defer= true; + } + else + { + until_log_names_cmp_result= UNTIL_LOG_NAMES_CMP_UNKNOWN; + until_relay_log_names_defer= false; + } + } + else if (!in_trans && until_relay_log_names_defer) + { + until_log_names_cmp_result= UNTIL_LOG_NAMES_CMP_UNKNOWN; + until_relay_log_names_defer= false; + } + } } DBUG_PRINT("info", ("group_master_log_name='%s', group_master_log_pos=%llu", @@ -1334,8 +1385,8 @@ bool Relay_log_info::is_until_satisfied(my_off_t master_beg_pos) } DBUG_RETURN(((until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_EQUAL && - log_pos >= until_log_pos) || - until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_GREATER)); + (log_pos >= until_log_pos && !in_trans)) || + until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_GREATER)); } diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 3dec5b0f044..29e47d302bc 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -402,7 +402,7 @@ public: void close_temporary_tables(); /* Check if UNTIL condition is satisfied. See slave.cc for more. */ - bool is_until_satisfied(my_off_t); + bool is_until_satisfied(Log_event *ev); inline ulonglong until_pos() { DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || @@ -410,7 +410,13 @@ public: return ((until_condition == UNTIL_MASTER_POS) ? group_master_log_pos : group_relay_log_pos); } - + inline char *until_name() + { + DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || + until_condition == UNTIL_RELAY_POS); + return ((until_condition == UNTIL_MASTER_POS) ? group_master_log_name : + group_relay_log_name); + } /** Helper function to do after statement completion. @@ -483,6 +489,15 @@ public: } private: + /* + Hint for when to stop event distribution by sql driver thread. + The flag is set ON by a non-group event when this event is in the middle + of a group (e.g a transaction group) so it's too early + to refresh the current-relay-log vs until-log cached comparison result. + And it is checked and to decide whether it's a right time to do so + when the being processed group has been fully scheduled. + */ + bool until_relay_log_names_defer; /* Holds the state of the data in the relay log. diff --git a/sql/slave.cc b/sql/slave.cc index 0ce9c4cbbcd..06f2b0d955a 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3784,12 +3784,8 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && (ev->server_id != global_system_variables.server_id || rli->replicate_same_server_id) && - rli->is_until_satisfied((rli->get_flag(Relay_log_info::IN_TRANSACTION) || !ev->log_pos) - ? rli->group_master_log_pos - : ev->log_pos - ev->data_written)) + rli->is_until_satisfied(ev)) { - sql_print_information("Slave SQL thread stopped because it reached its" - " UNTIL position %llu", rli->until_pos()); /* Setting abort_slave flag because we do not want additional message about error in query execution to be printed. @@ -4943,10 +4939,14 @@ pthread_handler_t handle_slave_sql(void *arg) } if ((rli->until_condition == Relay_log_info::UNTIL_MASTER_POS || rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && - rli->is_until_satisfied(rli->group_master_log_pos)) + rli->is_until_satisfied(NULL)) { sql_print_information("Slave SQL thread stopped because it reached its" - " UNTIL position %llu", rli->until_pos()); + " UNTIL position %llu in %s %s file", + rli->until_pos(), rli->until_name(), + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); mysql_mutex_unlock(&rli->data_lock); goto err; } @@ -5011,7 +5011,24 @@ pthread_handler_t handle_slave_sql(void *arg) err: if (mi->using_parallel()) rli->parallel.wait_for_done(thd, rli); + /* Gtid_list_log_event::do_apply_event has already reported the GTID until */ + if (rli->stop_for_until && rli->until_condition != Relay_log_info::UNTIL_GTID) + { + if (global_system_variables.log_warnings > 2) + sql_print_information("Slave SQL thread UNTIL stop was requested at position " + "%llu in %s %s file", + rli->until_log_pos, rli->until_log_name, + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); + sql_print_information("Slave SQL thread stopped because it reached its" + " UNTIL position %llu in %s %s file", + rli->until_pos(), rli->until_name(), + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); + }; /* Thread stopped. Print the current replication position to the log */ { StringBuffer<100> tmp; From 0c1f97b3ab7befdae522aaa738be00f6700cee8e Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Tue, 5 May 2020 20:32:32 +0300 Subject: [PATCH 07/13] MDEV-15152 Optimistic parallel slave doesnt cope well with START SLAVE UNTIL The immediate bug was caused by a failure to recognize a correct position to stop the slave applier run in optimistic parallel mode. There were the following set of issues that the analysis unveil. 1 incorrect estimate for the event binlog position passed to is_until_satisfied 2 wait for workers to complete by the driver thread did not account non-group events that could be left unprocessed and thus to mix up the last executed binlog group's file and position: the file remained old and the position related to the new rotated file 3 incorrect 'slave reached file:pos' by the parallel slave report in the error log 4 relay log UNTIL missed out the parallel slave branch in is_until_satisfied. The patch addresses all of them to simplify logics of log change notification in either the master and relay-log until case. P.1 is addressed with passing the event into is_until_satisfied() for proper analisis by the function. P.2 is fixed by changes in handle_queued_pos_update(). P.4 required removing relay-log change notification by workers. Instead the driver thread updates the notion of the current relay-log fully itself with aid of introduced bool Relay_log_info::until_relay_log_names_defer. An extra print out of the requested until file:pos is arranged with --log-warning=3. --- .../r/rpl_parallel_optimistic_until.result | 291 +++++++++++ .../rpl/t/rpl_parallel_optimistic_until.test | 465 ++++++++++++++++++ sql/rpl_parallel.cc | 6 +- sql/rpl_rli.cc | 65 ++- sql/rpl_rli.h | 21 +- sql/slave.cc | 31 +- 6 files changed, 859 insertions(+), 20 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result create mode 100644 mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test diff --git a/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result b/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result new file mode 100644 index 00000000000..a83a9b61b9f --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_parallel_optimistic_until.result @@ -0,0 +1,291 @@ +include/master-slave.inc +[connection master] +connection slave; +include/stop_slave.inc +RESET MASTER; +RESET SLAVE; +connection master; +RESET MASTER; +CREATE TABLE t1 (a int primary key, b text) ENGINE=InnoDB; +INSERT INTO t1 SET a=25, b='trx0'; +connection slave; +include/start_slave.inc +connection master; +connection slave; +include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=2; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; +connection slave; +SET @old_max_relay_log_size = @@global.max_relay_log_size; +SET @@global.max_relay_log_size=4096; +connection master; +BEGIN; +INSERT INTO t1 SET a=1, b='trx1'; +INSERT INTO t1 SET a=2, b='trx1'; +INSERT INTO t1 SET a=3, b='trx1'; +INSERT INTO t1 SET a=4, b='trx1'; +INSERT INTO t1 SET a=5, b='trx1'; +INSERT INTO t1 SET a=6, b='trx1'; +INSERT INTO t1 SET a=7, b='trx1'; +INSERT INTO t1 SET a=8, b='trx1'; +INSERT INTO t1 SET a=9, b='trx1'; +INSERT INTO t1 SET a=10, b='trx1'; +INSERT INTO t1 SET a=11, b='trx1'; +INSERT INTO t1 SET a=12, b='trx1'; +INSERT INTO t1 SET a=13, b='trx1'; +INSERT INTO t1 SET a=14, b='trx1'; +INSERT INTO t1 SET a=15, b='trx1'; +INSERT INTO t1 SET a=16, b='trx1'; +INSERT INTO t1 SET a=17, b='trx1'; +INSERT INTO t1 SET a=18, b='trx1'; +INSERT INTO t1 SET a=19, b='trx1'; +INSERT INTO t1 SET a=20, b='trx1'; +INSERT INTO t1 SET a=21, b='trx1'; +INSERT INTO t1 SET a=22, b='trx1'; +INSERT INTO t1 SET a=23, b='trx1'; +INSERT INTO t1 SET a=24, b='trx1'; +COMMIT; +FLUSH LOGS; +BEGIN; +UPDATE t1 SET b='trx2_0' WHERE a = 25; +UPDATE t1 SET b='trx2' WHERE a = 25; +COMMIT; +INSERT INTO t1 SET a=26,b='trx3'; +*** case 1 UNTIL inside trx2 +connection slave1; +BEGIN; +INSERT INTO t1 SET a= 1; +connection slave; +SELECT <= AND < as "pos_until < trx0 and is within trx2"; +pos_until < trx0 and is within trx2 +1 +CHANGE MASTER TO MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +connection slave1; +ROLLBACK; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2'; +trx2 is committed +1 +SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3'; +trx3 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +*** case 2 UNTIL inside trx2 +connection slave; +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +connection slave1; +BEGIN; +INSERT INTO t1 SET a= 1; +connection slave; +include/stop_slave.inc +SELECT <= AND < as "pos_until >= trx0 and is within trx2"; +pos_until >= trx0 and is within trx2 +1 +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +connection slave1; +ROLLBACK; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2'; +trx2 is committed +1 +SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3'; +trx3 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +*** case 3 UNTIL inside trx1 +connection slave; +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +connection slave1; +BEGIN; +INSERT INTO t1 SET a= 1; # block trx1; +connection slave; +include/stop_slave.inc +SELECT < as "pos_until before trx2 start position"; +pos_until before trx2 start position +1 +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE UNTIL MASTER_LOG_FILE = 'file_2', MASTER_LOG_POS = ; +connection slave1; +ROLLBACK; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 25-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1'; +trx1 is committed +1 +SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2'; +trx2 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +*** case 4 Relay-log UNTIL inside trx1 +connection slave; +DELETE FROM t1 WHERE a <> 25; +UPDATE t1 SET b='trx0' WHERE a = 25; +connection slave1; +BEGIN; +INSERT INTO t1 SET a= 1; # block trx1; +connection slave; +include/stop_slave.inc +CHANGE MASTER TO MASTER_LOG_FILE = 'file_1', MASTER_LOG_POS = , MASTER_USE_GTID=no; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +connection slave1; +ROLLBACK; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +SELECT count(*) = 25-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1'; +trx1 is committed +1 +SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2'; +trx2 is not committed +1 +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +*** case 5 Relay-log UNTIL inside a "big" trx that spawns few relay logs +connection master; +CREATE TABLE t2 (a TEXT) ENGINE=InnoDB; +FLUSH LOGS; +connection slave; +connection slave; +include/stop_slave.inc +connection master; +BEGIN; +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +INSERT INTO t2 SET a=repeat('a',1024); +COMMIT; +INSERT INTO t2 SET a='a'; +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +include/diff_tables.inc [master:t2,slave:t2] +*** case 6 Relay-log UNTIL inside a small trx inside a sequence of relay logs +connection slave; +include/stop_slave.inc +connection master; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +BEGIN; +DELETE FROM t2 LIMIT 1; +COMMIT; +COMMIT; +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_to_start.inc +connection master; +include/sync_slave_io_with_master.inc +connection slave; +START SLAVE UNTIL RELAY_LOG_FILE = 'file_2', RELAY_LOG_POS = ; +Proof 1: Correct stop +connection slave; +include/wait_for_slave_sql_to_stop.inc +Proof 2: Resume works out +include/start_slave.inc +connection master; +connection slave; +include/diff_tables.inc [master:t2,slave:t2] +connection slave; +include/stop_slave.inc +SET GLOBAL max_relay_log_size=@old_max_relay_log_size; +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +include/start_slave.inc +connection master; +DROP TABLE t1, t2; +connection slave; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test b/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test new file mode 100644 index 00000000000..508213c9075 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_parallel_optimistic_until.test @@ -0,0 +1,465 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/master-slave.inc +# Format is restricted because the test expects a specific result of +# relay-logging that splits a transaction into two different files. +--source include/have_binlog_format_row.inc + +# +# MDEV-15152 Optimistic parallel slave doesn't cope well with START SLAVE UNTIL +# +--connection slave +--source include/stop_slave.inc +RESET MASTER; +RESET SLAVE; + +--connection master +RESET MASTER; +CREATE TABLE t1 (a int primary key, b text) ENGINE=InnoDB; +--let $a0 = 25 +--eval INSERT INTO t1 SET a=$a0, b='trx0' +# Memorize the position for replication restart from it +--let $pos_trx0 = query_get_value(SHOW MASTER STATUS, Position, 1) + +--connection slave +--source include/start_slave.inc + +--connection master +# --connection slave +--sync_slave_with_master +--source include/stop_slave.inc +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; +SET GLOBAL slave_parallel_threads=2; +SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; +SET GLOBAL slave_parallel_mode='optimistic'; + +# Run the slave in the following modes one by one. +# +# 1. the until position is set in the middle of trx2 +# below $pos_trx0 of the last exec position in the first file +# 2. and above $pos_trx0 +# In either case trx2 must commit before slave stops. +# 3. the until postion is inside trx1 +# 4. RELAY log until inside trx1 +# 5. RELAY log until inside a "big" trx +# 6. RELAY log until inside a trx within a sequence of relay logs +# +# Execution flaw for Until_Master_Pos cases follows as: +# create the transaction trx1, trx2 +# logged at the beginning of two subsequent binlog files. +# Set the until position to at the middle of the 2rd transaction. +# Engage the optimistic scheduler while having trx1 execution blocked. +# Lift the block after trx2 has reached waiting its order to commit. +# *Proof 1* +# Observe that the slave applier stops at a correct position. +# In the bug condition it would stop prematurely having the stop position +# in the first file, therefore trx2 not committed. +# Specifically, an internal transaction position until makes the applier to run +# beyond it to commit commit the current transaction. +# *Proof 2* +# Observe the following START SLAVE resumes OK. +# +# Auxiliary third trx3 on master is just for triggering the actual stop +# (whihc is a legacy UNTIL's property). +# trx0 is to produce a specific value of the last executed binlog file:pos +# to emulate the bug condition. +# +# Intermediate checks via SELECT are supposed to succeed +# with putting out value 1. +# +# NOTE: Relay log until tests have to use explicit log names and position +# which may require to adjust with future changes to event formats etc. +# + +--connection slave +SET @old_max_relay_log_size = @@global.max_relay_log_size; +SET @@global.max_relay_log_size=4096; + +--connection master +# trx1 +--let $a=1 +BEGIN; +while (`SELECT $a < $a0`) +{ + --eval INSERT INTO t1 SET a=$a, b='trx1' +--inc $a +} +COMMIT; +--let $fil_1 = query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx1 = query_get_value(SHOW MASTER STATUS, Position, 1) + +FLUSH LOGS; + +# $pos_0 the offset of the first event of trx2 in new file +--let $pos_0=query_get_value(SHOW MASTER STATUS, Position, 1) +# trx2 +--let $a=$a0 +BEGIN; +--eval UPDATE t1 SET b='trx2_0' WHERE a = $a +--eval UPDATE t1 SET b='trx2' WHERE a = $a +COMMIT; +--let $fil_2=query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx2=query_get_value(SHOW MASTER STATUS, Position, 1) + +# trx3 +--let $a=$a0 +--inc $a +--eval INSERT INTO t1 SET a=$a,b='trx3' +--let $pos_trx3=query_get_value(SHOW MASTER STATUS, Position, 1) +--let $a= + + +--echo *** case 1 UNTIL inside trx2 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1 +--connection slave +--let $pos_until=`SELECT $pos_trx0 - 1` +--replace_result $pos_0 $pos_until $pos_trx2 +--eval SELECT $pos_0 <= $pos_until AND $pos_until < $pos_trx2 as "pos_until < trx0 and is within trx2" +CHANGE MASTER TO MASTER_USE_GTID=no; +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--let $wait_condition= SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2' +--eval SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 2 UNTIL inside trx2 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1 + +--connection slave +--source include/stop_slave.inc + +--let $pos_until=`SELECT $pos_trx2 - 1` +--replace_result $pos_trx0 $pos_until $pos_trx2 +--eval SELECT $pos_trx0 <= $pos_until AND $pos_until < $pos_trx2 as "pos_until >= trx0 and is within trx2" +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--let $wait_condition= SELECT COUNT(*) > 0 FROM information_schema.processlist WHERE state = "Waiting for prior transaction to commit" +--source include/wait_condition.inc + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = 1 as 'trx2 is committed' FROM t1 WHERE b = 'trx2' +--eval SELECT count(*) = 0 as 'trx3 is not committed' FROM t1 WHERE b = 'trx3' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 3 UNTIL inside trx1 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1; # block trx1 + +--connection slave +--source include/stop_slave.inc + +--let $pos_until=`SELECT $pos_0 - 1` +--replace_result $pos_0 $pos_until $pos_trx2 +--eval SELECT $pos_until < $pos_0 as "pos_until before trx2 start position" +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +--replace_result $fil_2 file_2 $pos_until +--eval START SLAVE UNTIL MASTER_LOG_FILE = '$fil_2', MASTER_LOG_POS = $pos_until + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +if (`SELECT "$file_stop" != "$fil_2" OR $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $fil_2:$pos_until. + --die +} +--eval SELECT count(*) = $a0-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1' +--eval SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + +--echo *** case 4 Relay-log UNTIL inside trx1 + +--connection slave +--eval DELETE FROM t1 WHERE a <> $a0 +--eval UPDATE t1 SET b='trx0' WHERE a = $a0 + +--connection slave1 +# Blocker to hold off EXEC_MASTER_LOG_POS advance +BEGIN; + --eval INSERT INTO t1 SET a= 1; # block trx1 + +--connection slave +--source include/stop_slave.inc +--replace_result $fil_1 file_1 $pos_trx0 +--eval CHANGE MASTER TO MASTER_LOG_FILE = '$fil_1', MASTER_LOG_POS = $pos_trx0, MASTER_USE_GTID=no +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +# The following test sets the stop coordinate is set to inside the first event +# of a relay log that holds events of a transaction started in an earlier log. +# Peek the stop position in the middle of trx1, not even on a event boundary. +--let $pos_until=255 +--let $file_rl=slave-relay-bin.000003 +--let $binlog_file=$file_rl + +--let $pos_xid=508 +--let $info= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_xid LIMIT 1, Info, 1) + +if (`SELECT "$info" NOT LIKE "COMMIT /* xid=% */" OR $pos_xid < $pos_until`) +{ + --echo *** Unexpected offset. Refine it to point to the correct XID event! + --die +} + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--connection slave1 +# unblock to see the slave applier stops at $until +ROLLBACK; + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +if (`SELECT strcmp("$file_rl","$file_stop") > -1`) +{ + --echo *** ERROR: Slave stopped at $file_stop:$pos_stop which is not $file_rl:$pos_until. + --die +} + +--eval SELECT count(*) = $a0-1 as 'trx1 is committed' FROM t1 WHERE b = 'trx1' +--eval SELECT count(*) = 0 as 'trx2 is not committed' FROM t1 WHERE b = 'trx2' + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + + + +--echo *** case 5 Relay-log UNTIL inside a "big" trx that spawns few relay logs + +--connection master +CREATE TABLE t2 (a TEXT) ENGINE=InnoDB; +FLUSH LOGS; + +--sync_slave_with_master +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +--let $records=`SELECT floor(4*@@global.max_relay_log_size / 1024) + 1` + +--connection slave +--source include/stop_slave.inc + +--connection master +# trx4 +BEGIN; +--let $i=$records +while ($i) +{ + INSERT INTO t2 SET a=repeat('a',1024); + +--dec $i +} +COMMIT; + +# slave will stop there: +--let $file_trx4 = query_get_value(SHOW MASTER STATUS, File, 1) +--let $pos_trx4 = query_get_value(SHOW MASTER STATUS, Position, 1) + +# trx5 +INSERT INTO t2 SET a='a'; +--let $pos_trx5 = query_get_value(SHOW MASTER STATUS, Position, 1) + +--connection slave +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +# Set position inside the transaction though the value +# specified is beyond that relay log file. +# The coordianate may point to in a different event in future changes +# but should not move away from inside this big group of events. +# So we don't test which event in the transaction it points to. +--let $pos_until= 4500 +--let $file_rl= slave-relay-bin.000010 + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Master_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Exec_Master_Log_Pos, 1) +# It's showed the actual stop occurred before trx5 +if (`SELECT strcmp("$file_trx4", "$file_stop") <> 0 OR $pos_stop >= $pos_trx5 OR count(*) <> $records FROM t2`) +{ + --echo *** ERROR: Slave stopped at *binlog* $file_stop:$pos_stop which is not $file_trx4:$pos_trx4. + --die +} + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + +--let $diff_tables=master:t2,slave:t2 +--source include/diff_tables.inc + + + +--echo *** case 6 Relay-log UNTIL inside a small trx inside a sequence of relay logs + +--connection slave +--source include/stop_slave.inc + +--connection master +# trx6 +--let $records=`SELECT count(*) FROM t2` +while ($records) +{ + BEGIN; + DELETE FROM t2 LIMIT 1; + COMMIT; +--dec $records +} +COMMIT; + +--connection slave +START SLAVE IO_THREAD; +--source include/wait_for_slave_io_to_start.inc + +--connection master +--source include/sync_slave_io_with_master.inc + +--connection slave +# The relay-log coordinate is not at an event boundary and +# also may change across the server version. +# The test makes its best to check its coherance. +--let $pos_until= 3130 +--let $file_rl= slave-relay-bin.000018 + +--let $pos_gtid = 2987 +--let $info= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_gtid LIMIT 1, Info, 1) + +if (`SELECT "$info" != "BEGIN GTID 0-1-23"`) +{ + --echo *** Unexpected offset. Refine it to point to the correct GTID! + --die +} +--let $pos_event = 3120 +--let $type= query_get_value(SHOW RELAYLOG EVENTS IN '$file_rl' FROM $pos_event LIMIT 1, Event_type, 1) +if (`SELECT "$type" != "Delete_rows_v1"`) +{ + --echo *** Unexpected offset. Refine it to point to the expected event! + --die +} + +--replace_result $file_rl file_2 $pos_until +--eval START SLAVE UNTIL RELAY_LOG_FILE = '$file_rl', RELAY_LOG_POS = $pos_until + +--echo Proof 1: Correct stop +--connection slave +--source include/wait_for_slave_sql_to_stop.inc +--let $file_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_File, 1) +--let $pos_stop= query_get_value(SHOW SLAVE STATUS, Relay_Log_Pos, 1) +if (`SELECT strcmp("$file_stop", "$file_rl") = -1 OR + strcmp("$file_stop", "$file_rl") = 0 AND $pos_stop < $pos_until`) +{ + --echo *** ERROR: Slave stopped at *relay* $file_stop:$pos_stop which is not $file_rl:$pos_until. + --die +} + +--echo Proof 2: Resume works out +--source include/start_slave.inc +--connection master +--sync_slave_with_master + +--let $diff_tables=master:t2,slave:t2 +--source include/diff_tables.inc + +# +# Clean up. +# +--connection slave +--source include/stop_slave.inc +SET GLOBAL max_relay_log_size=@old_max_relay_log_size; +SET GLOBAL slave_parallel_mode=@old_parallel_mode; +SET GLOBAL slave_parallel_threads=@old_parallel_threads; +--source include/start_slave.inc + +--connection master +DROP TABLE t1, t2; + +--sync_slave_with_master +--source include/rpl_end.inc diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 027aa5f628c..3871ddcf8ef 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -75,18 +75,18 @@ handle_queued_pos_update(THD *thd, rpl_parallel_thread::queued_event *qev) /* Do not update position if an earlier event group caused an error abort. */ DBUG_ASSERT(qev->typ == rpl_parallel_thread::queued_event::QUEUED_POS_UPDATE); + rli= qev->rgi->rli; e= qev->entry_for_queued; - if (e->stop_on_error_sub_id < (uint64)ULONGLONG_MAX || e->force_abort) + if (e->stop_on_error_sub_id < (uint64)ULONGLONG_MAX || + (e->force_abort && !rli->stop_for_until)) return; - rli= qev->rgi->rli; mysql_mutex_lock(&rli->data_lock); cmp= strcmp(rli->group_relay_log_name, qev->event_relay_log_name); if (cmp < 0) { rli->group_relay_log_pos= qev->future_event_relay_log_pos; strmake_buf(rli->group_relay_log_name, qev->event_relay_log_name); - rli->notify_group_relay_log_name_update(); } else if (cmp == 0 && rli->group_relay_log_pos < qev->future_event_relay_log_pos) rli->group_relay_log_pos= qev->future_event_relay_log_pos; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 3e1bb28f701..7215cdd4b96 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -60,6 +60,7 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery) slave_running(MYSQL_SLAVE_NOT_RUN), until_condition(UNTIL_NONE), until_log_pos(0), retried_trans(0), executed_entries(0), sql_delay(0), sql_delay_end(0), + until_relay_log_names_defer(false), m_flags(0) { DBUG_ENTER("Relay_log_info::Relay_log_info"); @@ -499,6 +500,8 @@ void Relay_log_info::clear_until_condition() until_condition= Relay_log_info::UNTIL_NONE; until_log_name[0]= 0; until_log_pos= 0; + until_relay_log_names_defer= false; + DBUG_VOID_RETURN; } @@ -989,7 +992,6 @@ void Relay_log_info::inc_group_relay_log_pos(ulonglong log_pos, { group_relay_log_pos= rgi->future_event_relay_log_pos; strmake_buf(group_relay_log_name, rgi->event_relay_log_name); - notify_group_relay_log_name_update(); } else if (cmp == 0 && group_relay_log_pos < rgi->future_event_relay_log_pos) group_relay_log_pos= rgi->future_event_relay_log_pos; @@ -1279,29 +1281,78 @@ err: autoincrement or if we have transactions). Should be called ONLY if until_condition != UNTIL_NONE ! + + In the parallel execution mode and UNTIL_MASTER_POS the file name is + presented by future_event_master_log_name which may be ahead of + group_master_log_name. Log_event::log_pos does relate to it nevertheless + so the pair comprises a correct binlog coordinate. + Internal group events and events that have zero log_pos also + produce the zero for the local log_pos which may not lead to the + function falsely return true. + In UNTIL_RELAY_POS the original caching and notification are simplified + to straightforward files comparison when the current event can't be + a part of an event group. + RETURN VALUE true - condition met or error happened (condition seems to have bad log file name) false - condition not met */ -bool Relay_log_info::is_until_satisfied(my_off_t master_beg_pos) +bool Relay_log_info::is_until_satisfied(Log_event *ev) { const char *log_name; ulonglong log_pos; + /* Prevents stopping within transaction; needed solely for Relay UNTIL. */ + bool in_trans= false; + DBUG_ENTER("Relay_log_info::is_until_satisfied"); if (until_condition == UNTIL_MASTER_POS) { log_name= (mi->using_parallel() ? future_event_master_log_name : group_master_log_name); - log_pos= master_beg_pos; + log_pos= (get_flag(Relay_log_info::IN_TRANSACTION) || !ev || !ev->log_pos) ? + (mi->using_parallel() ? 0 : group_master_log_pos) : + ev->log_pos - ev->data_written; } else { DBUG_ASSERT(until_condition == UNTIL_RELAY_POS); - log_name= group_relay_log_name; - log_pos= group_relay_log_pos; + if (!mi->using_parallel()) + { + log_name= group_relay_log_name; + log_pos= group_relay_log_pos; + } + else + { + log_name= event_relay_log_name; + log_pos= event_relay_log_pos; + in_trans= get_flag(Relay_log_info::IN_TRANSACTION); + /* + until_log_names_cmp_result is set to UNKNOWN either + - by a non-group event *and* only when it is in the middle of a group + - or by a group event when the preceding group made the above + non-group event to defer the resetting. + */ + if ((ev && !Log_event::is_group_event(ev->get_type_code()))) + { + if (in_trans) + { + until_relay_log_names_defer= true; + } + else + { + until_log_names_cmp_result= UNTIL_LOG_NAMES_CMP_UNKNOWN; + until_relay_log_names_defer= false; + } + } + else if (!in_trans && until_relay_log_names_defer) + { + until_log_names_cmp_result= UNTIL_LOG_NAMES_CMP_UNKNOWN; + until_relay_log_names_defer= false; + } + } } DBUG_PRINT("info", ("group_master_log_name='%s', group_master_log_pos=%llu", @@ -1355,8 +1406,8 @@ bool Relay_log_info::is_until_satisfied(my_off_t master_beg_pos) } DBUG_RETURN(((until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_EQUAL && - log_pos >= until_log_pos) || - until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_GREATER)); + (log_pos >= until_log_pos && !in_trans)) || + until_log_names_cmp_result == UNTIL_LOG_NAMES_CMP_GREATER)); } diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 4ec4821b67d..2bc0a80268a 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -219,7 +219,7 @@ public: */ char future_event_master_log_name[FN_REFLEN]; - /* + /* Original log name and position of the group we're currently executing (whose coordinates are group_relay_log_name/pos in the relay log) in the master's binlog. These concern the *group*, because in the master's @@ -419,7 +419,7 @@ public: void close_temporary_tables(); /* Check if UNTIL condition is satisfied. See slave.cc for more. */ - bool is_until_satisfied(my_off_t); + bool is_until_satisfied(Log_event *ev); inline ulonglong until_pos() { DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || @@ -427,7 +427,13 @@ public: return ((until_condition == UNTIL_MASTER_POS) ? group_master_log_pos : group_relay_log_pos); } - + inline char *until_name() + { + DBUG_ASSERT(until_condition == UNTIL_MASTER_POS || + until_condition == UNTIL_RELAY_POS); + return ((until_condition == UNTIL_MASTER_POS) ? group_master_log_name : + group_relay_log_name); + } /** Helper function to do after statement completion. @@ -564,6 +570,15 @@ private: relay_log.info had 4 lines. Now it has 5 lines. */ static const int LINES_IN_RELAY_LOG_INFO_WITH_DELAY= 5; + /* + Hint for when to stop event distribution by sql driver thread. + The flag is set ON by a non-group event when this event is in the middle + of a group (e.g a transaction group) so it's too early + to refresh the current-relay-log vs until-log cached comparison result. + And it is checked and to decide whether it's a right time to do so + when the being processed group has been fully scheduled. + */ + bool until_relay_log_names_defer; /* Holds the state of the data in the relay log. diff --git a/sql/slave.cc b/sql/slave.cc index eb95afd140b..1bf83aa9652 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3921,12 +3921,8 @@ static int exec_relay_log_event(THD* thd, Relay_log_info* rli, rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && (ev->server_id != global_system_variables.server_id || rli->replicate_same_server_id) && - rli->is_until_satisfied((rli->get_flag(Relay_log_info::IN_TRANSACTION) || !ev->log_pos) - ? rli->group_master_log_pos - : ev->log_pos - ev->data_written)) + rli->is_until_satisfied(ev)) { - sql_print_information("Slave SQL thread stopped because it reached its" - " UNTIL position %llu", rli->until_pos()); /* Setting abort_slave flag because we do not want additional message about error in query execution to be printed. @@ -5136,10 +5132,14 @@ pthread_handler_t handle_slave_sql(void *arg) } if ((rli->until_condition == Relay_log_info::UNTIL_MASTER_POS || rli->until_condition == Relay_log_info::UNTIL_RELAY_POS) && - rli->is_until_satisfied(rli->group_master_log_pos)) + rli->is_until_satisfied(NULL)) { sql_print_information("Slave SQL thread stopped because it reached its" - " UNTIL position %llu", rli->until_pos()); + " UNTIL position %llu in %s %s file", + rli->until_pos(), rli->until_name(), + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); mysql_mutex_unlock(&rli->data_lock); goto err; } @@ -5205,7 +5205,24 @@ pthread_handler_t handle_slave_sql(void *arg) err: if (mi->using_parallel()) rli->parallel.wait_for_done(thd, rli); + /* Gtid_list_log_event::do_apply_event has already reported the GTID until */ + if (rli->stop_for_until && rli->until_condition != Relay_log_info::UNTIL_GTID) + { + if (global_system_variables.log_warnings > 2) + sql_print_information("Slave SQL thread UNTIL stop was requested at position " + "%llu in %s %s file", + rli->until_log_pos, rli->until_log_name, + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); + sql_print_information("Slave SQL thread stopped because it reached its" + " UNTIL position %llu in %s %s file", + rli->until_pos(), rli->until_name(), + rli->until_condition == + Relay_log_info::UNTIL_MASTER_POS ? + "binlog" : "relaylog"); + }; /* Thread stopped. Print the current replication position to the log */ { StringBuffer<100> tmp; From 403dacf6a9a373c25a808a1be9169ab2cd001448 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 26 May 2020 20:04:47 +0300 Subject: [PATCH 08/13] Fixed crash in aria recovery when using bulk insert MDEV-20578 Got error 126 when executing undo undo_key_delete upon Aria crash recovery The crash happens in this scenario: - Table with unique keys and non unique keys - Batch insert (LOAD DATA or INSERT ... SELECT) with REPLACE - Some insert succeeds followed by duplicate key error In the above scenario the table gets corrupted. The bug was that we don't generate any undo entry for the failed insert as the whole insert can be ignored by undo. The code did however not take into account that when bulk insert is used, we would write cached keys to the file on failure and undo would wrongly ignore these. Fixed by moving the writing of the cache keys after we write the aborted-insert event to the log. --- mysql-test/suite/maria/bulk_insert_crash.opt | 2 ++ .../suite/maria/bulk_insert_crash.result | 13 +++++++ mysql-test/suite/maria/bulk_insert_crash.test | 36 +++++++++++++++++++ sql/handler.cc | 13 +++++++ sql/handler.h | 8 +---- storage/maria/ma_recovery.c | 21 +++++++---- storage/maria/ma_write.c | 15 ++++---- 7 files changed, 89 insertions(+), 19 deletions(-) create mode 100644 mysql-test/suite/maria/bulk_insert_crash.opt create mode 100644 mysql-test/suite/maria/bulk_insert_crash.result create mode 100644 mysql-test/suite/maria/bulk_insert_crash.test diff --git a/mysql-test/suite/maria/bulk_insert_crash.opt b/mysql-test/suite/maria/bulk_insert_crash.opt new file mode 100644 index 00000000000..f85a8d9c973 --- /dev/null +++ b/mysql-test/suite/maria/bulk_insert_crash.opt @@ -0,0 +1,2 @@ +--skip-stack-trace --skip-core-file +--default-storage-engine=Aria diff --git a/mysql-test/suite/maria/bulk_insert_crash.result b/mysql-test/suite/maria/bulk_insert_crash.result new file mode 100644 index 00000000000..fc28bf325ba --- /dev/null +++ b/mysql-test/suite/maria/bulk_insert_crash.result @@ -0,0 +1,13 @@ +create table t1 (a int primary key, b int, c int, unique key(b), key(c)) engine=aria transactional=1; +insert into t1 values (1000,1000,1000); +insert into t1 select seq,seq+100, seq+200 from seq_1_to_10; +SET GLOBAL debug_dbug="+d,crash_end_bulk_insert"; +REPLACE into t1 select seq+20,seq+95, seq + 300 from seq_1_to_10; +ERROR HY000: Lost connection to MySQL server during query +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select sum(a),sum(b),sum(c) from t1; +sum(a) sum(b) sum(c) +1055 2055 3055 +drop table t1; diff --git a/mysql-test/suite/maria/bulk_insert_crash.test b/mysql-test/suite/maria/bulk_insert_crash.test new file mode 100644 index 00000000000..d9167c3f0d7 --- /dev/null +++ b/mysql-test/suite/maria/bulk_insert_crash.test @@ -0,0 +1,36 @@ +--source include/not_embedded.inc +--source include/not_valgrind.inc +# Avoid CrashReporter popup on Mac +--source include/not_crashrep.inc +# Binary must be compiled with debug for crash to occur +--source include/have_debug.inc +--source include/have_sequence.inc + +# +# MDEV-20578 Got error 126 when executing undo undo_key_delete upon Aria crash +# recovery +# + +# Write file to make mysql-test-run.pl expect crash and restart +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +create table t1 (a int primary key, b int, c int, unique key(b), key(c)) engine=aria transactional=1; +insert into t1 values (1000,1000,1000); +insert into t1 select seq,seq+100, seq+200 from seq_1_to_10; + +# Insert into t1 with batch insert where we get a rows replaced after +# a few sucessful inserts + +SET GLOBAL debug_dbug="+d,crash_end_bulk_insert"; + +--error 2013 +REPLACE into t1 select seq+20,seq+95, seq + 300 from seq_1_to_10; + +# Wait until restarted +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect + +check table t1; +select sum(a),sum(b),sum(c) from t1; +drop table t1; diff --git a/sql/handler.cc b/sql/handler.cc index d739ce7cd5b..d6e1680143c 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4164,6 +4164,19 @@ int handler::ha_repair(THD* thd, HA_CHECK_OPT* check_opt) } +/** + End bulk insert +*/ + +int handler::ha_end_bulk_insert() +{ + DBUG_ENTER("handler::ha_end_bulk_insert"); + DBUG_EXECUTE_IF("crash_end_bulk_insert", + { extra(HA_EXTRA_FLUSH) ; DBUG_SUICIDE();}); + estimation_rows_to_insert= 0; + DBUG_RETURN(end_bulk_insert()); +} + /** Bulk update row: public interface. diff --git a/sql/handler.h b/sql/handler.h index a0a43daf144..d1c4b79ee48 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -2911,13 +2911,7 @@ public: start_bulk_insert(rows, flags); DBUG_VOID_RETURN; } - int ha_end_bulk_insert() - { - DBUG_ENTER("handler::ha_end_bulk_insert"); - estimation_rows_to_insert= 0; - int ret= end_bulk_insert(); - DBUG_RETURN(ret); - } + int ha_end_bulk_insert(); int ha_bulk_update_row(const uchar *old_data, uchar *new_data, uint *dup_key_found); int ha_delete_all_rows(); diff --git a/storage/maria/ma_recovery.c b/storage/maria/ma_recovery.c index c8031030361..96b678b8695 100644 --- a/storage/maria/ma_recovery.c +++ b/storage/maria/ma_recovery.c @@ -949,6 +949,7 @@ prototype_redo_exec_hook(REDO_RENAME_TABLE) char *old_name, *new_name; int error= 1; MARIA_HA *info= NULL; + my_bool from_table_is_crashed= 0; DBUG_ENTER("exec_REDO_LOGREC_REDO_RENAME_TABLE"); if (skip_DDLs) @@ -1018,15 +1019,15 @@ prototype_redo_exec_hook(REDO_RENAME_TABLE) } if (maria_is_crashed(info)) { - tprint(tracef, ", is crashed, can't rename it"); - ALERT_USER(); - goto end; + tprint(tracef, "is crashed, can't be used for rename ; new-name table "); + from_table_is_crashed= 1; } if (close_one_table(info->s->open_file_name.str, rec->lsn) || maria_close(info)) goto end; info= NULL; - tprint(tracef, ", is ok for renaming; new-name table "); + if (!from_table_is_crashed) + tprint(tracef, "is ok for renaming; new-name table "); } else /* one or two files absent, or header corrupted... */ { @@ -1091,11 +1092,19 @@ prototype_redo_exec_hook(REDO_RENAME_TABLE) goto end; info= NULL; /* abnormal situation */ - tprint(tracef, ", exists but is older than record, can't rename it"); + tprint(tracef, "exists but is older than record, can't rename it"); goto end; } else /* one or two files absent, or header corrupted... */ - tprint(tracef, ", can't be opened, probably does not exist"); + tprint(tracef, "can't be opened, probably does not exist"); + + if (from_table_is_crashed) + { + eprint(tracef, "Aborting rename as old table was crashed"); + ALERT_USER(); + goto end; + } + tprint(tracef, ", renaming '%s'", old_name); if (maria_rename(old_name, new_name)) { diff --git a/storage/maria/ma_write.c b/storage/maria/ma_write.c index 126554fffdb..826ff40baaf 100644 --- a/storage/maria/ma_write.c +++ b/storage/maria/ma_write.c @@ -335,12 +335,6 @@ err: my_errno == HA_ERR_NULL_IN_SPATIAL || my_errno == HA_ERR_OUT_OF_MEM) { - if (info->bulk_insert) - { - uint j; - for (j=0 ; j < share->base.keys ; j++) - maria_flush_bulk_insert(info, j); - } info->errkey= i < share->base.keys ? (int) i : -1; /* We delete keys in the reverse order of insertion. This is the order that @@ -366,6 +360,7 @@ err: { if (_ma_ft_del(info,i,buff,record,filepos)) { + fatal_error= 1; if (local_lock_tree) mysql_rwlock_unlock(&keyinfo->root_lock); break; @@ -380,6 +375,7 @@ err: filepos, info->trn->trid))) { + fatal_error= 1; if (local_lock_tree) mysql_rwlock_unlock(&keyinfo->root_lock); break; @@ -399,6 +395,13 @@ err: fatal_error= 1; } + if (info->bulk_insert) + { + uint j; + for (j=0 ; j < share->base.keys ; j++) + maria_flush_bulk_insert(info, j); + } + if (fatal_error) { maria_print_error(info->s, HA_ERR_CRASHED); From 18d8f06f31cbe3913e1c80c0c5120f23e036bd3e Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sun, 17 May 2020 16:24:40 +0300 Subject: [PATCH 09/13] intrusive::list fixes namespace intrusive: removed split class into two: ilist and sized_ilist which has a size field. ilist no more NULLify pointers to bring a slignly better performance. As a consequence, fil_space_t::is_in_unflushed_spaces and fil_space_t::is_in_rotation_list boolean members are needed now. --- include/{intrusive_list.h => ilist.h} | 101 ++++++++++++++++++++------ storage/innobase/fil/fil0crypt.cc | 3 +- storage/innobase/fil/fil0fil.cc | 52 +++++-------- storage/innobase/include/dyn0buf.h | 8 +- storage/innobase/include/fil0fil.h | 22 +++--- 5 files changed, 114 insertions(+), 72 deletions(-) rename include/{intrusive_list.h => ilist.h} (68%) diff --git a/include/intrusive_list.h b/include/ilist.h similarity index 68% rename from include/intrusive_list.h rename to include/ilist.h index 23534596b93..a94f878f49b 100644 --- a/include/intrusive_list.h +++ b/include/ilist.h @@ -1,5 +1,5 @@ /* - Copyright (c) 2019, MariaDB + Copyright (c) 2019, 2020, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -21,26 +21,28 @@ #include #include -namespace intrusive -{ - // Derive your class from this struct to insert to a linked list. -template struct list_node +template struct ilist_node { - list_node(list_node *next= NULL, list_node *prev= NULL) - : next(next), prev(prev) + ilist_node() + : +#ifndef DBUG_OFF + next(NULL), prev(NULL) +#endif { } - list_node *next; - list_node *prev; + ilist_node(ilist_node *next, ilist_node *prev) : next(next), prev(prev) {} + + ilist_node *next; + ilist_node *prev; }; // Modelled after std::list -template class list +template class ilist { public: - typedef list_node ListNode; + typedef ilist_node ListNode; class Iterator; // All containers in C++ should define these types to implement generic @@ -103,10 +105,10 @@ public: private: ListNode *node_; - friend class list; + friend class ilist; }; - list() : sentinel_(&sentinel_, &sentinel_), size_(0) {} + ilist() : sentinel_(&sentinel_, &sentinel_) {} reference front() { return *begin(); } reference back() { return *--end(); } @@ -127,14 +129,18 @@ public: reverse_iterator rend() { return reverse_iterator(begin()); } const_reverse_iterator rend() const { return reverse_iterator(begin()); } - bool empty() const { return size_ == 0; } - size_type size() const { return size_; } + bool empty() const { return sentinel_.next == &sentinel_; } + + // Not implemented because it's O(N) + // size_type size() const + // { + // return static_cast(std::distance(begin(), end())); + // } void clear() { sentinel_.next= &sentinel_; sentinel_.prev= &sentinel_; - size_= 0; } iterator insert(iterator pos, reference value) @@ -148,7 +154,6 @@ public: static_cast(value).prev= prev; static_cast(value).next= curr; - ++size_; return iterator(&value); } @@ -160,13 +165,12 @@ public: prev->next= next; next->prev= prev; - // This is not required for list functioning. But maybe it'll prevent bugs - // and ease debugging. +#ifndef DBUG_OFF ListNode *curr= pos.node_; curr->prev= NULL; curr->next= NULL; +#endif - --size_; return next; } @@ -177,12 +181,63 @@ public: void pop_front() { erase(begin()); } // STL version is O(n) but this is O(1) because an element can't be inserted - // several times in the same intrusive list. + // several times in the same ilist. void remove(reference value) { erase(iterator(&value)); } private: ListNode sentinel_; - size_type size_; }; -} // namespace intrusive +// Similar to ilist but also has O(1) size() method. +template class sized_ilist : public ilist +{ + typedef ilist BASE; + +public: + // All containers in C++ should define these types to implement generic + // container interface. + using typename BASE::const_iterator; + using typename BASE::const_pointer; + using typename BASE::const_reference; + using typename BASE::const_reverse_iterator; + using typename BASE::difference_type; + using typename BASE::iterator; + using typename BASE::pointer; + using typename BASE::reference; + using typename BASE::reverse_iterator; + using typename BASE::size_type; + using typename BASE::value_type; + + sized_ilist() : size_(0) {} + + size_type size() const { return size_; } + + void clear() + { + BASE::clear(); + size_= 0; + } + + iterator insert(iterator pos, reference value) + { + ++size_; + return BASE::insert(pos, value); + } + + iterator erase(iterator pos) + { + --size_; + return BASE::erase(pos); + } + + void push_back(reference value) { insert(BASE::end(), value); } + void pop_back() { erase(BASE::end()); } + + void push_front(reference value) { insert(BASE::begin(), value); } + void pop_front() { erase(BASE::begin()); } + + void remove(reference value) { erase(iterator(&value)); } + +private: + size_t size_; +}; diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index a28c9f63797..f06aa7606ee 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2261,7 +2261,7 @@ static void fil_crypt_rotation_list_fill() space != NULL; space = UT_LIST_GET_NEXT(space_list, space)) { if (space->purpose != FIL_TYPE_TABLESPACE - || space->is_in_rotation_list() + || space->is_in_rotation_list || space->is_stopping() || UT_LIST_GET_LEN(space->chain) == 0) { continue; @@ -2306,6 +2306,7 @@ static void fil_crypt_rotation_list_fill() } fil_system->rotation_list.push_back(*space); + space->is_in_rotation_list = true; } } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index d4b0d3f4377..bb7fc0d015f 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2019, MariaDB Corporation. +Copyright (c) 2014, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -832,7 +832,7 @@ static void fil_flush_low(fil_space_t* space, bool metadata = false) /* No need to flush. User has explicitly disabled buffering. */ - ut_ad(!space->is_in_unflushed_spaces()); + ut_ad(!space->is_in_unflushed_spaces); ut_ad(fil_space_is_flushed(space)); ut_ad(space->n_pending_flushes == 0); @@ -895,10 +895,11 @@ static void fil_flush_low(fil_space_t* space, bool metadata = false) skip_flush: #endif /* _WIN32 */ if (!node->needs_flush) { - if (space->is_in_unflushed_spaces() + if (space->is_in_unflushed_spaces && fil_space_is_flushed(space)) { fil_system->unflushed_spaces.remove(*space); + space->is_in_unflushed_spaces = false; } } @@ -1193,13 +1194,14 @@ fil_node_close_to_free( if (fil_buffering_disabled(space)) { - ut_ad(!space->is_in_unflushed_spaces()); + ut_ad(!space->is_in_unflushed_spaces); ut_ad(fil_space_is_flushed(space)); - } else if (space->is_in_unflushed_spaces() + } else if (space->is_in_unflushed_spaces && fil_space_is_flushed(space)) { fil_system->unflushed_spaces.remove(*space); + space->is_in_unflushed_spaces = false; } fil_node_close_file(node); @@ -1226,16 +1228,18 @@ fil_space_detach( HASH_DELETE(fil_space_t, name_hash, fil_system->name_hash, ut_fold_string(space->name), space); - if (space->is_in_unflushed_spaces()) { + if (space->is_in_unflushed_spaces) { ut_ad(!fil_buffering_disabled(space)); fil_system->unflushed_spaces.remove(*space); + space->is_in_unflushed_spaces = false; } - if (space->is_in_rotation_list()) { + if (space->is_in_rotation_list) { fil_system->rotation_list.remove(*space); + space->is_in_rotation_list = false; } UT_LIST_REMOVE(fil_system->space_list, space); @@ -1462,6 +1466,7 @@ fil_space_create( /* Key rotation is not enabled, need to inform background encryption threads. */ fil_system->rotation_list.push_back(*space); + space->is_in_rotation_list = true; mutex_exit(&fil_system->mutex); mutex_enter(&fil_crypt_threads_mutex); os_event_set(fil_crypt_threads_event); @@ -4751,16 +4756,17 @@ fil_node_complete_io(fil_node_t* node, const IORequest& type) /* We don't need to keep track of unflushed changes as user has explicitly disabled buffering. */ - ut_ad(!node->space->is_in_unflushed_spaces()); + ut_ad(!node->space->is_in_unflushed_spaces); ut_ad(node->needs_flush == false); } else { node->needs_flush = true; - if (!node->space->is_in_unflushed_spaces()) { + if (!node->space->is_in_unflushed_spaces) { fil_system->unflushed_spaces.push_front( *node->space); + node->space->is_in_unflushed_spaces = true; } } } @@ -5247,7 +5253,7 @@ fil_flush_file_spaces( n_space_ids = 0; - for (intrusive::list::iterator it + for (sized_ilist::iterator it = fil_system->unflushed_spaces.begin(), end = fil_system->unflushed_spaces.end(); it != end; ++it) { @@ -5974,9 +5980,10 @@ fil_space_remove_from_keyrotation(fil_space_t* space) ut_ad(mutex_own(&fil_system->mutex)); ut_ad(space); - if (space->n_pending_ops == 0 && space->is_in_rotation_list()) { + if (space->n_pending_ops == 0 && space->is_in_rotation_list) { ut_a(!fil_system->rotation_list.empty()); fil_system->rotation_list.remove(*space); + space->is_in_rotation_list = false; } } @@ -6005,7 +6012,7 @@ fil_space_t *fil_system_t::keyrotate_next(fil_space_t *prev_space, don't remove the last processed tablespace from the rotation list. */ const bool remove= (!recheck || prev_space->crypt_data) && !key_version == !srv_encrypt_tables; - intrusive::list::iterator it= + sized_ilist::iterator it= prev_space == NULL ? fil_system->rotation_list.end() : prev_space; if (it == fil_system->rotation_list.end()) @@ -6117,24 +6124,3 @@ fil_space_set_punch_hole( { node->space->punch_hole = val; } - -/** Checks that this tablespace in a list of unflushed tablespaces. -@return true if in a list */ -bool fil_space_t::is_in_unflushed_spaces() const -{ - ut_ad(mutex_own(&fil_system->mutex)); - - return static_cast *>( - this) - ->next; -} - -/** Checks that this tablespace needs key rotation. -@return true if in a rotation list */ -bool fil_space_t::is_in_rotation_list() const -{ - ut_ad(mutex_own(&fil_system->mutex)); - - return static_cast *>(this) - ->next; -} diff --git a/storage/innobase/include/dyn0buf.h b/storage/innobase/include/dyn0buf.h index 311f6518943..b3302409e5f 100644 --- a/storage/innobase/include/dyn0buf.h +++ b/storage/innobase/include/dyn0buf.h @@ -29,7 +29,7 @@ Created 2013-03-16 Sunny Bains #include "mem0mem.h" #include "dyn0types.h" -#include "intrusive_list.h" +#include "ilist.h" /** Class that manages dynamic buffers. It uses a UT_LIST of @@ -43,7 +43,7 @@ template class dyn_buf_t { public: - class block_t : public intrusive::list_node<> { + class block_t : public ilist_node<> { public: block_t() @@ -153,7 +153,7 @@ public: /** SIZE - sizeof(m_node) + sizeof(m_used) */ enum { MAX_DATA_SIZE = SIZE - - sizeof(intrusive::list_node<>) + - sizeof(ilist_node<>) + sizeof(ib_uint32_t) }; @@ -167,7 +167,7 @@ public: friend class dyn_buf_t; }; - typedef intrusive::list list_t; + typedef sized_ilist list_t; enum { MAX_DATA_SIZE = block_t::MAX_DATA_SIZE}; diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9c722944665..62d87ce2c06 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -33,7 +33,7 @@ Created 10/25/1995 Heikki Tuuri #include "dict0types.h" #include "page0size.h" #include "ibuf0types.h" -#include "intrusive_list.h" +#include "ilist.h" #include @@ -82,8 +82,8 @@ fil_type_is_data( struct fil_node_t; /** Tablespace or log data space */ -struct fil_space_t : intrusive::list_node, - intrusive::list_node +struct fil_space_t : ilist_node, + ilist_node { ulint id; /*!< space id */ hash_node_t hash; /*!< hash chain node */ @@ -165,18 +165,18 @@ struct fil_space_t : intrusive::list_node, UT_LIST_NODE_T(fil_space_t) named_spaces; /*!< list of spaces for which MLOG_FILE_NAME records have been issued */ - /** Checks that this tablespace in a list of unflushed tablespaces. - @return true if in a list */ - bool is_in_unflushed_spaces() const; UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - /** Checks that this tablespace needs key rotation. - @return true if in a rotation list */ - bool is_in_rotation_list() const; /** MariaDB encryption data */ fil_space_crypt_t* crypt_data; + /** Checks that this tablespace in a list of unflushed tablespaces. */ + bool is_in_unflushed_spaces; + + /** Checks that this tablespace needs key rotation. */ + bool is_in_rotation_list; + /** True if the device this filespace is on supports atomic writes */ bool atomic_write_supported; @@ -507,7 +507,7 @@ struct fil_system_t { not put to this list: they are opened after the startup, and kept open until shutdown */ - intrusive::list unflushed_spaces; + sized_ilist unflushed_spaces; /*!< list of those tablespaces whose files contain unflushed writes; those spaces have @@ -530,7 +530,7 @@ struct fil_system_t { record has been written since the latest redo log checkpoint. Protected only by log_sys->mutex. */ - intrusive::list rotation_list; + ilist rotation_list; /*!< list of all file spaces needing key rotation.*/ From 67496281eaa11098ad4482c2bb5aa5976b4f5cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 27 May 2020 09:51:46 +0300 Subject: [PATCH 10/13] Fix the RelWithDebInfo build The build was broken in the parent commit 18d8f06f31cbe3913e1c80c0c5120f23e036bd3e. --- include/ilist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ilist.h b/include/ilist.h index a94f878f49b..ae87b79d54e 100644 --- a/include/ilist.h +++ b/include/ilist.h @@ -25,8 +25,8 @@ template struct ilist_node { ilist_node() - : #ifndef DBUG_OFF + : next(NULL), prev(NULL) #endif { From bf1aa7569e7a29fae2b1cdf36b840e98130fda4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 27 May 2020 09:53:36 +0300 Subject: [PATCH 11/13] Add an end-of-test marker to ease merges --- mysql-test/r/partition_alter.result | 1 + mysql-test/t/partition_alter.test | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mysql-test/r/partition_alter.result b/mysql-test/r/partition_alter.result index 8de3d4b45ad..699377751d1 100644 --- a/mysql-test/r/partition_alter.result +++ b/mysql-test/r/partition_alter.result @@ -165,3 +165,4 @@ select * from t1 partition (p0); k x 1 2000-01-01 00:00:00.000000 drop table t1; +# End of 10.2 tests diff --git a/mysql-test/t/partition_alter.test b/mysql-test/t/partition_alter.test index 3c783ebe375..b52e92445a8 100644 --- a/mysql-test/t/partition_alter.test +++ b/mysql-test/t/partition_alter.test @@ -151,3 +151,5 @@ insert into t1 (k, x) values (1, '2000-01-01 00:00'), (2, '2000-01-01 00:01'); alter table t1 drop key u, algorithm=inplace; select * from t1 partition (p0); drop table t1; + +--echo # End of 10.2 tests From 1b3adaab25c4f0e4288b261f973906bbf6b394c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 27 May 2020 10:58:28 +0300 Subject: [PATCH 12/13] Fix the build with GCC 4.1.2 The build was broken on CentOS 5 and CentOS 6 as a result of commit 18d8f06f31cbe3913e1c80c0c5120f23e036bd3e introducing some constructs that old GCC versions could not cope with. --- include/ilist.h | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/include/ilist.h b/include/ilist.h index ae87b79d54e..746061c397d 100644 --- a/include/ilist.h +++ b/include/ilist.h @@ -210,7 +210,7 @@ public: sized_ilist() : size_(0) {} - size_type size() const { return size_; } + std::size_t size() const { return size_; } void clear() { @@ -218,25 +218,30 @@ public: size_= 0; } - iterator insert(iterator pos, reference value) + typename ilist::iterator + insert(typename ilist::iterator pos, + typename ilist::reference value) { ++size_; return BASE::insert(pos, value); } - iterator erase(iterator pos) + typename ilist::iterator erase(typename ilist::iterator pos) { --size_; return BASE::erase(pos); } - void push_back(reference value) { insert(BASE::end(), value); } + void push_back(typename ilist::reference value) + { insert(BASE::end(), value); } void pop_back() { erase(BASE::end()); } - void push_front(reference value) { insert(BASE::begin(), value); } + void push_front(typename ilist::reference value) + { insert(BASE::begin(), value); } void pop_front() { erase(BASE::begin()); } - void remove(reference value) { erase(iterator(&value)); } + void remove(typename ilist::reference value) + { erase(iterator(&value)); } private: size_t size_; From 5139cfabb3bfc62be38e651ff176580e0aba09b8 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Wed, 27 May 2020 12:59:27 +0300 Subject: [PATCH 13/13] fix compilation --- include/ilist.h | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/include/ilist.h b/include/ilist.h index 746061c397d..5d0a425a9cc 100644 --- a/include/ilist.h +++ b/include/ilist.h @@ -196,21 +196,21 @@ template class sized_ilist : public ilist public: // All containers in C++ should define these types to implement generic // container interface. - using typename BASE::const_iterator; - using typename BASE::const_pointer; - using typename BASE::const_reference; - using typename BASE::const_reverse_iterator; - using typename BASE::difference_type; - using typename BASE::iterator; - using typename BASE::pointer; - using typename BASE::reference; - using typename BASE::reverse_iterator; - using typename BASE::size_type; - using typename BASE::value_type; + typedef T value_type; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + typedef value_type &reference; + typedef const value_type &const_reference; + typedef T *pointer; + typedef const T *const_pointer; + typedef typename BASE::Iterator iterator; + typedef const typename BASE::Iterator const_iterator; + typedef std::reverse_iterator reverse_iterator; + typedef std::reverse_iterator const_reverse_iterator; sized_ilist() : size_(0) {} - std::size_t size() const { return size_; } + size_type size() const { return size_; } void clear() { @@ -218,31 +218,26 @@ public: size_= 0; } - typename ilist::iterator - insert(typename ilist::iterator pos, - typename ilist::reference value) + iterator insert(iterator pos, reference value) { ++size_; return BASE::insert(pos, value); } - typename ilist::iterator erase(typename ilist::iterator pos) + iterator erase(iterator pos) { --size_; return BASE::erase(pos); } - void push_back(typename ilist::reference value) - { insert(BASE::end(), value); } + void push_back(reference value) { insert(BASE::end(), value); } void pop_back() { erase(BASE::end()); } - void push_front(typename ilist::reference value) - { insert(BASE::begin(), value); } + void push_front(reference value) { insert(BASE::begin(), value); } void pop_front() { erase(BASE::begin()); } - void remove(typename ilist::reference value) - { erase(iterator(&value)); } + void remove(reference value) { erase(iterator(&value)); } private: - size_t size_; + size_type size_; };