From b1ab2ba5833ad466ea9c08b89d89bb51e1d445ed Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Thu, 7 Nov 2019 15:24:21 +0530 Subject: [PATCH 1/4] MDEV-20519: Query plan regression with optimizer_use_condition_selectivity > 1 The issue here is the wrong estimate of the cardinality of a partial join, the cardinality is too high because the function table_cond_selectivity() returns an absurd number 100 while selectivity cannot be greater than 1. When accessing table t by outer reference t1.a via index we do not perform any range analysis for t. Yet we see TABLE::quick_key_parts[key] and TABLE->quick_rows[key] contain a non-zero value though these should have been remained untouched and equal to 0. Thus real cause of the problem is that TABLE::init does not clean the arrays TABLE::quick_key_parts[] and TABLE::>quick_rows[]. It should have done it because the TABLE structure created for any instance of a table can be reused for many queries. --- mysql-test/r/selectivity.result | 56 ++++++++++++++++++++++++++ mysql-test/r/selectivity_innodb.result | 56 ++++++++++++++++++++++++++ mysql-test/t/selectivity.test | 32 +++++++++++++++ sql/sql_select.cc | 4 -- sql/table.cc | 21 ++++++++++ sql/table.h | 1 + 6 files changed, 166 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/selectivity.result b/mysql-test/r/selectivity.result index 5fe6986e9ff..dc7e9494946 100644 --- a/mysql-test/r/selectivity.result +++ b/mysql-test/r/selectivity.result @@ -1810,4 +1810,60 @@ b a a b 9 9 10 10 set optimizer_use_condition_selectivity= @@optimizer_use_condition_selectivity; drop table t1,t2,t3; +# +# MDEV-20519: Query plan regression with optimizer_use_condition_selectivity=4 +# +create table t1 (id int, a int, PRIMARY KEY(id), key(a)); +insert into t1 select seq,seq from seq_1_to_100; +create table t2 (id int, a int, b int, PRIMARY KEY(id), key(a), key(b)); +insert into t2 select seq,seq,seq from seq_1_to_100; +set optimizer_switch='exists_to_in=off'; +set optimizer_use_condition_selectivity=2; +SELECT * FROM t1 +WHERE +EXISTS (SELECT * FROM t1 A INNER JOIN t2 ON t2.a = A.id +WHERE A.a=t1.a AND t2.b < 20); +id a +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +explain SELECT * FROM t1 +WHERE +EXISTS (SELECT * FROM t1 A INNER JOIN t2 ON t2.a = A.id +WHERE A.a=t1.a AND t2.b < 20); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 100 Using where +2 DEPENDENT SUBQUERY A ref PRIMARY,a a 5 test.t1.a 1 +2 DEPENDENT SUBQUERY t2 ref a,b a 5 test.A.id 1 Using where +EXPLAIN SELECT * FROM t1 A, t1 B WHERE A.a = B.a and A.id = 65; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE A const PRIMARY,a PRIMARY 4 const 1 +1 SIMPLE B ref a a 5 const 1 +explain SELECT * FROM t1 +WHERE +EXISTS (SELECT * FROM t1 A INNER JOIN t2 ON t2.a = A.id +WHERE A.a=t1.a AND t2.b < 20); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 100 Using where +2 DEPENDENT SUBQUERY A ref PRIMARY,a a 5 test.t1.a 1 +2 DEPENDENT SUBQUERY t2 ref a,b a 5 test.A.id 1 Using where +set optimizer_switch= @save_optimizer_switch; +set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +drop table t1,t2; # End of 10.1 tests diff --git a/mysql-test/r/selectivity_innodb.result b/mysql-test/r/selectivity_innodb.result index 70ddd5bf1a9..f0cbf1662aa 100644 --- a/mysql-test/r/selectivity_innodb.result +++ b/mysql-test/r/selectivity_innodb.result @@ -1820,6 +1820,62 @@ b a a b 9 9 10 10 set optimizer_use_condition_selectivity= @@optimizer_use_condition_selectivity; drop table t1,t2,t3; +# +# MDEV-20519: Query plan regression with optimizer_use_condition_selectivity=4 +# +create table t1 (id int, a int, PRIMARY KEY(id), key(a)); +insert into t1 select seq,seq from seq_1_to_100; +create table t2 (id int, a int, b int, PRIMARY KEY(id), key(a), key(b)); +insert into t2 select seq,seq,seq from seq_1_to_100; +set optimizer_switch='exists_to_in=off'; +set optimizer_use_condition_selectivity=2; +SELECT * FROM t1 +WHERE +EXISTS (SELECT * FROM t1 A INNER JOIN t2 ON t2.a = A.id +WHERE A.a=t1.a AND t2.b < 20); +id a +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +11 11 +12 12 +13 13 +14 14 +15 15 +16 16 +17 17 +18 18 +19 19 +explain SELECT * FROM t1 +WHERE +EXISTS (SELECT * FROM t1 A INNER JOIN t2 ON t2.a = A.id +WHERE A.a=t1.a AND t2.b < 20); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 index NULL a 5 NULL 100 Using where; Using index +2 DEPENDENT SUBQUERY A ref PRIMARY,a a 5 test.t1.a 1 Using index +2 DEPENDENT SUBQUERY t2 ref a,b a 5 test.A.id 1 Using where +EXPLAIN SELECT * FROM t1 A, t1 B WHERE A.a = B.a and A.id = 65; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE A const PRIMARY,a PRIMARY 4 const 1 +1 SIMPLE B ref a a 5 const 1 Using index +explain SELECT * FROM t1 +WHERE +EXISTS (SELECT * FROM t1 A INNER JOIN t2 ON t2.a = A.id +WHERE A.a=t1.a AND t2.b < 20); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 index NULL a 5 NULL 100 Using where; Using index +2 DEPENDENT SUBQUERY A ref PRIMARY,a a 5 test.t1.a 1 Using index +2 DEPENDENT SUBQUERY t2 ref a,b a 5 test.A.id 1 Using where +set optimizer_switch= @save_optimizer_switch; +set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +drop table t1,t2; # End of 10.1 tests set optimizer_switch=@save_optimizer_switch_for_selectivity_test; set @tmp_ust= @@use_stat_tables; diff --git a/mysql-test/t/selectivity.test b/mysql-test/t/selectivity.test index a93ad5efb07..7df326edb44 100644 --- a/mysql-test/t/selectivity.test +++ b/mysql-test/t/selectivity.test @@ -1234,5 +1234,37 @@ set optimizer_use_condition_selectivity= @@optimizer_use_condition_selectivity; drop table t1,t2,t3; + +--echo # +--echo # MDEV-20519: Query plan regression with optimizer_use_condition_selectivity=4 +--echo # + + + +create table t1 (id int, a int, PRIMARY KEY(id), key(a)); +insert into t1 select seq,seq from seq_1_to_100; + +create table t2 (id int, a int, b int, PRIMARY KEY(id), key(a), key(b)); +insert into t2 select seq,seq,seq from seq_1_to_100; + +set optimizer_switch='exists_to_in=off'; +set optimizer_use_condition_selectivity=2; + +let $query= SELECT * FROM t1 + WHERE + EXISTS (SELECT * FROM t1 A INNER JOIN t2 ON t2.a = A.id + WHERE A.a=t1.a AND t2.b < 20); + +eval $query; +eval explain $query; + +EXPLAIN SELECT * FROM t1 A, t1 B WHERE A.a = B.a and A.id = 65; + +eval explain $query; + +set optimizer_switch= @save_optimizer_switch; +set optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; +drop table t1,t2; + --echo # End of 10.1 tests diff --git a/sql/sql_select.cc b/sql/sql_select.cc index dfc9f729118..55d371bdbfd 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -7654,7 +7654,6 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, something went wrong. */ sel /= (double)table->quick_rows[key] / (double) table->stat_records(); - DBUG_ASSERT(0 < sel && sel <= 2.0); set_if_smaller(sel, 1.0); used_range_selectivity= true; } @@ -7703,7 +7702,6 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, if (table->field[fldno]->cond_selectivity > 0) { sel /= table->field[fldno]->cond_selectivity; - DBUG_ASSERT(0 < sel && sel <= 2.0); set_if_smaller(sel, 1.0); } /* @@ -7761,7 +7759,6 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, if (field->cond_selectivity > 0) { sel/= field->cond_selectivity; - DBUG_ASSERT(0 < sel && sel <= 2.0); set_if_smaller(sel, 1.0); } break; @@ -7773,7 +7770,6 @@ double table_cond_selectivity(JOIN *join, uint idx, JOIN_TAB *s, sel*= table_multi_eq_cond_selectivity(join, idx, s, rem_tables, keyparts, ref_keyuse_steps); - DBUG_ASSERT(0.0 < sel && sel <= 1.0); return sel; } diff --git a/sql/table.cc b/sql/table.cc index 94cd174ffd7..52e63ea7389 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -4162,6 +4162,8 @@ void TABLE::init(THD *thd, TABLE_LIST *tl) created= TRUE; cond_selectivity= 1.0; cond_selectivity_sampling_explain= NULL; + quick_condition_rows=0; + initialize_quick_structures(); #ifdef HAVE_REPLICATION /* used in RBR Triggers */ master_had_triggers= 0; @@ -7546,3 +7548,22 @@ bool fk_modifies_child(enum_fk_option opt) static bool can_write[]= { false, false, true, true, false, true }; return can_write[opt]; } + + +/* + @brief + Initialize all the quick structures that are used to stored the + estimates when the range optimizer is run. + @details + This is specifically needed when we read the TABLE structure from the + table cache. There can be some garbage data from previous queries + that need to be reset here. +*/ + +void TABLE::initialize_quick_structures() +{ + bzero(quick_rows, sizeof(quick_rows)); + bzero(quick_key_parts, sizeof(quick_key_parts)); + bzero(quick_costs, sizeof(quick_costs)); + bzero(quick_n_ranges, sizeof(quick_n_ranges)); +} diff --git a/sql/table.h b/sql/table.h index 98ec9f005ea..44803b5aacd 100644 --- a/sql/table.h +++ b/sql/table.h @@ -1450,6 +1450,7 @@ public: } bool update_const_key_parts(COND *conds); + void initialize_quick_structures(); my_ptrdiff_t default_values_offset() const { return (my_ptrdiff_t) (s->default_values - record[0]); } From c4a844ca5143080426f92ef9de3f0a069a764d53 Mon Sep 17 00:00:00 2001 From: Hartmut Holzgraefe Date: Fri, 8 Nov 2019 08:03:49 +0100 Subject: [PATCH 2/4] MDEV-20981 wsrep_sst_mariabackup fails silently when mariabackup is not installed (#1406) Make sure failure to find mariabackup binary does not terminate the script silently, terminate with a clear error message instead --- scripts/wsrep_sst_mariabackup.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index cb4cb3bd1d4..59d2e12817b 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -83,7 +83,14 @@ fi pcmd="pv $pvopts" declare -a RC +set +e INNOBACKUPEX_BIN=$(which mariabackup) +if test -z $INNOBACKUPEX_BIN +then + wsrep_log_error 'mariabackup binary not found in $PATH' + exit 42 +fi +set -e XBSTREAM_BIN=mbstream XBCRYPT_BIN=xbcrypt # Not available in MariaBackup From dfd2d3d861fa7fad6747489dbbd173b7f680f44d Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Fri, 8 Nov 2019 09:50:14 -0500 Subject: [PATCH 3/4] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8a84b79a423..c4402b8a39e 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,3 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=1 -MYSQL_VERSION_PATCH=43 +MYSQL_VERSION_PATCH=44 From 13db50fc03e7312e6c01b06c7e4af69f69ba5382 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Wed, 6 Nov 2019 17:05:58 +0200 Subject: [PATCH 4/4] MDEV-19376 Repl_semi_sync_master::commit_trx assertion failure: ... || !m_active_tranxs->is_tranx_end_pos(trx_wait_binlog_name, trx_wait_binlog_pos) The assert indicates that the current transaction got caught uncleaned from the semisync master's cache when it is signaled to proceed upon its ack receive. The reason of missed cleanup turns out to be a flaw in the gtid connect mode. A submitted by connecting slave value of its last received event's binlog file *name* was adopted into {{Repl_semi_sync_master::m_reply_file_name}} as a part of semisync initialization. Notice that the initialization still refines the position part of the submitted last received event's binlog coordinates. The master side binlog filename:pos refinement is specific to the gtid connect mode for purpose of computing the latest binlog file to resume slave feeding from. Effectively in the gtid connect mode the computed resumption filename:pos may appear smaller in which case a new post-connect time committing transaction may be logged with its filename:pos also less than the submitted coordinates and that triggers the assert. Fixed with making the semisync initialization to use the refined filename:pos. It is guaranteed to be less than any new generated transaction's binlog:pos. --- .../rpl/r/rpl_semi_sync_gtid_reconnect.result | 28 +++++++ .../rpl/t/rpl_semi_sync_gtid_reconnect.test | 75 +++++++++++++++++++ sql/sql_repl.cc | 6 +- 3 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/rpl/r/rpl_semi_sync_gtid_reconnect.result create mode 100644 mysql-test/suite/rpl/t/rpl_semi_sync_gtid_reconnect.test diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_gtid_reconnect.result b/mysql-test/suite/rpl/r/rpl_semi_sync_gtid_reconnect.result new file mode 100644 index 00000000000..3c720b94fde --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_semi_sync_gtid_reconnect.result @@ -0,0 +1,28 @@ +include/master-slave.inc +[connection master] +RESET MASTER; +SET @@GLOBAL.rpl_semi_sync_master_enabled = 1; +include/stop_slave.inc +SET @@GLOBAL. rpl_semi_sync_slave_enabled = 1; +include/start_slave.inc +CREATE TABLE t1 (a INT); +INSERT INTO t1 SET a = 1; +include/save_master_gtid.inc +FLUSH LOGS; +INSERT INTO t1 SET a = 2; +include/stop_slave_sql.inc +INSERT INTO t1 SET a = 3; +include/sync_slave_io_with_master.inc +include/stop_slave_io.inc +RESET MASTER; +SET @@global.gtid_binlog_state = '0-1-2'; +CHANGE MASTER TO MASTER_USE_GTID = slave_pos; +SET @@global.gtid_slave_pos = '0-1-2'; +include/start_slave.inc +INSERT INTO t1 SET a = 4; +DROP TABLE t1; +SET @@GLOBAL. rpl_semi_sync_master_enabled = 0; +include/stop_slave.inc +SET @@GLOBAL. rpl_semi_sync_slave_enabled = 0; +include/start_slave.inc +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_gtid_reconnect.test b/mysql-test/suite/rpl/t/rpl_semi_sync_gtid_reconnect.test new file mode 100644 index 00000000000..c524fcff3a4 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_gtid_reconnect.test @@ -0,0 +1,75 @@ +source include/not_embedded.inc; +source include/have_binlog_format_mixed.inc; +source include/have_semisync.inc; +source include/master-slave.inc; + +# +# Semisync initialization +# +--connection master +RESET MASTER; +--let $sav_enabled_master=`SELECT @@GLOBAL.rpl_semi_sync_master_enabled` +SET @@GLOBAL.rpl_semi_sync_master_enabled = 1; + +--connection slave +source include/stop_slave.inc; +--let $sav_enabled_slave=`SELECT @@GLOBAL.rpl_semi_sync_slave_enabled` +SET @@GLOBAL. rpl_semi_sync_slave_enabled = 1; +source include/start_slave.inc; + +# Prove fixes to +# MDEV-19376 Assert (!m_active_tranxs->is_tranx_end_pos(trx_wait_binlog_name...) +# +# +# Run few queries to replicate/execute on slave. +# Stop the slave applier. +# Replicate/not-executed few more. +# Restart the slave. +# +--connection master +CREATE TABLE t1 (a INT); +INSERT INTO t1 SET a = 1; +--source include/save_master_gtid.inc +--let $resume_gtid = $master_pos +FLUSH LOGS; +INSERT INTO t1 SET a = 2; + +--sync_slave_with_master +--connection slave +--source include/stop_slave_sql.inc + +--connection master +INSERT INTO t1 SET a = 3; + +# the sync connection is 'slave' by default +--source include/sync_slave_io_with_master.inc +--connection slave +--source include/stop_slave_io.inc + +--connection master +RESET MASTER; +--eval SET @@global.gtid_binlog_state = '$resume_gtid' + +# The resume gtid is set up to point to the very first binlog file +--connection slave +CHANGE MASTER TO MASTER_USE_GTID = slave_pos; +--eval SET @@global.gtid_slave_pos = '$resume_gtid' +# Yet the slave io first submits the last received binlog file name:pos. +--source include/start_slave.inc + +# Here goes the cracker. +--connection master +INSERT INTO t1 SET a = 4; + +# +# Clean up +# +--connection master +DROP TABLE t1; +--eval SET @@GLOBAL. rpl_semi_sync_master_enabled = $sav_enabled_master + +--sync_slave_with_master +source include/stop_slave.inc; +--eval SET @@GLOBAL. rpl_semi_sync_slave_enabled = $sav_enabled_slave +source include/start_slave.inc; +--source include/rpl_end.inc diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index ef6d0706da0..48d2781cd46 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2725,7 +2725,11 @@ void mysql_binlog_send(THD* thd, char* log_ident, my_off_t pos, run hook first when all check has been made that slave seems to be requesting a reasonable position. i.e when transmit actually starts */ - if (RUN_HOOK(binlog_transmit, transmit_start, (thd, flags, log_ident, pos))) + + DBUG_ASSERT(pos == linfo.pos); + + if (RUN_HOOK(binlog_transmit, transmit_start, + (thd, flags, linfo.log_file_name, linfo.pos))) { info->errmsg= "Failed to run hook 'transmit_start'"; info->error= ER_UNKNOWN_ERROR;