From d61947550a38de96de44b4a526e6da90b910ff2b Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 31 May 2024 19:37:51 +0200 Subject: [PATCH 01/59] fix a typo --- scripts/mysql_install_db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index e2b2109373a..94677bc1211 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -332,8 +332,8 @@ parse_arguments PICK-ARGS-FROM-ARGV "$@" rel_mysqld="$dirname0/@INSTALL_SBINDIR@/mariadbd" if test "$cross_bootstrap" -eq 0 -a "$in_rpm" -eq 0 -a "$force" -eq 0 - do_resolve=1 then + do_resolve=1 fi # Configure paths to support files From a6b7203d6564728089df66231739e0cc77b039d7 Mon Sep 17 00:00:00 2001 From: Denis Protivensky Date: Tue, 7 May 2024 16:42:13 +0300 Subject: [PATCH 02/59] MDEV-33952: Fix flaky galera_create_table_as_select test with debug sync The test that triggers multi-master conflict between two CTAS commands uses LOCK/UNLOCK TABLES to block local CTAS from progress. It could result in a race when UNLOCK TABLES command is issued a bit earlier then needed, causing local CTAS to run further and change wsrep transaction state, so that a different code path is taken later and the original error gets overridden, causing the test to fail. The solution is to replace LOCK/UNLOCK TABLES with debug sync points. Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera/disabled.def | 1 - .../r/galera_create_table_as_select.result | 13 +++++------ .../t/galera_create_table_as_select.test | 22 +++++++++---------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 84229dd4e2c..45c04bdfd4b 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -19,4 +19,3 @@ galera_slave_replay : MDEV-32780 galera_as_slave_replay: assertion in the wsrep: galera_sst_mysqldump_with_key : MDEV-32782 galera_sst_mysqldump_with_key test failed galera_var_ignore_apply_errors : MENT-1997 galera_var_ignore_apply_errors test freezes galera_desync_overlapped : MDEV-21538 galera_desync_overlapped MTR failed: Result content mismatch -galera_create_table_as_select : MDEV-33952 fails sporadically diff --git a/mysql-test/suite/galera/r/galera_create_table_as_select.result b/mysql-test/suite/galera/r/galera_create_table_as_select.result index beda5f30fe2..4ee313fd683 100644 --- a/mysql-test/suite/galera/r/galera_create_table_as_select.result +++ b/mysql-test/suite/galera/r/galera_create_table_as_select.result @@ -76,22 +76,21 @@ EXECUTE stmt; DEALLOCATE PREPARE stmt; DROP TABLE t1, t2; connection node_1; +SET GLOBAL DEBUG_DBUG = 'd,sync.wsrep_apply_cb'; CREATE TABLE t2 (f1 INTEGER) ENGINE=InnoDB; INSERT INTO t2 VALUES (1),(2),(3),(4),(5); -connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; -LOCK TABLE t2 WRITE; -connection node_1; +SET DEBUG_SYNC = 'create_table_select_before_create WAIT_FOR sync.wsrep_apply_cb_reached'; +SET DEBUG_SYNC = 'create_table_select_before_lock SIGNAL signal.wsrep_apply_cb WAIT_FOR bf_abort'; CREATE TABLE t1 AS SELECT * FROM t2;; -connection node_1a; connection node_2; SELECT COUNT(*) = 5 FROM t2; COUNT(*) = 5 1 CREATE TABLE t1 AS SELECT * FROM t2; -connection node_1a; -UNLOCK TABLES; connection node_1; -Got one of the listed errors +ERROR 70100: Query execution was interrupted +SET GLOBAL DEBUG_DBUG = ''; +SET DEBUG_SYNC = 'RESET'; DROP TABLE t1, t2; CREATE TABLE t2 (f1 INTEGER) ENGINE=InnoDB; INSERT INTO t2 VALUES (1),(2),(3),(4),(5); diff --git a/mysql-test/suite/galera/t/galera_create_table_as_select.test b/mysql-test/suite/galera/t/galera_create_table_as_select.test index cfee63e5e27..63af9b552e0 100644 --- a/mysql-test/suite/galera/t/galera_create_table_as_select.test +++ b/mysql-test/suite/galera/t/galera_create_table_as_select.test @@ -3,6 +3,8 @@ # --source include/galera_cluster.inc +--source include/have_debug_sync.inc +--source include/have_debug.inc --connection node_1 SET SESSION default_storage_engine=InnoDB; @@ -103,31 +105,27 @@ DROP TABLE t1, t2; # --connection node_1 +# Pause applying CTAS command from the other node +SET GLOBAL DEBUG_DBUG = 'd,sync.wsrep_apply_cb'; CREATE TABLE t2 (f1 INTEGER) ENGINE=InnoDB; INSERT INTO t2 VALUES (1),(2),(3),(4),(5); ---connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 -LOCK TABLE t2 WRITE; - ---connection node_1 +# Wait until local CTAS grabs MDL lock and let applied CTAS BF-abort it +SET DEBUG_SYNC = 'create_table_select_before_create WAIT_FOR sync.wsrep_apply_cb_reached'; +SET DEBUG_SYNC = 'create_table_select_before_lock SIGNAL signal.wsrep_apply_cb WAIT_FOR bf_abort'; --send CREATE TABLE t1 AS SELECT * FROM t2; ---connection node_1a ---let $wait_condition = SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE STATE LIKE 'Waiting for table metadata lock%' ---source include/wait_condition.inc - --connection node_2 SELECT COUNT(*) = 5 FROM t2; CREATE TABLE t1 AS SELECT * FROM t2; ---connection node_1a -UNLOCK TABLES; - --connection node_1 ---error ER_TABLE_EXISTS_ERROR,ER_QUERY_INTERRUPTED +--error ER_QUERY_INTERRUPTED --reap +SET GLOBAL DEBUG_DBUG = ''; +SET DEBUG_SYNC = 'RESET'; DROP TABLE t1, t2; # From 0cc9b49751fc86f7942e19fe6ed8f7227f847c02 Mon Sep 17 00:00:00 2001 From: Denis Protivensky Date: Mon, 25 Mar 2024 14:40:55 +0100 Subject: [PATCH 03/59] MDEV-32633: Fix Galera cluster <-> native replication interaction It's possible to establish Galera multi-cluster setups connected through the native replication when every Galera cluster is configured to have a separate domain ID. For this setup to work, we need to replace domain ID values in generated GTID events when they are written at transaction commit to the values configured by Wsrep replication. At the same time, it's possible that the GTID event already contains a correct domain ID if it comes through the native replication from another Galera cluster. In this case, when such an event is applied either through a native replication slave thread or through Wsrep applier, we write GTID event on transaction start and avoid writing it during transaction commit. The code contained multiple problems that were fixed: - applying GTID events didn't work because it's applied without a running server transaction and Wsrep transaction was not started - GTID event generation on transaction start didn't contain proper "standalone" and "is_transactional" flags that the original applied GTID event contained - condition determining that GTID event is written on transaction start to avoid writing it on commit relied on the fact that the GTID event is the first found in transaction/statement caches, which wasn't the case and resulted in duplicate GTID events written - instead of relying on the caches to find a GTID event, a simple check is introduced that follows the exact rules for checking if event is written at transaction start as described above - the test case is improved to check that exact GTID events are applied after two Galera clusters have synced. Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera_3nodes/disabled.def | 1 - .../r/galera_gtid_2_cluster.result | 68 ++++++++++++++++--- .../t/galera_gtid_2_cluster.test | 12 +++- sql/log.cc | 24 +++++-- sql/rpl_gtid.cc | 2 + sql/wsrep_mysqld.cc | 9 ++- 6 files changed, 98 insertions(+), 18 deletions(-) diff --git a/mysql-test/suite/galera_3nodes/disabled.def b/mysql-test/suite/galera_3nodes/disabled.def index de7d8e755c7..f2e6f9802ce 100644 --- a/mysql-test/suite/galera_3nodes/disabled.def +++ b/mysql-test/suite/galera_3nodes/disabled.def @@ -11,7 +11,6 @@ ############################################################################## galera_2_cluster : MDEV-32631 galera_2_cluster: before_rollback(): Assertion `0' failed -galera_gtid_2_cluster : MDEV-32633 galera_gtid_2_cluster: Assertion `thd->wsrep_next_trx_id() != (0x7fffffffffffffffLL * 2ULL + 1)' galera_ssl_reload : MDEV-32778 galera_ssl_reload failed with warning message galera_ipv6_mariabackup : temporarily disabled at the request of Codership galera_pc_bootstrap : temporarily disabled at the request of Codership diff --git a/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result b/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result index 7d4751e79af..ee622ba23c8 100644 --- a/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result +++ b/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result @@ -75,15 +75,15 @@ insert into t1 values (2, 21, 1); select @@gtid_binlog_state; @@gtid_binlog_state 1-11-2,2-21-1 -select * from t1; -cluster_domain_id node_server_id seq_no -1 11 2 -2 21 1 #wait for sync cluster 1 and 2 connection node_1; include/save_master_gtid.inc connection node_4; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +2 21 1 cluster 1 node 2 connection node_2; select @@gtid_binlog_state; @@ -98,6 +98,11 @@ connection node_1; include/save_master_gtid.inc connection node_4; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +2 21 1 cluster 1 node 3 connection node_3; select @@gtid_binlog_state; @@ -112,6 +117,12 @@ connection node_1; include/save_master_gtid.inc connection node_4; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 cluster 2 node 2 connection node_5; select @@gtid_binlog_state; @@ -126,6 +137,13 @@ connection node_4; include/save_master_gtid.inc connection node_1; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 cluster 2 node 3 connection node_6; select @@gtid_binlog_state; @@ -140,6 +158,14 @@ connection node_4; include/save_master_gtid.inc connection node_1; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 cluster 1 node 1 connection node_1; select @@gtid_binlog_state; @@ -220,15 +246,15 @@ insert into t1 values (2, 21, 1); select @@gtid_binlog_state; @@gtid_binlog_state 1-11-7,2-21-4 -select * from t1; -cluster_domain_id node_server_id seq_no -1 11 2 -2 21 1 #wait for sync cluster 1 and 2 connection node_1; include/save_master_gtid.inc connection node_4; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +2 21 1 cluster 1 node 2 connection node_2; select @@gtid_binlog_state; @@ -243,6 +269,11 @@ connection node_1; include/save_master_gtid.inc connection node_4; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +2 21 1 cluster 1 node 3 connection node_3; select @@gtid_binlog_state; @@ -257,6 +288,12 @@ connection node_1; include/save_master_gtid.inc connection node_4; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 cluster 2 node 2 connection node_5; select @@gtid_binlog_state; @@ -271,6 +308,13 @@ connection node_4; include/save_master_gtid.inc connection node_1; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 cluster 2 node 3 connection node_6; select @@gtid_binlog_state; @@ -285,6 +329,14 @@ connection node_4; include/save_master_gtid.inc connection node_1; include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 cluster 1 node 1 connection node_1; select @@gtid_binlog_state; diff --git a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test index 925600ffaa8..efbfad36e35 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test +++ b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test @@ -75,12 +75,12 @@ select @@gtid_binlog_state; select @@gtid_binlog_state; insert into t1 values (2, 21, 1); select @@gtid_binlog_state; -select * from t1; --echo #wait for sync cluster 1 and 2 --connection node_1 --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 1 node 2 @@ -94,6 +94,7 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 1 node 3 --connection node_3 @@ -106,6 +107,7 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 2 node 2 --connection node_5 @@ -118,6 +120,7 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 2 node 3 --connection node_6 @@ -130,6 +133,7 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 1 node 1 @@ -226,13 +230,13 @@ select @@gtid_binlog_state; --connection node_4 insert into t1 values (2, 21, 1); select @@gtid_binlog_state; -select * from t1; --echo #wait for sync cluster 1 and 2 --connection node_1 --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 1 node 2 @@ -246,6 +250,7 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 1 node 3 --connection node_3 @@ -258,6 +263,7 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 2 node 2 --connection node_5 @@ -270,6 +276,7 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 2 node 3 --connection node_6 @@ -282,6 +289,7 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +select * from t1 order by 1, 2, 3; --echo cluster 1 node 1 diff --git a/sql/log.cc b/sql/log.cc index 32c8ad46321..cb46fc1230f 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -5254,6 +5254,16 @@ MYSQL_BIN_LOG::is_xidlist_idle_nolock() return true; } +#ifdef WITH_WSREP +static bool is_gtid_written_on_trans_start(const THD *thd) +{ + return wsrep_gtid_mode && WSREP(thd) && + (thd->variables.gtid_seq_no || thd->variables.wsrep_gtid_seq_no) && + ((thd->slave_thread && wsrep_thd_is_local(thd)) || + (!thd->slave_thread && (wsrep_thd_is_applying(thd)))); +} +#endif + /** Create a new log file name. @@ -5893,9 +5903,7 @@ THD::binlog_start_trans_and_stmt() Ha_trx_info *ha_info; ha_info= this->ha_data[binlog_hton->slot].ha_info + (mstmt_mode ? 1 : 0); - if (!ha_info->is_started() && - (this->variables.gtid_seq_no || this->variables.wsrep_gtid_seq_no) && - wsrep_on(this) && + if (!ha_info->is_started() && is_gtid_written_on_trans_start(this) && (this->wsrep_cs().mode() == wsrep::client_state::m_local)) { uchar *buf= 0; @@ -5914,8 +5922,14 @@ THD::binlog_start_trans_and_stmt() domain_id= wsrep_gtid_server.domain_id; server_id= wsrep_gtid_server.server_id; } - Gtid_log_event gtid_event(this, seqno, domain_id, true, - LOG_EVENT_SUPPRESS_USE_F, true, 0); + rpl_group_info* rgi = this->slave_thread ? this->rgi_slave : this->wsrep_rgi; + const bool standalone = + rgi->gtid_ev_flags2 & Gtid_log_event::FL_STANDALONE; + const bool is_transactional = + rgi->gtid_ev_flags2 & Gtid_log_event::FL_TRANSACTIONAL; + Gtid_log_event gtid_event(this, seqno, domain_id, + standalone, LOG_EVENT_SUPPRESS_USE_F, + is_transactional, 0); // Replicated events in writeset doesn't have checksum gtid_event.checksum_alg= BINLOG_CHECKSUM_ALG_OFF; gtid_event.server_id= server_id; diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 3f519f67eb6..6177a703383 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -704,6 +704,8 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, { thd->wsrep_ignore_table= false; table->file->row_logging= 1; // replication requires binary logging + if (thd->wsrep_next_trx_id() == WSREP_UNDEFINED_TRX_ID) + thd->set_query_id(next_query_id()); wsrep_start_trx_if_not_started(thd); } else diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index e584c2cc144..6d95ea33acc 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -1827,8 +1827,13 @@ int wsrep_to_buf_helper( domain_id= wsrep_gtid_server.domain_id; server_id= wsrep_gtid_server.server_id; } - Gtid_log_event gtid_event(thd, seqno, domain_id, true, - LOG_EVENT_SUPPRESS_USE_F, true, 0); + rpl_group_info* rgi = thd->slave_thread ? thd->rgi_slave : thd->wsrep_rgi; + const bool standalone = + rgi->gtid_ev_flags2 & Gtid_log_event::FL_STANDALONE; + const bool is_transactional = + rgi->gtid_ev_flags2 & Gtid_log_event::FL_TRANSACTIONAL; + Gtid_log_event gtid_event(thd, seqno, domain_id, standalone, + LOG_EVENT_SUPPRESS_USE_F, is_transactional, 0); gtid_event.server_id= server_id; if (!gtid_event.is_valid()) ret= 0; ret= writer.write(>id_event); From a4838721a252ea5570b5fed9ab56e38e5b234865 Mon Sep 17 00:00:00 2001 From: Denis Protivensky Date: Wed, 27 Mar 2024 13:39:59 +0300 Subject: [PATCH 04/59] MDEV-32633: Fix Galera cluster <-> native replication interaction GTID events are applied without a running server transaction, we need to set next transaction ID for Wsrep transaction. The whole Galera cluster now has a single GTID value (including the server ID throughout the cluster), fix the config accordingly. Add force restart so that repeated MTR test execution prints consistent GTID values, otherwise they would have been recovered from the previous run. Signed-off-by: Julius Goryavsky --- .../r/galera_gtid_2_cluster.result | 98 +++++++++++++++++++ .../galera_3nodes/t/galera_gtid_2_cluster.cnf | 8 +- .../t/galera_gtid_2_cluster.test | 27 +++++ 3 files changed, 129 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result b/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result index ee622ba23c8..1cb14cd3eff 100644 --- a/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result +++ b/mysql-test/suite/galera_3nodes/r/galera_gtid_2_cluster.result @@ -166,6 +166,55 @@ cluster_domain_id node_server_id seq_no 2 21 1 2 22 2 2 23 3 +# check other nodes are consistent +connection node_2; +select @@gtid_binlog_state; +@@gtid_binlog_state +1-11-4,2-21-3 +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 +connection node_3; +select @@gtid_binlog_state; +@@gtid_binlog_state +1-11-4,2-21-3 +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 +connection node_5; +select @@gtid_binlog_state; +@@gtid_binlog_state +1-11-4,2-21-3 +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 +connection node_6; +select @@gtid_binlog_state; +@@gtid_binlog_state +1-11-4,2-21-3 +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 cluster 1 node 1 connection node_1; select @@gtid_binlog_state; @@ -337,6 +386,55 @@ cluster_domain_id node_server_id seq_no 2 21 1 2 22 2 2 23 3 +# check other nodes are consistent +connection node_2; +select @@gtid_binlog_state; +@@gtid_binlog_state +1-11-9,2-21-6 +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 +connection node_3; +select @@gtid_binlog_state; +@@gtid_binlog_state +1-11-9,2-21-6 +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 +connection node_5; +select @@gtid_binlog_state; +@@gtid_binlog_state +1-11-9,2-21-6 +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 +connection node_6; +select @@gtid_binlog_state; +@@gtid_binlog_state +1-11-9,2-21-6 +select * from t1 order by 1, 2, 3; +cluster_domain_id node_server_id seq_no +1 11 2 +1 12 3 +1 13 4 +2 21 1 +2 22 2 +2 23 3 cluster 1 node 1 connection node_1; select @@gtid_binlog_state; diff --git a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.cnf b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.cnf index dc5535ef34a..bc64d114275 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.cnf +++ b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.cnf @@ -9,11 +9,11 @@ server-id=11 [mysqld.2] wsrep_gtid_domain_id=1 -server-id=12 +server-id=11 [mysqld.3] wsrep_gtid_domain_id=1 -server-id=13 +server-id=11 [mysqld.4] wsrep_gtid_domain_id=2 @@ -21,8 +21,8 @@ server-id=21 [mysqld.5] wsrep_gtid_domain_id=2 -server-id=22 +server-id=21 [mysqld.6] wsrep_gtid_domain_id=2 -server-id=23 +server-id=21 diff --git a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test index efbfad36e35..c8247f15aba 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test +++ b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test @@ -11,6 +11,7 @@ --source include/big_test.inc --source include/galera_cluster.inc --source include/have_innodb.inc +--source include/force_restart.inc --connection node_1 --echo cluster 1 node 1 @@ -135,6 +136,19 @@ select @@gtid_binlog_state; --source include/sync_with_master_gtid.inc select * from t1 order by 1, 2, 3; +--echo # check other nodes are consistent +--connection node_2 +select @@gtid_binlog_state; +select * from t1 order by 1, 2, 3; +--connection node_3 +select @@gtid_binlog_state; +select * from t1 order by 1, 2, 3; +--connection node_5 +select @@gtid_binlog_state; +select * from t1 order by 1, 2, 3; +--connection node_6 +select @@gtid_binlog_state; +select * from t1 order by 1, 2, 3; --echo cluster 1 node 1 --connection node_1 @@ -291,6 +305,19 @@ select @@gtid_binlog_state; --source include/sync_with_master_gtid.inc select * from t1 order by 1, 2, 3; +--echo # check other nodes are consistent +--connection node_2 +select @@gtid_binlog_state; +select * from t1 order by 1, 2, 3; +--connection node_3 +select @@gtid_binlog_state; +select * from t1 order by 1, 2, 3; +--connection node_5 +select @@gtid_binlog_state; +select * from t1 order by 1, 2, 3; +--connection node_6 +select @@gtid_binlog_state; +select * from t1 order by 1, 2, 3; --echo cluster 1 node 1 --connection node_1 From c21aa486a86d9db29ad16ff279380969443df00f Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 14 May 2024 04:52:53 +0200 Subject: [PATCH 05/59] MDEV-32633: additional post-merge changes for 10.5+ --- sql/log.cc | 18 +++++------------- sql/wsrep_mysqld.cc | 9 ++------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index cb46fc1230f..b065657e9a5 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -5257,10 +5257,9 @@ MYSQL_BIN_LOG::is_xidlist_idle_nolock() #ifdef WITH_WSREP static bool is_gtid_written_on_trans_start(const THD *thd) { - return wsrep_gtid_mode && WSREP(thd) && + return wsrep_on(thd) && (thd->variables.gtid_seq_no || thd->variables.wsrep_gtid_seq_no) && - ((thd->slave_thread && wsrep_thd_is_local(thd)) || - (!thd->slave_thread && (wsrep_thd_is_applying(thd)))); + (thd->wsrep_cs().mode() == wsrep::client_state::m_local); } #endif @@ -5903,8 +5902,7 @@ THD::binlog_start_trans_and_stmt() Ha_trx_info *ha_info; ha_info= this->ha_data[binlog_hton->slot].ha_info + (mstmt_mode ? 1 : 0); - if (!ha_info->is_started() && is_gtid_written_on_trans_start(this) && - (this->wsrep_cs().mode() == wsrep::client_state::m_local)) + if (!ha_info->is_started() && is_gtid_written_on_trans_start(this)) { uchar *buf= 0; size_t len= 0; @@ -5922,14 +5920,8 @@ THD::binlog_start_trans_and_stmt() domain_id= wsrep_gtid_server.domain_id; server_id= wsrep_gtid_server.server_id; } - rpl_group_info* rgi = this->slave_thread ? this->rgi_slave : this->wsrep_rgi; - const bool standalone = - rgi->gtid_ev_flags2 & Gtid_log_event::FL_STANDALONE; - const bool is_transactional = - rgi->gtid_ev_flags2 & Gtid_log_event::FL_TRANSACTIONAL; - Gtid_log_event gtid_event(this, seqno, domain_id, - standalone, LOG_EVENT_SUPPRESS_USE_F, - is_transactional, 0); + Gtid_log_event gtid_event(this, seqno, domain_id, true, + LOG_EVENT_SUPPRESS_USE_F, true, 0); // Replicated events in writeset doesn't have checksum gtid_event.checksum_alg= BINLOG_CHECKSUM_ALG_OFF; gtid_event.server_id= server_id; diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 6d95ea33acc..e584c2cc144 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -1827,13 +1827,8 @@ int wsrep_to_buf_helper( domain_id= wsrep_gtid_server.domain_id; server_id= wsrep_gtid_server.server_id; } - rpl_group_info* rgi = thd->slave_thread ? thd->rgi_slave : thd->wsrep_rgi; - const bool standalone = - rgi->gtid_ev_flags2 & Gtid_log_event::FL_STANDALONE; - const bool is_transactional = - rgi->gtid_ev_flags2 & Gtid_log_event::FL_TRANSACTIONAL; - Gtid_log_event gtid_event(thd, seqno, domain_id, standalone, - LOG_EVENT_SUPPRESS_USE_F, is_transactional, 0); + Gtid_log_event gtid_event(thd, seqno, domain_id, true, + LOG_EVENT_SUPPRESS_USE_F, true, 0); gtid_event.server_id= server_id; if (!gtid_event.is_valid()) ret= 0; ret= writer.write(>id_event); From 58a0e1e3dd96628cb119a36361bdddb049873a73 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 3 Jun 2024 14:01:42 +0530 Subject: [PATCH 06/59] MDEV-34223 Innodb - add status variable for number of bulk inserts - Added a counter innodb_num_bulk_insert_operation in INFORMATION_SCHEMA.GLOBAL_STATUS. This counter is incremented whenever a InnoDB undergoes bulk insert operation. - Change the innodb_instant_alter_column to atomic variable. --- .../suite/innodb/r/innodb_status_variables.result | 1 + mysql-test/suite/innodb/r/insert_into_empty.result | 8 ++++++++ mysql-test/suite/innodb/t/insert_into_empty.test | 10 ++++++++++ storage/innobase/handler/ha_innodb.cc | 5 ++++- storage/innobase/handler/handler0alter.cc | 1 + storage/innobase/include/srv0srv.h | 5 ++++- storage/innobase/row/row0ins.cc | 1 + 7 files changed, 29 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_status_variables.result b/mysql-test/suite/innodb/r/innodb_status_variables.result index 5b8ca678795..47d202aa3ac 100644 --- a/mysql-test/suite/innodb/r/innodb_status_variables.result +++ b/mysql-test/suite/innodb/r/innodb_status_variables.result @@ -124,3 +124,4 @@ INNODB_ENCRYPTION_N_ROWLOG_BLOCKS_DECRYPTED INNODB_ENCRYPTION_N_TEMP_BLOCKS_ENCRYPTED INNODB_ENCRYPTION_N_TEMP_BLOCKS_DECRYPTED INNODB_ENCRYPTION_NUM_KEY_REQUESTS +INNODB_BULK_OPERATIONS diff --git a/mysql-test/suite/innodb/r/insert_into_empty.result b/mysql-test/suite/innodb/r/insert_into_empty.result index fdf66681b5d..2fdea1bce38 100644 --- a/mysql-test/suite/innodb/r/insert_into_empty.result +++ b/mysql-test/suite/innodb/r/insert_into_empty.result @@ -20,9 +20,17 @@ DROP TEMPORARY TABLE t; SET @save_ahi = @@global.innodb_adaptive_hash_index; SET GLOBAL innodb_adaptive_hash_index = 1; CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; +SET @old_bulk_op= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_bulk_operations'); BEGIN; INSERT INTO t1 SELECT * FROM seq_1_to_65536; ROLLBACK; +SELECT variable_value-@old_bulk_op bulk_operations +FROM information_schema.global_status +WHERE variable_name = 'innodb_bulk_operations'; +bulk_operations +1 CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb/t/insert_into_empty.test b/mysql-test/suite/innodb/t/insert_into_empty.test index 67afaeb8eba..aa9ef08b30d 100644 --- a/mysql-test/suite/innodb/t/insert_into_empty.test +++ b/mysql-test/suite/innodb/t/insert_into_empty.test @@ -26,9 +26,19 @@ DROP TEMPORARY TABLE t; SET @save_ahi = @@global.innodb_adaptive_hash_index; SET GLOBAL innodb_adaptive_hash_index = 1; CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; + +SET @old_bulk_op= +(SELECT variable_value FROM information_schema.global_status +WHERE variable_name = 'innodb_bulk_operations'); + BEGIN; INSERT INTO t1 SELECT * FROM seq_1_to_65536; ROLLBACK; + +SELECT variable_value-@old_bulk_op bulk_operations +FROM information_schema.global_status +WHERE variable_name = 'innodb_bulk_operations'; + CHECK TABLE t1; --echo # --echo # MDEV-24832 Root page AHI Removal fails fails during diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 7099680d2b2..7d1463039b9 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1061,7 +1061,7 @@ static SHOW_VAR innodb_status_variables[]= { {"defragment_count", &export_vars.innodb_defragment_count, SHOW_SIZE_T}, {"instant_alter_column", - &export_vars.innodb_instant_alter_column, SHOW_ULONG}, + &export_vars.innodb_instant_alter_column, SHOW_SIZE_T}, /* Online alter table status variables */ {"onlineddl_rowlog_rows", @@ -1104,6 +1104,9 @@ static SHOW_VAR innodb_status_variables[]= { {"encryption_num_key_requests", &export_vars.innodb_encryption_key_requests, SHOW_LONGLONG}, + /* InnoDB bulk operations */ + {"bulk_operations", &export_vars.innodb_bulk_operations, SHOW_SIZE_T}, + {NullS, NullS, SHOW_LONG} }; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 321b7dfe576..471d6915159 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -7355,6 +7355,7 @@ error_handling_drop_uncached: ut_d(dict_table_check_for_dup_indexes(user_table, CHECK_PARTIAL_OK)); if (ctx->need_rebuild()) { + export_vars.innodb_bulk_operations++; ctx->new_table->acquire(); } diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index ffed1090a44..dc0546ac0f3 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -713,7 +713,10 @@ struct export_var_t{ operations*/ /** Number of instant ALTER TABLE operations that affect columns */ - ulong innodb_instant_alter_column; + Atomic_counter innodb_instant_alter_column; + + /* Number of InnoDB bulk operations */ + Atomic_counter innodb_bulk_operations; ulint innodb_onlineddl_rowlog_rows; /*!< Online alter rows */ ulint innodb_onlineddl_rowlog_pct_used; /*!< Online alter percentage diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 6995a0d5c7b..0f3f7dd3166 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2749,6 +2749,7 @@ err_exit: #else /* BTR_CUR_HASH_ADAPT */ index->table->bulk_trx_id = trx->id; #endif /* BTR_CUR_HASH_ADAPT */ + export_vars.innodb_bulk_operations++; } trx->bulk_insert = true; From 581712b989f47e4cc0206b0b470b8fcd80acbc83 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Fri, 10 May 2024 10:01:15 +1000 Subject: [PATCH 07/59] MDEV-33490 MENT-1504 Fix some english strings in spider. --- storage/spider/spd_db_mysql.cc | 4 ++-- storage/spider/spd_db_oracle.cc | 2 +- storage/spider/spd_err.h | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index d5f693c2cde..232a2b02c25 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -325,7 +325,7 @@ SPIDER_DBTON spider_dbton_mysql = { spider_mysql_create_conn, spider_mysql_support_direct_join, &spider_db_mysql_utility, - "For communicating to MySQL using native protocol", + "For communication with MySQL using the native protocol", "3.4.0", SPIDER_MATURITY_STABLE }; @@ -342,7 +342,7 @@ SPIDER_DBTON spider_dbton_mariadb = { spider_mariadb_create_conn, spider_mariadb_support_direct_join, &spider_db_mariadb_utility, - "For communicating to MariaDB using native protocol", + "For communication with MariaDB using the native protocol", "3.4.0", SPIDER_MATURITY_STABLE }; diff --git a/storage/spider/spd_db_oracle.cc b/storage/spider/spd_db_oracle.cc index a452cd2c8dc..362a77536ae 100644 --- a/storage/spider/spd_db_oracle.cc +++ b/storage/spider/spd_db_oracle.cc @@ -332,7 +332,7 @@ SPIDER_DBTON spider_dbton_oracle = { spider_oracle_create_conn, spider_oracle_support_direct_join, &spider_db_oracle_utility, - "For communicating Oracle using native protocol", + "For communication with Oracle using the native protocol", "1.0.0", SPIDER_MATURITY_BETA }; diff --git a/storage/spider/spd_err.h b/storage/spider/spd_err.h index 60b2a084714..368a9789fbd 100644 --- a/storage/spider/spd_err.h +++ b/storage/spider/spd_err.h @@ -80,13 +80,13 @@ #define ER_SPIDER_XA_PREPARED_NUM 12604 #define ER_SPIDER_XA_PREPARED_STR "This xid is prepared" #define ER_SPIDER_XA_EXISTS_NUM 12605 -#define ER_SPIDER_XA_EXISTS_STR "This xid is already exist" +#define ER_SPIDER_XA_EXISTS_STR "This xid already exists" #define ER_SPIDER_XA_MEMBER_EXISTS_NUM 12606 -#define ER_SPIDER_XA_MEMBER_EXISTS_STR "This xid member is already exist" +#define ER_SPIDER_XA_MEMBER_EXISTS_STR "This xid member already exists" #define ER_SPIDER_XA_NOT_EXISTS_NUM 12607 -#define ER_SPIDER_XA_NOT_EXISTS_STR "This xid is not exist" +#define ER_SPIDER_XA_NOT_EXISTS_STR "This xid does not exist" #define ER_SPIDER_XA_MEMBER_NOT_EXISTS_NUM 12608 -#define ER_SPIDER_XA_MEMBER_NOT_EXISTS_STR "This xid member is not exist" +#define ER_SPIDER_XA_MEMBER_NOT_EXISTS_STR "This xid member does not exist" #define ER_SPIDER_SYS_TABLE_VERSION_NUM 12609 #define ER_SPIDER_SYS_TABLE_VERSION_STR "System table %s is different version" #define ER_SPIDER_WRONG_CHARACTER_IN_NAME_NUM 12611 From 76e0dc18b66025c273c5ec428e0e08d39f7cc8bf Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 4 Jun 2024 12:00:20 +0400 Subject: [PATCH 08/59] MDEV-34288 SET NAMES DEFAULT crashes `mariadbd --collation-server=utf8mb4_unicode_ci` The @@global.character_set_client variable could erroneously be set to a non-default collation of its character set, which further made the `SET NAMES DEFAULT` statement crash the server. Fixing the code to make sure that the global value these variables: @@character_set_client @@character_set_connection @@character_set_server @@character_set_database @@character_set_connection point to the default compiled collations of the character set. --- .../main/ctype_utf8mb4_unicode_ci_def.result | 45 +++++++++++++++++++ .../main/ctype_utf8mb4_unicode_ci_def.test | 42 +++++++++++++++++ sql/lex_charset.h | 6 +++ sql/mysqld.cc | 6 ++- sql/sql_lex.cc | 23 +++++++++- sql/sql_lex.h | 3 +- sql/sql_yacc.yy | 11 ++--- sql/sys_vars.cc | 11 ++--- sql/sys_vars.inl | 25 +++++++++++ 9 files changed, 156 insertions(+), 16 deletions(-) diff --git a/mysql-test/main/ctype_utf8mb4_unicode_ci_def.result b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.result index 2e15931248b..a80d6160ed8 100644 --- a/mysql-test/main/ctype_utf8mb4_unicode_ci_def.result +++ b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.result @@ -9,3 +9,48 @@ DROP TABLE t1; # # End of 10.3 tests # +# +# Start of 10.11 tests +# +# +# MDEV-34288 SET NAMES DEFAULT crashes `mariadbd --collation-server=utf8mb4_unicode_ci` +# +SET NAMES DEFAULT COLLATE latin1_bin; +ERROR 42000: COLLATION 'latin1_bin' is not valid for CHARACTER SET 'utf8mb4' +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; +@@character_set_connection @@collation_connection @@character_set_results +latin1 latin1_swedish_ci latin1 +SET NAMES DEFAULT COLLATE utf8mb4_bin; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; +@@character_set_connection @@collation_connection @@character_set_results +utf8mb4 utf8mb4_bin utf8mb4 +SET NAMES DEFAULT COLLATE uca1400_ai_ci; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; +@@character_set_connection @@collation_connection @@character_set_results +utf8mb4 utf8mb4_uca1400_ai_ci utf8mb4 +SET @@global.character_set_client=latin1; +SET NAMES DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; +@@character_set_connection @@collation_connection @@character_set_results +latin1 latin1_swedish_ci latin1 +SET @@global.character_set_client=utf8mb3; +SET NAMES DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; +@@character_set_connection @@collation_connection @@character_set_results +utf8mb3 utf8mb3_general_ci utf8mb3 +SET @@global.character_set_client=DEFAULT; +SET NAMES DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; +@@character_set_connection @@collation_connection @@character_set_results +utf8mb4 utf8mb4_general_ci utf8mb4 +SET NAMES DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; +@@character_set_connection @@collation_connection @@character_set_results +utf8mb4 utf8mb4_general_ci utf8mb4 +SET NAMES DEFAULT COLLATE DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; +@@character_set_connection @@collation_connection @@character_set_results +utf8mb4 utf8mb4_general_ci utf8mb4 +# +# End of 10.11 tests +# diff --git a/mysql-test/main/ctype_utf8mb4_unicode_ci_def.test b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.test index fb7fbe04e3b..3e55d1e6b1e 100644 --- a/mysql-test/main/ctype_utf8mb4_unicode_ci_def.test +++ b/mysql-test/main/ctype_utf8mb4_unicode_ci_def.test @@ -13,3 +13,45 @@ DROP TABLE t1; --echo # --echo # End of 10.3 tests --echo # + + +--echo # +--echo # Start of 10.11 tests +--echo # + +--echo # +--echo # MDEV-34288 SET NAMES DEFAULT crashes `mariadbd --collation-server=utf8mb4_unicode_ci` +--echo # + +--error ER_COLLATION_CHARSET_MISMATCH +SET NAMES DEFAULT COLLATE latin1_bin; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; + +SET NAMES DEFAULT COLLATE utf8mb4_bin; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; + +SET NAMES DEFAULT COLLATE uca1400_ai_ci; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; + +SET @@global.character_set_client=latin1; +SET NAMES DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; + +SET @@global.character_set_client=utf8mb3; +SET NAMES DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; + +SET @@global.character_set_client=DEFAULT; +SET NAMES DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; + +SET NAMES DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; + +SET NAMES DEFAULT COLLATE DEFAULT; +SELECT @@character_set_connection, @@collation_connection, @@character_set_results; + + +--echo # +--echo # End of 10.11 tests +--echo # diff --git a/sql/lex_charset.h b/sql/lex_charset.h index 2bbeff8a4a6..f593681d27b 100644 --- a/sql/lex_charset.h +++ b/sql/lex_charset.h @@ -288,6 +288,12 @@ public: DBUG_ASSERT(0); return m_ci->coll_name; } + static Lex_extended_collation_st collate_default() + { + Lex_extended_collation_st res; + res.set_collate_default(); + return res; + } void set_collate_default() { m_ci= &my_collation_contextually_typed_default; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index d0619aa26c2..5ae30282729 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4329,8 +4329,10 @@ static int init_common_variables() if (is_supported_parser_charset(default_charset_info)) { global_system_variables.collation_connection= default_charset_info; - global_system_variables.character_set_results= default_charset_info; - global_system_variables.character_set_client= default_charset_info; + global_system_variables.character_set_results= + global_system_variables.character_set_client= + Lex_exact_charset_opt_extended_collate(default_charset_info, true). + find_default_collation(); } else { diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 7d3810dd326..bbae1f3da99 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -12131,10 +12131,31 @@ bool LEX::sp_create_set_password_instr(THD *thd, } +/* + Handle the SET NAMES statement variants, e.g.: + SET NAMES DEFAULT; + SET NAMES DEFAULT COLLATE DEFAULT; + SET NAMES DEFAULT COLLATE latin1_bin; + SET NAMES latin1; + SET NAMES latin1 COLLATE DEFAULT; + SET NAMES latin1 COLLATE latin1_bin; + SET NAMES utf8mb4 COLLATE uca1400_ai_ci; + + @param pos - The position of the keyword `NAMES` inside the query + @param cs - The character set part, or nullptr if DEFAULT + @param cl - The collation (explicit or contextually typed) + @param no_lookahead - The tokinizer lookahead state +*/ bool LEX::set_names(const char *pos, - const Lex_exact_charset_opt_extended_collate &cscl, + CHARSET_INFO *cs, + const Lex_extended_collation_st &cl, bool no_lookahead) { + CHARSET_INFO *def= global_system_variables.character_set_client; + Lex_exact_charset_opt_extended_collate cscl(cs ? cs : def, true); + if (cscl.merge_collation_override(cl)) + return true; + if (sp_create_assignment_lex(thd, pos)) return true; CHARSET_INFO *ci= cscl.collation().charset_info(); diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 73f7e557c49..8eb7c20eb50 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -3873,7 +3873,8 @@ public: int case_stmt_action_then(); bool setup_select_in_parentheses(); bool set_names(const char *pos, - const Lex_exact_charset_opt_extended_collate &cs, + CHARSET_INFO *cs, + const Lex_extended_collation_st &coll, bool no_lookahead); bool set_trigger_new_row(const LEX_CSTRING *name, Item *val); bool set_trigger_field(const LEX_CSTRING *name1, const LEX_CSTRING *name2, diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 577c90e660a..cfd31635643 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -16671,18 +16671,15 @@ option_value_no_option_type: } | NAMES_SYM charset_name_or_default { - CHARSET_INFO *def= global_system_variables.character_set_client; - Lex_exact_charset_opt_extended_collate tmp($2 ? $2 : def, false); - if (Lex->set_names($1.pos(), tmp, yychar == YYEMPTY)) + if (Lex->set_names($1.pos(), $2, + Lex_extended_collation_st::collate_default(), + yychar == YYEMPTY)) MYSQL_YYABORT; } | NAMES_SYM charset_name_or_default COLLATE_SYM collation_name_or_default { - CHARSET_INFO *def= global_system_variables.character_set_client; - Lex_exact_charset_opt_extended_collate tmp($2 ? $2 : def, false); - if (tmp.merge_collation($4) || - Lex->set_names($1.pos(), tmp, yychar == YYEMPTY)) + if (Lex->set_names($1.pos(), $2, $4, yychar == YYEMPTY)) MYSQL_YYABORT; } | DEFAULT ROLE_SYM grant_role diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 8057ecb8626..418507d5c64 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -824,7 +824,7 @@ static Sys_var_struct Sys_character_set_system( READ_ONLY GLOBAL_VAR(system_charset_info), NO_CMD_LINE, offsetof(CHARSET_INFO, cs_name.str), DEFAULT(0)); -static Sys_var_struct Sys_character_set_server( +static Sys_var_charset Sys_character_set_server( "character_set_server", "The default character set", SESSION_VAR(collation_server), NO_CMD_LINE, offsetof(CHARSET_INFO, cs_name.str), DEFAULT(&default_charset_info), @@ -838,7 +838,7 @@ static bool check_charset_db(sys_var *self, THD *thd, set_var *var) var->save_result.ptr= thd->db_charset; return false; } -static Sys_var_struct Sys_character_set_database( +static Sys_var_charset Sys_character_set_database( "character_set_database", "The character set used by the default database", SESSION_VAR(collation_database), NO_CMD_LINE, @@ -862,7 +862,8 @@ static bool fix_thd_charset(sys_var *self, THD *thd, enum_var_type type) thd->update_charset(); return false; } -static Sys_var_struct Sys_character_set_client( + +static Sys_var_charset Sys_character_set_client( "character_set_client", "The character set for statements " "that arrive from the client", NO_SET_STMT SESSION_VAR(character_set_client), NO_CMD_LINE, @@ -872,7 +873,7 @@ static Sys_var_struct Sys_character_set_client( // for check changing export sys_var *Sys_character_set_client_ptr= &Sys_character_set_client; -static Sys_var_struct Sys_character_set_connection( +static Sys_var_charset Sys_character_set_connection( "character_set_connection", "The character set used for " "literals that do not have a character set introducer and for " "number-to-string conversion", @@ -883,7 +884,7 @@ static Sys_var_struct Sys_character_set_connection( // for check changing export sys_var *Sys_character_set_connection_ptr= &Sys_character_set_connection; -static Sys_var_struct Sys_character_set_results( +static Sys_var_charset Sys_character_set_results( "character_set_results", "The character set used for returning " "query results to the client", SESSION_VAR(character_set_results), NO_CMD_LINE, diff --git a/sql/sys_vars.inl b/sql/sys_vars.inl index 932b12fcb2e..655d72a6663 100644 --- a/sql/sys_vars.inl +++ b/sql/sys_vars.inl @@ -2166,6 +2166,31 @@ public: { return valptr(thd, *(uchar**)option.def_value); } }; + +/** + The class to store character sets. +*/ +class Sys_var_charset: public Sys_var_struct +{ +public: + using Sys_var_struct::Sys_var_struct; + void global_save_default(THD *thd, set_var *var) + { + /* + The default value can point to an arbitrary collation, + e.g. default_charset_info. + Let's convert it to the compiled default collation. + This makes the code easier in various places such as SET NAMES. + */ + void **default_value= reinterpret_cast(option.def_value); + var->save_result.ptr= + Lex_exact_charset_opt_extended_collate((CHARSET_INFO *) *default_value, + true). + find_default_collation(); + } +}; + + /** The class for variables that store time zones From 5e12d4920557ad7a678bdf41d6c0b4673fca838c Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 4 Jun 2024 15:06:37 +0400 Subject: [PATCH 09/59] MDEV-34295 CAST(char_col AS DOUBLE) prints redundant spaces in a warning Field_string::val_int(), Field_string::val_real(), Field_string::val_decimal() passed the whole buffer of field_length bytes to data type conversion routines. This made conversion routines to print redundant trailing spaces in case of warnings. Adding a method Field_string::to_lex_cstring() and using it inside val_int(), val_real(), val_decimal(), val_str(). After this change conversion routines get the same value with what val_str() returns, and no redundant trailing spaces are displayed. --- mysql-test/main/sp-vars.result | 2 +- mysql-test/main/type_char.result | 40 +++++++++++++++++++ mysql-test/main/type_char.test | 19 +++++++++ mysql-test/main/type_varchar.result | 8 ++-- mysql-test/main/xml.result | 12 +++--- .../suite/funcs_1/r/innodb_func_view.result | 16 ++++---- .../suite/funcs_1/r/memory_func_view.result | 16 ++++---- .../suite/funcs_1/r/myisam_func_view.result | 16 ++++---- sql/field.cc | 40 +++++++++++-------- sql/field.h | 1 + 10 files changed, 118 insertions(+), 52 deletions(-) create mode 100644 mysql-test/main/type_char.result create mode 100644 mysql-test/main/type_char.test diff --git a/mysql-test/main/sp-vars.result b/mysql-test/main/sp-vars.result index f0931ca48ff..e8e7b66b3ab 100644 --- a/mysql-test/main/sp-vars.result +++ b/mysql-test/main/sp-vars.result @@ -1171,7 +1171,7 @@ SET j= 1 + i; END| CALL ctest(); Warnings: -Warning 1292 Truncated incorrect DOUBLE value: 'string ' +Warning 1292 Truncated incorrect DOUBLE value: 'string' DROP PROCEDURE ctest; CREATE PROCEDURE vctest() BEGIN diff --git a/mysql-test/main/type_char.result b/mysql-test/main/type_char.result new file mode 100644 index 00000000000..07dfe7cdc29 --- /dev/null +++ b/mysql-test/main/type_char.result @@ -0,0 +1,40 @@ +# +# Start of 10.5 tests +# +# +# MDEV-34295 CAST(char_col AS DOUBLE) prints redundant spaces in a warning +# +CREATE TABLE t1 (a CHAR(5) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci); +INSERT INTO t1 VALUES ('1x'), ('x'); +SELECT a, CAST(a AS DOUBLE) FROM t1 ORDER BY a; +a CAST(a AS DOUBLE) +1x 1 +x 0 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: '1x' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +SELECT a, CAST(a AS DECIMAL(20,2)) FROM t1 ORDER BY a; +a CAST(a AS DECIMAL(20,2)) +1x 1.00 +x 0.00 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '1x' +Warning 1292 Truncated incorrect DECIMAL value: 'x' +SELECT a, CAST(a AS SIGNED) FROM t1 ORDER BY a; +a CAST(a AS SIGNED) +1x 1 +x 0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: '1x' +Warning 1292 Truncated incorrect INTEGER value: 'x' +SELECT a, CAST(a AS UNSIGNED) FROM t1 ORDER BY a; +a CAST(a AS UNSIGNED) +1x 1 +x 0 +Warnings: +Warning 1292 Truncated incorrect INTEGER value: '1x' +Warning 1292 Truncated incorrect INTEGER value: 'x' +DROP TABLE t1; +# +# End of 10.5 tests +# diff --git a/mysql-test/main/type_char.test b/mysql-test/main/type_char.test new file mode 100644 index 00000000000..053613c5dad --- /dev/null +++ b/mysql-test/main/type_char.test @@ -0,0 +1,19 @@ +--echo # +--echo # Start of 10.5 tests +--echo # + +--echo # +--echo # MDEV-34295 CAST(char_col AS DOUBLE) prints redundant spaces in a warning +--echo # + +CREATE TABLE t1 (a CHAR(5) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci); +INSERT INTO t1 VALUES ('1x'), ('x'); +SELECT a, CAST(a AS DOUBLE) FROM t1 ORDER BY a; +SELECT a, CAST(a AS DECIMAL(20,2)) FROM t1 ORDER BY a; +SELECT a, CAST(a AS SIGNED) FROM t1 ORDER BY a; +SELECT a, CAST(a AS UNSIGNED) FROM t1 ORDER BY a; +DROP TABLE t1; + +--echo # +--echo # End of 10.5 tests +--echo # diff --git a/mysql-test/main/type_varchar.result b/mysql-test/main/type_varchar.result index d00531018a4..6c6e95e3721 100644 --- a/mysql-test/main/type_varchar.result +++ b/mysql-test/main/type_varchar.result @@ -465,7 +465,7 @@ a (a + 0) t 0 Warnings: Warning 1292 Truncated incorrect DOUBLE value: '1a' -Warning 1292 Truncated incorrect DOUBLE value: 't ' +Warning 1292 Truncated incorrect DOUBLE value: 't' SELECT a,(a DIV 2) FROM t1 ORDER BY a; a (a DIV 2) 10 5 @@ -476,7 +476,7 @@ a (a DIV 2) t 0 Warnings: Warning 1292 Truncated incorrect DECIMAL value: '1a' -Warning 1292 Truncated incorrect DECIMAL value: 't ' +Warning 1292 Truncated incorrect DECIMAL value: 't' SELECT a,CAST(a AS SIGNED) FROM t1 ORDER BY a; a CAST(a AS SIGNED) 10 10 @@ -508,8 +508,8 @@ SELECT 5 = a FROM t1; 0 0 Warnings: -Warning 1292 Truncated incorrect DECIMAL value: 's ' -Warning 1292 Truncated incorrect DECIMAL value: ' ' +Warning 1292 Truncated incorrect DECIMAL value: 's' +Warning 1292 Truncated incorrect DECIMAL value: '' DROP TABLE t1; # # MDEV-13530 VARBINARY doesn't convert to to BLOB for sizes 65533, 65534 and 65535 diff --git a/mysql-test/main/xml.result b/mysql-test/main/xml.result index efaca961b4a..d0acb0debf3 100644 --- a/mysql-test/main/xml.result +++ b/mysql-test/main/xml.result @@ -991,20 +991,20 @@ CALL spxml('b1b2', '1 and string'); ExtractValue(xml,'/a/b[$i]') b1 Warnings: -Warning 1292 Truncated incorrect INTEGER value: '1 and string ' -Warning 1292 Truncated incorrect INTEGER value: '1 and string ' +Warning 1292 Truncated incorrect INTEGER value: '1 and string' +Warning 1292 Truncated incorrect INTEGER value: '1 and string' CALL spxml('b1b2', 'string and 1'); ExtractValue(xml,'/a/b[$i]') Warnings: -Warning 1292 Truncated incorrect INTEGER value: 'string and 1 ' -Warning 1292 Truncated incorrect INTEGER value: 'string and 1 ' +Warning 1292 Truncated incorrect INTEGER value: 'string and 1' +Warning 1292 Truncated incorrect INTEGER value: 'string and 1' CALL spxml('b1b2', 'string'); ExtractValue(xml,'/a/b[$i]') Warnings: -Warning 1292 Truncated incorrect INTEGER value: 'string ' -Warning 1292 Truncated incorrect INTEGER value: 'string ' +Warning 1292 Truncated incorrect INTEGER value: 'string' +Warning 1292 Truncated incorrect INTEGER value: 'string' DROP PROCEDURE spxml; select UpdateXML('a',repeat('a b ',1000),''); ERROR HY000: XPATH syntax error: 'b a b a b a b a b a b a b a b...' diff --git a/mysql-test/suite/funcs_1/r/innodb_func_view.result b/mysql-test/suite/funcs_1/r/innodb_func_view.result index e9fa172c321..90f2f3d3dc0 100644 --- a/mysql-test/suite/funcs_1/r/innodb_func_view.result +++ b/mysql-test/suite/funcs_1/r/innodb_func_view.result @@ -2202,9 +2202,9 @@ IS NOT TRUE <--------30 characters-------> 3 IS NOT TRUE ---äÖüß@µ*$-- 4 IS TRUE -1 5 Warnings: -Warning 1292 Truncated incorrect DOUBLE value: ' ' +Warning 1292 Truncated incorrect DOUBLE value: '' Warning 1292 Truncated incorrect DOUBLE value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$--' SHOW CREATE VIEW v1; View Create View character_set_client collation_connection v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select if(`t1_values`.`my_char_30`,'IS TRUE','IS NOT TRUE') AS `IF(my_char_30, 'IS TRUE', 'IS NOT TRUE')`,`t1_values`.`my_char_30` AS `my_char_30`,`t1_values`.`id` AS `id` from `t1_values` latin1 latin1_swedish_ci @@ -2218,9 +2218,9 @@ IS NOT TRUE <--------30 characters-------> 3 IS NOT TRUE ---äÖüß@µ*$-- 4 IS TRUE -1 5 Warnings: -Warning 1292 Truncated incorrect DOUBLE value: ' ' +Warning 1292 Truncated incorrect DOUBLE value: '' Warning 1292 Truncated incorrect DOUBLE value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$--' DROP VIEW v1; @@ -3523,9 +3523,9 @@ NULL NULL 1 -1.00 -1 5 -3333.33 -3333.3333 26 Warnings: -Warning 1292 Truncated incorrect DECIMAL value: ' ' +Warning 1292 Truncated incorrect DECIMAL value: '' Warning 1292 Truncated incorrect DECIMAL value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$--' SHOW CREATE VIEW v1; View Create View character_set_client collation_connection v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select cast(`t1_values`.`my_char_30` as decimal(37,2)) AS `CAST(my_char_30 AS DECIMAL(37,2))`,`t1_values`.`my_char_30` AS `my_char_30`,`t1_values`.`id` AS `id` from `t1_values` latin1 latin1_swedish_ci @@ -3540,9 +3540,9 @@ NULL NULL 1 -1.00 -1 5 -3333.33 -3333.3333 26 Warnings: -Warning 1292 Truncated incorrect DECIMAL value: ' ' +Warning 1292 Truncated incorrect DECIMAL value: '' Warning 1292 Truncated incorrect DECIMAL value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$--' DROP VIEW v1; diff --git a/mysql-test/suite/funcs_1/r/memory_func_view.result b/mysql-test/suite/funcs_1/r/memory_func_view.result index 744166bb849..74b6206959a 100644 --- a/mysql-test/suite/funcs_1/r/memory_func_view.result +++ b/mysql-test/suite/funcs_1/r/memory_func_view.result @@ -2203,9 +2203,9 @@ IS NOT TRUE <--------30 characters-------> 3 IS NOT TRUE ---äÖüß@µ*$-- 4 IS TRUE -1 5 Warnings: -Warning 1292 Truncated incorrect DOUBLE value: ' ' +Warning 1292 Truncated incorrect DOUBLE value: '' Warning 1292 Truncated incorrect DOUBLE value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$--' SHOW CREATE VIEW v1; View Create View character_set_client collation_connection v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select if(`t1_values`.`my_char_30`,'IS TRUE','IS NOT TRUE') AS `IF(my_char_30, 'IS TRUE', 'IS NOT TRUE')`,`t1_values`.`my_char_30` AS `my_char_30`,`t1_values`.`id` AS `id` from `t1_values` latin1 latin1_swedish_ci @@ -2219,9 +2219,9 @@ IS NOT TRUE <--------30 characters-------> 3 IS NOT TRUE ---äÖüß@µ*$-- 4 IS TRUE -1 5 Warnings: -Warning 1292 Truncated incorrect DOUBLE value: ' ' +Warning 1292 Truncated incorrect DOUBLE value: '' Warning 1292 Truncated incorrect DOUBLE value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$--' DROP VIEW v1; @@ -3524,9 +3524,9 @@ NULL NULL 1 -1.00 -1 5 -3333.33 -3333.3333 26 Warnings: -Warning 1292 Truncated incorrect DECIMAL value: ' ' +Warning 1292 Truncated incorrect DECIMAL value: '' Warning 1292 Truncated incorrect DECIMAL value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$--' SHOW CREATE VIEW v1; View Create View character_set_client collation_connection v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select cast(`t1_values`.`my_char_30` as decimal(37,2)) AS `CAST(my_char_30 AS DECIMAL(37,2))`,`t1_values`.`my_char_30` AS `my_char_30`,`t1_values`.`id` AS `id` from `t1_values` latin1 latin1_swedish_ci @@ -3541,9 +3541,9 @@ NULL NULL 1 -1.00 -1 5 -3333.33 -3333.3333 26 Warnings: -Warning 1292 Truncated incorrect DECIMAL value: ' ' +Warning 1292 Truncated incorrect DECIMAL value: '' Warning 1292 Truncated incorrect DECIMAL value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$--' DROP VIEW v1; diff --git a/mysql-test/suite/funcs_1/r/myisam_func_view.result b/mysql-test/suite/funcs_1/r/myisam_func_view.result index 744166bb849..74b6206959a 100644 --- a/mysql-test/suite/funcs_1/r/myisam_func_view.result +++ b/mysql-test/suite/funcs_1/r/myisam_func_view.result @@ -2203,9 +2203,9 @@ IS NOT TRUE <--------30 characters-------> 3 IS NOT TRUE ---äÖüß@µ*$-- 4 IS TRUE -1 5 Warnings: -Warning 1292 Truncated incorrect DOUBLE value: ' ' +Warning 1292 Truncated incorrect DOUBLE value: '' Warning 1292 Truncated incorrect DOUBLE value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$--' SHOW CREATE VIEW v1; View Create View character_set_client collation_connection v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select if(`t1_values`.`my_char_30`,'IS TRUE','IS NOT TRUE') AS `IF(my_char_30, 'IS TRUE', 'IS NOT TRUE')`,`t1_values`.`my_char_30` AS `my_char_30`,`t1_values`.`id` AS `id` from `t1_values` latin1 latin1_swedish_ci @@ -2219,9 +2219,9 @@ IS NOT TRUE <--------30 characters-------> 3 IS NOT TRUE ---äÖüß@µ*$-- 4 IS TRUE -1 5 Warnings: -Warning 1292 Truncated incorrect DOUBLE value: ' ' +Warning 1292 Truncated incorrect DOUBLE value: '' Warning 1292 Truncated incorrect DOUBLE value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DOUBLE value: ' ---äÖüß@µ*$--' DROP VIEW v1; @@ -3524,9 +3524,9 @@ NULL NULL 1 -1.00 -1 5 -3333.33 -3333.3333 26 Warnings: -Warning 1292 Truncated incorrect DECIMAL value: ' ' +Warning 1292 Truncated incorrect DECIMAL value: '' Warning 1292 Truncated incorrect DECIMAL value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$--' SHOW CREATE VIEW v1; View Create View character_set_client collation_connection v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select cast(`t1_values`.`my_char_30` as decimal(37,2)) AS `CAST(my_char_30 AS DECIMAL(37,2))`,`t1_values`.`my_char_30` AS `my_char_30`,`t1_values`.`id` AS `id` from `t1_values` latin1 latin1_swedish_ci @@ -3541,9 +3541,9 @@ NULL NULL 1 -1.00 -1 5 -3333.33 -3333.3333 26 Warnings: -Warning 1292 Truncated incorrect DECIMAL value: ' ' +Warning 1292 Truncated incorrect DECIMAL value: '' Warning 1292 Truncated incorrect DECIMAL value: '<--------30 characters------->' -Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$-- ' +Warning 1292 Truncated incorrect DECIMAL value: ' ---äÖüß@µ*$--' DROP VIEW v1; diff --git a/sql/field.cc b/sql/field.cc index f03135a63bc..a423fdd0fa3 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7485,11 +7485,11 @@ double Field_string::val_real(void) { DBUG_ASSERT(marked_for_read()); THD *thd= get_thd(); - return Converter_strntod_with_warn(get_thd(), + const LEX_CSTRING str= to_lex_cstring(); + return Converter_strntod_with_warn(thd, Warn_filter_string(thd, this), Field_string::charset(), - (const char *) ptr, - field_length).result(); + str.str, str.length).result(); } @@ -7497,10 +7497,10 @@ longlong Field_string::val_int(void) { DBUG_ASSERT(marked_for_read()); THD *thd= get_thd(); + const LEX_CSTRING str= to_lex_cstring(); return Converter_strntoll_with_warn(thd, Warn_filter_string(thd, this), Field_string::charset(), - (const char *) ptr, - field_length).result(); + str.str, str.length).result(); } @@ -7516,20 +7516,26 @@ sql_mode_t Field_string::can_handle_sql_mode_dependency_on_store() const } -String *Field_string::val_str(String *val_buffer __attribute__((unused)), - String *val_ptr) +LEX_CSTRING Field_string::to_lex_cstring() const { DBUG_ASSERT(marked_for_read()); /* See the comment for Field_long::store(long long) */ DBUG_ASSERT(!table || table->in_use == current_thd); - size_t length; - if (get_thd()->variables.sql_mode & - MODE_PAD_CHAR_TO_FULL_LENGTH) - length= field_charset()->charpos(ptr, ptr + field_length, - Field_string::char_length()); - else - length= field_charset()->lengthsp((const char*) ptr, field_length); - val_ptr->set((const char*) ptr, length, field_charset()); + if (get_thd()->variables.sql_mode & MODE_PAD_CHAR_TO_FULL_LENGTH) + return Lex_cstring((const char*) ptr, + field_charset()->charpos(ptr, ptr + field_length, + Field_string::char_length())); + return Lex_cstring((const char *) ptr, + field_charset()->lengthsp((const char*) ptr, field_length)); +} + + +String *Field_string::val_str(String *val_buffer __attribute__((unused)), + String *val_ptr) +{ + DBUG_ASSERT(marked_for_read()); + const LEX_CSTRING str= to_lex_cstring(); + val_ptr->set(str.str, str.length, field_charset()); return val_ptr; } @@ -7538,12 +7544,12 @@ my_decimal *Field_string::val_decimal(my_decimal *decimal_value) { DBUG_ASSERT(marked_for_read()); THD *thd= get_thd(); + const LEX_CSTRING str= to_lex_cstring(); Converter_str2my_decimal_with_warn(thd, Warn_filter_string(thd, this), E_DEC_FATAL_ERROR & ~E_DEC_BAD_NUM, Field_string::charset(), - (const char *) ptr, - field_length, decimal_value); + str.str, str.length, decimal_value); return decimal_value; } diff --git a/sql/field.h b/sql/field.h index 3da30453e1b..2eafb471d4b 100644 --- a/sql/field.h +++ b/sql/field.h @@ -4013,6 +4013,7 @@ class Field_string final :public Field_longstr { field_length >= 4 && orig_table->s->frm_version < FRM_VER_TRUE_VARCHAR; } + LEX_CSTRING to_lex_cstring() const; public: bool can_alter_field_type; Field_string(uchar *ptr_arg, uint32 len_arg,uchar *null_ptr_arg, From e9f4b87e530bc3806062c2b91a25052edd2c4325 Mon Sep 17 00:00:00 2001 From: Tuukka Pasanen Date: Tue, 16 Apr 2024 10:44:00 +0300 Subject: [PATCH 10/59] MDEV-33919: Remove less standard format directive an-trap Few man pages have less standard format directive: .it 1 an-trap which specifying a formatting instruction related to indentation (adds tab in man page in this) There is no traces what an-trap should do and removing it does not affect rendering of man page --- man/myisamchk.1 | 4 ++-- man/mysql_upgrade.1 | 4 ++-- man/mysqladmin.1 | 2 +- man/mysqlbinlog.1 | 8 ++++---- man/mysqlcheck.1 | 2 +- man/mysqld_safe.1 | 2 +- man/mysqldump.1 | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/man/myisamchk.1 b/man/myisamchk.1 index 456ffbbbf5c..612f48d513a 100644 --- a/man/myisamchk.1 +++ b/man/myisamchk.1 @@ -38,7 +38,7 @@ with partitioned tables is not supported\&. .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br @@ -168,7 +168,7 @@ the section called \(lqMYISAMCHK MEMORY USAGE\(rq\&. .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br diff --git a/man/mysql_upgrade.1 b/man/mysql_upgrade.1 index 3014ed52df5..312dcd04693 100644 --- a/man/mysql_upgrade.1 +++ b/man/mysql_upgrade.1 @@ -36,7 +36,7 @@ performs a table check\&. If any problems are found, a table repair is attempted .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br @@ -54,7 +54,7 @@ with administrator privileges\&. You can do this by running a Command Prompt as .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br diff --git a/man/mysqladmin.1 b/man/mysqladmin.1 index 02cf8b275a3..7756d4347d7 100644 --- a/man/mysqladmin.1 +++ b/man/mysqladmin.1 @@ -396,7 +396,7 @@ shell> \fBmysqladmin password "my new password"\fR .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br diff --git a/man/mysqlbinlog.1 b/man/mysqlbinlog.1 index 989749548b0..ca41a7b768a 100644 --- a/man/mysqlbinlog.1 +++ b/man/mysqlbinlog.1 @@ -167,7 +167,7 @@ option is given\&. .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br @@ -424,7 +424,7 @@ USE\&. (In particular, no cross\-database updates should be used\&.) .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br @@ -1307,7 +1307,7 @@ capability enabled\&. .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br @@ -1981,7 +1981,7 @@ shell> \fBmysqlbinlog \-v \-\-base64\-output=DECODE\-ROWS \fR\fB\fIlog_file\fR\f .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br diff --git a/man/mysqlcheck.1 b/man/mysqlcheck.1 index 2119e632a4b..a5606080ba3 100644 --- a/man/mysqlcheck.1 +++ b/man/mysqlcheck.1 @@ -101,7 +101,7 @@ with partitioned tables is not supported\&. .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br diff --git a/man/mysqld_safe.1 b/man/mysqld_safe.1 index 06e3dbdee0b..56ed00d4360 100644 --- a/man/mysqld_safe.1 +++ b/man/mysqld_safe.1 @@ -733,7 +733,7 @@ If none of these options is given, the default is .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br diff --git a/man/mysqldump.1 b/man/mysqldump.1 index c532228c652..f23c7e2491b 100644 --- a/man/mysqldump.1 +++ b/man/mysqldump.1 @@ -2345,7 +2345,7 @@ file that contains its data\&. The option value is the directory in which to wri .sp .\} .RS 4 -.it 1 an-trap +.it 1 .nr an-no-space-flag 1 .nr an-break-flag 1 .br From 1bf0950b191aaaa181068551fa241c9d09e6c7c3 Mon Sep 17 00:00:00 2001 From: Tuukka Pasanen Date: Mon, 13 May 2024 12:08:22 +0300 Subject: [PATCH 11/59] MDEV-34146: Remove duplicate #DEBHELPER# from MariaDB server postrm Moving to use Debian systemd install and uninstall scripts caused duplicated DEBHELP to server postrm script. Commit removes unneeded and makes postrm work better and pass lintian tests --- debian/mariadb-server.postrm | 2 -- 1 file changed, 2 deletions(-) diff --git a/debian/mariadb-server.postrm b/debian/mariadb-server.postrm index 841be00b6c6..601fe87f7e9 100644 --- a/debian/mariadb-server.postrm +++ b/debian/mariadb-server.postrm @@ -12,8 +12,6 @@ fi ${DEBIAN_SCRIPT_TRACE:+ echo "#42#DEBUG# RUNNING $0 $*" 1>&2 } -#DEBHELPER# - # # - Purge logs and data only if they are ours (#307473) # - Remove the mysql user only after all his owned files are purged. From 042a0d85ad4f96229ab440cadb4ad3842c928c47 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Thu, 9 May 2024 10:28:55 +1000 Subject: [PATCH 12/59] MDEV-27186 spider/partition: Report error on info() failure Like MDEV-28105, spider may attempt to connect to remote server in info(), and it may emit an error upon failure to connect. In this case, the downstream caller ha_partition::open() should return the error to avoid inconsistency. This fixes MDEV-27186, MDEV-27237, MDEV-27334, MDEV-28241, MDEV-34101. --- sql/ha_partition.cc | 3 +- .../spider/bugfix/r/mdev_27186.result | 36 ++++++++++++++++++ .../mysql-test/spider/bugfix/t/mdev_27186.opt | 1 + .../spider/bugfix/t/mdev_27186.test | 37 +++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_27186.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_27186.opt create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_27186.test diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 735d5c4442d..0d2abc7d860 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -3854,7 +3854,8 @@ int ha_partition::open(const char *name, int mode, uint test_if_locked) m_part_info->part_expr->get_monotonicity_info(); else if (m_part_info->list_of_part_fields) m_part_func_monotonicity_info= MONOTONIC_STRICT_INCREASING; - info(HA_STATUS_VARIABLE | HA_STATUS_CONST | HA_STATUS_OPEN); + if ((error= info(HA_STATUS_VARIABLE | HA_STATUS_CONST | HA_STATUS_OPEN))) + goto err_handler; DBUG_RETURN(0); err_handler: diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_27186.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_27186.result new file mode 100644 index 00000000000..2b88b5da34c --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_27186.result @@ -0,0 +1,36 @@ +INSTALL SONAME 'ha_spider'; +Warnings: +Warning 1105 Cannot enable tc-log at run-time. XA features of SPIDER are disabled +CREATE TABLE t (s INT) ENGINE=SPIDER PARTITION BY HASH (s); +LOAD INDEX INTO CACHE t PARTITION (p,p1); +Table Op Msg_type Msg_text +test.t preload_keys Error Unable to connect to foreign data source: localhost +test.t preload_keys Error Unable to connect to foreign data source: localhost +test.t preload_keys Error Unable to connect to foreign data source: localhost +test.t preload_keys Error Unable to connect to foreign data source: localhost +test.t preload_keys Error Unable to connect to foreign data source: localhost +test.t preload_keys Error Unable to connect to foreign data source: localhost +test.t preload_keys error Corrupt +DROP TABLE t; +CREATE TABLE t (c INT PRIMARY KEY) ENGINE=SPIDER PARTITION BY KEY() PARTITIONS 2; +HANDLER t OPEN AS h; +ERROR HY000: Unable to connect to foreign data source: localhost +DROP TABLE t; +CREATE TABLE t (c INT PRIMARY KEY) ENGINE=SPIDER PARTITION BY HASH (c) PARTITIONS 2; +CREATE TRIGGER t AFTER INSERT ON t FOR EACH ROW INSERT INTO t VALUES(0); +ERROR HY000: Unable to connect to foreign data source: localhost +DROP TABLE t; +CREATE TABLE t (b INT) ENGINE=InnoDB; +PREPARE s FROM 'SELECT * FROM t LIMIT 2'; +DROP TABLE t; +CREATE TABLE t (a INT) ENGINE=Spider PARTITION BY LIST (a) PARTITIONS 2 (PARTITION p1 VALUES IN (0,1),PARTITION p2 VALUES IN (2,3)); +EXECUTE s; +ERROR HY000: Unable to connect to foreign data source: localhost +DROP TABLE t; +CREATE TABLE t (c INT) ENGINE=InnoDB; +LOCK TABLES t WRITE; +CREATE OR REPLACE TABLE t (d INT) ENGINE=Spider PARTITION BY LIST COLUMNS (d) (PARTITION p VALUES IN (0)); +ERROR HY000: Unable to connect to foreign data source: localhost +drop table t; +Warnings: +Warning 1620 Plugin is busy and will be uninstalled on shutdown diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_27186.opt b/storage/spider/mysql-test/spider/bugfix/t/mdev_27186.opt new file mode 100644 index 00000000000..789275fa25e --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_27186.opt @@ -0,0 +1 @@ +--skip-log-bin diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_27186.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_27186.test new file mode 100644 index 00000000000..67c038009ba --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_27186.test @@ -0,0 +1,37 @@ +INSTALL SONAME 'ha_spider'; + +# MDEV-27186 +CREATE TABLE t (s INT) ENGINE=SPIDER PARTITION BY HASH (s); +LOAD INDEX INTO CACHE t PARTITION (p,p1); +DROP TABLE t; + +# MDEV-27237 +CREATE TABLE t (c INT PRIMARY KEY) ENGINE=SPIDER PARTITION BY KEY() PARTITIONS 2; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +HANDLER t OPEN AS h; +DROP TABLE t; + +# MDEV-27334 +CREATE TABLE t (c INT PRIMARY KEY) ENGINE=SPIDER PARTITION BY HASH (c) PARTITIONS 2; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +CREATE TRIGGER t AFTER INSERT ON t FOR EACH ROW INSERT INTO t VALUES(0); +DROP TABLE t; + +# MDEV-28241 +CREATE TABLE t (b INT) ENGINE=InnoDB; +PREPARE s FROM 'SELECT * FROM t LIMIT 2'; +DROP TABLE t; +CREATE TABLE t (a INT) ENGINE=Spider PARTITION BY LIST (a) PARTITIONS 2 (PARTITION p1 VALUES IN (0,1),PARTITION p2 VALUES IN (2,3)); +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +EXECUTE s; +DROP TABLE t; + +# MDEV-34101 +CREATE TABLE t (c INT) ENGINE=InnoDB; +LOCK TABLES t WRITE; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +CREATE OR REPLACE TABLE t (d INT) ENGINE=Spider PARTITION BY LIST COLUMNS (d) (PARTITION p VALUES IN (0)); +drop table t; + +--disable_query_log +--source ../../include/clean_up_spider.inc From 4d38267fc7e886fc36fff66d14877c3f35f10ae4 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Sat, 1 Jun 2024 01:11:40 -0700 Subject: [PATCH 13/59] MDEV-29307 Wrong result when joining two derived tables over the same view This bug could affect queries containing a join of derived tables over grouping views such that one of the derived tables contains a window function while another uses view V with dependent subquery DSQ containing a set function aggregated outside of the subquery in the view V. The subquery also refers to the fields from the group clause of the view.Due to this bug execution of such queries could produce wrong result sets. When the fix_fields() method performs context analysis of a set function AF first, at the very beginning the function Item_sum::init_sum_func_check() is called. The function copies the pointer to the embedding set function, if any, stored in THD::LEX::in_sum_func into the corresponding field of the set function AF simultaneously changing the value of THD::LEX::in_sum_func to point to AF. When at the very end of the fix_fields() method the function Item_sum::check_sum_func() is called it is supposed to restore the value of THD::LEX::in_sum_func to point to the embedding set function. And in fact Item_sum::check_sum_func() did it, but only for regular set functions, not for those used in window functions. As a result after the context analysis of AF had finished THD::LEX::in_sum_func still pointed to AF. It confused the further context analysis. In particular it led to wrong resolution of Item_outer_ref objects in the fix_inner_refs() function. This wrong resolution forced reading the values of grouping fields referred in DSQ not from the temporary table used for aggregation from which they were supposed to be read, but from the table used as the source table for aggregation. This patch guarantees that the value of THD::LEX::in_sum_func is properly restored after the call of fix_fields() for any set function. --- mysql-test/main/win.result | 174 ++++++++++++++++++ mysql-test/main/win.test | 93 ++++++++++ .../encryption/r/tempfiles_encrypted.result | 174 ++++++++++++++++++ sql/item_sum.cc | 4 + 4 files changed, 445 insertions(+) diff --git a/mysql-test/main/win.result b/mysql-test/main/win.result index 96b13010516..668f7b793cb 100644 --- a/mysql-test/main/win.result +++ b/mysql-test/main/win.result @@ -4391,3 +4391,177 @@ row_number() OVER (order by a) 2 3 drop table t1; +# +# MDEV-29307: join of 2 derived tables over the same grouping view such +# that the first of the joined tables contains a window +# function and the view's specification contains a subquery +# with a set function aggregated on the top level +# +CREATE TABLE t1 ( +tst int NOT NULL, +flat tinyint unsigned NOT NULL, +type tinyint unsigned NOT NULL, +val int NOT NULL, +PRIMARY KEY (tst,flat,type) +) ENGINE=ARIA; +INSERT INTO t1 VALUES +(5, 20, 2, 100), +(7, 20, 2, 150), +(9, 20, 1, 200); +CREATE VIEW v1 AS ( +SELECT +flat, +type, +( SELECT val FROM t1 sw +WHERE sw.tst = MAX(w.tst) AND sw.flat = w.flat AND sw.type = w.type) +AS total +FROM t1 w +GROUP BY flat, type +); +EXPLAIN EXTENDED SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( +SELECT flat, type, total +FROM v1 +WHERE type = 1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 3 100.00 Using where +1 PRIMARY ref key0 key0 1 v1.flat 2 100.00 Using where +6 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using where; Using index; Using temporary; Using filesort +7 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +4 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using where; Using index; Using temporary; Using filesort +5 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +Warnings: +Note 1276 Field or reference 'test.w.tst' of SELECT #5 was resolved in SELECT #4 +Note 1981 Aggregate function 'max()' of SELECT #5 belongs to SELECT #4 +Note 1276 Field or reference 'test.w.flat' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.type' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.tst' of SELECT #7 was resolved in SELECT #6 +Note 1981 Aggregate function 'max()' of SELECT #7 belongs to SELECT #6 +Note 1276 Field or reference 'test.w.flat' of SELECT #7 was resolved in SELECT #6 +Note 1276 Field or reference 'test.w.type' of SELECT #7 was resolved in SELECT #6 +Note 1003 /* select#1 */ select `v1`.`total` AS `w2_total`,`v1`.`total` AS `w1_total` from `test`.`v1` join `test`.`v1` where `v1`.`flat` = `v1`.`flat` and `v1`.`type` = 2 and `v1`.`type` = 1 +SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( +SELECT flat, type, total +FROM v1 +WHERE type = 1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; +w2_total w1_total +150 200 +EXPLAIN EXTENDED SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( +SELECT flat, type, total, +COUNT(total) OVER (PARTITION BY type ORDER BY type) AS u +FROM v1 +WHERE type = 1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 3 100.00 Using where +1 PRIMARY ref key0 key0 1 w1.flat 2 100.00 Using where +6 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using where; Using index; Using temporary; Using filesort +7 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +2 DERIVED ALL NULL NULL NULL NULL 3 100.00 Using where; Using temporary +4 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using where; Using index; Using temporary; Using filesort +5 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +Warnings: +Note 1276 Field or reference 'test.w.tst' of SELECT #5 was resolved in SELECT #4 +Note 1981 Aggregate function 'max()' of SELECT #5 belongs to SELECT #4 +Note 1276 Field or reference 'test.w.flat' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.type' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.tst' of SELECT #7 was resolved in SELECT #6 +Note 1981 Aggregate function 'max()' of SELECT #7 belongs to SELECT #6 +Note 1276 Field or reference 'test.w.flat' of SELECT #7 was resolved in SELECT #6 +Note 1276 Field or reference 'test.w.type' of SELECT #7 was resolved in SELECT #6 +Note 1003 /* select#1 */ select `v1`.`total` AS `w2_total`,`w1`.`total` AS `w1_total` from (/* select#2 */ select `v1`.`flat` AS `flat`,`v1`.`type` AS `type`,`v1`.`total` AS `total`,count(`v1`.`total`) over ( partition by `v1`.`type` order by `v1`.`type`) AS `u` from `test`.`v1` where `v1`.`type` = 1) `w1` join `test`.`v1` where `v1`.`flat` = `w1`.`flat` and `v1`.`type` = 2 +SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( +SELECT flat, type, total, +COUNT(total) OVER (PARTITION BY type ORDER BY type) AS u +FROM v1 +WHERE type = 1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; +w2_total w1_total +150 200 +EXPLAIN EXTENDED SELECT w2.total AS w2_total, w1.total AS w1_total, u +FROM +( +SELECT flat, type, total, +COUNT(total) OVER (PARTITION BY flat ORDER BY flat) AS u +FROM v1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +) AS w2; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 3 100.00 +1 PRIMARY ALL NULL NULL NULL NULL 3 100.00 Using join buffer (flat, BNL join) +6 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using index; Using temporary; Using filesort +7 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +2 DERIVED ALL NULL NULL NULL NULL 3 100.00 Using temporary +4 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using index; Using temporary; Using filesort +5 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +Warnings: +Note 1276 Field or reference 'test.w.tst' of SELECT #5 was resolved in SELECT #4 +Note 1981 Aggregate function 'max()' of SELECT #5 belongs to SELECT #4 +Note 1276 Field or reference 'test.w.flat' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.type' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.tst' of SELECT #7 was resolved in SELECT #6 +Note 1981 Aggregate function 'max()' of SELECT #7 belongs to SELECT #6 +Note 1276 Field or reference 'test.w.flat' of SELECT #7 was resolved in SELECT #6 +Note 1276 Field or reference 'test.w.type' of SELECT #7 was resolved in SELECT #6 +Note 1003 /* select#1 */ select `v1`.`total` AS `w2_total`,`w1`.`total` AS `w1_total`,`w1`.`u` AS `u` from (/* select#2 */ select `v1`.`flat` AS `flat`,`v1`.`type` AS `type`,`v1`.`total` AS `total`,count(`v1`.`total`) over ( partition by `v1`.`flat` order by `v1`.`flat`) AS `u` from `test`.`v1`) `w1` join `test`.`v1` +SELECT w2.total AS w2_total, w1.total AS w1_total, u +FROM +( +SELECT flat, type, total, +COUNT(total) OVER (PARTITION BY flat ORDER BY flat) AS u +FROM v1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +) AS w2; +w2_total w1_total u +150 150 2 +150 200 2 +200 150 2 +200 200 2 +DROP VIEW v1; +DROP TABLE t1; +# End of 10.5 tests diff --git a/mysql-test/main/win.test b/mysql-test/main/win.test index cba6b0fd7af..363bf8f8ad9 100644 --- a/mysql-test/main/win.test +++ b/mysql-test/main/win.test @@ -2873,3 +2873,96 @@ create table t1 (a int); insert into t1 values (1),(2),(3); SELECT row_number() OVER (order by a) FROM t1 order by NAME_CONST('myname',NULL); drop table t1; + +--echo # +--echo # MDEV-29307: join of 2 derived tables over the same grouping view such +--echo # that the first of the joined tables contains a window +--echo # function and the view's specification contains a subquery +--echo # with a set function aggregated on the top level +--echo # + +CREATE TABLE t1 ( + tst int NOT NULL, + flat tinyint unsigned NOT NULL, + type tinyint unsigned NOT NULL, + val int NOT NULL, + PRIMARY KEY (tst,flat,type) +) ENGINE=ARIA; + +INSERT INTO t1 VALUES +(5, 20, 2, 100), +(7, 20, 2, 150), +(9, 20, 1, 200); + +CREATE VIEW v1 AS ( + SELECT + flat, + type, + ( SELECT val FROM t1 sw + WHERE sw.tst = MAX(w.tst) AND sw.flat = w.flat AND sw.type = w.type) + AS total + FROM t1 w + GROUP BY flat, type +); + +let $q1= +SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( + SELECT flat, type, total + FROM v1 + WHERE type = 1 +) AS w1 +JOIN +( + SELECT flat, type, total + FROM v1 + WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; + +eval EXPLAIN EXTENDED $q1; +eval $q1; + +let $q2= +SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( + SELECT flat, type, total, + COUNT(total) OVER (PARTITION BY type ORDER BY type) AS u + FROM v1 + WHERE type = 1 +) AS w1 +JOIN +( + SELECT flat, type, total + FROM v1 + WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; + +eval EXPLAIN EXTENDED $q2; +eval $q2; + +let $q3= +SELECT w2.total AS w2_total, w1.total AS w1_total, u +FROM +( + SELECT flat, type, total, + COUNT(total) OVER (PARTITION BY flat ORDER BY flat) AS u + FROM v1 +) AS w1 +JOIN +( + SELECT flat, type, total + FROM v1 +) AS w2; + +eval EXPLAIN EXTENDED $q3; +--sorted_result +eval $q3; + +DROP VIEW v1; +DROP TABLE t1; + +--echo # End of 10.5 tests diff --git a/mysql-test/suite/encryption/r/tempfiles_encrypted.result b/mysql-test/suite/encryption/r/tempfiles_encrypted.result index 5b9dab62c01..b913b295ad3 100644 --- a/mysql-test/suite/encryption/r/tempfiles_encrypted.result +++ b/mysql-test/suite/encryption/r/tempfiles_encrypted.result @@ -4398,6 +4398,180 @@ row_number() OVER (order by a) 3 drop table t1; # +# MDEV-29307: join of 2 derived tables over the same grouping view such +# that the first of the joined tables contains a window +# function and the view's specification contains a subquery +# with a set function aggregated on the top level +# +CREATE TABLE t1 ( +tst int NOT NULL, +flat tinyint unsigned NOT NULL, +type tinyint unsigned NOT NULL, +val int NOT NULL, +PRIMARY KEY (tst,flat,type) +) ENGINE=ARIA; +INSERT INTO t1 VALUES +(5, 20, 2, 100), +(7, 20, 2, 150), +(9, 20, 1, 200); +CREATE VIEW v1 AS ( +SELECT +flat, +type, +( SELECT val FROM t1 sw +WHERE sw.tst = MAX(w.tst) AND sw.flat = w.flat AND sw.type = w.type) +AS total +FROM t1 w +GROUP BY flat, type +); +EXPLAIN EXTENDED SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( +SELECT flat, type, total +FROM v1 +WHERE type = 1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 3 100.00 Using where +1 PRIMARY ref key0 key0 1 v1.flat 2 100.00 Using where +6 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using where; Using index; Using temporary; Using filesort +7 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +4 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using where; Using index; Using temporary; Using filesort +5 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +Warnings: +Note 1276 Field or reference 'test.w.tst' of SELECT #5 was resolved in SELECT #4 +Note 1981 Aggregate function 'max()' of SELECT #5 belongs to SELECT #4 +Note 1276 Field or reference 'test.w.flat' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.type' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.tst' of SELECT #7 was resolved in SELECT #6 +Note 1981 Aggregate function 'max()' of SELECT #7 belongs to SELECT #6 +Note 1276 Field or reference 'test.w.flat' of SELECT #7 was resolved in SELECT #6 +Note 1276 Field or reference 'test.w.type' of SELECT #7 was resolved in SELECT #6 +Note 1003 /* select#1 */ select `v1`.`total` AS `w2_total`,`v1`.`total` AS `w1_total` from `test`.`v1` join `test`.`v1` where `v1`.`flat` = `v1`.`flat` and `v1`.`type` = 2 and `v1`.`type` = 1 +SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( +SELECT flat, type, total +FROM v1 +WHERE type = 1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; +w2_total w1_total +150 200 +EXPLAIN EXTENDED SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( +SELECT flat, type, total, +COUNT(total) OVER (PARTITION BY type ORDER BY type) AS u +FROM v1 +WHERE type = 1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 3 100.00 Using where +1 PRIMARY ref key0 key0 1 w1.flat 2 100.00 Using where +6 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using where; Using index; Using temporary; Using filesort +7 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +2 DERIVED ALL NULL NULL NULL NULL 3 100.00 Using where; Using temporary +4 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using where; Using index; Using temporary; Using filesort +5 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +Warnings: +Note 1276 Field or reference 'test.w.tst' of SELECT #5 was resolved in SELECT #4 +Note 1981 Aggregate function 'max()' of SELECT #5 belongs to SELECT #4 +Note 1276 Field or reference 'test.w.flat' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.type' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.tst' of SELECT #7 was resolved in SELECT #6 +Note 1981 Aggregate function 'max()' of SELECT #7 belongs to SELECT #6 +Note 1276 Field or reference 'test.w.flat' of SELECT #7 was resolved in SELECT #6 +Note 1276 Field or reference 'test.w.type' of SELECT #7 was resolved in SELECT #6 +Note 1003 /* select#1 */ select `v1`.`total` AS `w2_total`,`w1`.`total` AS `w1_total` from (/* select#2 */ select `v1`.`flat` AS `flat`,`v1`.`type` AS `type`,`v1`.`total` AS `total`,count(`v1`.`total`) over ( partition by `v1`.`type` order by `v1`.`type`) AS `u` from `test`.`v1` where `v1`.`type` = 1) `w1` join `test`.`v1` where `v1`.`flat` = `w1`.`flat` and `v1`.`type` = 2 +SELECT w2.total AS w2_total, w1.total AS w1_total +FROM +( +SELECT flat, type, total, +COUNT(total) OVER (PARTITION BY type ORDER BY type) AS u +FROM v1 +WHERE type = 1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +WHERE type = 2 +) AS w2 +ON w1.flat = w2.flat; +w2_total w1_total +150 200 +EXPLAIN EXTENDED SELECT w2.total AS w2_total, w1.total AS w1_total, u +FROM +( +SELECT flat, type, total, +COUNT(total) OVER (PARTITION BY flat ORDER BY flat) AS u +FROM v1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +) AS w2; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 3 100.00 +1 PRIMARY ALL NULL NULL NULL NULL 3 100.00 Using join buffer (flat, BNL join) +6 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using index; Using temporary; Using filesort +7 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +2 DERIVED ALL NULL NULL NULL NULL 3 100.00 Using temporary +4 DERIVED w index NULL PRIMARY 6 NULL 3 100.00 Using index; Using temporary; Using filesort +5 DEPENDENT SUBQUERY sw eq_ref PRIMARY PRIMARY 6 func,func,func 1 100.00 Using index condition +Warnings: +Note 1276 Field or reference 'test.w.tst' of SELECT #5 was resolved in SELECT #4 +Note 1981 Aggregate function 'max()' of SELECT #5 belongs to SELECT #4 +Note 1276 Field or reference 'test.w.flat' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.type' of SELECT #5 was resolved in SELECT #4 +Note 1276 Field or reference 'test.w.tst' of SELECT #7 was resolved in SELECT #6 +Note 1981 Aggregate function 'max()' of SELECT #7 belongs to SELECT #6 +Note 1276 Field or reference 'test.w.flat' of SELECT #7 was resolved in SELECT #6 +Note 1276 Field or reference 'test.w.type' of SELECT #7 was resolved in SELECT #6 +Note 1003 /* select#1 */ select `v1`.`total` AS `w2_total`,`w1`.`total` AS `w1_total`,`w1`.`u` AS `u` from (/* select#2 */ select `v1`.`flat` AS `flat`,`v1`.`type` AS `type`,`v1`.`total` AS `total`,count(`v1`.`total`) over ( partition by `v1`.`flat` order by `v1`.`flat`) AS `u` from `test`.`v1`) `w1` join `test`.`v1` +SELECT w2.total AS w2_total, w1.total AS w1_total, u +FROM +( +SELECT flat, type, total, +COUNT(total) OVER (PARTITION BY flat ORDER BY flat) AS u +FROM v1 +) AS w1 +JOIN +( +SELECT flat, type, total +FROM v1 +) AS w2; +w2_total w1_total u +150 150 2 +150 200 2 +200 150 2 +200 200 2 +DROP VIEW v1; +DROP TABLE t1; +# End of 10.5 tests +# # MDEV-23867: select crash in compute_window_func # set @save_sort_buffer_size=@@sort_buffer_size; diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 4cf403c1618..a7fae9ea4c0 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -177,7 +177,11 @@ bool Item_sum::check_sum_func(THD *thd, Item **ref) } if (window_func_sum_expr_flag) + { + thd->lex->in_sum_func= in_sum_func; return false; + } + /* The value of max_arg_level is updated if an argument of the set function contains a column reference resolved against a subquery whose level is From ecf4a261076c461e467c17d7664c11a885d2babb Mon Sep 17 00:00:00 2001 From: ilyasa1211 Date: Wed, 29 May 2024 13:59:46 +1000 Subject: [PATCH 14/59] Fix Indonesian month name. Noticed on MySQL: https://github.com/mysql/mysql-server/pull/531 Matches https://icu4c-demos.unicode.org/icu-bin/locexp?d_=en&_=in_IN. --- sql/sql_locale.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/sql_locale.cc b/sql/sql_locale.cc index 60e7abc3fa2..ba5c12153c7 100644 --- a/sql/sql_locale.cc +++ b/sql/sql_locale.cc @@ -968,9 +968,9 @@ MY_LOCALE my_locale_hu_HU /***** LOCALE BEGIN id_ID: Indonesian - Indonesia *****/ static const char *my_locale_month_names_id_ID[13] = - {"Januari","Pebruari","Maret","April","Mei","Juni","Juli","Agustus","September","Oktober","November","Desember", NullS }; + {"Januari","Februari","Maret","April","Mei","Juni","Juli","Agustus","September","Oktober","November","Desember", NullS }; static const char *my_locale_ab_month_names_id_ID[13] = - {"Jan","Peb","Mar","Apr","Mei","Jun","Jul","Agu","Sep","Okt","Nov","Des", NullS }; + {"Jan","Feb","Mar","Apr","Mei","Jun","Jul","Agu","Sep","Okt","Nov","Des", NullS }; static const char *my_locale_day_names_id_ID[8] = {"Senin","Selasa","Rabu","Kamis","Jumat","Sabtu","Minggu", NullS }; static const char *my_locale_ab_day_names_id_ID[8] = From c6d36c3e7c58a968cc0aaada0b5194cea8e85f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 5 Jun 2024 09:25:20 +0300 Subject: [PATCH 15/59] MDEV-34297 get_rnd_value() of ib_counter_t is unnecessarily complex The shared counter template ib_counter_t uses the function my_timer_cycles() as a source of pseudo-random numbers to pick a shard. On some platforms, my_timer_cycles() could return the constant value 0. get_rnd_value(): Remove. my_pseudo_random(): Implement as an alias of my_timer_cycles() or a wrapper for pthread_self(). Reviewed by: Vladislav Vaintroub --- include/my_rdtsc.h | 13 +++++++++++++ storage/innobase/include/sync0arr.inl | 2 +- storage/innobase/include/ut0counter.h | 26 +------------------------- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/include/my_rdtsc.h b/include/my_rdtsc.h index e81015166d2..9729a47d4a5 100644 --- a/include/my_rdtsc.h +++ b/include/my_rdtsc.h @@ -177,10 +177,23 @@ static inline ulonglong my_timer_cycles(void) /* gethrtime may appear as either cycle or nanosecond counter */ return (ulonglong) gethrtime(); #else +# define MY_TIMER_CYCLES_IS_ZERO return 0; #endif } +#ifdef MY_TIMER_CYCLES_IS_ZERO +static inline size_t my_pseudo_random(void) +{ + /* In some platforms, pthread_self() might return a structure + that cannot be converted to a number like this. Possible alternatives + could include gettid() or sched_getcpu(). */ + return ((size_t) pthread_self()) / 16; +} +#else +# define my_pseudo_random my_timer_cycles +#endif + /** A nanosecond timer. @return the current timer value, in nanoseconds. diff --git a/storage/innobase/include/sync0arr.inl b/storage/innobase/include/sync0arr.inl index 962226b4934..e5eb126aea2 100644 --- a/storage/innobase/include/sync0arr.inl +++ b/storage/innobase/include/sync0arr.inl @@ -44,7 +44,7 @@ sync_array_get() return(sync_wait_array[0]); } - return(sync_wait_array[get_rnd_value() % sync_array_size]); + return(sync_wait_array[my_pseudo_random() % sync_array_size]); } /******************************************************************//** diff --git a/storage/innobase/include/ut0counter.h b/storage/innobase/include/ut0counter.h index 646a5f367c2..0083627b91f 100644 --- a/storage/innobase/include/ut0counter.h +++ b/storage/innobase/include/ut0counter.h @@ -41,30 +41,6 @@ Created 2012/04/12 by Sunny Bains /** Default number of slots to use in ib_counter_t */ #define IB_N_SLOTS 64 -/** Use the result of my_timer_cycles(), which mainly uses RDTSC for cycles -as a random value. See the comments for my_timer_cycles() */ -/** @return result from RDTSC or similar functions. */ -static inline size_t -get_rnd_value() -{ - size_t c = static_cast(my_timer_cycles()); - - if (c != 0) { - return c; - } - - /* We may go here if my_timer_cycles() returns 0, - so we have to have the plan B for the counter. */ -#if !defined(_WIN32) - return (size_t)os_thread_get_curr_id(); -#else - LARGE_INTEGER cnt; - QueryPerformanceCounter(&cnt); - - return static_cast(cnt.QuadPart); -#endif /* !_WIN32 */ -} - /** Class for using fuzzy counters. The counter is multi-instance relaxed atomic so the results are not guaranteed to be 100% accurate but close enough. Creates an array of counters and separates each element by the @@ -80,7 +56,7 @@ struct ib_counter_t { /** Add to the counter. @param[in] n amount to be added */ - void add(Type n) { add(get_rnd_value(), n); } + void add(Type n) { add(my_pseudo_random(), n); } /** Add to the counter. @param[in] index a reasonably thread-unique identifier From 38cbef8b3f7fac55182009fd141175a062728657 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 5 Jun 2024 10:22:27 +0300 Subject: [PATCH 16/59] MDEV-22935 Erroneous Aria Index / Optimizer behaviour The problem was in the Aria part of the range optimizer, maria_records_in_range(), which wrong concluded that there was no rows in the range. This error would happen in the unlikely case when searching for a range on a partial key and there was a match for the first key part in the upper part of the b-tree (node) and also a match in the underlying node page. In other words, for this bug to happen one have to use Aria, have a multi part key with a lot of identical values for the first key part and do a range search on the second part of the key. Fixed by ensuring that we do not stop searching for partial keys found on node. Other things: - Added some comments - Changed a variable name to more clearly explain it's purpose. - Fixed wrong cast in _ma_record_pos() that could cause problems on 32 bit systems. --- mysql-test/suite/maria/range.result | 6 ++++++ mysql-test/suite/maria/range.test | 22 ++++++++++++++++++++++ storage/maria/ma_range.c | 15 ++++++++------- 3 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 mysql-test/suite/maria/range.result create mode 100644 mysql-test/suite/maria/range.test diff --git a/mysql-test/suite/maria/range.result b/mysql-test/suite/maria/range.result new file mode 100644 index 00000000000..49b8dc0fdc0 --- /dev/null +++ b/mysql-test/suite/maria/range.result @@ -0,0 +1,6 @@ +# +# MDEV-22935 Erroneous Aria Index / Optimizer behaviour +# +create table t1 (a char(255), b datetime, primary key(a,b)) engine=aria transactional=0 pack_keys=0; +insert into t1 select concat("hello world hello world", truncate(seq/100,0)),from_unixtime(seq+1) from seq_1_to_20000; +drop table t1; diff --git a/mysql-test/suite/maria/range.test b/mysql-test/suite/maria/range.test new file mode 100644 index 00000000000..00a74ad30c4 --- /dev/null +++ b/mysql-test/suite/maria/range.test @@ -0,0 +1,22 @@ +--source include/have_sequence.inc + +--echo # +--echo # MDEV-22935 Erroneous Aria Index / Optimizer behaviour +--echo # + +create table t1 (a char(255), b datetime, primary key(a,b)) engine=aria transactional=0 pack_keys=0; +insert into t1 select concat("hello world hello world", truncate(seq/100,0)),from_unixtime(seq+1) from seq_1_to_20000; + +let $i= 200; +--disable_query_log +while ($i) +{ + let $tmp= `select count(*) from t1 where a="hello world hello world$i" and b <= from_unixtime($i*100+1)`; + if (`SELECT $tmp != 1`) + { + --echo "Found $tmp rows, expected 1, for value $i" + } + dec $i; +} +--enable_query_log +drop table t1; diff --git a/storage/maria/ma_range.c b/storage/maria/ma_range.c index 442adc35858..525bf27b90c 100644 --- a/storage/maria/ma_range.c +++ b/storage/maria/ma_range.c @@ -191,8 +191,8 @@ static ha_rows _ma_record_pos(MARIA_HA *info, const uchar *key_data, info->s->state.key_root[inx], final_page); if (pos >= 0.0) { - DBUG_PRINT("exit",("pos: %ld",(ulong) (pos*info->state->records))); - DBUG_RETURN((ulong) (pos*info->state->records+0.5)); + DBUG_PRINT("exit",("pos: %lld",(longlong) (pos*info->state->records))); + DBUG_RETURN((ha_rows) (pos*info->state->records+0.5)); } DBUG_RETURN(HA_POS_ERROR); } @@ -214,7 +214,7 @@ static double _ma_search_pos(MARIA_HA *info, MARIA_KEY *key, { int flag; uint keynr, UNINIT_VAR(max_keynr); - my_bool after_key; + my_bool last_key_on_page; uchar *keypos; double offset; MARIA_KEYDEF *keyinfo= key->keyinfo; @@ -230,7 +230,7 @@ static double _ma_search_pos(MARIA_HA *info, MARIA_KEY *key, goto err; *final_page= pos; flag= (*keyinfo->bin_search)(key, &page, nextflag, &keypos, - info->lastkey_buff, &after_key); + info->lastkey_buff, &last_key_on_page); keynr= _ma_keynr(&page, keypos, &max_keynr); if (flag) @@ -274,7 +274,7 @@ static double _ma_search_pos(MARIA_HA *info, MARIA_KEY *key, There may be identical keys in the tree. Try to match on of those. Matches keynr + [0-1] */ - if ((offset= _ma_search_pos(info, key, SEARCH_FIND, + if ((offset= _ma_search_pos(info, key, nextflag, _ma_kpos(page.node,keypos), final_page)) < 0) DBUG_RETURN(offset); /* Read error */ @@ -290,9 +290,10 @@ err: /* - Get keynummer of current key and max number of keys in nod + Get keynumber of current key and max number of keys in nod - keynr >= 0 && key_nr <= max_key + @return key position on page (0 - (ret_max_key - 1)) + ret_max_key contains how many keys there was on the page */ static uint _ma_keynr(MARIA_PAGE *page, uchar *keypos, uint *ret_max_key) From b20481798659ac2b58f2479e9d23ce6926d81fc8 Mon Sep 17 00:00:00 2001 From: Tuukka Pasanen Date: Thu, 30 May 2024 10:45:58 +0300 Subject: [PATCH 17/59] MDEV-34261: Detect if build is running under 32-bit container When building on 64-bit kernel machine in 32-bit docker container CMake falsely (but it works as expected) detects that container runtime in also 64-bits. Use linux32 command to change runtime enviroment to 32-bit and then CMake will correctly disable for example ColumnStore and not try to build it This commit only works with debian/autobake-debs.sh --- debian/autobake-deb.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index a0c0f2d30de..3e8dcb68b72 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -62,6 +62,7 @@ replace_uring_with_aio() } architecture=$(dpkg-architecture -q DEB_BUILD_ARCH) +uname_machine=$(uname -m) # Parse release name and number from Linux standard base release # Example: @@ -161,6 +162,14 @@ then BUILDPACKAGE_DPKGCMD+=("eatmydata") fi +# If running autobake-debs.sh inside docker/podman host machine which +# has 64 bits cpu but container image is 32 bit make sure that we set +# correct arch with linux32 for 32 bit enviroment +if [ "$architecture" = "i386" ] && [ "$uname_machine" = "x86_64" ] +then + BUILDPACKAGE_DPKGCMD+=("linux32") +fi + BUILDPACKAGE_DPKGCMD+=("dpkg-buildpackage") # Using dpkg-buildpackage args From 40abd973ab3813a7f3ac1ae961ed094ab251f3cf Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 5 Jun 2024 11:54:34 +0200 Subject: [PATCH 18/59] MDEV-34236 Mroonga build with ASAN/UBSAN with GCC 12+ extremely slow. Workaround by disabling sanitizer for single source file. --- storage/mroonga/vendor/groonga/lib/CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt index 4f076458a36..1149458d523 100644 --- a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt @@ -188,3 +188,15 @@ IF(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64" AND CMAKE_C_COMPILER_VERSION VERSION_GREATER "5") ADD_COMPILE_FLAGS(ts/ts_expr_node.c COMPILE_FLAGS "-fno-tree-loop-vectorize") ENDIF() + +# Workaround long compile times with GCC and sanitizers +IF(CMAKE_C_COMPILER_ID STREQUAL "GNU" + AND CMAKE_C_COMPILER_VERSION VERSION_GREATER "11.99") + IF(WITH_UBSAN) + ADD_COMPILE_FLAGS(expr.c COMPILE_FLAGS "-fno-sanitize=undefined") + ENDIF() + IF(WITH_ASAN) + ADD_COMPILE_FLAGS(expr.c COMPILE_FLAGS "-fno-sanitize=address") + ENDIF() +ENDIF() + From b242b44f0ace0fdaf9f63861ca4ad72a36d35937 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 5 Jun 2024 12:13:33 +0200 Subject: [PATCH 19/59] Appveyor build - skip irrelevant commits Since we're only building on Windows, skip changes to debian directory and to shell scripts. --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index b0820784bd9..f7fc71d291d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,4 +27,9 @@ test_script: - set /A parallel=4*%NUMBER_OF_PROCESSORS% - perl mysql-test-run.pl --force --max-test-fail=10 --retry=2 --parallel=%parallel% --testcase-timeout=4 --suite=main --skip-test-list=unstable-tests --mysqld=--loose-innodb-flush-log-at-trx-commit=2 +skip_commits: + files: + - debian/ + - '**/*.sh' + image: Visual Studio 2022 From bfd3f45e8e041a650a32eeca08e506ed70f847a1 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 5 Jun 2024 12:26:46 +0200 Subject: [PATCH 20/59] Appveyor - better filtering for branches to match buildbot --- appveyor.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index f7fc71d291d..c28d9a4d5db 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -32,4 +32,9 @@ skip_commits: - debian/ - '**/*.sh' +branches: + only: + - /bb-/ + - /d+\.\d+/ + image: Visual Studio 2022 From db9c2d225e1085b5c23f7776ee6936be7952aeb6 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 5 Jun 2024 13:14:20 +0200 Subject: [PATCH 21/59] fix typo --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index c28d9a4d5db..de8f52577f7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -35,6 +35,6 @@ skip_commits: branches: only: - /bb-/ - - /d+\.\d+/ + - /\d+\.\d+$/ image: Visual Studio 2022 From b12c14e3b42c44ff3a48e82679c2fab8148b372f Mon Sep 17 00:00:00 2001 From: mariadb-DebarunBanerjee Date: Thu, 30 May 2024 17:14:01 +0530 Subject: [PATCH 22/59] MDEV-34265 Possible hang during IO burst with innodb_flush_sync enabled When checkpoint age goes beyond the sync flush threshold and buf_flush_sync_lsn is set, page cleaner enters into "furious flush" stage to aggressively flush dirty pages from flush list and pull checkpoint LSN above safe margin. In this stage, page cleaner skips doing LRU flush and eviction. In 10.6, all other threads entirely rely on page cleaner to generate free pages. If free pages get over while page cleaner is busy in "furious flush" stage, a session thread could wait for free page in the middle of a min-transaction(mtr) while holding latches on other pages. It, in turn, can prevent page cleaner to flush such pages preventing checkpoint LSN to move forward creating a deadlock situation. Even otherwise, it could create a stall and hang like situation for large BP with plenty of dirty pages to flush before the stage could finish. Fix: During furious flush, check and evict LRU pages after each flush iteration. --- .../suite/innodb/r/insert_into_empty.result | 13 ++ .../suite/innodb/t/insert_into_empty.test | 30 +++- storage/innobase/buf/buf0flu.cc | 136 +++++++++++------- storage/innobase/buf/buf0lru.cc | 15 ++ 4 files changed, 138 insertions(+), 56 deletions(-) diff --git a/mysql-test/suite/innodb/r/insert_into_empty.result b/mysql-test/suite/innodb/r/insert_into_empty.result index 2fdea1bce38..42385a833ce 100644 --- a/mysql-test/suite/innodb/r/insert_into_empty.result +++ b/mysql-test/suite/innodb/r/insert_into_empty.result @@ -268,4 +268,17 @@ PARTITION BY KEY(a) PARTITIONS 16; INSERT INTO t1 VALUES(1); UPDATE t1 SET a = 2 WHERE a = 1; DROP TABLE t1; +# +# MDEV-34265 Possible hang during IO burst with innodb_flush_sync enabled +# +CREATE TABLE t1(f1 MEDIUMTEXT)ENGINE=InnoDB; +SET @save_dbug=@@GLOBAL.debug_dbug; +SET @@GLOBAL.debug_dbug='+d,ib_page_cleaner_sleep'; +SET STATEMENT debug_dbug='+d,ib_free_page_sleep' FOR +INSERT INTO t1 VALUES(REPEAT(1, 8459264)); +SET @@GLOBAL.debug_dbug=@save_dbug; +SELECT length(f1) FROM t1; +length(f1) +8459264 +DROP TABLE t1; # End of 10.6 tests diff --git a/mysql-test/suite/innodb/t/insert_into_empty.test b/mysql-test/suite/innodb/t/insert_into_empty.test index aa9ef08b30d..d7f884ecc4d 100644 --- a/mysql-test/suite/innodb/t/insert_into_empty.test +++ b/mysql-test/suite/innodb/t/insert_into_empty.test @@ -163,9 +163,9 @@ DROP TABLE t1,t2; CREATE TABLE t1 (a INT KEY) ENGINE=InnoDB; ---error 0,1193 +--error 0,ER_UNKNOWN_SYSTEM_VARIABLE SET @save_limit = @@GLOBAL.innodb_limit_optimistic_insert_debug; ---error 0,1193 +--error 0,ER_UNKNOWN_SYSTEM_VARIABLE SET GLOBAL innodb_limit_optimistic_insert_debug = 2; BEGIN; @@ -289,4 +289,30 @@ CREATE TABLE t1(a INT KEY)ENGINE=InnoDB INSERT INTO t1 VALUES(1); UPDATE t1 SET a = 2 WHERE a = 1; DROP TABLE t1; + +--echo # +--echo # MDEV-34265 Possible hang during IO burst with innodb_flush_sync enabled +--echo # +CREATE TABLE t1(f1 MEDIUMTEXT)ENGINE=InnoDB; + +--error 0,ER_UNKNOWN_SYSTEM_VARIABLE +SET @save_dbug=@@GLOBAL.debug_dbug; +--error 0,ER_UNKNOWN_SYSTEM_VARIABLE +SET @@GLOBAL.debug_dbug='+d,ib_page_cleaner_sleep'; + +if ($have_debug) { + SET STATEMENT debug_dbug='+d,ib_free_page_sleep' FOR + INSERT INTO t1 VALUES(REPEAT(1, 8459264)); +} +if (!$have_debug) { + --echo SET STATEMENT debug_dbug='+d,ib_free_page_sleep' FOR + INSERT INTO t1 VALUES(REPEAT(1, 8459264)); +} + +--error 0,ER_UNKNOWN_SYSTEM_VARIABLE +SET @@GLOBAL.debug_dbug=@save_dbug; + +SELECT length(f1) FROM t1; +DROP TABLE t1; + --echo # End of 10.6 tests diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index fd1f2643324..46129a037b0 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -2025,70 +2025,94 @@ ATTRIBUTE_COLD void buf_flush_ahead(lsn_t lsn, bool furious) /** Conduct checkpoint-related flushing for innodb_flush_sync=ON, and try to initiate checkpoints until the target is met. @param lsn minimum value of buf_pool.get_oldest_modification(LSN_MAX) */ -ATTRIBUTE_COLD static void buf_flush_sync_for_checkpoint(lsn_t lsn) +ATTRIBUTE_COLD ATTRIBUTE_NOINLINE +static void buf_flush_sync_for_checkpoint(lsn_t lsn) { ut_ad(!srv_read_only_mode); mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex); - for (;;) + /* During furious flush, we need to keep generating free pages. Otherwise + concurrent mtrs could be blocked holding latches for the pages to be flushed + causing deadlock in rare occasion. + + Ideally we should be acquiring buffer pool mutex for the check but it is more + expensive and we are not using the mutex while calling need_LRU_eviction() as + of today. It is a quick and dirty read of the LRU and free list length. + Atomic read of try_LRU_scan should eventually let us do the eviction. + Correcting the inaccuracy would need more consideration to avoid any possible + performance regression. */ + if (buf_pool.need_LRU_eviction()) { - if (ulint n_flushed= buf_flush_list(srv_max_io_capacity, lsn)) - { - MONITOR_INC_VALUE_CUMULATIVE(MONITOR_FLUSH_SYNC_TOTAL_PAGE, - MONITOR_FLUSH_SYNC_COUNT, - MONITOR_FLUSH_SYNC_PAGES, n_flushed); - } - - switch (srv_file_flush_method) { - case SRV_NOSYNC: - case SRV_O_DIRECT_NO_FSYNC: - break; - default: - fil_flush_file_spaces(); - } - - mysql_mutex_lock(&log_sys.mutex); - const lsn_t newest_lsn= log_sys.get_lsn(); - mysql_mutex_lock(&log_sys.flush_order_mutex); mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t measure= buf_pool.get_oldest_modification(0); - mysql_mutex_unlock(&log_sys.flush_order_mutex); - const lsn_t checkpoint_lsn= measure ? measure : newest_lsn; - - if (!recv_recovery_is_on() && - checkpoint_lsn > log_sys.last_checkpoint_lsn + SIZE_OF_FILE_CHECKPOINT) - { - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - log_checkpoint_low(checkpoint_lsn, newest_lsn); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - measure= buf_pool.get_oldest_modification(LSN_MAX); - } - else - { - mysql_mutex_unlock(&log_sys.mutex); - if (!measure) - measure= LSN_MAX; - } - - mysql_mutex_assert_not_owner(&log_sys.mutex); - - /* After attempting log checkpoint, check if we have reached our target. */ - const lsn_t target= buf_flush_sync_lsn; - - if (measure >= target) - buf_flush_sync_lsn= 0; - else if (measure >= buf_flush_async_lsn) - buf_flush_async_lsn= 0; - - /* wake up buf_flush_wait() */ - pthread_cond_broadcast(&buf_pool.done_flush_list); + buf_pool.page_cleaner_set_idle(false); + buf_pool.n_flush_inc(); mysql_mutex_unlock(&buf_pool.flush_list_mutex); - lsn= std::max(lsn, target); + mysql_mutex_lock(&buf_pool.mutex); + /* Confirm that eviction is needed after acquiring buffer pool mutex. */ + if (buf_pool.need_LRU_eviction()) + /* We intend to only evict pages keeping maximum flush bandwidth for + flush list pages advancing checkpoint. However, if the LRU tail is full + of dirty pages, we might need some flushing. */ + std::ignore= buf_flush_LRU(srv_io_capacity); + mysql_mutex_unlock(&buf_pool.mutex); - if (measure >= lsn) - return; + mysql_mutex_lock(&buf_pool.flush_list_mutex); + buf_pool.n_flush_dec(); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); } + + if (ulint n_flushed= buf_flush_list(srv_max_io_capacity, lsn)) + { + MONITOR_INC_VALUE_CUMULATIVE(MONITOR_FLUSH_SYNC_TOTAL_PAGE, + MONITOR_FLUSH_SYNC_COUNT, + MONITOR_FLUSH_SYNC_PAGES, n_flushed); + } + + switch (srv_file_flush_method) { + case SRV_NOSYNC: + case SRV_O_DIRECT_NO_FSYNC: + break; + default: + fil_flush_file_spaces(); + } + + mysql_mutex_lock(&log_sys.mutex); + const lsn_t newest_lsn= log_sys.get_lsn(); + mysql_mutex_lock(&log_sys.flush_order_mutex); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t measure= buf_pool.get_oldest_modification(0); + mysql_mutex_unlock(&log_sys.flush_order_mutex); + const lsn_t checkpoint_lsn= measure ? measure : newest_lsn; + + if (!recv_recovery_is_on() && + checkpoint_lsn > log_sys.last_checkpoint_lsn + SIZE_OF_FILE_CHECKPOINT) + { + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + log_checkpoint_low(checkpoint_lsn, newest_lsn); + mysql_mutex_lock(&buf_pool.flush_list_mutex); + measure= buf_pool.get_oldest_modification(LSN_MAX); + } + else + { + mysql_mutex_unlock(&log_sys.mutex); + if (!measure) + measure= LSN_MAX; + } + + mysql_mutex_assert_not_owner(&log_sys.mutex); + + /* After attempting log checkpoint, check if we have reached our target. */ + const lsn_t target= buf_flush_sync_lsn; + + if (measure >= target) + buf_flush_sync_lsn= 0; + else if (measure >= buf_flush_async_lsn) + buf_flush_async_lsn= 0; + + /* wake up buf_flush_wait() */ + pthread_cond_broadcast(&buf_pool.done_flush_list); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); } /** Check if the adpative flushing threshold is recommended based on @@ -2298,6 +2322,10 @@ static void buf_flush_page_cleaner() for (;;) { + DBUG_EXECUTE_IF("ib_page_cleaner_sleep", + { + std::this_thread::sleep_for(std::chrono::seconds(1)); + }); lsn_limit= buf_flush_sync_lsn; if (UNIV_UNLIKELY(lsn_limit != 0) && UNIV_LIKELY(srv_flush_sync)) diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 737df0a6913..87a071d1eda 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -406,17 +406,32 @@ got_block: ut_ad(LRU_size <= BUF_LRU_MIN_LEN || available >= scan_depth || buf_pool.need_LRU_eviction()); + ut_d(bool signalled = false); + if (UNIV_UNLIKELY(available < scan_depth) && LRU_size > BUF_LRU_MIN_LEN) { mysql_mutex_lock(&buf_pool.flush_list_mutex); if (!buf_pool.page_cleaner_active()) + { buf_pool.page_cleaner_wakeup(true); + ut_d(signalled = true); + } mysql_mutex_unlock(&buf_pool.flush_list_mutex); } if (!have_mutex) mysql_mutex_unlock(&buf_pool.mutex); + DBUG_EXECUTE_IF("ib_free_page_sleep", + { + static bool do_sleep = true; + if (do_sleep && signalled) + { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + do_sleep = false; + } + }); + block->page.zip.clear(); return block; } From 7d86751de5626d685873bc3c7d9e585a922fc5aa Mon Sep 17 00:00:00 2001 From: Nikita Malyavin Date: Sat, 1 Jun 2024 00:04:45 +0200 Subject: [PATCH 23/59] mtr: run check-testcase client process under debugger --- mysql-test/mysql-test-run.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index d1ad3bdea77..39df025ecbd 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -5526,6 +5526,8 @@ sub start_check_testcase ($$$) { mtr_add_arg($args, "--record"); } my $errfile= "$opt_vardir/tmp/$name.err"; + + My::Debugger::setup_client_args(\$args, \$exe_mysqltest); my $proc= My::SafeProcess->new ( name => $name, From ce9efb4e0223c978dcf4511ed33f5118e3b3c3ce Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 5 Jun 2024 16:34:50 +0200 Subject: [PATCH 24/59] MDEV-34296 tpool - declare thread_local_waiter "static thread_local" --- storage/innobase/log/log0sync.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/log/log0sync.cc b/storage/innobase/log/log0sync.cc index 2a6e1b8b853..0129ad1e324 100644 --- a/storage/innobase/log/log0sync.cc +++ b/storage/innobase/log/log0sync.cc @@ -186,7 +186,7 @@ void group_commit_lock::set_pending(group_commit_lock::value_type num) } const unsigned int MAX_SPINS = 1; /** max spins in acquire */ -thread_local group_commit_waiter_t thread_local_waiter; +static thread_local group_commit_waiter_t thread_local_waiter; group_commit_lock::lock_return_code group_commit_lock::acquire(value_type num) { From 0406b2a4ed11fd32532eadcc2d78f936b137f0cd Mon Sep 17 00:00:00 2001 From: Rucha Deodhar Date: Wed, 15 May 2024 17:06:20 +0530 Subject: [PATCH 25/59] MDEV-34143: Server crashes when executing JSON_EXTRACT after setting non-default collation_connection Analysis: Due to different collation, the string has nothing to chop off. Fix: Got rid of chop(), only append " ," only when we have more elements to add to the result. --- mysql-test/main/func_json.result | 11 +++++++++++ mysql-test/main/func_json.test | 12 ++++++++++++ sql/item_jsonfunc.cc | 20 +++++++++++--------- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/mysql-test/main/func_json.result b/mysql-test/main/func_json.result index 591e8813a0f..50861c4f7cb 100644 --- a/mysql-test/main/func_json.result +++ b/mysql-test/main/func_json.result @@ -1717,5 +1717,16 @@ SELECT JSON_REMOVE('{"A": { "B": 1 }}', '$.A.B.C.D'); JSON_REMOVE('{"A": { "B": 1 }}', '$.A.B.C.D') {"A": {"B": 1}} # +# MDEV-34143: Server crashes when executing JSON_EXTRACT after setting non-default collation_connection +# +SET @save_collation_connection= @@collation_connection; +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a": 1,"b": 2}','$.a'); +JSON_EXTRACT('{"a": 1,"b": 2}','$.a') +NULL +Warnings: +Warning 4036 Character disallowed in JSON in argument 1 to function 'json_extract' at position 2 +SET @@collation_connection= @save_collation_connection; +# # End of 10.5 tests # diff --git a/mysql-test/main/func_json.test b/mysql-test/main/func_json.test index 9f08c30f7cf..9abdded34d2 100644 --- a/mysql-test/main/func_json.test +++ b/mysql-test/main/func_json.test @@ -1147,6 +1147,18 @@ SELECT JSON_TYPE(json_value(JSON_OBJECT("id", 1, "name", 'Monty', "date", Cast(' SELECT JSON_REMOVE('{"A": { "B": 1 }}', '$.A.B.C.D'); + +--echo # +--echo # MDEV-34143: Server crashes when executing JSON_EXTRACT after setting non-default collation_connection +--echo # + +SET @save_collation_connection= @@collation_connection; + +SET collation_connection='utf16_bin'; +SELECT JSON_EXTRACT('{"a": 1,"b": 2}','$.a'); + +SET @@collation_connection= @save_collation_connection; + --echo # --echo # End of 10.5 tests --echo # diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 442c4cbed90..f4cf9022ba9 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -1003,11 +1003,18 @@ String *Item_func_json_extract::read_json(String *str, je= sav_je; } - for (int count= 0; count < count_path; count++) + if ((not_first_value && str->append(", ", 2))) + goto error; + while(count_path) { - if (str->append((const char *) value, v_len) || - str->append(", ", 2)) - goto error; /* Out of memory. */ + if (str->append((const char *) value, v_len)) + goto error; + count_path--; + if (count_path) + { + if (str->append(", ", 2)) + goto error; + } } not_first_value= 1; @@ -1029,11 +1036,6 @@ String *Item_func_json_extract::read_json(String *str, goto return_null; } - if (str->length()>2) - { - str->chop(); - str->chop(); - } if (possible_multiple_values && str->append("]", 1)) goto error; /* Out of memory. */ From bc3660925d6aa9b7b2b282b73b53f32c1405992d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 6 Jun 2024 10:18:42 +0300 Subject: [PATCH 26/59] MDEV-34307 On startup, [FATAL] InnoDB: Page ... still fixed or dirty buf_pool_invalidate(): Properly wait for os_aio_wait_until_no_pending_writes() to ensure so that there are no pending buf_page_t::write_complete() or buf_page_write_complete() operations. This will avoid a failure of buf_pool.assert_all_freed(). This bug should affect debug builds only. At this point, the buf_pool.flush_list should be clear and all changes should have been written out. The loop around buf_LRU_scan_and_free_block() should have eventually completed and freed all pages as soon as buf_page_t::write_complete() had a chance to release the page latches. It is worth noting that buf_flush_wait() is working as intended. As soon as buf_flush_page_cleaner() invokes buf_pool.get_oldest_modification() it will observe that buf_page_t::write_complete() had assigned oldest_modification_ to 1, and remove such blocks from buf_pool.flush_list. Upon reaching buf_pool.flush_list.count=0 the buf_flush_page_cleaner() will mark itself idle and wake buf_flush_wait() by broadcasting buf_pool.done_flush_list. This regression was introduced in commit a55b951e6082a4ce9a1f2ed5ee176ea7dbbaf1f2 (MDEV-26827). Reviewed by: Debarun Banerjee --- storage/innobase/buf/buf0buf.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 16592c05405..5a4f8057012 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -3700,15 +3700,13 @@ void buf_refresh_io_stats() All pages must be in a replaceable state (not modified or latched). */ void buf_pool_invalidate() { - mysql_mutex_lock(&buf_pool.mutex); - /* It is possible that a write batch that has been posted earlier is still not complete. For buffer pool invalidation to proceed we must ensure there is NO write activity happening. */ - ut_d(mysql_mutex_unlock(&buf_pool.mutex)); + os_aio_wait_until_no_pending_writes(false); ut_d(buf_pool.assert_all_freed()); - ut_d(mysql_mutex_lock(&buf_pool.mutex)); + mysql_mutex_lock(&buf_pool.mutex); while (UT_LIST_GET_LEN(buf_pool.LRU)) { buf_LRU_scan_and_free_block(); From 9fac857f26044861bab59d94f9617e10696b3691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 6 Jun 2024 13:03:34 +0300 Subject: [PATCH 27/59] MDEV-34283 A misplaced btr_cur_need_opposite_intention() check may fail to prevent hangs btr_cur_t::search_leaf(): Invoke btr_cur_need_opposite_intention() after positioning page_cur.rec so that the record will be in the intended page. This is something that was broken in commit f2096478d5750b983f9a9cc4691d20e152dafd4a or commit de4030e4d49805a7ded5c0bfee01cc3fd7623522 or related changes. btr_cur_need_opposite_intention(): Add a debug assertion that would catch the misuse. The "next line of defence" that should have caught this bug in debug builds are assertions that mtr_t::m_memo contains MTR_MEMO_X_LOCK for the dict_index_t::lock. When btr_cur_need_opposite_intention() holds, we should escalate to acquiring an exclusive index->lock in btr_cur_t::pessimistic_search_leaf(). Reviewed by: Debarun Banerjee --- storage/innobase/btr/btr0cur.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index b706281aab5..cff9e664c5d 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -756,6 +756,7 @@ static bool btr_cur_need_opposite_intention(const buf_page_t &bpage, ulint compress_limit, const rec_t *rec) { + ut_ad(bpage.frame == page_align(rec)); if (UNIV_LIKELY_NULL(bpage.zip.data) && !page_zip_available(&bpage.zip, is_clust, node_ptr_max_size, 1)) return true; @@ -1437,11 +1438,6 @@ release_tree: !btr_block_get(*index(), btr_page_get_next(block->page.frame), RW_X_LATCH, false, mtr, &err)) goto func_exit; - if (btr_cur_need_opposite_intention(block->page, index()->is_clust(), - lock_intention, - node_ptr_max_size, compress_limit, - page_cur.rec)) - goto need_opposite_intention; } reached_latched_leaf: @@ -1463,6 +1459,13 @@ release_tree: ut_ad(up_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); ut_ad(low_match != ULINT_UNDEFINED || mode != PAGE_CUR_LE); + if (latch_mode == BTR_MODIFY_TREE && + btr_cur_need_opposite_intention(block->page, index()->is_clust(), + lock_intention, + node_ptr_max_size, compress_limit, + page_cur.rec)) + goto need_opposite_intention; + #ifdef BTR_CUR_HASH_ADAPT /* We do a dirty read of btr_search_enabled here. We will properly check btr_search_enabled again in From 699d38d9512218531b9fe7569600ec0619cc6f71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 6 Jun 2024 14:38:42 +0300 Subject: [PATCH 28/59] MDEV-34296 extern thread_local is a CPU waste MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit 99bd22605938c42d876194f2ec75b32e658f00f5 (MDEV-31558) we wrongly thought that there would be minimal overhead for accessing a thread-local variable mariadb_stats. It turns out that in C++11, each access to an extern thread_local variable requires conditionally invoking an initialization function. In fact, the initializer expression of mariadb_stats is dynamic, and those calls were actually unavoidable. In C++20, one could declare constinit thread_local variables, but the address of a thread_local variable (&mariadb_dummy_stats) is not a compile-time constant. We did not want to declare mariadb_dummy_stats without thread_local, because then the dummy accesses could lead to cache line contention between threads. mariadb_stats: Declare as __thread or __declspec(thread) so that there will be no dynamic initialization, but zero-initialization. mariadb_dummy_stats: Remove. It is a lesser evil to let the environment perform zero-initialization and check if !mariadb_stats. Reviewed by: Sergei Petrunia --- storage/innobase/buf/buf0buf.cc | 22 +++++---- storage/innobase/buf/buf0rea.cc | 16 +++--- storage/innobase/handler/ha_innodb.cc | 3 +- storage/innobase/include/mariadb_stats.h | 62 ++++++++++++++---------- 4 files changed, 60 insertions(+), 43 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 5a4f8057012..8fa8b92470b 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2165,6 +2165,12 @@ void buf_page_free(fil_space_t *space, uint32_t page, mtr_t *mtr) mtr->memo_push(block, MTR_MEMO_PAGE_X_MODIFY); } +static void buf_inc_get(ha_handler_stats *stats) +{ + mariadb_increment_pages_accessed(stats); + ++buf_pool.stat.n_page_gets; +} + /** Get read access to a compressed page (usually of type FIL_PAGE_TYPE_ZBLOB or FIL_PAGE_TYPE_ZBLOB2). The page must be released with unfix(). @@ -2180,8 +2186,8 @@ buf_page_t* buf_page_get_zip(const page_id_t page_id, ulint zip_size) { ut_ad(zip_size); ut_ad(ut_is_2pow(zip_size)); - ++buf_pool.stat.n_page_gets; - mariadb_increment_pages_accessed(); + ha_handler_stats *const stats= mariadb_stats; + buf_inc_get(stats); buf_pool_t::hash_chain &chain= buf_pool.page_hash.cell_get(page_id.fold()); page_hash_latch &hash_lock= buf_pool.page_hash.lock_get(chain); @@ -2283,7 +2289,7 @@ must_read_page: switch (dberr_t err= buf_read_page(page_id, zip_size)) { case DB_SUCCESS: case DB_SUCCESS_LOCKED_REC: - mariadb_increment_pages_read(); + mariadb_increment_pages_read(stats); goto lookup; default: ib::error() << "Reading compressed page " << page_id @@ -2460,9 +2466,8 @@ buf_page_get_low( ut_ad(!mtr || !ibuf_inside(mtr) || ibuf_page_low(page_id, zip_size, FALSE, NULL)); - ++buf_pool.stat.n_page_gets; - mariadb_increment_pages_accessed(); - + ha_handler_stats* const stats = mariadb_stats; + buf_inc_get(stats); auto& chain= buf_pool.page_hash.cell_get(page_id.fold()); page_hash_latch& hash_lock = buf_pool.page_hash.lock_get(chain); loop: @@ -2533,7 +2538,7 @@ loop: switch (dberr_t local_err = buf_read_page(page_id, zip_size)) { case DB_SUCCESS: case DB_SUCCESS_LOCKED_REC: - mariadb_increment_pages_read(); + mariadb_increment_pages_read(stats); buf_read_ahead_random(page_id, zip_size, ibuf_inside(mtr)); break; default: @@ -3127,8 +3132,7 @@ buf_block_t *buf_page_try_get(const page_id_t page_id, mtr_t *mtr) ut_ad(block->page.buf_fix_count()); ut_ad(block->page.id() == page_id); - ++buf_pool.stat.n_page_gets; - mariadb_increment_pages_accessed(); + buf_inc_get(mariadb_stats); return block; } diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 76a5e7104a7..1c1e1150635 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -300,12 +300,15 @@ buf_read_page_low( } ut_ad(bpage->in_file()); - ulonglong mariadb_timer= 0; + ulonglong mariadb_timer = 0; if (sync) { thd_wait_begin(nullptr, THD_WAIT_DISKIO); - if (mariadb_stats_active()) - mariadb_timer= mariadb_measure(); + if (const ha_handler_stats *stats = mariadb_stats) { + if (stats->active) { + mariadb_timer = mariadb_measure(); + } + } } DBUG_LOG("ib_buf", @@ -324,15 +327,16 @@ buf_read_page_low( if (UNIV_UNLIKELY(fio.err != DB_SUCCESS)) { buf_pool.corrupted_evict(bpage, buf_page_t::READ_FIX); } else if (sync) { - thd_wait_end(NULL); + thd_wait_end(nullptr); /* The i/o was already completed in space->io() */ fio.err = bpage->read_complete(*fio.node); space->release(); if (fio.err == DB_FAIL) { fio.err = DB_PAGE_CORRUPTED; } - if (mariadb_timer) - mariadb_increment_pages_read_time(mariadb_timer); + if (mariadb_timer) { + mariadb_increment_pages_read_time(mariadb_timer); + } } return fio.err; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 7d1463039b9..fc4a82983fd 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -110,8 +110,7 @@ extern my_bool opt_readonly; #include "ut0mem.h" #include "row0ext.h" #include "mariadb_stats.h" -thread_local ha_handler_stats mariadb_dummy_stats; -thread_local ha_handler_stats *mariadb_stats= &mariadb_dummy_stats; +simple_thread_local ha_handler_stats *mariadb_stats; #include #include // TT_FOR_UPGRADE diff --git a/storage/innobase/include/mariadb_stats.h b/storage/innobase/include/mariadb_stats.h index e9051c0c08b..3a2790d02f0 100644 --- a/storage/innobase/include/mariadb_stats.h +++ b/storage/innobase/include/mariadb_stats.h @@ -16,54 +16,67 @@ this program; if not, write to the Free Software Foundation, Inc., *****************************************************************************/ -#ifndef mariadb_stats_h -#define mariadb_stats_h - -/* Include file to handle mariadbd handler specific stats */ +#pragma once #include "ha_handler_stats.h" #include "my_rdtsc.h" -/* Not active threads are ponting to this structure */ -extern thread_local ha_handler_stats mariadb_dummy_stats; +/* We do not want a dynamic initialization function to be +conditionally invoked on each access to a C++11 extern thread_local. */ +#if __cplusplus >= 202002L +# define simple_thread_local constinit thread_local +#else +# define simple_thread_local IF_WIN(__declspec(thread),__thread) +#endif -/* Points to either THD->handler_stats or mariad_dummy_stats */ -extern thread_local ha_handler_stats *mariadb_stats; +/** Pointer to handler::active_handler_stats or nullptr (via .tbss) */ +extern simple_thread_local ha_handler_stats *mariadb_stats; /* - Returns 1 if MariaDB wants engine status + Returns nonzero if MariaDB wants engine status */ -inline bool mariadb_stats_active() +inline uint mariadb_stats_active() { - return mariadb_stats->active != 0; -} - -inline bool mariadb_stats_active(ha_handler_stats *stats) -{ - return stats->active != 0; + if (ha_handler_stats *stats= mariadb_stats) + return stats->active; + return 0; } /* The following functions increment different engine status */ +inline void mariadb_increment_pages_accessed(ha_handler_stats *stats) +{ + if (stats) + stats->pages_accessed++; +} + inline void mariadb_increment_pages_accessed() { - mariadb_stats->pages_accessed++; + mariadb_increment_pages_accessed(mariadb_stats); } inline void mariadb_increment_pages_updated(ulonglong count) { - mariadb_stats->pages_updated+= count; + if (ha_handler_stats *stats= mariadb_stats) + stats->pages_updated+= count; +} + +inline void mariadb_increment_pages_read(ha_handler_stats *stats) +{ + if (stats) + stats->pages_read_count++; } inline void mariadb_increment_pages_read() { - mariadb_stats->pages_read_count++; + mariadb_increment_pages_read(mariadb_stats); } inline void mariadb_increment_undo_records_read() { - mariadb_stats->undo_records_read++; + if (ha_handler_stats *stats= mariadb_stats) + stats->undo_records_read++; } /* @@ -92,7 +105,7 @@ inline void mariadb_increment_pages_read_time(ulonglong start_time) ulonglong end_time= mariadb_measure(); /* Check that we only call this if active, see example! */ DBUG_ASSERT(start_time); - DBUG_ASSERT(mariadb_stats_active(stats)); + DBUG_ASSERT(stats->active); stats->pages_read_time+= (end_time - start_time); } @@ -105,15 +118,12 @@ inline void mariadb_increment_pages_read_time(ulonglong start_time) class mariadb_set_stats { public: - uint flag; mariadb_set_stats(ha_handler_stats *stats) { - mariadb_stats= stats ? stats : &mariadb_dummy_stats; + mariadb_stats= stats; } ~mariadb_set_stats() { - mariadb_stats= &mariadb_dummy_stats; + mariadb_stats= nullptr; } }; - -#endif /* mariadb_stats_h */ From a02773f7c000273dd823df6bb386c4d735c3dcc7 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 6 Jun 2024 19:09:13 +0530 Subject: [PATCH 29/59] MDEV-34057 Inconsistent FTS state in concurrent scenarios Problem: ======= - This commit is a merge of mysql commit 129ee47ef994652081a11ee9040c0488e5275b14. InnoDB FTS can be in inconsistent state when sync operation terminates the server before committing the operation. This could lead to incorrect synced doc id and incorrect query results. Solution: ======== - During sync commit operation, InnoDB should pass the sync transaction to update the max doc id in the config table. fts_read_synced_doc_id() : This function is used to read only synced doc id from the config table. --- .../r/fts_sync_commit_resiliency.result | 63 +++++++ .../t/fts_sync_commit_resiliency.opt | 1 + .../t/fts_sync_commit_resiliency.test | 47 +++++ storage/innobase/fts/fts0fts.cc | 168 ++++++++++-------- 4 files changed, 201 insertions(+), 78 deletions(-) create mode 100644 mysql-test/suite/innodb_fts/r/fts_sync_commit_resiliency.result create mode 100644 mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.opt create mode 100644 mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.test diff --git a/mysql-test/suite/innodb_fts/r/fts_sync_commit_resiliency.result b/mysql-test/suite/innodb_fts/r/fts_sync_commit_resiliency.result new file mode 100644 index 00000000000..1908502c9db --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/fts_sync_commit_resiliency.result @@ -0,0 +1,63 @@ +CREATE TABLE opening_lines ( +id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, +opening_line TEXT(500), +author VARCHAR(200), +title VARCHAR(200) +) ENGINE=InnoDB; +CREATE FULLTEXT INDEX idx ON opening_lines(opening_line); +CREATE FULLTEXT INDEX ft_idx1 ON opening_lines(title); +INSERT INTO opening_lines(opening_line,author,title) VALUES +('Call me Ishmael.','Herman Melville','Moby Dick'), +('A screaming comes across the sky.','Thomas Pynchon','Gravity\'s Rainbow'), + ('I am an invisible man.','Ralph Ellison','Invisible Man'), + ('Where now? Who now? When now?','Samuel Beckett','The Unnamable'), + ('It was love at first sight.','Joseph Heller','Catch-22'), + ('All this happened, more or less.','Kurt Vonnegut','Slaughterhouse-Five'), + ('Mrs. Dalloway said she would buy the flowers herself.','Virginia Woolf','Mrs. Dalloway'), + ('It was a pleasure to burn.','Ray Bradbury','Fahrenheit 451'); +SET GLOBAL innodb_ft_aux_table='test/opening_lines'; +SELECT * FROM information_schema.innodb_ft_config; +KEY VALUE +optimize_checkpoint_limit 180 +synced_doc_id 0 +stopword_table_name +use_stopword 1 +SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael'); +id opening_line author title +1 Call me Ishmael. Herman Melville Moby Dick +SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible'); +id opening_line author title +3 I am an invisible man. Ralph Ellison Invisible Man +SELECT * FROM opening_lines; +id opening_line author title +1 Call me Ishmael. Herman Melville Moby Dick +2 A screaming comes across the sky. Thomas Pynchon Gravity's Rainbow +3 I am an invisible man. Ralph Ellison Invisible Man +4 Where now? Who now? When now? Samuel Beckett The Unnamable +5 It was love at first sight. Joseph Heller Catch-22 +6 All this happened, more or less. Kurt Vonnegut Slaughterhouse-Five +7 Mrs. Dalloway said she would buy the flowers herself. Virginia Woolf Mrs. Dalloway +8 It was a pleasure to burn. Ray Bradbury Fahrenheit 451 +SET GLOBAL innodb_optimize_fulltext_only=ON; +SET DEBUG_SYNC='fts_crash_before_commit_sync SIGNAL hung WAIT_FOR ever'; +OPTIMIZE TABLE opening_lines; +connect con1,localhost,root,,; +SET DEBUG_SYNC='now WAIT_FOR hung'; +# restart +SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael'); +id opening_line author title +1 Call me Ishmael. Herman Melville Moby Dick +SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible'); +id opening_line author title +3 I am an invisible man. Ralph Ellison Invisible Man +SELECT * FROM opening_lines; +id opening_line author title +1 Call me Ishmael. Herman Melville Moby Dick +2 A screaming comes across the sky. Thomas Pynchon Gravity's Rainbow +3 I am an invisible man. Ralph Ellison Invisible Man +4 Where now? Who now? When now? Samuel Beckett The Unnamable +5 It was love at first sight. Joseph Heller Catch-22 +6 All this happened, more or less. Kurt Vonnegut Slaughterhouse-Five +7 Mrs. Dalloway said she would buy the flowers herself. Virginia Woolf Mrs. Dalloway +8 It was a pleasure to burn. Ray Bradbury Fahrenheit 451 +DROP TABLE opening_lines; diff --git a/mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.opt b/mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.opt new file mode 100644 index 00000000000..9e0e66f6620 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.opt @@ -0,0 +1 @@ +--innodb_ft_config diff --git a/mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.test b/mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.test new file mode 100644 index 00000000000..357d001f6e6 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.test @@ -0,0 +1,47 @@ +# Test database resiliency against scenario where the server crashes +# right before fts_sync_commit commits its transaction +source include/have_innodb.inc; +source include/have_debug.inc; +source include/not_embedded.inc; +source include/have_debug_sync.inc; + +CREATE TABLE opening_lines ( + id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, + opening_line TEXT(500), + author VARCHAR(200), + title VARCHAR(200) + ) ENGINE=InnoDB; + +CREATE FULLTEXT INDEX idx ON opening_lines(opening_line); +CREATE FULLTEXT INDEX ft_idx1 ON opening_lines(title); + +INSERT INTO opening_lines(opening_line,author,title) VALUES + ('Call me Ishmael.','Herman Melville','Moby Dick'), + ('A screaming comes across the sky.','Thomas Pynchon','Gravity\'s Rainbow'), + ('I am an invisible man.','Ralph Ellison','Invisible Man'), + ('Where now? Who now? When now?','Samuel Beckett','The Unnamable'), + ('It was love at first sight.','Joseph Heller','Catch-22'), + ('All this happened, more or less.','Kurt Vonnegut','Slaughterhouse-Five'), + ('Mrs. Dalloway said she would buy the flowers herself.','Virginia Woolf','Mrs. Dalloway'), + ('It was a pleasure to burn.','Ray Bradbury','Fahrenheit 451'); + +SET GLOBAL innodb_ft_aux_table='test/opening_lines'; +SELECT * FROM information_schema.innodb_ft_config; + +SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael'); +SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible'); +SELECT * FROM opening_lines; + +SET GLOBAL innodb_optimize_fulltext_only=ON; +SET DEBUG_SYNC='fts_crash_before_commit_sync SIGNAL hung WAIT_FOR ever'; +send OPTIMIZE TABLE opening_lines; + +connect(con1,localhost,root,,); +SET DEBUG_SYNC='now WAIT_FOR hung'; +let $shutdown_timeout=0; +--source include/restart_mysqld.inc + +SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael'); +SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible'); +SELECT * FROM opening_lines; +DROP TABLE opening_lines; diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 3e57773d32c..ba28eead0b2 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -2607,89 +2607,85 @@ fts_get_next_doc_id( return(DB_SUCCESS); } -/*********************************************************************//** -This function fetch the Doc ID from CONFIG table, and compare with +/** Read the synced document id from the fts configuration table +@param table fts table +@param doc_id document id to be read +@param trx transaction to read from config table +@return DB_SUCCESS in case of success */ +static +dberr_t fts_read_synced_doc_id(const dict_table_t *table, + doc_id_t *doc_id, + trx_t *trx) +{ + dberr_t error; + que_t* graph= NULL; + char table_name[MAX_FULL_NAME_LEN]; + + fts_table_t fts_table; + fts_table.suffix= "CONFIG"; + fts_table.table_id= table->id; + fts_table.type= FTS_COMMON_TABLE; + fts_table.table= table; + ut_a(table->fts->doc_col != ULINT_UNDEFINED); + + trx->op_info = "update the next FTS document id"; + pars_info_t *info= pars_info_create(); + pars_info_bind_function(info, "my_func", fts_fetch_store_doc_id, + doc_id); + + fts_get_table_name(&fts_table, table_name); + pars_info_bind_id(info, "config_table", table_name); + + graph= fts_parse_sql( + &fts_table, info, + "DECLARE FUNCTION my_func;\n" + "DECLARE CURSOR c IS SELECT value FROM $config_table" + " WHERE key = 'synced_doc_id' FOR UPDATE;\n" + "BEGIN\n" + "" + "OPEN c;\n" + "WHILE 1 = 1 LOOP\n" + " FETCH c INTO my_func();\n" + " IF c % NOTFOUND THEN\n" + " EXIT;\n" + " END IF;\n" + "END LOOP;\n" + "CLOSE c;"); + + *doc_id = 0; + error = fts_eval_sql(trx, graph); + fts_que_graph_free_check_lock(&fts_table, NULL, graph); + return error; +} + +/** This function fetch the Doc ID from CONFIG table, and compare with the Doc ID supplied. And store the larger one to the CONFIG table. +@param table fts table +@param cmp_doc_id Doc ID to compare +@param doc_id larger document id after comparing "cmp_doc_id" to + the one stored in CONFIG table +@param trx transaction @return DB_SUCCESS if OK */ -static MY_ATTRIBUTE((nonnull)) +static dberr_t fts_cmp_set_sync_doc_id( -/*====================*/ - const dict_table_t* table, /*!< in: table */ - doc_id_t cmp_doc_id, /*!< in: Doc ID to compare */ - ibool read_only, /*!< in: TRUE if read the - synced_doc_id only */ - doc_id_t* doc_id) /*!< out: larger document id - after comparing "cmp_doc_id" - to the one stored in CONFIG - table */ + const dict_table_t *table, + doc_id_t cmp_doc_id, + doc_id_t *doc_id, + trx_t *trx=nullptr) { - trx_t* trx; - pars_info_t* info; - dberr_t error; - fts_table_t fts_table; - que_t* graph = NULL; - fts_cache_t* cache = table->fts->cache; - char table_name[MAX_FULL_NAME_LEN]; -retry: - ut_a(table->fts->doc_col != ULINT_UNDEFINED); + fts_cache_t* cache= table->fts->cache; + dberr_t error = DB_SUCCESS; + const trx_t* const caller_trx = trx; - fts_table.suffix = "CONFIG"; - fts_table.table_id = table->id; - fts_table.type = FTS_COMMON_TABLE; - fts_table.table = table; - - trx = trx_create(); - if (srv_read_only_mode) { + if (trx == nullptr) { + trx = trx_create(); trx_start_internal_read_only(trx); - } else { - trx_start_internal(trx); } +retry: + error = fts_read_synced_doc_id(table, doc_id, trx); - trx->op_info = "update the next FTS document id"; - - info = pars_info_create(); - - pars_info_bind_function( - info, "my_func", fts_fetch_store_doc_id, doc_id); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "config_table", table_name); - - graph = fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $config_table" - " WHERE key = 'synced_doc_id' FOR UPDATE;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - *doc_id = 0; - - error = fts_eval_sql(trx, graph); - - fts_que_graph_free_check_lock(&fts_table, NULL, graph); - - // FIXME: We need to retry deadlock errors - if (error != DB_SUCCESS) { - goto func_exit; - } - - if (read_only) { - /* InnoDB stores actual synced_doc_id value + 1 in - FTS_CONFIG table. Reduce the value by 1 while reading - after startup. */ - if (*doc_id) *doc_id -= 1; - goto func_exit; - } + if (error != DB_SUCCESS) goto func_exit; if (cmp_doc_id == 0 && *doc_id) { cache->synced_doc_id = *doc_id - 1; @@ -2714,6 +2710,10 @@ retry: func_exit: + if (caller_trx) { + return error; + } + if (UNIV_LIKELY(error == DB_SUCCESS)) { fts_sql_commit(trx); } else { @@ -2721,6 +2721,7 @@ func_exit: ib::error() << "(" << error << ") while getting next doc id " "for table " << table->name; + fts_sql_rollback(trx); if (error == DB_DEADLOCK) { @@ -4201,8 +4202,8 @@ fts_sync_commit( /* After each Sync, update the CONFIG table about the max doc id we just sync-ed to index table */ - error = fts_cmp_set_sync_doc_id(sync->table, sync->max_doc_id, FALSE, - &last_doc_id); + error = fts_cmp_set_sync_doc_id(sync->table, sync->max_doc_id, + &last_doc_id, trx); /* Get the list of deleted documents that are either in the cache or were headed there but were deleted before the add @@ -4228,6 +4229,7 @@ fts_sync_commit( rw_lock_x_unlock(&cache->lock); if (UNIV_LIKELY(error == DB_SUCCESS)) { + DEBUG_SYNC_C("fts_crash_before_commit_sync"); fts_sql_commit(trx); } else { fts_sql_rollback(trx); @@ -4901,7 +4903,7 @@ fts_init_doc_id( /* Then compare this value with the ID value stored in the CONFIG table. The larger one will be our new initial Doc ID */ - fts_cmp_set_sync_doc_id(table, 0, FALSE, &max_doc_id); + fts_cmp_set_sync_doc_id(table, 0, &max_doc_id); /* If DICT_TF2_FTS_ADD_DOC_ID is set, we are in the process of creating index (and add doc id column. No need to recovery @@ -6376,7 +6378,17 @@ fts_init_index( start_doc = cache->synced_doc_id; if (!start_doc) { - fts_cmp_set_sync_doc_id(table, 0, TRUE, &start_doc); + trx_t *trx = trx_create(); + trx_start_internal_read_only(trx); + dberr_t err= fts_read_synced_doc_id(table, &start_doc, trx); + fts_sql_commit(trx); + trx->free(); + if (err != DB_SUCCESS) { + goto func_exit; + } + if (start_doc) { + start_doc--; + } cache->synced_doc_id = start_doc; } From d328705a12139efad45fa42d9c32642069b7820d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 21 May 2024 15:34:13 +0300 Subject: [PATCH 30/59] MDEV-34170 : table gtid_slave_pos entries never been deleted with wsrep_gtid_mode = 0 Problem was that updates to mysql.gtid_slave_pos table were replicated even when they were newer used and because that newer deleted. Avoid replication of mysql.gtid_slave_pos table if wsrep_gtid_mode=OFF. Signed-off-by: Julius Goryavsky --- .../r/galera_as_slave_gtid_myisam.result | 6 - .../galera/r/galera_replica_no_gtid.result | 71 ++++++++++ .../galera/t/galera_as_slave_gtid_myisam.test | 14 +- .../suite/galera/t/galera_replica_no_gtid.cnf | 9 ++ .../galera/t/galera_replica_no_gtid.test | 124 ++++++++++++++++++ sql/log_event_server.cc | 5 + sql/rpl_gtid.cc | 14 +- 7 files changed, 220 insertions(+), 23 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_replica_no_gtid.result create mode 100644 mysql-test/suite/galera/t/galera_replica_no_gtid.cnf create mode 100644 mysql-test/suite/galera/t/galera_replica_no_gtid.test diff --git a/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result b/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result index 6559c7828be..5698ebf9fd3 100644 --- a/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result +++ b/mysql-test/suite/galera/r/galera_as_slave_gtid_myisam.result @@ -22,12 +22,6 @@ EXPECT_1 1 gtid_binlog_state_equal 0 -connection node_2; -SELECT COUNT(*) AS EXPECT_1 FROM t1; -EXPECT_1 -1 -gtid_binlog_state_equal -0 #cleanup connection node_3; DROP TABLE t1; diff --git a/mysql-test/suite/galera/r/galera_replica_no_gtid.result b/mysql-test/suite/galera/r/galera_replica_no_gtid.result new file mode 100644 index 00000000000..80d713c2348 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_replica_no_gtid.result @@ -0,0 +1,71 @@ +connection node_2; +connection node_1; +connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3; +create user repl@'%' identified by 'repl'; +grant all on *.* to repl@'%'; +flush privileges; +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +connection node_1; +connection node_2; +connection node_2; +START SLAVE; +connection node_3; +CREATE TABLE t1 (id bigint primary key, msg varchar(100)) engine=innodb; +SELECT COUNT(*) AS EXPECT_10000 FROM t1; +EXPECT_10000 +10000 +connection node_2; +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +EXPECT_1 +1 +SELECT COUNT(*) AS EXPECT_10000 FROM t1; +EXPECT_10000 +10000 +connection node_1; +SELECT COUNT(*) AS EXPECT_0 FROM mysql.gtid_slave_pos; +EXPECT_0 +0 +SELECT COUNT(*) AS EXPECT_10000 FROM t1; +EXPECT_10000 +10000 +connection node_2; +# Verify that graceful shutdown succeeds. +# Force SST +connection node_1; +# Waiting until node_2 is not part of cluster anymore +connection node_2; +# Start node_2 again +¤ Wait until node_2 is back on cluster +connection node_2; +call mtr.add_suppression("Slave: Operation CREATE USER failed for .*"); +SELECT COUNT(*) AS EXPECT_0 FROM mysql.gtid_slave_pos; +EXPECT_0 +0 +SELECT COUNT(*) AS EXPECT_10000 FROM t1; +EXPECT_10000 +10000 +connection node_1; +SELECT COUNT(*) AS EXPECT_0 FROM mysql.gtid_slave_pos; +EXPECT_0 +0 +SELECT COUNT(*) AS EXPECT_10000 FROM t1; +EXPECT_10000 +10000 +connection node_3; +SELECT COUNT(*) AS EXPECT_10000 FROM t1; +EXPECT_10000 +10000 +connection node_2; +STOP SLAVE; +RESET SLAVE ALL; +connection node_3; +RESET MASTER; +drop table t1; +connection node_2; +DROP TABLE t1; +connection node_1; +connection node_1; +disconnect node_3; +disconnect node_2; +disconnect node_1; +# End of test diff --git a/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test index 60476bc45a7..e465237dfc3 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test +++ b/mysql-test/suite/galera/t/galera_as_slave_gtid_myisam.test @@ -46,18 +46,8 @@ SELECT LENGTH(@@global.gtid_binlog_state) > 1; SELECT COUNT(*) AS EXPECT_1 FROM t1; ---disable_query_log ---eval SELECT '$gtid_binlog_state_node1' = @@global.gtid_binlog_state AS gtid_binlog_state_equal; ---enable_query_log - ---connection node_2 ---let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; ---source include/wait_condition.inc - ---let $wait_condition = SELECT COUNT(*) = 1 FROM t1; ---source include/wait_condition.inc - -SELECT COUNT(*) AS EXPECT_1 FROM t1; +# Note that MyISAM tables are not replicated by Galera so we do not here +# check node_2 --disable_query_log --eval SELECT '$gtid_binlog_state_node1' = @@global.gtid_binlog_state AS gtid_binlog_state_equal; diff --git a/mysql-test/suite/galera/t/galera_replica_no_gtid.cnf b/mysql-test/suite/galera/t/galera_replica_no_gtid.cnf new file mode 100644 index 00000000000..916c1c1847f --- /dev/null +++ b/mysql-test/suite/galera/t/galera_replica_no_gtid.cnf @@ -0,0 +1,9 @@ +!include ../galera_2nodes_as_slave.cnf + +[mysqld] +wsrep-debug=1 +server_id=15 +wsrep_gtid_mode=OFF +wsrep_gtid_domain_id=16 +gtid_domain_id=11 +gtid_strict_mode=OFF diff --git a/mysql-test/suite/galera/t/galera_replica_no_gtid.test b/mysql-test/suite/galera/t/galera_replica_no_gtid.test new file mode 100644 index 00000000000..8cc88ef211a --- /dev/null +++ b/mysql-test/suite/galera/t/galera_replica_no_gtid.test @@ -0,0 +1,124 @@ +# +# Test Galera as a replica to a MySQL async replication +# +# The galera/galera_2node_slave.cnf describes the setup of the nodes +# +--source include/force_restart.inc +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_sequence.inc + +# As node #3 is not a Galera node, and galera_cluster.inc does not open connetion to it +# we open the node_3 connection here +--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3 + +create user repl@'%' identified by 'repl'; +grant all on *.* to repl@'%'; +flush privileges; +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; + +--let $node_1 = node_1 +--let $node_2 = node_2 +--source include/auto_increment_offset_save.inc + +--connection node_2 +--disable_query_log +--eval CHANGE MASTER TO master_host='127.0.0.1', master_user='repl', master_password='repl', master_port=$NODE_MYPORT_3, master_use_gtid=slave_pos; +--enable_query_log +START SLAVE; + +--connection node_3 + +CREATE TABLE t1 (id bigint primary key, msg varchar(100)) engine=innodb; +--disable_query_log +INSERT INTO t1 SELECT seq, 'test' from seq_1_to_10000; +--enable_query_log +SELECT COUNT(*) AS EXPECT_10000 FROM t1; + +--connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 10000 FROM t1; +--source include/wait_condition.inc + +# +# Node_2 is slave so mysql.gtid_slave_pos table is also replicated +# +SELECT COUNT(*) > 0 AS EXPECT_1 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) AS EXPECT_10000 FROM t1; + +--connection node_1 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT COUNT(*) = 10000 FROM t1; +--source include/wait_condition.inc + +# +# mysql-gtid_slave_pos table should not be replicated by Galera +# +SELECT COUNT(*) AS EXPECT_0 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) AS EXPECT_10000 FROM t1; + +--connection node_2 +--echo # Verify that graceful shutdown succeeds. +--source include/shutdown_mysqld.inc +--echo # Force SST +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat + +--connection node_1 +--echo # Waiting until node_2 is not part of cluster anymore +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; +--source include/wait_condition.inc + +--connection node_2 +--echo # Start node_2 again +--source include/start_mysqld.inc + +--echo ¤ Wait until node_2 is back on cluster +--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; +--source include/wait_condition.inc +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +--connection node_2 +call mtr.add_suppression("Slave: Operation CREATE USER failed for .*"); +SELECT COUNT(*) AS EXPECT_0 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) AS EXPECT_10000 FROM t1; + +--connection node_1 +SELECT COUNT(*) AS EXPECT_0 FROM mysql.gtid_slave_pos; +SELECT COUNT(*) AS EXPECT_10000 FROM t1; + +--connection node_3 +SELECT COUNT(*) AS EXPECT_10000 FROM t1; + +# +# Cleanup +# +--connection node_2 +STOP SLAVE; +RESET SLAVE ALL; + +--connection node_3 +RESET MASTER; +drop table t1; + +--connection node_2 +DROP TABLE t1; + +--connection node_1 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +--connection node_1 +--disconnect node_3 + +--source include/auto_increment_offset_restore.inc +--source include/galera_end.inc +--echo # End of test diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index 0a64b45cf49..83bbb4a3285 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -1726,6 +1726,11 @@ int Query_log_event::do_apply_event(rpl_group_info *rgi, thd->variables.pseudo_thread_id= thread_id; // for temp tables DBUG_PRINT("query",("%s", thd->query())); +#ifdef WITH_WSREP + WSREP_DEBUG("Query_log_event thread=%llu for query=%s", + thd_get_thread_id(thd), wsrep_thd_query(thd)); +#endif + if (unlikely(!(expected_error= error_code)) || ignored_error_code(expected_error) || !unexpected_error_code(expected_error)) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 6177a703383..70f61b7e300 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -697,10 +697,11 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, #ifdef WITH_WSREP /* - We should replicate local gtid_slave_pos updates to other nodes. + We should replicate local gtid_slave_pos updates to other nodes if + wsrep gtid mode is set. In applier we should not append them to galera writeset. */ - if (WSREP_ON_ && wsrep_thd_is_local(thd)) + if (WSREP_ON_ && wsrep_gtid_mode && wsrep_thd_is_local(thd)) { thd->wsrep_ignore_table= false; table->file->row_logging= 1; // replication requires binary logging @@ -877,10 +878,12 @@ rpl_slave_state::gtid_delete_pending(THD *thd, #ifdef WITH_WSREP /* - We should replicate local gtid_slave_pos updates to other nodes. + We should replicate local gtid_slave_pos updates to other nodes if + wsrep gtid mode is set. In applier we should not append them to galera writeset. */ - if (WSREP_ON_ && wsrep_thd_is_local(thd) && + if (WSREP_ON_ && wsrep_gtid_mode && + wsrep_thd_is_local(thd) && thd->wsrep_cs().state() != wsrep::client_state::s_none) { if (thd->wsrep_trx().active() == false) @@ -891,7 +894,8 @@ rpl_slave_state::gtid_delete_pending(THD *thd, } thd->wsrep_ignore_table= false; } - thd->wsrep_ignore_table= true; + else + thd->wsrep_ignore_table= true; #endif thd_saved_option= thd->variables.option_bits; From c1dc03974b036419925b6ebdd00351dc7222959f Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Thu, 28 Mar 2024 11:24:27 +0100 Subject: [PATCH 31/59] MDEV-33523 Spurious deadlock error when wsrep_on=OFF Avoid starting transactions in wsrep-lib side when wsrep is disabled. It is unnecessary, and causes spurious deadlock errors on transaction clean up. Signed-off-by: Julius Goryavsky --- .../suite/galera/galera_2nodes_as_master.cnf | 3 ++ .../suite/galera/galera_2nodes_as_slave.cnf | 3 ++ mysql-test/suite/galera/galera_2x2nodes.cnf | 6 ++-- .../suite/galera/galera_3nodes_as_slave.cnf | 3 ++ mysql-test/suite/galera/galera_4nodes.cnf | 3 ++ mysql-test/suite/galera/r/MDEV-33523.result | 6 ++++ .../suite/galera/r/galera_bf_kill.result | 11 ++++++-- mysql-test/suite/galera/t/MDEV-33523.test | 11 ++++++++ mysql-test/suite/galera/t/galera_bf_kill.test | 28 +++++++++++++++---- .../suite/galera_3nodes/galera_2x3nodes.cnf | 3 ++ .../suite/galera_3nodes/galera_3nodes.cnf | 3 ++ sql/transaction.cc | 2 +- 12 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 mysql-test/suite/galera/r/MDEV-33523.result create mode 100644 mysql-test/suite/galera/t/MDEV-33523.test diff --git a/mysql-test/suite/galera/galera_2nodes_as_master.cnf b/mysql-test/suite/galera/galera_2nodes_as_master.cnf index ba53d6062c1..11d7401535a 100644 --- a/mysql-test/suite/galera/galera_2nodes_as_master.cnf +++ b/mysql-test/suite/galera/galera_2nodes_as_master.cnf @@ -47,6 +47,9 @@ wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port' wsrep-on=OFF server-id=3 +[sst] +sst-log-archive-dir=@ENV.MYSQLTEST_VARDIR/log + [ENV] NODE_MYPORT_1= @mysqld.1.port NODE_MYSOCK_1= @mysqld.1.socket diff --git a/mysql-test/suite/galera/galera_2nodes_as_slave.cnf b/mysql-test/suite/galera/galera_2nodes_as_slave.cnf index f25c7cc6d48..c6f6299ca92 100644 --- a/mysql-test/suite/galera/galera_2nodes_as_slave.cnf +++ b/mysql-test/suite/galera/galera_2nodes_as_slave.cnf @@ -46,6 +46,9 @@ wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port' wsrep-on=OFF server-id=3 +[sst] +sst-log-archive-dir=@ENV.MYSQLTEST_VARDIR/log + [ENV] NODE_MYPORT_1= @mysqld.1.port NODE_MYSOCK_1= @mysqld.1.socket diff --git a/mysql-test/suite/galera/galera_2x2nodes.cnf b/mysql-test/suite/galera/galera_2x2nodes.cnf index 922906eac6f..a35baed0c32 100644 --- a/mysql-test/suite/galera/galera_2x2nodes.cnf +++ b/mysql-test/suite/galera/galera_2x2nodes.cnf @@ -11,7 +11,6 @@ default-storage-engine=innodb wsrep_gtid_mode=1 gtid_ignore_duplicates auto_increment_increment=3 - wsrep-provider=@ENV.WSREP_PROVIDER # enforce read-committed characteristics across the cluster # wsrep-causal-reads=ON @@ -61,6 +60,9 @@ wsrep_node_address='127.0.0.1:@mysqld.4.#galera_port' wsrep_node_incoming_address=127.0.0.1:@mysqld.4.port wsrep_sst_receive_address='127.0.0.1:@mysqld.4.#sst_port' +[sst] +sst-log-archive-dir=@ENV.MYSQLTEST_VARDIR/log + [ENV] NODE_MYPORT_1= @mysqld.1.port NODE_MYSOCK_1= @mysqld.1.socket @@ -73,5 +75,3 @@ NODE_MYSOCK_3= @mysqld.3.socket NODE_MYPORT_4= @mysqld.4.port NODE_MYSOCK_4= @mysqld.4.socket - - diff --git a/mysql-test/suite/galera/galera_3nodes_as_slave.cnf b/mysql-test/suite/galera/galera_3nodes_as_slave.cnf index 0ecf877b5a3..228147c0248 100644 --- a/mysql-test/suite/galera/galera_3nodes_as_slave.cnf +++ b/mysql-test/suite/galera/galera_3nodes_as_slave.cnf @@ -59,6 +59,9 @@ wsrep_sst_receive_address='127.0.0.1:@mysqld.3.#sst_port' wsrep-on=OFF server-id=4 +[sst] +sst-log-archive-dir=@ENV.MYSQLTEST_VARDIR/log + [ENV] NODE_MYPORT_1= @mysqld.1.port NODE_MYSOCK_1= @mysqld.1.socket diff --git a/mysql-test/suite/galera/galera_4nodes.cnf b/mysql-test/suite/galera/galera_4nodes.cnf index 3ad1a66b9e6..4be5b496370 100644 --- a/mysql-test/suite/galera/galera_4nodes.cnf +++ b/mysql-test/suite/galera/galera_4nodes.cnf @@ -59,6 +59,9 @@ wsrep_node_incoming_address=127.0.0.1:@mysqld.4.port wsrep_sst_receive_address='127.0.0.1:@mysqld.4.#sst_port' auto-increment-offset=4 +[sst] +sst-log-archive-dir=@ENV.MYSQLTEST_VARDIR/log + [ENV] NODE_MYPORT_1= @mysqld.1.port NODE_MYSOCK_1= @mysqld.1.socket diff --git a/mysql-test/suite/galera/r/MDEV-33523.result b/mysql-test/suite/galera/r/MDEV-33523.result new file mode 100644 index 00000000000..3ce8371fc63 --- /dev/null +++ b/mysql-test/suite/galera/r/MDEV-33523.result @@ -0,0 +1,6 @@ +connection node_2; +connection node_1; +SET SESSION wsrep_on=OFF; +BEGIN; +ROLLBACK; +SET SESSION wsrep_on=OFF; diff --git a/mysql-test/suite/galera/r/galera_bf_kill.result b/mysql-test/suite/galera/r/galera_bf_kill.result index 3738e8c9684..71b0366081b 100644 --- a/mysql-test/suite/galera/r/galera_bf_kill.result +++ b/mysql-test/suite/galera/r/galera_bf_kill.result @@ -49,16 +49,23 @@ a b disconnect node_2a; disconnect node_2b; connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2; connection node_2a; SET SESSION wsrep_on=OFF; begin; update t1 set a =5, b=2; connection node_2; ALTER TABLE t1 ADD UNIQUE KEY b3(b); +connection node_2b; +SET SESSION wsrep_sync_wait=0; +connection node_2a; select * from t1; a b -2 1 +5 2 +commit; +connection node_2; disconnect node_2a; +disconnect node_2b; connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; connection node_2a; SET SESSION wsrep_on=OFF; @@ -67,7 +74,7 @@ update t1 set a =5, b=2; connection node_2; select * from t1; a b -2 1 +5 2 disconnect node_2a; connection node_1; drop table t1; diff --git a/mysql-test/suite/galera/t/MDEV-33523.test b/mysql-test/suite/galera/t/MDEV-33523.test new file mode 100644 index 00000000000..5a2463d1973 --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-33523.test @@ -0,0 +1,11 @@ +# +# MDEV-33523: Spurious deadlock error when wsrep_on=OFF +# +--source include/galera_cluster.inc + +SET SESSION wsrep_on=OFF; +BEGIN; +# If bug is present, the following rollback +# results in ER_LOCK_DEADLOCK error. +ROLLBACK; +SET SESSION wsrep_on=OFF; diff --git a/mysql-test/suite/galera/t/galera_bf_kill.test b/mysql-test/suite/galera/t/galera_bf_kill.test index c8564bc9219..1b35b609a96 100644 --- a/mysql-test/suite/galera/t/galera_bf_kill.test +++ b/mysql-test/suite/galera/t/galera_bf_kill.test @@ -94,27 +94,43 @@ select * from t1; --disconnect node_2b # -# Test case 5: Start a transaction on node_2a with wsrep disabled -# and start a DDL on other transaction that will then abort node_2a -# transactions +# Test case 5: Start a transaction on node_2a with wsrep disabled. +# A conflicting DDL on other transaction can't BF abort +# transaction from node_2a (wsrep disabled). # --connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2 +--connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2 --connection node_2a SET SESSION wsrep_on=OFF; begin; update t1 set a =5, b=2; --connection node_2 -ALTER TABLE t1 ADD UNIQUE KEY b3(b); +--send ALTER TABLE t1 ADD UNIQUE KEY b3(b) +--connection node_2b +SET SESSION wsrep_sync_wait=0; +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE = 'Waiting for table metadata lock'; +--source include/wait_condition.inc + +--connection node_2a select * from t1; +# We expect that ALTER should not be able to BF abort +# this transaction, it must wait for it to finish. +# Expect commit to succeed. +commit; + +--connection node_2 +--reap + --disconnect node_2a +--disconnect node_2b # -# Test case 6: Start a transaction on node_2a with wsrep disabled -# and kill it from other connection on same node +# Test case 6: Start a transaction on node_2a with wsrep disabled +# and kill it from other connection on same node. # --connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2 diff --git a/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf b/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf index cd7a892f4c9..22e160c21b4 100644 --- a/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf +++ b/mysql-test/suite/galera_3nodes/galera_2x3nodes.cnf @@ -83,6 +83,9 @@ wsrep_node_address='127.0.0.1:@mysqld.6.#galera_port' wsrep_node_incoming_address=127.0.0.1:@mysqld.6.port wsrep_sst_receive_address='127.0.0.1:@mysqld.6.#sst_port' +[sst] +sst-log-archive-dir=@ENV.MYSQLTEST_VARDIR/log + [ENV] NODE_MYPORT_1= @mysqld.1.port NODE_MYSOCK_1= @mysqld.1.socket diff --git a/mysql-test/suite/galera_3nodes/galera_3nodes.cnf b/mysql-test/suite/galera_3nodes/galera_3nodes.cnf index a7dd4d21bc7..abd778c482c 100644 --- a/mysql-test/suite/galera_3nodes/galera_3nodes.cnf +++ b/mysql-test/suite/galera_3nodes/galera_3nodes.cnf @@ -46,6 +46,9 @@ wsrep_node_address='127.0.0.1:@mysqld.3.#galera_port' wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port wsrep_sst_receive_address='127.0.0.1:@mysqld.3.#sst_port' +[sst] +sst-log-archive-dir=@ENV.MYSQLTEST_VARDIR/log + [ENV] NODE_MYPORT_1= @mysqld.1.port NODE_MYSOCK_1= @mysqld.1.socket diff --git a/sql/transaction.cc b/sql/transaction.cc index 1e855781f44..e8ac7148334 100644 --- a/sql/transaction.cc +++ b/sql/transaction.cc @@ -191,7 +191,7 @@ bool trans_begin(THD *thd, uint flags) } #ifdef WITH_WSREP - if (wsrep_thd_is_local(thd)) + if (WSREP(thd) && wsrep_thd_is_local(thd)) { if (wsrep_sync_wait(thd)) DBUG_RETURN(TRUE); From c2d9762011aa74e076390af1500d89dd3146796d Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 6 Jun 2024 19:26:50 +0200 Subject: [PATCH 32/59] =?UTF-8?q?mtr:=20=D1=81hange=20the=20default=20sett?= =?UTF-8?q?ing=20for=20the=20port=20group=20size=20parameter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some galera tests starts 6 galera nodes. Each galera node requires three ports: 6*3 = 18. Plus 6 ports are needed for 6 mariadbd servers. Since the number of ports is rounded up to 10 everywhere in mtr, we will take 30 as the default value for the port group size parameter. --- mysql-test/mysql-test-run.pl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 39df025ecbd..7127bedffdc 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -335,7 +335,11 @@ my $opt_max_test_fail= env_or_val(MTR_MAX_TEST_FAIL => 10); my $opt_core_on_failure= 0; my $opt_parallel= $ENV{MTR_PARALLEL} || 1; -my $opt_port_group_size = $ENV{MTR_PORT_GROUP_SIZE} || 20; +# Some galera tests starts 6 galera nodes. Each galera node requires +# three ports: 6*3 = 18. Plus 6 ports are needed for 6 mariadbd servers. +# Since the number of ports is rounded up to 10 everywhere, we will +# take 30 as the default value: +my $opt_port_group_size = $ENV{MTR_PORT_GROUP_SIZE} || 30; # lock file to stop tests my $opt_stop_file= $ENV{MTR_STOP_FILE}; From 654f6ecec4fbca348b7ebeb04e3c499c6346089b Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 6 Jun 2024 19:37:31 +0200 Subject: [PATCH 33/59] galera: wsrep-lib submodule update --- wsrep-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wsrep-lib b/wsrep-lib index dfc4bdb8a5d..31db8476768 160000 --- a/wsrep-lib +++ b/wsrep-lib @@ -1 +1 @@ -Subproject commit dfc4bdb8a5dcbd6fbea007ad3beff899a6b5b7bd +Subproject commit 31db8476768ba68296ad91b6785bb06a6a9abf71 From 238798d978dccaa7c4d47fbb34f36d01d8cb10f0 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 6 Jun 2024 20:24:06 +0200 Subject: [PATCH 34/59] MDEV-32158: wsrep_sst_mariabackup use /tmp dir during SST rather then user defined tmpdir wsrep_sst_mariabackup should use the tmpdir defined by the user under the '[mysqld]' section of the configuration file rather than the default '/tmp' directory. --- scripts/wsrep_sst_mariabackup.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index dd206e38ec2..2cae93ed694 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -714,7 +714,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]+' || :) + pgid=$(ps -o 'pgid=' $$ 2>/dev/null | grep -o -E '[0-9]+' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -1086,17 +1086,19 @@ if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then tmpdir=$(parse_cnf "$encgroups" 'tmpdir') if [ -z "$tmpdir" ]; then xtmpdir="$(mktemp -d)" + itmpdir="$(mktemp -d)" elif [ "$OS" = 'Linux' ]; then - xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir") + xtmpdir=$(mktemp -d "--tmpdir=$tmpdir") + itmpdir=$(mktemp -d "--tmpdir=$tmpdir") else - xtmpdir=$(TMPDIR="$tmpdir"; mktemp '-d') + xtmpdir=$(TMPDIR="$tmpdir"; mktemp -d) + itmpdir=$(TMPDIR="$tmpdir"; mktemp -d) fi wsrep_log_info "Using '$xtmpdir' as mariadb-backup temporary directory" tmpopts=" --tmpdir='$xtmpdir'" - itmpdir="$(mktemp -d)" - wsrep_log_info "Using '$itmpdir' as mariadb-abackup working directory" + wsrep_log_info "Using '$itmpdir' as mariadb-backup working directory" usrst=0 if [ -n "$WSREP_SST_OPT_USER" ]; then From b7a75fbb8a93cf732b1b562647486246ea7f70dc Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 7 Jun 2024 13:11:17 +0530 Subject: [PATCH 35/59] MDEV-34169 Don't allow innodb_open_files to be lesser than number of non-user tablespace. - InnoDB only closes the user tablespace when the number of open files exceeds innodb_open_files limit. In that case, InnoDB should make sure that innodb_open_files value should be greater than number of undo tablespace, system and temporary tablespace files. --- mysql-test/suite/innodb/r/open_files_limit.result | 4 ++++ mysql-test/suite/innodb/t/open_files_limit.opt | 3 +++ mysql-test/suite/innodb/t/open_files_limit.test | 9 +++++++++ storage/innobase/handler/ha_innodb.cc | 15 +++++++++++++++ 4 files changed, 31 insertions(+) create mode 100644 mysql-test/suite/innodb/r/open_files_limit.result create mode 100644 mysql-test/suite/innodb/t/open_files_limit.opt create mode 100644 mysql-test/suite/innodb/t/open_files_limit.test diff --git a/mysql-test/suite/innodb/r/open_files_limit.result b/mysql-test/suite/innodb/r/open_files_limit.result new file mode 100644 index 00000000000..f2f63376495 --- /dev/null +++ b/mysql-test/suite/innodb/r/open_files_limit.result @@ -0,0 +1,4 @@ +call mtr.add_suppression("\\[Warning\\] InnoDB: innodb_open_files=.* is not greater than the number of system tablespace files, temporary tablespace files, innodb_undo_tablespaces=.*"); +FOUND 1 /\[Warning\] InnoDB: innodb_open_files=.* is not greater than the number of system tablespace files, temporary tablespace files, innodb_undo_tablespaces=.*/ in mysqld.1.err +CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/open_files_limit.opt b/mysql-test/suite/innodb/t/open_files_limit.opt new file mode 100644 index 00000000000..cd71a062da8 --- /dev/null +++ b/mysql-test/suite/innodb/t/open_files_limit.opt @@ -0,0 +1,3 @@ +--innodb_undo_tablespaces=8 +--innodb_open_files=10 +--innodb_temp_data_file_path=ibtmp1:32M;ibtmp2:32M:autoextend diff --git a/mysql-test/suite/innodb/t/open_files_limit.test b/mysql-test/suite/innodb/t/open_files_limit.test new file mode 100644 index 00000000000..ea7383150a8 --- /dev/null +++ b/mysql-test/suite/innodb/t/open_files_limit.test @@ -0,0 +1,9 @@ +--source include/have_innodb.inc +--source include/not_embedded.inc +call mtr.add_suppression("\\[Warning\\] InnoDB: innodb_open_files=.* is not greater than the number of system tablespace files, temporary tablespace files, innodb_undo_tablespaces=.*"); +let SEARCH_PATTERN= \[Warning\] InnoDB: innodb_open_files=.* is not greater than the number of system tablespace files, temporary tablespace files, innodb_undo_tablespaces=.*; +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +--source include/search_pattern_in_file.inc + +CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; +DROP TABLE t1; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 7bed0ed73eb..ae79e7f39a0 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3860,6 +3860,21 @@ static int innodb_init_params() } } + ulint min_open_files_limit = srv_undo_tablespaces + + srv_sys_space.m_files.size() + + srv_tmp_space.m_files.size() + 1; + if (min_open_files_limit > innobase_open_files) { + sql_print_warning( + "InnoDB: innodb_open_files=%lu is not greater " + "than the number of system tablespace files, " + "temporary tablespace files, " + "innodb_undo_tablespaces=%lu; adjusting " + "to innodb_open_files=%zu", + innobase_open_files, srv_undo_tablespaces, + min_open_files_limit); + innobase_open_files = (ulong) min_open_files_limit; + } + srv_max_n_open_files = innobase_open_files; srv_innodb_status = (ibool) innobase_create_status_file; From d9d0e8fd604bacf21e62a78d81c9763b90a79843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 7 Jun 2024 13:51:46 +0300 Subject: [PATCH 36/59] MDEV-34321: call to crc32c_3way through pointer to incorrect function type In commit 9ec7819c585d139c8fe64d0f7f0f0f51dcafa01f the CRC-32 function signatures had been unified somewhat, but not enough. clang -fsanitize=undefined would flag a function pointer signature mismatch between const char* and const void*, but not between uint32_t and unsigned. We try to fix both inconsistencies anyway. Reviewed by: Vladislav Vaintroub --- mysys/crc32/crc32c.cc | 2 +- mysys/crc32/crc32c_amd64.cc | 4 ++-- mysys/crc32/crc32c_x86.cc | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mysys/crc32/crc32c.cc b/mysys/crc32/crc32c.cc index 32a45478e94..6de7af70db7 100644 --- a/mysys/crc32/crc32c.cc +++ b/mysys/crc32/crc32c.cc @@ -526,7 +526,7 @@ extern "C" const char *my_crc32c_implementation() } // namespace crc32c } // namespace mysys_namespace -extern "C" unsigned my_crc32c(unsigned int crc, const char *buf, size_t size) +extern "C" uint32 my_crc32c(uint32 crc, const void *buf, size_t size) { return mysys_namespace::crc32c::ChosenExtend(crc,buf, size); } diff --git a/mysys/crc32/crc32c_amd64.cc b/mysys/crc32/crc32c_amd64.cc index 147c0ccaba6..6706ed0fadc 100644 --- a/mysys/crc32/crc32c_amd64.cc +++ b/mysys/crc32/crc32c_amd64.cc @@ -184,9 +184,9 @@ static inline uint64_t CombineCRC( // Compute CRC-32C using the Intel hardware instruction. extern "C" USE_PCLMUL -uint32_t crc32c_3way(uint32_t crc, const char *buf, size_t len) +uint32_t crc32c_3way(uint32_t crc, const void *buf, size_t len) { - const unsigned char* next = (const unsigned char*)buf; + const unsigned char* next = static_cast(buf); uint64_t count; uint64_t crc0, crc1, crc2; crc0 = crc ^ 0xffffffffu; diff --git a/mysys/crc32/crc32c_x86.cc b/mysys/crc32/crc32c_x86.cc index 1ea0689a14b..6d9169f9384 100644 --- a/mysys/crc32/crc32c_x86.cc +++ b/mysys/crc32/crc32c_x86.cc @@ -54,9 +54,9 @@ static uint32_t cpuid_ecx() #endif } -typedef unsigned (*my_crc32_t)(unsigned, const void *, size_t); -extern "C" unsigned int crc32_pclmul(unsigned int, const void *, size_t); -extern "C" unsigned int crc32c_3way(unsigned int, const void *, size_t); +typedef uint32_t (*my_crc32_t)(uint32_t, const void *, size_t); +extern "C" uint32_t crc32_pclmul(uint32_t, const void *, size_t); +extern "C" uint32_t crc32c_3way(uint32_t, const void *, size_t); #ifdef USE_VPCLMULQDQ # include From 77c4c0f256f3c268d3f72625b04240d24a70513c Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Fri, 7 Jun 2024 12:13:21 +0200 Subject: [PATCH 37/59] MDEV-34203 Sandbox mode \- is not compatible with --binary-mode "Process" sandbox short command put by masqldump to avoid an error. --- client/mysql.cc | 30 +++++++++++++++++++++++++----- mysql-test/main/mysql.result | 11 +++++++++++ mysql-test/main/mysql.test | 16 ++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/client/mysql.cc b/client/mysql.cc index 5c79dd7890b..1e235c8a34d 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -1117,6 +1117,8 @@ inline int get_command_index(char cmd_char) static int delimiter_index= -1; static int charset_index= -1; +static int sandbox_index= -1; + static bool real_binary_mode= FALSE; @@ -1127,7 +1129,8 @@ int main(int argc,char *argv[]) MY_INIT(argv[0]); DBUG_ENTER("main"); DBUG_PROCESS(argv[0]); - + + sandbox_index= get_command_index('-'); charset_index= get_command_index('C'); delimiter_index= get_command_index('d'); delimiter_str= delimiter; @@ -2234,8 +2237,9 @@ static int read_and_execute(bool interactive) /** It checks if the input is a short form command. It returns the command's - pointer if a command is found, else return NULL. Note that if binary-mode - is set, then only \C is searched for. + pointer if a command is found, else return NULL. + + Note that if binary-mode is set, then only \C and \- are searched for. @param cmd_char A character of one byte. @@ -2250,13 +2254,23 @@ static COMMANDS *find_command(char cmd_char) int index= -1; /* - In binary-mode, we disallow all mysql commands except '\C' - and DELIMITER. + In binary-mode, we disallow all client commands except '\C', + DELIMITER (see long comand finding find_command(char *)) + and '\-' (sandbox, see following comment). */ if (real_binary_mode) { if (cmd_char == 'C') index= charset_index; + /* + binary-mode enforces stricter controls compared to sandbox mode. + Whether sandbox mode is enabled or not is irrelevant when + binary-mode is active. + The only purpose of processing sandbox mode here is to avoid error + messages on files made by mysqldump. + */ + else if (cmd_char == '-') + index= sandbox_index; } else index= get_command_index(cmd_char); @@ -2312,6 +2326,12 @@ static COMMANDS *find_command(char *name) len= (uint) strlen(name); int index= -1; + /* + In binary-mode, we disallow all client commands except DELIMITER + and short commands '\C' and '\-' (see short command finding + find_command(char)). + */ + if (real_binary_mode) { if (is_delimiter_command(name, len)) diff --git a/mysql-test/main/mysql.result b/mysql-test/main/mysql.result index c6834d0fc20..445aa602241 100644 --- a/mysql-test/main/mysql.result +++ b/mysql-test/main/mysql.result @@ -658,4 +658,15 @@ tee source ^^^ 3 +# +# MDEV-34203: Sandbox mode \- is not compatible with --binary-mode +# +create table t1 (a int); +drop table t1; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +drop table t1; # End of 10.5 tests diff --git a/mysql-test/main/mysql.test b/mysql-test/main/mysql.test index 2f189e70b18..fe79b2c87cb 100644 --- a/mysql-test/main/mysql.test +++ b/mysql-test/main/mysql.test @@ -757,4 +757,20 @@ source $MYSQL_TMP_DIR/mysql_in;" $MYSQL_TMP_DIR/mysql_in2; --remove_file $MYSQL_TMP_DIR/mysql_in --remove_file $MYSQL_TMP_DIR/mysql_in2 +--echo # +--echo # MDEV-34203: Sandbox mode \- is not compatible with --binary-mode +--echo # + +create table t1 (a int); + +--exec $MYSQL_DUMP test t1 > $MYSQLTEST_VARDIR/tmp/MDEV-34203.sql + +drop table t1; + +--exec $MYSQL --binary-mode test 2>&1 < $MYSQLTEST_VARDIR/tmp/MDEV-34203.sql + +show create table t1; +drop table t1; +--remove_file $MYSQLTEST_VARDIR/tmp/MDEV-34203.sql + --echo # End of 10.5 tests From 4b4dbb23eac79040b39173693a53124183108317 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 7 Jun 2024 20:24:39 +0530 Subject: [PATCH 38/59] MDEV-34169 Don't allow innodb_open_files to be lesser than number of non-user tablespace. fil_space_t::try_to_close(): Don't try to close the tablespace which is acquired by the caller of the function Added the suppression message in open_files_limit test case --- mysql-test/suite/innodb/r/open_files_limit.result | 1 + mysql-test/suite/innodb/t/open_files_limit.test | 1 + storage/innobase/fil/fil0fil.cc | 12 +++++++----- storage/innobase/include/fil0fil.h | 3 ++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/innodb/r/open_files_limit.result b/mysql-test/suite/innodb/r/open_files_limit.result index f2f63376495..1c66bab7533 100644 --- a/mysql-test/suite/innodb/r/open_files_limit.result +++ b/mysql-test/suite/innodb/r/open_files_limit.result @@ -1,4 +1,5 @@ call mtr.add_suppression("\\[Warning\\] InnoDB: innodb_open_files=.* is not greater than the number of system tablespace files, temporary tablespace files, innodb_undo_tablespaces=.*"); +call mtr.add_suppression("\\[Warning\\] InnoDB: innodb_open_files=.* is exceeded \\(.* files stay open\\)"); FOUND 1 /\[Warning\] InnoDB: innodb_open_files=.* is not greater than the number of system tablespace files, temporary tablespace files, innodb_undo_tablespaces=.*/ in mysqld.1.err CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/open_files_limit.test b/mysql-test/suite/innodb/t/open_files_limit.test index ea7383150a8..c10f62f3333 100644 --- a/mysql-test/suite/innodb/t/open_files_limit.test +++ b/mysql-test/suite/innodb/t/open_files_limit.test @@ -1,6 +1,7 @@ --source include/have_innodb.inc --source include/not_embedded.inc call mtr.add_suppression("\\[Warning\\] InnoDB: innodb_open_files=.* is not greater than the number of system tablespace files, temporary tablespace files, innodb_undo_tablespaces=.*"); +call mtr.add_suppression("\\[Warning\\] InnoDB: innodb_open_files=.* is exceeded \\(.* files stay open\\)"); let SEARCH_PATTERN= \[Warning\] InnoDB: innodb_open_files=.* is not greater than the number of system tablespace files, temporary tablespace files, innodb_undo_tablespaces=.*; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; --source include/search_pattern_in_file.inc diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 88c1115969a..ba4fbc49204 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -66,9 +66,10 @@ inline bool fil_is_user_tablespace_id(ulint space_id) } /** Try to close a file to adhere to the innodb_open_files limit. +@param ignore_space Ignore the tablespace which is acquired by caller @param print_info whether to diagnose why a file cannot be closed @return whether a file was closed */ -bool fil_space_t::try_to_close(bool print_info) +bool fil_space_t::try_to_close(fil_space_t *ignore_space, bool print_info) { ut_ad(mutex_own(&fil_system.mutex)); for (fil_space_t *space= UT_LIST_GET_FIRST(fil_system.space_list); space; @@ -80,7 +81,8 @@ bool fil_space_t::try_to_close(bool print_info) case FIL_TYPE_IMPORT: break; case FIL_TYPE_TABLESPACE: - if (!fil_is_user_tablespace_id(space->id)) + if (space == ignore_space + || !fil_is_user_tablespace_id(space->id)) continue; } @@ -354,7 +356,7 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, n_pending.fetch_and(~CLOSING, std::memory_order_relaxed); if (++fil_system.n_open >= srv_max_n_open_files) { reacquire(); - try_to_close(true); + try_to_close(this, true); release(); } } @@ -405,7 +407,7 @@ static bool fil_node_open_file_low(fil_node_t *node) /* The following call prints an error message */ if (os_file_get_last_error(true) == EMFILE + 100 && - fil_space_t::try_to_close(true)) + fil_space_t::try_to_close(nullptr, true)) continue; ib::warn() << "Cannot open '" << node->name << "'."; @@ -449,7 +451,7 @@ static bool fil_node_open_file(fil_node_t *node) for (ulint count= 0; fil_system.n_open >= srv_max_n_open_files; count++) { - if (fil_space_t::try_to_close(count > 1)) + if (fil_space_t::try_to_close(nullptr, count > 1)) count= 0; else if (count >= 2) { diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 1f5589a4a20..059a3456c50 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -590,9 +590,10 @@ private: public: /** Try to close a file to adhere to the innodb_open_files limit. + @param ignore_space Ignore the tablespace which is acquired by caller @param print_info whether to diagnose why a file cannot be closed @return whether a file was closed */ - static bool try_to_close(bool print_info); + static bool try_to_close(fil_space_t *ignore_space, bool print_info); /** Close all tablespace files at shutdown */ static void close_all(); From e255837eaf90e72eaf122d88b30011db17f7ecf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 30 May 2024 09:16:42 +0300 Subject: [PATCH 39/59] MDEV-34266 safe_strcpy() includes an unnecessary conditional branch The strncpy() wrapper that was introduced in commit 567b68129943a1cceab1d7b4c68e2a4ba011cdc0 is checking whether the output was truncated even in cases where the caller does not care about it. Let us introduce a separate function safe_strcpy_truncated() that indidates whether the output was truncated. --- client/mysql_plugin.c | 8 +++--- client/mysqltest.cc | 5 ++-- include/m_string.h | 63 ++++++++++++++++++++++--------------------- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/client/mysql_plugin.c b/client/mysql_plugin.c index 73fa43752a9..679fbf3204c 100644 --- a/client/mysql_plugin.c +++ b/client/mysql_plugin.c @@ -686,7 +686,7 @@ static int load_plugin_data(char *plugin_name, char *config_file) if (i == -1) /* if first pass, read this line as so_name */ { /* Add proper file extension for soname */ - if (safe_strcpy(line + line_len - 1, sizeof(line), FN_SOEXT)) + if (safe_strcpy_truncated(line + line_len - 1, sizeof line, FN_SOEXT)) { reason= "Plugin name too long."; fclose(file_ptr); @@ -749,7 +749,7 @@ static int check_options(int argc, char **argv, char *operation) const char *plugin_dir_prefix = "--plugin_dir="; size_t plugin_dir_len= strlen(plugin_dir_prefix); - strcpy(plugin_name, ""); + *plugin_name= '\0'; for (i = 0; i < argc && num_found < 5; i++) { @@ -787,8 +787,8 @@ static int check_options(int argc, char **argv, char *operation) /* read the plugin config file and check for match against argument */ else { - if (safe_strcpy(plugin_name, sizeof(plugin_name), argv[i]) || - safe_strcpy(config_file, sizeof(config_file), argv[i]) || + if (safe_strcpy_truncated(plugin_name, sizeof plugin_name, argv[i]) || + safe_strcpy_truncated(config_file, sizeof config_file, argv[i]) || safe_strcat(config_file, sizeof(config_file), ".ini")) { fprintf(stderr, "ERROR: argument is too long.\n"); diff --git a/client/mysqltest.cc b/client/mysqltest.cc index fdee3e09851..fa542fe7787 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -6265,7 +6265,7 @@ int do_done(struct st_command *command) if (*cur_block->delim) { /* Restore "old" delimiter after false if block */ - if (safe_strcpy(delimiter, sizeof(delimiter), cur_block->delim)) + if (safe_strcpy_truncated(delimiter, sizeof delimiter, cur_block->delim)) die("Delimiter too long, truncated"); delimiter_length= strlen(delimiter); @@ -6526,7 +6526,8 @@ void do_block(enum block_cmd cmd, struct st_command* command) else { /* Remember "old" delimiter if entering a false if block */ - if (safe_strcpy(cur_block->delim, sizeof(cur_block->delim), delimiter)) + if (safe_strcpy_truncated(cur_block->delim, sizeof cur_block->delim, + delimiter)) die("Delimiter too long, truncated"); } diff --git a/include/m_string.h b/include/m_string.h index c45e62770ef..3da252b0912 100644 --- a/include/m_string.h +++ b/include/m_string.h @@ -239,15 +239,14 @@ static inline void lex_string_set3(LEX_CSTRING *lex_str, const char *c_str, lex_str->length= len; } -/* - Copies src into dst and ensures dst is a NULL terminated C string. +/** + Copies a string. - Returns 1 if the src string was truncated due to too small size of dst. - Returns 0 if src completely fit within dst. Pads the remaining dst with '\0' - - Note: dst_size must be > 0 + @param dst destination buffer, will be NUL padded. + @param dst_size size of dst buffer, must be > 0 + @param src NUL terminated source string */ -static inline int safe_strcpy(char *dst, size_t dst_size, const char *src) +static inline void safe_strcpy(char *dst, size_t dst_size, const char *src) { DBUG_ASSERT(dst_size > 0); @@ -256,45 +255,49 @@ static inline int safe_strcpy(char *dst, size_t dst_size, const char *src) * * 2) IF there is no 0 byte in the first dst_size bytes of src, strncpy will * copy dst_size bytes, and the final byte won't be 0. - * - * In GCC 8+, the `-Wstringop-truncation` warning will object to strncpy() - * being used in this way, so we need to disable this warning for this - * single statement. */ -#if defined(__GNUC__) && __GNUC__ >= 8 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wstringop-truncation" -#endif strncpy(dst, src, dst_size); -#if defined(__GNUC__) && __GNUC__ >= 8 -#pragma GCC diagnostic pop -#endif + dst[dst_size - 1]= 0; +} - if (dst[dst_size-1]) +/** + Copies a string, checking for truncation. + + @param dst destination buffer, will be NUL padded. + @param dst_size size of dst buffer, must be > 0 + @param src NUL terminated source string + + @retval 1 if the src string was truncated due to too small size of dst. + @retval 0 if src completely fit within dst, +*/ +static inline int safe_strcpy_truncated(char *dst, size_t dst_size, + const char *src) +{ + DBUG_ASSERT(dst_size > 0); + + strncpy(dst, src, dst_size); + if (dst[dst_size - 1]) { - /* Only possible in case (2), meaning src was truncated. */ - dst[dst_size-1]= 0; + dst[dst_size - 1]= 0; return 1; } return 0; } -/* - Appends src to dst and ensures dst is a NULL terminated C string. +/** + Appends src to dst and ensures dst is a NUL terminated C string. - Returns 1 if the src string was truncated due to too small size of dst. - Returns 0 if src completely fit within the remaining dst space. Pads the - remaining dst with '\0'. - - Note: dst_size must be > 0 + @retval 1 if the src string was truncated due to too small size of dst. + @retval 0 if src completely fit within the remaining dst space, + including NUL termination. */ static inline int safe_strcat(char *dst, size_t dst_size, const char *src) { size_t init_len= strlen(dst); - if (unlikely(init_len >= dst_size - 1)) + if (unlikely(init_len > dst_size)) return 1; - return safe_strcpy(dst + init_len, dst_size - init_len, src); + return safe_strcpy_truncated(dst + init_len, dst_size - init_len, src); } #ifdef __cplusplus From 0172887980cd6297cbda1dda79ecbde0568c4664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 6 Jun 2024 10:35:56 +0300 Subject: [PATCH 40/59] MDEV-34269 : 10.11.8 cluster becomes inconsistent when using composite primary key and partitioning This is regression from commit 3228c08fa8. Problem is that when table storage engine is determined there should be check is table partitioned and if it is then determine partition implementing storage engine. Reported bug is reproducible only with --log-bin so make sure tests changed by 3228c08fa8 and new test are run with --log-bin and binlog disabled. Signed-off-by: Julius Goryavsky --- mysql-test/include/log_bin.combinations | 4 ++ mysql-test/include/log_bin.inc | 3 + .../galera/r/galera_partition_key.result | 46 ++++++++++++++++ .../galera/t/galera_myisam_autocommit.test | 1 + .../suite/galera/t/galera_partition.test | 1 + .../suite/galera/t/galera_partition_key.test | 55 +++++++++++++++++++ mysql-test/suite/galera/t/mdev-22063.test | 2 +- sql/sql_parse.cc | 4 +- 8 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 mysql-test/include/log_bin.combinations create mode 100644 mysql-test/include/log_bin.inc create mode 100644 mysql-test/suite/galera/r/galera_partition_key.result create mode 100644 mysql-test/suite/galera/t/galera_partition_key.test diff --git a/mysql-test/include/log_bin.combinations b/mysql-test/include/log_bin.combinations new file mode 100644 index 00000000000..1eeb8fb4614 --- /dev/null +++ b/mysql-test/include/log_bin.combinations @@ -0,0 +1,4 @@ +[binlogoff] + +[binlogon] +log-bin diff --git a/mysql-test/include/log_bin.inc b/mysql-test/include/log_bin.inc new file mode 100644 index 00000000000..d318c7967e1 --- /dev/null +++ b/mysql-test/include/log_bin.inc @@ -0,0 +1,3 @@ +# include file for test files that can be run with and without log-bin +# (see include/log_bin.combinations) + diff --git a/mysql-test/suite/galera/r/galera_partition_key.result b/mysql-test/suite/galera/r/galera_partition_key.result new file mode 100644 index 00000000000..b4965482779 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_partition_key.result @@ -0,0 +1,46 @@ +connection node_2; +connection node_1; +connection node_1; +CREATE TABLE `t1` ( +`id` int(10) unsigned NOT NULL, +`other_id` int(10) unsigned NOT NULL, +PRIMARY KEY (`id`,`other_id`) +) ENGINE=InnoDB +PARTITION BY LIST (`id` MOD 2) +(PARTITION `p0` VALUES IN (0) ENGINE = InnoDB, +PARTITION `p1` VALUES IN (1) ENGINE = InnoDB); +INSERT INTO t1 VALUES (1, 0); +CREATE TABLE t2 LIKE t1; +START TRANSACTION; +INSERT INTO t2(SELECT * FROM t1 WHERE id = 1); +DELETE FROM t1 WHERE id = 1; +COMMIT; +connection node_2; +SELECT * from t1; +id other_id +SELECT * from t2; +id other_id +1 0 +DROP TABLE t1, t2; +connection node_1; +CREATE TABLE `t1` ( +`id` int(10) unsigned NOT NULL, +`other_id` int(10) unsigned NOT NULL, +PRIMARY KEY (`id`) +) ENGINE=InnoDB +PARTITION BY LIST (`id` MOD 2) +(PARTITION `p0` VALUES IN (0) ENGINE = InnoDB, +PARTITION `p1` VALUES IN (1) ENGINE = InnoDB); +INSERT INTO t1 VALUES (1, 0); +CREATE TABLE t2 LIKE t1; +START TRANSACTION; +INSERT INTO t2(SELECT * FROM t1 WHERE id = 1); +DELETE FROM t1 WHERE id = 1; +COMMIT; +connection node_2; +SELECT * from t1; +id other_id +SELECT * from t2; +id other_id +1 0 +DROP TABLE t1, t2; diff --git a/mysql-test/suite/galera/t/galera_myisam_autocommit.test b/mysql-test/suite/galera/t/galera_myisam_autocommit.test index 65f957e8422..00799cf2479 100644 --- a/mysql-test/suite/galera/t/galera_myisam_autocommit.test +++ b/mysql-test/suite/galera/t/galera_myisam_autocommit.test @@ -1,5 +1,6 @@ --source include/galera_cluster.inc --source include/have_innodb.inc +--source include/log_bin.inc # # This tests simple autocommit replication of MyISAM tables. diff --git a/mysql-test/suite/galera/t/galera_partition.test b/mysql-test/suite/galera/t/galera_partition.test index 269291311f1..d69f87b6472 100644 --- a/mysql-test/suite/galera/t/galera_partition.test +++ b/mysql-test/suite/galera/t/galera_partition.test @@ -2,6 +2,7 @@ --source include/have_partition.inc --source include/big_test.inc --source include/force_restart.inc +--source include/log_bin.inc --connection node_1 diff --git a/mysql-test/suite/galera/t/galera_partition_key.test b/mysql-test/suite/galera/t/galera_partition_key.test new file mode 100644 index 00000000000..ac1fcf72f06 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_partition_key.test @@ -0,0 +1,55 @@ +--source include/galera_cluster.inc +--source include/have_partition.inc +--source include/log_bin.inc + +--connection node_1 +CREATE TABLE `t1` ( + `id` int(10) unsigned NOT NULL, + `other_id` int(10) unsigned NOT NULL, + PRIMARY KEY (`id`,`other_id`) +) ENGINE=InnoDB + PARTITION BY LIST (`id` MOD 2) +(PARTITION `p0` VALUES IN (0) ENGINE = InnoDB, + PARTITION `p1` VALUES IN (1) ENGINE = InnoDB); + +INSERT INTO t1 VALUES (1, 0); + +CREATE TABLE t2 LIKE t1; + +START TRANSACTION; + +INSERT INTO t2(SELECT * FROM t1 WHERE id = 1); +DELETE FROM t1 WHERE id = 1; + +COMMIT; + +--connection node_2 +SELECT * from t1; +SELECT * from t2; +DROP TABLE t1, t2; + +--connection node_1 +CREATE TABLE `t1` ( + `id` int(10) unsigned NOT NULL, + `other_id` int(10) unsigned NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB + PARTITION BY LIST (`id` MOD 2) +(PARTITION `p0` VALUES IN (0) ENGINE = InnoDB, + PARTITION `p1` VALUES IN (1) ENGINE = InnoDB); + +INSERT INTO t1 VALUES (1, 0); + +CREATE TABLE t2 LIKE t1; + +START TRANSACTION; + +INSERT INTO t2(SELECT * FROM t1 WHERE id = 1); +DELETE FROM t1 WHERE id = 1; + +COMMIT; + +--connection node_2 +SELECT * from t1; +SELECT * from t2; +DROP TABLE t1, t2; diff --git a/mysql-test/suite/galera/t/mdev-22063.test b/mysql-test/suite/galera/t/mdev-22063.test index a8e7b87ad38..ef16c0c8716 100644 --- a/mysql-test/suite/galera/t/mdev-22063.test +++ b/mysql-test/suite/galera/t/mdev-22063.test @@ -1,6 +1,6 @@ --source include/galera_cluster.inc --source include/have_innodb.inc ---source include/have_log_bin.inc +--source include/log_bin.inc --source include/have_sequence.inc --source include/have_aria.inc diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index b071c74b245..ca991b34e9f 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4747,7 +4747,9 @@ mysql_execute_command(THD *thd) #ifdef WITH_WSREP if (wsrep && !first_table->view) { - bool is_innodb= (first_table->table->file->ht->db_type == DB_TYPE_INNODB); + const handlerton *hton = first_table->table->file->partition_ht() ? + first_table->table->file->partition_ht() : first_table->table->file->ht; + bool is_innodb= (hton->db_type == DB_TYPE_INNODB); // For consistency check inserted table needs to be InnoDB if (!is_innodb && thd->wsrep_consistency_check != NO_CONSISTENCY_CHECK) From 0d85c905c44f9d88e4a17f56594cc389d777abaf Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Fri, 7 Jun 2024 14:53:11 +0200 Subject: [PATCH 41/59] MDEV-34269: post-fix code simplification The code is slightly simplified taking into account the fact that partition_ht() always returns a normal hton when there is no partitioning. --- sql/sql_load.cc | 4 +--- sql/sql_parse.cc | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/sql/sql_load.cc b/sql/sql_load.cc index 0eced4e74b4..7329a7d2db5 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -117,10 +117,8 @@ public: */ if (WSREP(thd) && wsrep_load_data_splitting) { - handlerton *ht= table->s->db_type(); // For partitioned tables find underlying hton - if (table->file->partition_ht()) - ht= table->file->partition_ht(); + handlerton *ht= table->file->partition_ht(); if (ht->db_type != DB_TYPE_INNODB) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index ca991b34e9f..69164a95be6 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -4747,9 +4747,7 @@ mysql_execute_command(THD *thd) #ifdef WITH_WSREP if (wsrep && !first_table->view) { - const handlerton *hton = first_table->table->file->partition_ht() ? - first_table->table->file->partition_ht() : first_table->table->file->ht; - bool is_innodb= (hton->db_type == DB_TYPE_INNODB); + bool is_innodb= first_table->table->file->partition_ht()->db_type == DB_TYPE_INNODB; // For consistency check inserted table needs to be InnoDB if (!is_innodb && thd->wsrep_consistency_check != NO_CONSISTENCY_CHECK) From bf0aa99aeb7dd2bb7497f5ae9a82d1e72e7ee1d5 Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Thu, 6 Jun 2024 11:46:27 -0600 Subject: [PATCH 42/59] MDEV-34237: On Startup: UBSAN: runtime error: call to function MDL_lock::lf_hash_initializer lf_hash_insert through pointer to incorrect function type 'void (*)(st_lf_hash *, void *, const void *)' A few different incorrect function type UBSAN issues have been grouped into this patch. The only real potentially undefined behavior is an error about show_func_mutex_instances_lost, which when invoked in sql_show.cc::show_status_array(), puts 5 arguments onto the stack; however, the implementing function only actually has 3 parameters (so only 3 would be popped). This was fixed by adding in the remaining parameters to satisfy the type mysql_show_var_func. The rest of the findings are pointer type mismatches that wouldn't lead to actual undefined behavior. The lf_hash_initializer function type definition is typedef void (*lf_hash_initializer)(LF_HASH *hash, void *dst, const void *src); but the MDL_lock and table cache's implementations of this function do not have that signature. The MDL_lock has specific MDL object parameters: static void lf_hash_initializer(LF_HASH *hash __attribute__((unused)), MDL_lock *lock, MDL_key *key_arg) and the table cache has specific TDC parameters: static void tdc_hash_initializer(LF_HASH *, TDC_element *element, LEX_STRING *key) leading to UBSAN runtime errors when invoking these functions. This patch fixes these type mis-matches by changing the implementing functions to use void * and const void * for their respective parameters, and later casting them to their expected type in the function body. Note too the functions tdc_hash_key and tc_purge_callback had a similar problem to tdc_hash_initializer and was fixed similarly. Reviewed By: ============ Sergei Golubchik --- sql/mdl.cc | 4 +++- sql/sql_show.cc | 2 +- sql/table_cache.cc | 13 +++++++++---- storage/perfschema/ha_perfschema.cc | 4 +++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/sql/mdl.cc b/sql/mdl.cc index 6b8d3599c00..fb106a1af47 100644 --- a/sql/mdl.cc +++ b/sql/mdl.cc @@ -673,8 +673,10 @@ public: { ((MDL_lock*)(arg + LF_HASH_OVERHEAD))->~MDL_lock(); } static void lf_hash_initializer(LF_HASH *hash __attribute__((unused)), - MDL_lock *lock, MDL_key *key_arg) + void *_lock, const void *_key_arg) { + MDL_lock *lock= static_cast(_lock); + const MDL_key *key_arg= static_cast(_key_arg); DBUG_ASSERT(key_arg->mdl_namespace() != MDL_key::BACKUP); new (&lock->key) MDL_key(key_arg); if (key_arg->mdl_namespace() == MDL_key::SCHEMA) diff --git a/sql/sql_show.cc b/sql/sql_show.cc index eb84ef56419..ca291bd9e5d 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -3789,7 +3789,7 @@ static bool show_status_array(THD *thd, const char *wild, */ for (var=variables; var->type == SHOW_FUNC || var->type == SHOW_SIMPLE_FUNC; var= &tmp) - ((mysql_show_var_func)(var->value))(thd, &tmp, buff, + ((mysql_show_var_func)(var->value))(thd, &tmp, (void *) buff, status_var, scope); SHOW_TYPE show_type=var->type; diff --git a/sql/table_cache.cc b/sql/table_cache.cc index 499e82fafea..b76ba4fd239 100644 --- a/sql/table_cache.cc +++ b/sql/table_cache.cc @@ -293,9 +293,11 @@ static void tc_remove_all_unused_tables(TDC_element *element, periodicly flush all not used tables. */ -static my_bool tc_purge_callback(TDC_element *element, - Share_free_tables::List *purge_tables) +static my_bool tc_purge_callback(void *_element, void *_purge_tables) { + TDC_element *element= static_cast(_element); + Share_free_tables::List *purge_tables= + static_cast(_purge_tables); mysql_mutex_lock(&element->LOCK_table_share); tc_remove_all_unused_tables(element, purge_tables); mysql_mutex_unlock(&element->LOCK_table_share); @@ -566,17 +568,20 @@ static void lf_alloc_destructor(uchar *arg) static void tdc_hash_initializer(LF_HASH *, - TDC_element *element, LEX_STRING *key) + void *_element, const void *_key) { + TDC_element *element= static_cast(_element); + const LEX_STRING *key= static_cast(_key); memcpy(element->m_key, key->str, key->length); element->m_key_length= (uint)key->length; tdc_assert_clean_share(element); } -static uchar *tdc_hash_key(const TDC_element *element, size_t *length, +static uchar *tdc_hash_key(const unsigned char *_element, size_t *length, my_bool) { + const TDC_element *element= (const TDC_element *) _element; *length= element->m_key_length; return (uchar*) element->m_key; } diff --git a/storage/perfschema/ha_perfschema.cc b/storage/perfschema/ha_perfschema.cc index c6d2f23d653..8b7825f3816 100644 --- a/storage/perfschema/ha_perfschema.cc +++ b/storage/perfschema/ha_perfschema.cc @@ -133,7 +133,9 @@ static int pfs_done_func(void *p) DBUG_RETURN(0); } -static int show_func_mutex_instances_lost(THD *thd, SHOW_VAR *var, char *buff) +static int show_func_mutex_instances_lost(THD *thd, SHOW_VAR *var, void *buff, + struct system_status_var *status_var, + enum enum_var_type) { var->type= SHOW_LONG; var->value= buff; From 21f56583bf209e00907203ca1e313f8d0a7d8d7f Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 10 Jun 2024 09:31:14 +0400 Subject: [PATCH 43/59] MDEV-32376 SHOW CREATE DATABASE statement crashes the server when db name contains some unicode characters, ASAN stack-buffer-overflow Adding the test for the length of lex->name into show_create_db(). Without this test writes beyond the end of db_name_buff were possible upon a too long database name. --- mysql-test/main/create.result | 7 +++++++ mysql-test/main/create.test | 9 +++++++++ sql/sql_parse.cc | 17 +++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/mysql-test/main/create.result b/mysql-test/main/create.result index fc508108a97..95812106c7f 100644 --- a/mysql-test/main/create.result +++ b/mysql-test/main/create.result @@ -2061,4 +2061,11 @@ DROP TABLE t1; # CREATE TABLE t1 (id1 INT, id2 INT, primary key (id1), unique index (id2) visible); drop table t1; +# +# MDEV-32376 SHOW CREATE DATABASE statement crashes the server when db name contains some unicode characters, ASAN stack-buffer-overflow +# +SET NAMES utf8mb3; +SHOW CREATE DATABASE `#testone#￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭`; +ERROR 42000: Incorrect database name '#testone#￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭...' +SET NAMES DEFAULT; # End of 10.5 Test diff --git a/mysql-test/main/create.test b/mysql-test/main/create.test index e9470edefab..14541d84dc9 100644 --- a/mysql-test/main/create.test +++ b/mysql-test/main/create.test @@ -1935,4 +1935,13 @@ DROP TABLE t1; CREATE TABLE t1 (id1 INT, id2 INT, primary key (id1), unique index (id2) visible); drop table t1; +--echo # +--echo # MDEV-32376 SHOW CREATE DATABASE statement crashes the server when db name contains some unicode characters, ASAN stack-buffer-overflow +--echo # + +SET NAMES utf8mb3; +--error ER_WRONG_DB_NAME +SHOW CREATE DATABASE `#testone#￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭￭`; +SET NAMES DEFAULT; + --echo # End of 10.5 Test diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 69164a95be6..559af2fc813 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -6637,6 +6637,23 @@ show_create_db(THD *thd, LEX *lex) DBUG_EXECUTE_IF("4x_server_emul", my_error(ER_UNKNOWN_ERROR, MYF(0)); return 1;); +#if MYSQL_VERSION_ID<=110301 + /* + This piece of the code was added in 10.5 to fix MDEV-32376. + It should not get to 11.3 or higer, as MDEV-32376 was fixed + in a different way in 11.3.1 (see MDEV-31948). + */ + if (lex->name.length > sizeof(db_name_buff) - 1) + { + my_error(ER_WRONG_DB_NAME, MYF(0), + ErrConvString(lex->name.str, lex->name.length, + system_charset_info).ptr()); + return 1; + } +#else +#error Remove this preprocessor-conditional code in 11.3.1+ +#endif + db_name.str= db_name_buff; db_name.length= lex->name.length; strmov(db_name_buff, lex->name.str); From 246c0b3a353d0831fb00ac4cd46599a33a808d0c Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 10 Jun 2024 12:17:01 +0400 Subject: [PATCH 44/59] MDEV-34227 On startup: UBSAN: runtime error: applying non-zero offset in JOIN::make_aggr_tables_info in sql/sql_select.cc Avoid undefined behaviour (applying offset to nullptr). The reported scenario is covered in mysql-test/connect-no-db.test No new tests needed. --- sql/sql_select.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 215a7113b71..b99ea744ae8 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -3325,7 +3325,8 @@ bool JOIN::make_aggr_tables_info() { List *curr_all_fields= &all_fields; List *curr_fields_list= &fields_list; - JOIN_TAB *curr_tab= join_tab + const_tables; + // Avoid UB (applying .. offset to nullptr) when join_tab is nullptr + JOIN_TAB *curr_tab= join_tab ? join_tab + const_tables : nullptr; TABLE *exec_tmp_table= NULL; bool distinct= false; bool keep_row_order= false; @@ -3883,9 +3884,9 @@ bool JOIN::make_aggr_tables_info() - duplicate value removal Both of these operations are done after window function computation step. */ - curr_tab= join_tab + total_join_tab_cnt(); if (select_lex->window_funcs.elements) { + curr_tab= join_tab + total_join_tab_cnt(); if (!(curr_tab->window_funcs_step= new Window_funcs_computation)) DBUG_RETURN(true); if (curr_tab->window_funcs_step->setup(thd, &select_lex->window_funcs, From a2bd936c528adb17415b35f38ba4fef5792adb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 10 Jun 2024 12:35:33 +0300 Subject: [PATCH 45/59] MDEV-33161 Function pointer signature mismatch in LF_HASH In cmake -DWITH_UBSAN=ON builds with clang but not with GCC, -fsanitize=undefined will flag several runtime errors on function pointer mismatch related to the lock-free hash table LF_HASH. Let us use matching function signatures and remove function pointer casts in order to avoid potential bugs due to undefined behaviour. These errors could be caught at compilation time by -Wcast-function-type-strict, which is available starting with clang-16, but not available in any version of GCC as of now. The old GCC flag -Wcast-function-type is enabled as part of -Wextra, but it specifically does not catch these errors. Reviewed by: Vladislav Vaintroub --- mysys/lf_alloc-pin.c | 29 ++++++++------- mysys/thr_mutex.c | 58 ++++++++++++++---------------- mysys/waiting_threads.c | 4 ++- sql/mdl.cc | 11 +++--- sql/sql_base.cc | 16 ++++----- sql/sql_servers.cc | 9 +++-- sql/sql_test.cc | 2 +- sql/table_cache.cc | 11 +++--- sql/xa.cc | 42 +++++++++++----------- storage/innobase/include/trx0sys.h | 55 ++++++++++++++-------------- storage/innobase/lock/lock0lock.cc | 22 +++++++----- storage/innobase/trx/trx0trx.cc | 9 ++--- 12 files changed, 137 insertions(+), 131 deletions(-) diff --git a/mysys/lf_alloc-pin.c b/mysys/lf_alloc-pin.c index 4dc41645530..8726349daa4 100644 --- a/mysys/lf_alloc-pin.c +++ b/mysys/lf_alloc-pin.c @@ -242,8 +242,10 @@ void lf_pinbox_put_pins(LF_PINS *pins) return; } -static int ptr_cmp(void **a, void **b) +static int ptr_cmp(const void *pa, const void *pb) { + const void *const*a= pa; + const void *const*b= pb; return *a < *b ? -1 : *a == *b ? 0 : 1; } @@ -283,8 +285,10 @@ struct st_harvester { callback forlf_dynarray_iterate: scan all pins of all threads and accumulate all pins */ -static int harvest_pins(LF_PINS *el, struct st_harvester *hv) +static int harvest_pins(void *e, void *h) { + LF_PINS *el= e; + struct st_harvester *hv= h; int i; LF_PINS *el_end= el+MY_MIN(hv->npins, LF_DYNARRAY_LEVEL_LENGTH); for (; el < el_end; el++) @@ -310,8 +314,9 @@ static int harvest_pins(LF_PINS *el, struct st_harvester *hv) callback forlf_dynarray_iterate: scan all pins of all threads and see if addr is present there */ -static int match_pins(LF_PINS *el, void *addr) +static int match_pins(void *e, void *addr) { + LF_PINS *el= e; int i; LF_PINS *el_end= el+LF_DYNARRAY_LEVEL_LENGTH; for (; el < el_end; el++) @@ -352,13 +357,12 @@ static void lf_pinbox_real_free(LF_PINS *pins) hv.granary= addr; hv.npins= npins; /* scan the dynarray and accumulate all pinned addresses */ - lf_dynarray_iterate(&pinbox->pinarray, - (lf_dynarray_func)harvest_pins, &hv); + lf_dynarray_iterate(&pinbox->pinarray, harvest_pins, &hv); npins= (int)(hv.granary-addr); /* and sort them */ if (npins) - qsort(addr, npins, sizeof(void *), (qsort_cmp)ptr_cmp); + qsort(addr, npins, sizeof(void *), ptr_cmp); } } #endif @@ -387,8 +391,7 @@ static void lf_pinbox_real_free(LF_PINS *pins) } else /* no alloca - no cookie. linear search here */ { - if (lf_dynarray_iterate(&pinbox->pinarray, - (lf_dynarray_func)match_pins, cur)) + if (lf_dynarray_iterate(&pinbox->pinarray, match_pins, cur)) goto found; } } @@ -416,10 +419,11 @@ found: 'first' and 'last' are the ends of the linked list of nodes: first->el->el->....->el->last. Use first==last to free only one element. */ -static void alloc_free(uchar *first, - uchar volatile *last, - LF_ALLOCATOR *allocator) +static void alloc_free(void *f, void *l, void *alloc) { + uchar *first= f; + uchar volatile *last= l; + LF_ALLOCATOR *allocator= alloc; /* we need a union here to access type-punned pointer reliably. otherwise gcc -fstrict-aliasing will not see 'tmp' changed in the loop @@ -448,8 +452,7 @@ static void alloc_free(uchar *first, */ void lf_alloc_init(LF_ALLOCATOR *allocator, uint size, uint free_ptr_offset) { - lf_pinbox_init(&allocator->pinbox, free_ptr_offset, - (lf_pinbox_free_func *)alloc_free, allocator); + lf_pinbox_init(&allocator->pinbox, free_ptr_offset, alloc_free, allocator); allocator->top= 0; allocator->mallocs= 0; allocator->element_size= size; diff --git a/mysys/thr_mutex.c b/mysys/thr_mutex.c index 2a8e54621c0..5b5f4360b73 100644 --- a/mysys/thr_mutex.c +++ b/mysys/thr_mutex.c @@ -62,14 +62,10 @@ my_bool safe_mutex_deadlock_detector= 1; /* On by default */ static struct st_safe_mutex_create_info_t *safe_mutex_create_root= NULL; #endif -static my_bool add_used_to_locked_mutex(safe_mutex_t *used_mutex, - safe_mutex_deadlock_t *locked_mutex); -static my_bool add_to_locked_mutex(safe_mutex_deadlock_t *locked_mutex, - safe_mutex_t *current_mutex); -static my_bool remove_from_locked_mutex(safe_mutex_t *mp, - safe_mutex_t *delete_mutex); -static my_bool remove_from_used_mutex(safe_mutex_deadlock_t *locked_mutex, - safe_mutex_t *mutex); +static my_bool add_used_to_locked_mutex(void *used, void *locked); +static my_bool add_to_locked_mutex(void *locked, void *current); +static my_bool remove_from_locked_mutex(void *m, void* remove); +static my_bool remove_from_used_mutex(void *locked, void *m); static void print_deadlock_warning(safe_mutex_t *new_mutex, safe_mutex_t *conflicting_mutex); #endif @@ -373,8 +369,7 @@ int safe_mutex_lock(safe_mutex_t *mp, myf my_flags, const char *file, are now locking (C) in B->C, then we would add C into B->locked_mutex and A->locked_mutex */ - my_hash_iterate(mutex_root->used_mutex, - (my_hash_walk_action) add_used_to_locked_mutex, + my_hash_iterate(mutex_root->used_mutex, add_used_to_locked_mutex, deadlock); /* @@ -654,12 +649,8 @@ void safe_mutex_free_deadlock_data(safe_mutex_t *mp) if (!(mp->create_flags & MYF_NO_DEADLOCK_DETECTION) && mp->used_mutex != NULL) { pthread_mutex_lock(&THR_LOCK_mutex); - my_hash_iterate(mp->used_mutex, - (my_hash_walk_action) remove_from_locked_mutex, - mp); - my_hash_iterate(mp->locked_mutex, - (my_hash_walk_action) remove_from_used_mutex, - mp); + my_hash_iterate(mp->used_mutex, remove_from_locked_mutex, mp); + my_hash_iterate(mp->locked_mutex, remove_from_used_mutex, mp); pthread_mutex_unlock(&THR_LOCK_mutex); my_hash_free(mp->used_mutex); @@ -709,15 +700,15 @@ void safe_mutex_end(FILE *file __attribute__((unused))) #endif /* SAFE_MUTEX_DETECT_DESTROY */ } -static my_bool add_used_to_locked_mutex(safe_mutex_t *used_mutex, - safe_mutex_deadlock_t *locked_mutex) +static my_bool add_used_to_locked_mutex(void *used, void *locked) { + safe_mutex_t *used_mutex= used; + safe_mutex_deadlock_t *locked_mutex= locked; /* Add mutex to all parent of the current mutex */ if (!locked_mutex->warning_only) { (void) my_hash_iterate(locked_mutex->mutex->locked_mutex, - (my_hash_walk_action) add_to_locked_mutex, - used_mutex); + add_to_locked_mutex, used_mutex); /* mark that locked_mutex is locked after used_mutex */ (void) add_to_locked_mutex(locked_mutex, used_mutex); } @@ -729,12 +720,13 @@ static my_bool add_used_to_locked_mutex(safe_mutex_t *used_mutex, register that locked_mutex was locked after current_mutex */ -static my_bool add_to_locked_mutex(safe_mutex_deadlock_t *locked_mutex, - safe_mutex_t *current_mutex) +static my_bool add_to_locked_mutex(void *locked, void *current) { + safe_mutex_deadlock_t *locked_mutex= locked; + safe_mutex_t *current_mutex= current; DBUG_ENTER("add_to_locked_mutex"); - DBUG_PRINT("info", ("inserting 0x%lx into 0x%lx (id: %lu -> %lu)", - (ulong) locked_mutex, (long) current_mutex, + DBUG_PRINT("info", ("inserting %p into %p (id: %lu -> %lu)", + locked_mutex, current_mutex, locked_mutex->id, current_mutex->id)); if (my_hash_insert(current_mutex->locked_mutex, (uchar*) locked_mutex)) { @@ -762,13 +754,14 @@ static my_bool add_to_locked_mutex(safe_mutex_deadlock_t *locked_mutex, When counter goes to 0, we delete the safe_mutex_deadlock_t entry. */ -static my_bool remove_from_locked_mutex(safe_mutex_t *mp, - safe_mutex_t *delete_mutex) +static my_bool remove_from_locked_mutex(void *m, void *remove) { + safe_mutex_t *mp= m; + safe_mutex_t *delete_mutex= remove; safe_mutex_deadlock_t *found; DBUG_ENTER("remove_from_locked_mutex"); - DBUG_PRINT("enter", ("delete_mutex: 0x%lx mutex: 0x%lx (id: %lu <- %lu)", - (ulong) delete_mutex, (ulong) mp, + DBUG_PRINT("enter", ("delete_mutex: %p mutex: %p (id: %lu <- %lu)", + delete_mutex, mp, delete_mutex->id, mp->id)); found= (safe_mutex_deadlock_t *) my_hash_search(mp->locked_mutex, @@ -786,12 +779,13 @@ static my_bool remove_from_locked_mutex(safe_mutex_t *mp, DBUG_RETURN(0); } -static my_bool remove_from_used_mutex(safe_mutex_deadlock_t *locked_mutex, - safe_mutex_t *mutex) +static my_bool remove_from_used_mutex(void *locked, void *m) { + safe_mutex_deadlock_t *locked_mutex= locked; + safe_mutex_t *mutex= m; DBUG_ENTER("remove_from_used_mutex"); - DBUG_PRINT("enter", ("delete_mutex: 0x%lx mutex: 0x%lx (id: %lu <- %lu)", - (ulong) mutex, (ulong) locked_mutex, + DBUG_PRINT("enter", ("delete_mutex: %p mutex: %p (id: %lu <- %lu)", + mutex, locked_mutex, mutex->id, locked_mutex->id)); if (my_hash_delete(locked_mutex->mutex->used_mutex, (uchar*) mutex)) { diff --git a/mysys/waiting_threads.c b/mysys/waiting_threads.c index dd60088d534..445e9f4781a 100644 --- a/mysys/waiting_threads.c +++ b/mysys/waiting_threads.c @@ -424,8 +424,10 @@ static void wt_resource_destroy(uchar *arg) It's called from lf_hash when an element is inserted. */ static void wt_resource_init(LF_HASH *hash __attribute__((unused)), - WT_RESOURCE *rc, WT_RESOURCE_ID *id) + void *resource, const void *ident) { + WT_RESOURCE *rc= resource; + const WT_RESOURCE_ID *id= ident; DBUG_ENTER("wt_resource_init"); rc->id= *id; rc->waiter_count= 0; diff --git a/sql/mdl.cc b/sql/mdl.cc index fb106a1af47..669f2c142db 100644 --- a/sql/mdl.cc +++ b/sql/mdl.cc @@ -762,8 +762,10 @@ struct mdl_iterate_arg }; -static my_bool mdl_iterate_lock(MDL_lock *lock, mdl_iterate_arg *arg) +static my_bool mdl_iterate_lock(void *lk, void *a) { + MDL_lock *lock= static_cast(lk); + mdl_iterate_arg *arg= static_cast(a); /* We can skip check for m_strategy here, becase m_granted must be empty for such locks anyway. @@ -786,14 +788,13 @@ int mdl_iterate(mdl_iterator_callback callback, void *arg) { DBUG_ENTER("mdl_iterate"); mdl_iterate_arg argument= { callback, arg }; - LF_PINS *pins= mdl_locks.get_pins(); int res= 1; - if (pins) + if (LF_PINS *pins= mdl_locks.get_pins()) { res= mdl_iterate_lock(mdl_locks.m_backup_lock, &argument) || - lf_hash_iterate(&mdl_locks.m_locks, pins, - (my_hash_walk_action) mdl_iterate_lock, &argument); + lf_hash_iterate(&mdl_locks.m_locks, pins, mdl_iterate_lock, + &argument); lf_hash_put_pins(pins); } DBUG_RETURN(res); diff --git a/sql/sql_base.cc b/sql/sql_base.cc index eb66d33b5b0..bac6026cb1e 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -252,9 +252,10 @@ public: }; -static my_bool list_open_tables_callback(TDC_element *element, - list_open_tables_arg *arg) +static my_bool list_open_tables_callback(void *el, void *a) { + TDC_element *element= static_cast(el); + list_open_tables_arg *arg= static_cast(a); const Lex_ident_db db= Lex_ident_db(Lex_cstring_strlen((const char*) element->m_key)); const char *table_name= db.str + db.length + 1; @@ -302,8 +303,7 @@ OPEN_TABLE_LIST *list_open_tables(THD *thd, DBUG_ENTER("list_open_tables"); list_open_tables_arg argument(thd, db, wild); - if (tdc_iterate(thd, (my_hash_walk_action) list_open_tables_callback, - &argument, true)) + if (tdc_iterate(thd, list_open_tables_callback, &argument, true)) DBUG_RETURN(0); DBUG_RETURN(argument.open_list); @@ -462,9 +462,10 @@ struct tc_collect_arg flush_tables_type flush_type; }; -static my_bool tc_collect_used_shares(TDC_element *element, - tc_collect_arg *arg) +static my_bool tc_collect_used_shares(void *el, void *a) { + TDC_element *element= static_cast(el); + tc_collect_arg *arg= static_cast(a); my_bool result= FALSE; DYNAMIC_ARRAY *shares= &arg->shares; @@ -573,8 +574,7 @@ bool flush_tables(THD *thd, flush_tables_type flag) my_init_dynamic_array(PSI_INSTRUMENT_ME, &collect_arg.shares, sizeof(TABLE_SHARE*), 100, 100, MYF(0)); collect_arg.flush_type= flag; - if (tdc_iterate(thd, (my_hash_walk_action) tc_collect_used_shares, - &collect_arg, true)) + if (tdc_iterate(thd, tc_collect_used_shares, &collect_arg, true)) { /* Release already collected shares */ for (uint i= 0 ; i < collect_arg.shares.elements ; i++) diff --git a/sql/sql_servers.cc b/sql/sql_servers.cc index d52d6071e89..d63b0d66a5e 100644 --- a/sql/sql_servers.cc +++ b/sql/sql_servers.cc @@ -133,9 +133,10 @@ struct close_cached_connection_tables_arg }; -static my_bool close_cached_connection_tables_callback( - TDC_element *element, close_cached_connection_tables_arg *arg) +static my_bool close_cached_connection_tables_callback(void *el, void *a) { + TDC_element *element= static_cast(el); + auto arg= static_cast(a); TABLE_LIST *tmp; mysql_mutex_lock(&element->LOCK_table_share); @@ -188,9 +189,7 @@ static bool close_cached_connection_tables(THD *thd, LEX_CSTRING *connection) close_cached_connection_tables_arg argument= { thd, connection, 0 }; DBUG_ENTER("close_cached_connections"); - if (tdc_iterate(thd, - (my_hash_walk_action) close_cached_connection_tables_callback, - &argument)) + if (tdc_iterate(thd, close_cached_connection_tables_callback, &argument)) DBUG_RETURN(true); DBUG_RETURN(argument.tables ? diff --git a/sql/sql_test.cc b/sql/sql_test.cc index 47acdaf8d7c..b625341fe5e 100644 --- a/sql/sql_test.cc +++ b/sql/sql_test.cc @@ -116,7 +116,7 @@ static void print_cached_tables(void) /* purecov: begin tested */ puts("DB Table Version Thread Open Lock"); - tdc_iterate(0, (my_hash_walk_action) print_cached_tables_callback, NULL, true); + tdc_iterate(0, print_cached_tables_callback, NULL, true); fflush(stdout); /* purecov: end */ diff --git a/sql/table_cache.cc b/sql/table_cache.cc index b76ba4fd239..f073dd1ec87 100644 --- a/sql/table_cache.cc +++ b/sql/table_cache.cc @@ -309,7 +309,7 @@ void tc_purge() { Share_free_tables::List purge_tables; - tdc_iterate(0, (my_hash_walk_action) tc_purge_callback, &purge_tables); + tdc_iterate(0, tc_purge_callback, &purge_tables); while (auto table= purge_tables.pop_front()) intern_close_table(table); } @@ -1126,7 +1126,7 @@ struct eliminate_duplicates_arg static uchar *eliminate_duplicates_get_key(const uchar *element, size_t *length, - my_bool not_used __attribute__((unused))) + my_bool) { LEX_STRING *key= (LEX_STRING *) element; *length= key->length; @@ -1134,9 +1134,10 @@ static uchar *eliminate_duplicates_get_key(const uchar *element, size_t *length, } -static my_bool eliminate_duplicates(TDC_element *element, - eliminate_duplicates_arg *arg) +static my_bool eliminate_duplicates(void *el, void *a) { + TDC_element *element= static_cast(el); + eliminate_duplicates_arg *arg= static_cast(a); LEX_STRING *key= (LEX_STRING *) alloc_root(&arg->root, sizeof(LEX_STRING)); if (!key || !(key->str= (char*) memdup_root(&arg->root, element->m_key, @@ -1182,7 +1183,7 @@ int tdc_iterate(THD *thd, my_hash_walk_action action, void *argument, hash_flags); no_dups_argument.action= action; no_dups_argument.argument= argument; - action= (my_hash_walk_action) eliminate_duplicates; + action= eliminate_duplicates; argument= &no_dups_argument; } diff --git a/sql/xa.cc b/sql/xa.cc index 7935c06c2ad..c4cd548b132 100644 --- a/sql/xa.cc +++ b/sql/xa.cc @@ -126,10 +126,11 @@ public: } return true; } - static void lf_hash_initializer(LF_HASH *hash __attribute__((unused)), - XID_cache_element *element, - XID_cache_insert_element *new_element) + static void lf_hash_initializer(LF_HASH *, void *el, const void *ie) { + XID_cache_element *element= static_cast(el); + XID_cache_insert_element *new_element= + static_cast(const_cast(ie)); DBUG_ASSERT(!element->is_set(ACQUIRED | RECOVERED)); element->rm_error= 0; element->xa_state= new_element->xa_state; @@ -146,11 +147,11 @@ public: DBUG_ASSERT(!reinterpret_cast(ptr + LF_HASH_OVERHEAD) ->is_set(ACQUIRED)); } - static uchar *key(const XID_cache_element *element, size_t *length, - my_bool not_used __attribute__((unused))) + static uchar *key(const unsigned char *el, size_t *length, my_bool) { - *length= element->xid.key_length(); - return element->xid.key(); + const XID &xid= reinterpret_cast(el)->xid; + *length= xid.key_length(); + return xid.key(); } }; @@ -221,11 +222,10 @@ void xid_cache_init() { xid_cache_inited= true; lf_hash_init(&xid_cache, sizeof(XID_cache_element), LF_HASH_UNIQUE, 0, 0, - (my_hash_get_key) XID_cache_element::key, &my_charset_bin); + XID_cache_element::key, &my_charset_bin); xid_cache.alloc.constructor= XID_cache_element::lf_alloc_constructor; xid_cache.alloc.destructor= XID_cache_element::lf_alloc_destructor; - xid_cache.initializer= - (lf_hash_initializer) XID_cache_element::lf_hash_initializer; + xid_cache.initializer= XID_cache_element::lf_hash_initializer; } @@ -331,9 +331,10 @@ struct xid_cache_iterate_arg void *argument; }; -static my_bool xid_cache_iterate_callback(XID_cache_element *element, - xid_cache_iterate_arg *arg) +static my_bool xid_cache_iterate_callback(void *el, void *a) { + XID_cache_element *element= static_cast(el); + xid_cache_iterate_arg *arg= static_cast(a); my_bool res= FALSE; if (element->lock()) { @@ -348,8 +349,7 @@ static int xid_cache_iterate(THD *thd, my_hash_walk_action action, void *arg) xid_cache_iterate_arg argument= { action, arg }; return thd->fix_xid_hash_pins() ? -1 : lf_hash_iterate(&xid_cache, thd->xid_hash_pins, - (my_hash_walk_action) xid_cache_iterate_callback, - &argument); + xid_cache_iterate_callback, &argument); } @@ -1039,17 +1039,19 @@ static my_bool xa_recover_callback(XID_cache_element *xs, Protocol *protocol, } -static my_bool xa_recover_callback_short(XID_cache_element *xs, - Protocol *protocol) +static my_bool xa_recover_callback_short(void *x, void *p) { + XID_cache_element *xs= static_cast(x); + Protocol *protocol= static_cast(p); return xa_recover_callback(xs, protocol, xs->xid.data, xs->xid.gtrid_length + xs->xid.bqual_length, &my_charset_bin); } -static my_bool xa_recover_callback_verbose(XID_cache_element *xs, - Protocol *protocol) +static my_bool xa_recover_callback_verbose(void *x, void *p) { + XID_cache_element *xs= static_cast(x); + Protocol *protocol= static_cast(p); char buf[SQL_XIDSIZE]; uint len= get_sql_xid(&xs->xid, buf); return xa_recover_callback(xs, protocol, buf, len, @@ -1082,13 +1084,13 @@ bool mysql_xa_recover(THD *thd) { len= SQL_XIDSIZE; cs= &my_charset_utf8mb3_general_ci; - action= (my_hash_walk_action) xa_recover_callback_verbose; + action= xa_recover_callback_verbose; } else { len= XIDDATASIZE; cs= &my_charset_bin; - action= (my_hash_walk_action) xa_recover_callback_short; + action= xa_recover_callback_short; } field_list.push_back(new (mem_root) diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index caac9af29ed..acd18935914 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -453,10 +453,10 @@ class rw_trx_hash_t not accessible by concurrent threads. */ - static void rw_trx_hash_initializer(LF_HASH *, - rw_trx_hash_element_t *element, - trx_t *trx) + static void rw_trx_hash_initializer(LF_HASH *, void *el, const void *t) { + rw_trx_hash_element_t *element= static_cast(el); + trx_t *trx= static_cast(const_cast(t)); ut_ad(element->trx == 0); element->trx= trx; element->id= trx->id; @@ -470,7 +470,7 @@ class rw_trx_hash_t Pins are used to protect object from being destroyed or reused. They are normally stored in trx object for quick access. If caller doesn't have trx - available, we try to get it using currnet_trx(). If caller doesn't have trx + available, we try to get it using current_trx(). If caller doesn't have trx at all, temporary pins are allocated. */ @@ -496,9 +496,10 @@ class rw_trx_hash_t template - static my_bool eliminate_duplicates(rw_trx_hash_element_t *element, - eliminate_duplicates_arg *arg) + static my_bool eliminate_duplicates(void *el, void *a) { + rw_trx_hash_element_t *element= static_cast(el); + auto arg= static_cast*>(a); for (trx_ids_t::iterator it= arg->ids.begin(); it != arg->ids.end(); it++) { if (*it == element->id) @@ -524,17 +525,17 @@ class rw_trx_hash_t } - template struct debug_iterator_arg + struct debug_iterator_arg { - walk_action *action; - T *argument; + my_hash_walk_action action; + void *argument; }; - template - static my_bool debug_iterator(rw_trx_hash_element_t *element, - debug_iterator_arg *arg) + static my_bool debug_iterator(void *el, void *a) { + rw_trx_hash_element_t *element= static_cast(el); + debug_iterator_arg *arg= static_cast(a); mutex_enter(&element->mutex); if (element->trx) validate_element(element->trx); @@ -744,7 +745,7 @@ public: @param caller_trx used to get/set pins @param action called for every element in hash - @param argument opque argument passed to action + @param argument opaque argument passed to action May return the same element multiple times if hash is under contention. If caller doesn't like to see the same transaction multiple times, it has @@ -767,28 +768,24 @@ public: @retval 1 iteration was interrupted (action returned 1) */ - template - int iterate(trx_t *caller_trx, walk_action *action, T *argument= nullptr) + int iterate(trx_t *caller_trx, my_hash_walk_action action, + void *argument= nullptr) { LF_PINS *pins= caller_trx ? get_pins(caller_trx) : lf_hash_get_pins(&hash); ut_a(pins); #ifdef UNIV_DEBUG - debug_iterator_arg debug_arg= { action, argument }; - action= reinterpret_cast(debug_iterator); - argument= reinterpret_cast(&debug_arg); + debug_iterator_arg debug_arg= { action, argument }; + action= debug_iterator; + argument= reinterpret_cast(&debug_arg); #endif - int res= lf_hash_iterate(&hash, pins, - reinterpret_cast(action), - const_cast(static_cast - (argument))); + int res= lf_hash_iterate(&hash, pins, action, argument); if (!caller_trx) lf_hash_put_pins(pins); return res; } - template - int iterate(walk_action *action, T *argument= nullptr) + int iterate(my_hash_walk_action action, void *argument= nullptr) { return iterate(current_trx(), action, argument); } @@ -1176,9 +1173,10 @@ public: } private: - static my_bool get_min_trx_id_callback(rw_trx_hash_element_t *element, - trx_id_t *id) + static my_bool get_min_trx_id_callback(void *el, void *i) { + auto element= static_cast(el); + auto id= static_cast(i); if (element->id < *id) { mutex_enter(&element->mutex); @@ -1200,9 +1198,10 @@ private: }; - static my_bool copy_one_id(rw_trx_hash_element_t *element, - snapshot_ids_arg *arg) + static my_bool copy_one_id(void* el, void *a) { + auto element= static_cast(el); + auto arg= static_cast(a); if (element->id < arg->m_id) { trx_id_t no= element->no; diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index d9a3c96aab0..05a3260f705 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -5234,8 +5234,9 @@ static void lock_rec_block_validate(const page_id_t page_id) } -static my_bool lock_validate_table_locks(rw_trx_hash_element_t *element, void*) +static my_bool lock_validate_table_locks(void *el, void*) { + rw_trx_hash_element_t *element= static_cast(el); ut_ad(lock_mutex_own()); mutex_enter(&element->mutex); if (element->trx) @@ -5499,10 +5500,10 @@ struct lock_rec_other_trx_holds_expl_arg }; -static my_bool lock_rec_other_trx_holds_expl_callback( - rw_trx_hash_element_t *element, - lock_rec_other_trx_holds_expl_arg *arg) +static my_bool lock_rec_other_trx_holds_expl_callback(void *el, void *a) { + auto element= static_cast(el); + auto arg= static_cast(a); mutex_enter(&element->mutex); if (element->trx) { @@ -6325,13 +6326,14 @@ lock_table_get_n_locks( /** Do an exhaustive check for any locks (table or rec) against the table. - @param[in] table check if there are any locks held on records in this table - or on the table itself + @param t check if there are any locks held on records in this table + or on the table itself */ -static my_bool lock_table_locks_lookup(rw_trx_hash_element_t *element, - const dict_table_t *table) +static my_bool lock_table_locks_lookup(void *el, void *t) { + auto element= static_cast(el); + const dict_table_t *table= static_cast(t); ut_ad(lock_mutex_own()); mutex_enter(&element->mutex); if (element->trx) @@ -6381,7 +6383,9 @@ lock_table_has_locks( #ifdef UNIV_DEBUG if (!has_locks) { - trx_sys.rw_trx_hash.iterate(lock_table_locks_lookup, table); + trx_sys.rw_trx_hash.iterate(lock_table_locks_lookup, + const_cast + (static_cast(table))); } #endif /* UNIV_DEBUG */ diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 7f1f26547dc..284437497b5 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -2053,9 +2053,9 @@ static my_bool trx_recover_for_mysql_callback(rw_trx_hash_element_t *element, } -static my_bool trx_recover_reset_callback(rw_trx_hash_element_t *element, - void*) +static my_bool trx_recover_reset_callback(void *el, void*) { + rw_trx_hash_element_t *element= static_cast(el); mutex_enter(&element->mutex); if (trx_t *trx= element->trx) { @@ -2107,9 +2107,10 @@ struct trx_get_trx_by_xid_callback_arg }; -static my_bool trx_get_trx_by_xid_callback(rw_trx_hash_element_t *element, - trx_get_trx_by_xid_callback_arg *arg) +static my_bool trx_get_trx_by_xid_callback(void *el, void *a) { + auto element= static_cast(el); + auto arg= static_cast(a); my_bool found= 0; mutex_enter(&element->mutex); if (trx_t *trx= element->trx) From 3b80d23d022f462c34cee2ec33cdc46162eb2656 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Sun, 2 Jun 2024 13:17:51 +0400 Subject: [PATCH 46/59] mtr --skip-not-found did not skip suites --skip-not-found switch tells mtr to skip not found tests instead of aborting. But it failed to skip the test if the suite name was not found. This problem also made the *last-N-failed builbot builders fail to run `mtr --skip-not-found` if the last commit removed a file in the mysql-test/include/ directory. This commit fixes it, now the not found test is properly skipped, no matter what component of the test name was not found: $ ./mtr main.foo --skip-not-found foo.main ... ============================================================================== TEST WORKER RESULT TIME (ms) or COMMENT -------------------------------------------------------------------------- foo.main [ skipped ] not found main.foo [ skipped ] not found -------------------------------------------------------------------------- --- mysql-test/lib/mtr_cases.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/lib/mtr_cases.pm b/mysql-test/lib/mtr_cases.pm index c72717578cb..d9ab10b60f6 100644 --- a/mysql-test/lib/mtr_cases.pm +++ b/mysql-test/lib/mtr_cases.pm @@ -398,7 +398,8 @@ sub collect_suite_name($$) { my @dirs = my_find_dir(dirname($::glob_mysql_test_dir), ["mysql-test/suite", @plugin_suitedirs ], - $suitename); + $suitename, + $::opt_skip_not_found ? NOT_REQUIRED : undef); # # if $suitename contained wildcards, we'll have many suites and # their overlays here. Let's group them appropriately. From fcd21d3e40a8739ca6813ac3a098ebf8d5fd5475 Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Mon, 10 Jun 2024 12:27:18 -0600 Subject: [PATCH 47/59] =?UTF-8?q?MDEV-34355:=20rpl.rpl=5Fsemi=5Fsync=5Fno?= =?UTF-8?q?=5Fmissed=5Fack=5Fafter=5Fadd=5Fslave=20=E2=80=98server=5F3=20s?= =?UTF-8?q?hould=20have=20sent=E2=80=A6=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The problem is that the test could query the status variable Rpl_semi_sync_slave_send_ack before the slave actually updated it. This would result in an immediate --die assertion killing the rest of the test. The bottom of this commit message has a small patch that can be applied to reproduce the test failure. This patch fixes the test failure by waiting for the variable to be updated before querying its value. diff --git a/sql/semisync_slave.cc b/sql/semisync_slave.cc index 9ddd4c5c8d7..60538079fce 100644 --- a/sql/semisync_slave.cc +++ b/sql/semisync_slave.cc @@ -303,7 +303,10 @@ int Repl_semi_sync_slave::slave_reply(Master_info *mi) reply_res= DBUG_EVALUATE_IF("semislave_failed_net_flush", 1, net_flush(net)); if (!reply_res) + { + sleep(1); rpl_semi_sync_slave_send_ack++; + } } DBUG_RETURN(reply_res); } --- .../rpl/t/rpl_semi_sync_no_missed_ack_after_add_slave.test | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_no_missed_ack_after_add_slave.test b/mysql-test/suite/rpl/t/rpl_semi_sync_no_missed_ack_after_add_slave.test index c8870e47e00..d3523a149ef 100644 --- a/mysql-test/suite/rpl/t/rpl_semi_sync_no_missed_ack_after_add_slave.test +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_no_missed_ack_after_add_slave.test @@ -70,6 +70,10 @@ if (`SELECT $slave1_sent_ack`) --connection server_3 --echo # Verifying server_3 did send ACK +--let $status_var= Rpl_semi_sync_slave_send_ack +--let $status_var_comparsion= > +--let $status_var_value= 0 +--source include/wait_for_status_var.inc --let $slave2_sent_ack= query_get_value(SHOW STATUS LIKE 'rpl_semi_sync_slave_send_ack', Value, 1) if (`SELECT NOT $slave2_sent_ack`) { From 90d376e01710fbc6f7e9eeef3f8c4653d5d4d82e Mon Sep 17 00:00:00 2001 From: Dave Gosselin Date: Mon, 13 May 2024 10:36:11 -0400 Subject: [PATCH 48/59] MDEV-34129 mariadb-install-db appears to hang on macOS Immediately close down the signal handler loop when we decide to break connections as it's the start of process termination anyway, and there's no need to wait once we've invoked break_connections. --- sql/mysqld.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index cfc16209251..a9960400c17 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2953,6 +2953,15 @@ static void start_signal_handler(void) DBUG_VOID_RETURN; } +/** Called only from signal_hand function. */ +static void* exit_signal_handler() +{ + my_thread_end(); + signal_thread_in_use= 0; + pthread_exit(0); // Safety + return nullptr; // Avoid compiler warnings +} + /** This threads handles all signals and alarms. */ /* ARGSUSED */ @@ -3013,10 +3022,7 @@ pthread_handler_t signal_hand(void *arg __attribute__((unused))) if (abort_loop) { DBUG_PRINT("quit",("signal_handler: calling my_thread_end()")); - my_thread_end(); - signal_thread_in_use= 0; - pthread_exit(0); // Safety - return 0; // Avoid compiler warnings + return exit_signal_handler(); } switch (sig) { case SIGTERM: @@ -3035,6 +3041,7 @@ pthread_handler_t signal_hand(void *arg __attribute__((unused))) PSI_CALL_delete_current_thread(); my_sigset(sig, SIG_IGN); break_connect_loop(); // MIT THREAD has a alarm thread + return exit_signal_handler(); } break; case SIGHUP: From 40dd5b8676c96593f0f234ec5ced7674fc4d01f6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 10 Jun 2024 20:38:49 +0200 Subject: [PATCH 49/59] fix the test for --view --- mysql-test/main/func_json.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mysql-test/main/func_json.test b/mysql-test/main/func_json.test index 9abdded34d2..a1ac2368014 100644 --- a/mysql-test/main/func_json.test +++ b/mysql-test/main/func_json.test @@ -1155,7 +1155,9 @@ SELECT JSON_REMOVE('{"A": { "B": 1 }}', '$.A.B.C.D'); SET @save_collation_connection= @@collation_connection; SET collation_connection='utf16_bin'; +--disable_service_connection SELECT JSON_EXTRACT('{"a": 1,"b": 2}','$.a'); +--enable_service_connection SET @@collation_connection= @save_collation_connection; From d524cb5b3ddf4dfcaf01c989b1d6fc76be854e15 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 4 Jun 2024 14:07:12 +1000 Subject: [PATCH 50/59] MDEV-34002 Initialise fields in spider_db_handler Otherwise it may result in nonsensical values like 190 for a boolean. --- .../spider/mysql-test/spider/bugfix/r/mdev_34002.result | 9 +++++++++ .../spider/mysql-test/spider/bugfix/t/mdev_34002.test | 7 +++++++ storage/spider/spd_db_include.h | 7 ++----- 3 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_34002.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_34002.test diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_34002.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_34002.result new file mode 100644 index 00000000000..7e266156e0a --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_34002.result @@ -0,0 +1,9 @@ +INSTALL PLUGIN Spider SONAME 'ha_spider.so'; +CREATE TABLE t (c DATE, c2 VARCHAR(1025) CHARACTER SET utf8mb3, UNIQUE KEY k(c2)) ENGINE=SPIDER; +UPDATE t SET c='２'; +ERROR HY000: Unable to connect to foreign data source: localhost +drop table t; +Warnings: +Warning 1620 Plugin is busy and will be uninstalled on shutdown +Note 1305 PLUGIN SPIDER_ALLOC_MEM does not exist +Note 1305 PLUGIN SPIDER_WRAPPER_PROTOCOLS does not exist diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_34002.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_34002.test new file mode 100644 index 00000000000..575a306589d --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_34002.test @@ -0,0 +1,7 @@ +INSTALL PLUGIN Spider SONAME 'ha_spider.so'; +CREATE TABLE t (c DATE, c2 VARCHAR(1025) CHARACTER SET utf8mb3, UNIQUE KEY k(c2)) ENGINE=SPIDER; +--error ER_CONNECT_TO_FOREIGN_DATA_SOURCE +UPDATE t SET c='２'; +drop table t; +--disable_query_log +--source ../../include/clean_up_spider.inc diff --git a/storage/spider/spd_db_include.h b/storage/spider/spd_db_include.h index 4f25b0a2fe1..0632a816995 100644 --- a/storage/spider/spd_db_include.h +++ b/storage/spider/spd_db_include.h @@ -1136,11 +1136,8 @@ public: ha_spider *spider; spider_db_share *db_share; int first_link_idx; -#ifdef SPIDER_HAS_GROUP_BY_HANDLER - SPIDER_LINK_IDX_CHAIN *link_idx_chain; -#endif - bool strict_group_by; - bool no_where_cond; + bool strict_group_by= false; + bool no_where_cond= false; spider_db_handler(ha_spider *spider, spider_db_share *db_share) : dbton_id(db_share->dbton_id), spider(spider), db_share(db_share), first_link_idx(-1) {} From f2eda615798d6d879a913a44c22d5dd0fc7ed8c9 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 10 Jun 2024 21:56:22 +0200 Subject: [PATCH 51/59] MDEV-33616 workaround libmariadb bug : mysql_errno = 0 on failed connection The bug can happens on macOS, if server closes the socket without sending error packet to client. Closing the socket on server side is legitimate, and happen e.g when write timeout occurs, perhaps also other situations. However mysqltest is not prepared to handle mysql_errno 0, and erroneously thinks connection was successfully established. The fix/workaround in mysqltest is to treat client failure with mysql_errno 0 the same as CR_SERVER_LOST (generic client-side communication error) The real fix in client library would ensure that mysql_errno is set on errors. --- client/mysqltest.cc | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index fa542fe7787..7a31c6d6802 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -5908,14 +5908,20 @@ int connect_n_handle_errors(struct st_command *command, stay clear of trying to work out which exact user-limit was exceeded. */ + auto my_err= mysql_errno(con); + if(my_err == 0) + { + /* Workaround client library bug, not indicating connection error. */ + my_err= CR_SERVER_LOST; + } - if (((mysql_errno(con) == ER_TOO_MANY_USER_CONNECTIONS) || - (mysql_errno(con) == ER_USER_LIMIT_REACHED)) && + if (((my_err == ER_TOO_MANY_USER_CONNECTIONS) || + (my_err == ER_USER_LIMIT_REACHED)) && (failed_attempts++ < opt_max_connect_retries)) { int i; - i= match_expected_error(command, mysql_errno(con), mysql_sqlstate(con)); + i= match_expected_error(command, my_err, mysql_sqlstate(con)); if (i >= 0) goto do_handle_error; /* expected error, handle */ @@ -5925,9 +5931,9 @@ int connect_n_handle_errors(struct st_command *command, } do_handle_error: - var_set_errno(mysql_errno(con)); - handle_error(command, mysql_errno(con), mysql_error(con), - mysql_sqlstate(con), ds); + var_set_errno(my_err); + handle_error(command, my_err, mysql_error(con), + mysql_sqlstate(con), ds); return 0; /* Not connected */ } From 5b39ded713d021e7ec76f8f550b43678821eae22 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 11 Jun 2024 12:50:26 +0530 Subject: [PATCH 52/59] MDEV-34156 InnoDB fails to apply the redo log for compressed tablespace Problem: ======= During recovery, InnoDB fails to apply the redo log for compressed tablespace. The reason is that InnoDB assumes that pages has been freed while applying the redo log for it. During multiple scan of redo logs, InnoDB stores the freed page information when it have sufficient buffer pool pages. Once it ran out of memory, InnoDB doesn't store freed page information. But InnoDB assigns the freed page ranges to tablespace in recv_init_crash_recovery_spaces() even though InnoDB doesn't have complete freed range information. While applying the redo log, InnoDB wrongly assumes that page has been freed and it could lead to corruption of tablespace. This issue is caused by commit 941af1fa581a799e59ddc3afcae965852aeceb00 (MDEV-31803) and commit 2f9e264781f702b8da1ed418ac9f4f5e8f8aa843 (MDEV-29911). Solution: ======== During recovery, set recovery size and freed page information for all tablespace irrespective of memory. --- storage/innobase/log/log0recv.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 6b6a686823c..dc3e2db5d85 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2768,8 +2768,7 @@ restart: case INIT_PAGE: last_offset= FIL_PAGE_TYPE; free_or_init_page: - if (store) - store_freed_or_init_rec(id, (b & 0x70) == FREE_PAGE); + store_freed_or_init_rec(id, (b & 0x70) == FREE_PAGE); if (UNIV_UNLIKELY(rlen != 0)) goto record_corrupted; copy_if_needed: @@ -2833,7 +2832,7 @@ restart: { if (UNIV_UNLIKELY(rlen + last_offset > srv_page_size)) goto record_corrupted; - if (store && UNIV_UNLIKELY(!page_no) && file_checkpoint) + if (UNIV_UNLIKELY(!page_no) && file_checkpoint) { const bool has_size= last_offset <= FSP_HEADER_OFFSET + FSP_SIZE && last_offset + rlen >= FSP_HEADER_OFFSET + FSP_SIZE + 4; From d3a7e46bb46d1347c6d2738590d177e463376f56 Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Tue, 11 Jun 2024 08:21:28 -0600 Subject: [PATCH 53/59] MDEV-34365: UBSAN runtime error: call to function io_callback(tpool::aiocb*) On an UBSAN clang-15 build, if running with UBSAN option halt_on_error=1 (the issue doesn't show up without it), MTR fails during mysqld --bootstrap with UBSAN error: call to function io_callback(tpool::aiocb*) through pointer to incorrect function type 'void (*)(void *)' This patch corrects the parameter type of io_callback to match its expected type defined by callback_func, i.e. (void*). Reviewed By: ============ --- storage/innobase/os/os0file.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 4ef6799e9a6..ce4eafafcd5 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -3560,8 +3560,9 @@ os_file_get_status( extern void fil_aio_callback(const IORequest &request); -static void io_callback(tpool::aiocb* cb) +static void io_callback(void *_cb) { + tpool::aiocb* cb= static_cast(_cb); const IORequest request(*static_cast (static_cast(cb->m_userdata))); if (cb->m_err != DB_SUCCESS) From dd13243b0d22558f827130969b1164ee9b102488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 13 Jun 2024 19:42:18 +0300 Subject: [PATCH 54/59] MDEV-33161 fixup: CMAKE_CXX_FLAGS=-DEXTRA_DEBUG --- sql/sql_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/sql_test.cc b/sql/sql_test.cc index b625341fe5e..7a9e83bafdd 100644 --- a/sql/sql_test.cc +++ b/sql/sql_test.cc @@ -87,9 +87,9 @@ print_where(COND *cond,const char *info, enum_query_type query_type) #ifdef EXTRA_DEBUG /* This is for debugging purposes */ -static my_bool print_cached_tables_callback(TDC_element *element, - void *arg __attribute__((unused))) +static my_bool print_cached_tables_callback(void *el, void*) { + TDC_element *element= static_cast(el); TABLE *entry; mysql_mutex_lock(&element->LOCK_table_share); From c849952b71710e54f4c435c0ebfa255eb4373783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 13 Jun 2024 19:57:40 +0300 Subject: [PATCH 55/59] MDEV-33840: Fix GCC -Wreorder This fixes up the merge commit 829cb1a49c76abea0214560c2cb8a9f39b418bb4 --- tpool/tpool_generic.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tpool/tpool_generic.cc b/tpool/tpool_generic.cc index a4d6c405bdf..774576d5cde 100644 --- a/tpool/tpool_generic.cc +++ b/tpool/tpool_generic.cc @@ -244,7 +244,7 @@ class thread_pool_generic : public thread_pool unsigned int m_concurrency; /** True, if threadpool is being shutdown, false otherwise */ - bool m_in_shutdown; + bool m_in_shutdown= false; /** Maintenance timer state : true = active(ON),false = inactive(OFF)*/ enum class timer_state_t @@ -813,7 +813,6 @@ thread_pool_generic::thread_pool_generic(int min_threads, int max_threads) : m_wakeups(), m_spurious_wakeups(), m_timer_state(timer_state_t::ON), - m_in_shutdown(), m_timestamp(), m_long_tasks_count(), m_waiting_task_count(), From 3271588bb7c7e31ae9474c29b6c370fa3876e7b3 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 14 Jun 2024 12:46:02 +0530 Subject: [PATCH 56/59] MDEV-34381 During innodb_undo_truncate=ON recovery, InnoDB may fail to shrink undo* files - During recovery, InnoDB may fail to shrink the undo tablespaces when there are no pages to recover while applying the redo log. This issue exists only when innodb_undo_truncate is enabled. trx_lists_init_at_db_start() could've applied the redo logs for undo tablespace page0. --- storage/innobase/log/log0recv.cc | 47 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 620e9c9209d..f83f5f45af6 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2668,6 +2668,29 @@ void recv_sys_t::apply(bool last_batch) recv_no_ibuf_operations = !last_batch || srv_operation == SRV_OPERATION_RESTORE || srv_operation == SRV_OPERATION_RESTORE_EXPORT; + for (auto id= srv_undo_tablespaces_open; id--;) + { + const trunc& t= truncated_undo_spaces[id]; + if (t.lsn) + { + /* The entire undo tablespace will be reinitialized by + innodb_undo_log_truncate=ON. Discard old log for all pages. + Even though we recv_sys_t::parse() already invoked trim(), + this will be needed in case recovery consists of multiple batches + (there was an invocation with !last_batch). */ + trim({id + srv_undo_space_id_start, 0}, t.lsn); + if (fil_space_t *space = fil_space_get(id + srv_undo_space_id_start)) + { + ut_ad(UT_LIST_GET_LEN(space->chain) == 1); + ut_ad(space->recv_size >= t.pages); + fil_node_t *file= UT_LIST_GET_FIRST(space->chain); + ut_ad(file->is_open()); + os_file_truncate(file->name, file->handle, + os_offset_t{space->recv_size} << + srv_page_size_shift, true); + } + } + } mtr_t mtr; @@ -2683,30 +2706,6 @@ void recv_sys_t::apply(bool last_batch) apply_log_recs= true; apply_batch_on= true; - for (auto id= srv_undo_tablespaces_open; id--;) - { - const trunc& t= truncated_undo_spaces[id]; - if (t.lsn) - { - /* The entire undo tablespace will be reinitialized by - innodb_undo_log_truncate=ON. Discard old log for all pages. - Even though we recv_sys_t::parse() already invoked trim(), - this will be needed in case recovery consists of multiple batches - (there was an invocation with !last_batch). */ - trim({id + srv_undo_space_id_start, 0}, t.lsn); - if (fil_space_t *space = fil_space_get(id + srv_undo_space_id_start)) - { - ut_ad(UT_LIST_GET_LEN(space->chain) == 1); - ut_ad(space->recv_size >= t.pages); - fil_node_t *file= UT_LIST_GET_FIRST(space->chain); - ut_ad(file->is_open()); - os_file_truncate(file->name, file->handle, - os_offset_t{space->recv_size} << - srv_page_size_shift, true); - } - } - } - fil_system.extend_to_recv_size(); /* Release the log_sys mutex in non-last batches of multi-batch From 4b4c371fe79ebf471450fed042bb1bcb94544a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 14 Jun 2024 13:21:19 +0300 Subject: [PATCH 57/59] MDEV-34297 fixup: -Wconversion on 32-bit --- storage/innobase/include/ut0counter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/include/ut0counter.h b/storage/innobase/include/ut0counter.h index 0083627b91f..b30c3df3f0d 100644 --- a/storage/innobase/include/ut0counter.h +++ b/storage/innobase/include/ut0counter.h @@ -56,7 +56,7 @@ struct ib_counter_t { /** Add to the counter. @param[in] n amount to be added */ - void add(Type n) { add(my_pseudo_random(), n); } + void add(Type n) { add(size_t(my_pseudo_random()), n); } /** Add to the counter. @param[in] index a reasonably thread-unique identifier From fef32fd9ad844baa8b38b7f1a7da3b4ed2a772e5 Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 15 Jun 2024 14:26:07 +0300 Subject: [PATCH 58/59] MDEV-34406 Enhance mariadb_upgrade to print failing query in case of error To make this possible, it was also necessary to enhance the mariadb client with the option --print-query-on-error. This option can also be very useful when running a batch of queries through the mariadb client and one wants to find out where things goes wrong. TODO: It would be good to enhance mariadb_upgrade to not call the mariadb client for executing queries but instead do this internally. This would have made this patch much easier! Reviewed by: Sergei Golubchik --- client/mysql.cc | 66 +++++++++++++++++++++++++++----- client/mysql_upgrade.c | 62 +++++++++++++++++++----------- mysql-test/main/mysqldump.result | 6 ++- 3 files changed, 101 insertions(+), 33 deletions(-) diff --git a/client/mysql.cc b/client/mysql.cc index 1e235c8a34d..9591ffb096a 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -158,7 +158,8 @@ static my_bool ignore_errors=0,wait_flag=0,quick=0, default_pager_set= 0, opt_sigint_ignore= 0, auto_vertical_output= 0, show_warnings= 0, executing_query= 0, - ignore_spaces= 0, opt_binhex= 0, opt_progress_reports; + ignore_spaces= 0, opt_binhex= 0, opt_progress_reports, + opt_print_query_on_error; static my_bool debug_info_flag, debug_check_flag, batch_abort_on_error; static my_bool column_types_flag; static my_bool preserve_comments= 0; @@ -237,6 +238,7 @@ static int com_quit(String *str,char*), com_prompt(String *str, char*), com_delimiter(String *str, char*), com_warnings(String *str, char*), com_nowarnings(String *str, char*), com_sandbox(String *str, char*); +static void print_query_to_stderr(String *buffer); #ifdef USE_POPEN static int com_nopager(String *str, char*), com_pager(String *str, char*), @@ -1659,6 +1661,10 @@ static struct my_option my_long_options[] = #endif "built-in default (" STRINGIFY_ARG(MYSQL_PORT) ").", &opt_mysql_port, &opt_mysql_port, 0, GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"print-query-on-error", 0, + "Print the query if there was an error. Is only enabled in --batch mode if verbose is not set (as then the query would be printed anyway)", + &opt_print_query_on_error, &opt_print_query_on_error, 0, GET_BOOL, NO_ARG, + 1, 0, 0, 0, 0, 0}, {"progress-reports", 0, "Get progress reports for long running commands (like ALTER TABLE)", &opt_progress_reports, &opt_progress_reports, 0, GET_BOOL, NO_ARG, 1, 0, @@ -3086,6 +3092,11 @@ int mysql_real_query_for_lazy(const char *buf, size_t length) int error; if (!mysql_real_query(&mysql,buf,(ulong)length)) return 0; + if (opt_print_query_on_error) + { + String query(buf, length, charset_info); + (void) print_query_to_stderr(&query); + } error= put_error(&mysql); if (mysql_errno(&mysql) != CR_SERVER_GONE_ERROR || retry > 1 || !opt_reconnect) @@ -3291,7 +3302,6 @@ static int com_charset(String *, char *line) 1 if fatal error */ - static int com_go(String *buffer, char *) { char buff[200]; /* about 110 chars used so far */ @@ -3363,6 +3373,8 @@ static int com_go(String *buffer, char *) { if (!(result=mysql_use_result(&mysql)) && mysql_field_count(&mysql)) { + if (opt_print_query_on_error) + print_query_to_stderr(buffer); error= put_error(&mysql); goto end; } @@ -3416,7 +3428,11 @@ static int com_go(String *buffer, char *) (long) mysql_num_rows(result) == 1 ? "row" : "rows"); end_pager(); if (mysql_errno(&mysql)) + { + if (opt_print_query_on_error) + print_query_to_stderr(buffer); error= put_error(&mysql); + } } } else if (mysql_affected_rows(&mysql) == ~(ulonglong) 0) @@ -3443,13 +3459,21 @@ static int com_go(String *buffer, char *) put_info("",INFO_RESULT); // Empty row if (result && !mysql_eof(result)) /* Something wrong when using quick */ + { + if (opt_print_query_on_error) + print_query_to_stderr(buffer); error= put_error(&mysql); + } else if (unbuffered) fflush(stdout); mysql_free_result(result); } while (!(err= mysql_next_result(&mysql))); if (err >= 1) + { + if (opt_print_query_on_error) + print_query_to_stderr(buffer); error= put_error(&mysql); + } end: @@ -4375,14 +4399,35 @@ static int com_shell(String *, char *line) #endif +static void print_query(String *buffer, FILE *file) +{ + tee_puts("--------------", file); + (void) tee_fputs(buffer->c_ptr(), file); + if (!buffer->length() || (*buffer)[buffer->length()-1] != '\n') + tee_putc('\n', file); + tee_puts("--------------\n", file); +} + + +/* + Print query to stderr in batch mode if verbose is not set +*/ + +static void print_query_to_stderr(String *buffer) +{ + if ((status.batch || in_com_source) && !verbose) + { + fflush(stdout); + print_query(buffer, stderr); + fflush(stderr); + } +} + + static int com_print(String *buffer,char *) { - tee_puts("--------------", stdout); - (void) tee_fputs(buffer->c_ptr(), stdout); - if (!buffer->length() || (*buffer)[buffer->length()-1] != '\n') - tee_putc('\n', stdout); - tee_puts("--------------\n", stdout); - return 0; /* If empty buffer */ + print_query(buffer, stdout); + return 0; } @@ -5117,8 +5162,9 @@ put_info(const char *str,INFO_TYPE info_type, uint error, const char *sqlstate) static int put_error(MYSQL *con) { - return put_info(mysql_error(con), INFO_ERROR, mysql_errno(con), - mysql_sqlstate(con)); + DBUG_ENTER("put_error"); + DBUG_RETURN(put_info(mysql_error(con), INFO_ERROR, + mysql_errno(con), mysql_sqlstate(con))); } diff --git a/client/mysql_upgrade.c b/client/mysql_upgrade.c index b303ffd099a..f81cff2797c 100644 --- a/client/mysql_upgrade.c +++ b/client/mysql_upgrade.c @@ -628,7 +628,7 @@ static int run_query(const char *query, DYNAMIC_STRING *ds_res, { my_close(fd, MYF(MY_WME)); my_delete(query_file_path, MYF(0)); - die("Failed to write to '%s'", query_file_path); + die("Failed to write query to '%s'", query_file_path); } } @@ -637,7 +637,7 @@ static int run_query(const char *query, DYNAMIC_STRING *ds_res, { my_close(fd, MYF(MY_WME)); my_delete(query_file_path, MYF(0)); - die("Failed to write to '%s'", query_file_path); + die("Failed to write query to '%s'", query_file_path); } ret= run_tool(mysql_path, @@ -647,6 +647,7 @@ static int run_query(const char *query, DYNAMIC_STRING *ds_res, "--batch", /* Turns off pager etc. */ force ? "--force": "--skip-force", opt_verbose >= 5 ? "--verbose" : "", + "--print-query-on-error", ds_res || opt_silent ? "--silent": "", "<", query_file_path, @@ -1085,18 +1086,6 @@ static char* get_line(char* line) return line; } - -/* Print the current line to stderr */ -static void print_line(char* line) -{ - while (*line && *line != '\n') - { - fputc(*line, stderr); - line++; - } - fputc('\n', stderr); -} - static my_bool from_before_10_1() { my_bool ret= TRUE; @@ -1308,16 +1297,21 @@ static int check_slave_repositories(void) static int run_sql_fix_privilege_tables(void) { - int found_real_errors= 0; + int found_real_errors= 0, query_started= 0; const char **query_ptr; + const char *end; DYNAMIC_STRING ds_script; DYNAMIC_STRING ds_result; + DYNAMIC_STRING ds_query; DBUG_ENTER("run_sql_fix_privilege_tables"); - if (init_dynamic_string(&ds_script, "", 65536, 1024)) + if (init_dynamic_string(&ds_script, "", 96*1024, 8196)) die("Out of memory"); - if (init_dynamic_string(&ds_result, "", 512, 512)) + if (init_dynamic_string(&ds_result, "", 1024, 1024)) + die("Out of memory"); + + if (init_dynamic_string(&ds_query, "", 1024, 1024)) die("Out of memory"); verbose("Phase %d/%d: Running 'mysql_fix_privilege_tables'", @@ -1346,22 +1340,46 @@ static int run_sql_fix_privilege_tables(void) "Unknown column" and "Duplicate key name" since they just indicate the system tables are already up to date */ - char *line= ds_result.str; + const char *line= ds_result.str; do { + size_t length; + end= strchr(line, '\n'); + if (!end) + end= strend(line); + else + end++; /* Include end \n */ + length= (size_t) (end - line); + if (!is_expected_error(line)) { /* Something unexpected failed, dump error line to screen */ found_real_errors++; - print_line(line); + if (ds_query.length) + fwrite(ds_query.str, sizeof(char), ds_query.length, stderr); + fwrite(line, sizeof(char), length, stderr); + query_started= 0; } else if (strncmp(line, "WARNING", 7) == 0) { - print_line(line); + fwrite(line, sizeof(char), length, stderr); + query_started= 0; } - } while ((line= get_line(line)) && *line); + else if (!strncmp(line, "--------------\n", 16)) + { + /* mariadb separates query from the error with a line of '-' */ + if (!query_started++) + ds_query.length= 0; /* Truncate */ + else + query_started= 0; /* End of query */ + } + else if (query_started) + { + dynstr_append_mem(&ds_query, line, length); + } + } while (*(line= end)); } - + dynstr_free(&ds_query); dynstr_free(&ds_result); dynstr_free(&ds_script); DBUG_RETURN(found_real_errors); diff --git a/mysql-test/main/mysqldump.result b/mysql-test/main/mysqldump.result index 1541a4e6f72..1f59f31b7c8 100644 --- a/mysql-test/main/mysqldump.result +++ b/mysql-test/main/mysqldump.result @@ -6413,9 +6413,13 @@ j integer INSERT INTO t VALUES (1,1),(2,2),(3,3),(4,4); # Dump database 1 # Restore from database 1 to database 2 -ERROR 1100 (HY000) at line 46: Table 'seq_t_i' was not locked with LOCK TABLES SETVAL(`seq_t_i`, 1, 0) 1 +-------------- +INSERT INTO `t` VALUES (1,1),(2,2),(3,3),(4,4) +-------------- + +ERROR 1100 (HY000) at line 46: Table 'seq_t_i' was not locked with LOCK TABLES DROP DATABASE IF EXISTS test1; DROP DATABASE IF EXISTS test2; # From 956bcf8f49982ab99884321eb3f58dcfef3630ef Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 15 Jun 2024 16:55:08 +0300 Subject: [PATCH 59/59] Change mysqldump to use DO instead of 'SELECT' for storing sequences. This avoids a lot of SETVAL() results when applying a mysqldump with sequences. --- client/mysqldump.c | 2 +- mysql-test/main/mysql.result | 8 ++++++ mysql-test/main/mysqldump.result | 26 ------------------- .../suite/sql_sequence/mysqldump.result | 16 ++++++------ 4 files changed, 17 insertions(+), 35 deletions(-) diff --git a/client/mysqldump.c b/client/mysqldump.c index 521e048497d..1c9d92d6e09 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -2959,7 +2959,7 @@ static void get_sequence_structure(const char *seq, const char *db) row= mysql_fetch_row(result); if (row[0]) { - fprintf(sql_file, "SELECT SETVAL(%s, %s, 0);\n", result_seq, row[0]); + fprintf(sql_file, "DO SETVAL(%s, %s, 0);\n", result_seq, row[0]); } // Sequences will not use inserts, so no need for REPLACE and LOCKS mysql_free_result(result); diff --git a/mysql-test/main/mysql.result b/mysql-test/main/mysql.result index 445aa602241..0bb447df47a 100644 --- a/mysql-test/main/mysql.result +++ b/mysql-test/main/mysql.result @@ -137,6 +137,10 @@ c int(11) YES NULL drop table t1; 1 1 +-------------- + use +-------------- + ERROR 1064 (42000) at line 3: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '' at line 1 ERROR at line 1: USE must be followed by a database name 1 +1 @@ -166,6 +170,10 @@ count(*) drop table t17583; Test connect without db- or host-name => reconnect Test connect with dbname only => new dbname, old hostname +-------------- +connecttest +-------------- + ERROR 1064 (42000) at line 1: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'connecttest' at line 1 Test connect with _invalid_ dbname only => new invalid dbname, old hostname ERROR 1049 (42000) at line 1: Unknown database 'invalid' diff --git a/mysql-test/main/mysqldump.result b/mysql-test/main/mysqldump.result index 1f59f31b7c8..b69e6168c86 100644 --- a/mysql-test/main/mysqldump.result +++ b/mysql-test/main/mysqldump.result @@ -6326,14 +6326,6 @@ Table Create Table s4 CREATE SEQUENCE `s4` start with 400 minvalue 400 maxvalue 1400 increment by 40 cache 1000 cycle ENGINE=MyISAM # Dump sequence without `--no-data` # Restore from mysqldump -SETVAL(`s1`, 1101, 0) -1101 -SETVAL(`s2`, 1201, 0) -1201 -SETVAL(`s3`, 1301, 0) -1301 -SETVAL(`s4`, 1401, 0) -1401 # Show create after restore show create sequence d.s1; Table Create Table @@ -6352,14 +6344,6 @@ NEXTVAL(d.s1) NEXTVAL(d.s2) NEXTVAL(d.s3) NEXTVAL(d.s4) 100 200 300 400 # Dump sequence with `--no-data` # Restore from mysqldump -SETVAL(`s1`, 1101, 0) -1101 -SETVAL(`s2`, 1201, 0) -1201 -SETVAL(`s3`, 1301, 0) -1301 -SETVAL(`s4`, 1401, 0) -1401 # Show create after restore `--no-data` show create sequence d.s1; Table Create Table @@ -6378,14 +6362,6 @@ NEXTVAL(d.s1) NEXTVAL(d.s2) NEXTVAL(d.s3) NEXTVAL(d.s4) 100 200 300 400 # Restore to different database than original create database d2; -SETVAL(`s1`, 1101, 0) -1101 -SETVAL(`s2`, 1201, 0) -1201 -SETVAL(`s3`, 1301, 0) -1301 -SETVAL(`s4`, 1401, 0) -1401 show create sequence d2.s1; Table Create Table s1 CREATE SEQUENCE `s1` start with 100 minvalue 100 maxvalue 1100 increment by 10 cache 1000 cycle ENGINE=MyISAM @@ -6413,8 +6389,6 @@ j integer INSERT INTO t VALUES (1,1),(2,2),(3,3),(4,4); # Dump database 1 # Restore from database 1 to database 2 -SETVAL(`seq_t_i`, 1, 0) -1 -------------- INSERT INTO `t` VALUES (1,1),(2,2),(3,3),(4,4) -------------- diff --git a/mysql-test/suite/sql_sequence/mysqldump.result b/mysql-test/suite/sql_sequence/mysqldump.result index b07f07a605b..ea100fb0199 100644 --- a/mysql-test/suite/sql_sequence/mysqldump.result +++ b/mysql-test/suite/sql_sequence/mysqldump.result @@ -5,9 +5,9 @@ CREATE SEQUENCE x1 engine=innodb; # dump whole database /*!999999\- enable the sandbox mode */ CREATE SEQUENCE `a1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=Aria; -SELECT SETVAL(`a1`, 1, 0); +DO SETVAL(`a1`, 1, 0); CREATE SEQUENCE `x1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB; -SELECT SETVAL(`x1`, 1, 0); +DO SETVAL(`x1`, 1, 0); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `t1` ( @@ -19,9 +19,9 @@ INSERT INTO `t1` VALUES (1),(2); # dump by tables order 1 /*!999999\- enable the sandbox mode */ CREATE SEQUENCE `a1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=Aria; -SELECT SETVAL(`a1`, 1, 0); +DO SETVAL(`a1`, 1, 0); CREATE SEQUENCE `x1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB; -SELECT SETVAL(`x1`, 1, 0); +DO SETVAL(`x1`, 1, 0); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `t1` ( @@ -33,9 +33,9 @@ INSERT INTO `t1` VALUES (1),(2); # dump by tables order 2 /*!999999\- enable the sandbox mode */ CREATE SEQUENCE `a1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=Aria; -SELECT SETVAL(`a1`, 1, 0); +DO SETVAL(`a1`, 1, 0); CREATE SEQUENCE `x1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB; -SELECT SETVAL(`x1`, 1, 0); +DO SETVAL(`x1`, 1, 0); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `t1` ( @@ -57,9 +57,9 @@ INSERT INTO `t1` VALUES (1),(2); # dump by tables only sequences /*!999999\- enable the sandbox mode */ CREATE SEQUENCE `a1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=Aria; -SELECT SETVAL(`a1`, 1, 0); +DO SETVAL(`a1`, 1, 0); CREATE SEQUENCE `x1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=InnoDB; -SELECT SETVAL(`x1`, 1, 0); +DO SETVAL(`x1`, 1, 0); # end of dumps DROP TABLE a1,t1,x1; set default_storage_engine=InnoDB;