From e7442e5eb934980b981c1c68b4055241ff38f73b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 2 Apr 2025 12:53:21 +0300 Subject: [PATCH 001/125] MDEV-36226 fixup: format mismatch --- storage/innobase/buf/buf0dblwr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 8cc44eb7972..dc8d5e506c5 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -630,7 +630,7 @@ bool buf_dblwr_t::flush_buffered_writes(const ulint size) noexcept static_cast(srv_fatal_semaphore_wait_threshold); size_t log_count= first_log_count; - for (ulong count= 0;;) + for (size_t count= 0;;) { if (!active_slot->first_free) return false; From 58a36773090223c97d814a07d57ab35ebf803cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 2 Apr 2025 15:56:22 +0300 Subject: [PATCH 002/125] MDEV-29445 fixup: Do not skip a test --- mysql-test/suite/innodb/t/mdev-15707.opt | 2 +- storage/innobase/buf/buf0flu.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/innodb/t/mdev-15707.opt b/mysql-test/suite/innodb/t/mdev-15707.opt index fec3463c6d0..6add146aec9 100644 --- a/mysql-test/suite/innodb/t/mdev-15707.opt +++ b/mysql-test/suite/innodb/t/mdev-15707.opt @@ -1 +1 @@ ---innodb --innodb-buffer-pool-size=5MB --innodb-read-io-threads=1 --innodb-doublewrite=0 --innodb-flush-log-at-trx-commit=0 \ No newline at end of file +--innodb --innodb-buffer-pool-size=6MB --innodb-read-io-threads=1 --innodb-doublewrite=0 --innodb-flush-log-at-trx-commit=0 \ No newline at end of file diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 81f78a40527..061af1290cf 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1231,7 +1231,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, /* BUF_LRU_MIN_LEN (256) is too high value for low buffer pool(BP) size. For example, for BP size lower than 80M and 16 K page size, the limit is more than - 5% of total BP and for lowest BP 5M, it is 80% of the BP. Non-data objects + 5% of total BP and for lowest BP 6M, it is 80% of the BP. Non-data objects like explicit locks could occupy part of the BP pool reducing the pages available for LRU. If LRU reaches minimum limit and if no free pages are available, server would hang with page cleaner not able to free any more @@ -1745,7 +1745,7 @@ static ulint buf_flush_LRU(ulint max_n) noexcept pthread_cond_broadcast(&buf_pool.done_free); } else if (!pages && !buf_pool.try_LRU_scan) - /* For example, with the minimum innodb_buffer_pool_size=5M and + /* For example, with the minimum innodb_buffer_pool_size=6M and the default innodb_page_size=16k there are only a little over 316 pages in the buffer pool. The buffer pool can easily be exhausted by a workload of some dozen concurrent connections. The system could From 0d7ef4f4785bd0181e6a4c60f4000ecc5deb085d Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 3 Apr 2025 12:19:36 +0530 Subject: [PATCH 003/125] MDEV-36236 Instant alter aborts when InnoDB fails to rollback instant operation Problem: ======== - InnoDB does consecutive instant alter operation, first instant DDL fails, it fails to reset the old instant information in table during rollback. This lead to consecutive instant alter to have wrong assumption about the exisitng instant column information. Fix: ==== dict_table_t::instant_column(): Duplicate the instant information field of the table. By doing this, InnoDB alter retains the old instant information and reset it during rollback operation --- .../innodb/r/instant_alter_debug,redundant.rdiff | 7 ++++--- .../suite/innodb/r/instant_alter_debug.result | 11 +++++++++++ .../suite/innodb/t/instant_alter_debug.test | 12 ++++++++++-- storage/innobase/handler/handler0alter.cc | 15 +++++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/innodb/r/instant_alter_debug,redundant.rdiff b/mysql-test/suite/innodb/r/instant_alter_debug,redundant.rdiff index cf72c37bde8..ba4aeadaa50 100644 --- a/mysql-test/suite/innodb/r/instant_alter_debug,redundant.rdiff +++ b/mysql-test/suite/innodb/r/instant_alter_debug,redundant.rdiff @@ -1,8 +1,9 @@ -@@ -527,6 +527,6 @@ +@@ -576,7 +576,7 @@ FROM information_schema.global_status WHERE variable_name = 'innodb_instant_alter_column'; instants -37 +38 - SET GLOBAL innodb_stats_persistent = @save_stats_persistent; - # End of 10.6 tests + CREATE TABLE t1(f1 INT, f2 TEXT)ENGINE=InnoDB; + INSERT INTO t1 VALUES(1, 'a'); + ALTER TABLE t1 ADD COLUMN f3 TEXT FIRST; diff --git a/mysql-test/suite/innodb/r/instant_alter_debug.result b/mysql-test/suite/innodb/r/instant_alter_debug.result index 53da7b6f5dc..05211d3b55e 100644 --- a/mysql-test/suite/innodb/r/instant_alter_debug.result +++ b/mysql-test/suite/innodb/r/instant_alter_debug.result @@ -577,5 +577,16 @@ FROM information_schema.global_status WHERE variable_name = 'innodb_instant_alter_column'; instants 37 +CREATE TABLE t1(f1 INT, f2 TEXT)ENGINE=InnoDB; +INSERT INTO t1 VALUES(1, 'a'); +ALTER TABLE t1 ADD COLUMN f3 TEXT FIRST; +SET STATEMENT DEBUG_DBUG="+d,instant_insert_fail" FOR +ALTER TABLE t1 DROP COLUMN f1; +ERROR HY000: Internal error: InnoDB: Insert into SYS_COLUMNS failed +ALTER TABLE t1 DROP COLUMN f1; +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +DROP TABLE t1; SET GLOBAL innodb_stats_persistent = @save_stats_persistent; # End of 10.6 tests diff --git a/mysql-test/suite/innodb/t/instant_alter_debug.test b/mysql-test/suite/innodb/t/instant_alter_debug.test index dd11e09629c..07118aa2983 100644 --- a/mysql-test/suite/innodb/t/instant_alter_debug.test +++ b/mysql-test/suite/innodb/t/instant_alter_debug.test @@ -657,11 +657,19 @@ DROP TABLE t1; SET DEBUG_SYNC=RESET; --echo # End of 10.5 tests - SELECT variable_value-@old_instant instants FROM information_schema.global_status WHERE variable_name = 'innodb_instant_alter_column'; -SET GLOBAL innodb_stats_persistent = @save_stats_persistent; +CREATE TABLE t1(f1 INT, f2 TEXT)ENGINE=InnoDB; +INSERT INTO t1 VALUES(1, 'a'); +ALTER TABLE t1 ADD COLUMN f3 TEXT FIRST; +--error ER_INTERNAL_ERROR +SET STATEMENT DEBUG_DBUG="+d,instant_insert_fail" FOR +ALTER TABLE t1 DROP COLUMN f1; +ALTER TABLE t1 DROP COLUMN f1; +CHECK TABLE t1; +DROP TABLE t1; +SET GLOBAL innodb_stats_persistent = @save_stats_persistent; --echo # End of 10.6 tests diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 3fe02af5beb..3459be3a06d 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -621,6 +621,16 @@ inline bool dict_table_t::instant_column(const dict_table_t& table, } dict_index_t* index = dict_table_get_first_index(this); + if (instant) { + instant->field_map= static_cast( + mem_heap_dup(heap, instant->field_map, + (index->n_fields - + index->first_user_field()) * + sizeof *instant->field_map)); + instant= static_cast( + mem_heap_dup(heap, instant, sizeof *instant)); + } + bool metadata_changed; { const dict_index_t& i = *dict_table_get_first_index(&table); @@ -5515,6 +5525,11 @@ static bool innodb_insert_sys_columns( return false; } + DBUG_EXECUTE_IF("instant_insert_fail", + my_error(ER_INTERNAL_ERROR, MYF(0), + "InnoDB: Insert into SYS_COLUMNS failed"); + return true;); + if (DB_SUCCESS != que_eval_sql( info, "PROCEDURE ADD_COL () IS\n" From b11772d9a54b596d56e071543e34692830068bbd Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 3 Apr 2025 16:43:01 +0530 Subject: [PATCH 004/125] MDEV-33167 ASAN errors in dict_sys_t::load_table / get_foreign_key_info after failing to load a table Problem: ======= - While loading the foreign key constraints for the parent table, if child table wasn't open then InnoDB uses the parent table heap to store the child table name in fk_tables list. If the consecutive foreign key relation for the parent table fails with error, InnoDB evicts the parent table from memory. But InnoDB accesses the evicted table memory again in dict_sys.load_table() Solution: ======== dict_load_table_one(): In case of error, remove the child table names which was added during dict_load_foreigns() --- mysql-test/suite/innodb/r/foreign_key.result | 21 +++++++++++-- mysql-test/suite/innodb/t/foreign_key.test | 32 +++++++++++++++++--- storage/innobase/dict/dict0load.cc | 2 ++ 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/innodb/r/foreign_key.result b/mysql-test/suite/innodb/r/foreign_key.result index e7d0454df6e..236953795e6 100644 --- a/mysql-test/suite/innodb/r/foreign_key.result +++ b/mysql-test/suite/innodb/r/foreign_key.result @@ -155,7 +155,6 @@ INSERT INTO parent SET a=0; FLUSH TABLES; # restart disconnect incomplete; -SET @save_stats_persistent = @@GLOBAL.innodb_stats_persistent; SET GLOBAL innodb_stats_persistent = 0; INSERT INTO child SET a=0; INSERT INTO child SET a=1; @@ -1182,5 +1181,23 @@ ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fail ALTER TABLE t2 ADD KEY(b), ALGORITHM=NOCOPY; DELETE FROM t1; DROP TABLE t2, t1; +# +# MDEV-33167 ASAN errors after failing to load foreign key +# relation for the table +# +call mtr.add_suppression("InnoDB: Load table `test`.`t3` failed, the table has missing foreign key indexes. Turn off 'foreign_key_checks' and try again."); +SET STATEMENT FOREIGN_KEY_CHECKS = 0 FOR +CREATE TABLE t1(f1 VARCHAR(8), +FOREIGN KEY(f1) REFERENCES test.t3(f1))ENGINE=InnoDB; +SET STATEMENT FOREIGN_KEY_CHECKS = 0 FOR +CREATE TABLE t2(f1 VARCHAR(8), +FOREIGN KEY(f1) REFERENCES test.t3(f1)) +ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; +SET STATEMENT FOREIGN_KEY_CHECKS = 0 FOR +CREATE TABLE t3(f1 VARCHAR(8) PRIMARY KEY) +ENGINE=InnoDB DEFAULT CHARSET=latin1; +set GLOBAL innodb_fast_shutdown=0; +# restart +ALTER TABLE t2 FORCE; +DROP TABLE t2, t1, t3; # End of 10.6 tests -SET GLOBAL innodb_stats_persistent = @save_stats_persistent; diff --git a/mysql-test/suite/innodb/t/foreign_key.test b/mysql-test/suite/innodb/t/foreign_key.test index 6a65de0e9b0..e7761f3cf4b 100644 --- a/mysql-test/suite/innodb/t/foreign_key.test +++ b/mysql-test/suite/innodb/t/foreign_key.test @@ -133,7 +133,6 @@ FLUSH TABLES; --let $shutdown_timeout= disconnect incomplete; -SET @save_stats_persistent = @@GLOBAL.innodb_stats_persistent; SET GLOBAL innodb_stats_persistent = 0; INSERT INTO child SET a=0; @@ -1245,8 +1244,31 @@ ALTER TABLE t2 ADD KEY(b), ALGORITHM=NOCOPY; DELETE FROM t1; DROP TABLE t2, t1; +--echo # +--echo # MDEV-33167 ASAN errors after failing to load foreign key +--echo # relation for the table +--echo # +call mtr.add_suppression("InnoDB: Load table `test`.`t3` failed, the table has missing foreign key indexes. Turn off 'foreign_key_checks' and try again."); +SET STATEMENT FOREIGN_KEY_CHECKS = 0 FOR +CREATE TABLE t1(f1 VARCHAR(8), + FOREIGN KEY(f1) REFERENCES test.t3(f1))ENGINE=InnoDB; + +SET STATEMENT FOREIGN_KEY_CHECKS = 0 FOR +CREATE TABLE t2(f1 VARCHAR(8), + FOREIGN KEY(f1) REFERENCES test.t3(f1)) + ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; + +SET STATEMENT FOREIGN_KEY_CHECKS = 0 FOR +CREATE TABLE t3(f1 VARCHAR(8) PRIMARY KEY) + ENGINE=InnoDB DEFAULT CHARSET=latin1; + +set GLOBAL innodb_fast_shutdown=0; +--let $shutdown_timeout= +--source include/restart_mysqld.inc +# Error encountered while loading the foreign key +# constraint for t3. t1 wasn't loaded into memory yet +# t2 failed to find index for foreign key relation +ALTER TABLE t2 FORCE; +DROP TABLE t2, t1, t3; + --echo # End of 10.6 tests - -SET GLOBAL innodb_stats_persistent = @save_stats_persistent; - ---source include/wait_until_count_sessions.inc diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 917c4d51ee9..0071e7e8afe 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -2515,10 +2515,12 @@ corrupted: if (!table->is_readable()) { /* Don't attempt to load the indexes from disk. */ } else if (err == DB_SUCCESS) { + auto i = fk_tables.size(); err = dict_load_foreigns(table->name.m_name, nullptr, 0, true, ignore_err, fk_tables); if (err != DB_SUCCESS) { + fk_tables.erase(fk_tables.begin() + i, fk_tables.end()); ib::warn() << "Load table " << table->name << " failed, the table has missing" " foreign key indexes. Turn off" From 1b4efbeb8c1957d297a95957672e76650f4bb923 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Mon, 25 Nov 2024 19:48:23 +0200 Subject: [PATCH 005/125] MDEV-35207 ignored error at binlogging by CREATE-TABLE-SELECT leads to assert MDEV-35499 Errored-out CREATE-or-REPLACE-SELECT does not log DROP table into binlog MDEV-35502 Failed at ROW-format binlogging CREATE-TABLE-SELECT should not generate Incident event When a CREATE TABLE .. SELECT errors while inserting data, a user would expect that all changes are rolled back and the table would not exist after executing the query. However CREATE-TABLE-SELECT can face an error near the end of its execution select_create::send_eof() so that the error was never checked which led to various assert inside binlogging path that should not be attended at all. Specifically when binlog_commit() of ha_commit_one_phase() that CREATE-TABLE-SELECT employs errored out because of a limited cache size (binlog_commit may try writing to a transactional cache) the cache was not flushed to binlog. The missed error check allowed further execution down to trans_commit_implicit() in whose stack DBUG_ASSERT(!(entry->using_trx_cache && !mngr->trx_cache.empty() && mngr->get_binlog_cache_log(TRUE)->error)); fired. In a non-debug build that table remains created/populated inconsistently with binlog. The fixes need and install the error checking in select_create::send_eof(). That prevents from any further execution when ha_commit_one_phase() fails for any reason (typically due to binlog_commit()). This commit also covers CREATE-or-REPLACE-SELECT that additionally had a specific issue in that DROP TABLE was not logged the binary log, MDEV-35499. See changes select_create::abort_result_set(). The current commit also corrects an unnecessary Incident event logging when CREATE-TABLE-SELECT encounters a binloging issue, MDEV-35502. The Incident was actually only harmful in this case as the table was never going to be created, therefore replicated, in such a case. In "normal" cases when the SELECT phase errors due to binlogging, an internal incident flag gets reset inside select_create::abort_result_set(). A hunk in select_insert::prepare_eof() addresses a specific kind of this issue that deals with incorrect computation of the binlog cache type. Because of that in the OLD version execution was allowed to proceed along ha_commit_trans()..binlog_commit() while a Pending event was not flushed to the transactional cache. That might lead to the unnecessary binlogged Incident despite the select_create::abort_result_set() measures. However now with the corrected cache type any binlogging error to flush the Pending event is covered according to the normal case. non-transaction table, updates to the non-transactional table NOTE the commit contains few tests overlapping with unfixed yet MDEV-36027. Thanks to Brandon Nesterenko and Kristian Nielsen for thorough review, and Kristian additionally for ideas to simplify the patch and some code contribution. --- .../suite/rpl/r/rpl_create_select_row.result | 158 +++++++++++++++++ .../suite/rpl/t/rpl_create_select_row.test | 161 ++++++++++++++++++ sql/log.cc | 17 ++ sql/log.h | 1 + sql/sql_insert.cc | 44 ++++- 5 files changed, 374 insertions(+), 7 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_create_select_row.result create mode 100644 mysql-test/suite/rpl/t/rpl_create_select_row.test diff --git a/mysql-test/suite/rpl/r/rpl_create_select_row.result b/mysql-test/suite/rpl/r/rpl_create_select_row.result new file mode 100644 index 00000000000..3176415fc5b --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_create_select_row.result @@ -0,0 +1,158 @@ +include/master-slave.inc +[connection master] +connection master; +set @max_binlog_cache_size = @@global.max_binlog_cache_size; +set @binlog_cache_size = @@global.binlog_cache_size; +set @@global.max_binlog_cache_size = 4096; +set @@global. binlog_cache_size = 4096; +# +# MDEV-35207 ignored error at binlogging by CREATE-TABLE-SELECT leads to assert +# +connect conn_err,localhost,root,,; +call mtr.add_suppression("Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage"); +create table t engine=myisam select repeat ('a',4096*3) AS a; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mariadbd variable and try again +create table t engine=innodb select repeat ('a',4096*3) AS a; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mariadbd variable and try again +create table t (a int unique, b char) select 1 AS a, 'b' as b union select 1 as a, 'c' as b; +ERROR 23000: Duplicate entry '1' for key 'a' +select * from t; +ERROR 42S02: Table 'test.t' doesn't exist +disconnect conn_err; +connection master; + +# +# MDEV-35499 errored CREATE-OR-REPLACE-SELECT does not DROP table in binlog +# +# +# Engine = innodb +# +set statement binlog_format=statement for create table t (a int) select 1 as a; +set statement binlog_format=row for create or replace table t (a int primary key, b char) engine=innodb select 1 AS a, 'b' as b union select 1 as a, 'c' as b; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +select * from t; +ERROR 42S02: Table 'test.t' doesn't exist +# +# Prove an expected lonely `DROP table t' +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Query # # use `test`; DROP TABLE IF EXISTS `test`.`t`/* Generated to handle failed CREATE OR REPLACE */ +master-bin.000001 # Query # # ROLLBACK +set statement binlog_format=statement for create table t (a int) select 1 as a; +set statement binlog_format=row for create or replace table t (a text) engine=innodb select repeat ('a',1024) AS a union select repeat ('a',3*4096) AS a union select repeat ('a',3*4096) AS a; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mariadbd variable and try again +select * from t; +ERROR 42S02: Table 'test.t' doesn't exist +# +# Prove an expected lonely `DROP table t' +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Query # # use `test`; DROP TABLE IF EXISTS `test`.`t`/* Generated to handle failed CREATE OR REPLACE */ +master-bin.000001 # Query # # ROLLBACK +set statement binlog_format=statement for create table t (a int) select 1 as a; +set statement binlog_format=row for create or replace table t (a text) engine=innodb select repeat ('a',4096*3) AS a;; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mariadbd variable and try again +select * from t; +ERROR 42S02: Table 'test.t' doesn't exist +# +# Prove an expected lonely `DROP table t' +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Query # # use `test`; DROP TABLE IF EXISTS `test`.`t`/* Generated to handle failed CREATE OR REPLACE */ +master-bin.000001 # Query # # ROLLBACK +# +# Engine = myisam +# +set statement binlog_format=statement for create table t (a int) select 1 as a; +set statement binlog_format=row for create or replace table t (a int primary key, b char) engine=myisam select 1 AS a, 'b' as b union select 1 as a, 'c' as b; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +select * from t; +ERROR 42S02: Table 'test.t' doesn't exist +# +# Prove an expected lonely `DROP table t' +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Query # # use `test`; DROP TABLE IF EXISTS `test`.`t`/* Generated to handle failed CREATE OR REPLACE */ +master-bin.000001 # Query # # ROLLBACK +set statement binlog_format=statement for create table t (a int) select 1 as a; +set statement binlog_format=row for create or replace table t (a text) engine=myisam select repeat ('a',1024) AS a union select repeat ('a',3*4096) AS a union select repeat ('a',3*4096) AS a; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mariadbd variable and try again +select * from t; +ERROR 42S02: Table 'test.t' doesn't exist +# +# Prove an expected lonely `DROP table t' +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Query # # use `test`; DROP TABLE IF EXISTS `test`.`t`/* Generated to handle failed CREATE OR REPLACE */ +master-bin.000001 # Query # # ROLLBACK +set statement binlog_format=statement for create table t (a int) select 1 as a; +set statement binlog_format=row for create or replace table t (a text) engine=myisam select repeat ('a',4096*3) AS a;; +ERROR HY000: Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage; increase this mariadbd variable and try again +select * from t; +ERROR 42S02: Table 'test.t' doesn't exist +# +# Prove an expected lonely `DROP table t' +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Query # # use `test`; DROP TABLE IF EXISTS `test`.`t`/* Generated to handle failed CREATE OR REPLACE */ +master-bin.000001 # Query # # ROLLBACK +create table ti_pk (a int primary key) engine=innodb; +create table ta (a int) engine=aria; +create function f_ia(arg int) +returns integer +begin +insert into ti_pk set a=1; +insert into ta set a=1; +insert into ti_pk set a=arg; +return 1; +end | +set statement binlog_format = ROW for create table t_y (a int) engine=aria select f_ia(1 /* err */) as a; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +select * from t_y; +ERROR 42S02: Table 'test.t_y' doesn't exist +# correct execution: `ta` is modified and its new record is binlogged +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid # # BEGIN GTID #-#-# +master-bin.000001 # Table_map # # table_id: # (test.ta) +master-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +master-bin.000001 # Query # # COMMIT +select * from ta; +a +1 +select * from ti_pk; +a +connection slave; +include/diff_tables.inc [master:ta,slave:ta] +connection master; +delete from ta; +connection slave; +connection master; +set statement binlog_format = STATEMENT for create table t_y (a int) engine=aria select f_ia(1 /* err */) as a; +ERROR 23000: Duplicate entry '1' for key 'PRIMARY' +select * from t_y; +ERROR 42S02: Table 'test.t_y' doesn't exist +# ***TODO: fix MDEV-36027***. As of now `ta` is modified but that's not binlogged +include/show_binlog_events.inc +select *,'on_master' from ta; +a on_master +1 on_master +select * from ti_pk; +a +connection slave; +select *,'on_slave' from ta; +a on_slave +connection master; +drop function f_ia; +drop table ti_pk, ta; +SET @@global.max_binlog_cache_size = @max_binlog_cache_size; +SET @@global. binlog_cache_size = @binlog_cache_size; +connection slave; +End of the tests +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_create_select_row.test b/mysql-test/suite/rpl/t/rpl_create_select_row.test new file mode 100644 index 00000000000..90d2b85d4e8 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_create_select_row.test @@ -0,0 +1,161 @@ +--source include/have_binlog_format_row.inc +--source include/have_innodb.inc +--source include/master-slave.inc + +--connection master +set @max_binlog_cache_size = @@global.max_binlog_cache_size; +set @binlog_cache_size = @@global.binlog_cache_size; +set @@global.max_binlog_cache_size = 4096; +set @@global. binlog_cache_size = 4096; + +--echo # +--echo # MDEV-35207 ignored error at binlogging by CREATE-TABLE-SELECT leads to assert +--echo # +# fix the current (write) binlog position +--let $binlog_file_0= query_get_value(SHOW MASTER STATUS, File, 1) +--let $binlog_start_0 = query_get_value(SHOW MASTER STATUS, Position, 1) + +# use a separate connection also to validate its close will be clean +connect (conn_err,localhost,root,,); + +call mtr.add_suppression("Multi-statement transaction required more than 'max_binlog_cache_size' bytes of storage"); +--error ER_TRANS_CACHE_FULL +create table t engine=myisam select repeat ('a',4096*3) AS a; + +--error ER_TRANS_CACHE_FULL +create table t engine=innodb select repeat ('a',4096*3) AS a; + +--error ER_DUP_ENTRY +create table t (a int unique, b char) select 1 AS a, 'b' as b union select 1 as a, 'c' as b; +--error ER_NO_SUCH_TABLE +select * from t; + +--disconnect conn_err + +--connection master +--let $binlog_file_1= query_get_value(SHOW MASTER STATUS, File, 1) +--let $binlog_start_1= query_get_value(SHOW MASTER STATUS, Position, 1) + +--let $cmp = `select strcmp('$binlog_file_1', '$binlog_file_0') <> 0 OR $binlog_start_1 <> $binlog_start_0` +if (!$cmp) +{ + --echo *** Error: unexpected advance of binlog position + --die +} + +--echo +--echo # +--echo # MDEV-35499 errored CREATE-OR-REPLACE-SELECT does not DROP table in binlog +--echo # +--let $i = 2 +while ($i) +{ + --let $engine=`select if($i % 2, "myisam", "innodb")` + --echo # + --echo # Engine = $engine + --echo # + set statement binlog_format=statement for create table t (a int) select 1 as a; + --let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1) + --let $binlog_start = query_get_value(SHOW MASTER STATUS, Position, 1) + --error ER_DUP_ENTRY + --eval set statement binlog_format=row for create or replace table t (a int primary key, b char) engine=$engine select 1 AS a, 'b' as b union select 1 as a, 'c' as b + --error ER_NO_SUCH_TABLE + select * from t; + --echo # + --echo # Prove an expected lonely `DROP table t' + --source include/show_binlog_events.inc + + # error before stmt commit + set statement binlog_format=statement for create table t (a int) select 1 as a; + --let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1) + --let $binlog_start = query_get_value(SHOW MASTER STATUS, Position, 1) + --error ER_TRANS_CACHE_FULL + --eval set statement binlog_format=row for create or replace table t (a text) engine=$engine select repeat ('a',1024) AS a union select repeat ('a',3*4096) AS a union select repeat ('a',3*4096) AS a + --error ER_NO_SUCH_TABLE + select * from t; + --echo # + --echo # Prove an expected lonely `DROP table t' + --source include/show_binlog_events.inc + + # error at stmt commit + set statement binlog_format=statement for create table t (a int) select 1 as a; + --let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1) + --let $binlog_start = query_get_value(SHOW MASTER STATUS, Position, 1) + --error ER_TRANS_CACHE_FULL + --eval set statement binlog_format=row for create or replace table t (a text) engine=$engine select repeat ('a',4096*3) AS a; + --error ER_NO_SUCH_TABLE + select * from t; + --echo # + --echo # Prove an expected lonely `DROP table t' + --source include/show_binlog_events.inc + +--dec $i +} + +# Tests of mixed engines to demonstrate non-transaction table updates +# are binlogged or otherwise MDEV-36027. +create table ti_pk (a int primary key) engine=innodb; +create table ta (a int) engine=aria; +delimiter |; +create function f_ia(arg int) +returns integer +begin + insert into ti_pk set a=1; + insert into ta set a=1; + insert into ti_pk set a=arg; + return 1; +end | +delimiter ;| + +--let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1) +--let $binlog_start = query_get_value(SHOW MASTER STATUS, Position, 1) + +--error ER_DUP_ENTRY +set statement binlog_format = ROW for create table t_y (a int) engine=aria select f_ia(1 /* err */) as a; +--error ER_NO_SUCH_TABLE +select * from t_y; + +--echo # correct execution: `ta` is modified and its new record is binlogged +--source include/show_binlog_events.inc +select * from ta; +select * from ti_pk; + +--sync_slave_with_master +--let $diff_tables=master:ta,slave:ta +--source include/diff_tables.inc + +--connection master +delete from ta; +--sync_slave_with_master + +--connection master +# MDEV-36027 Errored-out CREATE-SELECT does not binlog results of any function modifying non-transactional table +--let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1) +--let $binlog_start = query_get_value(SHOW MASTER STATUS, Position, 1) +--error ER_DUP_ENTRY +set statement binlog_format = STATEMENT for create table t_y (a int) engine=aria select f_ia(1 /* err */) as a; +--error ER_NO_SUCH_TABLE +select * from t_y; + +--echo # ***TODO: fix MDEV-36027***. As of now `ta` is modified but that's not binlogged +--source include/show_binlog_events.inc +select *,'on_master' from ta; +select * from ti_pk; + +--sync_slave_with_master +select *,'on_slave' from ta; + +# Cleanup +--connection master +drop function f_ia; +drop table ti_pk, ta; + +SET @@global.max_binlog_cache_size = @max_binlog_cache_size; +SET @@global. binlog_cache_size = @binlog_cache_size; + +# test that binlog replicates correctly to slave +# --connection slave +--sync_slave_with_master + +--echo End of the tests +--source include/rpl_end.inc diff --git a/sql/log.cc b/sql/log.cc index 85f7a5adc11..2f9eab73812 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -322,6 +322,11 @@ public: incident= TRUE; } + void clear_incident(void) + { + incident= FALSE; + } + bool has_incident(void) { return(incident); @@ -2540,6 +2545,18 @@ void binlog_reset_cache(THD *thd) } +void binlog_clear_incident(THD *thd) +{ + binlog_cache_mngr *const cache_mngr= opt_bin_log ? + (binlog_cache_mngr*) thd_get_ha_data(thd, binlog_hton) : 0; + if (cache_mngr) + { + cache_mngr->stmt_cache.clear_incident(); + cache_mngr->trx_cache.clear_incident(); + } +} + + void MYSQL_BIN_LOG::set_write_error(THD *thd, bool is_transactional) { DBUG_ENTER("MYSQL_BIN_LOG::set_write_error"); diff --git a/sql/log.h b/sql/log.h index 90998d1a7bf..6da0851c8cc 100644 --- a/sql/log.h +++ b/sql/log.h @@ -1186,6 +1186,7 @@ File open_binlog(IO_CACHE *log, const char *log_file_name, void make_default_log_name(char **out, const char* log_ext, bool once); void binlog_reset_cache(THD *thd); +void binlog_clear_incident(THD *thd); bool write_annotated_row(THD *thd); extern MYSQL_PLUGIN_IMPORT MYSQL_BIN_LOG mysql_bin_log; diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index b80f6b93d60..f456bea19ad 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -4349,7 +4349,11 @@ bool select_insert::store_values(List &values) bool select_insert::prepare_eof() { int error; - bool const trans_table= table->file->has_transactions_and_rollback(); + // make sure any ROW format pending event is logged in the same binlog cache + bool const trans_table= (thd->is_current_stmt_binlog_format_row() && + table->file->row_logging) ? + table->file->row_logging_has_trans : + table->file->has_transactions_and_rollback(); bool changed; bool binary_logged= 0; killed_state killed_status= thd->killed; @@ -4574,7 +4578,8 @@ void select_insert::abort_result_set() query_cache_invalidate3(thd, table, 1); } DBUG_ASSERT(transactional_table || !changed || - thd->transaction->stmt.modified_non_trans_table); + (thd->transaction->stmt.modified_non_trans_table || + thd->transaction->all.modified_non_trans_table)); table->s->table_creation_was_logged|= binary_logged; table->file->ha_release_auto_increment(); @@ -5267,9 +5272,14 @@ bool select_create::send_eof() /* Remember xid's for the case of row based logging */ ddl_log_update_xid(&ddl_log_state_create, thd->binlog_xid); ddl_log_update_xid(&ddl_log_state_rm, thd->binlog_xid); - trans_commit_stmt(thd); - if (!(thd->variables.option_bits & OPTION_GTID_BEGIN)) - trans_commit_implicit(thd); + if (trans_commit_stmt(thd) || + (!(thd->variables.option_bits & OPTION_GTID_BEGIN) && + trans_commit_implicit(thd))) + { + abort_result_set(); + DBUG_RETURN(true); + } + thd->binlog_xid= 0; #ifdef WITH_WSREP @@ -5389,7 +5399,13 @@ void select_create::abort_result_set() /* possible error of writing binary log is ignored deliberately */ (void) thd->binlog_flush_pending_rows_event(TRUE, TRUE); + /* + In the error case, we remove any partially created table. So clear any + incident event generates due to cache error, as it no longer relevant. + */ + binlog_clear_incident(thd); + bool drop_table_was_logged= false; if (table) { bool tmp_table= table->s->tmp_table; @@ -5436,6 +5452,7 @@ void select_create::abort_result_set() create_info->db_type == partition_hton, &create_info->tabledef_version, tmp_table); + drop_table_was_logged= true; debug_crash_here("ddl_log_create_after_binlog"); thd->binlog_xid= 0; } @@ -5460,8 +5477,21 @@ void select_create::abort_result_set() if (create_info->table_was_deleted) { - /* Unlock locked table that was dropped by CREATE. */ - (void) trans_rollback_stmt(thd); + if (drop_table_was_logged) + { + /* for DROP binlogging the error status has to be canceled first */ + Diagnostics_area new_stmt_da(thd->query_id, false, true); + Diagnostics_area *old_stmt_da= thd->get_stmt_da(); + + thd->set_stmt_da(&new_stmt_da); + (void) trans_rollback_stmt(thd); + thd->set_stmt_da(old_stmt_da); + } + else + { + /* Unlock locked table that was dropped by CREATE. */ + (void) trans_rollback_stmt(thd); + } thd->locked_tables_list.unlock_locked_table(thd, create_info->mdl_ticket); } From c06c36218a59cf3ada01b096cb715a5e1cedfe2b Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Mon, 25 Nov 2024 19:06:51 +0200 Subject: [PATCH 006/125] MDEV-35506 commit policy of one-phase-commit even at errored-out binlogging leads to assert Currently execution of commit in one phase proceeds to commit by engines when binlog_commit() does not succeed. There are two issues with that: 1. absence of binlog_rollback() or lower-level `binlog_cache_data::reset()` along the following execution of the failing statement eventually will raise an assert on non-empty binlog cache, find in the MDEV description # --error assert(sql/log.cc:1712(binlog_close_connection)) # --disconnect default 2. engines, including ones that are rollback capable, commit in this particular error situation. Both effects can be observed with a new mtr test that would fail when run on a BASE of this commit. The BASE has to include MDEV-35207 et all fixes because the test is written with CREATE-TABLE-SELECTs. A new test file verifies the new behaviour to rollback including cases with a side effect of modified non-transactional engine which expose another MDEV-36027 (TODO: fix). --- .../suite/binlog/r/binlog_commit_fail.result | 116 +++++++++++++++ .../suite/binlog/t/binlog_commit_fail.test | 135 ++++++++++++++++++ sql/handler.cc | 8 +- 3 files changed, 257 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/binlog/r/binlog_commit_fail.result create mode 100644 mysql-test/suite/binlog/t/binlog_commit_fail.test diff --git a/mysql-test/suite/binlog/r/binlog_commit_fail.result b/mysql-test/suite/binlog/r/binlog_commit_fail.result new file mode 100644 index 00000000000..dd9e62be638 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_commit_fail.result @@ -0,0 +1,116 @@ +set @@session.gtid_domain_id=1; +set @save_gtid_stric_mode=@@global.gtid_strict_mode; +create table ta (a int) engine=aria; +create table ti (a int) engine=innodb; +create table ti_pk (a int primary key) engine=innodb; +create table t (a int) engine=innodb; +create function f_i() +returns integer +begin +insert into ti set a=1; +return 1; +end | +create function f_ia(arg int) +returns integer +begin +insert into ti_pk set a=1; +insert into ta set a=1; +insert into ti_pk set a=arg; +return 1; +end | +call mtr.add_suppression("Error writing file"); +select count(*) as zero from t; +zero +0 +select count(*) as zero from ta; +zero +0 +select count(*) as zero from ti; +zero +0 +# 1. simple Innodb test +set @@global.gtid_strict_mode=0; +set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +insert into t set a=1; +ERROR HY000: An attempt was made to binlog GTID VALUE which would create an out-of-order sequence number with existing GTID VALUE, and gtid strict mode is enabled +# observe effective rollback +select count(*) as zero from t; +zero +0 +# 2. simple Aira test +set @@global.gtid_strict_mode=0; +set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +insert into ta values (1),(2); +ERROR HY000: An attempt was made to binlog GTID VALUE which would create an out-of-order sequence number with existing GTID VALUE, and gtid strict mode is enabled +# note no rollback +select count(*) as '*NON-zero*' from ta; +*NON-zero* +2 +delete from ta; +# 3. multi-engine test +set @@global.gtid_strict_mode=0; +set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +insert into ta set a=f_i(); +ERROR HY000: An attempt was made to binlog GTID VALUE which would create an out-of-order sequence number with existing GTID VALUE, and gtid strict mode is enabled +# note no rollback.. +select count(*) as one from ta; +one +1 +# ..except transactional engine +select count(*) as zero from ti; +zero +0 +delete from ta; +set @@global.gtid_strict_mode=0; +set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +insert into t set a=f_ia(0); +ERROR HY000: An attempt was made to binlog GTID VALUE which would create an out-of-order sequence number with existing GTID VALUE, and gtid strict mode is enabled +# note no rollback.. +select count(*) as one from ta; +one +1 +# ..except transactional engine +select count(*) as zero from t; +zero +0 +select count(*) as zero from ti_pk; +zero +0 +delete from ta; +# 4. create-table-select-f() +set @@global.gtid_strict_mode=0; +set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +create table f_x (a int) select f_i() as a; +ERROR HY000: An attempt was made to binlog GTID VALUE which would create an out-of-order sequence number with existing GTID VALUE, and gtid strict mode is enabled +# rollback indeed takes place in the pure transactional case +select count(*) as zero from ti; +zero +0 +set @@global.gtid_strict_mode=0; +set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +create table t_x (a int) engine=aria select f_ia(0) as a; +ERROR HY000: An attempt was made to binlog GTID VALUE which would create an out-of-order sequence number with existing GTID VALUE, and gtid strict mode is enabled +select * from t_x; +ERROR 42S02: Table 'test.t_x' doesn't exist +# **TODO**: fix MDEV-36027 +# **TODO**: the empty binlog is buggy .. +include/show_binlog_events.inc +# .. as non-transactional `ta` (and `t_x` sic!) are modified +select count(*) as one from ta; +one +1 +select count(*) as zero from ti; +zero +0 +delete from ta; +#. +set @@global.gtid_strict_mode=@save_gtid_stric_mode; +drop function f_i; +drop function f_ia; +drop table t, ta, ti, ti_pk; diff --git a/mysql-test/suite/binlog/t/binlog_commit_fail.test b/mysql-test/suite/binlog/t/binlog_commit_fail.test new file mode 100644 index 00000000000..d88ad552b07 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_commit_fail.test @@ -0,0 +1,135 @@ +# Tests of commit time failures. +# At committing of an auto-commit statement a failure to commit in its +# binlog branch should rollback at least the transactional part of the statement. +# +# References: +# MDEV-35506 commit policy of one-phase-commit even at errored-out binlogging leads to assert +# MDEV-36027 Errored-out CREATE-SELECT does not binlog results of non-transactional table modification + +source include/have_innodb.inc; +source include/have_binlog_format_row.inc; + +set @@session.gtid_domain_id=1; +set @save_gtid_stric_mode=@@global.gtid_strict_mode; + +create table ta (a int) engine=aria; +create table ti (a int) engine=innodb; +create table ti_pk (a int primary key) engine=innodb; +create table t (a int) engine=innodb; +delimiter |; +create function f_i() +returns integer +begin + insert into ti set a=1; +return 1; +end | +create function f_ia(arg int) +returns integer +begin + insert into ti_pk set a=1; + insert into ta set a=1; + insert into ti_pk set a=arg; + return 1; +end | +delimiter ;| + +call mtr.add_suppression("Error writing file"); + +# Naturally all empty now +select count(*) as zero from t; +select count(*) as zero from ta; +select count(*) as zero from ti; + +# Force manual value assignement to gtid::seq_no while in the strict mode +# so that the value is rejected. Despite the errorred out statement +# being at its commit phase it will eventually be rolled back. +# Side effects of non-transactional engines, like Aria, are displayed. +--echo # 1. simple Innodb test +set @@global.gtid_strict_mode=0; set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +# mask possible allowed seq_no shift +--replace_regex /GTID 1-1-[0-9]+/GTID VALUE/ +--error ER_GTID_STRICT_OUT_OF_ORDER +insert into t set a=1; + +--echo # observe effective rollback +select count(*) as zero from t; + +--echo # 2. simple Aira test +set @@global.gtid_strict_mode=0; set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +--replace_regex /GTID 1-1-[0-9]+/GTID VALUE/ +--error ER_GTID_STRICT_OUT_OF_ORDER +insert into ta values (1),(2); + +--echo # note no rollback +select count(*) as '*NON-zero*' from ta; +# local cleanup +delete from ta; + +--echo # 3. multi-engine test +# A. non-transactional top-level +set @@global.gtid_strict_mode=0; set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +--replace_regex /GTID 1-1-[0-9]+/GTID VALUE/ +--error ER_GTID_STRICT_OUT_OF_ORDER +insert into ta set a=f_i(); +--echo # note no rollback.. +select count(*) as one from ta; +--echo # ..except transactional engine +select count(*) as zero from ti; +delete from ta; + +# B. non-transactional in the leaf +set @@global.gtid_strict_mode=0; set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +--replace_regex /GTID 1-1-[0-9]+/GTID VALUE/ +--error ER_GTID_STRICT_OUT_OF_ORDER +insert into t set a=f_ia(0); + +--echo # note no rollback.. +select count(*) as one from ta; +--echo # ..except transactional engine +select count(*) as zero from t; +select count(*) as zero from ti_pk; +delete from ta; + +--echo # 4. create-table-select-f() +--let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1) +--let $binlog_start = query_get_value(SHOW MASTER STATUS, Position, 1) +# A. two phase commit branch +set @@global.gtid_strict_mode=0; set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +--replace_regex /GTID 1-1-[0-9]+/GTID VALUE/ +--error ER_GTID_STRICT_OUT_OF_ORDER +create table f_x (a int) select f_i() as a; +--echo # rollback indeed takes place in the pure transactional case +select count(*) as zero from ti; + +# B. one phase commit branch +--let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1) +--let $binlog_start = query_get_value(SHOW MASTER STATUS, Position, 1) +set @@global.gtid_strict_mode=0; set @@session.gtid_seq_no=1; +set @@global.gtid_strict_mode=1; +--replace_regex /GTID 1-1-[0-9]+/GTID VALUE/ +--error ER_GTID_STRICT_OUT_OF_ORDER +create table t_x (a int) engine=aria select f_ia(0) as a; +--error ER_NO_SUCH_TABLE +select * from t_x; + +--echo # **TODO**: fix MDEV-36027 +--echo # **TODO**: the empty binlog is buggy .. +--source include/show_binlog_events.inc +--echo # .. as non-transactional `ta` (and `t_x` sic!) are modified +select count(*) as one from ta; +select count(*) as zero from ti; + +delete from ta; +--echo #. + +# cleanup + +set @@global.gtid_strict_mode=@save_gtid_stric_mode; +drop function f_i; +drop function f_ia; +drop table t, ta, ti, ti_pk; diff --git a/sql/handler.cc b/sql/handler.cc index 3d1415b3dec..0088b84a2ec 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1901,6 +1901,8 @@ int ha_commit_trans(THD *thd, bool all) } #endif /* WITH_WSREP */ error= ha_commit_one_phase(thd, all); + if (error) + goto err; #ifdef WITH_WSREP // Here in case of error we must return 2 for inconsistency if (run_wsrep_hooks && !error) @@ -2141,7 +2143,7 @@ commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) if (ha_info) { - int err; + int err= 0; if (has_binlog_hton(ha_info) && (err= binlog_commit(thd, all, @@ -2149,6 +2151,8 @@ commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) { my_error(ER_ERROR_DURING_COMMIT, MYF(0), err); error= 1; + + goto err; } for (; ha_info; ha_info= ha_info_next) { @@ -2184,7 +2188,7 @@ commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) if (count >= 2) statistic_increment(transactions_multi_engine, LOCK_status); } - + err: DBUG_RETURN(error); } From e176066a9e02cf2bdedf9e79fb150029dde430f6 Mon Sep 17 00:00:00 2001 From: Dmitry Shulga Date: Fri, 4 Apr 2025 11:16:31 +0700 Subject: [PATCH 007/125] MDEV-36462: Crash on `DECLARE spvar1 ROW TYPE OF cursor1` after a table recreation After a cursor's statement is re-parsed by the reason of metadata changes for tables the statement depends on, following memory allocations taken place on cursor execution is performed on a memory root already marked as read only despite the fact that a new memory root has been allocated for this goal. To fix the issue, bind the cursor lex with a new memory root created for re-parsing cursor's statement, clean up items stored on cursor's free list and nullify the data member sp_lex_cursor::free_list to avoid dangling pointer problem. --- mysql-test/main/sp-cursor.result | 63 ++++++++++++++++++++++++++++++++ mysql-test/main/sp-cursor.test | 56 ++++++++++++++++++++++++++++ sql/sp_instr.cc | 29 +++++++++++++-- sql/sp_instr.h | 3 +- 4 files changed, 147 insertions(+), 4 deletions(-) diff --git a/mysql-test/main/sp-cursor.result b/mysql-test/main/sp-cursor.result index 7afd4c33aef..9cc2c3ea1a8 100644 --- a/mysql-test/main/sp-cursor.result +++ b/mysql-test/main/sp-cursor.result @@ -923,4 +923,67 @@ SELECT pkg.f1(); pkg.f1() 1 2 DROP PACKAGE pkg; +# +# MDEV-36462: Crash on `DECLARE spvar1 ROW TYPE OF cursor1` after a table recreation +# +CREATE PROCEDURE p1() +BEGIN +DECLARE c CURSOR FOR SELECT a FROM t1; +BEGIN +DECLARE va ROW TYPE OF c; -- the crash happens here +END; +END; +/ +CREATE PROCEDURE p2() +BEGIN +FOR i IN 1..10 DO -- usually it crashes on the third iteration, but not always +SELECT i; +CREATE OR REPLACE TABLE t1 (a INT); +CALL p1; +CALL p1; +END FOR; +END; +/ +CALL p2; +i +1 +i +2 +i +3 +i +4 +i +5 +i +6 +i +7 +i +8 +i +9 +i +10 +# Clean up +DROP PROCEDURE p1; +DROP PROCEDURE p2; +DROP TABLE t1; +# The following test is taken from the task MDEV-36114 which is +# partially a duplicate of the task MDEV-36462 +CREATE PROCEDURE p() +BEGIN +DECLARE cur1 CURSOR FOR SELECT * FROM t; +BEGIN +DECLARE rec1 ROW TYPE OF cur1; +END; +END; +/ +CREATE TABLE t (id INT); +CALL p(); +CREATE OR REPLACE TABLE t (id INT); +CALL p(); +# Clean up +DROP PROCEDURE p; +DROP TABLE t; # End of 11.4 tests diff --git a/mysql-test/main/sp-cursor.test b/mysql-test/main/sp-cursor.test index d15a0c80a6f..b4e73aaf637 100644 --- a/mysql-test/main/sp-cursor.test +++ b/mysql-test/main/sp-cursor.test @@ -891,4 +891,60 @@ DROP PACKAGE pkg; SELECT pkg.f1(); DROP PACKAGE pkg; +--echo # +--echo # MDEV-36462: Crash on `DECLARE spvar1 ROW TYPE OF cursor1` after a table recreation +--echo # + +--delimiter / + +CREATE PROCEDURE p1() +BEGIN + DECLARE c CURSOR FOR SELECT a FROM t1; + BEGIN + DECLARE va ROW TYPE OF c; -- the crash happens here + END; +END; +/ + +CREATE PROCEDURE p2() +BEGIN + FOR i IN 1..10 DO -- usually it crashes on the third iteration, but not always + SELECT i; + CREATE OR REPLACE TABLE t1 (a INT); + CALL p1; + CALL p1; + END FOR; +END; +/ + +--delimiter ; + +CALL p2; + +--echo # Clean up +DROP PROCEDURE p1; +DROP PROCEDURE p2; +DROP TABLE t1; + +--echo # The following test is taken from the task MDEV-36114 which is +--echo # partially a duplicate of the task MDEV-36462 +--delimiter / +CREATE PROCEDURE p() +BEGIN + DECLARE cur1 CURSOR FOR SELECT * FROM t; + BEGIN + DECLARE rec1 ROW TYPE OF cur1; + END; +END; +/ +--delimiter ; +CREATE TABLE t (id INT); +CALL p(); +CREATE OR REPLACE TABLE t (id INT); +CALL p(); + +--echo # Clean up +DROP PROCEDURE p; +DROP TABLE t; + --echo # End of 11.4 tests diff --git a/sql/sp_instr.cc b/sql/sp_instr.cc index 7e403a24281..b15eeef6deb 100644 --- a/sql/sp_instr.cc +++ b/sql/sp_instr.cc @@ -662,11 +662,16 @@ bool sp_lex_instr::setup_table_fields_for_trigger( re-parsing. @param sphead The stored program. + @param[out] new_memroot_allocated true in case a new memory root for + re-parsing was created, else false meaning + that already allocated memory root is + reused @return false on success, true on error (OOM) */ -bool sp_lex_instr::setup_memroot_for_reparsing(sp_head *sphead) +bool sp_lex_instr::setup_memroot_for_reparsing(sp_head *sphead, + bool *new_memroot_allocated) { if (!m_mem_root_for_reparsing) { @@ -711,6 +716,8 @@ bool sp_lex_instr::setup_memroot_for_reparsing(sp_head *sphead) if (!m_mem_root_for_reparsing) return true; + + *new_memroot_allocated= true; } else { @@ -721,6 +728,7 @@ bool sp_lex_instr::setup_memroot_for_reparsing(sp_head *sphead) statement. */ free_root(m_mem_root_for_reparsing, MYF(0)); + *new_memroot_allocated= false; } init_sql_alloc(key_memory_sp_head_main_root, m_mem_root_for_reparsing, @@ -787,7 +795,8 @@ LEX* sp_lex_instr::parse_expr(THD *thd, sp_head *sp, LEX *sp_instr_lex) /* First, set up a men_root for the statement is going to re-compile. */ - if (setup_memroot_for_reparsing(sp)) + bool mem_root_allocated; + if (setup_memroot_for_reparsing(sp, &mem_root_allocated)) return nullptr; /* @@ -842,8 +851,22 @@ LEX* sp_lex_instr::parse_expr(THD *thd, sp_head *sp, LEX *sp_instr_lex) data member. So, for the sp_instr_cpush instruction by the time we reach this block cursor_lex->free_list is already empty. */ - cleanup_items(cursor_lex->free_list); + if (mem_root_allocated) + /* + If the new memory root for re-parsing has been just created, + then delete every item from the free item list of sp_lex_cursor. + In case the memory root for re-parsing is re-used from previous + re-parsing of failed instruction, don't do anything since all memory + allocated for items were already released on calling free_root + inside the method sp_lex_instr::setup_memroot_for_reparsing + */ + cursor_lex->free_items(); + + /* Nullify free_list to don't have a dangling pointer */ + cursor_lex->free_list= nullptr; + cursor_free_list= &cursor_lex->free_list; + cursor_lex->mem_root= m_mem_root_for_reparsing; DBUG_ASSERT(thd->lex == sp_instr_lex); lex_start(thd); } diff --git a/sql/sp_instr.h b/sql/sp_instr.h index a93f681e5d1..5d5af0a18f0 100644 --- a/sql/sp_instr.h +++ b/sql/sp_instr.h @@ -522,7 +522,8 @@ private: THD *thd, sp_head *sp, SQL_I_List *next_trig_items_list); - bool setup_memroot_for_reparsing(sp_head *sphead); + bool setup_memroot_for_reparsing(sp_head *sphead, + bool *new_memroot_allocated); }; From dca2e5509e38adca8fec18dbb9c330defbe9f131 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 4 Apr 2025 16:22:30 +1100 Subject: [PATCH 008/125] MDEV-36480 USAN: checking identifier names for 0 length names Identifier names can be empty in the grammar. The check_ident_length is used from everything from triggers, to partitions, to key names and UDF names. This change updates 0 length identifiers as valid without further checking. Primary keys are one clear case where a empty name is used and the name.str is a null pointer. Checking empty names where the key->name.str is a null pointer results in a UBSAN error in Well_formed_prefix_status further down the stack which we can avoid. --- sql/sql_parse.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 62bb759b3f3..54f08132c1c 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -10389,7 +10389,13 @@ bool check_string_char_length(const LEX_CSTRING *str, uint err_msg, bool check_ident_length(const LEX_CSTRING *ident) { - if (check_string_char_length(ident, 0, NAME_CHAR_LEN, system_charset_info, 1)) + /* + string_char_length desite the names, goes into Well_formed_prefix_status + so this is more than just a length comparison. Things like a primary key + doesn't have a name, therefore no length. Also the ident grammar allows + empty backtick. Check quickly the length, and if 0, accept that. + */ + if (ident->length && check_string_char_length(ident, 0, NAME_CHAR_LEN, system_charset_info, 1)) { my_error(ER_TOO_LONG_IDENT, MYF(0), ident->str); return 1; From ff4209fa0d1c66952bc9f00587f525314464e7be Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 1 Apr 2025 10:39:25 +0200 Subject: [PATCH 009/125] Fix broken clang-cl compilation - fix several Windows-specific "variable set but not used", or "variable unused" warnings. - correctly initialize std::atomic_flag (ATOMIC_FLAG_INIT) - fix Ninja build for spider on Windows - adjust check for sizeof(MYSQL) for Windows compilers --- sql/handle_connections_win.cc | 3 --- sql/mysqld.cc | 12 ++++++------ storage/innobase/include/fil0fil.h | 7 +++++-- storage/spider/CMakeLists.txt | 11 ----------- tests/mysql_client_fw.c | 4 +++- 5 files changed, 14 insertions(+), 23 deletions(-) diff --git a/sql/handle_connections_win.cc b/sql/handle_connections_win.cc index ffacfcab88f..9c98620e6d2 100644 --- a/sql/handle_connections_win.cc +++ b/sql/handle_connections_win.cc @@ -595,11 +595,8 @@ void network_init_win() void handle_connections_win() { - int n_waits; - create_shutdown_event(); wait_events.push_back(hEventShutdown); - n_waits= 1; for (size_t i= 0; i < all_listeners.size(); i++) { diff --git a/sql/mysqld.cc b/sql/mysqld.cc index b3e22cd7eed..e7a0daedc41 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1398,11 +1398,6 @@ bool unix_sock_is_online= false; static int systemd_sock_activation; /* systemd socket activation */ - -/** wakeup listening(main) thread by writing to this descriptor */ -static int termination_event_fd= -1; - - C_MODE_START #ifdef WITH_PERFSCHEMA_STORAGE_ENGINE /** @@ -1455,9 +1450,14 @@ static pthread_t select_thread; #endif /* OS specific variables */ - +#ifndef EMBEDDED_LIBRARY #ifdef _WIN32 +/** wakeup main thread by signaling this event */ HANDLE hEventShutdown; +#else +/** wakeup listening(main) thread by writing to this descriptor */ +static int termination_event_fd= -1; +#endif #endif diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 49449813bc3..c3514c96cf7 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -422,7 +422,7 @@ private: bool being_imported= false; /** Whether any corrupton of this tablespace has been reported */ - mutable std::atomic_flag is_corrupted{false}; + mutable std::atomic_flag is_corrupted= ATOMIC_FLAG_INIT; public: /** mutex to protect freed_ranges and last_freed_lsn */ @@ -1527,7 +1527,10 @@ extern fil_system_t fil_system; inline void fil_space_t::reacquire() noexcept { - ut_d(uint32_t n=) n_pending.fetch_add(1, std::memory_order_relaxed); +#ifdef SAFE_MUTEX + uint32_t n= +#endif + n_pending.fetch_add(1, std::memory_order_relaxed); #ifdef SAFE_MUTEX if (mysql_mutex_is_owner(&fil_system.mutex)) return; ut_ad(n & PENDING); diff --git a/storage/spider/CMakeLists.txt b/storage/spider/CMakeLists.txt index bc1387db236..f44fb3f8172 100644 --- a/storage/spider/CMakeLists.txt +++ b/storage/spider/CMakeLists.txt @@ -28,16 +28,5 @@ ELSE() MYSQL_ADD_PLUGIN(spider ${SPIDER_SOURCES} ${extra_options} STORAGE_ENGINE MODULE_ONLY) - IF(NOT TARGET spider) - RETURN() - ENDIF() ENDIF() -IF(MSVC AND (TARGET spider)) - IF (CMAKE_BUILD_TYPE STREQUAL "Debug") - ADD_CUSTOM_COMMAND(TARGET spider - POST_BUILD - COMMAND if not exist ..\\..\\sql\\lib mkdir ..\\..\\sql\\lib\\plugin - COMMAND copy Debug\\ha_spider.dll ..\\..\\sql\\lib\\plugin\\ha_spider.dll) - ENDIF() -ENDIF() diff --git a/tests/mysql_client_fw.c b/tests/mysql_client_fw.c index f1cd6deb9e2..b75d84ef19e 100644 --- a/tests/mysql_client_fw.c +++ b/tests/mysql_client_fw.c @@ -1438,7 +1438,9 @@ int main(int argc, char **argv) this limited check is enough, if sizeof(MYSQL) changes, it changes everywhere */ -#if defined __x86_64__ +#if defined _M_AMD64 + compile_time_assert(sizeof(MYSQL) == 1208); +#elif defined __x86_64__ compile_time_assert(sizeof(MYSQL) == 1272); #elif defined __i386__ compile_time_assert(sizeof(MYSQL) == 964); From b005b6097fa5d4aea3b14b93745ffd7b6c27ba22 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 3 Apr 2025 12:25:44 +0200 Subject: [PATCH 010/125] Cleanup CMake code (Windows-specific) Prepare for a more modern CMake version than the current minimum. - Use CMAKE_MSVC_RUNTIME_LIBRARY instead of the custom MSVC_CRT_TYPE. - Replace CMAKE_{C,CXX}_FLAGS modifications with add_compile_definitions/options and add_link_options. The older method already broke with new pcre2. - Fix clang-cl compilation and ASAN build. - Avoid modifying CMAKE_C_STANDARD_LIBRARIES/CMAKE_CXX_STANDARD_LIBRARIES, as this is discouraged by CMake. - Reduce system checks. --- CMakeLists.txt | 2 +- cmake/os/Windows.cmake | 528 +++++++++++------------------- cmake/os/WindowsCache.cmake | 19 +- cmake/plugin.cmake | 5 + config.h.cmake | 43 +-- mysys/CMakeLists.txt | 2 +- storage/connect/CMakeLists.txt | 6 +- win/packaging/ca/CMakeLists.txt | 5 +- win/upgrade_wizard/CMakeLists.txt | 20 +- 9 files changed, 235 insertions(+), 395 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6627d9ffc0..db08223093b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,7 @@ ENDIF() # in RPM's: #set(CPACK_RPM_SPEC_MORE_DEFINE "%define __spec_install_post /bin/true") -FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0056 CMP0067 CMP0074 CMP0075 CMP0069 CMP0135) +FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0056 CMP0067 CMP0074 CMP0075 CMP0069 CMP0135 CMP0091) IF(POLICY ${p}) CMAKE_POLICY(SET ${p} NEW) ENDIF() diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index 510413d915c..cdb16cde468 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -15,372 +15,212 @@ # This file includes Windows specific hacks, mostly around compiler flags -INCLUDE (CheckCSourceCompiles) -INCLUDE (CheckCXXSourceCompiles) -INCLUDE (CheckStructHasMember) -INCLUDE (CheckLibraryExists) -INCLUDE (CheckFunctionExists) -INCLUDE (CheckCSourceRuns) -INCLUDE (CheckSymbolExists) -INCLUDE (CheckTypeSize) -IF(MSVC) - IF(CMAKE_CXX_COMPILER_ARCHITECTURE_ID STREQUAL ARM64) - SET(MSVC_ARM64 1) - SET(MSVC_INTEL 0) - ELSE() - SET(MSVC_INTEL 1) - ENDIF() -ENDIF() +if(MSVC) + if(CMAKE_CXX_COMPILER_ARCHITECTURE_ID STREQUAL ARM64) + set(MSVC_ARM64 1) + set(MSVC_INTEL 0) + else() + set(MSVC_INTEL 1) + endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL Clang) + set(CLANG_CL TRUE) + endif() +endif() # avoid running system checks by using pre-cached check results # system checks are expensive on VS since every tiny program is to be compiled in # a VC solution. -GET_FILENAME_COMPONENT(_SCRIPT_DIR ${CMAKE_CURRENT_LIST_FILE} PATH) -INCLUDE(${_SCRIPT_DIR}/WindowsCache.cmake) - +get_filename_component(_SCRIPT_DIR ${CMAKE_CURRENT_LIST_FILE} PATH) +include(${_SCRIPT_DIR}/WindowsCache.cmake) # OS display name (version_compile_os etc). -# Used by the test suite to ignore bugs on some platforms, -IF(CMAKE_SIZEOF_VOID_P MATCHES 8) - SET(SYSTEM_TYPE "Win64") -ELSE() - SET(SYSTEM_TYPE "Win32") -ENDIF() +# Used by the test suite to ignore bugs on some platforms +if(CMAKE_SIZEOF_VOID_P EQUAL 8) + set(SYSTEM_TYPE "Win64") +else() + set(SYSTEM_TYPE "Win32") +endif() -# Intel compiler is almost Visual C++ -# (same compile flags etc). Set MSVC flag -IF(CMAKE_C_COMPILER MATCHES "icl") - SET(MSVC TRUE) -ENDIF() +function(find_asan_runtime result_list) + set(${result_list} "" PARENT_SCOPE) + if(CMAKE_C_COMPILER_VERSION) + set(CLANG_VERSION "${CMAKE_C_COMPILER_VERSION}") + else() + return() + endif() -IF(MSVC AND CMAKE_CXX_COMPILER_ID MATCHES Clang) - SET(CLANG_CL TRUE) -ENDIF() + get_filename_component(CLANG_BIN_DIR "${CMAKE_C_COMPILER}" DIRECTORY) + get_filename_component(LLVM_ROOT "${CLANG_BIN_DIR}" DIRECTORY) -ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE) -ADD_DEFINITIONS(-D_WIN32_WINNT=0x0A00) -# We do not want the windows.h , or winsvc.h macros min/max -ADD_DEFINITIONS(-DNOMINMAX -DNOSERVICE) -# Speed up build process excluding unused header files -ADD_DEFINITIONS(-DWIN32_LEAN_AND_MEAN) - -# Adjust compiler and linker flags -IF(MINGW AND CMAKE_SIZEOF_VOID_P EQUAL 4) - # mininal architecture flags, i486 enables GCC atomics - ADD_DEFINITIONS(-march=i486) -ENDIF() - -MACRO(ENABLE_SANITIZERS) - IF(NOT MSVC) - MESSAGE(FATAL_ERROR "clang-cl or MSVC necessary to enable asan/ubsan") - ENDIF() - # currently, asan is broken with static CRT. - IF(CLANG_CL AND NOT(MSVC_CRT_TYPE STREQUAL "/MD")) - SET(MSVC_CRT_TYPE "/MD" CACHE INTERNAL "" FORCE) - ENDIF() - IF(CMAKE_SIZEOF_VOID_P EQUAL 4) - SET(ASAN_ARCH i386) - ELSE() - SET(ASAN_ARCH x86_64) - ENDIF() - - # After installation, clang lib directory should be added to PATH - # (e.g C:/Program Files/LLVM/lib/clang/5.0.1/lib/windows) - SET(SANITIZER_LIBS) - SET(SANITIZER_LINK_LIBRARIES) - SET(SANITIZER_COMPILE_FLAGS) - IF(WITH_ASAN) - IF(CLANG_CL) - LIST(APPEND SANITIZER_LIBS - clang_rt.asan_dynamic-${ASAN_ARCH}.lib clang_rt.asan_dynamic_runtime_thunk-${ASAN_ARCH}.lib) - ENDIF() - STRING(APPEND SANITIZER_COMPILE_FLAGS " -fsanitize=address") - ENDIF() - IF(WITH_UBSAN) - STRING(APPEND SANITIZER_COMPILE_FLAGS " -fsanitize=undefined -fno-sanitize=alignment") - ENDIF() - FOREACH(lib ${SANITIZER_LIBS}) - FIND_LIBRARY(${lib}_fullpath ${lib}) - IF(NOT ${lib}_fullpath) - MESSAGE(FATAL_ERROR "Can't enable sanitizer : missing ${lib}") - ENDIF() - LIST(APPEND CMAKE_REQUIRED_LIBRARIES ${${lib}_fullpath}) - STRING(APPEND CMAKE_C_STANDARD_LIBRARIES " \"${${lib}_fullpath}\" ") - STRING(APPEND CMAKE_CXX_STANDARD_LIBRARIES " \"${${lib}_fullpath}\" ") - ENDFOREACH() - STRING(APPEND CMAKE_C_FLAGS ${SANITIZER_COMPILE_FLAGS}) - STRING(APPEND CMAKE_CXX_FLAGS ${SANITIZER_COMPILE_FLAGS}) -ENDMACRO() - - -IF(MSVC) - IF(MSVC_VERSION LESS 1920) - MESSAGE(FATAL_ERROR "Visual Studio 2019 or later is required") - ENDIF() - # Disable mingw based pkg-config found in Strawberry perl - SET(PKG_CONFIG_EXECUTABLE 0 CACHE INTERNAL "") - - SET(MSVC_CRT_TYPE /MD CACHE STRING - "Runtime library - specify runtime library for linking (/MT,/MTd,/MD,/MDd)" + # Determine target architecture + execute_process( + COMMAND "${CMAKE_C_COMPILER}" --version + OUTPUT_VARIABLE CLANG_VERSION_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET ) - SET(VALID_CRT_TYPES /MTd /MDd /MD /MT) - IF (NOT ";${VALID_CRT_TYPES};" MATCHES ";${MSVC_CRT_TYPE};") - MESSAGE(FATAL_ERROR "Invalid value ${MSVC_CRT_TYPE} for MSVC_CRT_TYPE, choose one of /MT,/MTd,/MD,/MDd ") - ENDIF() - # CMake version 3.15 and later uses CMAKE_MSVC_RUNTIME_LIBRARY - # variable for our MSVC_CRT_TYPE. - # Set CMAKE_MSVC_RUNTIME_LIBRARY and pass to external projects - # it is important to keep the same CRT type when linking - # - # Translation rules MSVC_CRT_TYPE -> CMAKE_MSVC_RUNTIME_LIBRARY - # /MT -> MultiThreaded - # /MTd -> MultiThreadedDebug - # /MD -> MultiThreadedDLL - # /MDd -> MultiThreadedDebugDLL + if(CLANG_VERSION_OUTPUT MATCHES "x86_64") + set(ARCH_SUFFIX "x86_64") + elseif(CLANG_VERSION_OUTPUT MATCHES "i686|i386") + set(ARCH_SUFFIX "i386") + elseif(CLANG_VERSION_OUTPUT MATCHES "aarch64") + set(ARCH_SUFFIX "aarch64") + else() + message(FATAL_ERROR "unknown arch") + endif() - SET(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded) - IF(MSVC_CRT_TYPE MATCHES "d$") - STRING(APPEND CMAKE_MSVC_RUNTIME_LIBRARY Debug) - ENDIF() - IF(MSVC_CRT_TYPE MATCHES "D") - STRING(APPEND CMAKE_MSVC_RUNTIME_LIBRARY DLL) - ENDIF() + string(REGEX MATCH "^[0-9]+" CLANG_MAJOR_VERSION "${CMAKE_C_COMPILER_VERSION}") + set(CLANG_VERSION_DIR "${LLVM_ROOT}/lib/clang/${CLANG_MAJOR_VERSION}") - IF(MSVC_CRT_TYPE MATCHES "/MD") - # Dynamic runtime (DLLs), need to install CRT libraries. - SET(CMAKE_INSTALL_SYSTEM_RUNTIME_COMPONENT VCCRT) - SET(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_NO_WARNINGS TRUE) - IF(MSVC_CRT_TYPE STREQUAL "/MDd") - SET (CMAKE_INSTALL_DEBUG_LIBRARIES_ONLY TRUE) - ENDIF() - INCLUDE(InstallRequiredSystemLibraries) - ENDIF() - - IF(WITH_ASAN AND (NOT CLANG_CL)) - SET(DYNAMIC_UCRT_LINK_DEFAULT OFF) - ELSE() - SET(DYNAMIC_UCRT_LINK_DEFAULT ON) - ENDIF() - - OPTION(DYNAMIC_UCRT_LINK "Link Universal CRT dynamically, if MSVC_CRT_TYPE=/MT" ${DYNAMIC_UCRT_LINK_DEFAULT}) - SET(DYNAMIC_UCRT_LINKER_OPTION " /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib") - - # Enable debug info also in Release build, - # and create PDB to be able to analyze crashes. - FOREACH(type EXE SHARED MODULE) - SET(CMAKE_${type}_LINKER_FLAGS_RELEASE - "${CMAKE_${type}_LINKER_FLAGS_RELEASE} /debug") - SET(CMAKE_${type}_LINKER_FLAGS_MINSIZEREL - "${CMAKE_${type}_LINKER_FLAGS_MINSIZEREL} /debug") - ENDFOREACH() - - # Force runtime libraries - # Compile with /Zi to get debugging information - - FOREACH(lang C CXX) - SET(CMAKE_${lang}_FLAGS_RELEASE "${CMAKE_${lang}_FLAGS_RELEASE} /Zi") - ENDFOREACH() - FOREACH(flag - CMAKE_C_FLAGS CMAKE_CXX_FLAGS - CMAKE_C_FLAGS_INIT CMAKE_CXX_FLAGS_INIT - CMAKE_C_FLAGS_RELEASE CMAKE_C_FLAGS_RELWITHDEBINFO - CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_DEBUG_INIT - CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_RELWITHDEBINFO - CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG_INIT - CMAKE_C_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_MINSIZEREL - ) - STRING(REGEX REPLACE "/M[TD][d]?" "${MSVC_CRT_TYPE}" "${flag}" "${${flag}}" ) - STRING(REPLACE "/ZI " "/Zi " "${flag}" "${${flag}}") - IF((NOT "${${flag}}" MATCHES "/Zi") AND (NOT "${${flag}}" MATCHES "/Z7")) - STRING(APPEND ${flag} " /Zi") - ENDIF() - # Remove inlining flags, added by CMake, if any. - # Compiler default is fine. - STRING(REGEX REPLACE "/Ob[0-3]" "" "${flag}" "${${flag}}" ) - ENDFOREACH() - - # Allow to overwrite the inlining flag - SET(MSVC_INLINE "" CACHE STRING - "MSVC Inlining option, either empty, or one of /Ob0,/Ob1,/Ob2,/Ob3") - IF(MSVC_INLINE MATCHES "/Ob[0-3]") - ADD_COMPILE_OPTIONS(${MSVC_INLINE}) - ELSEIF(NOT(MSVC_INLINE STREQUAL "")) - MESSAGE(FATAL_ERROR "Invalid option for MSVC_INLINE") - ENDIF() - - IF(WITH_ASAN OR WITH_UBSAN) - # Workaround something Linux specific - SET(SECURITY_HARDENED 0 CACHE INTERNAL "" FORCE) - ENABLE_SANITIZERS() - ENDIF() - - IF(CLANG_CL) - SET(CLANG_CL_FLAGS -"-Wno-unknown-warning-option -Wno-unused-private-field \ --Wno-unused-parameter -Wno-inconsistent-missing-override \ --Wno-unused-command-line-argument -Wno-pointer-sign \ --Wno-deprecated-register -Wno-missing-braces \ --Wno-unused-function -Wno-unused-local-typedef -msse4.2 " - ) - IF(CMAKE_SIZEOF_VOID_P MATCHES 8) - STRING(APPEND CLANG_CL_FLAGS "-mpclmul ") + set(out) + foreach(name clang_rt.asan_dynamic-${ARCH_SUFFIX}.lib + clang_rt.asan_dynamic_runtime_thunk-${ARCH_SUFFIX}.lib) + set(path "${CLANG_VERSION_DIR}/lib/windows/${name}") + if(EXISTS "${path}") + list(APPEND out ${path}) + else() + message(FATAL_ERROR "expected library ${path} not found") ENDIF() - STRING(APPEND CMAKE_C_FLAGS " ${CLANG_CL_FLAGS} ${MSVC_CRT_TYPE}") - STRING(APPEND CMAKE_CXX_FLAGS " ${CLANG_CL_FLAGS} ${MSVC_CRT_TYPE}") - ENDIF() + endforeach() + set(${result_list} ${out} PARENT_SCOPE) +endfunction() - FOREACH(type EXE SHARED MODULE) - STRING(REGEX REPLACE "/STACK:([^ ]+)" "" CMAKE_${type}_LINKER_FLAGS "${CMAKE_${type}_LINKER_FLAGS}") - IF(WITH_ASAN) - SET(build_types RELWITHDEBINFO DEBUG) - ELSE() - SET(build_types RELWITHDEBINFO) - ENDIF() - FOREACH(btype ${build_types}) - STRING(REGEX REPLACE "/INCREMENTAL:([^ ]+)" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_${btype} "${CMAKE_${type}_LINKER_FLAGS_${btype}}") - STRING(REGEX REPLACE "/INCREMENTAL$" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_${btype} "${CMAKE_${type}_LINKER_FLAGS_${btype}}") - ENDFOREACH() - IF(NOT CLANG_CL) - STRING(APPEND CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO " /release /OPT:REF,ICF") - ENDIF() - IF(DYNAMIC_UCRT_LINK AND (MSVC_CRT_TYPE STREQUAL "/MT")) - FOREACH(config RELEASE RELWITHDEBINFO DEBUG MINSIZEREL) - STRING(APPEND CMAKE_${type}_LINKER_FLAGS_${config} ${DYNAMIC_UCRT_LINKER_OPTION}) - ENDFOREACH() - ENDIF() - ENDFOREACH() +macro(enable_sanitizers) + # Remove the runtime checks from the compiler flags + # ASAN does the same thing, in many cases better + foreach(lang C CXX) + foreach(suffix "_DEBUG" "_DEBUG_INIT") + string(REGEX REPLACE "/RTC[1su]" "" CMAKE_${lang}_FLAGS${suffix} "${CMAKE_${lang}_FLAGS${suffix}}") + endforeach() + endforeach() + + if(WITH_ASAN) + add_compile_options($<$:/fsanitize=address>) + endif() + if(WITH_UBSAN) + include(CheckCCompilerFlag) + check_c_compiler_flag(/fsanitize=undefined HAVE_fsanitize_undefined) + if (HAVE_fsanitize_undefined) + add_compile_options($<$:/fsanitize=undefined>) + else() + message(FATAL_ERROR "UBSAN not supported by this compiler yet") + endif() + endif() + if(CLANG_CL) + find_asan_runtime(asan_libs) + foreach(lib ${asan_libs}) + link_libraries(${lib}) + string(APPEND CMAKE_C_STANDARD_LIBRARIES " \"${lib}\"") + string(APPEND CMAKE_CXX_STANDARD_LIBRARIES " \"${lib}\"") + endforeach() + else() + add_link_options(/INCREMENTAL:NO) + endif() +endmacro() + + +if(MSVC) + # Disable mingw based pkg-config found in Strawberry perl + set(PKG_CONFIG_EXECUTABLE 0 CACHE INTERNAL "") + + if(NOT DEFINED CMAKE_MSVC_RUNTIME_LIBRARY) + set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreadedDLL) + endif() + + if(CMAKE_MSVC_RUNTIME_LIBRARY MATCHES "DLL") + # Dynamic runtime (DLLs), need to install CRT libraries. + set(CMAKE_INSTALL_SYSTEM_RUNTIME_COMPONENT VCCRT) + set(CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_NO_WARNINGS TRUE) + if(CMAKE_MSVC_RUNTIME_LIBRARY STREQUAL "MultiThreadedDebugDLL") + set(CMAKE_INSTALL_DEBUG_LIBRARIES_ONLY TRUE) + endif() + include(InstallRequiredSystemLibraries) + endif() + + # Compile with /Zi to get debugging information + if (NOT DEFINED CMAKE_MSVC_DEBUG_INFORMATION_FORMAT) + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT "ProgramDatabase") + add_link_options(/DEBUG) # Ensure debugging info at link time + endif() + + if(WITH_ASAN OR WITH_UBSAN) + # Workaround something Linux specific + set(SECURITY_HARDENED 0 CACHE INTERNAL "" FORCE) + enable_sanitizers() + endif() + + add_compile_definitions( + _CRT_SECURE_NO_DEPRECATE + _CRT_NONSTDC_NO_WARNINGS + _WIN32_WINNT=0x0A00 + # We do not want the windows.h , or winsvc.h macros min/max + NOMINMAX NOSERVICE + # Speed up build process excluding unused header files + WIN32_LEAN_AND_MEAN + ) + if(CLANG_CL) + add_compile_options( + -Wno-unknown-warning-option + -Wno-unused-private-field + -Wno-unused-parameter + -Wno-inconsistent-missing-override + -Wno-unused-command-line-argument + -Wno-pointer-sign + -Wno-deprecated-register + -Wno-missing-braces + -Wno-unused-function + -Wno-unused-local-typedef + -Wno-microsoft-static-assert + -Wno-c++17-extensions + -msse4.2 + ) + if((CMAKE_SIZEOF_VOID_P MATCHES 8) AND MSVC_INTEL) + add_compile_options(-mpclmul) + endif() + endif() - # Mark 32 bit executables large address aware so they can # use > 2GB address space - IF(CMAKE_SIZEOF_VOID_P MATCHES 4) - STRING(APPEND CMAKE_EXE_LINKER_FLAGS " /LARGEADDRESSAWARE") - ENDIF() - - # Speed up multiprocessor build - IF (NOT CLANG_CL) - STRING(APPEND CMAKE_C_FLAGS " /MP") - STRING(APPEND CMAKE_CXX_FLAGS " /MP") - STRING(APPEND CMAKE_C_FLAGS_RELWITHDEBINFO " /Gw") - STRING(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " /Gw") - ENDIF() - - #TODO: update the code and remove the disabled warnings - STRING(APPEND CMAKE_C_FLAGS " /we4700 /we4311 /we4477 /we4302 /we4090") - STRING(APPEND CMAKE_CXX_FLAGS " /we4099 /we4700 /we4311 /we4477 /we4302 /we4090") - IF(MSVC_VERSION GREATER 1910 AND NOT CLANG_CL) - STRING(APPEND CMAKE_CXX_FLAGS " /permissive-") - STRING(APPEND CMAKE_C_FLAGS " /diagnostics:caret") - STRING(APPEND CMAKE_CXX_FLAGS " /diagnostics:caret") - ENDIF() - ADD_DEFINITIONS(-D_CRT_NONSTDC_NO_WARNINGS) - IF(MYSQL_MAINTAINER_MODE MATCHES "ERR") - STRING(APPEND CMAKE_C_FLAGS " /WX") - STRING(APPEND CMAKE_CXX_FLAGS " /WX") - FOREACH(type EXE SHARED MODULE) - FOREACH(cfg RELEASE DEBUG RELWITHDEBINFO) - SET(CMAKE_${type}_LINKER_FLAGS_${cfg} "${CMAKE_${type}_LINKER_FLAGS_${cfg}} /WX") - ENDFOREACH() - ENDFOREACH() - ENDIF() + if(CMAKE_SIZEOF_VOID_P MATCHES 4) + add_link_options(/LARGEADDRESSAWARE) + endif() - IF(FAST_BUILD) - STRING (REGEX REPLACE "/RTC(su|[1su])" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - ELSEIF (NOT CLANG_CL) - STRING(APPEND CMAKE_CXX_FLAGS_RELEASE " /d2OptimizeHugeFunctions") - STRING(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " /d2OptimizeHugeFunctions") - ENDIF() - ADD_COMPILE_OPTIONS($<$:/utf-8>) -ENDIF() + # RelWithDebInfo is deoptimized wrt inlining. + # Fix it to default + foreach(lang C CXX) + foreach(suffix "_RELWITHDEBINFO" "_RELWITHDEBINFO_INIT") + string(REGEX REPLACE "/Ob[0-1]" "" CMAKE_${lang}_FLAGS${suffix} "${CMAKE_${lang}_FLAGS${suffix}}") + endforeach() + endforeach() -# Always link with socket/synchronization libraries -STRING(APPEND CMAKE_C_STANDARD_LIBRARIES " ws2_32.lib synchronization.lib") -STRING(APPEND CMAKE_CXX_STANDARD_LIBRARIES " ws2_32.lib synchronization.lib") + if(NOT CLANG_CL) + add_link_options("$<$:/INCREMENTAL:NO;/RELEASE;/OPT:REF,ICF>") + add_compile_options($<$:$<$:/Gw>>) + add_compile_options($<$:/MP>) + add_compile_options("$<$:/we4099;/we4700;/we4311;/we4477;/we4302;/we4090>") + add_compile_options($<$:/permissive->) + add_compile_options($<$:/diagnostics:caret>) + add_compile_options($<$:/utf-8>) + if(NOT FAST_BUILD) + add_compile_options($<$:$<$:/d2OptimizeHugeFunctions>>) + endif() + endif() -# System checks -SET(SIGNAL_WITH_VIO_CLOSE 1) # Something that runtime team needs -SET(HAVE_UNACCESSIBLE_AFTER_MEM_DECOMMIT 1) + if(MYSQL_MAINTAINER_MODE MATCHES "ERR") + set(CMAKE_COMPILE_WARNING_AS_ERROR ON) + add_link_options(/WX) + endif() +endif() -# IPv6 constants appeared in Vista SDK first. We need to define them in any case if they are -# not in headers, to handle dual mode sockets correctly. -CHECK_SYMBOL_EXISTS(IPPROTO_IPV6 "winsock2.h" HAVE_IPPROTO_IPV6) -IF(NOT HAVE_IPPROTO_IPV6) - SET(HAVE_IPPROTO_IPV6 41) -ENDIF() -CHECK_SYMBOL_EXISTS(IPV6_V6ONLY "winsock2.h;ws2ipdef.h" HAVE_IPV6_V6ONLY) -IF(NOT HAVE_IPV6_V6ONLY) - SET(IPV6_V6ONLY 27) -ENDIF() +# avoid running system checks by using pre-cached check results +# system checks are expensive on VS generator +get_filename_component(_SCRIPT_DIR ${CMAKE_CURRENT_LIST_FILE} PATH) +include(${_SCRIPT_DIR}/WindowsCache.cmake) -# Some standard functions exist there under different -# names (e.g popen is _popen or strok_r is _strtok_s) -# If a replacement function exists, HAVE_FUNCTION is -# defined to 1. CMake variable will also -# be defined to the replacement name. -# So for example, CHECK_FUNCTION_REPLACEMENT(popen _popen) -# will define HAVE_POPEN to 1 and set variable named popen -# to _popen. If the header template, one needs to have -# cmakedefine popen @popen@ which will expand to -# define popen _popen after CONFIGURE_FILE +# this is out of place, not really a system check +set(FN_NO_CASE_SENSE 1) +set(USE_SYMDIR 1) +set(HAVE_UNACCESSIBLE_AFTER_MEM_DECOMMIT 1) -MACRO(CHECK_FUNCTION_REPLACEMENT function replacement) - STRING(TOUPPER ${function} function_upper) - CHECK_FUNCTION_EXISTS(${function} HAVE_${function_upper}) - IF(NOT HAVE_${function_upper}) - CHECK_FUNCTION_EXISTS(${replacement} HAVE_${replacement}) - IF(HAVE_${replacement}) - SET(HAVE_${function_upper} 1 ) - SET(${function} ${replacement}) - ENDIF() - ENDIF() -ENDMACRO() -MACRO(CHECK_SYMBOL_REPLACEMENT symbol replacement header) - STRING(TOUPPER ${symbol} symbol_upper) - CHECK_SYMBOL_EXISTS(${symbol} ${header} HAVE_${symbol_upper}) - IF(NOT HAVE_${symbol_upper}) - CHECK_SYMBOL_EXISTS(${replacement} ${header} HAVE_${replacement}) - IF(HAVE_${replacement}) - SET(HAVE_${symbol_upper} 1) - SET(${symbol} ${replacement}) - ENDIF() - ENDIF() -ENDMACRO() - -CHECK_SYMBOL_REPLACEMENT(S_IROTH _S_IREAD sys/stat.h) -CHECK_SYMBOL_REPLACEMENT(S_IFIFO _S_IFIFO sys/stat.h) -CHECK_SYMBOL_REPLACEMENT(SIGQUIT SIGTERM signal.h) -CHECK_SYMBOL_REPLACEMENT(SIGPIPE SIGINT signal.h) -CHECK_FUNCTION_REPLACEMENT(popen _popen) -CHECK_FUNCTION_REPLACEMENT(pclose _pclose) -CHECK_FUNCTION_REPLACEMENT(access _access) -CHECK_FUNCTION_REPLACEMENT(strcasecmp _stricmp) -CHECK_FUNCTION_REPLACEMENT(strncasecmp _strnicmp) -CHECK_SYMBOL_REPLACEMENT(snprintf _snprintf stdio.h) -CHECK_FUNCTION_REPLACEMENT(strtok_r strtok_s) -CHECK_FUNCTION_REPLACEMENT(strtoll _strtoi64) -CHECK_FUNCTION_REPLACEMENT(strtoull _strtoui64) -CHECK_FUNCTION_REPLACEMENT(vsnprintf _vsnprintf) -CHECK_TYPE_SIZE(ssize_t SIZE_OF_SSIZE_T) -IF(NOT HAVE_SIZE_OF_SSIZE_T) - SET(ssize_t SSIZE_T) -ENDIF() - -SET(FN_NO_CASE_SENSE 1) -SET(USE_SYMDIR 1) - -# Force static C runtime for targets in current directory -# (useful to get rid of MFC dll's dependency, or in installer) -MACRO(FORCE_STATIC_CRT) - FOREACH(flag - CMAKE_C_FLAGS_RELEASE CMAKE_C_FLAGS_RELWITHDEBINFO - CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_DEBUG_INIT - CMAKE_CXX_FLAGS_RELEASE CMAKE_CXX_FLAGS_RELWITHDEBINFO - CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG_INIT - CMAKE_C_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_MINSIZEREL - ) - STRING(REGEX REPLACE "/MD[d]?" "/MT" "${flag}" "${${flag}}" ) - STRING(REPLACE "${DYNAMIC_UCRT_LINKER_OPTION}" "" "${flag}" "${${flag}}") - ENDFOREACH() -ENDMACRO() diff --git a/cmake/os/WindowsCache.cmake b/cmake/os/WindowsCache.cmake index 75d21f6ca41..64c8f6d57da 100644 --- a/cmake/os/WindowsCache.cmake +++ b/cmake/os/WindowsCache.cmake @@ -203,10 +203,10 @@ SET(HAVE_STRING_H 1 CACHE INTERNAL "") SET(HAVE_STRNDUP CACHE INTERNAL "") SET(HAVE_STRNLEN 1 CACHE INTERNAL "") SET(HAVE_STRPBRK 1 CACHE INTERNAL "") -SET(HAVE_STRTOK_R CACHE INTERNAL "") -SET(HAVE_STRTOLL CACHE INTERNAL "") +SET(HAVE_STRTOK_R 1 CACHE INTERNAL "") +SET(HAVE_STRTOLL 1 CACHE INTERNAL "") SET(HAVE_STRTOUL 1 CACHE INTERNAL "") -SET(HAVE_STRTOULL CACHE INTERNAL "") +SET(HAVE_STRTOULL 1 CACHE INTERNAL "") SET(HAVE_SYNCH_H CACHE INTERNAL "") SET(HAVE_SYSENT_H CACHE INTERNAL "") SET(HAVE_SYS_DIR_H CACHE INTERNAL "") @@ -294,6 +294,7 @@ SET(HAVE_EVENT_H CACHE INTERNAL "") SET(HAVE_LINUX_UNISTD_H CACHE INTERNAL "") SET(HAVE_SYS_UTSNAME_H CACHE INTERNAL "") SET(HAVE_PTHREAD_ATTR_GETGUARDSIZE CACHE INTERNAL "") +SET(HAVE_PTHREAD_GETATTR_NP CACHE INTERNAL "") SET(HAVE_SOCKPEERCRED CACHE INTERNAL "") SET(HAVE_ABI_CXA_DEMANGLE CACHE INTERNAL "") SET(HAVE_GCC_C11_ATOMICS CACHE INTERNAL "") @@ -348,4 +349,16 @@ SET(HAVE_SYS_STATVFS_H CACHE INTERNAL "") SET(HAVE_GETPAGESIZES CACHE INTERNAL "") SET(HAVE_LINUX_LIMITS_H CACHE INTERNAL "") SET(HAVE_FILE_UCONTEXT_H CACHE INTERNAL "") +SET(have_C__Werror CACHE INTERNAL "") +SET(HAVE_SIGNAL_H 1 CACHE INTERNAL "") +SET(HAVE_UINT CACHE INTERNAL "") +SET(HAVE_SOCKET_LEN_T CACHE INTERNAL "") +SET(HAVE_GETTHRID CACHE INTERNAL "") +SET(HAVE_THREAD_LOCAL 1 CACHE INTERNAL "") +SET(have_CXX__Wno_unused_but_set_variable CACHE INTERNAL "") +SET(HAVE_UNISTD_H CACHE INTERNAL "") +SET(HAVE_LINUX_UNISTD_H CACHE INTERNAL "") +SET(OFF64_T CACHE INTERNAL "") +SET(Z_HAVE_UNISTD_H CACHE INTERNAL "") +SET(HAVE_OFF64_T CACHE FALSE INTERNAL "") ENDIF(MSVC) diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index 378fc5f5f1c..838a234dd00 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -214,6 +214,11 @@ MACRO(MYSQL_ADD_PLUGIN) TARGET_LINK_LIBRARIES (${target} mysqlservices ${ARG_LINK_LIBRARIES}) + IF(WIN32) + # A popular library, turns out many plugins need it for gethostname() + TARGET_LINK_LIBRARIES (${target} ws2_32) + ENDIF() + IF(CMAKE_SYSTEM_NAME MATCHES AIX) TARGET_LINK_OPTIONS(${target} PRIVATE "-Wl,-bE:${CMAKE_SOURCE_DIR}/libservices/mysqlservices_aix.def") ENDIF() diff --git a/config.h.cmake b/config.h.cmake index 6d2fbee55d1..751bca8528e 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -402,38 +402,27 @@ #cmakedefine SIGNAL_WITH_VIO_CLOSE 1 /* Windows stuff, mostly functions, that have Posix analogs but named differently */ -#cmakedefine S_IROTH @S_IROTH@ -#cmakedefine S_IFIFO @S_IFIFO@ -#cmakedefine IPPROTO_IPV6 @IPPROTO_IPV6@ -#cmakedefine IPV6_V6ONLY @IPV6_V6ONLY@ -#cmakedefine sigset_t @sigset_t@ -#cmakedefine mode_t @mode_t@ -#cmakedefine SIGQUIT @SIGQUIT@ -#cmakedefine SIGPIPE @SIGPIPE@ -#cmakedefine popen @popen@ -#cmakedefine pclose @pclose@ -#cmakedefine ssize_t @ssize_t@ -#cmakedefine strcasecmp @strcasecmp@ -#cmakedefine strncasecmp @strncasecmp@ -#cmakedefine snprintf @snprintf@ -#cmakedefine strtok_r @strtok_r@ -#cmakedefine strtoll @strtoll@ -#cmakedefine strtoull @strtoull@ -#cmakedefine vsnprintf @vsnprintf@ -#if defined(_MSC_VER) && (_MSC_VER > 1800) +#ifdef _WIN32 +#define S_IROTH _S_IREAD +#define S_IFIFO _S_IFIFO +#define SIGQUIT SIGTERM +#define SIGPIPE SIGINT +#define sigset_t int +#define mode_t int +#define popen _popen +#define pclose _pclose +#define ssize_t SSIZE_T +#define strcasecmp _stricmp +#define strncasecmp _strnicmp +#define strtok_r strtok_s #define tzname _tzname #define P_tmpdir "C:\\TEMP" -#endif -#if defined(_MSC_VER) && (_MSC_VER > 1310) -# define HAVE_SETENV #define setenv(a,b,c) _putenv_s(a,b) -#endif -#define PSAPI_VERSION 1 /* for GetProcessMemoryInfo() */ -/* We don't want the min/max macros */ -#ifdef _WIN32 +#define HAVE_SETENV #define NOMINMAX 1 -#endif +#define PSAPI_VERSION 2 /* for GetProcessMemoryInfo() */ +#endif /* _WIN32 */ /* MySQL features diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index ceae31f07eb..5970be64ca5 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -171,7 +171,7 @@ IF(HAVE_BFD_H) ENDIF(HAVE_BFD_H) IF (WIN32) - TARGET_LINK_LIBRARIES(mysys iphlpapi dbghelp) + TARGET_LINK_LIBRARIES(mysys iphlpapi dbghelp ws2_32 synchronization) ENDIF(WIN32) # Need explicit pthread for gcc -fsanitize=address diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index 31f7d2eb9e5..001e201dec8 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -413,14 +413,16 @@ IF(NOT TARGET connect) RETURN() ENDIF() -IF(MSVC AND (CMAKE_CXX_FLAGS MATCHES "/MP")) +IF(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # domdoc.cpp uses compiler directive #import which is not compatible # with the /MP option, resulting in compiler error C2813. # Remove /MP for this file. + GET_TARGET_PROPERTY(CURRENT_COMPILE_OPTIONS connect COMPILE_OPTIONS) + LIST(REMOVE_ITEM CURRENT_COMPILE_OPTIONS "$<$:/MP>") + SET_TARGET_PROPERTIES(connect PROPERTIES COMPILE_OPTIONS "${CURRENT_COMPILE_OPTIONS}") SET(src_list ${CONNECT_SOURCES}) LIST(FIND src_list domdoc.cpp idx) IF(idx GREATER -1) - STRING(REPLACE "/MP" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") LIST(REMOVE_AT src_list ${idx}) SET_SOURCE_FILES_PROPERTIES(${src_list} PROPERTIES COMPILE_FLAGS "/MP") ENDIF() diff --git a/win/packaging/ca/CMakeLists.txt b/win/packaging/ca/CMakeLists.txt index 368a844f830..652b76de11f 100644 --- a/win/packaging/ca/CMakeLists.txt +++ b/win/packaging/ca/CMakeLists.txt @@ -18,7 +18,8 @@ SET(WIXCA_SOURCES CustomAction.cpp CustomAction.def) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CMAKE_CURRENT_SOURCE_DIR} ${WIX_INCLUDE_DIR}) # Custom action should not depend on C runtime, since we do not know if CRT is installed. -FORCE_STATIC_CRT() ADD_VERSION_INFO(wixca SHARED WIXCA_SOURCES) ADD_LIBRARY(wixca SHARED EXCLUDE_FROM_ALL ${WIXCA_SOURCES} ${CMAKE_SOURCE_DIR}/sql/winservice.c) -TARGET_LINK_LIBRARIES(wixca ${WIX_WCAUTIL_LIBRARY} ${WIX_DUTIL_LIBRARY} msi version) +# Static linking with CRT, because it is called when MSVC libraries are not yet there, +SET_TARGET_PROPERTIES(wixca PROPERTIES MSVC_RUNTIME_LIBRARY MultiThreaded) +TARGET_LINK_LIBRARIES(wixca ${WIX_WCAUTIL_LIBRARY} ${WIX_DUTIL_LIBRARY} msi version ws2_32) diff --git a/win/upgrade_wizard/CMakeLists.txt b/win/upgrade_wizard/CMakeLists.txt index fd3560e1ee6..d849550e505 100644 --- a/win/upgrade_wizard/CMakeLists.txt +++ b/win/upgrade_wizard/CMakeLists.txt @@ -4,7 +4,6 @@ ENDIF() # We need MFC # /permissive- flag does not play well with MFC, disable it. -STRING(REPLACE "/permissive-" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") REMOVE_DEFINITIONS(-DNOSERVICE) # fixes "already defined" warning in an AFX header FIND_PACKAGE(MFC) @@ -17,31 +16,22 @@ IF(NOT MFC_FOUND) RETURN() ENDIF() -IF(MSVC_CRT_TYPE MATCHES "/MD") - # FORCE static CRT and MFC for upgrade wizard, - # so we do not have to redistribute MFC. - FORCE_STATIC_CRT() - SET(UPGRADE_WIZARD_SOURCES ${CMAKE_SOURCE_DIR}/sql/winservice.c) -ELSE() - SET(UPGRADE_WIZARD_LINK_LIBRARIES winservice) -ENDIF() - # MFC should be statically linked SET(CMAKE_MFC_FLAG 1) # Enable exception handling (avoids warnings) -SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc -DNO_WARN_MBCS_MFC_DEPRECATION") - +ADD_DEFINITIONS(-DNO_WARN_MBCS_MFC_DEPRECATION) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql) MYSQL_ADD_EXECUTABLE(mariadb-upgrade-wizard - upgrade.cpp upgradeDlg.cpp upgrade.rc ${UPGRADE_WIZARD_SOURCES} + upgrade.cpp upgradeDlg.cpp + ${CMAKE_SOURCE_DIR}/sql/winservice.c + upgrade.rc COMPONENT Server) - -TARGET_LINK_LIBRARIES(mariadb-upgrade-wizard ${UPGRADE_WIZARD_LINK_LIBRARIES}) # upgrade_wizard is Windows executable, set WIN32_EXECUTABLE so it does not # create a console. SET_TARGET_PROPERTIES(mariadb-upgrade-wizard PROPERTIES WIN32_EXECUTABLE 1 LINK_FLAGS "/MANIFESTUAC:level='requireAdministrator'" + MSVC_RUNTIME_LIBRARY MultiThreaded ) From db4763a0d1d1187d349b3a24d9e13e41dd5b33f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 7 Apr 2025 10:25:34 +0300 Subject: [PATCH 011/125] Fix a slow test When we expect a lock wait timeout, let us override the default innodb_lock_wait_timeout=50 with the minimum timeout of 1 second. --- mysql-test/suite/innodb/r/innodb_bug52663.result | 4 +++- mysql-test/suite/innodb/t/innodb_bug52663.test | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_bug52663.result b/mysql-test/suite/innodb/r/innodb_bug52663.result index 348a7b21022..a87638d6555 100644 --- a/mysql-test/suite/innodb/r/innodb_bug52663.result +++ b/mysql-test/suite/innodb/r/innodb_bug52663.result @@ -1,10 +1,11 @@ +SET @save_innodb_timeout=@@innodb_lock_wait_timeout; +SET GLOBAL innodb_lock_wait_timeout=1; set session transaction isolation level read committed; create table innodb_bug52663 (what varchar(5), id integer, count integer, primary key (what, id)) engine=innodb; insert into innodb_bug52663 values ('total', 0, 0); begin; connect addconroot, localhost, root,,; -connection addconroot; set session transaction isolation level read committed; begin; connection default; @@ -31,3 +32,4 @@ select * from innodb_bug52663; what id count total 0 2 drop table innodb_bug52663; +SET GLOBAL innodb_lock_wait_timeout=@save_innodb_timeout; diff --git a/mysql-test/suite/innodb/t/innodb_bug52663.test b/mysql-test/suite/innodb/t/innodb_bug52663.test index dd246297d94..1d12fb63ef8 100644 --- a/mysql-test/suite/innodb/t/innodb_bug52663.test +++ b/mysql-test/suite/innodb/t/innodb_bug52663.test @@ -1,6 +1,7 @@ ---source include/long_test.inc --source include/have_innodb.inc +SET @save_innodb_timeout=@@innodb_lock_wait_timeout; +SET GLOBAL innodb_lock_wait_timeout=1; set session transaction isolation level read committed; create table innodb_bug52663 (what varchar(5), id integer, count integer, primary key @@ -9,7 +10,6 @@ insert into innodb_bug52663 values ('total', 0, 0); begin; connect (addconroot, localhost, root,,); -connection addconroot; set session transaction isolation level read committed; begin; @@ -33,3 +33,4 @@ select * from innodb_bug52663; connection default; select * from innodb_bug52663; drop table innodb_bug52663; +SET GLOBAL innodb_lock_wait_timeout=@save_innodb_timeout; From 669f719cc21286020c95eec11f0d09b74f96639e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 7 Apr 2025 11:01:17 +0300 Subject: [PATCH 012/125] MDEV-36489 10.11 crashes during bootstrap on macOS buf_block_t::initialise(): Remove a redundant call to page.lock.init() that was already executed in buf_pool_t::create() or buf_pool_t::resize(). This fixes a regression that was introduced in commit b6923420f326ac030e4f3ef89a2acddb45eccb30 (MDEV-29445). --- .../suite/innodb/t/innodb_buffer_pool_resize_temporary.test | 4 ++-- storage/innobase/buf/buf0buf.cc | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/innodb/t/innodb_buffer_pool_resize_temporary.test b/mysql-test/suite/innodb/t/innodb_buffer_pool_resize_temporary.test index 59afed24521..d997c722edd 100644 --- a/mysql-test/suite/innodb/t/innodb_buffer_pool_resize_temporary.test +++ b/mysql-test/suite/innodb/t/innodb_buffer_pool_resize_temporary.test @@ -25,8 +25,8 @@ SET DEBUG_SYNC='buf_pool_shrink_before_wakeup SIGNAL blocked WAIT_FOR go'; send SET GLOBAL innodb_buffer_pool_size=8388608; connection default; SET DEBUG_SYNC='now WAIT_FOR blocked'; -# adjust for 32-bit ---replace_result 504/504 505/505 +# adjust for 32-bit and SUX_LOCK_GENERIC +--replace_regex /(5..)\/\1/505\/505/ SHOW STATUS LIKE 'innodb_buffer_pool_resize_status'; SET DEBUG_SYNC='now SIGNAL go'; connection con1; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 08d037c60e9..f7a9110a7ba 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -3409,7 +3409,6 @@ void buf_block_t::initialise(const page_id_t page_id, ulint zip_size, { ut_ad(!page.in_file()); buf_block_init_low(this); - page.lock.init(); page.init(fix, page_id); page.set_os_used(); page_zip_set_size(&page.zip, zip_size); From 2ba49bd8048ffea9d2150ca1295d9d256807ae44 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 28 Mar 2025 18:11:09 +1100 Subject: [PATCH 013/125] mroonga: correct offsetof calculation Using a null pointer as a basic for calculation was undefined behaviour. --- storage/mroonga/vendor/groonga/lib/db.c | 4 ++-- storage/mroonga/vendor/groonga/lib/hash.c | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/storage/mroonga/vendor/groonga/lib/db.c b/storage/mroonga/vendor/groonga/lib/db.c index 65463bdad20..3c2f98e4cde 100644 --- a/storage/mroonga/vendor/groonga/lib/db.c +++ b/storage/mroonga/vendor/groonga/lib/db.c @@ -969,8 +969,8 @@ calc_rec_size(grn_table_flags flags, uint32_t max_n_subrecs, uint32_t range_size *subrec_size = range_size + sizeof(uint32_t) + sizeof(uint32_t); break; } - *value_size = (uintptr_t)GRN_RSET_SUBRECS_NTH((((grn_rset_recinfo *)0)->subrecs), - *subrec_size, max_n_subrecs); + *value_size = (uintptr_t) GRN_RSET_SUBRECS_NTH(offsetof(grn_rset_recinfo, subrecs), + *subrec_size, max_n_subrecs); } else { *value_size = range_size; } diff --git a/storage/mroonga/vendor/groonga/lib/hash.c b/storage/mroonga/vendor/groonga/lib/hash.c index 8cdae7bad5e..a2254d2ea2f 100644 --- a/storage/mroonga/vendor/groonga/lib/hash.c +++ b/storage/mroonga/vendor/groonga/lib/hash.c @@ -1727,15 +1727,15 @@ grn_io_hash_calculate_entry_size(uint32_t key_size, uint32_t value_size, { if (flags & GRN_OBJ_KEY_VAR_SIZE) { if (flags & GRN_OBJ_KEY_LARGE) { - return (uintptr_t)((grn_io_hash_entry_large *)0)->value + value_size; + return offsetof(grn_io_hash_entry_large, value) + value_size; } else { - return (uintptr_t)((grn_io_hash_entry_normal *)0)->value + value_size; + return offsetof(grn_io_hash_entry_normal, value) + value_size; } } else { if (key_size == sizeof(uint32_t)) { - return (uintptr_t)((grn_plain_hash_entry *)0)->value + value_size; + return offsetof(grn_plain_hash_entry, value) + value_size; } else { - return (uintptr_t)((grn_rich_hash_entry *)0)->key_and_value + return offsetof(grn_rich_hash_entry, key_and_value) + key_size + value_size; } } @@ -1865,12 +1865,12 @@ grn_tiny_hash_calculate_entry_size(uint32_t key_size, uint32_t value_size, { uint32_t entry_size; if (flags & GRN_OBJ_KEY_VAR_SIZE) { - entry_size = (uintptr_t)((grn_tiny_hash_entry *)0)->value + value_size; + entry_size = offsetof(grn_tiny_hash_entry, value) + value_size; } else { if (key_size == sizeof(uint32_t)) { - entry_size = (uintptr_t)((grn_plain_hash_entry *)0)->value + value_size; + entry_size = offsetof(grn_plain_hash_entry, value) + value_size; } else { - entry_size = (uintptr_t)((grn_rich_hash_entry *)0)->key_and_value + entry_size = offsetof(grn_rich_hash_entry, key_and_value) + key_size + value_size; } } From b316a7135b146968d78b2c1d81bca61fdb7cbadb Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 28 Mar 2025 18:12:24 +1100 Subject: [PATCH 014/125] mroonga: undefined behaviour fix The value of dv[0].data being null showed up in the mtr tests: mroonga/storage.alter_table_fulltext_add_no_primary_key as: /source/storage/mroonga/vendor/groonga/lib/ii.c:2052:37: runtime error: applying non-zero offset 28 to null pointer Correct this by entrying the if condition on null pointer value. The free is valid, and the data of size is allocated. --- storage/mroonga/vendor/groonga/lib/ii.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/mroonga/vendor/groonga/lib/ii.c b/storage/mroonga/vendor/groonga/lib/ii.c index 3e402e15e7e..8ce4857bfc2 100644 --- a/storage/mroonga/vendor/groonga/lib/ii.c +++ b/storage/mroonga/vendor/groonga/lib/ii.c @@ -2049,7 +2049,7 @@ grn_p_decv(grn_ctx *ctx, uint8_t *data, uint32_t data_size, datavec *dv, uint32_ if ((df & 1)) { df >>= 1; size = nreq == dvlen ? data_size : df * nreq; - if (dv[dvlen].data < dv[0].data + size) { + if (!dv[0].data || dv[dvlen].data < dv[0].data + size) { if (dv[0].data) { GRN_FREE(dv[0].data); } if (!(rp = GRN_MALLOC(size * sizeof(uint32_t)))) { return 0; } dv[dvlen].data = rp + size; @@ -10653,7 +10653,7 @@ grn_ii_builder_options_fix(grn_ii_builder_options *options) } #define GRN_II_BUILDER_TERM_INPLACE_SIZE\ - (sizeof(grn_ii_builder_term) - (uintptr_t)&((grn_ii_builder_term *)0)->dummy) + (sizeof(grn_ii_builder_term) - offsetof(grn_ii_builder_term, dummy)) typedef struct { grn_id rid; /* Last record ID */ From 13dd07374228ba106c59c70b325379ad0db507bb Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 1 Apr 2025 16:11:09 +1100 Subject: [PATCH 015/125] MDEV-31846: enable cursor protocol for test federatedx_create_handlers Per comment it code - it can be removed 10.5+. Not in the comment, is the SELECT INTO ... parts still required cursor protocol exclusion. --- mysql-test/suite/federated/federatedx_create_handlers.test | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/federated/federatedx_create_handlers.test b/mysql-test/suite/federated/federatedx_create_handlers.test index 9cf08a34931..d42e389cd49 100644 --- a/mysql-test/suite/federated/federatedx_create_handlers.test +++ b/mysql-test/suite/federated/federatedx_create_handlers.test @@ -7,9 +7,6 @@ connection default; set global federated_pushdown=1; -#Enable after fix MDEV-31846 or in v. 10.5 and later ---disable_cursor_protocol - connection slave; DROP TABLE IF EXISTS federated.t1; @@ -165,11 +162,13 @@ insert into federated.t4 select * from federated.t1; --sorted_result select * from federated.t4; +--disable_cursor_protocol select name into @var from federated.t1 where id=3 limit 1 ; select @var; --disable_ps2_protocol select name into outfile 'tmp.txt' from federated.t1; --enable_ps2_protocol +--enable_cursor_protocol let $path=`select concat(@@datadir, 'test/tmp.txt')`; remove_file $path; @@ -436,7 +435,5 @@ DEALLOCATE PREPARE stmt; set global federated_pushdown=0; ---enable_cursor_protocol - source include/federated_cleanup.inc; From 3995de0318be240effb19384e3a8ddc077917dff Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 1 Apr 2025 16:22:05 +1100 Subject: [PATCH 016/125] MDEV-36341 UBSAN: FederatedX fill_server runtime error: applying non-zero offset to null pointer UBSAN is nuanced around null pointer additions, but there is no offset on schema, its the start of the server->key. --- storage/federatedx/ha_federatedx.cc | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/storage/federatedx/ha_federatedx.cc b/storage/federatedx/ha_federatedx.cc index 95a63366ac0..ee31ffc0a15 100644 --- a/storage/federatedx/ha_federatedx.cc +++ b/storage/federatedx/ha_federatedx.cc @@ -1489,20 +1489,20 @@ static void fill_server(MEM_ROOT *mem_root, FEDERATEDX_SERVER *server, sizeof(int) + 8); key.append(scheme); key.q_append('\0'); - server->hostname= (const char *) (intptr) key.length(); + size_t hostname_pos= key.length(); key.append(hostname); key.q_append('\0'); - server->database= (const char *) (intptr) key.length(); + size_t database_pos= key.length(); key.append(database); key.q_append('\0'); key.q_append((uint32) share->port); - server->socket= (const char *) (intptr) key.length(); + size_t socket_pos= key.length(); key.append(socket); key.q_append('\0'); - server->username= (const char *) (intptr) key.length(); + size_t username_pos= key.length(); key.append(username); key.q_append('\0'); - server->password= (const char *) (intptr) key.length(); + size_t password_pos= key.length(); key.append(password); key.c_ptr_safe(); // Ensure we have end \0 @@ -1510,13 +1510,12 @@ static void fill_server(MEM_ROOT *mem_root, FEDERATEDX_SERVER *server, /* Copy and add end \0 */ server->key= (uchar *) strmake_root(mem_root, key.ptr(), key.length()); - /* pointer magic */ - server->scheme+= (intptr) server->key; - server->hostname+= (intptr) server->key; - server->database+= (intptr) server->key; - server->username+= (intptr) server->key; - server->password+= (intptr) server->key; - server->socket+= (intptr) server->key; + server->scheme= (const char *)server->key; + server->hostname= (const char *)server->key + hostname_pos; + server->database= (const char *)server->key + database_pos; + server->username= (const char *)server->key + username_pos; + server->password= (const char *)server->key + password_pos; + server->socket= (const char*)server->key + socket_pos; server->port= share->port; if (!share->socket) From 93ea4f29a4cbacf0e849c320b47c550a874c471a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 1 Apr 2025 16:40:30 +1100 Subject: [PATCH 017/125] MDEV-36347 UBSAN: plugins.auth_v0100 - runtime error: call to function do_auth_0x0100 through pointer to incorrect function type. Redoing a new plugin interface for an obsolete protocol was too much so we just remove the UBSAN testing on the function. It wasn't possible to just disable funtion-type-mismatch --- plugin/auth_examples/auth_0x0100.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugin/auth_examples/auth_0x0100.c b/plugin/auth_examples/auth_0x0100.c index e23f495f43b..e0639dddf90 100644 --- a/plugin/auth_examples/auth_0x0100.c +++ b/plugin/auth_examples/auth_0x0100.c @@ -56,6 +56,10 @@ struct st_mysql_auth }; #endif +/* function-type-mismatch ignore */ +#if defined(__clang__) +__attribute__((no_sanitize("undefined"))) +#endif static int do_auth_0x0100(MYSQL_PLUGIN_VIO *vio, MYSQL_SERVER_AUTH_INFO *info) { info->password_used= 1; From 690b2cf776faa2e1a6832077cec2246c86cd00bf Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 20 Mar 2025 09:54:12 +1100 Subject: [PATCH 018/125] test: archive-big test too big for msan archive.archive-big w1 [ fail ] timeout after 900 seconds Test ended at 2025-03-19 22:27:30 Test case timeout after 900 seconds == /build/mysql-test/var/1/log/archive-big.log == CREATE TABLE t1(a BLOB) ENGINE=ARCHIVE; INSERT INTO t1 SELECT * FROM t1; ... == /build/mysql-test/var/1/tmp/analyze-timeout-mysqld.1.err == mysqltest: Could not open connection 'default' after 500 attempts: 2002 Can't connect to local MySQL server through socket '/build/mysql-test/var/tmp/1/mysqld.1.sock' (111) --- mysql-test/suite/archive/archive-big.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/archive/archive-big.test b/mysql-test/suite/archive/archive-big.test index fd59df66b43..8cd19b1d809 100644 --- a/mysql-test/suite/archive/archive-big.test +++ b/mysql-test/suite/archive/archive-big.test @@ -1,6 +1,7 @@ --source include/big_test.inc -# Valgrind is to slow for this test +# Valgrind and msan is to slow for this test --source include/not_valgrind.inc +--source include/not_msan.inc --source include/have_archive.inc CREATE TABLE t1(a BLOB) ENGINE=ARCHIVE; --disable_query_log From b02ad4a6f8ea09c5cdf0a44a9ee57a60f2989f48 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Sat, 5 Apr 2025 21:06:41 +0400 Subject: [PATCH 019/125] MDEV-36427 - FTBFS with libxml2 2.14.0 Connect engine fails to build with libxml2 2.14.0. Connect engine uses "#ifndef BASE_BUFFER_SIZE" to determine if libxml2 is available. If libxml2 is unavailable it did redefine xmlElementType enum of libxml/tree.h. The reasons for this redefinition is vague, most probably some of these constants were used when connect was compiled with MSXML, while libxml2 was disabled. However BASE_BUFFER_SIZE constant was removed from libxml2 recently, as a result connect fails to build due to xmlElementType constants redefinition. Use LIBXML2_SUPPORT instead of BASE_BUFFER_SIZE for libxml2 availability check. --- storage/connect/plgxml.h | 4 ++-- storage/connect/tabxml.cpp | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/storage/connect/plgxml.h b/storage/connect/plgxml.h index 82629e4c7db..6b49e6ac527 100644 --- a/storage/connect/plgxml.h +++ b/storage/connect/plgxml.h @@ -5,7 +5,7 @@ /******************************************************************/ /* Dual XML implementation base classes defines. */ /******************************************************************/ -#if !defined(BASE_BUFFER_SIZE) +#ifndef LIBXML2_SUPPORT enum ElementType { // libxml2 XML_ELEMENT_NODE = 1, XML_ATTRIBUTE_NODE = 2, @@ -28,7 +28,7 @@ enum ElementType { // libxml2 XML_XINCLUDE_START = 19, XML_XINCLUDE_END = 20, XML_DOCB_DOCUMENT_NODE = 21}; -#endif // !BASE_BUFFER_SIZE +#endif //#if !defined(NODE_TYPE_LIST) #ifdef NOT_USED diff --git a/storage/connect/tabxml.cpp b/storage/connect/tabxml.cpp index fae2375e5a1..461ac6849f8 100644 --- a/storage/connect/tabxml.cpp +++ b/storage/connect/tabxml.cpp @@ -26,6 +26,9 @@ #include #include //#include +#ifdef LIBXML2_SUPPORT +#include +#endif #include "osutil.h" #define _O_RDONLY O_RDONLY #endif // !_WIN32 From 59962ae2b3061dbd3694e005c779d9b2dfbf625c Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 1 Apr 2025 17:38:25 +1100 Subject: [PATCH 020/125] MDEV-36442 MDEV-35452 Fix --view-protocol for spider tests with SELECT arguments of SELECT statements Running mtr --view-protocol transforms SELECT statements to a CREATE OR REPLACE VIEW of the statement, followed by SELECT from the view. When thus when spider tests check the query log for select statements, it often output a different one with --view-protocol compared to without. By adding disable/enable_view_protocol pairs to these statements. Most of these statements are surrounded by existing disable/enable_ps[2]_protocol pairs. Acked-by: Yuchen Pei --- .../t/checksum_table_with_quick_mode_3.test | 2 ++ .../mysql-test/spider/bugfix/t/cp932_column.test | 2 ++ .../spider/bugfix/t/delete_with_float_column.inc | 2 ++ .../spider/bugfix/t/group_by_order_by_limit.test | 4 ++++ .../mysql-test/spider/bugfix/t/insert_select.test | 2 ++ .../mysql-test/spider/bugfix/t/mdev_19866.test | 4 ++++ .../mysql-test/spider/bugfix/t/mdev_20100.test | 2 ++ .../mysql-test/spider/bugfix/t/mdev_20502.test | 2 ++ .../mysql-test/spider/bugfix/t/mdev_21884.test | 2 ++ .../mysql-test/spider/bugfix/t/mdev_27172.test | 2 ++ .../mysql-test/spider/bugfix/t/mdev_29008.test | 2 ++ .../mysql-test/spider/bugfix/t/quick_mode_0.test | 12 ++++++++++++ .../mysql-test/spider/bugfix/t/quick_mode_1.test | 12 ++++++++++++ .../mysql-test/spider/bugfix/t/quick_mode_2.test | 12 ++++++++++++ .../mysql-test/spider/bugfix/t/quick_mode_3.test | 12 ++++++++++++ .../spider/bugfix/t/return_found_rows_insert.test | 6 ++++++ .../spider/bugfix/t/return_found_rows_update.test | 2 ++ .../spider/bugfix/t/select_by_null.test | 2 ++ .../spider/bugfix/t/select_with_backquote.test | 2 ++ .../spider/bugfix/t/slave_trx_isolation.test | 2 ++ .../mysql-test/spider/bugfix/t/sql_mode.inc | 2 ++ .../spider/bugfix/t/strict_group_by.test | 4 ++++ .../spider/bugfix/t/wrapper_mariadb.test | 2 ++ .../spider/mysql-test/spider/bugfix/t/xa_cmd.test | 2 ++ .../spider/feature/t/checksum_table_parallel.inc | 2 ++ .../e1121/t/direct_join_by_pkey_key.test | 2 ++ .../e1121/t/direct_join_by_pkey_pkey.test | 2 ++ .../spider/regression/e1121/t/load_data.inc | 2 ++ .../e112122/t/group_by_order_by_limit_ok.test | 4 ++++ .../regression/e112122/t/load_data_part.inc | 4 ++++ .../mysql-test/spider/t/auto_increment.test | 2 ++ .../t/checksum_table_with_quick_mode_3.test | 2 ++ .../spider/mysql-test/spider/t/direct_join.test | 4 ++++ .../mysql-test/spider/t/direct_join_using.test | 2 ++ .../mysql-test/spider/t/direct_left_join.test | 2 ++ .../spider/t/direct_left_join_nullable.test | 2 ++ .../spider/t/direct_left_right_join_nullable.test | 2 ++ .../t/direct_left_right_left_join_nullable.test | 2 ++ .../mysql-test/spider/t/direct_right_join.test | 2 ++ .../spider/t/direct_right_join_nullable.test | 2 ++ .../spider/t/direct_right_left_join_nullable.test | 2 ++ .../t/direct_right_left_right_join_nullable.test | 2 ++ .../mysql-test/spider/t/partition_cond_push.test | 6 ++++++ .../mysql-test/spider/t/partition_fulltext.test | 6 ++++++ ...tition_join_pushdown_for_single_partition.test | 6 ++++++ .../mysql-test/spider/t/pushdown_not_like.test | 2 ++ .../spider/mysql-test/spider/t/quick_mode_0.test | 10 ++++++++++ .../spider/mysql-test/spider/t/quick_mode_1.test | 12 ++++++++++++ .../spider/mysql-test/spider/t/quick_mode_2.test | 12 ++++++++++++ .../spider/mysql-test/spider/t/quick_mode_3.test | 12 ++++++++++++ .../mysql-test/spider/t/slave_trx_isolation.test | 2 ++ storage/spider/mysql-test/spider/t/timestamp.test | 15 +++++++++++++++ .../spider/mysql-test/spider/t/udf_pushdown.inc | 2 ++ 53 files changed, 225 insertions(+) diff --git a/storage/spider/mysql-test/spider/bugfix/t/checksum_table_with_quick_mode_3.test b/storage/spider/mysql-test/spider/bugfix/t/checksum_table_with_quick_mode_3.test index de2dd90a500..e650ef3bd3a 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/checksum_table_with_quick_mode_3.test +++ b/storage/spider/mysql-test/spider/bugfix/t/checksum_table_with_quick_mode_3.test @@ -52,9 +52,11 @@ TRUNCATE TABLE mysql.general_log; CHECKSUM TABLE tbl_a EXTENDED; --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --echo diff --git a/storage/spider/mysql-test/spider/bugfix/t/cp932_column.test b/storage/spider/mysql-test/spider/bugfix/t/cp932_column.test index 119179702ec..a0f70bd9e82 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/cp932_column.test +++ b/storage/spider/mysql-test/spider/bugfix/t/cp932_column.test @@ -59,9 +59,11 @@ SET NAMES utf8; --connection child2_1 SET NAMES cp932; +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; SET NAMES utf8; diff --git a/storage/spider/mysql-test/spider/bugfix/t/delete_with_float_column.inc b/storage/spider/mysql-test/spider/bugfix/t/delete_with_float_column.inc index a4def9b3dc8..6285cf5eae4 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/delete_with_float_column.inc +++ b/storage/spider/mysql-test/spider/bugfix/t/delete_with_float_column.inc @@ -78,9 +78,11 @@ sync_with_master; SET SESSION sql_log_bin= 0; --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection slave1_1 diff --git a/storage/spider/mysql-test/spider/bugfix/t/group_by_order_by_limit.test b/storage/spider/mysql-test/spider/bugfix/t/group_by_order_by_limit.test index d694230bf2a..553fb6adeaf 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/group_by_order_by_limit.test +++ b/storage/spider/mysql-test/spider/bugfix/t/group_by_order_by_limit.test @@ -76,11 +76,15 @@ SHOW STATUS LIKE 'Spider_direct_aggregate'; set spider_direct_aggregate=@old_spider_direct_aggregate; --connection child2_1 +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/insert_select.test b/storage/spider/mysql-test/spider/bugfix/t/insert_select.test index c2ac615e2ac..84ebd230849 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/insert_select.test +++ b/storage/spider/mysql-test/spider/bugfix/t/insert_select.test @@ -79,9 +79,11 @@ INSERT IGNORE INTO tbl_b (SELECT skey, CAST(CONCAT(dt, ' ', tm) AS datetime) FRO INSERT IGNORE INTO tbl_b (SELECT skey, CAST(CONCAT(dt, ' ', tm) AS datetime) FROM tbl_a WHERE skey = 5 AND dt > DATE_ADD('2012-12-01', INTERVAL -10 DAY)); --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --echo diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_19866.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_19866.test index 0b8ce69cd73..a953a250449 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_19866.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_19866.test @@ -71,11 +71,15 @@ SELECT * FROM tbl_a WHERE pkey = 2; SELECT * FROM tbl_a; --connection child2_1 +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_20100.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_20100.test index 3d1ccf3ce6a..a440b370077 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_20100.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_20100.test @@ -70,7 +70,9 @@ SELECT a, b, c FROM tbl_a PARTITION (pt2,pt3); --connection child2_1 --disable_ps2_protocol +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol --enable_ps2_protocol --disable_ps_protocol eval $CHILD2_1_SELECT_TABLES; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_20502.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_20502.test index 1cd1c689372..3174ace8bfb 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_20502.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_20502.test @@ -59,7 +59,9 @@ SELECT MAX(id) AS m, 0 AS const, val, (SELECT tbl_a.val+1 FROM tbl_a LIMIT 1) AS FROM tbl_a GROUP BY val; --connection child2_1 +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_21884.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_21884.test index c1a9aaa4e9d..01a2a1a1171 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_21884.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_21884.test @@ -81,9 +81,11 @@ SELECT STRAIGHT_JOIN b.a, b.b FROM tb_l a, tbl_a b WHERE a.a = b.a; --connection child2_1 SET NAMES utf8; +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol --disable_ps_protocol eval $CHILD2_1_SELECT_TABLES; --enable_ps_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_27172.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_27172.test index 0b2e065702d..8374713651a 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_27172.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_27172.test @@ -76,7 +76,9 @@ SELECT * FROM tbl_c WHERE greeting = "Aloha!" AND CASE greeting WHEN "Aloha!" THEN "one" ELSE 'more' END = "one"; # hack to disable GBH --connection child2_1 +--disable_view_protocol SELECT argument FROM mysql.general_log WHERE argument LIKE 'select %'; +--enable_view_protocol --enable_ps2_protocol --connection child2_1 diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29008.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29008.test index f36855d8e71..40455916596 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_29008.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29008.test @@ -38,9 +38,11 @@ SELECT MIN(t2.a) AS f1, t1.b AS f2 FROM tbl_a AS t1 JOIN tbl_a AS t2 GROUP BY f2 --connection master_1 DROP DATABASE IF EXISTS auto_test_local; --connection child2_1 +--disable_view_protocol --disable_ps2_protocol SELECT argument FROM mysql.general_log WHERE argument LIKE 'select %'; --enable_ps2_protocol +--enable_view_protocol set global log_output=@old_log_output; set global general_log=@old_general_log; DROP DATABASE IF EXISTS auto_test_remote; diff --git a/storage/spider/mysql-test/spider/bugfix/t/quick_mode_0.test b/storage/spider/mysql-test/spider/bugfix/t/quick_mode_0.test index 3fd9e623887..23bfcb065db 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/quick_mode_0.test +++ b/storage/spider/mysql-test/spider/bugfix/t/quick_mode_0.test @@ -78,13 +78,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --connection master_1 @@ -103,13 +107,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --connection master_1 @@ -129,12 +137,16 @@ SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ +--disable_view_protocol eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/quick_mode_1.test b/storage/spider/mysql-test/spider/bugfix/t/quick_mode_1.test index 940fa61c598..dd56ca135f0 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/quick_mode_1.test +++ b/storage/spider/mysql-test/spider/bugfix/t/quick_mode_1.test @@ -78,13 +78,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --connection master_1 @@ -103,13 +107,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --connection master_1 @@ -128,13 +136,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/quick_mode_2.test b/storage/spider/mysql-test/spider/bugfix/t/quick_mode_2.test index e9270a2ee93..cedc4b60449 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/quick_mode_2.test +++ b/storage/spider/mysql-test/spider/bugfix/t/quick_mode_2.test @@ -78,13 +78,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --connection master_1 @@ -103,13 +107,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --connection master_1 @@ -128,13 +136,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/quick_mode_3.test b/storage/spider/mysql-test/spider/bugfix/t/quick_mode_3.test index ad042aadfed..db3a6118743 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/quick_mode_3.test +++ b/storage/spider/mysql-test/spider/bugfix/t/quick_mode_3.test @@ -79,13 +79,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --connection master_1 @@ -104,13 +108,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --connection master_1 @@ -129,13 +137,17 @@ TRUNCATE TABLE mysql.general_log; SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; --connection child2_1 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/return_found_rows_insert.test b/storage/spider/mysql-test/spider/bugfix/t/return_found_rows_insert.test index d4c0db1e7cc..2921d74b1cc 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/return_found_rows_insert.test +++ b/storage/spider/mysql-test/spider/bugfix/t/return_found_rows_insert.test @@ -56,9 +56,11 @@ exec $MYSQL -v -v -u root -h localhost -P $MASTER_1_MYPORT -S $MASTER_1_MYSOCK - --enable_query_log --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; TRUNCATE TABLE mysql.general_log; @@ -69,9 +71,11 @@ exec $MYSQL -v -v -u root -h localhost -P $MASTER_1_MYPORT -S $MASTER_1_MYSOCK - --enable_query_log --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; TRUNCATE TABLE mysql.general_log; @@ -82,9 +86,11 @@ exec $MYSQL -v -v -u root -h localhost -P $MASTER_1_MYPORT -S $MASTER_1_MYSOCK - --enable_query_log --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --echo diff --git a/storage/spider/mysql-test/spider/bugfix/t/return_found_rows_update.test b/storage/spider/mysql-test/spider/bugfix/t/return_found_rows_update.test index a5f63978a22..64aa4d7593e 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/return_found_rows_update.test +++ b/storage/spider/mysql-test/spider/bugfix/t/return_found_rows_update.test @@ -58,9 +58,11 @@ exec $MYSQL -v -v -u root -h localhost --default-character-set=latin1 -P $MASTER --enable_query_log --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --echo diff --git a/storage/spider/mysql-test/spider/bugfix/t/select_by_null.test b/storage/spider/mysql-test/spider/bugfix/t/select_by_null.test index feb7df57f02..dfe8b9ec2ef 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/select_by_null.test +++ b/storage/spider/mysql-test/spider/bugfix/t/select_by_null.test @@ -52,9 +52,11 @@ TRUNCATE TABLE mysql.general_log; SELECT pkey FROM tbl_a WHERE NULL; --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --echo diff --git a/storage/spider/mysql-test/spider/bugfix/t/select_with_backquote.test b/storage/spider/mysql-test/spider/bugfix/t/select_with_backquote.test index b511b05bf01..53daaf06f68 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/select_with_backquote.test +++ b/storage/spider/mysql-test/spider/bugfix/t/select_with_backquote.test @@ -55,7 +55,9 @@ SELECT `pkey`, LEFT(`txt_utf8`, 4) FROM `auto_test_local`.`tbl_a` ORDER BY LEFT( --connection child2_1 SET NAMES utf8; +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/slave_trx_isolation.test b/storage/spider/mysql-test/spider/bugfix/t/slave_trx_isolation.test index 3d4280e35e4..66e044a5a5c 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/slave_trx_isolation.test +++ b/storage/spider/mysql-test/spider/bugfix/t/slave_trx_isolation.test @@ -71,10 +71,12 @@ SET SESSION sql_log_bin= 0; --connection child2_1 +--disable_view_protocol --disable_ps2_protocol --replace_regex /-[0-9a-f]{12}-[0-9a-f]+-/-xxxxxxxxxxxx-xxxxx-/ eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection slave1_1 diff --git a/storage/spider/mysql-test/spider/bugfix/t/sql_mode.inc b/storage/spider/mysql-test/spider/bugfix/t/sql_mode.inc index c4b17f6a837..1b3a46590b0 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/sql_mode.inc +++ b/storage/spider/mysql-test/spider/bugfix/t/sql_mode.inc @@ -49,8 +49,10 @@ TRUNCATE TABLE mysql.general_log; SELECT * FROM tbl_a ORDER BY pkey; --connection child2_1 +--disable_view_protocol --replace_regex /-[0-9a-f]{12}-[0-9a-f]+-/-xxxxxxxxxxxx-xxxxx-/ eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/strict_group_by.test b/storage/spider/mysql-test/spider/bugfix/t/strict_group_by.test index eaf149fa5da..bc9ee2a6833 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/strict_group_by.test +++ b/storage/spider/mysql-test/spider/bugfix/t/strict_group_by.test @@ -78,15 +78,19 @@ SHOW STATUS LIKE 'Spider_direct_aggregate'; set spider_direct_aggregate=@old_spider_direct_aggregate; --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_2_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --echo diff --git a/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test b/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test index d32af31674a..ae7e6643e3b 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test +++ b/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test @@ -50,9 +50,11 @@ TRUNCATE TABLE mysql.general_log; SELECT * FROM tbl_a ORDER BY pkey; --connection child2_1 +--disable_view_protocol # in --ps a query is logged differently in a general log replace_regex /order by t0.`pkey`/order by `pkey`/; eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/xa_cmd.test b/storage/spider/mysql-test/spider/bugfix/t/xa_cmd.test index 1e831453e41..690fe3e5b50 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/xa_cmd.test +++ b/storage/spider/mysql-test/spider/bugfix/t/xa_cmd.test @@ -49,9 +49,11 @@ XA PREPARE 'test'; XA COMMIT 'test'; --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --echo diff --git a/storage/spider/mysql-test/spider/feature/t/checksum_table_parallel.inc b/storage/spider/mysql-test/spider/feature/t/checksum_table_parallel.inc index 83c3ffd6c54..79792b2cacf 100644 --- a/storage/spider/mysql-test/spider/feature/t/checksum_table_parallel.inc +++ b/storage/spider/mysql-test/spider/feature/t/checksum_table_parallel.inc @@ -74,10 +74,12 @@ send_eval $MASTER_1_CHECKSUM_TABLE; --connection child2_1_2 SELECT SLEEP(1); +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --connection child2_2_2 eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol --connection child2_1 UNLOCK TABLES; diff --git a/storage/spider/mysql-test/spider/regression/e1121/t/direct_join_by_pkey_key.test b/storage/spider/mysql-test/spider/regression/e1121/t/direct_join_by_pkey_key.test index ec4639ed666..fd61b986cb1 100644 --- a/storage/spider/mysql-test/spider/regression/e1121/t/direct_join_by_pkey_key.test +++ b/storage/spider/mysql-test/spider/regression/e1121/t/direct_join_by_pkey_key.test @@ -67,7 +67,9 @@ SELECT a.val, a.akey FROM tbl_a a, tbl_b b WHERE a.akey = b.akey AND b.bkey = 5; --connection child2_1 --disable_ps_protocol +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --enable_ps_protocol diff --git a/storage/spider/mysql-test/spider/regression/e1121/t/direct_join_by_pkey_pkey.test b/storage/spider/mysql-test/spider/regression/e1121/t/direct_join_by_pkey_pkey.test index 52d9b52ddb5..cc272be51b5 100644 --- a/storage/spider/mysql-test/spider/regression/e1121/t/direct_join_by_pkey_pkey.test +++ b/storage/spider/mysql-test/spider/regression/e1121/t/direct_join_by_pkey_pkey.test @@ -67,7 +67,9 @@ SELECT a.val, a.akey FROM tbl_a a, tbl_b b WHERE a.akey = b.akey AND b.bkey = 5; --connection child2_1 --disable_ps_protocol +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --enable_ps_protocol diff --git a/storage/spider/mysql-test/spider/regression/e1121/t/load_data.inc b/storage/spider/mysql-test/spider/regression/e1121/t/load_data.inc index 325a7523974..55fff558338 100644 --- a/storage/spider/mysql-test/spider/regression/e1121/t/load_data.inc +++ b/storage/spider/mysql-test/spider/regression/e1121/t/load_data.inc @@ -67,9 +67,11 @@ eval LOAD DATA $OPTION_LOCAL INFILE '$MYSQLTEST_VARDIR/tmp/spider_outfile.tsv' $ --remove_file $MYSQLTEST_VARDIR/tmp/spider_outfile.tsv --connection child2_1 +--disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --echo diff --git a/storage/spider/mysql-test/spider/regression/e112122/t/group_by_order_by_limit_ok.test b/storage/spider/mysql-test/spider/regression/e112122/t/group_by_order_by_limit_ok.test index e4f3f2155bb..dae5a812888 100644 --- a/storage/spider/mysql-test/spider/regression/e112122/t/group_by_order_by_limit_ok.test +++ b/storage/spider/mysql-test/spider/regression/e112122/t/group_by_order_by_limit_ok.test @@ -76,11 +76,15 @@ SHOW STATUS LIKE 'Spider_direct_aggregate'; set spider_direct_aggregate=@old_spider_direct_aggregate; --connection child2_1 +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/regression/e112122/t/load_data_part.inc b/storage/spider/mysql-test/spider/regression/e112122/t/load_data_part.inc index f52d415a032..7b30cc0865d 100644 --- a/storage/spider/mysql-test/spider/regression/e112122/t/load_data_part.inc +++ b/storage/spider/mysql-test/spider/regression/e112122/t/load_data_part.inc @@ -84,11 +84,15 @@ eval LOAD DATA $OPTION_LOCAL INFILE '$MYSQLTEST_VARDIR/tmp/spider_outfile.tsv' $ --disable_ps2_protocol --connection child2_1 +--disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_1_SELECT_TABLES; --connection child2_2 +--disable_view_protocol eval $CHILD2_2_SELECT_ARGUMENT1; +--enable_view_protocol eval $CHILD2_2_SELECT_TABLES; --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/t/auto_increment.test b/storage/spider/mysql-test/spider/t/auto_increment.test index c5f272f5c44..5e3fff3dc55 100644 --- a/storage/spider/mysql-test/spider/t/auto_increment.test +++ b/storage/spider/mysql-test/spider/t/auto_increment.test @@ -156,7 +156,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/checksum_table_with_quick_mode_3.test b/storage/spider/mysql-test/spider/t/checksum_table_with_quick_mode_3.test index e0fa7a84674..44d2878cd08 100644 --- a/storage/spider/mysql-test/spider/t/checksum_table_with_quick_mode_3.test +++ b/storage/spider/mysql-test/spider/t/checksum_table_with_quick_mode_3.test @@ -96,9 +96,11 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --disable_ps2_protocol eval $CHILD2_1_SELECT_ARGUMENT1; --enable_ps2_protocol + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_join.test b/storage/spider/mysql-test/spider/t/direct_join.test index 634b958f9f9..23cfbb197c9 100644 --- a/storage/spider/mysql-test/spider/t/direct_join.test +++ b/storage/spider/mysql-test/spider/t/direct_join.test @@ -168,7 +168,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) @@ -315,7 +317,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_CONST_TABLE_JOIN; eval $CHILD2_1_SELECT_CONST_TABLE2_JOIN; diff --git a/storage/spider/mysql-test/spider/t/direct_join_using.test b/storage/spider/mysql-test/spider/t/direct_join_using.test index 3ecfb292c89..61942b1998e 100644 --- a/storage/spider/mysql-test/spider/t/direct_join_using.test +++ b/storage/spider/mysql-test/spider/t/direct_join_using.test @@ -168,7 +168,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_left_join.test b/storage/spider/mysql-test/spider/t/direct_left_join.test index c9784b047ed..31744aa199d 100644 --- a/storage/spider/mysql-test/spider/t/direct_left_join.test +++ b/storage/spider/mysql-test/spider/t/direct_left_join.test @@ -168,7 +168,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_left_join_nullable.test b/storage/spider/mysql-test/spider/t/direct_left_join_nullable.test index 7c69d73d335..949a1ecf5c7 100644 --- a/storage/spider/mysql-test/spider/t/direct_left_join_nullable.test +++ b/storage/spider/mysql-test/spider/t/direct_left_join_nullable.test @@ -183,7 +183,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_left_right_join_nullable.test b/storage/spider/mysql-test/spider/t/direct_left_right_join_nullable.test index dd407482713..e81dbbcccad 100644 --- a/storage/spider/mysql-test/spider/t/direct_left_right_join_nullable.test +++ b/storage/spider/mysql-test/spider/t/direct_left_right_join_nullable.test @@ -183,7 +183,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_left_right_left_join_nullable.test b/storage/spider/mysql-test/spider/t/direct_left_right_left_join_nullable.test index 3de0c310464..480ac8dcd1c 100644 --- a/storage/spider/mysql-test/spider/t/direct_left_right_left_join_nullable.test +++ b/storage/spider/mysql-test/spider/t/direct_left_right_left_join_nullable.test @@ -183,7 +183,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_right_join.test b/storage/spider/mysql-test/spider/t/direct_right_join.test index d124f75a230..823d5176f34 100644 --- a/storage/spider/mysql-test/spider/t/direct_right_join.test +++ b/storage/spider/mysql-test/spider/t/direct_right_join.test @@ -168,7 +168,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_right_join_nullable.test b/storage/spider/mysql-test/spider/t/direct_right_join_nullable.test index cdb2d4e6e58..0f0f614a03a 100644 --- a/storage/spider/mysql-test/spider/t/direct_right_join_nullable.test +++ b/storage/spider/mysql-test/spider/t/direct_right_join_nullable.test @@ -183,7 +183,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_right_left_join_nullable.test b/storage/spider/mysql-test/spider/t/direct_right_left_join_nullable.test index cff40bfdeef..75e1551bcbb 100644 --- a/storage/spider/mysql-test/spider/t/direct_right_left_join_nullable.test +++ b/storage/spider/mysql-test/spider/t/direct_right_left_join_nullable.test @@ -183,7 +183,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_right_left_right_join_nullable.test b/storage/spider/mysql-test/spider/t/direct_right_left_right_join_nullable.test index cd677a4bb8d..017a8174df7 100644 --- a/storage/spider/mysql-test/spider/t/direct_right_left_right_join_nullable.test +++ b/storage/spider/mysql-test/spider/t/direct_right_left_right_join_nullable.test @@ -183,7 +183,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/partition_cond_push.test b/storage/spider/mysql-test/spider/t/partition_cond_push.test index e47209ef220..b5eceead130 100644 --- a/storage/spider/mysql-test/spider/t/partition_cond_push.test +++ b/storage/spider/mysql-test/spider/t/partition_cond_push.test @@ -166,19 +166,25 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; --connection child2_3 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_3_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_3_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/partition_fulltext.test b/storage/spider/mysql-test/spider/t/partition_fulltext.test index 1b31fa05534..664617d2451 100644 --- a/storage/spider/mysql-test/spider/t/partition_fulltext.test +++ b/storage/spider/mysql-test/spider/t/partition_fulltext.test @@ -170,19 +170,25 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; --connection child2_3 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_3_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_3_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/partition_join_pushdown_for_single_partition.test b/storage/spider/mysql-test/spider/t/partition_join_pushdown_for_single_partition.test index 753014d5da7..e1916e1d9b0 100644 --- a/storage/spider/mysql-test/spider/t/partition_join_pushdown_for_single_partition.test +++ b/storage/spider/mysql-test/spider/t/partition_join_pushdown_for_single_partition.test @@ -169,19 +169,25 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; --connection child2_3 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_3_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_3_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/pushdown_not_like.test b/storage/spider/mysql-test/spider/t/pushdown_not_like.test index b39bb7785f6..5e7de1ff0ed 100644 --- a/storage/spider/mysql-test/spider/t/pushdown_not_like.test +++ b/storage/spider/mysql-test/spider/t/pushdown_not_like.test @@ -111,7 +111,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol SELECT argument FROM mysql.general_log WHERE argument LIKE '%select%'; + --enable_view_protocol } } --enable_ps2_protocol diff --git a/storage/spider/mysql-test/spider/t/quick_mode_0.test b/storage/spider/mysql-test/spider/t/quick_mode_0.test index 070d9e40548..b7f923ac609 100644 --- a/storage/spider/mysql-test/spider/t/quick_mode_0.test +++ b/storage/spider/mysql-test/spider/t/quick_mode_0.test @@ -135,15 +135,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) @@ -195,15 +199,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) @@ -243,6 +251,7 @@ if ($USE_CHILD_GROUP2) } } --connection master_1 +--disable_view_protocol --disable_ps2_protocol SELECT a.pkey FROM tbl_a a, tbl_b b WHERE a.pkey = b.pkey; if ($USE_CHILD_GROUP2) @@ -273,6 +282,7 @@ if ($USE_CHILD_GROUP2) } } --enable_ps2_protocol +--enable_view_protocol --echo --echo deinit diff --git a/storage/spider/mysql-test/spider/t/quick_mode_1.test b/storage/spider/mysql-test/spider/t/quick_mode_1.test index a2c2bf87844..979679249bc 100644 --- a/storage/spider/mysql-test/spider/t/quick_mode_1.test +++ b/storage/spider/mysql-test/spider/t/quick_mode_1.test @@ -135,15 +135,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) @@ -195,15 +199,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) @@ -255,15 +263,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/quick_mode_2.test b/storage/spider/mysql-test/spider/t/quick_mode_2.test index 12a48a904a2..2db2d1415bf 100644 --- a/storage/spider/mysql-test/spider/t/quick_mode_2.test +++ b/storage/spider/mysql-test/spider/t/quick_mode_2.test @@ -135,15 +135,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) @@ -195,15 +199,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) @@ -255,15 +263,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/quick_mode_3.test b/storage/spider/mysql-test/spider/t/quick_mode_3.test index 65851a5bf3e..bb9bc8c6e48 100644 --- a/storage/spider/mysql-test/spider/t/quick_mode_3.test +++ b/storage/spider/mysql-test/spider/t/quick_mode_3.test @@ -135,15 +135,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) @@ -195,15 +199,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) @@ -255,15 +263,19 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLES; --connection child2_2 if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /tmp_spider_bka_0x[0-9a-f]*/tmp_spider_bka_xxxx/ eval $CHILD2_2_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_2_SELECT_TABLES; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/slave_trx_isolation.test b/storage/spider/mysql-test/spider/t/slave_trx_isolation.test index a3a3fad8677..b88d038de46 100644 --- a/storage/spider/mysql-test/spider/t/slave_trx_isolation.test +++ b/storage/spider/mysql-test/spider/t/slave_trx_isolation.test @@ -109,8 +109,10 @@ if ($USE_CHILD_GROUP2) --disable_ps2_protocol if ($USE_GENERAL_LOG) { + --disable_view_protocol --replace_regex /-[0-9a-f]{12}-[0-9a-f]+-/-xxxxxxxxxxxx-xxxxx-/ eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } --enable_ps2_protocol eval $CHILD2_1_SELECT_TABLES; diff --git a/storage/spider/mysql-test/spider/t/timestamp.test b/storage/spider/mysql-test/spider/t/timestamp.test index 47d637bb0c5..fddfbfd3880 100644 --- a/storage/spider/mysql-test/spider/t/timestamp.test +++ b/storage/spider/mysql-test/spider/t/timestamp.test @@ -183,7 +183,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLE; if (!$OUTPUT_CHILD_GROUP2) @@ -216,6 +218,7 @@ if ($USE_CHILD_GROUP2) } --connection master_1 DELETE FROM tbl_a WHERE col_ts='1970-01-01 01:00:01'; +--disable_view_protocol --disable_ps2_protocol SELECT *, unix_timestamp(col_ts) FROM tbl_a; if ($USE_CHILD_GROUP2) @@ -228,7 +231,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLE; if (!$OUTPUT_CHILD_GROUP2) @@ -272,7 +277,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLE; if (!$OUTPUT_CHILD_GROUP2) @@ -317,7 +324,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLE; if (!$OUTPUT_CHILD_GROUP2) @@ -367,7 +376,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLE; if (!$OUTPUT_CHILD_GROUP2) @@ -447,7 +458,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLE; if (!$OUTPUT_CHILD_GROUP2) @@ -499,7 +512,9 @@ if ($USE_CHILD_GROUP2) --connection child2_1 if ($USE_GENERAL_LOG) { + --disable_view_protocol eval $CHILD2_1_SELECT_ARGUMENT1; + --enable_view_protocol } eval $CHILD2_1_SELECT_TABLE_F; if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/udf_pushdown.inc b/storage/spider/mysql-test/spider/t/udf_pushdown.inc index df9742a35b8..93e570a656d 100644 --- a/storage/spider/mysql-test/spider/t/udf_pushdown.inc +++ b/storage/spider/mysql-test/spider/t/udf_pushdown.inc @@ -9,7 +9,9 @@ SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32); if ($USE_CHILD_GROUP2) { --connection child2_1 + --disable_view_protocol SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%"; + --enable_view_protocol --disable_query_log TRUNCATE TABLE mysql.general_log; --enable_query_log From 25f1e6f565d5b1effb5cfcd858660e268bb1b1fe Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Wed, 2 Apr 2025 15:45:23 +1100 Subject: [PATCH 021/125] MDEV-36307 MDEV-35452 Do not create spider group by handler when dealing with derived tables or view and at least one select item is constant If one of the selected field is a MIN or MAX and it has been optimized into a constant, it is not added to the temp table used by a group by handler (GBH). The GBH therefore cannot store results to this missing field. On the other hand, when SELECTing from a view or a derived table, TMP_TABLE_ALL_COLUMNS is set. If the query has no group by or order by, an Item_temptable_field is created for this MIN/MAX field and added to the JOIN. Since the GBH could not store results to the corresponding field in the temp table, the value of this Item_temptable_field remains NULL. And the NULL value is passed to the record, then the temp row, and finally output as the (wrong) result. To fix this, we opt to not creating a spider GBH when a view or derived table is involved. This fixes spider/bugfix.mdev_26345 for --view-protocol Also fixed a comment: TABLE_LIST::belong_to_derived is NULL if the table belongs to a derived table that has non-MERGE type. --- sql/table.h | 2 +- .../mysql-test/spider/bugfix/r/mdev_26345.result | 3 +++ .../spider/mysql-test/spider/bugfix/t/mdev_26345.test | 1 + .../spider/mysql-test/spider/bugfix/t/mdev_29502.test | 4 ++++ storage/spider/spd_group_by_handler.cc | 10 ++++++++++ 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/sql/table.h b/sql/table.h index d46f91ee2dd..981fc5d23c5 100644 --- a/sql/table.h +++ b/sql/table.h @@ -2526,7 +2526,7 @@ struct TABLE_LIST List *view_tables; /* most upper view this table belongs to */ TABLE_LIST *belong_to_view; - /* A derived table this table belongs to */ + /* A merged derived table this table belongs to */ TABLE_LIST *belong_to_derived; /* The view directly referencing this table diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_26345.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_26345.result index 9f569795299..31a2cc452d0 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_26345.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_26345.result @@ -15,6 +15,9 @@ insert into t1 VALUES (1,4), (1,2), (2,11); SELECT MIN(b), a FROM t1 WHERE a=1; MIN(b) a 2 1 +select * from (SELECT MIN(b), a FROM t1 WHERE a=1) as v; +MIN(b) a +2 1 SELECT MAX(b), a FROM t1 WHERE a<3; MAX(b) a 11 1 diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_26345.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_26345.test index b14df3bf332..e3a0631721d 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_26345.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_26345.test @@ -22,6 +22,7 @@ create table t1 (a int, b int, PRIMARY KEY (a, b)) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; insert into t1 VALUES (1,4), (1,2), (2,11); SELECT MIN(b), a FROM t1 WHERE a=1; +select * from (SELECT MIN(b), a FROM t1 WHERE a=1) as v; SELECT MAX(b), a FROM t1 WHERE a<3; drop table t1, t2; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29502.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29502.test index 91a8cad5f8b..7e0bcc61df9 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_29502.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29502.test @@ -24,10 +24,14 @@ eval CREATE TABLE t1 (a INT KEY) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "$sr SELECT MAX(a) FROM t1; SELECT SUM(a) FROM t1; SELECT COUNT(a) FROM t1; +# Spider does not create a GBH with view protocol in these cases which +# would cause extra direct aggregate counts than without view protocol +--disable_view_protocol SELECT MAX(a), SUM(a) FROM t1; SELECT COUNT(a), MAX(a), SUM(a) FROM t1; SELECT MAX(a), COUNT(a), SUM(a) FROM t1; SELECT MAX(a), MAX(COALESCE(a)) FROM t1; +--enable_view_protocol SHOW STATUS LIKE 'Spider_direct_aggregate'; DROP TABLE t, t1; diff --git a/storage/spider/spd_group_by_handler.cc b/storage/spider/spd_group_by_handler.cc index 0c62e6cf7eb..70f9ef3c0b5 100644 --- a/storage/spider/spd_group_by_handler.cc +++ b/storage/spider/spd_group_by_handler.cc @@ -1566,6 +1566,16 @@ group_by_handler *spider_create_group_by_handler( DBUG_PRINT("info",("spider select item=%p", item)); if (item->const_item()) { + /* + Do not create the GBH when a derived table or view is + involved + */ + if (thd->derived_tables != NULL) + { + keep_going= FALSE; + break; + } + /* Do not handle the complex case where there's a const item in the auxiliary fields. It is too unlikely (if at all) to From 0f7c9146cf4c0d93e3970be8f1c45037748a778d Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Wed, 2 Apr 2025 14:28:10 +1100 Subject: [PATCH 022/125] MDEV-36452 MDEV-35452 Fix udf output temp field name in spider tests with --view-protocol --- .../include/direct_sql_with_comma_pwd_init.inc | 2 +- .../include/direct_sql_with_tmp_table_init.inc | 2 +- .../spider/bugfix/r/direct_sql_with_comma_pwd.result | 4 ++-- .../spider/bugfix/r/direct_sql_with_tmp_table.result | 4 ++-- .../spider/bugfix/r/udf_mysql_func_early.result | 4 ++-- .../bugfix/r/udf_mysql_func_early_init_file.result | 4 ++-- .../spider/bugfix/t/udf_mysql_func_early.test | 2 +- .../mysql-test/spider/feature/r/pushdown_case.result | 12 ++++++------ .../mysql-test/spider/feature/t/pushdown_case.test | 4 ++-- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/storage/spider/mysql-test/spider/bugfix/include/direct_sql_with_comma_pwd_init.inc b/storage/spider/mysql-test/spider/bugfix/include/direct_sql_with_comma_pwd_init.inc index c87af2d02e4..5c5a14bf796 100644 --- a/storage/spider/mysql-test/spider/bugfix/include/direct_sql_with_comma_pwd_init.inc +++ b/storage/spider/mysql-test/spider/bugfix/include/direct_sql_with_comma_pwd_init.inc @@ -6,6 +6,6 @@ --enable_query_log --enable_warnings let $DIRECT_SQL_COMMAND= - SELECT spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test", password "pass,1234", user "tu"'); + SELECT spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test", password "pass,1234", user "tu"') as exp; --connection child2_1 GRANT ALL ON *.* TO tu@'%' IDENTIFIED BY 'pass,1234'; diff --git a/storage/spider/mysql-test/spider/bugfix/include/direct_sql_with_tmp_table_init.inc b/storage/spider/mysql-test/spider/bugfix/include/direct_sql_with_tmp_table_init.inc index 7b58bd3e726..bb2c7eb3aba 100644 --- a/storage/spider/mysql-test/spider/bugfix/include/direct_sql_with_tmp_table_init.inc +++ b/storage/spider/mysql-test/spider/bugfix/include/direct_sql_with_tmp_table_init.inc @@ -6,4 +6,4 @@ --enable_query_log --enable_warnings let $DIRECT_SQL_COMMAND= - SELECT spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test"'); + SELECT spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test"') as exp; diff --git a/storage/spider/mysql-test/spider/bugfix/r/direct_sql_with_comma_pwd.result b/storage/spider/mysql-test/spider/bugfix/r/direct_sql_with_comma_pwd.result index b485d645619..7d18527351e 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/direct_sql_with_comma_pwd.result +++ b/storage/spider/mysql-test/spider/bugfix/r/direct_sql_with_comma_pwd.result @@ -15,8 +15,8 @@ CREATE TEMPORARY TABLE tmp_a ( pkey int NOT NULL, PRIMARY KEY (pkey) ) MASTER_1_ENGINE2 -SELECT spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test", password "pass,1234", user "tu"'); -spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test", password "pass,1234", user "tu"') +SELECT spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test", password "pass,1234", user "tu"') as exp; +exp 1 SELECT pkey FROM tmp_a; pkey diff --git a/storage/spider/mysql-test/spider/bugfix/r/direct_sql_with_tmp_table.result b/storage/spider/mysql-test/spider/bugfix/r/direct_sql_with_tmp_table.result index 65beb8a43dd..467bc0ed6cd 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/direct_sql_with_tmp_table.result +++ b/storage/spider/mysql-test/spider/bugfix/r/direct_sql_with_tmp_table.result @@ -13,8 +13,8 @@ CREATE TEMPORARY TABLE tmp_a ( pkey int NOT NULL, PRIMARY KEY (pkey) ) MASTER_1_ENGINE2 -SELECT spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test"'); -spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test"') +SELECT spider_direct_sql('SELECT 22', 'tmp_a', 'srv "s_2_1", database "test"') as exp; +exp 1 SELECT pkey FROM tmp_a; pkey diff --git a/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early.result b/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early.result index b84f60a67fb..2504cda9cd2 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early.result +++ b/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early.result @@ -31,8 +31,8 @@ CREATE TABLE tbl_a ( a INT ) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='table "tbl_a", srv "s_2_1"'; create temporary table results (a int); -SELECT SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"'); -SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"') +SELECT SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"') as exp; +exp 1 select * from results; a diff --git a/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early_init_file.result b/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early_init_file.result index b84f60a67fb..2504cda9cd2 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early_init_file.result +++ b/storage/spider/mysql-test/spider/bugfix/r/udf_mysql_func_early_init_file.result @@ -31,8 +31,8 @@ CREATE TABLE tbl_a ( a INT ) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='table "tbl_a", srv "s_2_1"'; create temporary table results (a int); -SELECT SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"'); -SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"') +SELECT SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"') as exp; +exp 1 select * from results; a diff --git a/storage/spider/mysql-test/spider/bugfix/t/udf_mysql_func_early.test b/storage/spider/mysql-test/spider/bugfix/t/udf_mysql_func_early.test index 4edff9ca784..5a5fffc3940 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/udf_mysql_func_early.test +++ b/storage/spider/mysql-test/spider/bugfix/t/udf_mysql_func_early.test @@ -37,7 +37,7 @@ eval CREATE TABLE tbl_a ( create temporary table results (a int); --disable_ps_protocol -SELECT SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"'); +SELECT SPIDER_DIRECT_SQL('select * from tbl_a', 'results', 'srv "s_2_1", database "auto_test_remote"') as exp; --enable_ps_protocol select * from results; diff --git a/storage/spider/mysql-test/spider/feature/r/pushdown_case.result b/storage/spider/mysql-test/spider/feature/r/pushdown_case.result index 613ce377865..67adb4954fb 100644 --- a/storage/spider/mysql-test/spider/feature/r/pushdown_case.result +++ b/storage/spider/mysql-test/spider/feature/r/pushdown_case.result @@ -11,11 +11,11 @@ create table t2 (c int); create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; insert into t1 values (42), (3), (848), (100); -explain select case c when 3 then "three" when 42 then "answer" else "other" end from t1; +explain select case c when 3 then "three" when 42 then "answer" else "other" end as exp from t1; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY -select case c when 3 then "three" when 42 then "answer" else "other" end from t1; -case c when 3 then "three" when 42 then "answer" else "other" end +select case c when 3 then "three" when 42 then "answer" else "other" end as exp from t1; +exp answer three other @@ -29,11 +29,11 @@ answer three NULL NULL -explain select case when c = 3 then "three" when c = 42 then "answer" else "other" end from t1; +explain select case when c = 3 then "three" when c = 42 then "answer" else "other" end as exp from t1; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Storage engine handles GROUP BY -select case when c = 3 then "three" when c = 42 then "answer" else "other" end from t1; -case when c = 3 then "three" when c = 42 then "answer" else "other" end +select case when c = 3 then "three" when c = 42 then "answer" else "other" end as exp from t1; +exp answer three other diff --git a/storage/spider/mysql-test/spider/feature/t/pushdown_case.test b/storage/spider/mysql-test/spider/feature/t/pushdown_case.test index b86edceb615..9a77183d3d5 100644 --- a/storage/spider/mysql-test/spider/feature/t/pushdown_case.test +++ b/storage/spider/mysql-test/spider/feature/t/pushdown_case.test @@ -16,7 +16,7 @@ insert into t1 values (42), (3), (848), (100); # everything let $query= -select case c when 3 then "three" when 42 then "answer" else "other" end from t1; +select case c when 3 then "three" when 42 then "answer" else "other" end as exp from t1; eval explain $query; eval $query; @@ -28,7 +28,7 @@ eval $query; # no value let $query= -select case when c = 3 then "three" when c = 42 then "answer" else "other" end from t1; +select case when c = 3 then "three" when c = 42 then "answer" else "other" end as exp from t1; eval explain $query; eval $query; From dabd51c3912484b97f92f091b6ab7b20409d9f0c Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Wed, 2 Apr 2025 14:47:53 +1100 Subject: [PATCH 023/125] MDEV-36335 MDEV-35452 Disable view protocol when calling spider_copy_tables in tests Spider needs to lock the spider table when executing the udf, but the server layer would have already locked tables in view protocol because it transforms the query: select spider_copy_table('t', 0, 1) to two queries create or replace view mysqltest_tmp_v as select spider_copy_table('t', 0, 1); select * from mysqltest_tmp_v; So spider justifiably errors out in this case by checking on thd->derived_tables and thd->locks in spider_copy_tables_body() --- storage/spider/mysql-test/spider/bg/t/ha.test | 2 ++ storage/spider/mysql-test/spider/bg/t/ha_part.test | 2 ++ storage/spider/mysql-test/spider/bugfix/t/mdev_30649.test | 2 ++ storage/spider/mysql-test/spider/bugfix/t/mdev_30727.test | 4 ++++ storage/spider/mysql-test/spider/handler/t/ha.test | 2 ++ storage/spider/mysql-test/spider/handler/t/ha_part.test | 2 ++ storage/spider/mysql-test/spider/t/ha.test | 2 ++ storage/spider/mysql-test/spider/t/ha_part.test | 2 ++ 8 files changed, 18 insertions(+) diff --git a/storage/spider/mysql-test/spider/bg/t/ha.test b/storage/spider/mysql-test/spider/bg/t/ha.test index 00dcf776836..aca4d47b646 100644 --- a/storage/spider/mysql-test/spider/bg/t/ha.test +++ b/storage/spider/mysql-test/spider/bg/t/ha.test @@ -395,7 +395,9 @@ if ($USE_CHILD_GROUP3) --connection master_1 eval $MASTER_1_SET_RECOVERY_STATUS_2_1; eval $MASTER_1_CHECK_LINK_STATUS; +--disable_view_protocol eval $MASTER_1_COPY_TABLES_2_1; +--enable_view_protocol if ($USE_CHILD_GROUP3) { if (!$OUTPUT_CHILD_GROUP3) diff --git a/storage/spider/mysql-test/spider/bg/t/ha_part.test b/storage/spider/mysql-test/spider/bg/t/ha_part.test index 72ddcfd1f10..b7d27b0d69d 100644 --- a/storage/spider/mysql-test/spider/bg/t/ha_part.test +++ b/storage/spider/mysql-test/spider/bg/t/ha_part.test @@ -460,7 +460,9 @@ if ($HAVE_PARTITION) --connection master_1 eval $MASTER_1_SET_RECOVERY_STATUS_P_2_1; eval $MASTER_1_CHECK_LINK_STATUS; + --disable_view_protocol eval $MASTER_1_COPY_TABLES_P_2_1; + --enable_view_protocol if ($USE_CHILD_GROUP3) { if (!$OUTPUT_CHILD_GROUP3) diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30649.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30649.test index 02df6032887..8a7141e5248 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_30649.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30649.test @@ -25,7 +25,9 @@ INSERT INTO dst VALUES (555, '1999-12-12'); # possibly a bug, e.g. an unnecessary requirement. evalp CREATE TABLE t (c INT, d DATE, PRIMARY KEY(c)) ENGINE=SPIDER COMMENT='table "src dst", srv "s_2_1 s_1"'; +--disable_view_protocol SELECT spider_copy_tables('t', '0', '1'); +--enable_view_protocol SELECT * FROM dst; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30727.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30727.test index ebd08edce24..294675d2f4c 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_30727.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30727.test @@ -7,8 +7,10 @@ CREATE FUNCTION spider_bg_direct_sql RETURNS INT SONAME 'ha_spider.so'; SELECT spider_bg_direct_sql ('SELECT * FROM s','a','srv "b"'); CREATE FUNCTION spider_copy_tables RETURNS INT SONAME 'ha_spider.so'; +--disable_view_protocol --error ER_CANT_INITIALIZE_UDF SELECT spider_copy_tables ('t', '0', '0'); +--enable_view_protocol # spider_flush_table_mon_cache does not require spider init to function CREATE FUNCTION spider_flush_table_mon_cache RETURNS INT SONAME 'ha_spider.so'; @@ -21,8 +23,10 @@ install soname 'ha_spider'; SELECT spider_direct_sql ('SELECT * FROM s','a','srv "b"'); call mtr.add_suppression(".*\\[Error\\] (mysqld|mariadbd): Can't find record in 'spider_tables'"); +--disable_view_protocol --error ER_KEY_NOT_FOUND SELECT spider_copy_tables ('t', '0', '0'); +--enable_view_protocol SELECT spider_flush_table_mon_cache (); diff --git a/storage/spider/mysql-test/spider/handler/t/ha.test b/storage/spider/mysql-test/spider/handler/t/ha.test index 00dcf776836..aca4d47b646 100644 --- a/storage/spider/mysql-test/spider/handler/t/ha.test +++ b/storage/spider/mysql-test/spider/handler/t/ha.test @@ -395,7 +395,9 @@ if ($USE_CHILD_GROUP3) --connection master_1 eval $MASTER_1_SET_RECOVERY_STATUS_2_1; eval $MASTER_1_CHECK_LINK_STATUS; +--disable_view_protocol eval $MASTER_1_COPY_TABLES_2_1; +--enable_view_protocol if ($USE_CHILD_GROUP3) { if (!$OUTPUT_CHILD_GROUP3) diff --git a/storage/spider/mysql-test/spider/handler/t/ha_part.test b/storage/spider/mysql-test/spider/handler/t/ha_part.test index 72ddcfd1f10..b7d27b0d69d 100644 --- a/storage/spider/mysql-test/spider/handler/t/ha_part.test +++ b/storage/spider/mysql-test/spider/handler/t/ha_part.test @@ -460,7 +460,9 @@ if ($HAVE_PARTITION) --connection master_1 eval $MASTER_1_SET_RECOVERY_STATUS_P_2_1; eval $MASTER_1_CHECK_LINK_STATUS; + --disable_view_protocol eval $MASTER_1_COPY_TABLES_P_2_1; + --enable_view_protocol if ($USE_CHILD_GROUP3) { if (!$OUTPUT_CHILD_GROUP3) diff --git a/storage/spider/mysql-test/spider/t/ha.test b/storage/spider/mysql-test/spider/t/ha.test index 00dcf776836..aca4d47b646 100644 --- a/storage/spider/mysql-test/spider/t/ha.test +++ b/storage/spider/mysql-test/spider/t/ha.test @@ -395,7 +395,9 @@ if ($USE_CHILD_GROUP3) --connection master_1 eval $MASTER_1_SET_RECOVERY_STATUS_2_1; eval $MASTER_1_CHECK_LINK_STATUS; +--disable_view_protocol eval $MASTER_1_COPY_TABLES_2_1; +--enable_view_protocol if ($USE_CHILD_GROUP3) { if (!$OUTPUT_CHILD_GROUP3) diff --git a/storage/spider/mysql-test/spider/t/ha_part.test b/storage/spider/mysql-test/spider/t/ha_part.test index 7e9d6f7715c..123b100a474 100644 --- a/storage/spider/mysql-test/spider/t/ha_part.test +++ b/storage/spider/mysql-test/spider/t/ha_part.test @@ -460,7 +460,9 @@ if ($HAVE_PARTITION) --connection master_1 eval $MASTER_1_SET_RECOVERY_STATUS_P_2_1; eval $MASTER_1_CHECK_LINK_STATUS; + --disable_view_protocol eval $MASTER_1_COPY_TABLES_P_2_1; + --enable_view_protocol if ($USE_CHILD_GROUP3) { if (!$OUTPUT_CHILD_GROUP3) From 8866fba00a03dbdcba479b42071089271cb6c415 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Thu, 3 Apr 2025 17:56:56 +1100 Subject: [PATCH 024/125] MDEV-36454 MDEV-35452 Fix spider view protocol test failures caused by different queries constructed without GBH With view protocol, often during optimization, the GBH is not created because join->tables_list is the view mysqltest_tmp_v which has MEMORY as engine which does not have GBH implemented. In such cases, if without view protocol the test takes a path that does create a spider GBH, the resulting queries sent to the data node often differ. Therefore we disable view protocol for these statements. --- .../mysql-test/spider/bugfix/t/mdev_20502.test | 4 ++++ .../mysql-test/spider/bugfix/t/mdev_27172.test | 6 ++++++ .../mysql-test/spider/bugfix/t/mdev_34659.test | 3 +++ .../spider/bugfix/t/wrapper_mariadb.test | 4 ++-- .../mysql-test/spider/t/auto_increment.test | 2 ++ .../mysql-test/spider/t/direct_join_using.test | 2 ++ .../mysql-test/spider/t/direct_left_join.test | 2 ++ .../mysql-test/spider/t/direct_right_join.test | 2 ++ .../mysql-test/spider/t/pushdown_not_like.test | 2 ++ storage/spider/mysql-test/spider/t/timestamp.test | 15 ++++++++++++++- .../spider/mysql-test/spider/t/udf_pushdown.inc | 2 ++ 11 files changed, 41 insertions(+), 3 deletions(-) diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_20502.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_20502.test index 3174ace8bfb..574a13c8e86 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_20502.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_20502.test @@ -49,14 +49,18 @@ TRUNCATE TABLE mysql.general_log; --disable_ps2_protocol --connection master_1 +--disable_view_protocol SELECT id, 0 AS const, val FROM tbl_a; SELECT 1+2, id, 0 AS const, val, val+10, (SELECT tbl_a.val+1 FROM tbl_a) AS sq FROM tbl_a; +--enable_view_protocol INSERT INTO tbl_a (val) VALUES (2), (1); +--disable_view_protocol SELECT val+10, 0 AS const, val, (SELECT tbl_a.val+1 FROM tbl_a LIMIT 1) AS sq FROM tbl_a GROUP BY val; SELECT MAX(id) AS m, 0 AS const, val, (SELECT tbl_a.val+1 FROM tbl_a LIMIT 1) AS sq FROM tbl_a GROUP BY val; +--enable_view_protocol --connection child2_1 --disable_view_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_27172.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_27172.test index 8374713651a..da7a1457a78 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_27172.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_27172.test @@ -50,8 +50,10 @@ eval CREATE TABLE tbl_a ( --disable_ps2_protocol INSERT INTO tbl_a VALUES (1, "Hi!"),(2, "Aloha!"),(3, "Aloha!!!"); +--disable_view_protocol SELECT * FROM tbl_a WHERE greeting = "Aloha!" AND CASE greeting WHEN "Aloha!" THEN "one" ELSE 'more' END = "one"; # hack to disable GBH +--enable_view_protocol # LIKE eval CREATE TABLE tbl_b ( @@ -61,8 +63,10 @@ eval CREATE TABLE tbl_b ( ) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='table "tbl_b", srv "s_2_1"'; INSERT INTO tbl_b VALUES (1, "Hi!"),(2, "Aloha!"),(3, "Aloha!!!"); +--disable_view_protocol SELECT * FROM tbl_b WHERE greeting = "Aloha!" AND CASE greeting WHEN "Aloha!" THEN "one" ELSE 'more' END = "one"; # hack to disable GBH +--enable_view_protocol # LIKE eval CREATE TABLE tbl_c ( @@ -72,8 +76,10 @@ eval CREATE TABLE tbl_c ( ) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='table "tbl_c", srv "s_2_1"'; INSERT INTO tbl_c VALUES (1, "Hi!"),(2, "Aloha!"),(3, "Aloha!!!"); +--disable_view_protocol SELECT * FROM tbl_c WHERE greeting = "Aloha!" AND CASE greeting WHEN "Aloha!" THEN "one" ELSE 'more' END = "one"; # hack to disable GBH +--enable_view_protocol --connection child2_1 --disable_view_protocol diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_34659.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_34659.test index 3b1dc206866..330b96db6f9 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_34659.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_34659.test @@ -12,7 +12,10 @@ CREATE TABLE t1 (c INT) ENGINE=MyISAM; CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",SRV "srv",TABLE "t1"'; insert into t2 values (456), (123); SELECT * FROM t2 ORDER BY CAST(c AS char(60)); +# extra warnings with view protocol +--disable_view_protocol SELECT * FROM t2 ORDER BY CAST(c AS INET6); +--enable_view_protocol SELECT * FROM t2 GROUP BY CAST(c AS char(60)); SELECT * FROM t2 GROUP BY CAST(c AS INET6); # Cleanup diff --git a/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test b/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test index ae7e6643e3b..72502b64f78 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test +++ b/storage/spider/mysql-test/spider/bugfix/t/wrapper_mariadb.test @@ -47,12 +47,12 @@ TRUNCATE TABLE mysql.general_log; --disable_ps2_protocol --connection master_1 +--disable_view_protocol SELECT * FROM tbl_a ORDER BY pkey; +--enable_view_protocol --connection child2_1 --disable_view_protocol -# in --ps a query is logged differently in a general log -replace_regex /order by t0.`pkey`/order by `pkey`/; eval $CHILD2_1_SELECT_ARGUMENT1; --enable_view_protocol eval $CHILD2_1_SELECT_TABLES; diff --git a/storage/spider/mysql-test/spider/t/auto_increment.test b/storage/spider/mysql-test/spider/t/auto_increment.test index 5e3fff3dc55..b4f0610fa58 100644 --- a/storage/spider/mysql-test/spider/t/auto_increment.test +++ b/storage/spider/mysql-test/spider/t/auto_increment.test @@ -145,7 +145,9 @@ if ($USE_CHILD_GROUP2) } --connection master_1 --disable_ps2_protocol +--disable_view_protocol SELECT * FROM tbl_a; +--enable_view_protocol if ($USE_CHILD_GROUP2) { if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/direct_join_using.test b/storage/spider/mysql-test/spider/t/direct_join_using.test index 61942b1998e..44d563b27ca 100644 --- a/storage/spider/mysql-test/spider/t/direct_join_using.test +++ b/storage/spider/mysql-test/spider/t/direct_join_using.test @@ -156,7 +156,9 @@ if ($USE_CHILD_GROUP2) --connection master_1 --disable_ps2_protocol +--disable_view_protocol SELECT a.a, c.b, c.c FROM tbl_a a join tbl_b b using(a) join tbl_c c using(a) ORDER BY a.b DESC; +--enable_view_protocol if ($USE_CHILD_GROUP2) { diff --git a/storage/spider/mysql-test/spider/t/direct_left_join.test b/storage/spider/mysql-test/spider/t/direct_left_join.test index 31744aa199d..ed36f580a52 100644 --- a/storage/spider/mysql-test/spider/t/direct_left_join.test +++ b/storage/spider/mysql-test/spider/t/direct_left_join.test @@ -156,7 +156,9 @@ if ($USE_CHILD_GROUP2) --connection master_1 --disable_ps2_protocol +--disable_view_protocol SELECT a.a, c.b, c.c FROM tbl_a a left join tbl_b b on a.a = b.a left join tbl_c c on a.a = c.a ORDER BY a.b DESC; +--enable_view_protocol if ($USE_CHILD_GROUP2) { diff --git a/storage/spider/mysql-test/spider/t/direct_right_join.test b/storage/spider/mysql-test/spider/t/direct_right_join.test index 823d5176f34..effa1121c8b 100644 --- a/storage/spider/mysql-test/spider/t/direct_right_join.test +++ b/storage/spider/mysql-test/spider/t/direct_right_join.test @@ -156,7 +156,9 @@ if ($USE_CHILD_GROUP2) --connection master_1 --disable_ps2_protocol +--disable_view_protocol SELECT a.a, c.b, c.c FROM tbl_a a right join tbl_b b on a.a = b.a right join tbl_c c on a.a = c.a ORDER BY a.b DESC; +--enable_view_protocol if ($USE_CHILD_GROUP2) { diff --git a/storage/spider/mysql-test/spider/t/pushdown_not_like.test b/storage/spider/mysql-test/spider/t/pushdown_not_like.test index 5e7de1ff0ed..cf2ed838d31 100644 --- a/storage/spider/mysql-test/spider/t/pushdown_not_like.test +++ b/storage/spider/mysql-test/spider/t/pushdown_not_like.test @@ -105,7 +105,9 @@ if ($USE_CHILD_GROUP2) --connection master_1 --disable_ps2_protocol +--disable_view_protocol select * from ta_l where b not like 'a%'; +--enable_view_protocol if ($USE_CHILD_GROUP2) { --connection child2_1 diff --git a/storage/spider/mysql-test/spider/t/timestamp.test b/storage/spider/mysql-test/spider/t/timestamp.test index fddfbfd3880..20fa69a6287 100644 --- a/storage/spider/mysql-test/spider/t/timestamp.test +++ b/storage/spider/mysql-test/spider/t/timestamp.test @@ -172,7 +172,9 @@ if ($USE_CHILD_GROUP2) } --connection master_1 --disable_ps2_protocol +--disable_view_protocol SELECT *, unix_timestamp(col_ts) FROM tbl_a; +--enable_view_protocol if ($USE_CHILD_GROUP2) { if (!$OUTPUT_CHILD_GROUP2) @@ -218,9 +220,10 @@ if ($USE_CHILD_GROUP2) } --connection master_1 DELETE FROM tbl_a WHERE col_ts='1970-01-01 01:00:01'; ---disable_view_protocol --disable_ps2_protocol +--disable_view_protocol SELECT *, unix_timestamp(col_ts) FROM tbl_a; +--enable_view_protocol if ($USE_CHILD_GROUP2) { if (!$OUTPUT_CHILD_GROUP2) @@ -266,7 +269,9 @@ SET @@timestamp=1; INSERT INTO tbl_a VALUES (1, now(), now()); SET @@timestamp=0; --disable_ps2_protocol +--disable_view_protocol SELECT *, unix_timestamp(col_ts) FROM tbl_a; +--enable_view_protocol if ($USE_CHILD_GROUP2) { if (!$OUTPUT_CHILD_GROUP2) @@ -313,7 +318,9 @@ if ($USE_CHILD_GROUP2) --connection master_1 UPDATE tbl_a SET col_ts=col_dt; --disable_ps2_protocol +--disable_view_protocol SELECT *, unix_timestamp(col_ts) FROM tbl_a; +--enable_view_protocol if ($USE_CHILD_GROUP2) { if (!$OUTPUT_CHILD_GROUP2) @@ -359,6 +366,7 @@ if ($USE_CHILD_GROUP2) } --connection master_1 --disable_ps2_protocol +--disable_view_protocol SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > '2018-01-01'; SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts < '2018-10-28 02:30:00'; SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE '2018-10-28 02:30:00' > col_ts; @@ -366,6 +374,7 @@ SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts BETWEEN '2018-10-28 01: SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts >= '2018-10-28 01:30:00' AND col_ts <= '2018-10-28 02:30:00'; SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 180325020000; SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 19700101010001; +--enable_view_protocol if ($USE_CHILD_GROUP2) { if (!$OUTPUT_CHILD_GROUP2) @@ -441,6 +450,7 @@ if ($USE_CHILD_GROUP2) } --connection master_1 --disable_ps2_protocol +--disable_view_protocol SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > '2018-01-01'; SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts < '2018-10-28 02:30:00'; SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE '2018-10-28 02:30:00' > col_ts; @@ -448,6 +458,7 @@ SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts BETWEEN '2018-10-28 01: SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts >= '2018-10-28 01:30:00' AND col_ts <= '2018-10-28 02:30:00'; SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 180325020000; SELECT *, unix_timestamp(col_ts) FROM tbl_a WHERE col_ts > 19700101010001; +--enable_view_protocol if ($USE_CHILD_GROUP2) { if (!$OUTPUT_CHILD_GROUP2) @@ -497,11 +508,13 @@ if ($USE_CHILD_GROUP2) } --connection master_1 --disable_ps2_protocol +--disable_view_protocol SELECT * FROM tbl_f; SELECT TIMESTAMP(col_d, col_t) FROM tbl_f; SELECT TIMESTAMP('2018-06-25', col_t) FROM tbl_f; SELECT TIMESTAMP(col_d, '10:43:21') FROM tbl_f; SELECT TIMESTAMP('2018-06-25', '10:43:21') FROM tbl_f; +--enable_view_protocol if ($USE_CHILD_GROUP2) { if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/mysql-test/spider/t/udf_pushdown.inc b/storage/spider/mysql-test/spider/t/udf_pushdown.inc index 93e570a656d..16f85dfffcd 100644 --- a/storage/spider/mysql-test/spider/t/udf_pushdown.inc +++ b/storage/spider/mysql-test/spider/t/udf_pushdown.inc @@ -2,8 +2,10 @@ --echo ##### test SELECTs ##### --connection master_1 --disable_ps2_protocol +--disable_view_protocol SELECT * FROM ta_l WHERE id = plusone(1); SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32); +--enable_view_protocol --enable_ps2_protocol if ($USE_CHILD_GROUP2) From b6392c292e8dc4539185dba4696e05c65d0f1065 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Thu, 3 Apr 2025 17:57:04 +1100 Subject: [PATCH 025/125] MDEV-36454 MDEV-35452 Fix spider view protocol test failures caused by tampering of order by items With view protocol, a SELECT statement is transformed into two statements: 1. CREATE OR REPLACE VIEW mysqltest_tmp_v AS SELECT ... 2. SELECT * FROM mysqltest_tmp_v The first statement reconstructed the query, which is executed in the second statement. The reconstruction often replaces aliases in ORDER BY by the original item. For example, in the test spider/bugfix.mdev_29008 the query SELECT MIN(t2.a) AS f1, t1.b AS f2 FROM tbl_a AS t1 JOIN tbl_a AS t2 GROUP BY f2 ORDER BY f1, f2; is transformed to "select min(`t2`.`a`) AS `f1`,`t1`.`b` AS `f2` from (`auto_test_local`.`tbl_a` `t1` join `auto_test_local`.`tbl_a` `t2`) group by `t1`.`b` order by min(`t2`.`a`),`t1`.`b`" In such cases, spider constructs different queries to execute at the data node. So we disable view protocol for such queries. --- .../mysql-test/spider/bugfix/t/group_by_order_by_limit.test | 2 ++ storage/spider/mysql-test/spider/bugfix/t/mdev_29008.test | 2 ++ .../spider/regression/e112122/t/group_by_order_by_limit_ok.test | 2 ++ 3 files changed, 6 insertions(+) diff --git a/storage/spider/mysql-test/spider/bugfix/t/group_by_order_by_limit.test b/storage/spider/mysql-test/spider/bugfix/t/group_by_order_by_limit.test index 553fb6adeaf..d35c25534e9 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/group_by_order_by_limit.test +++ b/storage/spider/mysql-test/spider/bugfix/t/group_by_order_by_limit.test @@ -71,7 +71,9 @@ TRUNCATE TABLE mysql.general_log; set @old_spider_direct_aggregate=@@session.spider_direct_aggregate; set spider_direct_aggregate=1; SHOW STATUS LIKE 'Spider_direct_aggregate'; +--disable_view_protocol SELECT skey, count(*) cnt FROM tbl_a GROUP BY skey ORDER BY cnt DESC, skey DESC LIMIT 5; +--enable_view_protocol SHOW STATUS LIKE 'Spider_direct_aggregate'; set spider_direct_aggregate=@old_spider_direct_aggregate; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29008.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29008.test index 40455916596..7e5e2ca2fd9 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_29008.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29008.test @@ -31,8 +31,10 @@ eval CREATE TABLE tbl_a ( ) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='table "tbl_a", srv "s_2_1"'; --disable_ps2_protocol +--disable_view_protocol SELECT MIN(t2.a) AS f1, t1.b AS f2 FROM tbl_a AS t1 JOIN tbl_a AS t2 GROUP BY f2 ORDER BY f1, f2; SELECT MIN(t2.a) AS f1, t1.b AS f2 FROM tbl_a AS t1 JOIN tbl_a AS t2 GROUP BY f2 ORDER BY MIN(t2.a), MAX(t2.a), f2; +--enable_view_protocol --enable_ps2_protocol --connection master_1 diff --git a/storage/spider/mysql-test/spider/regression/e112122/t/group_by_order_by_limit_ok.test b/storage/spider/mysql-test/spider/regression/e112122/t/group_by_order_by_limit_ok.test index dae5a812888..007fef77b62 100644 --- a/storage/spider/mysql-test/spider/regression/e112122/t/group_by_order_by_limit_ok.test +++ b/storage/spider/mysql-test/spider/regression/e112122/t/group_by_order_by_limit_ok.test @@ -71,7 +71,9 @@ TRUNCATE TABLE mysql.general_log; set @old_spider_direct_aggregate=@@session.spider_direct_aggregate; set spider_direct_aggregate=1; SHOW STATUS LIKE 'Spider_direct_aggregate'; +--disable_view_protocol SELECT skey, count(*) cnt FROM tbl_a GROUP BY skey ORDER BY cnt DESC, skey DESC LIMIT 5; +--enable_view_protocol SHOW STATUS LIKE 'Spider_direct_aggregate'; set spider_direct_aggregate=@old_spider_direct_aggregate; From 00fa5b86768f819de5aab5c381c931fb67a59385 Mon Sep 17 00:00:00 2001 From: Dave Gosselin Date: Tue, 4 Feb 2025 13:42:18 -0500 Subject: [PATCH 026/125] MDEV-35721 UBSAN: runtime error: -nan outside range Prevent division by zero when reading table statistics --- mysql-test/main/mdev-35721-ubsan.result | 21 +++++++++++++++++++++ mysql-test/main/mdev-35721-ubsan.test | 22 ++++++++++++++++++++++ sql/sql_statistics.cc | 2 +- 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 mysql-test/main/mdev-35721-ubsan.result create mode 100644 mysql-test/main/mdev-35721-ubsan.test diff --git a/mysql-test/main/mdev-35721-ubsan.result b/mysql-test/main/mdev-35721-ubsan.result new file mode 100644 index 00000000000..8a3e61ad6ac --- /dev/null +++ b/mysql-test/main/mdev-35721-ubsan.result @@ -0,0 +1,21 @@ +CREATE TABLE t (c1 VARCHAR(10),c2 VARCHAR(10),PRIMARY KEY(c1,c2),FULLTEXT KEY k (c2)) ENGINE=InnoDB; +INSERT INTO t VALUES ('a','b'); +DROP TABLE t; +CREATE TABLE t (c1 VARCHAR(10),c2 VARCHAR(10),PRIMARY KEY(c1,c2),FULLTEXT KEY k (c2)) ENGINE=InnoDB; +DELETE FROM t; +DROP TABLE t; +CREATE TABLE t (a INT(1),d INT(1),b VARCHAR(1),c CHAR(1),c3 INT(1) GENERATED ALWAYS AS ((a + LENGTH (d))) STORED,c2 CHAR(1) GENERATED ALWAYS AS (SUBSTR(b,0,0)) VIRTUAL,k1 CHAR(1) GENERATED ALWAYS AS (SUBSTR(b,0,0)) VIRTUAL,PRIMARY KEY(b (1),a,d),KEY d (d),KEY a (a),KEY c_renamed (c (1),b (1)),KEY b (b (1),c (1),a),KEY k1 (k1),KEY a_2 (a,k1),KEY k1_2 (k1,d)) DEFAULT CHARSET=latin1 ENGINE=InnoDB; +DELETE FROM t; +DROP TABLE t; +CREATE TABLE t (a INT,ROW_START TIMESTAMP(6) AS ROW START,ROW_END TIMESTAMP(6) AS ROW END,PERIOD FOR SYSTEM_TIME(ROW_START,ROW_END),INDEX (ROW_START),INDEX (ROW_END),PRIMARY KEY(ROW_END,a,ROW_START),INDEX (ROW_END,ROW_START,a)) WITH SYSTEM VERSIONING ENGINE=InnoDB; +SHOW INDEX FROM t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment Ignored +t 0 PRIMARY 1 ROW_END A 0 NULL NULL BTREE NO +t 0 PRIMARY 2 a A 0 NULL NULL BTREE NO +t 0 PRIMARY 3 ROW_START A 0 NULL NULL BTREE NO +t 1 ROW_START 1 ROW_START A 0 NULL NULL BTREE NO +t 1 ROW_END 1 ROW_END A 0 NULL NULL BTREE NO +t 1 ROW_END_2 1 ROW_END A 0 NULL NULL BTREE NO +t 1 ROW_END_2 2 ROW_START A 0 NULL NULL BTREE NO +t 1 ROW_END_2 3 a A 0 NULL NULL BTREE NO +DROP TABLE t; diff --git a/mysql-test/main/mdev-35721-ubsan.test b/mysql-test/main/mdev-35721-ubsan.test new file mode 100644 index 00000000000..24f580f82b4 --- /dev/null +++ b/mysql-test/main/mdev-35721-ubsan.test @@ -0,0 +1,22 @@ + +--source include/have_innodb.inc + +CREATE TABLE t (c1 VARCHAR(10),c2 VARCHAR(10),PRIMARY KEY(c1,c2),FULLTEXT KEY k (c2)) ENGINE=InnoDB; +INSERT INTO t VALUES ('a','b'); + +DROP TABLE t; + +CREATE TABLE t (c1 VARCHAR(10),c2 VARCHAR(10),PRIMARY KEY(c1,c2),FULLTEXT KEY k (c2)) ENGINE=InnoDB; +DELETE FROM t; + +DROP TABLE t; + +CREATE TABLE t (a INT(1),d INT(1),b VARCHAR(1),c CHAR(1),c3 INT(1) GENERATED ALWAYS AS ((a + LENGTH (d))) STORED,c2 CHAR(1) GENERATED ALWAYS AS (SUBSTR(b,0,0)) VIRTUAL,k1 CHAR(1) GENERATED ALWAYS AS (SUBSTR(b,0,0)) VIRTUAL,PRIMARY KEY(b (1),a,d),KEY d (d),KEY a (a),KEY c_renamed (c (1),b (1)),KEY b (b (1),c (1),a),KEY k1 (k1),KEY a_2 (a,k1),KEY k1_2 (k1,d)) DEFAULT CHARSET=latin1 ENGINE=InnoDB; +DELETE FROM t; + +DROP TABLE t; + +CREATE TABLE t (a INT,ROW_START TIMESTAMP(6) AS ROW START,ROW_END TIMESTAMP(6) AS ROW END,PERIOD FOR SYSTEM_TIME(ROW_START,ROW_END),INDEX (ROW_START),INDEX (ROW_END),PRIMARY KEY(ROW_END,a,ROW_START),INDEX (ROW_END,ROW_START,a)) WITH SYSTEM VERSIONING ENGINE=InnoDB; +SHOW INDEX FROM t; + +DROP TABLE t; diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 163cb8da9fc..de4bdeeaa31 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -3139,7 +3139,7 @@ read_statistics_for_table(THD *thd, TABLE *table, double avg_frequency= pk_read_stats->get_avg_frequency(j-1); set_if_smaller(avg_frequency, 1); double val= (pk_read_stats->get_avg_frequency(j) / - avg_frequency); + avg_frequency > 0 ? avg_frequency : 1); index_statistics->set_avg_frequency (l, val); } } From 0bcc03a2e6619f7c30b72bb4a148a4f332ab371e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 10 Apr 2025 12:58:10 +0300 Subject: [PATCH 027/125] Recovery cleanup recv_sys_t::report_progress(): Display the largest currently known LSN. recv_scan_log(): Display an error with fewer function calls. Reviewed by: Debarun Banerjee --- storage/innobase/log/log0recv.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 901e3ca3520..9a4bac91035 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3559,13 +3559,14 @@ void recv_sys_t::report_progress() const } else { + const lsn_t end{std::max(recv_sys.scanned_lsn, recv_sys.file_checkpoint)}; sql_print_information("InnoDB: To recover: LSN " LSN_PF "/" LSN_PF "; %zu pages", - recv_sys.lsn, recv_sys.scanned_lsn, n); + recv_sys.lsn, end, n); service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, "To recover: LSN " LSN_PF "/" LSN_PF "; %zu pages", - recv_sys.lsn, recv_sys.scanned_lsn, n); + recv_sys.lsn, end, n); } } @@ -4106,8 +4107,8 @@ static bool recv_scan_log(bool last_phase) {log_sys.buf + recv_sys.len, size})) { mysql_mutex_unlock(&recv_sys.mutex); - ib::error() << "Failed to read log at " << source_offset - << ": " << err; + sql_print_error("InnoDB: Failed to read log at %" PRIu64 ": %s", + source_offset, ut_strerr(err)); recv_sys.set_corrupt_log(); mysql_mutex_lock(&recv_sys.mutex); } From acd071f599f416ddb4821dec485c4d912844213f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 10 Apr 2025 13:02:17 +0300 Subject: [PATCH 028/125] MDEV-21923: LSN allocation is a bottleneck The parameter innodb_log_spin_wait_delay will be deprecated and ignored, because there is no spin loop anymore. Thanks to commit 685d958e38b825ad9829be311f26729cccf37c46 and commit a635c40648519fd6c3729c9657872a16a0a20821 multiple mtr_t::commit() can concurrently copy their slice of mtr_t::m_log to the shared log_sys.buf. Each writer would allocate their own log sequence number by invoking log_t::append_prepare() while holding a shared log_sys.latch. This function was too heavy, because it would invoke a minimum of 4 atomic read-modify-write operations as well as system calls in the supposedly fast code path. It turns out that with a simpler data structure, instead of having several data fields that needed to be kept consistent with each other, we only need one Atomic_relaxed write_lsn_offset, on which we can operate using fetch_add(), fetch_sub() as well as a single-bit fetch_or(), which reasonably modern compilers (GCC 7, Clang 15 or later) can translate into loop-free code on AMD64. Before anything can be written to the log, log_sys.clear_mmap() must be invoked. log_t::base_lsn: The LSN of the last write_buf() or persist(). This is a rough approximation of log_sys.lsn, which will be removed. log_t::write_lsn_offset: An Atomic_relaxed that buffers updates of write_to_buf and base_lsn. log_t::buf_free, log_t::max_buf_free, log_t::lsn. Remove. Replaced by base_lsn and write_lsn_offset. log_t::buf_size: Always reflects the usable size in append_prepare(). log_t::lsn_lock: Remove. For the memory-mapped log in resize_write(), there will be a resize_wrap_mutex. log_t::get_lsn_approx(): Return a lower bound of get_lsn(). This should be exact unless append_prepare_wait() is pending. log_get_lsn(): A wrapper for log_sys.get_lsn(), which must be invoked while holding an exclusive log_sys.latch. recv_recovery_from_checkpoint_start(): Do not invoke fil_names_clear(); it would seem to be unnecessary. In many places, references to log_sys.get_lsn() are replaced with log_sys.get_flushed_lsn(), which remains a simple std::atomic::load(). Reviewed by: Debarun Banerjee --- extra/mariabackup/xtrabackup.cc | 2 +- .../suite/sys_vars/r/sysvars_innodb.result | 2 +- storage/innobase/buf/buf0buf.cc | 14 +- storage/innobase/buf/buf0dblwr.cc | 5 +- storage/innobase/buf/buf0flu.cc | 43 +-- storage/innobase/handler/ha_innodb.cc | 58 ++-- storage/innobase/include/fil0fil.h | 2 +- storage/innobase/include/log0log.h | 174 +++++------ storage/innobase/include/mtr0mtr.h | 9 +- storage/innobase/log/log0crypt.cc | 2 +- storage/innobase/log/log0log.cc | 206 ++++++------- storage/innobase/log/log0recv.cc | 30 +- storage/innobase/mtr/mtr0mtr.cc | 273 ++++++------------ storage/innobase/row/row0mysql.cc | 4 +- storage/innobase/srv/srv0mon.cc | 8 +- storage/innobase/srv/srv0srv.cc | 6 +- storage/innobase/srv/srv0start.cc | 23 +- 17 files changed, 387 insertions(+), 474 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 837e82d82ca..52c5b5341aa 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2684,7 +2684,7 @@ static bool innodb_init() } recv_sys.lsn= log_sys.next_checkpoint_lsn= - log_sys.get_lsn() - SIZE_OF_FILE_CHECKPOINT; + log_get_lsn() - SIZE_OF_FILE_CHECKPOINT; log_sys.set_latest_format(false); // not encrypted log_hdr_init(); byte *b= &log_hdr_buf[log_t::START_OFFSET]; diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 565a4fbf22c..5810f23f14e 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1092,7 +1092,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL VARIABLE_TYPE INT UNSIGNED -VARIABLE_COMMENT Delay between log buffer spin lock polls (0 to use a blocking latch) +VARIABLE_COMMENT Deprecated parameter with no effect NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 6000 NUMERIC_BLOCK_SIZE 0 diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index f7a9110a7ba..c3da36ede3a 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -558,16 +558,18 @@ buf_page_is_checksum_valid_crc32( } #ifndef UNIV_INNOCHECKSUM -/** Checks whether the lsn present in the page is lesser than the -peek current lsn. -@param check_lsn lsn to check +/** Check whether a page is newer than the durable LSN. +@param check_lsn whether to check the LSN @param read_buf page frame -@return whether the FIL_PAGE_LSN is invalid */ -static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) +@return whether the FIL_PAGE_LSN is invalid (ahead of the durable LSN) */ +static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) noexcept { if (!check_lsn) return false; - lsn_t current_lsn= log_sys.get_lsn(); + /* A page may not be read before it is written, and it may not be + written before the corresponding log has been durably written. + Hence, we refer to the current durable LSN here */ + lsn_t current_lsn= log_sys.get_flushed_lsn(std::memory_order_relaxed); if (UNIV_UNLIKELY(current_lsn == log_sys.FIRST_LSN) && srv_force_recovery == SRV_FORCE_NO_LOG_REDO) return false; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index faea6d113df..7a4b9b6172f 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -365,7 +365,7 @@ void buf_dblwr_t::recover() noexcept ut_ad(log_sys.last_checkpoint_lsn); if (!is_created()) return; - const lsn_t max_lsn{log_sys.get_lsn()}; + const lsn_t max_lsn{log_sys.get_flushed_lsn(std::memory_order_relaxed)}; ut_ad(recv_sys.scanned_lsn == max_lsn); ut_ad(recv_sys.scanned_lsn >= recv_sys.lsn); @@ -780,6 +780,9 @@ void buf_dblwr_t::flush_buffered_writes_completed(const IORequest &request) ut_ad(lsn); ut_ad(lsn >= bpage->oldest_modification()); log_write_up_to(lsn, true); + ut_ad(!e.request.node->space->full_crc32() || + !buf_page_is_corrupted(true, static_cast(frame), + e.request.node->space->flags)); e.request.node->space->io(e.request, bpage->physical_offset(), e_size, frame, bpage); } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 061af1290cf..13eabd92f3b 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -694,7 +694,6 @@ not_compressed: { static_assert(FIL_PAGE_FCRC32_CHECKSUM == 4, "alignment"); mach_write_to_4(tmp + len - 4, my_crc32c(0, tmp, len - 4)); - ut_ad(!buf_page_is_corrupted(true, tmp, space->flags)); } d= tmp; @@ -854,6 +853,8 @@ bool buf_page_t::flush(fil_space_t *space) noexcept if (!space->is_temporary() && !space->is_being_imported() && lsn > log_sys.get_flushed_lsn()) log_write_up_to(lsn, true); + ut_ad(space->is_temporary() || !space->full_crc32() || + !buf_page_is_corrupted(true, write_frame, space->flags)); space->io(IORequest{type, this, slot}, physical_offset(), size, write_frame, this); } @@ -1773,8 +1774,9 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept { ut_ad(!srv_read_only_mode); ut_ad(end_lsn >= next_checkpoint_lsn); - ut_ad(end_lsn <= get_lsn()); - ut_ad(end_lsn + SIZE_OF_FILE_CHECKPOINT <= get_lsn() || + ut_d(const lsn_t current_lsn{get_lsn()}); + ut_ad(end_lsn <= current_lsn); + ut_ad(end_lsn + SIZE_OF_FILE_CHECKPOINT <= current_lsn || srv_shutdown_state > SRV_SHUTDOWN_INITIATED); DBUG_PRINT("ib_log", @@ -1903,7 +1905,8 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept ut_ad(!is_opened()); my_munmap(buf, file_size); buf= resize_buf; - set_buf_free(START_OFFSET + (get_lsn() - resizing)); + buf_size= unsigned(std::min(resize_target - START_OFFSET, + buf_size_max)); } else #endif @@ -2038,9 +2041,9 @@ static bool log_checkpoint() noexcept } /** Make a checkpoint. */ -ATTRIBUTE_COLD void log_make_checkpoint() +ATTRIBUTE_COLD void log_make_checkpoint() noexcept { - buf_flush_wait_flushed(log_sys.get_lsn(std::memory_order_acquire)); + buf_flush_wait_flushed(log_get_lsn()); while (!log_checkpoint()); } @@ -2048,8 +2051,6 @@ ATTRIBUTE_COLD void log_make_checkpoint() NOTE: The calling thread is not allowed to hold any buffer page latches! */ static void buf_flush_wait(lsn_t lsn) noexcept { - ut_ad(lsn <= log_sys.get_lsn()); - lsn_t oldest_lsn; while ((oldest_lsn= buf_pool.get_oldest_modification(lsn)) < lsn) @@ -2258,13 +2259,13 @@ static void buf_flush_sync_for_checkpoint(lsn_t lsn) noexcept mysql_mutex_unlock(&buf_pool.flush_list_mutex); } -/** Check if the adpative flushing threshold is recommended based on +/** Check if the adaptive flushing threshold is recommended based on redo log capacity filled threshold. @param oldest_lsn buf_pool.get_oldest_modification() @return true if adaptive flushing is recommended. */ static bool af_needed_for_redo(lsn_t oldest_lsn) noexcept { - lsn_t age= (log_sys.get_lsn() - oldest_lsn); + lsn_t age= log_sys.get_lsn_approx() - oldest_lsn; lsn_t af_lwm= static_cast(srv_adaptive_flushing_lwm * static_cast(log_sys.log_capacity) / 100); @@ -2324,7 +2325,7 @@ static ulint page_cleaner_flush_pages_recommendation(ulint last_pages_in, lsn_t lsn_rate; ulint n_pages = 0; - const lsn_t cur_lsn = log_sys.get_lsn(); + const lsn_t cur_lsn = log_sys.get_lsn_approx(); ut_ad(oldest_lsn <= cur_lsn); ulint pct_for_lsn = af_get_pct_for_lsn(cur_lsn - oldest_lsn); time_t curr_time = time(nullptr); @@ -2796,7 +2797,7 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool() noexcept NOTE: The calling thread is not allowed to hold any buffer page latches! */ void buf_flush_sync_batch(lsn_t lsn) noexcept { - lsn= std::max(lsn, log_sys.get_lsn()); + lsn= std::max(lsn, log_get_lsn()); mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_wait(lsn); mysql_mutex_unlock(&buf_pool.flush_list_mutex); @@ -2815,20 +2816,26 @@ void buf_flush_sync() noexcept thd_wait_begin(nullptr, THD_WAIT_DISKIO); tpool::tpool_wait_begin(); - mysql_mutex_lock(&buf_pool.flush_list_mutex); - for (;;) + log_sys.latch.wr_lock(SRW_LOCK_CALL); + + for (lsn_t lsn= log_sys.get_lsn();;) { - const lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_wait(lsn); /* Wait for the page cleaner to be idle (for log resizing at startup) */ while (buf_flush_sync_lsn) my_cond_wait(&buf_pool.done_flush_list, &buf_pool.flush_list_mutex.m_mutex); - if (lsn == log_sys.get_lsn()) + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t new_lsn= log_sys.get_lsn(); + if (lsn == new_lsn) break; + lsn= new_lsn; } - mysql_mutex_unlock(&buf_pool.flush_list_mutex); + log_sys.latch.wr_unlock(); tpool::tpool_wait_end(); thd_wait_end(nullptr); } @@ -2848,7 +2855,7 @@ ATTRIBUTE_COLD void buf_pool_t::print_flush_info() const noexcept "-------------------", lru_size, free_size, dirty_size, dirty_pct); - lsn_t lsn= log_sys.get_lsn(); + lsn_t lsn= log_get_lsn(); lsn_t clsn= log_sys.last_checkpoint_lsn; sql_print_information("InnoDB: LSN flush parameters\n" "-------------------\n" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index fc44c404d56..a2b75f43291 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -243,11 +243,11 @@ static void innodb_max_purge_lag_wait_update(THD *thd, st_mysql_sys_var *, if (thd_kill_level(thd)) break; /* Adjust for purge_coordinator_state::refresh() */ - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); const lsn_t last= log_sys.last_checkpoint_lsn, max_age= log_sys.max_checkpoint_age; - log_sys.latch.rd_unlock(); const lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); if ((lsn - last) / 4 >= max_age / 5) buf_flush_ahead(last + max_age / 5, false); purge_sys.wake_if_not_active(); @@ -1155,7 +1155,7 @@ innobase_rollback_to_savepoint_can_release_mdl( be rolled back to savepoint */ /** Request notification of log writes */ -static void innodb_log_flush_request(void *cookie); +static void innodb_log_flush_request(void *cookie) noexcept; /** Requests for log flushes */ struct log_flush_request @@ -4749,11 +4749,13 @@ void log_flush_notify(lsn_t flush_lsn) We put the request in a queue, so that we can notify upper layer about checkpoint complete when we have flushed the redo log. If we have already flushed all relevant redo log, we notify immediately.*/ -static void innodb_log_flush_request(void *cookie) +static void innodb_log_flush_request(void *cookie) noexcept { + log_sys.latch.wr_lock(SRW_LOCK_CALL); lsn_t flush_lsn= log_sys.get_flushed_lsn(); /* Load lsn relaxed after flush_lsn was loaded from the same cache line */ const lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); if (flush_lsn >= lsn) /* All log is already persistent. */; @@ -18440,11 +18442,16 @@ checkpoint_now_set(THD* thd, st_mysql_sys_var*, void*, const void *save) const auto size= log_sys.is_encrypted() ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT; mysql_mutex_unlock(&LOCK_global_system_variables); - lsn_t lsn; - while (!thd_kill_level(thd) && - log_sys.last_checkpoint_lsn.load(std::memory_order_acquire) + size < - (lsn= log_sys.get_lsn(std::memory_order_acquire))) + while (!thd_kill_level(thd)) + { + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t cp= log_sys.last_checkpoint_lsn.load(std::memory_order_relaxed), + lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + if (cp + size >= lsn) + break; log_make_checkpoint(); + } mysql_mutex_lock(&LOCK_global_system_variables); } @@ -18664,35 +18671,23 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, mysql_mutex_unlock(&buf_pool.flush_list_mutex); if (!resizing || !log_sys.resize_running(thd)) break; - if (resizing > log_sys.get_lsn()) + log_sys.latch.wr_lock(SRW_LOCK_CALL); + while (resizing > log_sys.get_lsn()) { ut_ad(!log_sys.is_mmap()); /* The server is almost idle. Write dummy FILE_CHECKPOINT records to ensure that the log resizing will complete. */ - log_sys.latch.wr_lock(SRW_LOCK_CALL); - while (resizing > log_sys.get_lsn()) - { - mtr_t mtr; - mtr.start(); - mtr.commit_files(log_sys.last_checkpoint_lsn); - } - log_sys.latch.wr_unlock(); + mtr_t mtr; + mtr.start(); + mtr.commit_files(log_sys.last_checkpoint_lsn); } + log_sys.latch.wr_unlock(); } } } mysql_mutex_lock(&LOCK_global_system_variables); } -static void innodb_log_spin_wait_delay_update(THD *, st_mysql_sys_var*, - void *, const void *save) -{ - log_sys.latch.wr_lock(SRW_LOCK_CALL); - mtr_t::spin_wait_delay= *static_cast(save); - mtr_t::finisher_update(); - log_sys.latch.wr_unlock(); -} - /** Update innodb_status_output or innodb_status_output_locks, which control InnoDB "status monitor" output to the error log. @param[out] var current value @@ -19551,11 +19546,12 @@ static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size, nullptr, innodb_log_file_size_update, 96 << 20, 4 << 20, std::numeric_limits::max(), 4096); -static MYSQL_SYSVAR_UINT(log_spin_wait_delay, mtr_t::spin_wait_delay, - PLUGIN_VAR_OPCMDARG, - "Delay between log buffer spin lock polls (0 to use a blocking latch)", - nullptr, innodb_log_spin_wait_delay_update, - 0, 0, 6000, 0); +static uint innodb_log_spin_wait_delay; + +static MYSQL_SYSVAR_UINT(log_spin_wait_delay, innodb_log_spin_wait_delay, + PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_DEPRECATED, + "Deprecated parameter with no effect", + nullptr, nullptr, 0, 0, 6000, 0); static MYSQL_SYSVAR_UINT(old_blocks_pct, innobase_old_blocks_pct, PLUGIN_VAR_RQCMDARG, diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index c3514c96cf7..689fbdbdbc7 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -351,7 +351,7 @@ struct fil_space_t final /** fil_system.spaces chain node */ fil_space_t *hash= nullptr; /** log_sys.get_lsn() of the most recent fil_names_write_if_was_clean(). - Reset to 0 by fil_names_clear(). Protected by log_sys.mutex. + Reset to 0 by fil_names_clear(). Protected by log_sys.latch_have_wr(). If and only if this is nonzero, the tablespace will be in named_spaces. */ lsn_t max_lsn= 0; /** base node for the chain of data files; multiple entries are diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 33df9d39ba6..55ffbc0df99 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -64,20 +64,19 @@ void log_write_up_to(lsn_t lsn, bool durable, /** Write to the log file up to the last log entry. @param durable whether to wait for a durable write to complete */ -void log_buffer_flush_to_disk(bool durable= true); - +void log_buffer_flush_to_disk(bool durable= true) noexcept; /** Prepare to invoke log_write_and_flush(), before acquiring log_sys.latch. */ -ATTRIBUTE_COLD void log_write_and_flush_prepare(); +ATTRIBUTE_COLD void log_write_and_flush_prepare() noexcept; /** Durably write the log up to log_sys.get_lsn(). */ -ATTRIBUTE_COLD void log_write_and_flush(); +ATTRIBUTE_COLD void log_write_and_flush() noexcept; /** Make a checkpoint */ -ATTRIBUTE_COLD void log_make_checkpoint(); +ATTRIBUTE_COLD void log_make_checkpoint() noexcept; /** Make a checkpoint at the latest lsn on shutdown. */ -ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown(); +ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() noexcept; /******************************************************//** Prints info of the log. */ @@ -167,40 +166,35 @@ struct log_t static constexpr lsn_t FIRST_LSN= START_OFFSET; private: - /** the lock bit in buf_free */ - static constexpr size_t buf_free_LOCK= ~(~size_t{0} >> 1); + /** the least significant bit of the write_to_buf buffer */ + static constexpr size_t WRITE_TO_BUF_SHIFT{34}; + /** write_lsn_offset component for incrementing write_to_buf */ + static constexpr uint64_t WRITE_TO_BUF{1ULL << WRITE_TO_BUF_SHIFT}; + /** write_lsn_offset flag to indicate that append_prepare_wait() is active */ + static constexpr uint64_t WRITE_BACKOFF{1ULL << 33}; + + /** The current log sequence number, relative to base_lsn, and flags; + may be modified while latch_have_any() */ alignas(CPU_LEVEL1_DCACHE_LINESIZE) - /** first free offset within buf used; - the most significant bit is set by lock_lsn() to protect this field - as well as write_to_buf, waits */ - std::atomic buf_free; -public: - /** number of write requests (to buf); protected by lock_lsn() or lsn_lock */ - size_t write_to_buf; - /** log record buffer, written to by mtr_t::commit() */ - byte *buf; -private: - /** The log sequence number of the last change of durable InnoDB files; - protected by lock_lsn() or lsn_lock or latch.wr_lock() */ - std::atomic lsn; + Atomic_relaxed write_lsn_offset; + /** the LSN of the last write_buf() or persist(); protected by latch */ + std::atomic base_lsn; /** the first guaranteed-durable log sequence number */ std::atomic flushed_to_disk_lsn; public: - /** number of append_prepare_wait(); protected by lock_lsn() or lsn_lock */ - size_t waits; - /** innodb_log_buffer_size (size of buf,flush_buf if !is_mmap(), in bytes) */ + /** innodb_log_buffer_size (usable append_prepare() size in bytes) */ unsigned buf_size; /** log file size in bytes, including the header */ lsn_t file_size; #ifdef LOG_LATCH_DEBUG typedef srw_lock_debug log_rwlock; - typedef srw_mutex log_lsn_lock; bool latch_have_wr() const { return latch.have_wr(); } bool latch_have_rd() const { return latch.have_rd(); } bool latch_have_any() const { return latch.have_any(); } #else + typedef srw_lock log_rwlock; # ifndef UNIV_DEBUG # elif defined SUX_LOCK_GENERIC bool latch_have_wr() const { return true; } @@ -211,23 +205,23 @@ public: bool latch_have_rd() const { return latch.is_locked(); } bool latch_have_any() const { return latch.is_locked(); } # endif -# ifdef __aarch64__ - /* On ARM, we spin more */ - typedef srw_spin_lock log_rwlock; - typedef pthread_mutex_wrapper log_lsn_lock; -# else - typedef srw_lock log_rwlock; - typedef srw_mutex log_lsn_lock; -# endif #endif - /** exclusive latch for checkpoint, shared for mtr_t::commit() to buf */ - alignas(CPU_LEVEL1_DCACHE_LINESIZE) log_rwlock latch; + /** latch_have_wr() for checkpoint, latch_have_any() for append_prepare() */ + log_rwlock latch; + + /** log record buffer, written to by mtr_t::commit() */ + alignas(CPU_LEVEL1_DCACHE_LINESIZE) byte *buf; + + /** number of write requests to buf, + excluding (write_lsn_offset & WRITE_TO_BUF); + protected by latch.wr_lock() */ + size_t write_to_buf; /** number of writes from buf or flush_buf to log; protected by latch.wr_lock() */ - ulint write_to_log; + size_t write_to_log; - /** Last written LSN */ + /** Last written LSN; protected by latch */ lsn_t write_lsn; /** Buffer for writing data to ib_logfile0, or nullptr if is_mmap(). @@ -241,8 +235,6 @@ public: Atomic_relaxed checkpoint_pending; /** next checkpoint number (protected by latch.wr_lock()) */ byte next_checkpoint_no; - /** recommended maximum buf_free size, after which the buffer is flushed */ - unsigned max_buf_free; /** Log sequence number when a log file overwrite (broken crash recovery) was noticed. Protected by latch.wr_lock(). */ lsn_t overwrite_warned; @@ -266,12 +258,6 @@ private: /** Buffer for writing to resize_log; @see flush_buf */ byte *resize_flush_buf; - /** Special implementation of lock_lsn() for IA-32 and AMD64 */ - void lsn_lock_bts() noexcept; - /** Acquire a lock for updating buf_free and related fields. - @return the value of buf_free */ - size_t lock_lsn() noexcept; - /** log sequence number when log resizing was initiated; 0 if the log is not being resized, 1 if resize_start() is in progress */ std::atomic resize_lsn; @@ -327,34 +313,24 @@ public: private: /** the thread that initiated resize_lsn() */ Atomic_relaxed resize_initiator; - /** A lock when the spin-only lock_lsn() is not being used */ - log_lsn_lock lsn_lock; +#ifdef HAVE_PMEM + /** mutex protecting wrap-around in resize_write() */ + srw_mutex resize_wrap_mutex; +#endif public: + /** number of long append_prepare_wait(); protected by latch_have_wr() */ + size_t waits; - bool is_initialised() const noexcept { return max_buf_free != 0; } - - /** whether there is capacity in the log buffer */ - bool buf_free_ok() const noexcept - { - ut_ad(!is_mmap()); - return (buf_free.load(std::memory_order_relaxed) & ~buf_free_LOCK) < - max_buf_free; - } - + bool is_initialised() const noexcept + { return base_lsn.load(std::memory_order_relaxed) != 0; } inline void set_recovered() noexcept; - void set_buf_free(size_t f) noexcept - { ut_ad(f < buf_free_LOCK); buf_free.store(f, std::memory_order_relaxed); } - bool is_mmap() const noexcept { return !flush_buf; } /** @return whether a handle to the log is open; is_mmap() && !is_opened() holds for PMEM */ bool is_opened() const noexcept { return log.is_opened(); } - /** @return target write LSN to react on !buf_free_ok() */ - inline lsn_t get_write_target() const; - /** @return LSN at which log resizing was started and is still in progress @retval 0 if no log resizing is in progress @retval 1 if resize_start() is in progress */ @@ -407,53 +383,64 @@ public: { return resize_buf + resize_target; } /** Initialise the redo log subsystem. */ - void create(); + void create() noexcept; /** Attach a log file. @return whether the memory allocation succeeded */ - bool attach(log_file_t file, os_offset_t size); + bool attach(log_file_t file, os_offset_t size) noexcept; /** Disable memory-mapped access (update log_mmap) */ - void clear_mmap(); - void close_file(bool really_close= true); + void clear_mmap() noexcept; + void close_file(bool really_close= true) noexcept; #if defined __linux__ || defined _WIN32 /** Try to enable or disable file system caching (update log_buffered) */ - void set_buffered(bool buffered); + void set_buffered(bool buffered) noexcept; #endif /** Calculate the checkpoint safety margins. */ - static void set_capacity(); + static void set_capacity() noexcept; /** Write a log file header. @param buf log header buffer @param lsn log sequence number corresponding to log_sys.START_OFFSET @param encrypted whether the log is encrypted */ - static void header_write(byte *buf, lsn_t lsn, bool encrypted); + static void header_write(byte *buf, lsn_t lsn, bool encrypted) noexcept; - lsn_t get_lsn(std::memory_order order= std::memory_order_relaxed) const - { return lsn.load(order); } + /** @return a lower bound estimate of get_lsn(), + using acquire-release ordering with write_buf() or persist(); + this is exact unless append_prepare_wait() is pending */ + lsn_t get_lsn_approx() const noexcept + { + /* acquire-release ordering with write_buf() and persist() */ + lsn_t lsn= base_lsn.load(std::memory_order_acquire); + lsn += write_lsn_offset.load(std::memory_order_relaxed) & + (WRITE_BACKOFF - 1); + return lsn; + } + + /** @return the current log sequence number (logical time stamp) */ + lsn_t get_lsn() const noexcept + { + ut_ad(latch_have_wr()); + return base_lsn.load(std::memory_order_relaxed) + + (write_lsn_offset & (WRITE_BACKOFF - 1)); + } lsn_t get_flushed_lsn(std::memory_order order= std::memory_order_acquire) const noexcept { return flushed_to_disk_lsn.load(order); } /** Initialize the LSN on initial log file creation. */ - lsn_t init_lsn() noexcept - { - latch.wr_lock(SRW_LOCK_CALL); - const lsn_t lsn{get_lsn()}; - flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); - write_lsn= lsn; - latch.wr_unlock(); - return lsn; - } + inline lsn_t init_lsn() noexcept; void set_recovered_lsn(lsn_t lsn) noexcept { ut_ad(latch_have_wr()); - write_lsn= lsn; - this->lsn.store(lsn, std::memory_order_relaxed); + uint64_t lsn_offset= ((write_size - 1) & (lsn - first_lsn)); + write_lsn_offset= lsn_offset; + base_lsn.store(lsn - lsn_offset, std::memory_order_relaxed); flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn= lsn; } #ifdef HAVE_PMEM @@ -491,22 +478,16 @@ private: void writer_update(bool resizing) noexcept; /** Wait in append_prepare() for buffer to become available - @tparam spin whether to use the spin-only lock_lsn() - @param b the value of buf_free - @param ex whether log_sys.latch is exclusively locked - @param lsn log sequence number to write up to - @return the new value of buf_free */ - template - ATTRIBUTE_COLD size_t append_prepare_wait(size_t b, bool ex, lsn_t lsn) - noexcept; + @param late whether the WRITE_BACKOFF flag had already been set + @param ex whether log_sys.latch is exclusively locked */ + ATTRIBUTE_COLD void append_prepare_wait(bool late, bool ex) noexcept; public: /** Reserve space in the log buffer for appending data. - @tparam spin whether to use the spin-only lock_lsn() @tparam mmap log_sys.is_mmap() @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ - template + template std::pair append_prepare(size_t size, bool ex) noexcept; /** Append a string of bytes to the redo log. @@ -577,7 +558,10 @@ extern log_t log_sys; /** Wait for a log checkpoint if needed. NOTE that this function may only be called while not holding any synchronization objects except dict_sys.latch. */ -void log_free_check(); +void log_free_check() noexcept; + +/** @return the current log sequence number (may be stale) */ +lsn_t log_get_lsn() noexcept; /** Release the latches that protect log resizing. */ -void log_resize_release(); +void log_resize_release() noexcept; diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index d6dd203cb10..1ba52a00c7e 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -700,19 +700,19 @@ private: @param mtr mini-transaction @param lsns {start_lsn,flush_ahead} */ template - static void commit_log(mtr_t *mtr, std::pair lsns); + static void commit_log(mtr_t *mtr, std::pair lsns) + noexcept; /** Append the redo log records to the redo log buffer. @return {start_lsn,flush_ahead} */ std::pair do_write(); /** Append the redo log records to the redo log buffer. - @tparam spin whether to use the spin-only log_sys.lock_lsn() @tparam mmap log_sys.is_mmap() @param mtr mini-transaction @param len number of bytes to write @return {start_lsn,flush_ahead} */ - template static + template static std::pair finish_writer(mtr_t *mtr, size_t len); /** The applicable variant of commit_log() */ @@ -723,9 +723,6 @@ private: std::pair finish_write(size_t len) { return finisher(this, len); } public: - /** Poll interval in log_sys.lock_lsn(); 0 to use log_sys.lsn_lock. - Protected by LOCK_global_system_variables and log_sys.latch. */ - static unsigned spin_wait_delay; /** Update finisher when spin_wait_delay is changing to or from 0. */ static void finisher_update(); private: diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index 8a7714101ba..92eaf4ce034 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -566,7 +566,7 @@ ATTRIBUTE_NOINLINE void mtr_t::encrypt() alignas(8) byte iv[MY_AES_BLOCK_SIZE]; - m_commit_lsn= log_sys.get_lsn(); + m_commit_lsn= log_sys.get_flushed_lsn(); ut_ad(m_commit_lsn); byte *tmp= static_cast(alloca(srv_page_size)), *t= tmp; byte *dst= static_cast(alloca(srv_page_size)); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 268871cc5af..22994e389c9 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -68,7 +68,7 @@ log_t log_sys; #define LOG_BUF_FLUSH_MARGIN ((4 * 4096) /* cf. log_t::append_prepare() */ \ + (4U << srv_page_size_shift)) -void log_t::set_capacity() +void log_t::set_capacity() noexcept { ut_ad(log_sys.latch_have_wr()); /* Margin for the free space in the smallest log, before a new query @@ -87,13 +87,15 @@ void log_t::set_capacity() log_sys.max_checkpoint_age = margin; } -void log_t::create() +void log_t::create() noexcept { ut_ad(this == &log_sys); ut_ad(!is_initialised()); + latch.SRW_LOCK_INIT(log_latch_key); + write_lsn_offset= 0; /* LSN 0 and 1 are reserved; @see buf_page_t::oldest_modification_ */ - lsn.store(FIRST_LSN, std::memory_order_relaxed); + base_lsn.store(FIRST_LSN, std::memory_order_relaxed); flushed_to_disk_lsn.store(FIRST_LSN, std::memory_order_relaxed); need_checkpoint.store(true, std::memory_order_relaxed); write_lsn= FIRST_LSN; @@ -102,10 +104,10 @@ void log_t::create() ut_ad(!buf); ut_ad(!flush_buf); ut_ad(!writer); - max_buf_free= 1; - latch.SRW_LOCK_INIT(log_latch_key); - lsn_lock.init(); +#ifdef HAVE_PMEM + resize_wrap_mutex.init(); +#endif last_checkpoint_lsn= FIRST_LSN; log_capacity= 0; @@ -114,8 +116,6 @@ void log_t::create() next_checkpoint_lsn= 0; checkpoint_pending= false; - set_buf_free(0); - ut_ad(is_initialised()); } @@ -306,7 +306,7 @@ remap: #if defined __linux__ || defined _WIN32 /** Display a message about opening the log */ -ATTRIBUTE_COLD static void log_file_message() +ATTRIBUTE_COLD static void log_file_message() noexcept { sql_print_information("InnoDB: %s (block size=%u bytes)", log_sys.log_mmap @@ -320,10 +320,10 @@ ATTRIBUTE_COLD static void log_file_message() log_sys.write_size); } #else -static inline void log_file_message() {} +static inline void log_file_message() noexcept {} #endif -bool log_t::attach(log_file_t file, os_offset_t size) +bool log_t::attach(log_file_t file, os_offset_t size) noexcept { log= file; ut_ad(!size || size >= START_OFFSET + SIZE_OF_FILE_CHECKPOINT); @@ -352,7 +352,6 @@ bool log_t::attach(log_file_t file, os_offset_t size) } # endif buf= static_cast(ptr); - max_buf_free= 1; writer_update(false); # ifdef HAVE_PMEM if (is_pmem) @@ -366,7 +365,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) if (!buf) { alloc_fail: - max_buf_free= 0; + base_lsn.store(0, std::memory_order_relaxed); sql_print_error("InnoDB: Cannot allocate memory;" " too large innodb_log_buffer_size?"); return false; @@ -394,7 +393,6 @@ bool log_t::attach(log_file_t file, os_offset_t size) TRASH_ALLOC(buf, buf_size); TRASH_ALLOC(flush_buf, buf_size); - max_buf_free= buf_size / LOG_BUF_FLUSH_RATIO - LOG_BUF_FLUSH_MARGIN; writer_update(false); memset_aligned<512>(checkpoint_buf, 0, write_size); @@ -407,7 +405,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) @param buf log header buffer @param lsn log sequence number corresponding to log_sys.START_OFFSET @param encrypted whether the log is encrypted */ -void log_t::header_write(byte *buf, lsn_t lsn, bool encrypted) +void log_t::header_write(byte *buf, lsn_t lsn, bool encrypted) noexcept { mach_write_to_4(my_assume_aligned<4>(buf) + LOG_HEADER_FORMAT, log_sys.FORMAT_10_8); @@ -436,8 +434,9 @@ void log_t::create(lsn_t lsn) noexcept ut_ad(is_latest()); ut_ad(this == &log_sys); - this->lsn.store(lsn, std::memory_order_relaxed); - this->flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn_offset= 0; + base_lsn.store(lsn, std::memory_order_relaxed); + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); first_lsn= lsn; write_lsn= lsn; @@ -452,14 +451,13 @@ void log_t::create(lsn_t lsn) noexcept mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); memset_aligned<4096>(buf, 0, 4096); log_sys.header_write(buf, lsn, is_encrypted()); - set_buf_free(START_OFFSET); pmem_persist(buf, 512); + buf_size= unsigned(std::min(capacity(), buf_size_max)); } else #endif { ut_ad(!is_mmap()); - set_buf_free(0); memset_aligned<4096>(flush_buf, 0, buf_size); memset_aligned<4096>(buf, 0, buf_size); log_sys.header_write(buf, lsn, is_encrypted()); @@ -468,12 +466,12 @@ void log_t::create(lsn_t lsn) noexcept } } -ATTRIBUTE_COLD static void log_close_failed(dberr_t err) +ATTRIBUTE_COLD static void log_close_failed(dberr_t err) noexcept { ib::fatal() << "closing ib_logfile0 failed: " << err; } -void log_t::close_file(bool really_close) +void log_t::close_file(bool really_close) noexcept { if (is_mmap()) { @@ -508,16 +506,25 @@ void log_t::close_file(bool really_close) log_close_failed(err); } +/** @return the current log sequence number (may be stale) */ +lsn_t log_get_lsn() noexcept +{ + log_sys.latch.wr_lock(SRW_LOCK_CALL); + lsn_t lsn= log_sys.get_lsn(); + log_sys.latch.wr_unlock(); + return lsn; +} + /** Acquire all latches that protect the log. */ -static void log_resize_acquire() +static void log_resize_acquire() noexcept { #ifdef HAVE_PMEM if (!log_sys.is_mmap()) #endif { - while (flush_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (flush_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); - while (write_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (write_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); } @@ -525,7 +532,7 @@ static void log_resize_acquire() } /** Release the latches that protect the log. */ -void log_resize_release() +void log_resize_release() noexcept { log_sys.latch.wr_unlock(); @@ -542,7 +549,7 @@ void log_resize_release() #if defined __linux__ || defined _WIN32 /** Try to enable or disable file system caching (update log_buffered) */ -void log_t::set_buffered(bool buffered) +void log_t::set_buffered(bool buffered) noexcept { if (!log_maybe_unbuffered || #ifdef HAVE_PMEM @@ -887,9 +894,7 @@ void log_t::persist(lsn_t lsn) noexcept ut_ad(!is_opened()); ut_ad(!write_lock.is_owner()); ut_ad(!flush_lock.is_owner()); -#ifdef LOG_LATCH_DEBUG - ut_ad(latch_have_any()); -#endif + ut_ad(latch_have_wr()); lsn_t old= flushed_to_disk_lsn.load(std::memory_order_relaxed); @@ -907,26 +912,26 @@ void log_t::persist(lsn_t lsn) noexcept else pmem_persist(buf + start, end - start); - old= flushed_to_disk_lsn.load(std::memory_order_relaxed); - - if (old < lsn) - { - while (!flushed_to_disk_lsn.compare_exchange_weak - (old, lsn, std::memory_order_release, std::memory_order_relaxed)) - if (old >= lsn) - break; - - log_flush_notify(lsn); - DBUG_EXECUTE_IF("crash_after_log_write_upto", DBUG_SUICIDE();); - } + uint64_t offset{write_lsn_offset}; + const lsn_t new_base_lsn= base_lsn.load(std::memory_order_relaxed) + + (offset & (WRITE_BACKOFF - 1)); + ut_ad(new_base_lsn >= lsn); + write_to_buf+= size_t(offset >> WRITE_TO_BUF_SHIFT); + /* This synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + write_lsn_offset.store(0, std::memory_order_relaxed); + base_lsn.store(new_base_lsn, std::memory_order_release); + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + log_flush_notify(lsn); + DBUG_EXECUTE_IF("crash_after_log_write_upto", DBUG_SUICIDE();); } ATTRIBUTE_NOINLINE static void log_write_persist(lsn_t lsn) noexcept { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); log_sys.persist(lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); } #endif @@ -977,7 +982,7 @@ lsn_t log_t::write_buf() noexcept ut_ad(resizing == RETAIN_LATCH || (resizing == RESIZING) == (resize_in_progress() > 1)); - const lsn_t lsn{get_lsn(std::memory_order_relaxed)}; + const lsn_t lsn{get_lsn()}; if (write_lsn >= lsn) { @@ -993,7 +998,8 @@ lsn_t log_t::write_buf() noexcept ut_ad(write_lsn >= get_flushed_lsn()); const size_t write_size_1{write_size - 1}; ut_ad(ut_is_2pow(write_size)); - size_t length{buf_free.load(std::memory_order_relaxed)}; + lsn_t base= base_lsn.load(std::memory_order_relaxed); + size_t length{size_t(lsn - base)}; lsn_t offset{calc_lsn_offset(write_lsn)}; ut_ad(length >= (offset & write_size_1)); ut_ad(write_size_1 >= 511); @@ -1015,14 +1021,8 @@ lsn_t log_t::write_buf() noexcept { ut_ad(!((length ^ (size_t(lsn) - size_t(first_lsn))) & write_size_1)); /* Keep filling the same buffer until we have more than one block. */ -#if 0 /* TODO: Pad the last log block with dummy records. */ - buf_free= log_pad(lsn, (write_size_1 + 1) - length, - buf + length, flush_buf); - ... /* TODO: Update the LSN and adjust other code. */ -#else MEM_MAKE_DEFINED(buf + length, (write_size_1 + 1) - length); buf[length]= 0; /* ensure that recovery catches EOF */ -#endif if (UNIV_LIKELY_NULL(re_write_buf)) { MEM_MAKE_DEFINED(re_write_buf + length, (write_size_1 + 1) - length); @@ -1033,8 +1033,13 @@ lsn_t log_t::write_buf() noexcept else { const size_t new_buf_free{length & write_size_1}; + base+= length & ~write_size_1; ut_ad(new_buf_free == ((lsn - first_lsn) & write_size_1)); - buf_free.store(new_buf_free, std::memory_order_relaxed); + write_to_buf+= size_t(write_lsn_offset >> WRITE_TO_BUF_SHIFT); + /* This synchronizes with get_lsn_approx(); + we must store write_lsn_offset before base_lsn. */ + write_lsn_offset.store(new_buf_free, std::memory_order_relaxed); + base_lsn.store(base, std::memory_order_release); if (new_buf_free) { @@ -1063,7 +1068,9 @@ lsn_t log_t::write_buf() noexcept std::swap(resize_buf, resize_flush_buf); } + ut_ad(base + (write_lsn_offset & (WRITE_TO_BUF - 1)) == lsn); write_to_log++; + if (resizing != RETAIN_LATCH) latch.wr_unlock(); @@ -1107,7 +1114,7 @@ bool log_t::flush(lsn_t lsn) noexcept @retval 0 if there are no pending callbacks on flush_lock or there is another group commit lead. */ -static lsn_t log_flush(lsn_t lsn) +static lsn_t log_flush(lsn_t lsn) noexcept { ut_ad(!log_sys.is_mmap()); ut_a(log_sys.flush(lsn)); @@ -1126,7 +1133,7 @@ wait and check if an already running write is covering the request. void log_write_up_to(lsn_t lsn, bool durable, const completion_callback *callback) noexcept { - ut_ad(!srv_read_only_mode || log_sys.buf_free_ok()); + ut_ad(!srv_read_only_mode); ut_ad(lsn != LSN_MAX); ut_ad(lsn != 0); ut_ad(!log_sys.is_mmap() || !callback || durable); @@ -1139,8 +1146,6 @@ void log_write_up_to(lsn_t lsn, bool durable, return; } - ut_ad(lsn <= log_sys.get_lsn()); - #ifdef HAVE_PMEM if (log_sys.is_mmap()) { @@ -1157,10 +1162,10 @@ repeat: if (flush_lock.acquire(lsn, callback) != group_commit_lock::ACQUIRED) return; /* Promise to other concurrent flush_lock.acquire() that we - will durable at least up to the current LSN. The LSN may still - advance until we acquire log_sys.latch below. */ - lsn= log_sys.get_lsn(); - flush_lock.set_pending(lsn); + will be durable at least up to the current LSN. The LSN may still + advance when we acquire log_sys.latch below. */ + if (lsn > log_sys.get_flushed_lsn()) + flush_lock.set_pending(lsn); } lsn_t pending_write_lsn= 0, pending_flush_lsn= 0; @@ -1206,33 +1211,41 @@ void log_t::writer_update(bool resizing) noexcept /** Write to the log file up to the last log entry. @param durable whether to wait for a durable write to complete */ -void log_buffer_flush_to_disk(bool durable) +void log_buffer_flush_to_disk(bool durable) noexcept { - log_write_up_to(log_sys.get_lsn(std::memory_order_acquire), durable); + log_write_up_to(log_get_lsn(), durable); } /** Prepare to invoke log_write_and_flush(), before acquiring log_sys.latch. */ -ATTRIBUTE_COLD void log_write_and_flush_prepare() +ATTRIBUTE_COLD void log_write_and_flush_prepare() noexcept { #ifdef HAVE_PMEM if (log_sys.is_mmap()) return; #endif - while (flush_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (flush_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); - while (write_lock.acquire(log_sys.get_lsn() + 1, nullptr) != + while (write_lock.acquire(log_get_lsn() + 1, nullptr) != group_commit_lock::ACQUIRED); } -void log_t::clear_mmap() +void log_t::clear_mmap() noexcept { - if (!is_mmap() || -#ifdef HAVE_PMEM - !is_opened() || -#endif - high_level_read_only) + if (!is_mmap() || high_level_read_only) return; +#ifdef HAVE_PMEM + if (!is_opened()) + { + latch.wr_lock(SRW_LOCK_CALL); + ut_ad(!resize_in_progress()); + ut_ad(get_lsn() == get_flushed_lsn(std::memory_order_relaxed)); + buf_size= unsigned(std::min(capacity(), buf_size_max)); + latch.wr_unlock(); + return; + } +#endif + log_resize_acquire(); ut_ad(!resize_in_progress()); ut_ad(write_lsn == get_lsn()); @@ -1242,10 +1255,10 @@ void log_t::clear_mmap() { alignas(16) byte log_block[4096]; const size_t bs{write_size}; - const size_t bf{buf_free.load(std::memory_order_relaxed)}; { - byte *const b= buf; - memcpy_aligned<16>(log_block, b + (bf & ~(bs - 1)), bs); + const size_t bf= + size_t(write_lsn - base_lsn.load(std::memory_order_relaxed)); + memcpy_aligned<16>(log_block, buf + (bf & ~(bs - 1)), bs); } close_file(false); @@ -1253,14 +1266,13 @@ void log_t::clear_mmap() ut_a(attach(log, file_size)); ut_ad(!is_mmap()); - set_buf_free(bf & (bs - 1)); - memcpy_aligned<16>(log_sys.buf, log_block, bs); + memcpy_aligned<16>(buf, log_block, bs); } log_resize_release(); } /** Durably write the log up to log_sys.get_lsn(). */ -ATTRIBUTE_COLD void log_write_and_flush() +ATTRIBUTE_COLD void log_write_and_flush() noexcept { ut_ad(!srv_read_only_mode); #ifdef HAVE_PMEM @@ -1280,17 +1292,17 @@ Tries to establish a big enough margin of free space in the log, such that a new log entry can be catenated without an immediate need for a checkpoint. NOTE: this function may only be called if the calling thread owns no synchronization objects! */ -ATTRIBUTE_COLD static void log_checkpoint_margin() +ATTRIBUTE_COLD static void log_checkpoint_margin() noexcept { while (log_sys.check_for_checkpoint()) { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); ut_ad(!recv_no_log_write); if (!log_sys.check_for_checkpoint()) { func_exit: - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); return; } @@ -1308,7 +1320,7 @@ func_exit: } DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", goto skip_checkpoint;); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); /* We must wait to prevent the tail of the log overwriting the head. */ buf_flush_wait_flushed(std::min(sync_lsn, checkpoint + (1U << 20))); @@ -1320,7 +1332,7 @@ func_exit: /** Wait for a log checkpoint if needed. NOTE that this function may only be called while not holding any synchronization objects except dict_sys.latch. */ -void log_free_check() +void log_free_check() noexcept { ut_ad(!lock_sys.is_holder()); if (log_sys.check_for_checkpoint()) @@ -1337,7 +1349,7 @@ inline void buf_mem_pressure_shutdown() noexcept {} #endif /** Make a checkpoint at the latest lsn on shutdown. */ -ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() +ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() noexcept { lsn_t lsn; ulint count = 0; @@ -1474,7 +1486,7 @@ wait_suspend_loop: ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT; - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); lsn = log_sys.get_lsn(); @@ -1482,7 +1494,7 @@ wait_suspend_loop: && lsn != log_sys.last_checkpoint_lsn + sizeof_cp; ut_ad(lsn >= log_sys.last_checkpoint_lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); if (lsn_changed) { goto loop; @@ -1500,7 +1512,7 @@ wait_suspend_loop: "Free innodb buffer pool"); ut_d(buf_pool.assert_all_freed()); - ut_a(lsn == log_sys.get_lsn() + ut_a(lsn == log_get_lsn() || srv_force_recovery == SRV_FORCE_NO_LOG_REDO); if (UNIV_UNLIKELY(lsn < recv_sys.lsn)) { @@ -1514,7 +1526,7 @@ wait_suspend_loop: /* Make some checks that the server really is quiet */ ut_ad(!srv_any_background_activity()); - ut_a(lsn == log_sys.get_lsn() + ut_a(lsn == log_get_lsn() || srv_force_recovery == SRV_FORCE_NO_LOG_REDO); } @@ -1525,44 +1537,42 @@ log_print( /*======*/ FILE* file) /*!< in: file where to print */ { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); const lsn_t lsn= log_sys.get_lsn(); mysql_mutex_lock(&buf_pool.flush_list_mutex); const lsn_t pages_flushed = buf_pool.get_oldest_modification(lsn); mysql_mutex_unlock(&buf_pool.flush_list_mutex); + const lsn_t flushed_lsn{log_sys.get_flushed_lsn()}; + const lsn_t checkpoint_lsn{log_sys.last_checkpoint_lsn}; + log_sys.latch.wr_unlock(); fprintf(file, "Log sequence number " LSN_PF "\n" "Log flushed up to " LSN_PF "\n" "Pages flushed up to " LSN_PF "\n" "Last checkpoint at " LSN_PF "\n", - lsn, - log_sys.get_flushed_lsn(), - pages_flushed, - lsn_t{log_sys.last_checkpoint_lsn}); - - log_sys.latch.rd_unlock(); + lsn, flushed_lsn, pages_flushed, checkpoint_lsn); } /** Shut down the redo log subsystem. */ void log_t::close() { ut_ad(this == &log_sys); - ut_ad(!(buf_free & buf_free_LOCK)); if (!is_initialised()) return; close_file(); ut_ad(!checkpoint_buf); ut_ad(!buf); ut_ad(!flush_buf); + base_lsn.store(0, std::memory_order_relaxed); latch.destroy(); - lsn_lock.destroy(); +#ifdef HAVE_PMEM + resize_wrap_mutex.destroy(); +#endif recv_sys.close(); - - max_buf_free= 0; } std::string get_log_file_path(const char *filename) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 9a4bac91035..30b99ea0bc7 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1466,6 +1466,7 @@ void recv_sys_t::debug_free() mysql_mutex_lock(&mutex); recovery_on= false; + recv_needed_recovery= false; pages.clear(); pages_it= pages.end(); @@ -1473,7 +1474,6 @@ void recv_sys_t::debug_free() log_sys.clear_mmap(); } - /** Free a redo log snippet. @param data buffer allocated in add() */ inline void recv_sys_t::free(const void *data) @@ -2972,7 +2972,7 @@ restart: l - recs + rlen))) { lsn= start_lsn; - if (lsn > log_sys.get_lsn()) + if (lsn > log_sys.get_flushed_lsn(std::memory_order_relaxed)) log_sys.set_recovered_lsn(start_lsn); l+= rlen; offset= begin.ptr - log_sys.buf; @@ -4582,19 +4582,16 @@ dberr_t recv_recovery_read_checkpoint() inline void log_t::set_recovered() noexcept { ut_ad(get_flushed_lsn() == get_lsn()); - ut_ad(recv_sys.lsn == get_lsn()); - size_t offset{recv_sys.offset}; + ut_ad(recv_sys.lsn == get_flushed_lsn()); if (!is_mmap()) { const size_t bs{log_sys.write_size}, bs_1{bs - 1}; - memmove_aligned<512>(buf, buf + (offset & ~bs_1), bs); - offset&= bs_1; + memmove_aligned<512>(buf, buf + (recv_sys.offset & ~bs_1), bs); } -#ifndef _WIN32 +#ifdef HAVE_PMEM else mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); #endif - set_buf_free(offset); } inline bool recv_sys_t::validate_checkpoint() const noexcept @@ -4668,7 +4665,7 @@ read_only_recovery: goto err_exit; } ut_ad(recv_sys.file_checkpoint); - ut_ad(log_sys.get_lsn() >= recv_sys.scanned_lsn); + ut_ad(log_sys.get_flushed_lsn() >= recv_sys.scanned_lsn); if (rewind) { recv_sys.lsn = log_sys.next_checkpoint_lsn; recv_sys.offset = 0; @@ -4730,7 +4727,7 @@ read_only_recovery: tablespaces (not individual pages), while retaining the initial recv_sys.pages. */ mysql_mutex_lock(&recv_sys.mutex); - ut_ad(log_sys.get_lsn() >= recv_sys.lsn); + ut_ad(log_sys.get_flushed_lsn() >= recv_sys.lsn); recv_sys.clear(); recv_sys.lsn = log_sys.next_checkpoint_lsn; mysql_mutex_unlock(&recv_sys.mutex); @@ -4738,7 +4735,8 @@ read_only_recovery: if (srv_operation <= SRV_OPERATION_EXPORT_RESTORED) { mysql_mutex_lock(&recv_sys.mutex); - deferred_spaces.deferred_dblwr(log_sys.get_lsn()); + deferred_spaces.deferred_dblwr( + log_sys.get_flushed_lsn()); buf_dblwr.recover(); mysql_mutex_unlock(&recv_sys.mutex); } @@ -4771,16 +4769,6 @@ err_exit: if (!srv_read_only_mode && log_sys.is_latest()) { log_sys.set_recovered(); - if (recv_needed_recovery - && srv_operation <= SRV_OPERATION_EXPORT_RESTORED - && recv_sys.lsn - log_sys.next_checkpoint_lsn - < log_sys.log_capacity) { - /* Write a FILE_CHECKPOINT marker as the first thing, - before generating any other redo log. This ensures - that subsequent crash recovery will be possible even - if the server were killed soon after this. */ - fil_names_clear(log_sys.next_checkpoint_lsn); - } } DBUG_EXECUTE_IF("before_final_redo_apply", goto err_exit;); diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index bb632bd66b7..716dac624d5 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -44,7 +44,6 @@ void (*mtr_t::commit_logger)(mtr_t *, std::pair); #endif std::pair (*mtr_t::finisher)(mtr_t *, size_t); -unsigned mtr_t::spin_wait_delay; void mtr_t::finisher_update() { @@ -53,15 +52,12 @@ void mtr_t::finisher_update() if (log_sys.is_mmap()) { commit_logger= mtr_t::commit_log; - finisher= spin_wait_delay - ? mtr_t::finish_writer : mtr_t::finish_writer; + finisher= mtr_t::finish_writer; return; } commit_logger= mtr_t::commit_log; #endif - finisher= - (spin_wait_delay - ? mtr_t::finish_writer : mtr_t::finish_writer); + finisher= mtr_t::finish_writer; } void mtr_memo_slot_t::release() const @@ -257,7 +253,7 @@ static void insert_imported(buf_block_t *block) { if (block->page.oldest_modification() <= 1) { - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); /* For unlogged mtrs (MTR_LOG_NO_REDO), we use the current system LSN. The mtr that generated the LSN is either already committed or in mtr_t::commit. Shared latch and relaxed atomics should be fine here as it is guaranteed @@ -269,7 +265,7 @@ static void insert_imported(buf_block_t *block) mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_pool.insert_into_flush_list (buf_pool.prepare_insert_into_flush_list(lsn), block, lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); mysql_mutex_unlock(&buf_pool.flush_list_mutex); } } @@ -339,24 +335,11 @@ void mtr_t::release() m_memo.clear(); } -inline lsn_t log_t::get_write_target() const -{ - ut_ad(latch_have_any()); - if (UNIV_LIKELY(buf_free_ok())) - return 0; - /* The LSN corresponding to the end of buf is - write_lsn - (first_lsn & 4095) + buf_free, - but we use simpler arithmetics to return a smaller write target in - order to minimize waiting in log_write_up_to(). */ - ut_ad(max_buf_free >= 4096 * 4); - return write_lsn + max_buf_free / 2; -} - template void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) + noexcept { size_t modified= 0; - const lsn_t write_lsn= mmap ? 0 : log_sys.get_write_target(); if (mtr->m_made_dirty) { @@ -475,9 +458,6 @@ void mtr_t::commit_log(mtr_t *mtr, std::pair lsns) if (UNIV_UNLIKELY(lsns.second != PAGE_FLUSH_NO)) buf_flush_ahead(mtr->m_commit_lsn, lsns.second == PAGE_FLUSH_SYNC); - - if (!mmap && UNIV_UNLIKELY(write_lsn != 0)) - log_write_up_to(write_lsn, false); } /** Commit a mini-transaction. */ @@ -690,7 +670,7 @@ bool mtr_t::commit_file(fil_space_t &space, const char *name) /* We will not encrypt any FILE_ records, but we will reserve a nonce at the end. */ size+= 8; - m_commit_lsn= log_sys.get_lsn(); + m_commit_lsn= log_sys.get_flushed_lsn(); } else m_commit_lsn= 0; @@ -775,7 +755,7 @@ ATTRIBUTE_COLD lsn_t mtr_t::commit_files(lsn_t checkpoint_lsn) /* We will not encrypt any FILE_ records, but we will reserve a nonce at the end. */ size+= 8; - m_commit_lsn= log_sys.get_lsn(); + m_commit_lsn= log_sys.get_flushed_lsn(); } else m_commit_lsn= 0; @@ -897,181 +877,110 @@ ATTRIBUTE_COLD static void log_overwrite_warning(lsn_t lsn) ? ". Shutdown is in progress" : ""); } -static ATTRIBUTE_NOINLINE void lsn_delay(size_t delay, size_t mult) noexcept +ATTRIBUTE_COLD void log_t::append_prepare_wait(bool late, bool ex) noexcept { - delay*= mult * 2; // GCC 13.2.0 -O2 targeting AMD64 wants to unroll twice - HMT_low(); - do - MY_RELAX_CPU(); - while (--delay); - HMT_medium(); -} - -#if defined __clang_major__ && __clang_major__ < 10 -/* Only clang-10 introduced support for asm goto */ -#elif defined __APPLE__ -/* At least some versions of Apple Xcode do not support asm goto */ -#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__) -# if SIZEOF_SIZE_T == 8 -# define LOCK_TSET \ - __asm__ goto("lock btsq $63, %0\n\t" "jnc %l1" \ - : : "m"(buf_free) : "cc", "memory" : got) -# else -# define LOCK_TSET \ - __asm__ goto("lock btsl $31, %0\n\t" "jnc %l1" \ - : : "m"(buf_free) : "cc", "memory" : got) -# endif -#elif defined _MSC_VER && (defined _M_IX86 || defined _M_X64) -# if SIZEOF_SIZE_T == 8 -# define LOCK_TSET \ - if (!_interlockedbittestandset64 \ - (reinterpret_cast(&buf_free), 63)) return -# else -# define LOCK_TSET \ - if (!_interlockedbittestandset \ - (reinterpret_cast(&buf_free), 31)) return -# endif -#endif - -#ifdef LOCK_TSET -ATTRIBUTE_NOINLINE -void log_t::lsn_lock_bts() noexcept -{ - LOCK_TSET; + if (UNIV_LIKELY(!ex)) { - const size_t m= mtr_t::spin_wait_delay; - constexpr size_t DELAY= 10, MAX_ITERATIONS= 10; - for (size_t delay_count= DELAY, delay_iterations= 1;; - lsn_delay(delay_iterations, m)) + latch.rd_unlock(); + if (!late) { - if (!(buf_free.load(std::memory_order_relaxed) & buf_free_LOCK)) - LOCK_TSET; - if (!delay_count); - else if (delay_iterations < MAX_ITERATIONS) - delay_count= DELAY, delay_iterations++; - else - delay_count--; + /* Wait for all threads to back off. */ + latch.wr_lock(SRW_LOCK_CALL); + goto got_ex; + } + + const auto delay= my_cpu_relax_multiplier / 4 * srv_spin_wait_delay; + const auto rounds= srv_n_spin_wait_rounds; + + for (;;) + { + HMT_low(); + for (auto r= rounds + 1; r--; ) + { + if (write_lsn_offset.load(std::memory_order_relaxed) & WRITE_BACKOFF) + { + for (auto d= delay; d--; ) + MY_RELAX_CPU(); + } + else + { + HMT_medium(); + goto done; + } + } + HMT_medium(); + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + } + else + { + got_ex: + const uint64_t l= write_lsn_offset.load(std::memory_order_relaxed); + const lsn_t lsn{base_lsn.load(std::memory_order_relaxed)}; + ut_d(lsn_t ll= lsn + (l & (WRITE_BACKOFF - 1))); + ut_ad(is_mmap() + ? ll - get_flushed_lsn(std::memory_order_relaxed) < capacity() + : ll - write_lsn - ((write_size - 1) & (write_lsn - first_lsn)) < + buf_size); + waits++; +#ifdef HAVE_PMEM + const bool is_pmem{is_mmap()}; + if (is_pmem) + persist(lsn + (l & (WRITE_BACKOFF - 1))); +#endif + latch.wr_unlock(); + /* write_buf() or persist() will clear the WRITE_BACKOFF flag, + which our caller will recheck. */ +#ifdef HAVE_PMEM + if (!is_pmem) +#endif + log_write_up_to(lsn + (l & (WRITE_BACKOFF - 1)), false); + if (ex) + { + latch.wr_lock(SRW_LOCK_CALL); + return; } } -# ifdef __GNUC__ - got: - return; -# endif -} - -inline -#else -ATTRIBUTE_NOINLINE -#endif -size_t log_t::lock_lsn() noexcept -{ -#ifdef LOCK_TSET - lsn_lock_bts(); - return ~buf_free_LOCK & buf_free.load(std::memory_order_relaxed); -# undef LOCK_TSET -#else - size_t b= buf_free.fetch_or(buf_free_LOCK, std::memory_order_acquire); - if (b & buf_free_LOCK) - { - const size_t m= mtr_t::spin_wait_delay; - constexpr size_t DELAY= 10, MAX_ITERATIONS= 10; - for (size_t delay_count= DELAY, delay_iterations= 1; - ((b= buf_free.load(std::memory_order_relaxed)) & buf_free_LOCK) || - (buf_free_LOCK & (b= buf_free.fetch_or(buf_free_LOCK, - std::memory_order_acquire))); - lsn_delay(delay_iterations, m)) - if (!delay_count); - else if (delay_iterations < MAX_ITERATIONS) - delay_count= DELAY, delay_iterations++; - else - delay_count--; - } - return b; -#endif -} - -template -ATTRIBUTE_COLD size_t log_t::append_prepare_wait(size_t b, bool ex, lsn_t lsn) - noexcept -{ - waits++; - ut_ad(buf_free.load(std::memory_order_relaxed) == - (spin ? (b | buf_free_LOCK) : b)); - if (spin) - buf_free.store(b, std::memory_order_release); - else - lsn_lock.wr_unlock(); - - if (ex) - latch.wr_unlock(); - else - latch.rd_unlock(); - - log_write_up_to(lsn, is_mmap()); - - if (ex) - latch.wr_lock(SRW_LOCK_CALL); - else - latch.rd_lock(SRW_LOCK_CALL); - - if (spin) - return lock_lsn(); - - lsn_lock.wr_lock(); - return buf_free.load(std::memory_order_relaxed); +done: + latch.rd_lock(SRW_LOCK_CALL); } /** Reserve space in the log buffer for appending data. -@tparam spin whether to use the spin-only lock_lsn() @tparam mmap log_sys.is_mmap() @param size total length of the data to append(), in bytes @param ex whether log_sys.latch is exclusively locked @return the start LSN and the buffer position for append() */ -template +template inline std::pair log_t::append_prepare(size_t size, bool ex) noexcept { ut_ad(ex ? latch_have_wr() : latch_have_rd()); ut_ad(mmap == is_mmap()); - if (!spin) - lsn_lock.wr_lock(); - size_t b{spin ? lock_lsn() : buf_free.load(std::memory_order_relaxed)}; - write_to_buf++; - - lsn_t l{lsn.load(std::memory_order_relaxed)}, end_lsn{l + size}; - - if (UNIV_UNLIKELY(mmap - ? (end_lsn - - get_flushed_lsn(std::memory_order_relaxed)) > capacity() - : b + size >= buf_size)) + ut_ad(!mmap || buf_size == std::min(capacity(), buf_size_max)); + const size_t buf_size{this->buf_size - size}; + uint64_t l; + static_assert(WRITE_TO_BUF == WRITE_BACKOFF << 1, ""); + while (UNIV_UNLIKELY((l= write_lsn_offset.fetch_add(size + WRITE_TO_BUF) & + (WRITE_TO_BUF - 1)) >= buf_size)) { - b= append_prepare_wait(b, ex, l); - /* While flushing log, we had released the lsn lock and LSN could have - progressed in the meantime. */ - l= lsn.load(std::memory_order_relaxed); - end_lsn= l + size; + /* The following is inlined here instead of being part of + append_prepare_wait(), in order to increase the locality of reference + and to set the WRITE_BACKOFF flag as soon as possible. */ + bool late(write_lsn_offset.fetch_or(WRITE_BACKOFF) & WRITE_BACKOFF); + /* Subtract our LSN overshoot. */ + write_lsn_offset.fetch_sub(size); + append_prepare_wait(late, ex); } - size_t new_buf_free= b + size; - if (mmap && new_buf_free >= file_size) - new_buf_free-= size_t(capacity()); - - lsn.store(end_lsn, std::memory_order_relaxed); + const lsn_t lsn{l + base_lsn.load(std::memory_order_relaxed)}, + end_lsn{lsn + size}; if (UNIV_UNLIKELY(end_lsn >= last_checkpoint_lsn + log_capacity)) set_check_for_checkpoint(true); - byte *our_buf= buf; - if (spin) - buf_free.store(new_buf_free, std::memory_order_release); - else - { - buf_free.store(new_buf_free, std::memory_order_relaxed); - lsn_lock.wr_unlock(); - } - - return {l, our_buf + b}; + return {lsn, + buf + size_t(mmap ? FIRST_LSN + (lsn - first_lsn) % capacity() : l)}; } /** Finish appending data to the log. @@ -1216,7 +1125,7 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, if (!resize_flush_buf) { ut_ad(is_mmap()); - lsn_lock.wr_lock(); + resize_wrap_mutex.wr_lock(); const size_t resize_capacity{resize_target - START_OFFSET}; { const lsn_t resizing{resize_in_progress()}; @@ -1227,7 +1136,7 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, if (UNIV_UNLIKELY(lsn < resizing)) { /* This function may execute in multiple concurrent threads - that hold a shared log_sys.latch. Before we got lsn_lock, + that hold a shared log_sys.latch. Before we got resize_wrap_mutex, another thread could have executed resize_lsn.store(lsn) below with a larger lsn than ours. @@ -1277,7 +1186,7 @@ inline void log_t::resize_write(lsn_t lsn, const byte *end, size_t len, ut_ad(resize_buf[s] <= 1); resize_buf[s]= 1; mmap_done: - lsn_lock.wr_unlock(); + resize_wrap_mutex.wr_unlock(); } else #endif @@ -1304,7 +1213,7 @@ inline void log_t::append(byte *&d, const void *s, size_t size) noexcept d+= size; } -template +template std::pair mtr_t::finish_writer(mtr_t *mtr, size_t len) { @@ -1315,7 +1224,7 @@ mtr_t::finish_writer(mtr_t *mtr, size_t len) const size_t size{mtr->m_commit_lsn ? 5U + 8U : 5U}; std::pair start= - log_sys.append_prepare(len, mtr->m_latch_ex); + log_sys.append_prepare(len, mtr->m_latch_ex); if (!mmap) { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 30b4f5975a1..cb06ec9f52a 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -69,7 +69,7 @@ Created 9/17/2000 Heikki Tuuri /** Delay an INSERT, DELETE or UPDATE operation if the purge is lagging. */ -static void row_mysql_delay_if_needed() +static void row_mysql_delay_if_needed() noexcept { const auto delay= srv_dml_needed_delay; if (UNIV_UNLIKELY(delay != 0)) @@ -78,8 +78,8 @@ static void row_mysql_delay_if_needed() log_sys.latch.rd_lock(SRW_LOCK_CALL); const lsn_t last= log_sys.last_checkpoint_lsn, max_age= log_sys.max_checkpoint_age; + const lsn_t lsn= log_sys.get_flushed_lsn(); log_sys.latch.rd_unlock(); - const lsn_t lsn= log_sys.get_lsn(); if ((lsn - last) / 4 >= max_age / 5) buf_flush_ahead(last + max_age / 5, false); purge_sys.wake_if_not_active(); diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index a9bc310753b..ff646ba1b79 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -1454,7 +1454,7 @@ srv_mon_process_existing_counter( /* innodb_os_log_written */ case MONITOR_OVLD_OS_LOG_WRITTEN: - value = log_sys.get_lsn() - recv_sys.lsn; + value = log_get_lsn() - recv_sys.lsn; break; /* innodb_log_waits */ @@ -1587,7 +1587,7 @@ srv_mon_process_existing_counter( break; case MONITOR_OVLD_LSN_CURRENT: - value = log_sys.get_lsn(); + value = log_get_lsn(); break; case MONITOR_OVLD_CHECKPOINTS: @@ -1595,10 +1595,10 @@ srv_mon_process_existing_counter( break; case MONITOR_LSN_CHECKPOINT_AGE: - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); value = static_cast(log_sys.get_lsn() - log_sys.last_checkpoint_lsn); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); break; case MONITOR_OVLD_BUF_OLDEST_LSN: diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 10f148a50da..0a21c69fb38 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -979,13 +979,13 @@ srv_export_innodb_status(void) mysql_mutex_unlock(&srv_innodb_monitor_mutex); - log_sys.latch.rd_lock(SRW_LOCK_CALL); + log_sys.latch.wr_lock(SRW_LOCK_CALL); export_vars.innodb_lsn_current = log_sys.get_lsn(); export_vars.innodb_lsn_flushed = log_sys.get_flushed_lsn(); export_vars.innodb_lsn_last_checkpoint = log_sys.last_checkpoint_lsn; export_vars.innodb_checkpoint_max_age = static_cast( log_sys.max_checkpoint_age); - log_sys.latch.rd_unlock(); + log_sys.latch.wr_unlock(); export_vars.innodb_os_log_written = export_vars.innodb_lsn_current - recv_sys.lsn; @@ -1072,7 +1072,7 @@ void srv_monitor_task(void*) /* Try to track a strange bug reported by Harald Fuchs and others, where the lsn seems to decrease at times */ - lsn_t new_lsn = log_sys.get_lsn(); + lsn_t new_lsn = log_get_lsn(); ut_a(new_lsn >= old_lsn); old_lsn = new_lsn; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index dc2ad75848f..3577cd7b118 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1057,7 +1057,7 @@ srv_init_abort_low( /** Prepare to delete the redo log file. Flush the dirty pages from all the buffer pools. Flush the redo log buffer to the redo log file. @return lsn upto which data pages have been flushed. */ -static lsn_t srv_prepare_to_delete_redo_log_file() +static lsn_t srv_prepare_to_delete_redo_log_file() noexcept { DBUG_ENTER("srv_prepare_to_delete_redo_log_file"); @@ -1071,7 +1071,7 @@ static lsn_t srv_prepare_to_delete_redo_log_file() log_sys.latch.wr_lock(SRW_LOCK_CALL); const bool latest_format{log_sys.is_latest()}; - lsn_t flushed_lsn{log_sys.get_lsn()}; + lsn_t flushed_lsn{log_sys.get_flushed_lsn(std::memory_order_relaxed)}; if (latest_format && !(log_sys.file_size & 4095) && flushed_lsn != log_sys.next_checkpoint_lsn + @@ -1079,6 +1079,11 @@ static lsn_t srv_prepare_to_delete_redo_log_file() ? SIZE_OF_FILE_CHECKPOINT + 8 : SIZE_OF_FILE_CHECKPOINT)) { +#ifdef HAVE_PMEM + if (!log_sys.is_opened()) + log_sys.buf_size= unsigned(std::min(log_sys.capacity(), + log_sys.buf_size_max)); +#endif fil_names_clear(flushed_lsn); flushed_lsn= log_sys.get_lsn(); } @@ -1119,7 +1124,7 @@ same_size: if (latest_format) log_write_up_to(flushed_lsn, false); - ut_ad(flushed_lsn == log_sys.get_lsn()); + ut_ad(flushed_lsn == log_get_lsn()); ut_ad(!os_aio_pending_reads()); ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); ut_ad(!buf_pool.get_oldest_modification(0)); @@ -1134,6 +1139,18 @@ static tpool::task rollback_all_recovered_task(trx_rollback_all_recovered, nullptr, &rollback_all_recovered_group); +inline lsn_t log_t::init_lsn() noexcept +{ + latch.wr_lock(SRW_LOCK_CALL); + ut_ad(!write_lsn_offset); + write_lsn_offset= 0; + const lsn_t lsn{base_lsn.load(std::memory_order_relaxed)}; + flushed_to_disk_lsn.store(lsn, std::memory_order_relaxed); + write_lsn= lsn; + latch.wr_unlock(); + return lsn; +} + /** Start InnoDB. @param[in] create_new_db whether to create a new database @return DB_SUCCESS or error code */ From 2a5a12b227845e03575f1b1eb0f6366dccc3e026 Mon Sep 17 00:00:00 2001 From: Nikita Malyavin Date: Mon, 7 Apr 2025 19:10:02 +0200 Subject: [PATCH 029/125] MDEV-36506 Build fails with cmake 4.0 Update cmake_minimum_required to 2.8...3.12 in root cmake and mroonga. This will update "Policy Version" to 3.12, which will not prevent the build by even higher cmake versions. There is also a reason to stay on the compatible with windows "policy version", so 3.12 is conservatively chosen. On the other hand, it will require at least version 2.8. --- CMakeLists.txt | 2 +- storage/mroonga/CMakeLists.txt | 2 +- storage/mroonga/vendor/groonga/CMakeLists.txt | 2 +- .../vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c1e2719e43..646e4256c97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA -CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8...3.12) IF(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) # Setting build type to RelWithDebInfo as none was specified. diff --git a/storage/mroonga/CMakeLists.txt b/storage/mroonga/CMakeLists.txt index 11eec13d520..e53568bc591 100644 --- a/storage/mroonga/CMakeLists.txt +++ b/storage/mroonga/CMakeLists.txt @@ -17,7 +17,7 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required(VERSION 2.8...3.12) project(mroonga) if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/storage/mroonga/vendor/groonga/CMakeLists.txt b/storage/mroonga/vendor/groonga/CMakeLists.txt index 5b25ada4f5b..0b03898b913 100644 --- a/storage/mroonga/vendor/groonga/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/CMakeLists.txt @@ -15,7 +15,7 @@ # https://buildbot.askmonty.org/buildbot/builders/work-amd64-valgrind/builds/5263/steps/compile/logs/stdio # says CMake 2.6.2... We want to drop old software support... -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required(VERSION 2.8...3.12) # cmake_minimum_required(VERSION 2.6.4) # CentOS 5 set(GRN_PROJECT_NAME "groonga") set(GRN_PROJECT_LABEL "Groonga") diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt index 96f9b3e6bbf..a555bdf6ac2 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/CMakeLists.txt @@ -15,7 +15,7 @@ # Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, # MA 02110-1335 USA -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required(VERSION 2.8...3.12) if(NOT DEFINED GROONGA_NORMALIZER_MYSQL_PROJECT_NAME) set(GROONGA_NORMALIZER_MYSQL_PROJECT_NAME "groonga-normalizer-mysql") endif() From 24a4d4ab0c0fbfe64d10d10098431f4cedc1d44a Mon Sep 17 00:00:00 2001 From: Nikita Malyavin Date: Mon, 7 Apr 2025 20:37:36 +0200 Subject: [PATCH 030/125] MDEV-36422 Build fails with cmake 4.0.0 due to wsrep update wsrep-lib --- wsrep-lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wsrep-lib b/wsrep-lib index 70cd967f5e2..e55f01ce1ee 160000 --- a/wsrep-lib +++ b/wsrep-lib @@ -1 +1 @@ -Subproject commit 70cd967f5e249b53d6cce90e2e4198641c564381 +Subproject commit e55f01ce1eed02e0781bc53bb23456c936667ccf From c6de1267dda77d78f843b26b4dafe0bc0473f7d5 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 10 Apr 2025 10:04:14 +0530 Subject: [PATCH 031/125] MDEV-35689 InnoDB system tables cannot be optimized or defragmented - With the help of MDEV-14795, InnoDB implemented a way to shrink the InnoDB system tablespace after undo tablespaces have been moved to separate files (MDEV-29986). There is no way to defragment any pages of InnoDB system tables. By doing that, shrinking of system tablespace can be more effective. This patch deals with defragment of system tables inside ibdata1. Following steps are done to do the defragmentation of system tablespace: 1) Make sure that there is no user tables exist in ibdata1 2) Iterate through all extent descriptor pages in system tablespace and note their states. 3) Find the free earlier extent to replace the lastly used extents in the system tablespace. 4) Iterate through all indexes of system tablespace and defragment the tree level by level. 5) Iterate the level from left page to right page and find out the page comes under the extent to be replaced. If it is then do step (6) else step(4) 6) Prepare the allocation of new extent by latching necessary pages. If any error happens then there is no modification of page happened till step (5). 7) Allocate the new page from the new extent 8) Prepare the associated pages for the block to be modified 9) Prepare the step of freeing of page 10) If any error happens during preparing of associated pages, freeing of page then restore the page which was modified during new page allocation 11) Copy the old page content to new page 12) Change the associative pages like left, right and parent page 13) Complete the freeing of old page Allocation of page from new extent, changing of relative pages, freeing of page are done by 2 steps. one is prepare which latches the to be modified pages and checks their validation. Other is complete(), Do the operation fseg_validate(): Validate the list exist in inode segment Defragmentation is enabled only when :autoextend exist in innodb_data_file_path variable. --- .../suite/innodb/r/sys_defragment.result | 25 + .../suite/innodb/r/sys_defragment_fail.result | 52 + mysql-test/suite/innodb/r/sys_truncate.result | 2 +- .../suite/innodb/r/sys_truncate_debug.result | 2 +- mysql-test/suite/innodb/t/sys_defragment.opt | 6 + mysql-test/suite/innodb/t/sys_defragment.test | 40 + .../suite/innodb/t/sys_defragment_fail.opt | 6 + .../suite/innodb/t/sys_defragment_fail.test | 90 + mysql-test/suite/innodb/t/sys_truncate.opt | 1 + mysql-test/suite/innodb/t/sys_truncate.test | 3 +- .../suite/innodb/t/sys_truncate_debug.opt | 1 + .../suite/innodb/t/sys_truncate_debug.test | 2 +- storage/innobase/fsp/fsp0fsp.cc | 1843 ++++++++++++++++- storage/innobase/fut/fut0lst.cc | 39 +- storage/innobase/include/dict0dict.h | 8 + storage/innobase/include/fil0fil.h | 6 + storage/innobase/include/fut0lst.h | 5 +- 17 files changed, 2067 insertions(+), 64 deletions(-) create mode 100644 mysql-test/suite/innodb/r/sys_defragment.result create mode 100644 mysql-test/suite/innodb/r/sys_defragment_fail.result create mode 100644 mysql-test/suite/innodb/t/sys_defragment.opt create mode 100644 mysql-test/suite/innodb/t/sys_defragment.test create mode 100644 mysql-test/suite/innodb/t/sys_defragment_fail.opt create mode 100644 mysql-test/suite/innodb/t/sys_defragment_fail.test diff --git a/mysql-test/suite/innodb/r/sys_defragment.result b/mysql-test/suite/innodb/r/sys_defragment.result new file mode 100644 index 00000000000..f279d694e6e --- /dev/null +++ b/mysql-test/suite/innodb/r/sys_defragment.result @@ -0,0 +1,25 @@ +# restart +SET GLOBAL innodb_file_per_table= 0; +Warnings: +Warning 1287 '@@innodb_file_per_table' is deprecated and will be removed in a future release +SET GLOBAL innodb_limit_optimistic_insert_debug = 2; +CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL, f3 INT NOT NULL)ENGINE=InnoDB; +INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; +SET GLOBAL innodb_file_per_table= default; +Warnings: +Warning 1287 '@@innodb_file_per_table' is deprecated and will be removed in a future release +CREATE TABLE t2(f1 INT NOT NULL PRIMARY KEY,f2 VARCHAR(40))ENGINE=InnoDB PARTITION BY KEY() PARTITIONS 256; +INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; +DROP TABLE t2, t1; +InnoDB 0 transactions not purged +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; +name file_size +innodb_system 205520896 +set GLOBAL innodb_fast_shutdown= 0; +# restart +FOUND 1 /InnoDB: Moving the data from extents 4096 through 22016/ in mysqld.1.err +FOUND 1 /InnoDB: Defragmentation of system tablespace is successful/ in mysqld.1.err +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; +name file_size +innodb_system 14680064 +# restart diff --git a/mysql-test/suite/innodb/r/sys_defragment_fail.result b/mysql-test/suite/innodb/r/sys_defragment_fail.result new file mode 100644 index 00000000000..5abf4e65772 --- /dev/null +++ b/mysql-test/suite/innodb/r/sys_defragment_fail.result @@ -0,0 +1,52 @@ +call mtr.add_suppression("InnoDB: Defragmentation of CLUST_IND in SYS_INDEXES failed: Data structure corruption"); +call mtr.add_suppression("InnoDB: Defragmentation of CLUST_IND in SYS_COLUMNS failed: Data structure corruption"); +call mtr.add_suppression("InnoDB: Cannot free the unused segments in system tablespace"); +# restart +set GLOBAL innodb_file_per_table = 0; +Warnings: +Warning 1287 '@@innodb_file_per_table' is deprecated and will be removed in a future release +set GLOBAL innodb_limit_optimistic_insert_debug = 2; +CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL)ENGINE=InnoDB; +INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096; +SET GLOBAL innodb_file_per_table= 1; +Warnings: +Warning 1287 '@@innodb_file_per_table' is deprecated and will be removed in a future release +CREATE TABLE t2(f1 INT NOT NULL PRIMARY KEY, +f2 VARCHAR(40))ENGINE=InnoDB PARTITION BY KEY() PARTITIONS 256; +INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096; +DROP TABLE t2; +InnoDB 0 transactions not purged +# restart +FOUND 1 /InnoDB: User table exists in the system tablespace/ in mysqld.1.err +DROP TABLE t1; +InnoDB 0 transactions not purged +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; +name file_size +innodb_system 58720256 +# restart: --debug_dbug=+d,fail_after_level_defragment +FOUND 1 /InnoDB: Defragmentation of CLUST_IND in SYS_COLUMNS failed./ in mysqld.1.err +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; +name file_size +innodb_system 58720256 +# restart: --debug_dbug=d,allocation_prepare_fail +FOUND 1 /InnoDB: Defragmentation of CLUST_IND in SYS_INDEXES failed./ in mysqld.1.err +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; +name file_size +innodb_system 58720256 +# restart: --debug_dbug=d,relation_page_prepare_fail +FOUND 2 /InnoDB: Defragmentation of CLUST_IND in SYS_INDEXES failed./ in mysqld.1.err +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; +name file_size +innodb_system 58720256 +# restart: --debug_dbug=d,remover_prepare_fail +FOUND 3 /InnoDB: Defragmentation of CLUST_IND in SYS_INDEXES failed./ in mysqld.1.err +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; +name file_size +innodb_system 58720256 +# restart +FOUND 5 /InnoDB: Moving the data from extents 4096 through 8960/ in mysqld.1.err +FOUND 1 /InnoDB: Defragmentation of system tablespace is successful/ in mysqld.1.err +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; +name file_size +innodb_system 15728640 +# restart diff --git a/mysql-test/suite/innodb/r/sys_truncate.result b/mysql-test/suite/innodb/r/sys_truncate.result index bb8eafbe584..909f363096e 100644 --- a/mysql-test/suite/innodb/r/sys_truncate.result +++ b/mysql-test/suite/innodb/r/sys_truncate.result @@ -4,7 +4,7 @@ Warning 1287 '@@innodb_file_per_table' is deprecated and will be removed in a fu SET UNIQUE_CHECKS=0, FOREIGN_KEY_CHECKS=0; CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL, f3 INT NOT NULL, INDEX(f1), -INDEX(f2), INDEX(f3))ENGINE=InnoDB; +INDEX(f2), INDEX(f3))STATS_PERSISTENT=0 ENGINE=InnoDB; BEGIN; INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; diff --git a/mysql-test/suite/innodb/r/sys_truncate_debug.result b/mysql-test/suite/innodb/r/sys_truncate_debug.result index b198d449a76..f51693d849c 100644 --- a/mysql-test/suite/innodb/r/sys_truncate_debug.result +++ b/mysql-test/suite/innodb/r/sys_truncate_debug.result @@ -9,7 +9,7 @@ Warning 1287 '@@innodb_file_per_table' is deprecated and will be removed in a fu SET UNIQUE_CHECKS=0, FOREIGN_KEY_CHECKS=0; CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL, f3 INT NOT NULL, INDEX(f1), -INDEX(f2), INDEX(f3))ENGINE=InnoDB; +INDEX(f2), INDEX(f3))STATS_PERSISTENT=0 ENGINE=InnoDB; BEGIN; INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; diff --git a/mysql-test/suite/innodb/t/sys_defragment.opt b/mysql-test/suite/innodb/t/sys_defragment.opt new file mode 100644 index 00000000000..beba09e7f48 --- /dev/null +++ b/mysql-test/suite/innodb/t/sys_defragment.opt @@ -0,0 +1,6 @@ +--innodb_page_size=4k +--innodb_data_file_path=ibdata1:1M:autoextend:autoshrink +--innodb_undo_tablespaces=0 +--innodb_stats_persistent=0 +--skip_partition=0 +--innodb_sys_tablespaces diff --git a/mysql-test/suite/innodb/t/sys_defragment.test b/mysql-test/suite/innodb/t/sys_defragment.test new file mode 100644 index 00000000000..93b219458a6 --- /dev/null +++ b/mysql-test/suite/innodb/t/sys_defragment.test @@ -0,0 +1,40 @@ +--source include/have_innodb.inc +--source include/have_sequence.inc +--source include/have_debug.inc + +--let MYSQLD_DATADIR= `SELECT @@datadir` +--source include/shutdown_mysqld.inc +--copy_file $MYSQLD_DATADIR/ibdata1 $MYSQLD_DATADIR/ibdata1_copy +--copy_file $MYSQLD_DATADIR/ib_logfile0 $MYSQLD_DATADIR/ib_logfile0_copy +--source include/start_mysqld.inc + +SET GLOBAL innodb_file_per_table= 0; +SET GLOBAL innodb_limit_optimistic_insert_debug = 2; +CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL, f3 INT NOT NULL)ENGINE=InnoDB; +INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; +SET GLOBAL innodb_file_per_table= default; +CREATE TABLE t2(f1 INT NOT NULL PRIMARY KEY,f2 VARCHAR(40))ENGINE=InnoDB PARTITION BY KEY() PARTITIONS 256; +INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; +DROP TABLE t2, t1; +--source include/wait_all_purged.inc + +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; + +set GLOBAL innodb_fast_shutdown= 0; +--source include/restart_mysqld.inc + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: Moving the data from extents 4096 through 22016; +--source include/search_pattern_in_file.inc + +let SEARCH_PATTERN=InnoDB: Defragmentation of system tablespace is successful; +--source include/search_pattern_in_file.inc + +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; + +--source include/shutdown_mysqld.inc + +--move_file $MYSQLD_DATADIR/ibdata1_copy $MYSQLD_DATADIR/ibdata1 +--move_file $MYSQLD_DATADIR/ib_logfile0_copy $MYSQLD_DATADIR/ib_logfile0 + +--source include/start_mysqld.inc diff --git a/mysql-test/suite/innodb/t/sys_defragment_fail.opt b/mysql-test/suite/innodb/t/sys_defragment_fail.opt new file mode 100644 index 00000000000..6326b7e90a3 --- /dev/null +++ b/mysql-test/suite/innodb/t/sys_defragment_fail.opt @@ -0,0 +1,6 @@ +--innodb_page_size=4k +--innodb_data_file_path=ibdata1:15M:autoextend:autoshrink +--innodb_undo_tablespaces=0 +--innodb_stats_persistent=0 +--skip_partition=0 +--innodb_sys_tablespaces diff --git a/mysql-test/suite/innodb/t/sys_defragment_fail.test b/mysql-test/suite/innodb/t/sys_defragment_fail.test new file mode 100644 index 00000000000..6136d43d601 --- /dev/null +++ b/mysql-test/suite/innodb/t/sys_defragment_fail.test @@ -0,0 +1,90 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_sequence.inc + +call mtr.add_suppression("InnoDB: Defragmentation of CLUST_IND in SYS_INDEXES failed: Data structure corruption"); +call mtr.add_suppression("InnoDB: Defragmentation of CLUST_IND in SYS_COLUMNS failed: Data structure corruption"); +call mtr.add_suppression("InnoDB: Cannot free the unused segments in system tablespace"); + +--let MYSQLD_DATADIR= `SELECT @@datadir` +--source include/shutdown_mysqld.inc +--copy_file $MYSQLD_DATADIR/ibdata1 $MYSQLD_DATADIR/ibdata1_copy +--copy_file $MYSQLD_DATADIR/ib_logfile0 $MYSQLD_DATADIR/ib_logfile0_copy +--source include/start_mysqld.inc + +set GLOBAL innodb_file_per_table = 0; +set GLOBAL innodb_limit_optimistic_insert_debug = 2; +CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL)ENGINE=InnoDB; +INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096; + +SET GLOBAL innodb_file_per_table= 1; +CREATE TABLE t2(f1 INT NOT NULL PRIMARY KEY, + f2 VARCHAR(40))ENGINE=InnoDB PARTITION BY KEY() PARTITIONS 256; + +INSERT INTO t1 SELECT seq, seq FROM seq_1_to_4096; +DROP TABLE t2; + +--source include/wait_all_purged.inc +let $restart_parameters=; +--source include/restart_mysqld.inc + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: User table exists in the system tablespace; +--source include/search_pattern_in_file.inc +DROP TABLE t1; + +--source include/wait_all_purged.inc +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; + +let $restart_parameters=--debug_dbug=+d,fail_after_level_defragment; +--source include/restart_mysqld.inc + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: Defragmentation of CLUST_IND in SYS_COLUMNS failed.; +--source include/search_pattern_in_file.inc + +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; + +let $restart_parameters=--debug_dbug=d,allocation_prepare_fail; +--source include/restart_mysqld.inc + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: Defragmentation of CLUST_IND in SYS_INDEXES failed.; +--source include/search_pattern_in_file.inc + +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; + +let $restart_parameters=--debug_dbug=d,relation_page_prepare_fail; +--source include/restart_mysqld.inc + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: Defragmentation of CLUST_IND in SYS_INDEXES failed.; +--source include/search_pattern_in_file.inc + +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; + +let $restart_parameters=--debug_dbug=d,remover_prepare_fail; +--source include/restart_mysqld.inc + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: Defragmentation of CLUST_IND in SYS_INDEXES failed.; +--source include/search_pattern_in_file.inc + +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; + +let $restart_parameters=; +--source include/restart_mysqld.inc + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN= InnoDB: Moving the data from extents 4096 through 8960; +--source include/search_pattern_in_file.inc + +let SEARCH_PATTERN=InnoDB: Defragmentation of system tablespace is successful; +--source include/search_pattern_in_file.inc + +select name, file_size from information_schema.innodb_sys_tablespaces where space = 0; + +--source include/shutdown_mysqld.inc +--move_file $MYSQLD_DATADIR/ibdata1_copy $MYSQLD_DATADIR/ibdata1 +--move_file $MYSQLD_DATADIR/ib_logfile0_copy $MYSQLD_DATADIR/ib_logfile0 +--source include/start_mysqld.inc diff --git a/mysql-test/suite/innodb/t/sys_truncate.opt b/mysql-test/suite/innodb/t/sys_truncate.opt index f940dadffd3..375d9847b65 100644 --- a/mysql-test/suite/innodb/t/sys_truncate.opt +++ b/mysql-test/suite/innodb/t/sys_truncate.opt @@ -1,2 +1,3 @@ --innodb_data_file_path=ibdata1:10M:autoextend:autoshrink --innodb_sys_tablespaces +--innodb_buffer_pool_size=75M diff --git a/mysql-test/suite/innodb/t/sys_truncate.test b/mysql-test/suite/innodb/t/sys_truncate.test index d5e05dea34c..fbba010dc09 100644 --- a/mysql-test/suite/innodb/t/sys_truncate.test +++ b/mysql-test/suite/innodb/t/sys_truncate.test @@ -1,10 +1,11 @@ --source include/have_innodb.inc --source include/have_sequence.inc + SET GLOBAL INNODB_FILE_PER_TABLE= 0; SET UNIQUE_CHECKS=0, FOREIGN_KEY_CHECKS=0; CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL, f3 INT NOT NULL, INDEX(f1), - INDEX(f2), INDEX(f3))ENGINE=InnoDB; + INDEX(f2), INDEX(f3))STATS_PERSISTENT=0 ENGINE=InnoDB; BEGIN; INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; diff --git a/mysql-test/suite/innodb/t/sys_truncate_debug.opt b/mysql-test/suite/innodb/t/sys_truncate_debug.opt index b8a0ed244e4..97647d9843b 100644 --- a/mysql-test/suite/innodb/t/sys_truncate_debug.opt +++ b/mysql-test/suite/innodb/t/sys_truncate_debug.opt @@ -1,3 +1,4 @@ --innodb_data_file_path=ibdata1:1M:autoextend:autoshrink --innodb_sys_tablespaces --innodb_page_size=4k +--innodb_buffer_pool_size=100M diff --git a/mysql-test/suite/innodb/t/sys_truncate_debug.test b/mysql-test/suite/innodb/t/sys_truncate_debug.test index be70ea743e9..7dcb5ffde2a 100644 --- a/mysql-test/suite/innodb/t/sys_truncate_debug.test +++ b/mysql-test/suite/innodb/t/sys_truncate_debug.test @@ -16,7 +16,7 @@ SET GLOBAL INNODB_FILE_PER_TABLE= 0; SET UNIQUE_CHECKS=0, FOREIGN_KEY_CHECKS=0; CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL, f3 INT NOT NULL, INDEX(f1), - INDEX(f2), INDEX(f3))ENGINE=InnoDB; + INDEX(f2), INDEX(f3))STATS_PERSISTENT=0 ENGINE=InnoDB; BEGIN; INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; INSERT INTO t1 SELECT seq, seq, seq FROM seq_1_to_16384; diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index cc55ddd66cc..63d68930e0e 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -42,6 +42,7 @@ Created 11/29/1995 Heikki Tuuri #ifndef DBUG_OFF # include "trx0purge.h" #endif +#include #include #include "trx0undo.h" @@ -237,15 +238,13 @@ inline void xdes_set_state(const buf_block_t &block, xdes_t *descr, Gets the state of an xdes. @return state */ UNIV_INLINE -ulint +uint32_t xdes_get_state( /*===========*/ const xdes_t* descr) /*!< in: descriptor */ { - ulint state; - ut_ad(descr); - state = mach_read_from_4(descr + XDES_STATE); + uint32_t state = mach_read_from_4(descr + XDES_STATE); ut_ad(state - 1 < XDES_FSEG); return(state); } @@ -1057,7 +1056,7 @@ fsp_alloc_from_free_frag(buf_block_t *header, buf_block_t *xdes, xdes_t *descr, @param[in,out] mtr mini-transaction @return block, initialized */ static buf_block_t* fsp_page_create(fil_space_t *space, uint32_t offset, - mtr_t *mtr) + mtr_t *mtr) noexcept { buf_block_t *free_block= buf_LRU_get_free_block(have_no_mutex), *block= buf_page_create(space, offset, space->zip_size(), mtr, free_block); @@ -3089,14 +3088,13 @@ std::ostream &fseg_header::to_stream(std::ostream &out) const } #endif /* UNIV_DEBUG */ -/** Get the latched extent descriptor page or -acquire the extent descriptor page. +/** Get the latched page page or acquire the page. @param page_id page identifier to be acquired @param mtr mini-transaction @param err error code @return block descriptor */ static -buf_block_t *fsp_get_latched_xdes_page( +buf_block_t *fsp_get_latched_page( page_id_t page_id, mtr_t *mtr, dberr_t *err) { buf_block_t *block= nullptr; @@ -3118,7 +3116,7 @@ class fsp_xdes_old_page const uint32_t m_space; public: fsp_xdes_old_page(uint32_t space):m_space(space) {} - ulint n_pages() + uint32_t n_pages() noexcept { uint32_t count=0; for (uint32_t i= 0; i < m_old_xdes_pages.size(); i++) @@ -3127,7 +3125,7 @@ public: } __attribute__((warn_unused_result)) - dberr_t insert(uint32_t page_no, mtr_t *mtr) + dberr_t insert(uint32_t page_no, mtr_t *mtr) noexcept { uint32_t m_index= page_no >> srv_page_size_shift; if (m_old_xdes_pages.size() > m_index && @@ -3137,7 +3135,7 @@ public: DBUG_EXECUTE_IF("shrink_buffer_pool_full", return DB_OUT_OF_MEMORY;); dberr_t err= DB_SUCCESS; - buf_block_t *block= fsp_get_latched_xdes_page( + buf_block_t *block= fsp_get_latched_page( page_id_t(m_space, page_no), mtr, &err); if (block) { @@ -3154,7 +3152,7 @@ public: return err; } - buf_block_t *search(uint32_t page_no) + buf_block_t *search(uint32_t page_no) noexcept { uint32_t m_index= page_no >> srv_page_size_shift; if (m_index > m_old_xdes_pages.size()) @@ -3162,7 +3160,7 @@ public: return m_old_xdes_pages[m_index]; } - void restore(mtr_t *mtr) + void restore(mtr_t *mtr) noexcept { for (uint32_t i= 0; i < m_old_xdes_pages.size(); i++) { @@ -3198,11 +3196,11 @@ static dberr_t fsp_lst_update_skip( buf_block_t *header, uint16_t hdr_offset, fil_addr_t cur_addr, fil_addr_t last_valid_addr, - uint32_t skip_len, mtr_t *mtr) + uint32_t skip_len, mtr_t *mtr) noexcept { dberr_t err= DB_SUCCESS; uint32_t space_id= header->page.id().space(); - buf_block_t *cur= fsp_get_latched_xdes_page( + buf_block_t *cur= fsp_get_latched_page( page_id_t(space_id, cur_addr.page), mtr, &err); if (!cur) return err; @@ -3229,7 +3227,7 @@ dberr_t fsp_lst_update_skip( prev= cur; else { - prev= fsp_get_latched_xdes_page( + prev= fsp_get_latched_page( page_id_t(space_id, last_valid_addr.page), mtr, &err); if (!prev) return err; @@ -3270,7 +3268,7 @@ dberr_t fsp_lst_write_end( buf_block_t *header, uint16_t hdr_offset, fil_addr_t cur_addr, uint32_t skip_len, uint32_t orig_len, - mtr_t *mtr) + mtr_t *mtr) noexcept { dberr_t err= DB_SUCCESS; byte *len_bytes= &header->page.frame[hdr_offset + FLST_LEN]; @@ -3312,7 +3310,7 @@ func_exit: header->page.frame + hdr_offset + FLST_LAST, cur_addr.page, cur_addr.boffset, mtr); - buf_block_t *cur_block= fsp_get_latched_xdes_page( + buf_block_t *cur_block= fsp_get_latched_page( page_id_t(header->page.id().space(), cur_addr.page), mtr, &err); @@ -3340,7 +3338,7 @@ func_exit: __attribute__((warn_unused_result)) static dberr_t fsp_shrink_list(buf_block_t *header, uint16_t hdr_offset, - uint32_t threshold, mtr_t *mtr) + uint32_t threshold, mtr_t *mtr) noexcept { ut_ad(mach_read_from_4(header->page.frame + FIL_PAGE_OFFSET) == 0); const uint32_t len= flst_get_len(hdr_offset + header->page.frame); @@ -3362,7 +3360,7 @@ dberr_t fsp_shrink_list(buf_block_t *header, uint16_t hdr_offset, ut_ad(!(addr.page & (srv_page_size - 1))); if (!descr_block || descr_block->page.id().page_no() != addr.page) { - descr_block= fsp_get_latched_xdes_page( + descr_block= fsp_get_latched_page( page_id_t(header->page.id().space(), addr.page), mtr, &err); if (!descr_block) return err; @@ -3426,7 +3424,7 @@ dberr_t fsp_xdes_reset(uint32_t space_id, uint32_t threshold, mtr_t *mtr) 0, (cur_descr_page + srv_page_size - 1)); last_descr_offset+= XDES_SIZE; dberr_t err= DB_SUCCESS; - buf_block_t *block= fsp_get_latched_xdes_page( + buf_block_t *block= fsp_get_latched_page( page_id_t(space_id, cur_descr_page), mtr, &err); if (!block) return err; @@ -3478,7 +3476,7 @@ dberr_t fsp_traverse_extents( { if (!block) { - block= fsp_get_latched_xdes_page( + block= fsp_get_latched_page( page_id_t(space->id, last_descr_page_no), mtr, &err); if (!block) return err; @@ -3546,32 +3544,31 @@ dberr_t fsp_traverse_extents( return err; } -#ifdef UNIV_DEBUG /** Validate the system tablespace list */ __attribute__((warn_unused_result)) -dberr_t fsp_tablespace_validate(fil_space_t *space) +static dberr_t fsp_tablespace_validate(fil_space_t *space, + mtr_t *mtr) noexcept { /* Validate all FSP list in system tablespace */ - mtr_t local_mtr; dberr_t err= DB_SUCCESS; - local_mtr.start(); - if (buf_block_t *header= fsp_get_header( - space, &local_mtr, &err)) + if (buf_block_t *header= fsp_get_header(space, mtr, &err)) { - flst_validate(header, FSP_FREE + FSP_HEADER_OFFSET, &local_mtr); - flst_validate(header, FSP_FREE_FRAG + FSP_HEADER_OFFSET, - &local_mtr); - flst_validate(header, FSP_HEADER_OFFSET + FSP_FULL_FRAG, - &local_mtr); - flst_validate(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FULL, - &local_mtr); - flst_validate(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FREE, - &local_mtr); + err= flst_validate(header, FSP_FREE + FSP_HEADER_OFFSET, mtr); + if (err == DB_SUCCESS) + err= flst_validate(header, FSP_FREE_FRAG + FSP_HEADER_OFFSET, + mtr); + if (err == DB_SUCCESS) + err= flst_validate(header, FSP_HEADER_OFFSET + FSP_FULL_FRAG, + mtr); + if (err == DB_SUCCESS) + err= flst_validate(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FULL, + mtr); + if (err == DB_SUCCESS) + err= flst_validate(header, FSP_HEADER_OFFSET + FSP_SEG_INODES_FREE, + mtr); } - local_mtr.commit(); return err; } -#endif /* UNIV_DEBUG */ /** Store the inode information which basically stores the page and offset */ @@ -4017,6 +4014,1748 @@ dberr_t fil_space_t::garbage_collect(bool shutdown) return unused_inodes.free_segs(); } +class SpaceDefragmenter; + +namespace flst +{ + /** Validate the file list node for the system tablespace. + @param addr file space address + @return true if validation successful or false */ + static bool node_valid(const fil_addr_t *addr) noexcept + { + return addr->boffset >= FIL_PAGE_DATA && + addr->boffset < (srv_page_size - FIL_PAGE_DATA_END); + } + + /** Prepare the steps for removing the file list node + @param descr_block descriptor block + @param xoffset descriptor offset within the block + @param free_limit maximum free limit in the tablespace + @param mtr mini-transaction + @param prev_block previous block in the list + @param next_block next block in the list + @return error code */ + static dberr_t remove_prepare(const buf_block_t &descr_block, + uint32_t xoffset, uint32_t free_limit, + mtr_t *mtr, buf_block_t **prev_block, + buf_block_t **next_block) noexcept + { + const xdes_t *descr= descr_block.page.frame + xoffset; + fil_addr_t prev_addr= flst_get_prev_addr(descr); + fil_addr_t next_addr= flst_get_next_addr(descr); + dberr_t err= DB_SUCCESS; + + if (prev_addr.page != FIL_NULL) + { + if (!node_valid(&prev_addr)) + return DB_CORRUPTION; + + *prev_block= fsp_get_latched_page(page_id_t{0, prev_addr.page}, + mtr, &err); + ut_ad(!*prev_block == (err != DB_SUCCESS)); + + if (!*prev_block) + return err; + + fil_addr_t cur_addr= + flst_get_next_addr((*prev_block)->page.frame + + prev_addr.boffset); + if (cur_addr.page != descr_block.page.id().page_no() || + cur_addr.boffset != xoffset) + return DB_CORRUPTION; + } + + if (next_addr.page != FIL_NULL) + { + if (!node_valid(&next_addr)) + return DB_CORRUPTION; + + *next_block= fsp_get_latched_page(page_id_t{0, next_addr.page}, + mtr, &err); + ut_ad(!*next_block == (err != DB_SUCCESS)); + if (!*next_block) + return err; + + fil_addr_t cur_addr= + flst_get_prev_addr((*next_block)->page.frame + next_addr.boffset); + if (cur_addr.page != descr_block.page.id().page_no() || + cur_addr.boffset != xoffset) + return DB_CORRUPTION; + } + + return err; + } + + /** Complete the steps for removing the file list node + @param base base block where free list starts + @param boffset offset where list starts + @param descr descriptor to be removed + @param mtr mini-transaction */ + static void remove_complete(buf_block_t *base, uint16_t boffset, + xdes_t *descr, mtr_t *mtr) noexcept + { + fil_addr_t prev_addr= flst_get_prev_addr(descr + XDES_FLST_NODE); + fil_addr_t next_addr= flst_get_next_addr(descr + XDES_FLST_NODE); + /* remove_prepare() checked these already */ + ut_ad(next_addr.page == FIL_NULL || node_valid(&next_addr)); + ut_ad(prev_addr.page == FIL_NULL || node_valid(&prev_addr)); + byte *list= base->page.frame + boffset; + + buf_block_t *prev_block= nullptr; + buf_block_t *next_block= nullptr; + + if (prev_addr.page != FIL_NULL) + { + prev_block= + mtr->get_already_latched(page_id_t{0, prev_addr.page}, + MTR_MEMO_PAGE_SX_FIX); + ut_ad(prev_block); + + flst_write_addr(*prev_block, prev_block->page.frame + + prev_addr.boffset + FLST_NEXT, + next_addr.page, next_addr.boffset, mtr); + } + else + flst_write_addr(*base, list + FLST_FIRST, + next_addr.page, next_addr.boffset, mtr); + + if (next_addr.page != FIL_NULL) + { + next_block= + mtr->get_already_latched(page_id_t{0, next_addr.page}, + MTR_MEMO_PAGE_SX_FIX); + ut_ad(next_block); + + flst_write_addr(*next_block, next_block->page.frame + + next_addr.boffset + FLST_PREV, + prev_addr.page, prev_addr.boffset, mtr); + } + else + flst_write_addr(*base, list + FLST_LAST, + prev_addr.page, prev_addr.boffset, mtr); + + /* All callers of remove_prepare() does check the FLST_LEN of + the list */ + byte *len= list + FLST_LEN; + mtr->write<4>(*base, len, mach_read_from_4(len) - 1); + } + + /** Prepare the steps for adding the block into last of the list + @param base block where list starts + @param boffset offset to find the list + @param free_limit maximum free limit in the tablespace + @param mtr mini-transaction + @param last_block_list last block in the list + @return error code */ + static dberr_t append_prepare(const buf_block_t &base, uint16_t boffset, + uint32_t free_limit, mtr_t *mtr, + buf_block_t **last_block_list) noexcept + { + ut_ad(!*last_block_list); + if (!flst_get_len(base.page.frame + boffset)) + return DB_SUCCESS; + + fil_addr_t addr= flst_get_last(base.page.frame + boffset); + + if (addr.page >= free_limit) + return DB_CORRUPTION; + + if (!node_valid(&addr)) + return DB_CORRUPTION; + + dberr_t err= DB_SUCCESS; + *last_block_list= fsp_get_latched_page(page_id_t{0, addr.page}, + mtr, &err); + return err; + } + + /** Complete the steps for adding the block into last of the list + @param base base block where free list starts + @param boffset offset where list starts + @param curr extent descriptor block + @param coffset offset to point the descriptor + @param mtr mini-transaction */ + static void append_complete(buf_block_t *base, uint16_t boffset, + buf_block_t *curr, uint16_t coffset, + mtr_t *mtr) noexcept + { + fil_addr_t last_addr= flst_get_last(base->page.frame + boffset); + ut_ad(last_addr.page == FIL_NULL || node_valid(&last_addr)); + buf_block_t *last_block_list= nullptr; + if (last_addr.page != FIL_NULL) + { + last_block_list= + mtr->get_already_latched(page_id_t{0, last_addr.page}, + MTR_MEMO_PAGE_SX_FIX); + ut_ad(last_block_list); + + fil_addr_t addr= flst_get_last(base->page.frame + boffset); + + flst_write_addr(*last_block_list, + last_block_list->page.frame + addr.boffset + + FLST_NEXT, + curr->page.id().page_no(), coffset, mtr); + flst_write_addr(*curr, + curr->page.frame + coffset + FLST_PREV, + addr.page, addr.boffset, mtr); + flst_write_addr(*base, base->page.frame + boffset + FLST_LAST, + curr->page.id().page_no(), coffset, mtr); + } + else + { + /* Encountered empty list. So add current block as FIRST + and LAST block in the list */ + flst_write_addr(*curr, + curr->page.frame + coffset + FLST_PREV, + FIL_NULL, 0, mtr); + flst_write_addr(*base, base->page.frame + boffset + FLST_FIRST, + curr->page.id().page_no(), coffset, mtr); + memcpy(base->page.frame + boffset + FLST_LAST, + base->page.frame + boffset + FLST_FIRST, FIL_ADDR_SIZE); + mtr->memmove(*base, boffset + FLST_LAST, + boffset + FLST_FIRST, FIL_ADDR_SIZE); + } + + flst_write_addr(*curr, + curr->page.frame + coffset + FLST_NEXT, + FIL_NULL, 0, mtr); + + byte *len= base->page.frame + boffset + FLST_LEN; + mtr->write<4>(*base, len, mach_read_from_4(len) + 1); + } +} /* namespace flst */ + +static dberr_t fseg_validate_low(fil_space_t *space, dict_index_t *index, + mtr_t *mtr) noexcept +{ + dberr_t err= DB_SUCCESS; + buf_block_t *root= btr_root_block_get(index, RW_SX_LATCH, mtr, &err); + if (UNIV_UNLIKELY(!root)) + return err; + + fseg_header_t *seg_header= + root->page.frame + PAGE_HEADER + PAGE_BTR_SEG_TOP; + buf_block_t *iblock; + fseg_inode_t *inode= fseg_inode_try_get(seg_header, 0, 0, mtr, + &iblock, &err); + if (!inode) + return err; + + uint16_t i_offset= uint16_t(inode - iblock->page.frame); + + err= flst_validate(iblock, uint16_t(i_offset + FSEG_FREE), mtr); + if (err == DB_SUCCESS) + err= flst_validate(iblock, uint16_t(i_offset + FSEG_NOT_FULL), mtr); + if (err == DB_SUCCESS) + err= flst_validate(iblock, uint16_t(i_offset + FSEG_FULL), mtr); + + if (err) return err; + + seg_header= root->page.frame + PAGE_HEADER + PAGE_BTR_SEG_LEAF; + inode= fseg_inode_try_get(seg_header, 0, 0, mtr, &iblock, &err); + if (!inode) + return err; + + i_offset= uint16_t(inode - iblock->page.frame); + + err= flst_validate(iblock, uint16_t(i_offset + FSEG_FREE), mtr); + if (err == DB_SUCCESS) + err= flst_validate(iblock, uint16_t(i_offset + FSEG_NOT_FULL), mtr); + if (err == DB_SUCCESS) + err= flst_validate(iblock, uint16_t(i_offset + FSEG_FULL), mtr); + return err; +} + +/** Validate the system tablespace list */ +__attribute__((warn_unused_result)) +static dberr_t fseg_validate(fil_space_t *space, + dict_index_t *index) noexcept +{ + /* Validate all FSP list in system tablespace */ + mtr_t mtr; + mtr.start(); + dberr_t err= fseg_validate_low(space, index, &mtr); + mtr.commit(); + return err; +} + +/** Prepare the associate pages of the current block and modify +the associated pages */ +class AssociatedPages final +{ + buf_block_t *m_left_block= nullptr; + buf_block_t *m_right_block= nullptr; + buf_block_t *m_parent_block= nullptr; + buf_block_t *const m_cur_block; + mtr_t *const m_mtr; + +public: + AssociatedPages(buf_block_t *cur_block, mtr_t *mtr) + : m_cur_block(cur_block), m_mtr(mtr) {} + + /** Fetch the left, right and parent page for the respective + current block and make sure that there is no issue exist */ + dberr_t prepare(uint32_t parent_page) noexcept + { + uint32_t left_page_no= btr_page_get_prev(m_cur_block->page.frame); + dberr_t err= DB_SUCCESS; + if (left_page_no != FIL_NULL) + { + m_left_block= fsp_get_latched_page(page_id_t{0, left_page_no}, + m_mtr, &err); + ut_ad(!m_left_block == (err != DB_SUCCESS)); + if (!m_left_block) + return err; + } + + uint32_t right_page_no= btr_page_get_next(m_cur_block->page.frame); + if (right_page_no != FIL_NULL) + { + m_right_block= fsp_get_latched_page(page_id_t{0, right_page_no}, + m_mtr, &err); + ut_ad(!m_right_block == (err != DB_SUCCESS)); + if (!m_right_block) + return err; + } + + m_parent_block= fsp_get_latched_page(page_id_t{0, parent_page}, + m_mtr, &err); + return err; + } + + /** Modify the FIL_PAGE_NEXT, FIL_PAGE_PREV, CHILD_PAGE of + respective left, right and parent block to new page number */ + void complete(uint32_t new_page_no, uint32_t parent_offset) noexcept + { + if (m_left_block) + m_mtr->write<4>(*m_left_block, + m_left_block->page.frame + FIL_PAGE_NEXT, + new_page_no); + + if (m_right_block) + m_mtr->write<4>(*m_right_block, + m_right_block->page.frame + FIL_PAGE_PREV, + new_page_no); + + m_mtr->write<4>(*m_parent_block, + m_parent_block->page.frame + parent_offset, + new_page_no); + } +}; + +/** page operation for the system tablespace does the 2 things: +1) Page Allocation +2) Page removal + +Steps for page allocation depends on new extent state. + +(1) If the xdes_get_state(new_descr) == XDES_FREE then +remove the new extent from FSP_FREE list + + (1.1) If the page has to be allocated for segment then + add the newly allocated extent descriptor to + FSEG_NOT_FULL list and make the xdes_set_state(new_descr) + as XDES_FSEG + + (1.2) If the page has to be non-segment page then add the + newly allocated extent descriptor to FSP_FREE_FRAG list + and make the xdes_set_state(new_descr) as XDES_FREE_FRAG + + (1.3) Allocate a page from the new extent + +(2) If the xdes_get_state(new_descr) == XDES_FREE_FRAG then + + (2.1) Allocate a page from the new extent + + (2.2) xdes_get_n_used(new_descr) is FSP_EXTENT_SIZE then + - Remove the new extent descriptor from FSP_FREE_FRAG list + - Add the new extent descriptor to FSP_FULL_FRAG list + and make xdes_set_state(new_descr) as XDES_FULL_FRAG + +(3) If the xdes_get_state(new_descr) == XDES_FSEG then + + (3.1) Allocate a page from extent + + (3.2) xdes_get_n_used(new_descr) is FSP_EXTENT_SIZE then + - Remove the new extent descriptor from FSEG_NOT_FULL list + - Add the new extent descriptor to FSEG_FULL list + + +Steps for removing the page from extent: + + (1) To remove the page from extent and number of used + pages in extent descriptor is FSP_EXTENT_SIZE + + (1a) If the xdes_get_state(m_old_descr) is XDES_FSEG then + move the extent descriptor from FSEG_FULL to FSEG_NOT_FULL + + (1b) If the xdes_get_stats(m_old_descr) is XDES_FREE_FRAG/XDES_FULL_FRAG + then move the extent descriptor from FSP_FULL_FRAG to + FSP_FREE_FRAG list + + (2) If the number of used pages in extent descriptor is 0 then + move the extent descriptor to FSP_FREE + + (3) Free the page and mark the XDES_FREE_BIT of the respective + page in current extent descriptor + +Above all scenario done by 2 steps to make sure that there +will be no error scenario once the modification of the pages +has started. +1) prepare - Basically validates the necessary condition +and make sure that pages are being latched +2) Complete - Completes the action by using the latched +pages in prepare step */ +class PageOperator final +{ + /** Header block for the tablespace */ + buf_block_t *const m_header_block= nullptr; + /** Index node block */ + buf_block_t *const m_iblock= nullptr; + /** Index node */ + fseg_inode_t *const m_inode= nullptr; + /** offset of index node within index node page*/ + uint16_t m_ioffset= 0; + /** Maximum free limit of the tablespace */ + uint32_t m_free_limit= 0; + /** Segment id */ + uint64_t m_seg_id= 0; + /** Extent size */ + uint32_t m_extent_size= 0; + + /** New block to be allocated */ + buf_block_t *m_new_block= nullptr; + /** New block extent descriptor */ + buf_block_t *m_new_xdes= nullptr; + /** New block descriptor */ + xdes_t *m_new_descr= nullptr; + /** New block descriptor offset within xdes page */ + uint16_t m_xoffset= 0; + /** New extent descriptor state */ + uint32_t m_new_state= 0; + /** Need segment allocation */ + bool m_need_segment= false; + /** Old pages during allocation to be saved */ + buf_block_t *m_old_pages[8]= {nullptr}; + /** Page to be removed */ + byte m_old_page_no[4]= {0}; + /** Old block extent descriptor page */ + buf_block_t *m_old_xdes= nullptr; + /** Old block descriptor */ + xdes_t *m_old_descr= nullptr; + /** Old block descriptor offset with descriptor page */ + uint16_t m_old_xoffset= 0; + /** Old descriptor state */ + uint32_t m_old_state= 0; + /** Mini-transaction to allocate & free a page */ + mtr_t *const m_mtr; + + /** Save the old page state of the block before + allocating a page + @param block block to be stored + @return error code */ + dberr_t save_old_page(buf_block_t *block) noexcept + { + if (!block) return DB_SUCCESS; + size_t first_free; + for (first_free= 0; first_free < array_elements(m_old_pages); first_free++) + { + const buf_block_t *b= m_old_pages[first_free]; + if (!b) + goto found; + if (b->page.hash == &block->page) + return DB_SUCCESS; + } + return DB_CORRUPTION; +found: + buf_block_t *old= buf_LRU_get_free_block(have_no_mutex_soft); + if (!old) return DB_OUT_OF_MEMORY; + memcpy_aligned( + old->page.frame, block->page.frame, srv_page_size); + m_old_pages[first_free]= old; + old->page.hash= &block->page; + return DB_SUCCESS; + } + + /** Prepare the steps for free extent allocation by validating + FLST_PREV, FLST_NEXT of choosen extent descriptor + and their FLST_LEN of FSP_FREE list in FSP_HEADER_PAGE. + @return error code or DB_SUCCESS */ + dberr_t free_extent_prepare() noexcept + { + /* At least there should be 1 element in FSP_FREE list */ + byte *len= + &m_header_block->page.frame[FSP_HEADER_OFFSET + FSP_FREE + + FLST_LEN]; + if (mach_read_from_4(len) == 0) + return DB_CORRUPTION; + + buf_block_t *fsp_free_prev= nullptr; + buf_block_t *fsp_free_next= nullptr; + + dberr_t err= flst::remove_prepare(*m_new_xdes, m_xoffset, + m_free_limit, m_mtr, + &fsp_free_prev, &fsp_free_next); + if (err == DB_SUCCESS) + { + err= save_old_page(fsp_free_prev); + if (err == DB_SUCCESS) + err= save_old_page(fsp_free_next); + } + return err; + } + + /** Complete the free extent allocation */ + void free_extent_complete() noexcept + { + flst::remove_complete(m_header_block, FSP_HEADER_OFFSET + FSP_FREE, + m_new_descr, m_mtr); + fil_system.sys_space->free_len--; + } + + /** Prepare the steps to do the following + 1) free extent allocation + 2) Add the extent to FSEG_NOT_FULL list by validating the + last extent descriptor in FSEG_NOT_FULL list of segment inode + @return error code */ + dberr_t initialize_segment_prepare() noexcept + { + dberr_t err= free_extent_prepare(); + if (err) return err; + + buf_block_t *fseg_not_full_last= nullptr; + err= flst::append_prepare(*m_iblock, + uint16_t(m_ioffset + FSEG_NOT_FULL), + m_free_limit, m_mtr, &fseg_not_full_last); + if (err == DB_SUCCESS) + err= save_old_page(fseg_not_full_last); + return err; + } + + /** This function does the following + 1) Allocating the free extent + 2) Appending the extent to FSEG_NOT_FULL list in segment inode + 3) Mark the extent state as XDES_FSEG */ + void initialize_segment_complete() noexcept + { + free_extent_complete(); + flst::append_complete(m_iblock, + uint16_t(m_ioffset + FSEG_NOT_FULL), + m_new_xdes, m_xoffset, m_mtr); + + /* Update the FSEG_NOT_FULL_N_USED in inode */ + byte *p_not_full= m_inode + FSEG_NOT_FULL_N_USED; + m_mtr->write<4>(*m_iblock, p_not_full, + mach_read_from_4(p_not_full) + 1); + xdes_set_state(*m_new_xdes, m_new_descr, XDES_FSEG, m_mtr); + m_mtr->write<8,mtr_t::MAYBE_NOP>(*m_new_xdes, + m_new_descr + XDES_ID, + m_seg_id); + xdes_set_free(*m_new_xdes, m_new_descr, + m_new_block->page.id().page_no() % m_extent_size, + m_mtr); + } + + /** Prepare the steps for + 1) Allocating the free extent + 2) Adding the extent to FSP_FREE_FRAG list by validating + the last extent descriptor in FSP_FREE_FRAG list of FSP_HEADER page + @return error code */ + dberr_t initialize_free_frag_prepare() noexcept + { + dberr_t err= free_extent_prepare(); + if (err) return err; + + buf_block_t *fsp_free_frag_last= nullptr; + err= flst::append_prepare(*m_header_block, + FSP_HEADER_OFFSET + FSP_FREE_FRAG, + m_free_limit, m_mtr, &fsp_free_frag_last); + + if (err == DB_SUCCESS) + err= save_old_page(fsp_free_frag_last); + return err; + } + + /** This function does the following + 1) Allocating the free extent + 2) Appending the extent to FSP_FREE_FRAG list in FSP_HEADER page + 3) Mark the extent state as XDES_FREE_FRAG */ + void initialize_free_frag_complete() noexcept + { + free_extent_complete(); + flst::append_complete(m_header_block, + FSP_HEADER_OFFSET + FSP_FREE_FRAG, + m_new_xdes, m_xoffset, m_mtr); + + byte *n_frag_used= m_header_block->page.frame + + FSP_HEADER_OFFSET + FSP_FRAG_N_USED; + m_mtr->write<4>(*m_header_block, n_frag_used, + mach_read_from_4(n_frag_used) + 1); + + /* Allocate the extent state to FREE_FRAG & update FSP_FRAG_N_USED */ + xdes_set_state(*m_new_xdes, m_new_descr, XDES_FREE_FRAG, m_mtr); + xdes_set_free(*m_new_xdes, m_new_descr, + m_new_block->page.id().page_no() % m_extent_size, + m_mtr); + } + + /** Prepare the steps to + 1) Allocate a page from XDES_FSEG extent + 2) If the extent size is FSP_EXTENT_SIZE then + prepare the extent to move from FSEG_NOT_FULL to FSEG_FULL + list in segment inode by validating the last extent descriptor in + FSEG_FULL list and previous and next extent in FSEG_NOT_FULL list. + @return error code */ + dberr_t alloc_from_fseg_prepare() noexcept + { + uint32_t n_used= xdes_get_n_used(m_new_descr); + if (n_used < 1 || n_used >= m_extent_size) + return DB_CORRUPTION; + + if (n_used < m_extent_size) + return DB_SUCCESS; + + byte *lst= m_iblock->page.frame + uint16_t(m_ioffset + FSEG_NOT_FULL); + if (!mach_read_from_4(lst + FLST_LEN)) + return DB_CORRUPTION; + + buf_block_t *fseg_not_full_prev= nullptr; + buf_block_t *fseg_not_full_next= nullptr; + dberr_t err= flst::remove_prepare(*m_new_xdes, m_xoffset, + m_free_limit, m_mtr, + &fseg_not_full_prev, + &fseg_not_full_next); + if (err) return err; + + buf_block_t *fseg_full_last= nullptr; + err= flst::append_prepare(*m_iblock, + uint16_t(m_ioffset + FSEG_FULL), + m_free_limit, m_mtr, &fseg_full_last); + if (err == DB_SUCCESS) + { + err= save_old_page(fseg_not_full_prev); + if (err == DB_SUCCESS) + err= save_old_page(fseg_not_full_next); + if (err == DB_SUCCESS) + err= save_old_page(fseg_full_last); + } + return err; + } + + /** Does the following + 1) Complete the page allocation from file segment. + 2) If the extent size is FSP_EXTENT_SIZE then + i) Remove the extent from FSEG_NOT_FULL list + ii) Add the extent to FSEG_FULL */ + void alloc_from_fseg_complete() noexcept + { + xdes_set_free(*m_new_xdes, m_new_descr, + m_new_block->page.id().page_no() % m_extent_size, + m_mtr); + + byte *p_not_full= m_inode + FSEG_NOT_FULL_N_USED; + uint32_t n_used_val= mach_read_from_4(p_not_full) + 1; + + if (xdes_get_n_used(m_new_descr) == m_extent_size) + { + n_used_val-= FSP_EXTENT_SIZE; + m_mtr->write<4>(*m_iblock, p_not_full, n_used_val); + flst::remove_complete(m_iblock, + uint16_t(m_ioffset + FSEG_NOT_FULL), + m_new_descr, m_mtr); + flst::append_complete(m_iblock, + uint16_t(m_ioffset + FSEG_FULL), + m_new_xdes, m_xoffset, m_mtr); + } + else + m_mtr->write<4>(*m_iblock, p_not_full, n_used_val); + } + + /** Prepare the steps to + 1) Allocate the page from free fragment extent. + 2) If the extent size is FSP_EXTENT_SIZE then prepare the + steps to move the extent from FSP_FREE_FRAG to FSP_FULL_FRAG + list by validating the next, previous extent descriptor of + current extent descriptor in FSP_FREE_FRAG list and + last extent descriptor in FSP_FULL_FRAG list + @return error code */ + dberr_t alloc_from_free_frag_prepare() noexcept + { + uint32_t n_used= xdes_get_n_used(m_new_descr); + if (n_used < 1 || n_used >= m_extent_size) + return DB_CORRUPTION; + + if (n_used < m_extent_size) + return DB_SUCCESS; + + byte *lst= m_header_block->page.frame + FSP_HEADER_OFFSET + FSP_FREE_FRAG; + if (!mach_read_from_4(lst + FLST_LEN)) + return DB_CORRUPTION; + + buf_block_t *fsp_free_frag_prev= nullptr; + buf_block_t *fsp_free_frag_next= nullptr; + dberr_t err= flst::remove_prepare(*m_new_xdes, m_xoffset, + m_free_limit, m_mtr, + &fsp_free_frag_prev, + &fsp_free_frag_next); + if (err) return err; + + buf_block_t *fsp_full_frag_last= nullptr; + err= flst::append_prepare(*m_header_block, + FSP_HEADER_OFFSET + FSP_FULL_FRAG, + m_free_limit, m_mtr, + &fsp_full_frag_last); + + if (err == DB_SUCCESS) + { + err= save_old_page(fsp_free_frag_prev); + if (err == DB_SUCCESS) + err= save_old_page(fsp_free_frag_next); + if (err == DB_SUCCESS) + err= save_old_page(fsp_full_frag_last); + } + return err; + } + + /** Does the following + 1) Allocate the page from fragment extent + 2) If the extent size is FSP_EXTENT_SIZE then + i) remove the extent descriptor from FSP_FREE_FRAG list + ii) Add the extent descriptor in FSP_FULL_FRAG list */ + void alloc_from_free_frag_complete() noexcept + { + xdes_set_free(*m_new_xdes, m_new_descr, + m_new_block->page.id().page_no() % m_extent_size, + m_mtr); + + byte *frag_n_used= m_header_block->page.frame + FSP_HEADER_OFFSET + + FSP_FRAG_N_USED; + uint32_t n_used_frag= mach_read_from_4(frag_n_used) + 1; + + if (xdes_get_n_used(m_new_descr) == m_extent_size) + { + n_used_frag-= FSP_EXTENT_SIZE; + m_mtr->write<4>(*m_header_block, frag_n_used, n_used_frag); + flst::remove_complete(m_header_block, + FSP_HEADER_OFFSET + FSP_FREE_FRAG, + m_new_descr, m_mtr); + + flst::append_complete(m_header_block, + FSP_HEADER_OFFSET + FSP_FULL_FRAG, + m_new_xdes, m_xoffset, m_mtr); + } + else + m_mtr->write<4>(*m_header_block, frag_n_used, n_used_frag); + } + + /** Prepare the steps to free the page from fragment pages. + 1) Check the page exist in segment fragment array + 2) If the extent descriptor is in XDES_FULL_FRAG then + prepare the steps to move the extent descriptor + from FSP_FULL_FRAG to FSP_FREE_FRAG list by validating + the FLST_PREV, FLST_NEXT of current extent descriptor + and FLST_LAST in FSP_FREE_FRAG list + 3) If the extent is about to empty then prepare the steps + to move the extent descriptor from FSP_FREE_FRAG to FSP_FREE list + by validating the FLST_PREV, FLST_NEXT of current extent + descriptor and FLST_LAST in FSP_FREE list + @return error code */ + dberr_t free_from_frag_prepare() noexcept + { + uint32_t n_arr_slots= m_extent_size / 2; + bool page_exist= false; + for (ulint i= 0; i < n_arr_slots; i++) + { + if (!memcmp(m_inode + FSEG_FRAG_ARR + i * FSEG_FRAG_SLOT_SIZE, + m_old_page_no, 4)) + { + page_exist= true; + break; + } + } + + if (!page_exist) return DB_CORRUPTION; + + buf_block_t *fsp_full_frag_prev= nullptr; + buf_block_t *fsp_full_frag_next= nullptr; + buf_block_t *fsp_free_frag_last= nullptr; + dberr_t err= DB_SUCCESS; + uint32_t n_used= xdes_get_n_used(m_old_descr); + + if (m_old_state == XDES_FULL_FRAG) + { + if (n_used != m_extent_size) + return DB_CORRUPTION; + + byte *lst= m_header_block->page.frame + FSP_HEADER_OFFSET + FSP_FULL_FRAG; + if (!mach_read_from_4(lst + FLST_LEN)) + return DB_CORRUPTION; + + err= flst::remove_prepare(*m_old_xdes, m_old_xoffset, m_free_limit, + m_mtr, &fsp_full_frag_prev, + &fsp_full_frag_next); + + if (err) return err; + + return flst::append_prepare(*m_header_block, + FSP_HEADER_OFFSET + FSP_FREE_FRAG, + m_free_limit, m_mtr, + &fsp_free_frag_last); + } + + if (n_used >= m_extent_size || n_used == 0) + return DB_CORRUPTION; + + buf_block_t *fsp_free_frag_prev= nullptr; + buf_block_t *fsp_free_frag_next= nullptr; + buf_block_t *fsp_free_last= nullptr; + + if (n_used == 1) + { + byte *lst= m_header_block->page.frame + FSP_HEADER_OFFSET + FSP_FREE_FRAG; + if (!mach_read_from_4(lst + FLST_LEN)) + return DB_CORRUPTION; + + err= flst::remove_prepare(*m_old_xdes, m_old_xoffset, m_free_limit, + m_mtr, &fsp_free_frag_prev, + &fsp_free_frag_next); + if (err) return err; + + return flst::append_prepare(*m_header_block, + FSP_HEADER_OFFSET + FSP_FREE, + m_free_limit, m_mtr, + &fsp_free_last); + } + return err; + } + + /** Complete the removal of page from XDES_FREE_FRAG + (or) XDES_FULL_FRAG list. + 1) If the extent is from FSP_FULL_FRAG then move the + extent descriptor from FSP_FULL_FRAG to FSP_FREE_FRAG + 2) If the extent is from FSP_FREE_FRAG and no pages + has been used in that descr then move the extent + from FSP_FREE_FRAG to FSP_FREE */ + void free_from_frag_complete() noexcept + { + uint32_t old_page_no= mach_read_from_4(m_old_page_no); + m_mtr->free(*fil_system.sys_space, old_page_no); + xdes_set_free(*m_old_xdes, m_old_descr, + old_page_no % m_extent_size, m_mtr); + uint32_t n_used= xdes_get_n_used(m_old_descr); + byte *frag_n_used= m_header_block->page.frame + FSP_HEADER_OFFSET + + FSP_FRAG_N_USED; + uint32_t n_frag_used= mach_read_from_4(frag_n_used) - 1; + + for (size_t i= 0, frag= m_ioffset + FSEG_FRAG_ARR; + i < m_extent_size / 2; i++, frag += FSEG_FRAG_SLOT_SIZE) + { + if (!memcmp(m_iblock->page.frame + frag, m_old_page_no, 4)) + { + m_mtr->memset(m_iblock, frag, 4, 0xff); + break; + } + } + + if (n_used == m_extent_size - 1) + { + flst::remove_complete(m_header_block, + FSP_HEADER_OFFSET + FSP_FULL_FRAG, + m_old_descr, m_mtr); + + xdes_set_state(*m_old_xdes, m_old_descr, XDES_FREE_FRAG, m_mtr); + + flst::append_complete(m_header_block, + FSP_HEADER_OFFSET + FSP_FREE_FRAG, + m_old_xdes, m_old_xoffset, m_mtr); + + n_frag_used += m_extent_size; + } + else if (n_used == 0) + { + flst::remove_complete(m_header_block, + FSP_HEADER_OFFSET + FSP_FREE_FRAG, + m_old_descr, m_mtr); + + xdes_set_state(*m_old_xdes, m_old_descr, XDES_FREE, m_mtr); + + flst::append_complete(m_header_block, + FSP_HEADER_OFFSET + FSP_FREE, + m_old_xdes, m_old_xoffset, m_mtr); + } + m_mtr->write<4>(*m_header_block, frag_n_used, n_frag_used); + } + + /** Prepare the removal of page from file segment + 1) If the number of used pages in extent descriptor is + FSP_EXTENT_SIZE then move the extent descriptor from + FSEG_FULL to FSEG_NOT_FULL list by validating the + FLST_PREV, FLST_NEXT of current extent descriptor + and last extent descriptor in FSEG_NOT_FULL list + 2) If the number of used pages in extent descriptor is 0 + then move the extent descriptor from FSEG_NOT_FULL to + FSP_FREE list by validating the FLST_PREV, FLST_NEXT + of current extent descriptor and last extent descriptor + in FSP_FREE list + @return error code */ + dberr_t free_from_fseg_prepare() noexcept + { + if (memcmp(m_old_descr, m_inode + FSEG_ID, 8)) + return DB_CORRUPTION; + + uint32_t n_used= xdes_get_n_used(m_old_descr); + if (n_used == 0 || n_used > m_extent_size) + return DB_CORRUPTION; + + buf_block_t *fseg_full_prev= nullptr; + buf_block_t *fseg_full_next= nullptr; + buf_block_t *fseg_not_full_last= nullptr; + buf_block_t *fseg_not_full_prev= nullptr; + buf_block_t *fseg_not_full_next= nullptr; + buf_block_t *fsp_free_last= nullptr; + + dberr_t err= DB_SUCCESS; + + if (n_used == m_extent_size) + { + byte *lst= m_iblock->page.frame + uint16_t(m_ioffset + FSEG_FULL); + if (!mach_read_from_4(lst + FLST_LEN)) + return DB_CORRUPTION; + + err= flst::remove_prepare(*m_old_xdes, m_old_xoffset, m_free_limit, + m_mtr, &fseg_full_prev, + &fseg_full_next); + if (err) return err; + + err= flst::append_prepare(*m_iblock, + uint16_t(FSEG_NOT_FULL + m_ioffset), + m_free_limit, m_mtr, + &fseg_not_full_last); + if (err) return err; + } + else + { + uint32_t not_full_n_used= + mach_read_from_4(m_inode + FSEG_NOT_FULL_N_USED); + if (!not_full_n_used) return DB_CORRUPTION; + } + + if (n_used == 1) + { + byte *lst= m_iblock->page.frame + uint16_t(m_ioffset + FSEG_NOT_FULL); + if (!mach_read_from_4(lst + FLST_LEN)) + return DB_CORRUPTION; + + err= flst::remove_prepare(*m_old_xdes, m_old_xoffset, m_free_limit, + m_mtr, &fseg_not_full_prev, + &fseg_not_full_next); + if (err) return err; + + err= flst::append_prepare(*m_header_block, + FSP_FREE + FSP_HEADER_OFFSET, + m_free_limit, m_mtr, &fsp_free_last); + } + return err; + } + + /** Complete the removal of page from file segment + 1) If the extent is from FSEG_FULL then move the + extent descriptor from FSEG_FULL to FSEG_NOT_FULL + 2) If the extent is from FSEG_NOT_FULL then move the + extent descriptor to FSP_FREE */ + void free_from_fseg_complete() noexcept + { + uint32_t n_used= xdes_get_n_used(m_old_descr); + uint32_t old_page_no= mach_read_from_4(m_old_page_no); + m_mtr->free(*fil_system.sys_space, old_page_no); + xdes_set_free(*m_old_xdes, m_old_descr, + old_page_no % m_extent_size, m_mtr); + + byte* p_not_full = m_inode + FSEG_NOT_FULL_N_USED; + uint32_t not_full_n_used = mach_read_from_4(p_not_full) - 1; + if (n_used == m_extent_size) + { + flst::remove_complete(m_iblock, uint16_t(m_ioffset + FSEG_FULL), + m_old_descr, m_mtr); + flst::append_complete(m_iblock, + uint16_t(m_ioffset + FSEG_NOT_FULL), + m_old_xdes, m_old_xoffset, m_mtr); + not_full_n_used += m_extent_size; + } + m_mtr->write<4>(*m_iblock, p_not_full, not_full_n_used); + + if (n_used == 1) + { + flst::remove_complete(m_iblock, + uint16_t(m_ioffset + FSEG_NOT_FULL), + m_old_descr, m_mtr); + + xdes_set_state(*m_old_xdes, m_old_descr, XDES_FREE, m_mtr); + flst::append_complete(m_header_block, + FSP_HEADER_OFFSET + FSP_FREE, + m_old_xdes, m_old_xoffset, m_mtr); + fil_system.sys_space->free_len++; + } + } +public: + PageOperator(buf_block_t *header_block, buf_block_t *iblock, + fseg_inode_t *inode, + uint32_t extent_size, byte* old_page_no, + mtr_t *mtr) : + m_header_block(header_block), + m_iblock(iblock), m_inode(inode), m_extent_size(extent_size), + m_mtr(mtr) + { + if (old_page_no) + memcpy(m_old_page_no, old_page_no, 4); + m_free_limit= mach_read_from_4(FSP_HEADER_OFFSET + FSP_FREE_LIMIT + + m_header_block->page.frame); + m_seg_id= mach_read_from_8(m_inode + FSEG_ID); + } + + ~PageOperator() + { + for (buf_block_t *old : m_old_pages) + if (old) + { + old->page.hash= nullptr; + buf_block_free(old); + } + } + + + /** Get allocated new block */ + buf_block_t* get_new_block() const noexcept { return m_new_block; } + + /** Prepare the new page allocation from the new given extent + @param new_extent starting page of new extent + @param segment segment allocation + @return error code */ + dberr_t prepare_new_page(uint32_t new_extent, bool segment) noexcept + { + dberr_t err= DB_SUCCESS; + uint32_t size= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SIZE + + m_header_block->page.frame); + if (new_extent >= size || new_extent >= m_free_limit) + return DB_CORRUPTION; + + uint32_t new_descr_page_no= xdes_calc_descriptor_page(0, new_extent); + m_new_xdes= fsp_get_latched_page(page_id_t{0, new_descr_page_no}, + m_mtr, &err); + if (!m_new_xdes) + return err; + + ut_ad(!m_new_block); + m_ioffset= uint16_t(m_inode - m_iblock->page.frame); + m_need_segment= segment; + m_xoffset= uint16_t(xdes_calc_descriptor_index(0, new_extent) * XDES_SIZE + + XDES_ARR_OFFSET + XDES_FLST_NODE); + m_new_descr= m_new_xdes->page.frame + m_xoffset - XDES_FLST_NODE; + m_new_state= uint32_t(xdes_get_state(m_new_descr)); + uint32_t new_page= 0; + + /* Allocate the new extent and initialize the extent state + with XDES_FSEG/XDES_FREE_FRAG */ + if (m_new_state == XDES_FREE) + { + if (segment) err= initialize_segment_prepare(); + else err= initialize_free_frag_prepare(); + + if (err) return err; +new_page: + new_page= xdes_find_free(m_new_descr); + if (new_page == FIL_NULL) + return DB_CORRUPTION; + + new_page+= new_extent; + m_new_block= fsp_page_create(fil_system.sys_space, new_page, m_mtr); + err= save_old_page(m_header_block); + if (err == DB_SUCCESS) + err= save_old_page(m_iblock); + if (err == DB_SUCCESS) + err= save_old_page(m_new_xdes); + if (err == DB_SUCCESS) + err= save_old_page(m_new_block); + return err; + } + + uint32_t n_used= xdes_get_n_used(m_new_descr); + if (n_used == 0 || n_used >= m_extent_size) + return DB_CORRUPTION; + + /* Allocate the page from file segment */ + if (m_seg_id != FIL_NULL && m_new_state == XDES_FSEG && + mach_read_from_8(m_new_descr + XDES_ID) == m_seg_id) + err= alloc_from_fseg_prepare(); + /* Allocate the page from free frag */ + else if (m_new_state == XDES_FREE_FRAG || m_new_state == XDES_FULL_FRAG) + err= alloc_from_free_frag_prepare(); + else return DB_CORRUPTION; + + if (err) return err; + goto new_page; + } + + /** Complete the page allocation from FREE extent descriptor + or XDES_FSEG/XDES_FREE_FRAG extent list */ + void complete_new_page() noexcept + { + if (m_new_state == XDES_FREE) + { + if (m_need_segment) + return initialize_segment_complete(); + return initialize_free_frag_complete(); + } + if (m_new_state == XDES_FSEG) + return alloc_from_fseg_complete(); + return alloc_from_free_frag_complete(); + } + + /** Assign the fragment slot of the index node. + This step should be done after removing the old page + because there is a possiblity that FRAGMENT ARRAY + could be full. */ + void assign_frag_slot() noexcept + { + if ((!m_need_segment && m_new_state == XDES_FREE) || + m_new_state == XDES_FULL_FRAG || + m_new_state == XDES_FREE_FRAG) + fseg_set_nth_frag_page_no(m_inode, m_iblock, + fseg_find_free_frag_page_slot(m_inode), + m_new_block->page.id().page_no(), m_mtr); + } + + /** Restore the page modified during page allocation */ + void restore_old_pages() noexcept + { + for (buf_block_t *old : m_old_pages) + if (old) + memcpy_aligned( + old->page.hash->frame, old->page.frame, srv_page_size); + } + + /** Prepare the steps to remove the page from file segment + (or) fragment extent. + @return error code */ + dberr_t prepare_old_page() noexcept + { + uint32_t old_page_no= mach_read_from_4(m_old_page_no); + uint32_t old_descr_page_no= + xdes_calc_descriptor_page(0, old_page_no); + dberr_t err= DB_SUCCESS; + m_old_xdes= fsp_get_latched_page(page_id_t{0, old_descr_page_no}, + m_mtr, &err); + if (!m_old_xdes) + return err; + + m_old_xoffset= + uint16_t(xdes_calc_descriptor_index(0, old_page_no) * XDES_SIZE + + XDES_ARR_OFFSET + XDES_FLST_NODE); + + m_old_descr= m_old_xdes->page.frame + m_old_xoffset - XDES_FLST_NODE; + m_old_state= uint32_t(xdes_get_state(m_old_descr)); + if (m_old_state == XDES_FREE) + return DB_CORRUPTION; + + if (xdes_is_free(m_old_descr, old_page_no & (m_extent_size -1))) + return DB_CORRUPTION; + + m_ioffset= uint16_t(m_inode - m_iblock->page.frame); + return m_old_state == XDES_FSEG + ? free_from_fseg_prepare() + : free_from_frag_prepare(); + } + + /** Complete the removal of page operation */ + void complete_free_old_page() noexcept + { + return m_old_state == XDES_FSEG + ? free_from_fseg_complete() + : free_from_frag_complete(); + } +}; + + +class IndexDefragmenter final +{ + /** Parent block and its associate offset where + we store the child page number. This is stored + in the form of */ + std::unordered_map m_parent_pages; + + dict_index_t &m_index; + + buf_block_t *m_root; + /** Iterate through the page and map the child_page_no + with the parent page and their associate offset + in m_parent_pages + @param block block to be traversed */ + dberr_t get_child_pages(buf_block_t *block) noexcept; + + /** Get the first block for the given level + @param level level + @param mtr mini-transaction + @param cur_page_no first page number for the given level + @return error code or DB_SUCCESS */ + dberr_t get_level_block(uint16_t level, mtr_t *mtr, + uint32_t *cur_page_no) noexcept; + + /** Defragment the level of the index + @param level level to be defragmented + @param mtr mini-transaction + @param space_defrag space defragmenter information + and also responsible for allocating new + segment or page from tablespace + @return error code or DB_SUCCESS */ + dberr_t defragment_level(uint16_t level, mtr_t *mtr, + SpaceDefragmenter *space_defrag) noexcept; + +public: + IndexDefragmenter(dict_index_t &index): m_index(index) {} + + /** Defragment the index with the help of space defragmenter. + 1) Iterate through each level of the index + 2) Find out what are the pages/segment + to be modified for the index. + 3) Allocate the page from the new segment/extent + 4) Copy the to be changed page content to new page + 5) Change the associative pages in the tree with + new page(left, right, parent block) + 6) Do step (4), (5) within single mini-transaction + and commit the mini-transaction + @return error code or DB_SUCCESS */ + dberr_t defragment(SpaceDefragmenter *space_defrag) noexcept; +}; + +class SpaceDefragmenter final +{ + /** Extent is already allocated for defragmentation */ + static constexpr uint32_t XDES_USED= ~0U; + /** Store the extent information in the tablespace */ + std::map m_extent_info; + /** Map of last used extent with early unused extent within + the tablespace */ + std::map m_extent_map; + + /** Collect the extent information from tablespace */ + dberr_t extract_extent_state() noexcept + { + mtr_t mtr; + dberr_t err= DB_SUCCESS; + uint32_t last_descr_page_no= 0; + fil_space_t *space= fil_system.sys_space; + mtr.start(); + mtr.x_lock_space(space); + buf_block_t *last_descr= buf_page_get_gen(page_id_t{space->id, 0}, 0, + RW_S_LATCH, nullptr, + BUF_GET_POSSIBLY_FREED, &mtr, + &err); + if (!last_descr) + { +func_exit: + mtr.commit(); + return err; + } + + for (uint32_t xdes_n= 0; xdes_n < space->free_limit; + xdes_n+= m_extent_size) + { + /* Ignore doublewrite buffer extent */ + if (buf_dblwr.is_inside(xdes_n)) + continue; + uint32_t descr_page_no= + xdes_calc_descriptor_page(space->id, xdes_n); + if (descr_page_no != last_descr_page_no) + { + last_descr= buf_page_get_gen(page_id_t{space->id, xdes_n}, + 0, RW_S_LATCH, nullptr, + BUF_GET_POSSIBLY_FREED, &mtr, + &err); + if (!last_descr) + goto func_exit; + } + xdes_t *descr= XDES_ARR_OFFSET + XDES_SIZE * + xdes_calc_descriptor_index(0, xdes_n) + last_descr->page.frame; + last_descr_page_no= descr_page_no; + /* Ignore the extent descriptor extent */ + if (xdes_n % srv_page_size == 0 && xdes_get_n_used(descr) == 2) + continue; + m_extent_info[xdes_n]= xdes_get_state(descr); + } + goto func_exit; + } + + /** Find the earlier free extent for the given used extent + @param max_limit Find the extent below max limit extent + @return value + @retval FIL_NULL if there is no extent */ + uint32_t find_free_extent(uint32_t max_limit) noexcept + { + for (auto &extent_info : m_extent_info) + { + if (max_limit <= extent_info.first) + return FIL_NULL; + + if (extent_info.second == XDES_FREE) + { + /* Mark the extent as used one */ + extent_info.second = XDES_USED; + return extent_info.first; + } + } + return FIL_NULL; + } + + /** Defragment the indexes */ + dberr_t defragment_index(dict_index_t &index) noexcept + { + IndexDefragmenter index_defrag(index); + return index_defrag.defragment(this); + } + + /** Defragment the table */ + dberr_t defragment_table(const dict_table_t *table) noexcept + { + for (dict_index_t *index= dict_table_get_first_index(table); + index; index= dict_table_get_next_index(index)) + { + dberr_t err= fseg_validate(fil_system.sys_space, index); + if (err == DB_SUCCESS) + err= defragment_index(*index); + + if (err) + { + sql_print_error("InnoDB: Defragmentation of %s in %s failed: %s", + index->name, table->name.m_name, ut_strerr(err)); + return err; + } + } + return DB_SUCCESS; + } +public: + const uint32_t m_extent_size; + + SpaceDefragmenter() noexcept : m_extent_size(FSP_EXTENT_SIZE) {} + + /** Find the new extent for the existing last used extent + Iterate the tablespace from last and find out the free + extent in the beginning of the tablespace */ + dberr_t find_new_extents() noexcept + { + dberr_t err= extract_extent_state(); + if (err) return err; + + uint32_t free_limit= fil_system.sys_space->free_limit; + uint32_t fixed_size= srv_sys_space.get_min_size(); + while (free_limit > fixed_size) + { + uint32_t state= m_extent_info[free_limit]; + + switch (state) { + case XDES_USED: + goto func_exit; + case XDES_FREE: + goto prev_extent; + case XDES_FSEG: + case XDES_FULL_FRAG: + case XDES_FREE_FRAG: + uint32_t dest= find_free_extent(free_limit); + if (dest == FIL_NULL) + goto func_exit; + m_extent_map[free_limit]= dest; + break; + } +prev_extent: + free_limit-= FSP_EXTENT_SIZE; + } +func_exit: + if (m_extent_map.empty()) + return DB_SUCCESS_LOCKED_REC; + + sql_print_information("InnoDB: System tablespace defragmentation " + "process starts"); + sql_print_information("InnoDB: Moving the data from extents %" + PRIu32 " through %" PRIu32, + m_extent_map.begin()->first, + m_extent_map.rbegin()->first); + return DB_SUCCESS; + } + + /** Defragment the system tables */ + dberr_t defragment_system_tables() noexcept + { + dberr_t err= defragment_table(dict_sys.sys_tables); + if (err == DB_SUCCESS) + err= defragment_table(dict_sys.sys_columns); + if (err == DB_SUCCESS) + err= defragment_table(dict_sys.sys_indexes); + if (err == DB_SUCCESS) + err= defragment_table(dict_sys.sys_fields); + if (err == DB_SUCCESS) + err= defragment_table(dict_sys.sys_foreign); + if (err == DB_SUCCESS) + err= defragment_table(dict_sys.sys_foreign_cols); + if (err == DB_SUCCESS) + err= defragment_table(dict_sys.sys_virtual); + + if (err == DB_SUCCESS) + sql_print_information("InnoDB: Defragmentation of system " + "tablespace is successful"); + return err; + } + + /** @return extent which replaces the later extent + or same extent if there is no replacement exist */ + uint32_t get_new_extent(uint32_t old_extent) const noexcept + { + auto it= m_extent_map.find(old_extent); + if (it != m_extent_map.end()) + return it->second; + return old_extent; + } + + /** @return state for the given extent */ + uint32_t get_state(uint32_t extent) noexcept + { + return m_extent_info[extent]; + } +}; + +dberr_t IndexDefragmenter::get_child_pages(buf_block_t *block) noexcept +{ + const byte *page= block->page.frame; + const rec_t *rec= page_rec_get_next_low(page + PAGE_OLD_INFIMUM, false); + while (rec != page + PAGE_OLD_SUPREMUM) + { + ulint len; + ulint offset= rec_get_nth_field_offs_old(rec, + rec_get_n_fields_old(rec) - 1, + &len); + if (len != 4) + return DB_CORRUPTION; + + if (offset >= srv_page_size) + return DB_CORRUPTION; + + const byte *field= rec + offset; + /* m_parent_pages[child_page_no] = + 1st 32 bit to indicate offset in parent page + 2nd 32 bit to indicate parent page number */ + m_parent_pages[mach_read_from_4(field)]= + uint64_t(page_offset(field)) << 32 | block->page.id().page_no(); + rec= page_rec_get_next_low(rec, false); + } + return DB_SUCCESS; +} + +dberr_t IndexDefragmenter::get_level_block(uint16_t level, mtr_t *mtr, + uint32_t *cur_page_no) noexcept +{ + uint32_t child_page_no= m_index.page; + dberr_t err= DB_SUCCESS; + uint16_t prev_level= UINT16_MAX; + while (1) + { + buf_block_t *block= fsp_get_latched_page(page_id_t{0, child_page_no}, + mtr, &err); + if (!block) + return err; + + page_t *page= buf_block_get_frame(block); + uint16_t cur_level= btr_page_get_level(page); + if (cur_level == level) + break; + + if (prev_level == UINT16_MAX) + prev_level= cur_level; + else if (prev_level != cur_level + 1) + return DB_CORRUPTION; + + const rec_t *rec= page_rec_get_next_low(page + PAGE_OLD_INFIMUM, false); + if (rec && rec != page + PAGE_OLD_SUPREMUM) + { + ulint len; + rec+= rec_get_nth_field_offs_old(rec, rec_get_n_fields_old(rec) - 1, + &len); + if (len != 4 || rec + len - page > page_header_get_field(page, + PAGE_HEAP_TOP)) + return DB_CORRUPTION; + child_page_no= mach_read_from_4(rec); + } + else + return DB_CORRUPTION; + if (cur_level == level + 1) + break; + prev_level= cur_level; + } + *cur_page_no= child_page_no; + return err; +} + +dberr_t IndexDefragmenter::defragment_level( + uint16_t level, + mtr_t *mtr, + SpaceDefragmenter *space_defrag) noexcept +{ + uint32_t cur_page_no= FIL_NULL; + dberr_t err= get_level_block(level, mtr, &cur_page_no); + if (err) + return err; + + fil_space_t *const space= fil_system.sys_space; + uint32_t extent_size= space_defrag->m_extent_size; + + buf_block_t *block= fsp_get_latched_page(page_id_t{0, cur_page_no}, + mtr, &err); + if (!block) + return err; + + for (;;) + { + page_t *page= buf_block_get_frame(block); + uint32_t next_page_no= btr_page_get_next(page); + uint32_t cur_extent= (cur_page_no / extent_size) * extent_size; + uint32_t old_state= space_defrag->get_state(cur_extent); + + if (old_state == XDES_FREE) + { +fetch_next_page: + if (next_page_no == FIL_NULL) + break; + mtr->commit(); + cur_page_no= next_page_no; + + mtr->start(); + mtr->x_lock_space(space); + block= fsp_get_latched_page(page_id_t{0, cur_page_no}, + mtr, &err); + if (!block) + return err; + continue; + } + + uint32_t new_extent= space_defrag->get_new_extent(cur_extent); + /* There is no need for extent to be changed */ + if (new_extent == cur_extent) + { + if (level) + { + /* Store the child page number and their offset + exist in the parent block records */ + err= get_child_pages(block); + if (err) return err; + } + goto fetch_next_page; + } + + buf_block_t *header_block= + fsp_get_latched_page(page_id_t{0, 0}, mtr, &err); + if (!header_block) + return err; + + const fseg_header_t *seg_header= m_root->page.frame + + (level ? PAGE_HEADER + PAGE_BTR_SEG_TOP + : PAGE_HEADER + PAGE_BTR_SEG_LEAF); + + buf_block_t *iblock; + fseg_inode_t *inode= fseg_inode_try_get(seg_header, 0, 0, mtr, + &iblock, &err); + if (!inode) + return err; + + auto parent_it= m_parent_pages.find(cur_page_no); + if (parent_it == m_parent_pages.end()) + { + err= DB_CORRUPTION; + return err; + } + + uint32_t parent_page_no= uint32_t(parent_it->second); + + uint32_t parent_offset= uint32_t(parent_it->second >> 32); + + if (parent_offset >= srv_page_size - FIL_PAGE_DATA_END) + { + err= DB_CORRUPTION; + return err; + } + + PageOperator operation(header_block, iblock, inode, extent_size, + page + FIL_PAGE_OFFSET, mtr); + + AssociatedPages related_pages(block, mtr); + + err= operation.prepare_new_page(new_extent, old_state == XDES_FSEG); + + DBUG_EXECUTE_IF("allocation_prepare_fail", err= DB_CORRUPTION;); + if (err) + { +err_exit: + operation.restore_old_pages(); + mtr->discard_modifications(); + return err; + } + + err= related_pages.prepare(parent_page_no); + DBUG_EXECUTE_IF("relation_page_prepare_fail", err= DB_CORRUPTION;); + + if (err) goto err_exit; + + operation.complete_new_page(); + + /* After allocating the new page, try to prepare the steps + of page removal function. Because there is a possiblity that + last block in FSEG_NOT_FULL/FSP_FREE_FRAG/FSP_FREE last block + could've changed while allocating the new block. */ + err= operation.prepare_old_page(); + + DBUG_EXECUTE_IF("remover_prepare_fail", err= DB_CORRUPTION;); + if (err) goto err_exit; + + /* Copy the data from old block to new block */ + buf_block_t *new_block= operation.get_new_block(); + uint32_t new_page_no= new_block->page.id().page_no(); + /* Copy FIL_PAGE_PREV, FIL_PAGE_NEXT */ + mtr->memcpy(*new_block, + new_block->page.frame + FIL_PAGE_PREV, + block->page.frame + FIL_PAGE_PREV, + page_has_next(block->page.frame) ? 8 : 4); + mtr->memcpy(*new_block, new_block->page.frame + FIL_PAGE_TYPE, + block->page.frame + FIL_PAGE_TYPE, + srv_page_size - FIL_PAGE_TYPE - 8); + + /* Assign the new block page number in left, right + and parent block */ + related_pages.complete(new_page_no, parent_offset); + + /* Complete the page free operation */ + operation.complete_free_old_page(); + /* Add the new page in inode fragment array */ + operation.assign_frag_slot(); + + if (level) + { + err= get_child_pages(new_block); + if (err) return err; + } + goto fetch_next_page; + } + + ut_a(!fsp_tablespace_validate(space, mtr)); + ut_a(!fseg_validate_low(space, &m_index, mtr)); + if (level > 1) + { + mtr->commit(); + mtr->start(); + mtr->x_lock_space(space); + } + return DB_SUCCESS; +} + +dberr_t IndexDefragmenter::defragment(SpaceDefragmenter *space_defrag) noexcept +{ + mtr_t mtr; + mtr.start(); + dberr_t err= DB_SUCCESS; + m_index.lock.x_lock(SRW_LOCK_CALL); + fil_space_t *const space= fil_system.sys_space; + mtr.x_lock_space(space); + m_root= btr_root_block_get(&m_index, RW_S_LATCH, &mtr, &err); + if (!m_root) + { + mtr.commit(); + m_index.lock.x_unlock(); + return err; + } + + m_root->page.fix(); + mtr.release_last_page(); + uint16_t level= btr_page_get_level(m_root->page.frame); + while (1) + { + err= defragment_level(level, &mtr, space_defrag); + DBUG_EXECUTE_IF("fail_after_level_defragment", + if (m_index.table->id == 2 && level == 1) + err= DB_CORRUPTION;); + if (err || !level) + break; + level--; + } + ut_ad(err == DB_SUCCESS || !mtr.has_modifications()); + mtr.commit(); + m_index.lock.x_unlock(); + m_root->page.unfix(); + return err; +} + +/** check whether any user table exist in system tablespace +@retval DB_SUCCESS_LOCKED_REC if user table exist +@retval DB_SUCCESS if no user table exist +@retval DB_CORRUPTION if any error encountered */ +static dberr_t user_tables_exists() noexcept +{ + mtr_t mtr; + btr_pcur_t pcur; + dberr_t err= DB_SUCCESS; + mtr.start(); + for (const rec_t *rec= dict_startscan_system(&pcur, &mtr, + dict_sys.sys_tables); + rec; rec= dict_getnext_system(&pcur, &mtr)) + { + const byte *field= nullptr; + ulint len= 0; + if (rec_get_deleted_flag(rec, 0)) + { +corrupt: + sql_print_error("InnoDB: Encountered corrupted record in SYS_TABLES"); + err= DB_CORRUPTION; + goto func_exit; + } + field= rec_get_nth_field_old(rec, DICT_FLD__SYS_TABLES__SPACE, &len); + if (len != 4) + goto corrupt; + if (mach_read_from_4(field) != 0) + continue; + field= rec_get_nth_field_old(rec, DICT_FLD__SYS_TABLES__ID, &len); + if (len != 8) + goto corrupt; + if (!dict_sys.is_sys_table(mach_read_from_8(field))) + { + err= DB_SUCCESS_LOCKED_REC; + btr_pcur_close(&pcur); + goto func_exit; + } + } +func_exit: + mtr.commit(); + return err; +} + +dberr_t fil_space_t::defragment() noexcept +{ + ut_ad(this == fil_system.sys_space); + dberr_t err= user_tables_exists(); + if (err == DB_SUCCESS_LOCKED_REC) + { + sql_print_information( + "InnoDB: User table exists in the system tablespace." + "Please try to move the data from system tablespace " + "to separate tablespace before defragment the " + "system tablespace."); + return DB_SUCCESS; + } else if (err) { return err; } + + SpaceDefragmenter defragmenter; + err= defragmenter.find_new_extents(); + /* There is no free extent exist */ + if (err == DB_SUCCESS_LOCKED_REC) + return DB_SUCCESS; + + if (err == DB_SUCCESS) + err= defragmenter.defragment_system_tables(); + return err; +} + void fsp_system_tablespace_truncate(bool shutdown) { ut_ad(!purge_sys.enabled()); @@ -4030,6 +5769,16 @@ void fsp_system_tablespace_truncate(bool shutdown) return; } + if (!shutdown) + { + err= space->defragment(); + if (err) + { + srv_sys_space.set_shrink_fail(); + return; + } + } + mtr_t mtr; mtr.start(); mtr.x_lock_space(space); @@ -4064,7 +5813,11 @@ err_exit: fil_system.set_use_doublewrite(false); buf_block_t *header= nullptr; - ut_ad(!fsp_tablespace_validate(space)); +#ifdef UNIV_DEBUG + mtr.start(); + ut_ad(!fsp_tablespace_validate(space, &mtr)); + mtr.commit(); +#endif /* UNIV_DEBUG */ mtr.start(); mtr.x_lock_space(space); @@ -4090,7 +5843,7 @@ err_exit: UINT32PF " to " UINT32PF " pages", space->size, last_used_extent); - header= fsp_get_latched_xdes_page( + header= fsp_get_latched_page( page_id_t(space->id, 0), &mtr, &err); if (!header) goto err_exit; @@ -4132,7 +5885,11 @@ mtr_max: old_xdes_list.restore(&mtr); mtr.discard_modifications(); mtr.commit(); - ut_ad(!fsp_tablespace_validate(space)); +#ifdef UNIV_DEBUG + mtr.start(); + ut_ad(!fsp_tablespace_validate(space, &mtr)); + mtr.commit(); +#endif /* UNIV_DEBUG */ sql_print_error( "InnoDB: Cannot shrink the system tablespace " "because the mini-transaction log size (%zu bytes) " @@ -4206,7 +5963,7 @@ func_exit: UINT32PF " to " UINT32PF " pages", space->size, last_used_extent); - buf_block_t *header= fsp_get_latched_xdes_page( + buf_block_t *header= fsp_get_latched_page( page_id_t(space->id, 0), &mtr, &err); if (!header) goto func_exit; diff --git a/storage/innobase/fut/fut0lst.cc b/storage/innobase/fut/fut0lst.cc index ff876801242..11d424ae303 100644 --- a/storage/innobase/fut/fut0lst.cc +++ b/storage/innobase/fut/fut0lst.cc @@ -409,45 +409,56 @@ dberr_t flst_remove(buf_block_t *base, uint16_t boffset, return err; } -#ifdef UNIV_DEBUG /** Validate a file-based list. */ -void flst_validate(const buf_block_t *base, uint16_t boffset, mtr_t *mtr) +dberr_t flst_validate(const buf_block_t *base, uint16_t boffset, + mtr_t *mtr) noexcept { - ut_ad(boffset < base->physical_size()); + if (boffset >= base->physical_size()) + return DB_CORRUPTION; + ut_ad(mtr->memo_contains_flagged(base, MTR_MEMO_PAGE_X_FIX | MTR_MEMO_PAGE_SX_FIX)); const uint32_t len= flst_get_len(base->page.frame + boffset); fil_addr_t addr= flst_get_first(base->page.frame + boffset); + dberr_t err= DB_SUCCESS; for (uint32_t i= len; i--; ) { - ut_ad(addr.boffset >= FIL_PAGE_DATA); - ut_ad(addr.boffset < base->physical_size() - FIL_PAGE_DATA_END); + if (addr.boffset < FIL_PAGE_DATA || + addr.boffset >= base->physical_size() - FIL_PAGE_DATA_END) + return DB_CORRUPTION; const buf_block_t *b= buf_page_get_gen(page_id_t(base->page.id().space(), addr.page), - base->zip_size(), RW_SX_LATCH, nullptr, BUF_GET, mtr); - ut_ad(b); + base->zip_size(), RW_SX_LATCH, nullptr, BUF_GET, mtr, + &err); + if (!b) + return err; addr= flst_get_next_addr(b->page.frame + addr.boffset); mtr->release_last_page(); } - ut_ad(addr.page == FIL_NULL); + if (addr.page != FIL_NULL) + return DB_CORRUPTION; addr= flst_get_last(base->page.frame + boffset); for (uint32_t i= len; i--; ) { - ut_ad(addr.boffset >= FIL_PAGE_DATA); - ut_ad(addr.boffset < base->physical_size() - FIL_PAGE_DATA_END); + if (addr.boffset < FIL_PAGE_DATA || + addr.boffset >= base->physical_size() - FIL_PAGE_DATA_END) + return DB_CORRUPTION; const buf_block_t *b= buf_page_get_gen(page_id_t(base->page.id().space(), addr.page), - base->zip_size(), RW_SX_LATCH, nullptr, BUF_GET, mtr); - ut_ad(b); + base->zip_size(), RW_SX_LATCH, nullptr, BUF_GET, mtr, + &err); + if (!b) + return err; addr= flst_get_prev_addr(b->page.frame + addr.boffset); mtr->release_last_page(); } - ut_ad(addr.page == FIL_NULL); + if (addr.page != FIL_NULL) + return DB_CORRUPTION; + return err; } -#endif diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 751f8744052..afe3f3eedcc 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -1504,6 +1504,14 @@ public: bool load_sys_tables() noexcept; /** Create or check system tables on startup */ dberr_t create_or_check_sys_tables() noexcept; + + bool is_sys_table(table_id_t table_id) const noexcept + { + return (table_id > 0 && table_id <= 4) || + table_id == sys_foreign->id || + table_id == sys_foreign_cols->id || + table_id == sys_virtual->id; + } }; /** the data dictionary cache */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index cd1aa8ca336..4e0744b2207 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1021,6 +1021,12 @@ public: @param shutdown called during slow shutdown @return error code */ dberr_t garbage_collect(bool shutdown); + + /** Move InnoDB system tables closer to the start of + the tablespace. + @return error code + @retval DB_SUCCESS on successful operation */ + dberr_t defragment() noexcept; private: /** @return whether the file is usable for io() */ ATTRIBUTE_COLD bool prepare_acquired() noexcept; diff --git a/storage/innobase/include/fut0lst.h b/storage/innobase/include/fut0lst.h index dc8806a5c74..4a701fa0d8b 100644 --- a/storage/innobase/include/fut0lst.h +++ b/storage/innobase/include/fut0lst.h @@ -161,9 +161,8 @@ inline fil_addr_t flst_get_prev_addr(const flst_node_t *node) void flst_write_addr(const buf_block_t &block, byte *faddr, uint32_t page, uint16_t boffset, mtr_t *mtr); -# ifdef UNIV_DEBUG /** Validate a file-based list. */ -void flst_validate(const buf_block_t *base, uint16_t boffset, mtr_t *mtr); -# endif +dberr_t flst_validate(const buf_block_t *base, uint16_t boffset, + mtr_t *mtr) noexcept; #endif /* !UNIV_INNOCHECKSUM */ From 4de5161a70d3bfc07fa35fc000db29f6bef446d1 Mon Sep 17 00:00:00 2001 From: Nikita Malyavin Date: Mon, 7 Apr 2025 22:02:35 +0200 Subject: [PATCH 032/125] fix build with -fno-elide-constructors Linker is trying to find a copy constructor to injector::transaction::transaction ld.lld: error: undefined symbol: injector::transaction::transaction (injector::transaction const&) >>> referenced by rpl_injector.cc:164 >>> rpl_injector.cc.o:(injector::new_trans(THD*)) This constructor is declared, but is not implemented. Ok if copy elision is enabled, but causes error otherwise. Remove the constructor declaration, so that operator= will be used in reality. --- sql/rpl_injector.h | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/rpl_injector.h b/sql/rpl_injector.h index 6a1c724809a..446b319535c 100644 --- a/sql/rpl_injector.h +++ b/sql/rpl_injector.h @@ -146,7 +146,6 @@ public: }; transaction() : m_thd(NULL) { } - transaction(transaction const&); ~transaction(); /* Clear transaction, i.e., make calls to 'good()' return false. */ From e6ea5d568c0fb972a391a8c64716846ddb7c57ef Mon Sep 17 00:00:00 2001 From: Nikita Malyavin Date: Wed, 5 Mar 2025 15:05:59 +0100 Subject: [PATCH 033/125] MDEV-36507 fix dbug_print_row concurrent access 7544fd4cae had to make use of a static array to avoid memory use-after-free or leak. Instead, let us make a function returning String, this is the only way to automatically manage the memory after the function returned. To make it all correct, move constructor is added. Normally, it is expected, that the constructor will be elided upon return of an object by value, but if something goes different, or -fno-elide-constructors is used, we can have a problem. So this was a move constructor avoids copy elision-related UB. dbug_print_row returning char* is still there for convenient use in a debugger. --- sql/filesort.cc | 46 +++++++++++++++++++++++++++++----------------- sql/handler.cc | 12 ++++++++---- sql/handler.h | 2 +- sql/sql_string.h | 6 ++++++ 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/sql/filesort.cc b/sql/filesort.cc index 442597efe49..93152770cdc 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -637,19 +637,8 @@ static uchar *read_buffpek_from_file(IO_CACHE *buffpek_pointers, uint count, static char dbug_row_print_buf[4096]; -/* - Print table's current row into a buffer and return a pointer to it. - This is intended to be used from gdb: - - (gdb) p dbug_print_table_row(table) - $33 = "SUBQUERY2_t1(col_int_key,col_varchar_nokey)=(7,c)" - (gdb) - - Only columns in table->read_set are printed -*/ - -const char* dbug_print_row(TABLE *table, const uchar *rec, bool print_names) +String dbug_format_row(TABLE *table, const uchar *rec, bool print_names) { Field **pfield; char row_buff_tmp[512]; @@ -717,14 +706,37 @@ const char* dbug_print_row(TABLE *table, const uchar *rec, bool print_names) output.append(tmp.ptr(), tmp.length()); } } - output.append(")"); - if (output.c_ptr() == dbug_row_print_buf) - return dbug_row_print_buf; - else - return "Couldn't fit into buffer"; + output.append(')'); + + return output; } +/** + A function to display a row in debugger. + Example usage: + (gdb) p dbug_print_row(table, table->record[1]) +*/ +const char *dbug_print_row(TABLE *table, const uchar *rec) +{ + String row= dbug_format_row(table, table->record[0]); + if (row.length() > sizeof dbug_row_print_buf - 1) + return "Couldn't fit into buffer"; + memcpy(dbug_row_print_buf, row.c_ptr(), row.length()); + return dbug_row_print_buf; +} + +/** + Print table's current row into a buffer and return a pointer to it. + + This is intended to be used from gdb: + + (gdb) p dbug_print_table_row(table) + $33 = "SUBQUERY2_t1(col_int_key,col_varchar_nokey)=(7,c)" + (gdb) + + Only columns in table->read_set are printed +*/ const char* dbug_print_table_row(TABLE *table) { return dbug_print_row(table, table->record[0]); diff --git a/sql/handler.cc b/sql/handler.cc index d243a2149a8..7970aec0743 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -7366,7 +7366,8 @@ int handler::ha_write_row(const uchar *buf) TABLE_IO_WAIT(tracker, PSI_TABLE_WRITE_ROW, MAX_KEY, error, { error= write_row(buf); }) - DBUG_PRINT("dml", ("INSERT: %s = %d", dbug_print_row(table, buf, false), error)); + DBUG_PRINT("dml", ("INSERT: %s = %d", + dbug_format_row(table, buf, false).c_ptr_safe(), error)); MYSQL_INSERT_ROW_DONE(error); if (likely(!error)) @@ -7425,8 +7426,10 @@ int handler::ha_update_row(const uchar *old_data, const uchar *new_data) TABLE_IO_WAIT(tracker, PSI_TABLE_UPDATE_ROW, active_index, 0, { error= update_row(old_data, new_data);}) - DBUG_PRINT("dml", ("UPDATE: %s => %s = %d", dbug_print_row(table, old_data, false), - dbug_print_row(table, new_data, false), error)); + DBUG_PRINT("dml", ("UPDATE: %s => %s = %d", + dbug_format_row(table, old_data, false).c_ptr_safe(), + dbug_format_row(table, new_data, false).c_ptr_safe(), + error)); MYSQL_UPDATE_ROW_DONE(error); if (likely(!error)) @@ -7506,7 +7509,8 @@ int handler::ha_delete_row(const uchar *buf) TABLE_IO_WAIT(tracker, PSI_TABLE_DELETE_ROW, active_index, error, { error= delete_row(buf);}) - DBUG_PRINT("dml", ("DELETE: %s = %d", dbug_print_row(table, buf, false), error)); + DBUG_PRINT("dml", ("DELETE: %s = %d", + dbug_format_row(table, buf, false).c_ptr_safe(), error)); MYSQL_DELETE_ROW_DONE(error); if (likely(!error)) { diff --git a/sql/handler.h b/sql/handler.h index c06c27b08c9..46b26bc33ac 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -5324,6 +5324,6 @@ int get_select_field_pos(Alter_info *alter_info, int select_field_count, bool versioned); #ifndef DBUG_OFF -const char* dbug_print_row(TABLE *table, const uchar *rec, bool print_names= true); +String dbug_format_row(TABLE *table, const uchar *rec, bool print_names= true); #endif /* DBUG_OFF */ #endif /* HANDLER_INCLUDED */ diff --git a/sql/sql_string.h b/sql/sql_string.h index c618990e84e..7fe955e996c 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -512,6 +512,10 @@ public: extra_alloc= 0; alloced= thread_specific= 0; } + Binary_string(Binary_string &&str) noexcept + { + move(str); + } ~Binary_string() { free(); } @@ -856,6 +860,8 @@ public: Binary_string(str, len) { } String(const String &str) = default; + String(String &&str) noexcept + :Charset(std::move(str)), Binary_string(std::move(str)){} void set(String &str,size_t offset,size_t arg_length) { From 4a701e8ce4a178faf955641caf59fc13fcc718e9 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2024 14:39:09 +1100 Subject: [PATCH 034/125] MDEV-35640 Protocol_text: handle errors in allocation Protocol_text constructor allocated memory on an optional prealloc argument. This was along with its subclass Protocol_local. Remove this functionality to an independent member function "allocate". Replace two use with existing error handling on insufficient resources at the err label. Protocol_local, that inherits from Protocol_text had one use that requested an allocation of 0. So it wasn't needed. --- sql/protocol.cc | 10 ++++++++-- sql/protocol.h | 10 ++++------ sql/sql_prepare.cc | 6 +++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/sql/protocol.cc b/sql/protocol.cc index d2ef52e0887..c1e0cd24701 100644 --- a/sql/protocol.cc +++ b/sql/protocol.cc @@ -1202,7 +1202,10 @@ bool Protocol::send_result_set_metadata(List *list, uint flags) { List_iterator_fast it(*list); Item *item; - Protocol_text prot(thd, thd->variables.net_buffer_length); + Protocol_text prot(thd); + + if (prot.allocate(thd->variables.net_buffer_length)) + goto err; #ifndef DBUG_OFF field_handlers= (const Type_handler **) thd->alloc( sizeof(field_handlers[0]) * list->elements); @@ -1251,7 +1254,10 @@ bool Protocol::send_list_fields(List *list, const TABLE_LIST *table_list) DBUG_ENTER("Protocol::send_list_fields"); List_iterator_fast it(*list); Field *fld; - Protocol_text prot(thd, thd->variables.net_buffer_length); + Protocol_text prot(thd); + + if (prot.allocate(thd->variables.net_buffer_length)) + goto err; #ifndef DBUG_OFF field_handlers= (const Type_handler **) thd->alloc(sizeof(field_handlers[0]) * diff --git a/sql/protocol.h b/sql/protocol.h index 09dbdfbde2b..22ddeb943fe 100644 --- a/sql/protocol.h +++ b/sql/protocol.h @@ -199,12 +199,10 @@ class Protocol_text :public Protocol StringBuffer buffer; bool store_numeric_string_aux(const char *from, size_t length); public: - Protocol_text(THD *thd_arg, ulong prealloc= 0) - :Protocol(thd_arg) - { - if (prealloc) - packet->alloc(prealloc); - } + Protocol_text(THD *thd_arg) + :Protocol(thd_arg) {}; + bool __attribute__((warn_unused_result)) + allocate(size_t size) { return packet->alloc(size); } void prepare_for_resend() override; bool store_null() override; bool store_tiny(longlong from) override; diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 56fa6e775a3..35f67b43a3d 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -5493,8 +5493,8 @@ public: my_bool do_log_bin; - Protocol_local(THD *thd_arg, THD *new_thd_arg, ulong prealloc) : - Protocol_text(thd_arg, prealloc), + Protocol_local(THD *thd_arg, THD *new_thd_arg) : + Protocol_text(thd_arg), cur_data(0), first_data(0), data_tail(&first_data), alloc(0), new_thd(new_thd_arg), do_log_bin(FALSE) {} @@ -6310,7 +6310,7 @@ extern "C" MYSQL *mysql_real_connect_local(MYSQL *mysql) else new_thd= NULL; - p= new Protocol_local(thd_orig, new_thd, 0); + p= new Protocol_local(thd_orig, new_thd); if (new_thd) new_thd->protocol= p; else From 60638a84e81144c94be873680960cc5c081fdce1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 13 Apr 2025 12:11:34 +0200 Subject: [PATCH 035/125] MDEV-36586 USER_STATISTICS.BUSY_TIME is in microseconds the bug was that MDEV-35720 missed two lines followup for 95975b921e9 --- mysql-test/main/userstat.result | 7 ++++++- mysql-test/main/userstat.test | 7 ++++++- plugin/userstat/client_stats.cc | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/mysql-test/main/userstat.result b/mysql-test/main/userstat.result index 5315317e33a..d7c5a66bde9 100644 --- a/mysql-test/main/userstat.result +++ b/mysql-test/main/userstat.result @@ -247,6 +247,11 @@ select f() from information_schema.index_statistics; ERROR 21000: Subquery returns more than 1 row set global userstat= 0; drop function f; -# # End of 10.2 tests # +# MDEV-36586 USER_STATISTICS.BUSY_TIME is in microseconds +# +select distinct busy_time>1e5, cpu_time>1e5 from information_schema.user_statistics; +busy_time>1e5 cpu_time>1e5 +0 0 +# End of 10.11 tests diff --git a/mysql-test/main/userstat.test b/mysql-test/main/userstat.test index 0c49acaa19e..d14228ed1b9 100644 --- a/mysql-test/main/userstat.test +++ b/mysql-test/main/userstat.test @@ -135,6 +135,11 @@ set global userstat= 0; drop function f; --enable_ps2_protocol ---echo # --echo # End of 10.2 tests + --echo # +--echo # MDEV-36586 USER_STATISTICS.BUSY_TIME is in microseconds +--echo # +select distinct busy_time>1e5, cpu_time>1e5 from information_schema.user_statistics; + +--echo # End of 10.11 tests diff --git a/plugin/userstat/client_stats.cc b/plugin/userstat/client_stats.cc index 72c71785606..42d510b4a9b 100644 --- a/plugin/userstat/client_stats.cc +++ b/plugin/userstat/client_stats.cc @@ -45,8 +45,8 @@ static int send_user_stats(THD* thd, HASH *all_user_stats, TABLE *table) table->field[j++]->store((longlong)user_stats->total_connections,TRUE); table->field[j++]->store((longlong)user_stats->concurrent_connections, TRUE); table->field[j++]->store((longlong)user_stats->connected_time, TRUE); - table->field[j++]->store((double)user_stats->busy_time); - table->field[j++]->store((double)user_stats->cpu_time); + table->field[j++]->store((double)user_stats->busy_time/1e6); + table->field[j++]->store((double)user_stats->cpu_time/1e6); table->field[j++]->store((longlong)user_stats->bytes_received, TRUE); table->field[j++]->store((longlong)user_stats->bytes_sent, TRUE); table->field[j++]->store((longlong)user_stats->binlog_bytes_written, TRUE); From 6cff704e578aa54dc8c2dfbeaf1960bf98219463 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 27 Nov 2024 16:10:09 +1100 Subject: [PATCH 036/125] MDEV-35512 remove all RPM compat packages The end of life of SLES 12 combined with the EOL for Centos/RHEL 7 marks a happy time in our distribution maintaince where we can drop all RPM compat packages. They where only needed for distros that we released at the time of MariaDB-10.1. All those have finally gone EOL. Horray! --- cmake/cpack_rpm.cmake | 47 ------------------------------------------- 1 file changed, 47 deletions(-) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index d11a303539b..cde918d7089 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -344,53 +344,6 @@ ENDIF() SET(PYTHON_SHEBANG "/usr/bin/python3" CACHE STRING "python shebang") -# If we want to build build MariaDB-shared-compat, -# extract compat libraries from MariaDB-shared-5.3 rpm -FILE(GLOB compat53 RELATIVE ${CMAKE_SOURCE_DIR} - "${CMAKE_SOURCE_DIR}/../MariaDB-shared-5.3.*.rpm") -FILE(GLOB compat101 RELATIVE ${CMAKE_SOURCE_DIR} - "${CMAKE_SOURCE_DIR}/../MariaDB-shared-10.1.*.rpm") -IF(compat53 AND compat101) - FOREACH(compat_rpm "${compat53}" "${compat101}") - MESSAGE(STATUS "Using ${compat_rpm} to build MariaDB-compat") - INSTALL(CODE "EXECUTE_PROCESS( - COMMAND rpm2cpio ${CMAKE_SOURCE_DIR}/${compat_rpm} - COMMAND cpio --extract --make-directories */libmysqlclient*.so.* - - WORKING_DIRECTORY \$ENV{DESTDIR}) - EXECUTE_PROCESS( - COMMAND chmod -R a+rX . - WORKING_DIRECTORY \$ENV{DESTDIR})" - COMPONENT Compat) - ENDFOREACH() - - EXECUTE_PROCESS( - COMMAND rpm -q --provides -p "${CMAKE_SOURCE_DIR}/${compat101}" - ERROR_QUIET - OUTPUT_VARIABLE compat_provides) - EXECUTE_PROCESS( - COMMAND rpm -q --obsoletes -p "${CMAKE_SOURCE_DIR}/${compat101}" - ERROR_QUIET - OUTPUT_VARIABLE compat_obsoletes) - - STRING(REPLACE "\n" " " compat_provides "${compat_provides}") - STRING(REPLACE "\n" " " compat_obsoletes "${compat_obsoletes}") - STRING(REGEX REPLACE "[^ ]+\\([^ ]+ *" "" compat_obsoletes "${compat_obsoletes}") - SETA(CPACK_RPM_compat_PACKAGE_PROVIDES "${compat_provides}") - SETA(CPACK_RPM_compat_PACKAGE_OBSOLETES "${compat_obsoletes}") - - SET(CPACK_COMPONENTS_ALL ${CPACK_COMPONENTS_ALL} Compat) - - # RHEL6/CentOS6 install Postfix by default, and it requires - # libmysqlclient.so.16 that pulls in mysql-libs-5.1.x - # And the latter conflicts with our rpms. - # Make sure that for these distributions all our rpms require - # MariaDB-compat, that will replace mysql-libs-5.1 - IF(RPM MATCHES "(rhel|centos)[67]") - SET(CPACK_RPM_common_PACKAGE_REQUIRES "MariaDB-compat") - SET(CPACK_RPM_compat_PACKAGE_CONFLICTS "mariadb-libs < 1:10.1.0") - ENDIF() -ENDIF() - ################ IF(CMAKE_VERSION VERSION_GREATER "3.9.99") From a096f12ff75595ce51fedf879b71640576f70e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 14 Apr 2025 08:33:10 +0300 Subject: [PATCH 037/125] MDEV-29445 fixup: debug assertion buf_buddy_alloc_from(): Pass the correct argument to buf_pool.contains_zip(). This fixes a failure of the test encryption.innochecksum when the code is built with cmake -DWITH_UBSAN=ON -DCMAKE_BUILD_TYPE=Debug --- storage/innobase/buf/buf0buddy.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/buf/buf0buddy.cc b/storage/innobase/buf/buf0buddy.cc index 12620ec8a2a..6da47d84307 100644 --- a/storage/innobase/buf/buf0buddy.cc +++ b/storage/innobase/buf/buf0buddy.cc @@ -385,7 +385,7 @@ static void *buf_buddy_alloc_from(void *buf, ulint i) ut_ad(i >= buf_buddy_get_slot(UNIV_ZIP_SIZE_MIN)); ut_ad(i <= BUF_BUDDY_SIZES); ut_ad(!ut_align_offset(buf, srv_page_size)); - ut_ad(!buf_pool.contains_zip(buf, srv_page_size)); + ut_ad(!buf_pool.contains_zip(buf, srv_page_size_shift)); /* Add the unused parts of the block to the free lists. */ for (ulint j = BUF_BUDDY_SIZES, offs = srv_page_size; j-- > i; ) { From 14389b69932a83a47e3fdbf1a6e2c40625013288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 14 Apr 2025 09:22:39 +0300 Subject: [PATCH 038/125] MDEV-36510 InnoDB fails to compile with clang++-20 ut_allocator(PSI_memory_key): Remove the explicit keyword, to allow implicit instantiation of purge_sys_t::unordered_map. --- storage/innobase/include/ut0new.h | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index 3ff5f8853e0..398dd0dcc9e 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -277,7 +277,6 @@ public: #ifdef UNIV_PFS_MEMORY /** Default constructor. */ - explicit ut_allocator(PSI_memory_key key = PSI_NOT_INSTRUMENTED) : m_key(key) { From a524ec5951d68cec7ef6228093ea1b1993a3243c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 14 Apr 2025 10:33:22 +0300 Subject: [PATCH 039/125] MDEV-36587 InnoDB uses too much memory log_t::clear_mmap(): Do not modify buf_size; we may have file_size==0 here during bootstrap. log_t::set_recovered(): If we are writing to a memory-mapped log, update log_sys.buf_size to the record payload area of log_sys.buf. This fixes up commit acd071f599f416ddb4821dec485c4d912844213f (MDEV-21923). --- storage/innobase/log/log0log.cc | 5 ++--- storage/innobase/log/log0recv.cc | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 22994e389c9..64ca9e6fff6 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1237,11 +1237,10 @@ void log_t::clear_mmap() noexcept #ifdef HAVE_PMEM if (!is_opened()) { - latch.wr_lock(SRW_LOCK_CALL); + ut_d(latch.wr_lock(SRW_LOCK_CALL)); ut_ad(!resize_in_progress()); ut_ad(get_lsn() == get_flushed_lsn(std::memory_order_relaxed)); - buf_size= unsigned(std::min(capacity(), buf_size_max)); - latch.wr_unlock(); + ut_d(latch.wr_unlock()); return; } #endif diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 30b99ea0bc7..4162f60fee5 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -4590,7 +4590,10 @@ inline void log_t::set_recovered() noexcept } #ifdef HAVE_PMEM else + { + buf_size= unsigned(std::min(capacity(), buf_size_max)); mprotect(buf, size_t(file_size), PROT_READ | PROT_WRITE); + } #endif } From ba34657cd22fc831bdfd06fd088644b288043652 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 22 Jan 2025 22:08:56 +0100 Subject: [PATCH 040/125] MDEV-35238 (MDEV-34922) Wrong results from a tables with a single record and an aggregate The problem is that copy function was used in field list but never copied in this execution path. So copy should be performed before returning result. Protection against uninitialized copy usage added. --- mysql-test/main/group_by.result | 64 +++++++++++++++++++ mysql-test/main/group_by.test | 22 +++++++ .../type_inet/type_inet6_engines.inc | 13 ++++ .../type_inet/type_inet6_innodb.result | 12 ++++ .../type_inet/type_inet6_memory.result | 12 ++++ .../type_inet/type_inet6_myisam.result | 12 ++++ sql/item.cc | 8 +++ sql/item.h | 22 ++++++- sql/sql_select.cc | 2 + 9 files changed, 166 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/group_by.result b/mysql-test/main/group_by.result index 881febb13a4..27f2c34a6d4 100644 --- a/mysql-test/main/group_by.result +++ b/mysql-test/main/group_by.result @@ -2997,5 +2997,69 @@ UPDATE t1 SET c=1 ORDER BY (SELECT c); ERROR 42S22: Reference 'c' not supported (forward reference in item list) DROP TABLE t1; # +# MDEV-35238: Wrong results from a tables with a single record and an aggregate +# +CREATE OR REPLACE TABLE t1 (a int) ENGINE=myisam; +SELECT 1+0, min(1) FROM t1 WHERE if(uuid_short(), a,1); +1+0 min(1) +1 NULL +explain format=json SELECT 1+0, min(1) FROM t1 WHERE if(uuid_short(), a,1); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "message": "Impossible WHERE noticed after reading const tables" + } + } +} +INSERT INTO t1 VALUES (NULL); +SELECT 1+0, min(1) FROM t1 WHERE if(uuid_short(), a,1); +1+0 min(1) +1 NULL +explain format=json SELECT 1+0, min(1) FROM t1 WHERE if(uuid_short(), a,1); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "pseudo_bits_condition": "if(uuid_short(),NULL,1)", + "table": { + "table_name": "t1", + "access_type": "system", + "rows": 1, + "filtered": 100 + } + } +} +DROP TABLE t1; +CREATE TABLE t1 (a int PRIMARY KEY) ENGINE=myisam; +INSERT INTO t1 VALUES (1); +CREATE TABLE t2 (a int NOT NULL) ENGINE=myisam; +INSERT INTO t2 VALUES (10); +SELECT 1+0, MIN(t1.a) FROM t1,t2 WHERE t2.a = rand(); +1+0 MIN(t1.a) +1 1 +explain format=json SELECT 1+0, MIN(t1.a) FROM t1,t2 WHERE t2.a = rand(); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "pseudo_bits_condition": "10 = rand()", + "table": { + "table_name": "t1", + "access_type": "system", + "rows": 1, + "filtered": 100 + }, + "table": { + "table_name": "t2", + "access_type": "system", + "rows": 1, + "filtered": 100 + } + } +} +DROP TABLE t1,t2; +# # End of 10.5 tests # diff --git a/mysql-test/main/group_by.test b/mysql-test/main/group_by.test index eaa0d060fe9..19f2e6582ae 100644 --- a/mysql-test/main/group_by.test +++ b/mysql-test/main/group_by.test @@ -2152,6 +2152,28 @@ UPDATE t1 SET c=1 ORDER BY (SELECT c); UPDATE t1 SET c=1 ORDER BY (SELECT c); DROP TABLE t1; +--echo # +--echo # MDEV-35238: Wrong results from a tables with a single record and an aggregate +--echo # +CREATE OR REPLACE TABLE t1 (a int) ENGINE=myisam; +SELECT 1+0, min(1) FROM t1 WHERE if(uuid_short(), a,1); +explain format=json SELECT 1+0, min(1) FROM t1 WHERE if(uuid_short(), a,1); +INSERT INTO t1 VALUES (NULL); +SELECT 1+0, min(1) FROM t1 WHERE if(uuid_short(), a,1); +explain format=json SELECT 1+0, min(1) FROM t1 WHERE if(uuid_short(), a,1); +DROP TABLE t1; + +CREATE TABLE t1 (a int PRIMARY KEY) ENGINE=myisam; +INSERT INTO t1 VALUES (1); + +CREATE TABLE t2 (a int NOT NULL) ENGINE=myisam; +INSERT INTO t2 VALUES (10); + +SELECT 1+0, MIN(t1.a) FROM t1,t2 WHERE t2.a = rand(); +explain format=json SELECT 1+0, MIN(t1.a) FROM t1,t2 WHERE t2.a = rand(); + +DROP TABLE t1,t2; + --echo # --echo # End of 10.5 tests --echo # diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6_engines.inc b/plugin/type_inet/mysql-test/type_inet/type_inet6_engines.inc index 596036fc0ee..9cf91169528 100644 --- a/plugin/type_inet/mysql-test/type_inet/type_inet6_engines.inc +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6_engines.inc @@ -36,3 +36,16 @@ SELECT * FROM t1 WHERE a=CAST('::ff' AS INET6); EXPLAIN EXTENDED SELECT * FROM t1 WHERE a=CAST('::ff' AS INET6); DROP TABLE t1; + +--echo # +--echo # MDEV-34922: Assertion `value.length() == FbtImpl::binary_length()' failed in +--echo # Type_handler_fbt::Field_fbt::store_native, +--echo # Assertion `item->null_value' failed in Type_handler::Item_send_str +--echo # + +CREATE TABLE t1 (a datetime); +INSERT INTO t1 VALUES (NULL); +SELECT * FROM (SELECT cast('::' AS INET6),min(1) FROM t1 WHERE if(uuid_short(), a,1)) dt; +DROP TABLE t1; + +--echo # End of 10.5 tests diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6_innodb.result b/plugin/type_inet/mysql-test/type_inet/type_inet6_innodb.result index deb1c718b49..dd5e3ba9f7f 100644 --- a/plugin/type_inet/mysql-test/type_inet/type_inet6_innodb.result +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6_innodb.result @@ -88,6 +88,18 @@ Warnings: Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = INET6'::ff' DROP TABLE t1; # +# MDEV-34922: Assertion `value.length() == FbtImpl::binary_length()' failed in +# Type_handler_fbt::Field_fbt::store_native, +# Assertion `item->null_value' failed in Type_handler::Item_send_str +# +CREATE TABLE t1 (a datetime); +INSERT INTO t1 VALUES (NULL); +SELECT * FROM (SELECT cast('::' AS INET6),min(1) FROM t1 WHERE if(uuid_short(), a,1)) dt; +cast('::' AS INET6) min(1) +:: NULL +DROP TABLE t1; +# End of 10.5 tests +# # MDEV-26742 Assertion `field->type_handler() == this' failed in FixedBinTypeBundle::Type_handler_fbt::stored_field_cmp_to_item # CREATE TABLE t1 (pk inet6, c text) engine=myisam; diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6_memory.result b/plugin/type_inet/mysql-test/type_inet/type_inet6_memory.result index e805b167136..9bf8e58fb0b 100644 --- a/plugin/type_inet/mysql-test/type_inet/type_inet6_memory.result +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6_memory.result @@ -155,5 +155,17 @@ Warnings: Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = INET6'::ff' DROP TABLE t1; # +# MDEV-34922: Assertion `value.length() == FbtImpl::binary_length()' failed in +# Type_handler_fbt::Field_fbt::store_native, +# Assertion `item->null_value' failed in Type_handler::Item_send_str +# +CREATE TABLE t1 (a datetime); +INSERT INTO t1 VALUES (NULL); +SELECT * FROM (SELECT cast('::' AS INET6),min(1) FROM t1 WHERE if(uuid_short(), a,1)) dt; +cast('::' AS INET6) min(1) +:: NULL +DROP TABLE t1; +# End of 10.5 tests +# # End of 10.5 tests # diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6_myisam.result b/plugin/type_inet/mysql-test/type_inet/type_inet6_myisam.result index c2577ef44df..c05830f55c7 100644 --- a/plugin/type_inet/mysql-test/type_inet/type_inet6_myisam.result +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6_myisam.result @@ -88,6 +88,18 @@ Warnings: Note 1003 select `test`.`t1`.`a` AS `a` from `test`.`t1` where `test`.`t1`.`a` = INET6'::ff' DROP TABLE t1; # +# MDEV-34922: Assertion `value.length() == FbtImpl::binary_length()' failed in +# Type_handler_fbt::Field_fbt::store_native, +# Assertion `item->null_value' failed in Type_handler::Item_send_str +# +CREATE TABLE t1 (a datetime); +INSERT INTO t1 VALUES (NULL); +SELECT * FROM (SELECT cast('::' AS INET6),min(1) FROM t1 WHERE if(uuid_short(), a,1)) dt; +cast('::' AS INET6) min(1) +:: NULL +DROP TABLE t1; +# End of 10.5 tests +# # MDEV-26742 Assertion `field->type_handler() == this' failed in FixedBinTypeBundle::Type_handler_fbt::stored_field_cmp_to_item # CREATE TABLE t1 (c varchar(64), key(c)) engine=myisam; diff --git a/sql/item.cc b/sql/item.cc index 7143bced98b..9498d816bb0 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -5218,6 +5218,7 @@ bool Item_param::assign_default(Field *field) double Item_copy_string::val_real() { + DBUG_ASSERT(copied_in); int err_not_used; char *end_not_used; return (null_value ? 0.0 : @@ -5228,6 +5229,7 @@ double Item_copy_string::val_real() longlong Item_copy_string::val_int() { + DBUG_ASSERT(copied_in); int err; return null_value ? 0 : str_value.charset()->strntoll(str_value.ptr(), str_value.length(), 10, @@ -5237,6 +5239,7 @@ longlong Item_copy_string::val_int() int Item_copy_string::save_in_field(Field *field, bool no_conversions) { + DBUG_ASSERT(copied_in); return save_str_value_in_field(field, &str_value); } @@ -5247,11 +5250,15 @@ void Item_copy_string::copy() if (res && res != &str_value) str_value.copy(*res); null_value=item->null_value; +#ifndef DBUG_OFF + copied_in= 1; +#endif } /* ARGSUSED */ String *Item_copy_string::val_str(String *str) { + DBUG_ASSERT(copied_in); // Item_copy_string is used without fix_fields call if (null_value) return (String*) 0; @@ -5261,6 +5268,7 @@ String *Item_copy_string::val_str(String *str) my_decimal *Item_copy_string::val_decimal(my_decimal *decimal_value) { + DBUG_ASSERT(copied_in); // Item_copy_string is used without fix_fields call if (null_value) return (my_decimal *) 0; diff --git a/sql/item.h b/sql/item.h index ce02b0edda1..d60d5c01985 100644 --- a/sql/item.h +++ b/sql/item.h @@ -6383,8 +6383,15 @@ protected: Type_std_attributes::set(item); name= item->name; set_handler(item->type_handler()); +#ifndef DBUG_OFF + copied_in= 0; +#endif } +#ifndef DBUG_OFF + bool copied_in; +#endif + public: /** @@ -6451,7 +6458,10 @@ public: double val_real() override; longlong val_int() override; bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override - { return get_date_from_string(thd, ltime, fuzzydate); } + { + DBUG_ASSERT(copied_in); + return get_date_from_string(thd, ltime, fuzzydate); + } void copy() override; int save_in_field(Field *field, bool no_conversions) override; Item *do_get_copy(THD *thd) const override @@ -6481,9 +6491,13 @@ public: null_value= tmp.is_null(); m_value= tmp.is_null() ? Timestamp_or_zero_datetime() : Timestamp_or_zero_datetime(tmp); +#ifndef DBUG_OFF + copied_in=1; +#endif } int save_in_field(Field *field, bool) override { + DBUG_ASSERT(copied_in); DBUG_ASSERT(sane()); if (null_value) return set_field_to_null(field); @@ -6492,30 +6506,35 @@ public: } longlong val_int() override { + DBUG_ASSERT(copied_in); DBUG_ASSERT(sane()); return null_value ? 0 : m_value.to_datetime(current_thd).to_longlong(); } double val_real() override { + DBUG_ASSERT(copied_in); DBUG_ASSERT(sane()); return null_value ? 0e0 : m_value.to_datetime(current_thd).to_double(); } String *val_str(String *to) override { + DBUG_ASSERT(copied_in); DBUG_ASSERT(sane()); return null_value ? NULL : m_value.to_datetime(current_thd).to_string(to, decimals); } my_decimal *val_decimal(my_decimal *to) override { + DBUG_ASSERT(copied_in); DBUG_ASSERT(sane()); return null_value ? NULL : m_value.to_datetime(current_thd).to_decimal(to); } bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate) override { + DBUG_ASSERT(copied_in); DBUG_ASSERT(sane()); bool res= m_value.to_TIME(thd, ltime, fuzzydate); DBUG_ASSERT(!res); @@ -6523,6 +6542,7 @@ public: } bool val_native(THD *thd, Native *to) override { + DBUG_ASSERT(copied_in); DBUG_ASSERT(sane()); return null_value || m_value.to_native(to, decimals); } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index f0da265f86b..bb7cc72e623 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -20788,6 +20788,8 @@ do_select(JOIN *join, Procedure *procedure) */ clear_tables(join, &cleared_tables); } + if (join->tmp_table_param.copy_funcs.elements) + copy_fields(&join->tmp_table_param); if (!join->having || join->having->val_int()) { List *columns_list= (procedure ? &join->procedure_fields_list : From cb7e39b75bfc18813f818404c83a4fb6dc519320 Mon Sep 17 00:00:00 2001 From: Nikita Malyavin Date: Tue, 8 Apr 2025 21:57:11 +0200 Subject: [PATCH 041/125] MDEV-36181 Field pointer may be uninitialized in fill_record Newer gcc reports: error: 'rfield' may be used uninitialized [-Werror=maybe-uninitialized] 9041 | unwind_stored_field_offsets(fields, rfield); After investigation, it turned to be an impossible case: 1. The only way it could be broken is if if (!(field= fld->field_for_view_update())) line case would succeed from the first time. 2. Consequent checks initialize rfield. fld may return NULL in field_for_view_update() only for views. 3. Before fill_record, UPDATE first calls check_fields, where field_for_view_update() result is already checked. INSERT calls check_view_insertability that checks that all view fields are updateable. It all means that field_for_view_update() cannot be NULL in fill_record, so the if can be converted to DBUG_ASSERT. This essentially shifts the responsibility on preliminary field_for_view_update() check to the caller. In this patch: 1. convert field_for_view_update() check to DBUG_ASSERT 2. harden unwind_stored_field_offsets function so that it can be used even if field_for_view_update() is NULL 3. As a consequence, `field` is passed instead of `rfield` as a terminator. 4. Initialize `field` to NULL to bypass a false-positive warning! --- sql/sql_base.cc | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 09dac838c3f..ac4bce1dce0 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -8485,14 +8485,15 @@ err_no_arena: } -static void unwind_stored_field_offsets(const List &fields, Field *end) +static void unwind_stored_field_offsets(const List &fields, Item_field *end) { - for (Item &item_field: fields) + for (Item &item: fields) { - Field *f= item_field.field_for_view_update()->field; - if (f == end) + Item_field *item_field= item.field_for_view_update(); + if (item_field == end) break; + Field *f= item_field->field; if (f->stored_in_db()) { TABLE *table= f->table; @@ -8537,7 +8538,7 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, { List_iterator_fast f(fields),v(values); Item *value, *fld; - Item_field *field; + Item_field *field= NULL; Field *rfield; TABLE *table; bool only_unvers_fields= update && table_arg->versioned(); @@ -8555,11 +8556,8 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, while ((fld= f++)) { - if (!(field= fld->field_for_view_update())) - { - my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), fld->name.str); - goto err_unwind_fields; - } + field= fld->field_for_view_update(); + DBUG_ASSERT(field); // ensured by check_fields or check_view_insertability. value=v++; DBUG_ASSERT(value); rfield= field->field; @@ -8621,7 +8619,7 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, DBUG_RETURN(thd->is_error()); err_unwind_fields: if (update && thd->variables.sql_mode & MODE_SIMULTANEOUS_ASSIGNMENT) - unwind_stored_field_offsets(fields, rfield); + unwind_stored_field_offsets(fields, field); err: DBUG_PRINT("error",("got error")); thd->abort_on_warning= save_abort_on_warning; From ec5068fe59b6711d47217c75c13ab45bdd2c6a54 Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko Date: Fri, 7 Jun 2024 00:52:29 +0300 Subject: [PATCH 042/125] MDEV-34998: master can stop responding after cluster vote to evict a node After cluster vote to evict a node that failed a transaction, current master can't commit anymore. Error voting for joiner in the JOINED state was broken because the group-wide commit cut (implicit SUCCESS vote) was not taken into account when processing error vote request from the JOINED node. This commit adds 3 MTR tests to verify the fix in the galera library works as designed. Requires Galera library commit 91f0090a05e96c3cc353b80d961ede45cefb9279 (galera library version > 26.4.19). Signed-off-by: Julius Goryavsky --- .../galera/r/galera_vote_during_ist.result | 109 ++++++++++++ .../galera/r/galera_vote_joined_apply.result | 93 +++++++++++ .../galera/r/galera_vote_joined_skip.result | 101 +++++++++++ .../suite/galera/t/galera_vote_during_ist.cnf | 20 +++ .../galera/t/galera_vote_during_ist.test | 158 ++++++++++++++++++ .../galera/t/galera_vote_joined_apply.cnf | 21 +++ .../galera/t/galera_vote_joined_apply.test | 73 ++++++++ .../galera/t/galera_vote_joined_begin.inc | 74 ++++++++ .../suite/galera/t/galera_vote_joined_end.inc | 33 ++++ .../galera/t/galera_vote_joined_skip.cnf | 21 +++ .../galera/t/galera_vote_joined_skip.test | 100 +++++++++++ scripts/wsrep_sst_rsync.sh | 7 + 12 files changed, 810 insertions(+) create mode 100644 mysql-test/suite/galera/r/galera_vote_during_ist.result create mode 100644 mysql-test/suite/galera/r/galera_vote_joined_apply.result create mode 100644 mysql-test/suite/galera/r/galera_vote_joined_skip.result create mode 100644 mysql-test/suite/galera/t/galera_vote_during_ist.cnf create mode 100644 mysql-test/suite/galera/t/galera_vote_during_ist.test create mode 100644 mysql-test/suite/galera/t/galera_vote_joined_apply.cnf create mode 100644 mysql-test/suite/galera/t/galera_vote_joined_apply.test create mode 100644 mysql-test/suite/galera/t/galera_vote_joined_begin.inc create mode 100644 mysql-test/suite/galera/t/galera_vote_joined_end.inc create mode 100644 mysql-test/suite/galera/t/galera_vote_joined_skip.cnf create mode 100644 mysql-test/suite/galera/t/galera_vote_joined_skip.test diff --git a/mysql-test/suite/galera/r/galera_vote_during_ist.result b/mysql-test/suite/galera/r/galera_vote_during_ist.result new file mode 100644 index 00000000000..2dd295cce85 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_vote_during_ist.result @@ -0,0 +1,109 @@ +connection node_4; +connection node_3; +connection node_2; +connection node_1; +connection node_1; +connection node_2; +connection node_3; +connection node_4; +connection node_1; +CREATE TABLE t1(pk INT AUTO_INCREMENT PRIMARY KEY); +CREATE PROCEDURE p1(IN max INT) +BEGIN +DECLARE i INT; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET i = 0; +WHILE i < max DO +INSERT IGNORE INTO t1 VALUES (DEFAULT); +SET i = i + 1; +END WHILE; +END| +CALL p1(130); +connection node_4; +Shutting down server 4... +connection node_1; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_2; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_3; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +Server 4 left the cluster +connection node_1; +CALL p1(130); +connection node_1; +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; +connection node_2; +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; +connection node_3; +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; +INSERT INTO t2 VALUES (DEFAULT); +CALL p1(130); +connection node_1; +SET GLOBAL debug = "+d,sync.wsrep_sst_donor_after_donation"; +Restarting server 4 +Wait for server 1 to become a donor +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_sst_donor_after_donation_reached"; +Server 1 got SST request from server 4 +SET SESSION DEBUG_SYNC = "now SIGNAL signal.wsrep_sst_donor_after_donation_continue"; +SET GLOBAL debug = ""; +SET DEBUG_SYNC='RESET'; +Waiting for server 4 to leave the cluster +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_2; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_3; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_4; +Server 4 left the cluster, killing it... +Killed server 4... +Restarting server 4... +connection node_1; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_1; +SELECT count(*) AS expect1_390 FROM t1; +expect1_390 +390 +SELECT count(*) AS expect1_1 FROM t2; +expect1_1 +1 +connection node_2; +SELECT count(*) AS expect2_390 FROM t1; +expect2_390 +390 +SELECT count(*) AS expect2_1 FROM t2; +expect2_1 +1 +connection node_3; +SELECT count(*) AS expect3_390 FROM t1; +expect3_390 +390 +SELECT count(*) AS expect3_1 FROM t2; +expect3_1 +1 +connection node_4; +SELECT count(*) AS expect4_390 FROM t1; +expect4_390 +390 +SELECT count(*) AS expect4_1 FROM t2; +expect4_1 +1 +DROP TABLE t1; +DROP TABLE t2; +DROP PROCEDURE p1; +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); +CALL mtr.add_suppression("Inconsistency detected: Failed on preordered"); +CALL mtr.add_suppression("Failed to apply write set"); diff --git a/mysql-test/suite/galera/r/galera_vote_joined_apply.result b/mysql-test/suite/galera/r/galera_vote_joined_apply.result new file mode 100644 index 00000000000..b71487c83b9 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_vote_joined_apply.result @@ -0,0 +1,93 @@ +connection node_4; +connection node_3; +connection node_2; +connection node_1; +connection node_1; +connection node_2; +connection node_3; +connection node_4; +connection node_1; +CREATE TABLE t1(pk INT AUTO_INCREMENT PRIMARY KEY); +CREATE PROCEDURE p1(IN max INT) +BEGIN +DECLARE i INT; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET i = 0; +WHILE i < max DO +INSERT IGNORE INTO t1 VALUES (DEFAULT); +SET i = i + 1; +END WHILE; +END| +CALL p1(130); +connection node_4; +Shutting down server 4... +connection node_1; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +SET GLOBAL debug = "+d,sync.wsrep_donor_state"; +connection node_4; +Restarting server 4... +connection node_1; +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_donor_state_reached"; +Tables on server 1 flushed and locked for SST to server 4 +SET SESSION DEBUG_SYNC = "now SIGNAL signal.wsrep_donor_state"; +SET GLOBAL debug = ""; +SET DEBUG_SYNC='RESET'; +Wait for the state snapshot to be copied to server 4 +SST script unlocked server 1 +connection node_1; +CALL p1(130); +connection node_1; +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; +connection node_2; +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; +connection node_3; +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; +INSERT INTO t2 VALUES (DEFAULT); +CALL p1(130); +Waiting for server 4 to leave the cluster +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_2; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_1; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_4; +Server 4 left the cluster, killing it... +Killed server 4... +Restarting server 4... +DROP TABLE t2; +connection node_1; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_1; +SELECT count(*) AS expect1_390 FROM t1; +expect1_390 +390 +connection node_2; +SELECT count(*) AS expect2_390 FROM t1; +expect2_390 +390 +connection node_3; +SELECT count(*) AS expect3_390 FROM t1; +expect3_390 +390 +connection node_4; +SELECT count(*) AS expect4_390 FROM t1; +expect4_390 +390 +DROP TABLE t1; +DROP PROCEDURE p1; +connection node_4; +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); +CALL mtr.add_suppression("Inconsistency detected: Inconsistent by consensus"); +CALL mtr.add_suppression("Failed to apply write set: gtid:"); diff --git a/mysql-test/suite/galera/r/galera_vote_joined_skip.result b/mysql-test/suite/galera/r/galera_vote_joined_skip.result new file mode 100644 index 00000000000..427c34f0f39 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_vote_joined_skip.result @@ -0,0 +1,101 @@ +connection node_4; +connection node_3; +connection node_2; +connection node_1; +connection node_1; +connection node_2; +connection node_3; +connection node_4; +connection node_1; +CREATE TABLE t1(pk INT AUTO_INCREMENT PRIMARY KEY); +CREATE PROCEDURE p1(IN max INT) +BEGIN +DECLARE i INT; +DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; +SET i = 0; +WHILE i < max DO +INSERT IGNORE INTO t1 VALUES (DEFAULT); +SET i = i + 1; +END WHILE; +END| +CALL p1(130); +connection node_4; +Shutting down server 4... +connection node_1; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +SET GLOBAL debug = "+d,sync.wsrep_donor_state"; +connection node_4; +Restarting server 4... +connection node_1; +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_donor_state_reached"; +Tables on server 1 flushed and locked for SST to server 4 +SET SESSION DEBUG_SYNC = "now SIGNAL signal.wsrep_donor_state"; +SET GLOBAL debug = ""; +SET DEBUG_SYNC='RESET'; +Wait for the state snapshot to be copied to server 4 +SST script unlocked server 1 +connection node_1; +CALL p1(130); +connection node_3; +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; +INSERT INTO t2 VALUES (DEFAULT); +SET SESSION wsrep_on = OFF; +connection node_1; +CALL p1(130); +Waiting for server 3 to leave the cluster +connection node_1; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_2; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_4; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_3; +Server 3 left the cluster, killing it... +Killed server 3. +Restarting server 3... +Waiting for server 3 to rejoin the cluster +connection node_1; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_3; +sleeping for 20 +Waiting ready +Server 3 restarted. +connection node_1; +SET SESSION wsrep_on = ON; +SET SESSION wsrep_sync_wait = 15; +connection node_1; +SELECT count(*) AS expect1_390 FROM t1; +expect1_390 +390 +connection node_2; +SELECT count(*) AS expect2_390 FROM t1; +expect2_390 +390 +connection node_3; +SELECT count(*) AS expect3_390 FROM t1; +expect3_390 +390 +connection node_4; +SELECT count(*) AS expect4_390 FROM t1; +expect4_390 +390 +DROP TABLE t1; +DROP PROCEDURE p1; +connection node_1; +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); +connection node_2; +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); +connection node_3; +CALL mtr.add_suppression("Vote 0 \\(success\\) on .* is inconsistent with group"); +connection node_4; +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); diff --git a/mysql-test/suite/galera/t/galera_vote_during_ist.cnf b/mysql-test/suite/galera/t/galera_vote_during_ist.cnf new file mode 100644 index 00000000000..d6c7d771d5d --- /dev/null +++ b/mysql-test/suite/galera/t/galera_vote_during_ist.cnf @@ -0,0 +1,20 @@ +!include ../galera_4nodes.cnf + +[mysqld] +wsrep-ignore-apply-errors=0 + +[mysqld.1] +wsrep_node_name='node_1' + +[mysqld.2] +wsrep_node_name='node_2' + +[mysqld.3] +wsrep_node_name='node_3' + +[mysqld.4] +wsrep_node_name='node_4' +wsrep_sst_donor='node_1' + +[ENV] +galera_cluster_size=4 diff --git a/mysql-test/suite/galera/t/galera_vote_during_ist.test b/mysql-test/suite/galera/t/galera_vote_during_ist.test new file mode 100644 index 00000000000..ee29d08567d --- /dev/null +++ b/mysql-test/suite/galera/t/galera_vote_during_ist.test @@ -0,0 +1,158 @@ +# +# Test a case where a joiner encounters an error during IST +# Instead of voting it should assume error and bail out. +# + +--source include/galera_cluster.inc +--source include/big_test.inc +--source include/have_debug_sync.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--let $node_4=node_4 +--source ../include/auto_increment_offset_save.inc + +# create table t1 and procedure p1 to generate wirtesets +--connection node_1 +CREATE TABLE t1(pk INT AUTO_INCREMENT PRIMARY KEY); + +DELIMITER |; +CREATE PROCEDURE p1(IN max INT) +BEGIN + DECLARE i INT; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + + SET i = 0; + WHILE i < max DO + INSERT IGNORE INTO t1 VALUES (DEFAULT); + SET i = i + 1; + END WHILE; +END| +DELIMITER ;| + +CALL p1(130); + +--connection node_4 +--echo Shutting down server 4... +--let $node_4_server_id= `SELECT @@server_id` +--let $node_4_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$node_4_server_id.expect +--let $node_4_pid_file= `SELECT @@pid_file` +--source include/shutdown_mysqld.inc + +# Wait for node #4 to leave cluster +--let $members = 3 +--connection node_1 +--source include/wsrep_wait_membership.inc +--connection node_2 +--source include/wsrep_wait_membership.inc +--connection node_3 +--source include/wsrep_wait_membership.inc +--echo Server 4 left the cluster + +# Create some writesets for IST +--connection node_1 +CALL p1(130); + +# Create a writeset that node 4 won't be able to apply by creating a table +# that won't be present in the replication stream +--connection node_1 +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; + +--connection node_2 +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; + +--connection node_3 +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; + +# This should cause error during IST +INSERT INTO t2 VALUES (DEFAULT); + +# make sure nodes 1,2,3 progress far enough for commit cut update +CALL p1(130); + +--connection node_1 +# prepare to stop SST donor thread when it receives a request from starting node #4 +SET GLOBAL debug = "+d,sync.wsrep_sst_donor_after_donation"; + +--echo Restarting server 4 +# Need to use this form instead of start_mysqld.inc because the latter is blocking +--exec echo "restart:$start_mysqld_params" > $node_4_expect_file_name + +--echo Wait for server 1 to become a donor +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_sst_donor_after_donation_reached"; +--echo Server 1 got SST request from server 4 +SET SESSION DEBUG_SYNC = "now SIGNAL signal.wsrep_sst_donor_after_donation_continue"; +SET GLOBAL debug = ""; +SET DEBUG_SYNC='RESET'; + +# +# After this point node #4 shall proceed to IST and bail out +# + +--echo Waiting for server 4 to leave the cluster +--let $members = 3 +--source include/wsrep_wait_membership.inc +--connection node_2 +--source include/wsrep_wait_membership.inc +--connection node_3 +--source include/wsrep_wait_membership.inc + +--connection node_4 +--echo Server 4 left the cluster, killing it... + +# Kill the connected server +--exec echo "wait" > $node_4_expect_file_name +--let KILL_NODE_PIDFILE = $node_4_pid_file +--perl + my $pid_filename = $ENV{'KILL_NODE_PIDFILE'}; + my $mysqld_pid = `cat $pid_filename`; + chomp($mysqld_pid); + system("kill -9 $mysqld_pid"); + exit(0); +EOF +--echo Killed server 4... +--source include/wait_until_disconnected.inc +--echo Restarting server 4... +--source include/start_mysqld.inc +--source include/galera_wait_ready.inc + +# Confirm node #4 has rejoined +--connection node_1 +--let $members = 4 +--source include/wsrep_wait_membership.inc + +# Confirm that all is good and all nodes have identical data + +--connection node_1 +SELECT count(*) AS expect1_390 FROM t1; +SELECT count(*) AS expect1_1 FROM t2; + +--connection node_2 +SELECT count(*) AS expect2_390 FROM t1; +SELECT count(*) AS expect2_1 FROM t2; + +--connection node_3 +SELECT count(*) AS expect3_390 FROM t1; +SELECT count(*) AS expect3_1 FROM t2; + +--connection node_4 +SELECT count(*) AS expect4_390 FROM t1; +SELECT count(*) AS expect4_1 FROM t2; + +DROP TABLE t1; +DROP TABLE t2; +DROP PROCEDURE p1; + +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); +CALL mtr.add_suppression("Inconsistency detected: Failed on preordered"); +CALL mtr.add_suppression("Failed to apply write set"); + +--source ../include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_vote_joined_apply.cnf b/mysql-test/suite/galera/t/galera_vote_joined_apply.cnf new file mode 100644 index 00000000000..88a1982bf66 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_vote_joined_apply.cnf @@ -0,0 +1,21 @@ +!include ../galera_4nodes.cnf + +[mysqld] +wsrep-ignore-apply-errors=0 + +[mysqld.1] +wsrep_node_name='node_1' + +[mysqld.2] +wsrep_node_name='node_2' + +[mysqld.3] +wsrep_node_name='node_3' + +[mysqld.4] +wsrep_node_name='node_4' +wsrep_sst_donor='node_1' + +[ENV] +galera_cluster_size=4 +MTR_SST_JOINER_DELAY=20 diff --git a/mysql-test/suite/galera/t/galera_vote_joined_apply.test b/mysql-test/suite/galera/t/galera_vote_joined_apply.test new file mode 100644 index 00000000000..d903b83dc0e --- /dev/null +++ b/mysql-test/suite/galera/t/galera_vote_joined_apply.test @@ -0,0 +1,73 @@ +# +# Test a case where a vote happens in JOINED state after SST on a writeset +# that should be applied. +# + +--source galera_vote_joined_begin.inc +# +# At this point state snapshot has been copied, node 1 is operational and +# we have about 10 seconds while everything we do will go into the replication +# queue on node 4 which it will have to apply on top of the snapshot. +# + +# Increase replication queue on node_4 +--connection node_1 +CALL p1(130); + +# Create a writeset that node 4 won't be able to apply by creating a table +# that won't be present in the replication stream +--connection node_1 +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; + +--connection node_2 +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; + +--connection node_3 +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; + +# This should cause node #4 to initiate a vote and leave the cluster +INSERT INTO t2 VALUES (DEFAULT); + +# make sure nodes 1,2,3 progress far enough for commit cut update +CALL p1(130); + +--echo Waiting for server 4 to leave the cluster +--let $members = 3 +--source include/wsrep_wait_membership.inc +--connection node_2 +--source include/wsrep_wait_membership.inc +--connection node_1 +--source include/wsrep_wait_membership.inc + +--connection node_4 +--echo Server 4 left the cluster, killing it... +# Kill the connected server +--exec echo "wait" > $node_4_expect_file_name +--let KILL_NODE_PIDFILE = $node_4_pid_file +--perl + my $pid_filename = $ENV{'KILL_NODE_PIDFILE'}; + my $mysqld_pid = `cat $pid_filename`; + chomp($mysqld_pid); + system("kill -9 $mysqld_pid"); + exit(0); +EOF +--echo Killed server 4... +--source include/wait_until_disconnected.inc +--echo Restarting server 4... +--source include/start_mysqld.inc +--source include/galera_wait_ready.inc +DROP TABLE t2; + +--source galera_vote_joined_end.inc + +--connection node_4 +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); +CALL mtr.add_suppression("Inconsistency detected: Inconsistent by consensus"); +CALL mtr.add_suppression("Failed to apply write set: gtid:"); diff --git a/mysql-test/suite/galera/t/galera_vote_joined_begin.inc b/mysql-test/suite/galera/t/galera_vote_joined_begin.inc new file mode 100644 index 00000000000..abc59caee81 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_vote_joined_begin.inc @@ -0,0 +1,74 @@ +# This file purpose is to set up node 4 to require SST which is artificaially +# prolonged and as a result accumulate sufficient relication queue. +# The contents of the qeuee are controlled in the sourcing test files. + +--source include/galera_cluster.inc +--source include/big_test.inc +--source include/have_debug_sync.inc + +--let $node_1=node_1 +--let $node_2=node_2 +--let $node_3=node_3 +--let $node_4=node_4 +--source ../include/auto_increment_offset_save.inc + +# create table t1 and procedure p1 to generate wirtesets +--connection node_1 +CREATE TABLE t1(pk INT AUTO_INCREMENT PRIMARY KEY); + +DELIMITER |; +CREATE PROCEDURE p1(IN max INT) +BEGIN + DECLARE i INT; + DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; + + SET i = 0; + WHILE i < max DO + INSERT IGNORE INTO t1 VALUES (DEFAULT); + SET i = i + 1; + END WHILE; +END| +DELIMITER ;| + +# 130 events move the commit cut, it is essential in voting +CALL p1(130); + +--connection node_4 +--echo Shutting down server 4... +--let $node_4_server_id= `SELECT @@server_id` +--let $node_4_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$node_4_server_id.expect +--let $node_4_pid_file= `SELECT @@pid_file` +--source include/shutdown_mysqld.inc +# enforce SST +--exec rm -rf $MYSQLTEST_VARDIR/mysqld.4/data/grastate.dat + +# Wait for node #4 to leave cluster +--connection node_1 +--let $members = 3 +--source include/wsrep_wait_membership.inc + +# prepare to stop SST donor thread when node is in donor state +SET GLOBAL debug = "+d,sync.wsrep_donor_state"; + +--connection node_4 +--echo Restarting server 4... +# Need to use this form instead of start_mysqld.inc because the latter is blocking +--exec echo "restart:$start_mysqld_params" > $node_4_expect_file_name + +# Wait for node #1 to become a donor +--connection node_1 +SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_donor_state_reached"; +--echo Tables on server 1 flushed and locked for SST to server 4 +SET SESSION DEBUG_SYNC = "now SIGNAL signal.wsrep_donor_state"; +SET GLOBAL debug = ""; +SET DEBUG_SYNC='RESET'; + +--echo Wait for the state snapshot to be copied to server 4 +--source include/galera_wait_ready.inc +--echo SST script unlocked server 1 + +# +# At this point state snapshot has been copied, node 1 is operational and +# we have about 20 seconds while everything we do will go into the replication +# queue on node 4 which it will have to apply on top of the snapshot. +# diff --git a/mysql-test/suite/galera/t/galera_vote_joined_end.inc b/mysql-test/suite/galera/t/galera_vote_joined_end.inc new file mode 100644 index 00000000000..1b652634001 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_vote_joined_end.inc @@ -0,0 +1,33 @@ +# Confirm node #4 has rejoined +--connection node_1 +--let $members = 4 +--source include/wsrep_wait_membership.inc +#DROP TABLE IF EXISTS t2; + +# Confirm that all is good and all nodes have identical data + +--connection node_1 +SELECT count(*) AS expect1_390 FROM t1; + +#CALL mtr.add_suppression("Replica SQL: Could not execute Delete_rows"); +#CALL mtr.add_suppression("Event 3 Delete_rows apply failed: 120, seqno [0-9]*"); + +--connection node_2 +SELECT count(*) AS expect2_390 FROM t1; + +#CALL mtr.add_suppression("mysqld: Can't find record in 't1'"); +#CALL mtr.add_suppression("Replica SQL: Could not execute Delete_rows"); +#CALL mtr.add_suppression("Event 3 Delete_rows apply failed: 120, seqno seqno [0-9]*"); + +--connection node_3 +SELECT count(*) AS expect3_390 FROM t1; + +--connection node_4 +SELECT count(*) AS expect4_390 FROM t1; + +DROP TABLE t1; +DROP PROCEDURE p1; + +#CALL mtr.add_suppression("inconsistent with group"); + +--source ../include/auto_increment_offset_restore.inc diff --git a/mysql-test/suite/galera/t/galera_vote_joined_skip.cnf b/mysql-test/suite/galera/t/galera_vote_joined_skip.cnf new file mode 100644 index 00000000000..88a1982bf66 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_vote_joined_skip.cnf @@ -0,0 +1,21 @@ +!include ../galera_4nodes.cnf + +[mysqld] +wsrep-ignore-apply-errors=0 + +[mysqld.1] +wsrep_node_name='node_1' + +[mysqld.2] +wsrep_node_name='node_2' + +[mysqld.3] +wsrep_node_name='node_3' + +[mysqld.4] +wsrep_node_name='node_4' +wsrep_sst_donor='node_1' + +[ENV] +galera_cluster_size=4 +MTR_SST_JOINER_DELAY=20 diff --git a/mysql-test/suite/galera/t/galera_vote_joined_skip.test b/mysql-test/suite/galera/t/galera_vote_joined_skip.test new file mode 100644 index 00000000000..8ccd8dabbfb --- /dev/null +++ b/mysql-test/suite/galera/t/galera_vote_joined_skip.test @@ -0,0 +1,100 @@ +# +# Test a case where a vote happens in JOINED state after SST on a writeset +# that should be skipped. I.e. JOINED node should continue operation. +# + +--source galera_vote_joined_begin.inc +# +# At this point state snapshot has been copied, node 1 is operational and +# we have about 10 seconds while everything we do will go into the replication +# queue on node 4 which it will have to apply on top of the snapshot. +# + +# Increase replication queue on node_4 +--connection node_1 +CALL p1(130); + +# +# Create a writeset that node 4 won't be able to apply by making node 3 +# inconsisitent +# +--connection node_3 +--let $node_3_server_id= `SELECT @@server_id` +--let $node_3_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$node_3_server_id.expect +--let $node_3_pid_file= `SELECT @@pid_file` +SET SESSION wsrep_on = OFF; +CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); +SET SESSION wsrep_on = ON; + +# This should cause nodes #1 and #2 to initiate a vote and kick node #3 +# out of the cluster, node #4 should recover the vote when fails to apply +# the event and continue +INSERT INTO t2 VALUES (DEFAULT); +SET SESSION wsrep_on = OFF; + +# make sure nodes 1,2 progress far enough for commit cut update +--connection node_1 +CALL p1(130); + +--let $members = 3 +--echo Waiting for server 3 to leave the cluster +--connection node_1 +--source include/wsrep_wait_membership.inc +--connection node_2 +--source include/wsrep_wait_membership.inc +--connection node_4 +# need to wait for extra SST delay on joiner +--sleep $MTR_SST_JOINER_DELAY +--sleep $MTR_SST_JOINER_DELAY +--enable_reconnect +--let $wait_timeout = 60 +--source include/wsrep_wait_membership.inc + +--connection node_3 +--echo Server 3 left the cluster, killing it... +# Kill the connected server +--exec echo "wait" > $node_3_expect_file_name +--let KILL_NODE_PIDFILE = $node_3_pid_file +--perl + my $pid_filename = $ENV{'KILL_NODE_PIDFILE'}; + my $mysqld_pid = `cat $pid_filename`; + chomp($mysqld_pid); + system("kill -9 $mysqld_pid"); + exit(0); +EOF +--echo Killed server 3. +--source include/wait_until_disconnected.inc +--echo Restarting server 3... +--exec echo "restart:$start_mysqld_params" > $node_3_expect_file_name + +--echo Waiting for server 3 to rejoin the cluster +--connection node_1 +--let $members = 3 +--source include/wsrep_wait_membership.inc + +--connection node_3 +--echo sleeping for $MTR_SST_JOINER_DELAY +# need to wait for extra SST delay on joiner +--sleep $MTR_SST_JOINER_DELAY +--sleep $MTR_SST_JOINER_DELAY +--echo Waiting ready +--enable_reconnect +--source include/galera_wait_ready.inc +--echo Server 3 restarted. + +--source galera_vote_joined_end.inc + +--connection node_1 +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); + +--connection node_2 +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); + +--connection node_3 +CALL mtr.add_suppression("Vote 0 \\(success\\) on .* is inconsistent with group"); + +--connection node_4 +CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 2b90f2047b3..6e0336713ea 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -915,6 +915,13 @@ EOF fi fi + # Delay for MTR tests if needed to simulate long SST + if [ ${MTR_SST_JOINER_DELAY:=0} -gt 0 ] + then + wsrep_log_info "Sleeping $MTR_SST_JOINER_DELAY seconds for MTR test" + sleep $MTR_SST_JOINER_DELAY + fi + # Remove special tags from the magic file, and from the output: coords=$(head -n1 "$MAGIC_FILE") wsrep_log_info "Galera co-ords from recovery: $coords" From bbd0e4b2c92f6456989e9540ee161ca7d8e4fa0e Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 3 Apr 2025 14:15:34 +0200 Subject: [PATCH 043/125] MDEV-34998 addendum: post-fix corrections for SST scripts Delaying scripts on joiner after SST/IST has been made a common debug feature for all suitable SST/IST methods. Also some minor fixes have been made for new tests. --- .../suite/galera/r/galera_vote_during_ist.result | 3 ++- .../suite/galera/r/galera_vote_joined_apply.result | 3 ++- .../suite/galera/r/galera_vote_joined_skip.result | 9 +++++---- .../suite/galera/t/galera_vote_during_ist.test | 7 ++++++- .../suite/galera/t/galera_vote_joined_apply.test | 2 +- .../suite/galera/t/galera_vote_joined_begin.inc | 5 +++++ .../suite/galera/t/galera_vote_joined_end.inc | 4 ++-- .../suite/galera/t/galera_vote_joined_skip.test | 8 ++++---- scripts/wsrep_sst_common.sh | 13 +++++++++++++ scripts/wsrep_sst_mariabackup.sh | 2 ++ scripts/wsrep_sst_mysqldump.sh | 4 ++++ scripts/wsrep_sst_rsync.sh | 7 +------ 12 files changed, 47 insertions(+), 20 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_vote_during_ist.result b/mysql-test/suite/galera/r/galera_vote_during_ist.result index 2dd295cce85..0949742a21e 100644 --- a/mysql-test/suite/galera/r/galera_vote_during_ist.result +++ b/mysql-test/suite/galera/r/galera_vote_during_ist.result @@ -2,6 +2,7 @@ connection node_4; connection node_3; connection node_2; connection node_1; +# Correct Galera library found connection node_1; connection node_2; connection node_3; @@ -103,7 +104,7 @@ expect4_1 DROP TABLE t1; DROP TABLE t2; DROP PROCEDURE p1; -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); CALL mtr.add_suppression("Inconsistency detected: Failed on preordered"); CALL mtr.add_suppression("Failed to apply write set"); diff --git a/mysql-test/suite/galera/r/galera_vote_joined_apply.result b/mysql-test/suite/galera/r/galera_vote_joined_apply.result index b71487c83b9..4458dd32fb8 100644 --- a/mysql-test/suite/galera/r/galera_vote_joined_apply.result +++ b/mysql-test/suite/galera/r/galera_vote_joined_apply.result @@ -2,6 +2,7 @@ connection node_4; connection node_3; connection node_2; connection node_1; +# Correct Galera library found connection node_1; connection node_2; connection node_3; @@ -87,7 +88,7 @@ expect4_390 DROP TABLE t1; DROP PROCEDURE p1; connection node_4; -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); CALL mtr.add_suppression("Inconsistency detected: Inconsistent by consensus"); CALL mtr.add_suppression("Failed to apply write set: gtid:"); diff --git a/mysql-test/suite/galera/r/galera_vote_joined_skip.result b/mysql-test/suite/galera/r/galera_vote_joined_skip.result index 427c34f0f39..187028fd55c 100644 --- a/mysql-test/suite/galera/r/galera_vote_joined_skip.result +++ b/mysql-test/suite/galera/r/galera_vote_joined_skip.result @@ -2,6 +2,7 @@ connection node_4; connection node_3; connection node_2; connection node_1; +# Correct Galera library found connection node_1; connection node_2; connection node_3; @@ -89,13 +90,13 @@ expect4_390 DROP TABLE t1; DROP PROCEDURE p1; connection node_1; -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); connection node_2; -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); connection node_3; -CALL mtr.add_suppression("Vote 0 \\(success\\) on .* is inconsistent with group"); +CALL mtr.add_suppression("Vote 0 \\(success\\) on .+ is inconsistent with group"); connection node_4; -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); diff --git a/mysql-test/suite/galera/t/galera_vote_during_ist.test b/mysql-test/suite/galera/t/galera_vote_during_ist.test index ee29d08567d..d18014551c1 100644 --- a/mysql-test/suite/galera/t/galera_vote_during_ist.test +++ b/mysql-test/suite/galera/t/galera_vote_during_ist.test @@ -5,8 +5,13 @@ --source include/galera_cluster.inc --source include/big_test.inc +--source include/have_debug.inc --source include/have_debug_sync.inc +# Make sure that the test is operating on the right version of galera library. +--let $galera_version=26.4.19 +source ../wsrep/include/check_galera_version.inc; + --let $node_1=node_1 --let $node_2=node_2 --let $node_3=node_3 @@ -150,7 +155,7 @@ DROP TABLE t1; DROP TABLE t2; DROP PROCEDURE p1; -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); CALL mtr.add_suppression("Inconsistency detected: Failed on preordered"); CALL mtr.add_suppression("Failed to apply write set"); diff --git a/mysql-test/suite/galera/t/galera_vote_joined_apply.test b/mysql-test/suite/galera/t/galera_vote_joined_apply.test index d903b83dc0e..bbd40323f87 100644 --- a/mysql-test/suite/galera/t/galera_vote_joined_apply.test +++ b/mysql-test/suite/galera/t/galera_vote_joined_apply.test @@ -67,7 +67,7 @@ DROP TABLE t2; --source galera_vote_joined_end.inc --connection node_4 -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); CALL mtr.add_suppression("Inconsistency detected: Inconsistent by consensus"); CALL mtr.add_suppression("Failed to apply write set: gtid:"); diff --git a/mysql-test/suite/galera/t/galera_vote_joined_begin.inc b/mysql-test/suite/galera/t/galera_vote_joined_begin.inc index abc59caee81..3e8befc3ece 100644 --- a/mysql-test/suite/galera/t/galera_vote_joined_begin.inc +++ b/mysql-test/suite/galera/t/galera_vote_joined_begin.inc @@ -4,8 +4,13 @@ --source include/galera_cluster.inc --source include/big_test.inc +--source include/have_debug.inc --source include/have_debug_sync.inc +# Make sure that the test is operating on the right version of galera library. +--let $galera_version=26.4.19 +source ../wsrep/include/check_galera_version.inc; + --let $node_1=node_1 --let $node_2=node_2 --let $node_3=node_3 diff --git a/mysql-test/suite/galera/t/galera_vote_joined_end.inc b/mysql-test/suite/galera/t/galera_vote_joined_end.inc index 1b652634001..3ba138bbbf9 100644 --- a/mysql-test/suite/galera/t/galera_vote_joined_end.inc +++ b/mysql-test/suite/galera/t/galera_vote_joined_end.inc @@ -10,14 +10,14 @@ SELECT count(*) AS expect1_390 FROM t1; #CALL mtr.add_suppression("Replica SQL: Could not execute Delete_rows"); -#CALL mtr.add_suppression("Event 3 Delete_rows apply failed: 120, seqno [0-9]*"); +#CALL mtr.add_suppression("Event 3 Delete_rows apply failed: 120, seqno [0-9]+"); --connection node_2 SELECT count(*) AS expect2_390 FROM t1; #CALL mtr.add_suppression("mysqld: Can't find record in 't1'"); #CALL mtr.add_suppression("Replica SQL: Could not execute Delete_rows"); -#CALL mtr.add_suppression("Event 3 Delete_rows apply failed: 120, seqno seqno [0-9]*"); +#CALL mtr.add_suppression("Event 3 Delete_rows apply failed: 120, seqno seqno [0-9]+"); --connection node_3 SELECT count(*) AS expect3_390 FROM t1; diff --git a/mysql-test/suite/galera/t/galera_vote_joined_skip.test b/mysql-test/suite/galera/t/galera_vote_joined_skip.test index 8ccd8dabbfb..2019b639be1 100644 --- a/mysql-test/suite/galera/t/galera_vote_joined_skip.test +++ b/mysql-test/suite/galera/t/galera_vote_joined_skip.test @@ -85,16 +85,16 @@ EOF --source galera_vote_joined_end.inc --connection node_1 -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); --connection node_2 -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); --connection node_3 -CALL mtr.add_suppression("Vote 0 \\(success\\) on .* is inconsistent with group"); +CALL mtr.add_suppression("Vote 0 \\(success\\) on .+ is inconsistent with group"); --connection node_4 -CALL mtr.add_suppression("BF applier failed to open_and_lock_tables: 1146"); +CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 550165b0c3a..38aa18d5f69 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1910,4 +1910,17 @@ create_data SST_PID="$DATA/wsrep_sst.pid" +if [ -n "${MTR_SST_JOINER_DELAY:-}" ]; then + MTR_SST_JOINER_DELAY=$(trim_string "$MTR_SST_JOINER_DELAY") +fi + +simulate_long_sst() +{ + # Delay for MTR tests if needed to simulate long SST/IST: + if [ ${MTR_SST_JOINER_DELAY:-0} -gt 0 ]; then + wsrep_log_info "Sleeping $MTR_SST_JOINER_DELAY seconds for MTR test" + sleep $MTR_SST_JOINER_DELAY + fi +} + wsrep_log_info "$WSREP_METHOD $WSREP_TRANSFER_TYPE started on $WSREP_SST_OPT_ROLE" diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 3e13efb6c6d..d6de8fc1127 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -1519,6 +1519,8 @@ else # joiner exit 2 fi + simulate_long_sst + # use donor magic file, if present # if IST was used, donor magic file was not created # Remove special tags from the magic file, and from the output: diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index f8fbda24356..eee38c136c8 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -184,5 +184,9 @@ else echo "$SET_START_POSITION" | $MYSQL || exit $? fi +if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then + simulate_long_sst +fi + wsrep_log_info "$WSREP_METHOD $WSREP_TRANSFER_TYPE completed on $WSREP_SST_OPT_ROLE" exit 0 diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 6e0336713ea..90d439b9316 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -915,12 +915,7 @@ EOF fi fi - # Delay for MTR tests if needed to simulate long SST - if [ ${MTR_SST_JOINER_DELAY:=0} -gt 0 ] - then - wsrep_log_info "Sleeping $MTR_SST_JOINER_DELAY seconds for MTR test" - sleep $MTR_SST_JOINER_DELAY - fi + simulate_long_sst # Remove special tags from the magic file, and from the output: coords=$(head -n1 "$MAGIC_FILE") From 77391482bdd3b78f977cd1a4d6bd8f6fc6cec98e Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko Date: Fri, 11 Apr 2025 22:22:41 +0200 Subject: [PATCH 044/125] MDEV-34998: master can stop responding after cluster vote to evict a node Additional corrections: there is a natural race between closing connection to cluster in case of applying error and finishing the IST and sometimes IST finishes and tries to send JOIN message over a closed connection. This does not affect the correctness of the test or node behavior. Added error message suppression. Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera/r/galera_vote_during_ist.result | 2 ++ mysql-test/suite/galera/t/galera_vote_during_ist.test | 2 ++ 2 files changed, 4 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_vote_during_ist.result b/mysql-test/suite/galera/r/galera_vote_during_ist.result index 0949742a21e..adbe0e2fe7d 100644 --- a/mysql-test/suite/galera/r/galera_vote_during_ist.result +++ b/mysql-test/suite/galera/r/galera_vote_during_ist.result @@ -108,3 +108,5 @@ CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables fo CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); CALL mtr.add_suppression("Inconsistency detected: Failed on preordered"); CALL mtr.add_suppression("Failed to apply write set"); +CALL mtr.add_suppression("Sending JOIN failed: -103"); +CALL mtr.add_suppression("Failed to JOIN the cluster after SST"); diff --git a/mysql-test/suite/galera/t/galera_vote_during_ist.test b/mysql-test/suite/galera/t/galera_vote_during_ist.test index d18014551c1..eff7f527300 100644 --- a/mysql-test/suite/galera/t/galera_vote_during_ist.test +++ b/mysql-test/suite/galera/t/galera_vote_during_ist.test @@ -159,5 +159,7 @@ CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables fo CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); CALL mtr.add_suppression("Inconsistency detected: Failed on preordered"); CALL mtr.add_suppression("Failed to apply write set"); +CALL mtr.add_suppression("Sending JOIN failed: -103"); +CALL mtr.add_suppression("Failed to JOIN the cluster after SST"); --source ../include/auto_increment_offset_restore.inc From 08f902e9ce525e0512c5d6d43b6548200c6fc673 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Mon, 7 Apr 2025 17:28:32 +0200 Subject: [PATCH 045/125] MDEV-36116: correction for error codes --- mysql-test/suite/galera/r/MDEV-36116.result | 2 +- mysql-test/suite/galera/t/MDEV-36116.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/galera/r/MDEV-36116.result b/mysql-test/suite/galera/r/MDEV-36116.result index 11899b9c09b..fe992d118f0 100644 --- a/mysql-test/suite/galera/r/MDEV-36116.result +++ b/mysql-test/suite/galera/r/MDEV-36116.result @@ -9,7 +9,7 @@ connection con1; SET DEBUG_SYNC = 'now WAIT_FOR may_kill'; SET DEBUG_SYNC = 'now SIGNAL continue'; connection node_1; -ERROR HY000: Lost connection to MySQL server during query +Got one of the listed errors connection node_2; SHOW TABLES LIKE 't1'; Tables_in_test (t1) diff --git a/mysql-test/suite/galera/t/MDEV-36116.test b/mysql-test/suite/galera/t/MDEV-36116.test index eb166e69066..c216e00652f 100644 --- a/mysql-test/suite/galera/t/MDEV-36116.test +++ b/mysql-test/suite/galera/t/MDEV-36116.test @@ -27,7 +27,7 @@ SET DEBUG_SYNC = 'now WAIT_FOR may_kill'; SET DEBUG_SYNC = 'now SIGNAL continue'; --connection node_1 ---error 2013 +--error 2013,2026 --reap # Verify no tables created on either nodes. From d3c9a2ee2126cd045d5d66ee0d4fa4c8880a0f56 Mon Sep 17 00:00:00 2001 From: Dave Gosselin Date: Mon, 3 Feb 2025 10:42:16 -0500 Subject: [PATCH 046/125] MDEV-35510 ASAN build crashes during bootstrap Avoid ASAN failure by collecting statistics from Result objects before cleaning them up. In related single-table cases, statistics are maintained directly by the single-table update and delete functions. --- sql/sql_cmd.h | 7 ++++++ sql/sql_delete.cc | 7 ++++-- sql/sql_delete.h | 7 ++++++ sql/sql_select.cc | 56 ++++++++++++++++++++++++----------------------- sql/sql_update.cc | 9 +++++++- sql/sql_update.h | 8 +++++++ 6 files changed, 64 insertions(+), 30 deletions(-) diff --git a/sql/sql_cmd.h b/sql/sql_cmd.h index 6a43772975a..81bd66db1a6 100644 --- a/sql/sql_cmd.h +++ b/sql/sql_cmd.h @@ -20,6 +20,8 @@ #ifndef SQL_CMD_INCLUDED #define SQL_CMD_INCLUDED +#include + /* When a command is added here, be sure it's also added in mysqld.cc in "struct show_var_st status_vars[]= {" ... @@ -225,6 +227,11 @@ public: */ virtual bool is_dml() const { return false; } + virtual void get_dml_stat (ha_rows &found, ha_rows &changed) + { + found= changed= 0; + } + /** @brief Unprepare prepared statement for the command @param thd global context of the processed statement diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index 0fb061eb553..1f9e413b994 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -328,7 +328,6 @@ bool Sql_cmd_delete::delete_from_single_table(THD *thd) SQL_SELECT *select= 0; SORT_INFO *file_sort= 0; READ_RECORD info; - ha_rows deleted= 0; bool reverse= FALSE; bool binlog_is_row; killed_state killed_status= NOT_KILLED; @@ -427,6 +426,7 @@ bool Sql_cmd_delete::delete_from_single_table(THD *thd) has_triggers= table->triggers && table->triggers->has_delete_triggers(); transactional_table= table->file->has_transactions_and_rollback(); + deleted= 0; if (!returning && !using_limit && const_cond_result && (!thd->is_current_stmt_binlog_format_row() && !has_triggers) @@ -975,7 +975,7 @@ cleanup: result->send_eof(); else my_ok(thd, deleted); - DBUG_PRINT("info",("%ld records deleted",(long) deleted)); + DBUG_PRINT("info", ("%ld records deleted", (long) deleted)); } delete file_sort; free_underlaid_joins(thd, select_lex); @@ -1861,6 +1861,9 @@ bool Sql_cmd_delete::execute_inner(THD *thd) if (result) { + /* In single table case, this->deleted set by delete_from_single_table */ + if (res && multitable) + deleted= ((multi_delete*)get_result())->num_deleted(); res= false; delete result; } diff --git a/sql/sql_delete.h b/sql/sql_delete.h index 1842c5c04c7..c3afbe8bc6c 100644 --- a/sql/sql_delete.h +++ b/sql/sql_delete.h @@ -43,6 +43,7 @@ template class SQL_I_List; class Sql_cmd_delete final : public Sql_cmd_dml { public: + ha_rows deleted{0}; Sql_cmd_delete(bool multitable_arg) : orig_multitable(multitable_arg), multitable(multitable_arg), save_protocol(NULL) @@ -66,6 +67,12 @@ public: void remove_order_by_without_limit(THD *thd); + void get_dml_stat (ha_rows &found, ha_rows &changed) override + { + found= 0; + changed= deleted; + } + protected: /** @brief Perform precheck of table privileges for delete statements diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 395fb277cc9..c16f37eb93f 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -34268,39 +34268,38 @@ static void MYSQL_DML_START(THD *thd) } -static void MYSQL_DML_DONE(THD *thd, int rc) +static void MYSQL_DML_GET_STAT(THD * thd, ha_rows &found, ha_rows &changed) { switch (thd->lex->sql_command) { - case SQLCOM_UPDATE: - MYSQL_UPDATE_DONE( - rc, - (rc ? 0 : - ((multi_update*)(((Sql_cmd_dml*)(thd->lex->m_sql_cmd))->get_result())) - ->num_found()), - (rc ? 0 : - ((multi_update*)(((Sql_cmd_dml*)(thd->lex->m_sql_cmd))->get_result())) - ->num_updated())); - break; case SQLCOM_UPDATE_MULTI: - MYSQL_MULTI_UPDATE_DONE( - rc, - (rc ? 0 : - ((multi_update*)(((Sql_cmd_dml*)(thd->lex->m_sql_cmd))->get_result())) - ->num_found()), - (rc ? 0 : - ((multi_update*)(((Sql_cmd_dml*)(thd->lex->m_sql_cmd))->get_result())) - ->num_updated())); + case SQLCOM_DELETE_MULTI: + thd->lex->m_sql_cmd->get_dml_stat(found, changed); break; case SQLCOM_DELETE: - MYSQL_DELETE_DONE(rc, (rc ? 0 : (ulong) (thd->get_row_count_func()))); + found= 0; + changed= (thd->get_row_count_func()); + break; + default: + DBUG_ASSERT(0); + } +} + + +static void MYSQL_DML_DONE(THD *thd, int rc, ha_rows found, ha_rows changed) +{ + switch (thd->lex->sql_command) { + case SQLCOM_UPDATE: + MYSQL_UPDATE_DONE(rc, found, changed); + break; + case SQLCOM_UPDATE_MULTI: + MYSQL_MULTI_UPDATE_DONE(rc, found, changed); + break; + case SQLCOM_DELETE: + MYSQL_DELETE_DONE(rc, changed); break; case SQLCOM_DELETE_MULTI: - MYSQL_MULTI_DELETE_DONE( - rc, - (rc ? 0 : - ((multi_delete*)(((Sql_cmd_dml*)(thd->lex->m_sql_cmd))->get_result())) - ->num_deleted())); + MYSQL_MULTI_DELETE_DONE(rc, changed); break; default: DBUG_ASSERT(0); @@ -34389,6 +34388,7 @@ err: bool Sql_cmd_dml::execute(THD *thd) { lex = thd->lex; + ha_rows found= 0, changed= 0; bool res; SELECT_LEX_UNIT *unit = &lex->unit; @@ -34439,6 +34439,8 @@ bool Sql_cmd_dml::execute(THD *thd) if (res) goto err; + else + MYSQL_DML_GET_STAT(thd, found, changed); res= unit->cleanup(); @@ -34447,13 +34449,13 @@ bool Sql_cmd_dml::execute(THD *thd) THD_STAGE_INFO(thd, stage_end); - MYSQL_DML_DONE(thd, res); + MYSQL_DML_DONE(thd, 0, found, changed); return res; err: DBUG_ASSERT(thd->is_error() || thd->killed); - MYSQL_DML_DONE(thd, 1); + MYSQL_DML_DONE(thd, 1, 0, 0); THD_STAGE_INFO(thd, stage_end); (void)unit->cleanup(); if (is_prepared()) diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 2e1c7428854..8e3b4a80e7a 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -366,7 +366,7 @@ bool Sql_cmd_update::update_single_table(THD *thd) ha_rows dup_key_found; bool need_sort= TRUE; bool reverse= FALSE; - ha_rows updated, updated_or_same, found; + ha_rows updated_or_same; key_map old_covering_keys; TABLE *table; SQL_SELECT *select= NULL; @@ -3140,6 +3140,13 @@ bool Sql_cmd_update::execute_inner(THD *thd) if (result) { + /* In single table case, this->updated set by update_single_table */ + if (res && multitable) + { + found= ((multi_update*)get_result())->num_found(); + updated= ((multi_update*)get_result())->num_updated(); + } + res= false; delete result; } diff --git a/sql/sql_update.h b/sql/sql_update.h index d01ecb7354a..272c35a378a 100644 --- a/sql/sql_update.h +++ b/sql/sql_update.h @@ -45,6 +45,7 @@ bool compare_record(const TABLE *table); class Sql_cmd_update final : public Sql_cmd_dml { public: + ha_rows found{0}, updated{0}; Sql_cmd_update(bool multitable_arg) : orig_multitable(multitable_arg), multitable(multitable_arg) {} @@ -65,6 +66,13 @@ public: void set_as_multitable() { multitable= true; } + void get_dml_stat (ha_rows &found, ha_rows &changed) override + { + + found= this->found; + changed= this->updated; + } + protected: /** @brief Perform precheck of table privileges for update statements From 0403f0147f176190b6a2ca61996c12eada275e4c Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Mon, 7 Apr 2025 17:40:33 +0200 Subject: [PATCH 047/125] MDEV-33136: backport corrections from 10.11+ --- mysql-test/suite/galera/r/MDEV-33136.result | 2 +- mysql-test/suite/galera/t/MDEV-33136.test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/galera/r/MDEV-33136.result b/mysql-test/suite/galera/r/MDEV-33136.result index 36159fa05cd..fb9945235d6 100644 --- a/mysql-test/suite/galera/r/MDEV-33136.result +++ b/mysql-test/suite/galera/r/MDEV-33136.result @@ -4,7 +4,7 @@ connect node_1a,127.0.0.1,root,,test,$NODE_MYPORT_1; connection node_1; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB; connection node_1a; -TRUNCATE TABLE t1; +RENAME TABLE t1 TO tmp, tmp TO t1; SET SESSION wsrep_retry_autocommit = 0; SET DEBUG_SYNC = 'dict_stats_mdl_acquired SIGNAL may_toi WAIT_FOR bf_abort'; INSERT INTO t1 VALUES (1); diff --git a/mysql-test/suite/galera/t/MDEV-33136.test b/mysql-test/suite/galera/t/MDEV-33136.test index 4b484eb47da..dc1ae1aecdd 100644 --- a/mysql-test/suite/galera/t/MDEV-33136.test +++ b/mysql-test/suite/galera/t/MDEV-33136.test @@ -20,8 +20,8 @@ CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB; --connection node_1a -TRUNCATE TABLE t1; -# TRUNCATE forces the next statement to re-read statistics from persistent storage, +RENAME TABLE t1 TO tmp, tmp TO t1; +# RENAME forces the next statement to re-read statistics from persistent storage, # which will acquire MDL locks on the statistics tables in InnoDB. SET SESSION wsrep_retry_autocommit = 0; SET DEBUG_SYNC = 'dict_stats_mdl_acquired SIGNAL may_toi WAIT_FOR bf_abort'; From ee947fae80dc5f82d2c964cd0d8e897e9c244002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 2 Apr 2025 14:44:41 +0300 Subject: [PATCH 048/125] MDEV-36464 : Galera test failure on galera_3nodes.galera_gtid_2_cluster Test changes only. Add wait conditions after INSERT-clauses to make sure that they are replicated before checking gtid position or table contents. Signed-off-by: Julius Goryavsky --- .../t/galera_gtid_2_cluster.test | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test index 750f840c470..fe71303277c 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test +++ b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test @@ -77,6 +77,8 @@ select @@gtid_binlog_state; --echo cluster 2 node 1 --connection node_4 +--let $wait_condition = SELECT COUNT(*) = 1 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 21, 1); select @@gtid_binlog_state; @@ -85,11 +87,16 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 2 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 1 node 2 --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 2 FROM test.t1; +--source include/wait_condition.inc + select @@gtid_binlog_state; insert into t1 values (1, 12, 3); select @@gtid_binlog_state; @@ -99,10 +106,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 3 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 1 node 3 --connection node_3 +--let $wait_condition = SELECT COUNT(*) = 3 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (1, 13, 4); select @@gtid_binlog_state; @@ -112,10 +123,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 2 node 2 --connection node_5 +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 22, 2); select @@gtid_binlog_state; @@ -125,37 +140,55 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 5 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 2 node 3 --connection node_6 +--let $wait_condition = SELECT COUNT(*) = 5 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 23, 3); select @@gtid_binlog_state; --echo #wait for sync cluster 2 and 1 --connection node_4 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo # check other nodes are consistent --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_3 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_5 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_6 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --echo cluster 1 node 1 --connection node_1 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; drop table t1; stop slave; @@ -250,6 +283,8 @@ select @@gtid_binlog_state; --sleep 2 --echo cluster 2 node 1 --connection node_4 +--let $wait_condition = SELECT COUNT(*) = 1 FROM test.t1; +--source include/wait_condition.inc insert into t1 values (2, 21, 1); select @@gtid_binlog_state; @@ -258,11 +293,16 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 2 FROM test.t1; +--source include/wait_condition.inc + select * from t1 order by 1, 2, 3; --echo cluster 1 node 2 --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 2 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (1, 12, 3); select @@gtid_binlog_state; @@ -272,10 +312,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 3 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 1 node 3 --connection node_3 +--let $wait_condition = SELECT COUNT(*) = 3 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (1, 13, 4); select @@gtid_binlog_state; @@ -285,10 +329,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 2 node 2 --connection node_5 +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 22, 2); select @@gtid_binlog_state; @@ -298,10 +346,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 5 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 2 node 3 --connection node_6 +--let $wait_condition = SELECT COUNT(*) = 5 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 23, 3); select @@gtid_binlog_state; @@ -311,24 +363,36 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo # check other nodes are consistent --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_3 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_5 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_6 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --echo cluster 1 node 1 --connection node_1 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; drop table t1; stop slave; From 5f2562291c8c2792c33048c1bd21c3ccd18ac75d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 8 Apr 2025 09:56:47 +0300 Subject: [PATCH 049/125] MDEV-36509 : Galera test failure on galera_sr.mysql-wsrep-features#165 Problem was that thread was holding lock_sys.wait_mutex when streaming replication transaction rollback was handled and in wsrep-lib requests THD::LOCK_thd_kill mutex causing wrong mutex usage (thd->reset_globals()). Fix is to remove streaming replication rollback handling from Deadlock::report() i.e. wsrep_handle_SR_rollback call. Purpose of Deadloc::report() is to find a cycle in the waits-for graph if exists, report it, mark victim transaction as deadlock victim and release locks it is waiting for. Actual streaming replication rollback that can take longer time can be handled later at trx_t::rollback where lock_sys.wait_mutex is not held. Signed-off-by: Julius Goryavsky --- storage/innobase/lock/lock0lock.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index d258e5c6b27..7ecdc034bbc 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -7108,10 +7108,6 @@ and less modified rows. Bit 0 is used to prefer orig_trx in case of a tie. victim->lock.was_chosen_as_deadlock_victim= true; DEBUG_SYNC_C("deadlock_report_before_lock_releasing"); lock_cancel_waiting_and_release(victim->lock.wait_lock); -#ifdef WITH_WSREP - if (victim->is_wsrep() && wsrep_thd_is_SR(victim->mysql_thd)) - wsrep_handle_SR_rollback(trx->mysql_thd, victim->mysql_thd); -#endif } func_exit: From 1f5d2b201083578e2262e4e089c45267e9c31f25 Mon Sep 17 00:00:00 2001 From: Mohanad Date: Sat, 29 Mar 2025 20:36:08 +0200 Subject: [PATCH 050/125] MDEV-33671: Remove hardcoded open-files-limit in safe_process.cc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fixed limit of 1024 open files was preventing proper concurrency testing in MTR. This commit removes the hardcoded value and adds a new option to control the limit when running tests: --open-files-limit=X. The default is still 1024, but it can now be changed when needed, making it easier to test scenarios that require different number of open file descriptors at the same time e.g: partition_notwin.test Documentation is added to mtr's help file as well. Signed-off-by: VicenÈ›iu Ciorbaru --- mysql-test/lib/My/SafeProcess.pm | 3 +++ mysql-test/lib/My/SafeProcess/safe_process.cc | 17 +++++++++++------ mysql-test/mariadb-test-run.pl | 6 ++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/mysql-test/lib/My/SafeProcess.pm b/mysql-test/lib/My/SafeProcess.pm index e3b46b3709e..3649d207b71 100644 --- a/mysql-test/lib/My/SafeProcess.pm +++ b/mysql-test/lib/My/SafeProcess.pm @@ -138,6 +138,7 @@ sub new { my $error = delete($opts{'error'}); my $verbose = delete($opts{'verbose'}) || $::opt_verbose; my $nocore = delete($opts{'nocore'}); + my $open_files_limit = delete($opts{'open_files_limit'}); my $host = delete($opts{'host'}); my $shutdown = delete($opts{'shutdown'}); my $user_data= delete($opts{'user_data'}); @@ -161,6 +162,8 @@ sub new { push(@safe_args, "--verbose") if $verbose > 0; push(@safe_args, "--nocore") if $nocore; + push(@safe_args, "--open-files-limit=$open_files_limit") if $open_files_limit; + # Point the safe_process at the right parent if running on cygwin push(@safe_args, "--parent-pid=".Cygwin::pid_to_winpid($$)) if IS_CYGWIN; diff --git a/mysql-test/lib/My/SafeProcess/safe_process.cc b/mysql-test/lib/My/SafeProcess/safe_process.cc index dcf9491d2d6..462e72ed976 100644 --- a/mysql-test/lib/My/SafeProcess/safe_process.cc +++ b/mysql-test/lib/My/SafeProcess/safe_process.cc @@ -220,6 +220,7 @@ int main(int argc, char* const argv[] ) pid_t own_pid= getpid(); pid_t parent_pid= getppid(); bool nocore = false; + int open_files_limit = 1024; struct sigaction sa,sa_abort; sa.sa_handler= handle_signal; @@ -268,7 +269,14 @@ int main(int argc, char* const argv[] ) } else if ( strncmp (arg, "--env ", 6) == 0 ) { - putenv(strdup(arg+6)); + putenv(strdup(arg+6)); + } + else if ( strncmp(arg, "--open-files-limit=", 19) == 0 ) + { + const char* start = arg + 19; + open_files_limit = atoi(start); + if (open_files_limit <= 0) + die("Invalid value '%s' passed to --open-files-limit", start); } else die("Unknown option: %s", arg); @@ -318,11 +326,8 @@ int main(int argc, char* const argv[] ) if (nocore) setlimit(RLIMIT_CORE, 0, 0); - /* - mysqld defaults depend on that. make test results stable and independent - from the environment - */ - setlimit(RLIMIT_NOFILE, 1024, 1024); + // Set open files limit + setlimit(RLIMIT_NOFILE, open_files_limit, open_files_limit); // Signal that child is ready buf= 37; diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 69411317f65..8b3dbf9881f 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -130,6 +130,8 @@ our $path_language; our $path_current_testlog; our $path_testlog; +our $opt_open_files_limit; + our $default_vardir; our $opt_vardir; # Path to use for var/ dir our $plugindir; @@ -1277,6 +1279,7 @@ sub command_line_setup { 'list-options' => \$opt_list_options, 'skip-test-list=s' => \@opt_skip_test_list, 'xml-report=s' => \$opt_xml_report, + 'open-files-limit=i', => \$opt_open_files_limit, My::Debugger::options(), My::CoreDump::options(), @@ -5767,6 +5770,7 @@ sub start_mysqltest ($) { append => 1, error => $path_current_testlog, verbose => $opt_verbose, + open_files_limit => $opt_open_files_limit, ); mtr_verbose("Started $proc"); return $proc; @@ -6065,6 +6069,8 @@ Misc options timediff With --timestamp, also print time passed since *previous* test started max-connections=N Max number of open connection to server in mysqltest + open-files-limit=N Max number of open files allowed for any of the children + of my_safe_process. Default is 1024. report-times Report how much time has been spent on different phases of test execution. stress=ARGS Run stress test, providing options to From 6edfdae44dcb6ba50d2cd55b2c4cd1968ab61ebf Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 9 Apr 2025 17:00:42 +0200 Subject: [PATCH 051/125] MDEV-35983 Avoid install failures by using retry logic for file operations Added retry logic to certain file operations during installation as a workaround for issues caused by buggy antivirus software on Windows. Retry logic added for WritePrivateProfileString (mysql_install_db.cc) and renaming file in Innodb. --- sql/mysql_install_db.cc | 21 ++++++++++++++++++++- storage/innobase/os/os0file.cc | 16 ++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index 61fa17596aa..776762a52ad 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -344,10 +344,29 @@ static char *init_bootstrap_command_line(char *cmdline, size_t size) static char my_ini_path[MAX_PATH]; +/** + Wrapper for WritePrivateProfileStringA, with retries and sleeps + if file is locked by another process. +*/ +static BOOL write_private_profile_string_with_retries(const char *appname, + const char *key, const char *val, const char *filename) +{ + static constexpr int RETRIES=50; + static constexpr int SLEEP_MS=10; + for (int n= RETRIES;; n--) + { + if (WritePrivateProfileStringA(appname, key, val, filename)) + return TRUE; + if (GetLastError() != ERROR_ACCESS_DENIED || !n) + return FALSE; + Sleep(SLEEP_MS); + } +} + static void write_myini_str(const char *key, const char* val, const char *section="mysqld") { DBUG_ASSERT(my_ini_path[0]); - if (!WritePrivateProfileString(section, key, val, my_ini_path)) + if (!write_private_profile_string_with_retries(section, key, val, my_ini_path)) { die("Can't write to ini file key=%s, val=%s, section=%s, Windows error %u",key,val,section, GetLastError()); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index a6379190d4a..c0bb760cea7 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2314,8 +2314,20 @@ os_file_rename_func( ut_ad(exists); #endif /* UNIV_DEBUG */ - if (MoveFileEx(oldpath, newpath, MOVEFILE_REPLACE_EXISTING)) { - return(true); + for (int retry= 50;; retry--){ + if (MoveFileEx(oldpath, newpath, MOVEFILE_REPLACE_EXISTING)) + return true; + + if (!retry) + break; + + if (GetLastError() != ERROR_SHARING_VIOLATION) + break; + + // oldpath was opened by someone else (antivirus?) + //without FILE_SHARE_DELETE flag. Retry operation + + Sleep(10); } os_file_handle_rename_error(oldpath, newpath); From 4c9ff3c1bec8d652f9ec822ee3831d918948a224 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 15 Apr 2025 15:42:10 +0200 Subject: [PATCH 052/125] Improve AppVeyor CI configuration - For non-main branches, only build the latest commit and skip intermediate ones if multiple are pushed in quick succession. - Use 'skip_branch_with_pr' to avoid redundant branch builds when a PR is open. - Increase clone_depth from 1 to 10 to reduce failures when multiple commits are pushed rapidly. --- appveyor.yml | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 3192436dfbb..65ba8305323 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,41 @@ version: build-{build}~branch-{branch} -clone_depth: 1 +clone_depth: 10 + +skip_branch_with_pr: true +before_build: + - ps: | + function Get-Remote-Ref($ref) { + try { + $result = git ls-remote origin $ref 2>$null + if (-not $result) { + "Warning: Could not fetch remote ref '$ref'" + return $null + } + return ($result -split "`t")[0] + } catch { + "Warning: Exception while running git ls-remote for '$ref': $_" + return $null + } + } + + $commit = $env:APPVEYOR_REPO_COMMIT + $branch = $env:APPVEYOR_REPO_BRANCH + $latest = $null + $mainBranch = $branch -match '^(main|\d+\.\d+)$' + if ($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null) { + "Branch build detected" + $latest = Get-Remote-Ref "refs/heads/$branch" + } else { + $pr = $env:APPVEYOR_PULL_REQUEST_NUMBER + $latest = Get-Remote-Ref "refs/pull/$pr/head" + $mainBranch = $False + "Pull Request build detected" + } + if ($latest -and ($commit -ne $latest) -and (-not $mainBranch)) { + "Skipping outdated commit $commit (latest is $latest)" + Exit-AppVeyorBuild + } build_script: # dump some system info From 2dc9b8b78e34537a03412ffdc6eeca607851438b Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 7 Apr 2025 17:17:44 +0400 Subject: [PATCH 053/125] MDEV-36216 TO_CHAR FM format not recognized in SQL_MODE=Oracle Adding support for the "FM" format in function TO_CHAR(date_time, fmt). "FM" in the format string disables padding of all components following it. So now TO_CHAR() works as follows: - By default string format components DAY (weekday name) and MONTH (month name) are right-padded with spaces to the maximum possible DAY and MONTH name lengths respectively, according to the current locale specified in @@lc_time_names. So for example, with lc_time_names='en_US' all month names are right-padded with spaces up to 9 characters ('September' is the longest). SET lc_time_names='en_US'; SELECT TO_CHAR('0001-02-03', 'MONTH'); -> 'February ' (padded to 9 chars) NEW: When typed after FM, DAY and MONTH names are not right-padded with trailing spaces any more: SET lc_time_names='en_US'; SELECT TO_CHAR('0001-02-03', 'FMMONTH'); -> 'February' (not padded) - By default numeric components YYYY, YYY, YY, Y, DD, H12, H24, MI, SS are left-padded with leading digits '0' up to the maximum possible number of digits in the component (e.g. 4 for YYYY): SELECT TO_CHAR('0001-02-03', 'YYYY'); -> '0001' (padded to 4 chars) NEW: When typed after FM, these numeric components are not left-padded with leading zeros any more: SELECT TO_CHAR('0001-02-03', 'FMYYYY'); -> '1' (not padded) - If FM is specified multiple times in a format string, every FM negates the previous padding state: * an odd FM disables padding * an even FM enables padding Implementation details: - Adding a helper class Date_time_format_oracle. - Adding a helper method Date_time_format_oracle::append_lex_cstring() - Moving the function append_val() to Date_time_format_oracle as a method. - Moving the function make_date_time_oracle() to Date_time_format_oracle as a method format(). - Adding helper methods month_name() and day_name() in class MY_LOCALE, to return the corresponding components as LEX_CSTRINGs. --- .../suite/compat/oracle/r/func_to_char.result | 236 ++++++++++++++++++ .../suite/compat/oracle/t/func_to_char.test | 71 ++++++ sql/item_timefunc.cc | 130 +++++++--- sql/sql_locale.h | 20 ++ 4 files changed, 426 insertions(+), 31 deletions(-) diff --git a/mysql-test/suite/compat/oracle/r/func_to_char.result b/mysql-test/suite/compat/oracle/r/func_to_char.result index 1f95acece5e..03c0de2ac2d 100644 --- a/mysql-test/suite/compat/oracle/r/func_to_char.result +++ b/mysql-test/suite/compat/oracle/r/func_to_char.result @@ -446,3 +446,239 @@ SELECT TO_CHAR((VALUES('2022-12-12','2020-10-10'))); ERROR HY000: Illegal parameter data type row for operation 'to_char' SELECT TO_CHAR((STR_TO_DATE('2023-01-01', '%d-%m-%Y'), 'YYYY-MM-DD') ); ERROR HY000: Illegal parameter data type row for operation 'to_char' +# +# MDEV-36216 TO_CHAR FM format not recognized in SQL_MODE=Oracle +# +SET NAMES utf8mb3; +CREATE TABLE t1 (fmt VARCHAR(256)); +INSERT INTO t1 VALUES +/* Add the slash character before FM to see the position of FM in the results */ +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('/FMYYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-/FMMM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-/FMDD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD /FMHH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:/FMMI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:/FMSS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS /FMDAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY /FMMONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; /FMYYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-/FMMM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-/FMDD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD /FMHH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:/FMMI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:/FMSS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS /FMDAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY /FMMONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH/FM;'), +/*Formats not covered above */ +('YYY YY Y HH12; /FMYYY YY Y HH12;'), +/*FM specified multiple times*/ +('FMFMFMFMFMFMFMFMYYYY-MM-DD [DAY] [MONTH]'), +('FMFMFMFMFMFMFMFMFMYYYY-MM-DD [DAY] [MONTH]'), +( +'YYYY-MM-DD [DAY] [MONTH]; FMYYYY-MM-DD [DAY] [MONTH]; ' + 'FMYYYY-MM-DD [DAY] [MONTH]; FMYYYY-MM-DD [DAY] [MONTH]; ' + 'FMYYYY-MM-DD [DAY] [MONTH];' +), +/*Corner cases*/ +('FX') /*Unknown format starting with 'F'*/, +('F') /*Unexpected end of the format string*/; +SET lc_time_names='en_US'; +SELECT TO_CHAR('0001-02-03 04:05:06', fmt) AS c1 FROM t1; +c1 +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 Saturday February ; +/1-2-3 4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-/2-3 4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-/3 4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 /4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:/5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:/6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 /Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday /February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; /1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-/2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-/3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 /4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:/5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:/6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 /Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 Saturday /February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 Saturday February /; +001 01 1 04; /1 1 1 4; +0001-02-03 [Saturday ] [February ] +1-2-3 [Saturday] [February] +0001-02-03 [Saturday ] [February ]; 1-2-3 [Saturday] [February]; 0001-02-03 [Saturday ] [February ]; 1-2-3 [Saturday] [February]; 0001-02-03 [Saturday ] [February ]; +NULL +NULL +Warnings: +Warning 3047 Invalid argument error: date format not recognized at FX in function to_char. +Warning 3047 Invalid argument error: date format not recognized at F in function to_char. +SET sql_mode=ORACLE; +CREATE VIEW v1 AS SELECT TO_CHAR('0001-02-03 04:05:06', fmt) AS c1 FROM t1; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE VIEW "v1" AS select to_char('0001-02-03 04:05:06',"t1"."fmt") AS "c1" from "t1" utf8mb3 utf8mb3_general_ci +SELECT * FROM v1; +c1 +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 Saturday February ; +/1-2-3 4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-/2-3 4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-/3 4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 /4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:/5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:/6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 /Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday /February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; /1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-/2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-/3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 /4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:/5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:/6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 /Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 Saturday /February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 Saturday February /; +001 01 1 04; /1 1 1 4; +0001-02-03 [Saturday ] [February ] +1-2-3 [Saturday] [February] +0001-02-03 [Saturday ] [February ]; 1-2-3 [Saturday] [February]; 0001-02-03 [Saturday ] [February ]; 1-2-3 [Saturday] [February]; 0001-02-03 [Saturday ] [February ]; +NULL +NULL +Warnings: +Warning 3047 Invalid argument error: date format not recognized at FX in function to_char. +Warning 3047 Invalid argument error: date format not recognized at F in function to_char. +SET sql_mode=DEFAULT; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select to_char('0001-02-03 04:05:06',`t1`.`fmt`) AS `c1` from `t1` utf8mb3 utf8mb3_general_ci +SELECT * FROM v1; +c1 +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 Saturday February ; +/1-2-3 4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-/2-3 4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-/3 4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 /4:5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:/5:6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:/6 Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 /Saturday February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday /February; 1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; /1-2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-/2-3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-/3 4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 /4:5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:/5:6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:/6 Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 /Saturday February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 Saturday /February; +0001-02-03 04:05:06 Saturday February ; 0001-02-03 04:05:06 Saturday February /; +001 01 1 04; /1 1 1 4; +0001-02-03 [Saturday ] [February ] +1-2-3 [Saturday] [February] +0001-02-03 [Saturday ] [February ]; 1-2-3 [Saturday] [February]; 0001-02-03 [Saturday ] [February ]; 1-2-3 [Saturday] [February]; 0001-02-03 [Saturday ] [February ]; +NULL +NULL +Warnings: +Warning 3047 Invalid argument error: date format not recognized at FX in function to_char. +Warning 3047 Invalid argument error: date format not recognized at F in function to_char. +DROP VIEW v1; +SET lc_time_names='zh_CN'; +SELECT TO_CHAR('0001-02-03 04:05:06', fmt) AS c1 FROM t1; +c1 +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 星期六 二月 ; +/1-2-3 4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-/2-3 4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-/3 4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 /4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:/5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:/6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 /星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 /二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; /1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-/2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-/3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 /4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:/5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:/6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 /星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 星期六 /二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 星期六 二月 /; +001 01 1 04; /1 1 1 4; +0001-02-03 [星期六] [二月 ] +1-2-3 [星期六] [二月] +0001-02-03 [星期六] [二月 ]; 1-2-3 [星期六] [二月]; 0001-02-03 [星期六] [二月 ]; 1-2-3 [星期六] [二月]; 0001-02-03 [星期六] [二月 ]; +NULL +NULL +Warnings: +Warning 3047 Invalid argument error: date format not recognized at FX in function to_char. +Warning 3047 Invalid argument error: date format not recognized at F in function to_char. +SET sql_mode=ORACLE; +CREATE VIEW v1 AS SELECT TO_CHAR('0001-02-03 04:05:06', fmt) AS c1 FROM t1; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE VIEW "v1" AS select to_char('0001-02-03 04:05:06',"t1"."fmt") AS "c1" from "t1" utf8mb3 utf8mb3_general_ci +SELECT * FROM v1; +c1 +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 星期六 二月 ; +/1-2-3 4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-/2-3 4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-/3 4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 /4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:/5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:/6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 /星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 /二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; /1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-/2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-/3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 /4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:/5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:/6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 /星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 星期六 /二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 星期六 二月 /; +001 01 1 04; /1 1 1 4; +0001-02-03 [星期六] [二月 ] +1-2-3 [星期六] [二月] +0001-02-03 [星期六] [二月 ]; 1-2-3 [星期六] [二月]; 0001-02-03 [星期六] [二月 ]; 1-2-3 [星期六] [二月]; 0001-02-03 [星期六] [二月 ]; +NULL +NULL +Warnings: +Warning 3047 Invalid argument error: date format not recognized at FX in function to_char. +Warning 3047 Invalid argument error: date format not recognized at F in function to_char. +SET sql_mode=DEFAULT; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select to_char('0001-02-03 04:05:06',`t1`.`fmt`) AS `c1` from `t1` utf8mb3 utf8mb3_general_ci +SELECT * FROM v1; +c1 +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 星期六 二月 ; +/1-2-3 4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-/2-3 4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-/3 4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 /4:5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:/5:6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:/6 星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 /星期六 二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 /二月; 1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; /1-2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-/2-3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-/3 4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 /4:5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:/5:6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:/6 星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 /星期六 二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 星期六 /二月; +0001-02-03 04:05:06 星期六 二月 ; 0001-02-03 04:05:06 星期六 二月 /; +001 01 1 04; /1 1 1 4; +0001-02-03 [星期六] [二月 ] +1-2-3 [星期六] [二月] +0001-02-03 [星期六] [二月 ]; 1-2-3 [星期六] [二月]; 0001-02-03 [星期六] [二月 ]; 1-2-3 [星期六] [二月]; 0001-02-03 [星期六] [二月 ]; +NULL +NULL +Warnings: +Warning 3047 Invalid argument error: date format not recognized at FX in function to_char. +Warning 3047 Invalid argument error: date format not recognized at F in function to_char. +DROP VIEW v1; +SET lc_time_names=DEFAULT; +DROP TABLE t1; diff --git a/mysql-test/suite/compat/oracle/t/func_to_char.test b/mysql-test/suite/compat/oracle/t/func_to_char.test index 7a40321538d..df5d0065f50 100644 --- a/mysql-test/suite/compat/oracle/t/func_to_char.test +++ b/mysql-test/suite/compat/oracle/t/func_to_char.test @@ -234,3 +234,74 @@ SELECT TO_CHAR((VALUES('2022-12-12','2020-10-10'))); --error ER_ILLEGAL_PARAMETER_DATA_TYPE_FOR_OPERATION SELECT TO_CHAR((STR_TO_DATE('2023-01-01', '%d-%m-%Y'), 'YYYY-MM-DD') ); + +--echo # +--echo # MDEV-36216 TO_CHAR FM format not recognized in SQL_MODE=Oracle +--echo # + +SET NAMES utf8mb3; +CREATE TABLE t1 (fmt VARCHAR(256)); +INSERT INTO t1 VALUES + +/* Add the slash character before FM to see the position of FM in the results */ +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('/FMYYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-/FMMM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-/FMDD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD /FMHH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:/FMMI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:/FMSS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS /FMDAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY /FMMONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH;'), + +('YYYY-MM-DD HH24:MI:SS DAY MONTH; /FMYYYY-MM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-/FMMM-DD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-/FMDD HH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD /FMHH24:MI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:/FMMI:SS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:/FMSS DAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS /FMDAY MONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY /FMMONTH;'), +('YYYY-MM-DD HH24:MI:SS DAY MONTH; YYYY-MM-DD HH24:MI:SS DAY MONTH/FM;'), + +/*Formats not covered above */ +('YYY YY Y HH12; /FMYYY YY Y HH12;'), + +/*FM specified multiple times*/ +('FMFMFMFMFMFMFMFMYYYY-MM-DD [DAY] [MONTH]'), +('FMFMFMFMFMFMFMFMFMYYYY-MM-DD [DAY] [MONTH]'), +( + 'YYYY-MM-DD [DAY] [MONTH]; FMYYYY-MM-DD [DAY] [MONTH]; ' + 'FMYYYY-MM-DD [DAY] [MONTH]; FMYYYY-MM-DD [DAY] [MONTH]; ' + 'FMYYYY-MM-DD [DAY] [MONTH];' +), + +/*Corner cases*/ +('FX') /*Unknown format starting with 'F'*/, +('F') /*Unexpected end of the format string*/; + +SET lc_time_names='en_US'; +SELECT TO_CHAR('0001-02-03 04:05:06', fmt) AS c1 FROM t1; +SET sql_mode=ORACLE; +CREATE VIEW v1 AS SELECT TO_CHAR('0001-02-03 04:05:06', fmt) AS c1 FROM t1; +SHOW CREATE VIEW v1; +SELECT * FROM v1; +SET sql_mode=DEFAULT; +SHOW CREATE VIEW v1; +SELECT * FROM v1; +DROP VIEW v1; + +SET lc_time_names='zh_CN'; +SELECT TO_CHAR('0001-02-03 04:05:06', fmt) AS c1 FROM t1; +SET sql_mode=ORACLE; +CREATE VIEW v1 AS SELECT TO_CHAR('0001-02-03 04:05:06', fmt) AS c1 FROM t1; +SHOW CREATE VIEW v1; +SELECT * FROM v1; +SET sql_mode=DEFAULT; +SHOW CREATE VIEW v1; +SELECT * FROM v1; +DROP VIEW v1; + +SET lc_time_names=DEFAULT; + +DROP TABLE t1; diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 1200fa33382..224a7674b84 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -1962,6 +1962,79 @@ null_date: return 0; } + +/* A class to print TO_CHAR(date_time, format) */ +class Date_time_format_oracle +{ + // m_fm is true if "FM" was found in the format string odd number of times + bool m_fm; +public: + Date_time_format_oracle() + :m_fm(false) + { } + + /* + Append a numeric value to a String. + If m_fm is false, then left-pad the numeric value with '0'. + + @param val - the numeric value to be appended to str + @param size - the maximum number of digits possible in val + @param [OUT] str - the result String (val will be appended to it) + + @retval false - on success + @retval true - on error (e.g. EOM) + */ + bool append_val(int val, uint size, String *str) const + { + if (m_fm) + return str->append_longlong(val); + return str->append_zerofill(val, size); + } + + /* + Append a LEX_CSTRING value to a String. + If m_fm is false, then right-pad the appended value with spaces. + + @param ls - the LEX_CSTRING to be append to str + @param max_char_length - the maximum possible length of ls, in characters + @param [OUT] str - the result String (ls will be appended to it) + + @retval false - on success + @retval true - on error (e.g. EOM) + */ + bool append_lex_cstring(const LEX_CSTRING ls, uint max_char_length, + String *str) const + { + // Locale data uses utf8mb3 + static constexpr CHARSET_INFO *cs= &my_charset_utf8mb3_general_ci; + str->append(ls.str, ls.length, cs); + if (!m_fm) + { + size_t char_length= cs->numchars(ls.str, ls.str + ls.length); + if (char_length < max_char_length) + return str->fill(str->length() + max_char_length - char_length, ' '); + } + return false; + } + + /* + Print a date/time value to a String according to the given format + @param fmt_array - the format array + @param l_time - the date/time value + @param locale - the locale to use for textual components + (MONTH and DAY) + @param [OUT] str - the string to print into. + + @retval false - on success + @retval true - on error (e.g. EOM) + */ + bool format(const uint16 *fmt_array, + const MYSQL_TIME *l_time, + const MY_LOCALE *locale, + String *str); +}; + + /* Oracle has many formatting models, we list all but only part of them are implemented, because some models depend on oracle functions @@ -2138,7 +2211,8 @@ bool Item_func_tochar::parse_format_string(const String *format, uint *fmt_len) /* Oracle datetime format support text in double quotation marks like 'YYYY"abc"MM"xyz"DD', When this happens, store the text and quotation - marks, and use the text as a separator in make_date_time_oracle. + marks, and use the text as a separator in + Date_time_format_oracle::format(). NOTE: the quotation mark is not print in return value. for example: select TO_CHAR(sysdate, 'YYYY"abc"MM"xyzDD"') will return 2021abc01xyz11 @@ -2419,6 +2493,17 @@ bool Item_func_tochar::parse_format_string(const String *format, uint *fmt_len) tmp_fmt--; break; + case 'F': + if (ptr + 1 == end) + goto error; + if (my_toupper(system_charset_info, ptr[1]) == 'M') + { + *tmp_fmt= FMT_FM; + ptr+= 1; + continue; + } + goto error; + default: offset= parse_special(cfmt, ptr, end, tmp_fmt); if (!offset) @@ -2441,16 +2526,10 @@ error: } -static inline bool append_val(int val, int size, String *str) -{ - return str->append_zerofill(val, size); -} - - -static bool make_date_time_oracle(const uint16 *fmt_array, - const MYSQL_TIME *l_time, - const MY_LOCALE *locale, - String *str) +bool Date_time_format_oracle::format(const uint16 *fmt_array, + const MYSQL_TIME *l_time, + const MY_LOCALE *locale, + String *str) { bool quotation_flag= false; const uint16 *ptr= fmt_array; @@ -2558,16 +2637,8 @@ static bool make_date_time_oracle(const uint16 *fmt_array, } else { - const char *month_name= (locale->month_names-> - type_names[l_time->month-1]); - size_t month_byte_len= strlen(month_name); - size_t month_char_len; - str->append(month_name, month_byte_len, system_charset_info); - month_char_len= my_numchars_mb(&my_charset_utf8mb3_general_ci, - month_name, month_name + - month_byte_len); - if (str->fill(str->length() + locale->max_month_name_length - - month_char_len, ' ')) + if (append_lex_cstring(locale->month_name(l_time->month - 1), + locale->max_month_name_length, str)) goto err_exit; } } @@ -2598,17 +2669,10 @@ static bool make_date_time_oracle(const uint16 *fmt_array, str->append("00", 2, system_charset_info); else { - const char *day_name; - size_t day_byte_len, day_char_len; weekday=calc_weekday(calc_daynr(l_time->year,l_time->month, l_time->day), 0); - day_name= locale->day_names->type_names[weekday]; - day_byte_len= strlen(day_name); - str->append(day_name, day_byte_len, system_charset_info); - day_char_len= my_numchars_mb(&my_charset_utf8mb3_general_ci, - day_name, day_name + day_byte_len); - if (str->fill(str->length() + locale->max_day_name_length - - day_char_len, ' ')) + if (append_lex_cstring(locale->day_name(weekday), + locale->max_day_name_length, str)) goto err_exit; } } @@ -2636,6 +2700,10 @@ static bool make_date_time_oracle(const uint16 *fmt_array, goto err_exit; break; + case FMT_FM: + m_fm= !m_fm; + break; + default: str->append((char) *ptr); } @@ -2743,7 +2811,7 @@ String *Item_func_tochar::val_str(String* str) /* Create the result string */ str->set_charset(collation.collation); - if (!make_date_time_oracle(fmt_array, &l_time, lc, str)) + if (!Date_time_format_oracle().format(fmt_array, &l_time, lc, str)) return str; null_date: diff --git a/sql/sql_locale.h b/sql/sql_locale.h index b7ce9f7ba1d..e7cee1ef732 100644 --- a/sql/sql_locale.h +++ b/sql/sql_locale.h @@ -62,6 +62,26 @@ public: {} my_repertoire_t repertoire() const { return is_ascii ? MY_REPERTOIRE_ASCII : MY_REPERTOIRE_EXTENDED; } + /* + Get a non-abbreviated month name by index + @param month - the month index 0..11 + */ + LEX_CSTRING month_name(uint month) const + { + if (month > 11) + return Lex_cstring("##", 2); + return Lex_cstring_strlen(month_names->type_names[month]); + } + /* + Get a non-abbreviated weekday name by index + @param weekday - the weekday index 0..6 + */ + LEX_CSTRING day_name(uint weekday) const + { + if (weekday > 6) + return Lex_cstring("##", 2); + return Lex_cstring_strlen(day_names->type_names[weekday]); + } }; /* Exported variables */ From 11324875b42d6c14509a8452c2c22f489206cfc4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 14 Apr 2025 16:50:16 +0200 Subject: [PATCH 054/125] MDEV-33474 Windows packaging - install runtime dependencies Use CMake 3.21 (semi)automatic dependency resolution to install server and plugin dependencies (DLLs). This is especially useful with 3rd party dependencies, such as Curl, zlib etc --- CMakeLists.txt | 12 ++++++++++++ cmake/install_macros.cmake | 29 ++++++++++++++++++++++++++-- cmake/mysql_add_executable.cmake | 6 +++--- win/packaging/CPackWixConfig.cmake | 5 +++-- win/packaging/ComponentsIgnore.cmake | 2 +- 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 567d5ab07a5..40023f28cf5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,6 +125,18 @@ ENDIF() FIND_PACKAGE(Git) +IF(WIN32 AND (CMAKE_VERSION VERSION_GREATER "3.21")) + # Install runtime dependency by default, when using vcpkg + IF(NOT DEFINED INSTALL_RUNTIME_DEPENDENCIES_DEFAULT) + IF("${VCPKG_INSTALLED_DIR}") + SET(INSTALL_RUNTIME_DEPENDENCIES_DEFAULT OFF) + ELSE() + SET(INSTALL_RUNTIME_DEPENDENCIES_DEFAULT ON) + ENDIF() + ENDIF() + OPTION(INSTALL_RUNTIME_DEPENDENCIES "Install runtime dependencies" "${INSTALL_RUNTIME_DEPENDENCIES_DEFAULT}") +ENDIF() + # Following autotools tradition, add preprocessor definitions # specified in environment variable CPPFLAGS IF(DEFINED ENV{CPPFLAGS}) diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake index 0c07c5c7afc..dd072abdfb8 100644 --- a/cmake/install_macros.cmake +++ b/cmake/install_macros.cmake @@ -199,7 +199,7 @@ FUNCTION(SIGN_TARGET target) ENDFUNCTION() # Installs targets, also installs pdbs on Windows. -# +# Also installs runtime dependencies # FUNCTION(MYSQL_INSTALL_TARGETS) @@ -230,9 +230,34 @@ FUNCTION(MYSQL_INSTALL_TARGETS) ENDIF() ENDFOREACH() - INSTALL(TARGETS ${TARGETS} DESTINATION ${ARG_DESTINATION} ${COMP}) + IF(WIN32 AND INSTALL_RUNTIME_DEPENDENCIES) + STRING(JOIN "." runtime_deps_set_name ${TARGETS}) + SET(RUNTIME_DEPS RUNTIME_DEPENDENCY_SET "${runtime_deps_set_name}") + ENDIF() + + INSTALL(TARGETS ${TARGETS} DESTINATION ${ARG_DESTINATION} ${COMP} ${RUNTIME_DEPS}) INSTALL_DEBUG_SYMBOLS(${TARGETS} ${COMP} INSTALL_LOCATION ${ARG_DESTINATION}) + IF(WIN32 AND INSTALL_RUNTIME_DEPENDENCIES) + INSTALL( + RUNTIME_DEPENDENCY_SET + "${runtime_deps_set_name}" + COMPONENT RuntimeDeps + DESTINATION ${INSTALL_BINDIR} + PRE_EXCLUDE_REGEXES + "api-ms-" # Windows stuff + "ext-ms-" + "server\\.dll" # main server DLL, installed separately + "clang_rt" # ASAN libraries + "vcruntime" + POST_EXCLUDE_REGEXES + ".*system32/.*\\.dll" # Windows stuff + POST_INCLUDE_REGEXES + DIRECTORIES + ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin + $<$:${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/bin> + ) + ENDIF() ENDFUNCTION() # Optionally install mysqld/client/embedded from debug build run. outside of the current build dir diff --git a/cmake/mysql_add_executable.cmake b/cmake/mysql_add_executable.cmake index f9faabab9cd..f548534a056 100644 --- a/cmake/mysql_add_executable.cmake +++ b/cmake/mysql_add_executable.cmake @@ -48,9 +48,9 @@ FUNCTION (MYSQL_ADD_EXECUTABLE) ENDIF() IF (ARG_WIN32) - SET(WIN32 WIN32) + SET(ARG_WIN32 WIN32) ELSE() - UNSET(WIN32) + UNSET(ARG_WIN32) ENDIF() IF (ARG_MACOSX_BUNDLE) SET(MACOSX_BUNDLE MACOSX_BUNDLE) @@ -63,7 +63,7 @@ FUNCTION (MYSQL_ADD_EXECUTABLE) UNSET(EXCLUDE_FROM_ALL) ENDIF() - ADD_EXECUTABLE(${target} ${WIN32} ${MACOSX_BUNDLE} ${EXCLUDE_FROM_ALL} ${sources}) + ADD_EXECUTABLE(${target} ${ARG_WIN32} ${MACOSX_BUNDLE} ${EXCLUDE_FROM_ALL} ${sources}) # tell CPack where to install IF(NOT ARG_EXCLUDE_FROM_ALL) diff --git a/win/packaging/CPackWixConfig.cmake b/win/packaging/CPackWixConfig.cmake index f6ef25d5b46..2400a6543ba 100644 --- a/win/packaging/CPackWixConfig.cmake +++ b/win/packaging/CPackWixConfig.cmake @@ -37,7 +37,7 @@ add_component_group(AlwaysInstall HIDDEN DISPLAY_NAME "AlwaysInstall" DESCRIPTION "Always installed components") -foreach(c Readme Common VCCRT) +foreach(c Readme Common VCCRT RuntimeDeps) add_component(${c} GROUP AlwaysInstall) endforeach() @@ -53,7 +53,8 @@ add_component(Backup DESCRIPTION "Installs backup utilities(mariabackup and mbstream)") #Miscellaneous hidden components, part of server / or client programs -foreach(comp connect-engine connect-engine-jdbc ClientPlugins aws-key-management rocksdb-engine) +foreach(comp connect-engine connect-engine-jdbc ClientPlugins aws-key-management rocksdb-engine + hashicorp-key-management) add_component(${comp} GROUP MySQLServer HIDDEN) endforeach() diff --git a/win/packaging/ComponentsIgnore.cmake b/win/packaging/ComponentsIgnore.cmake index 5f037e365af..6653964ed18 100644 --- a/win/packaging/ComponentsIgnore.cmake +++ b/win/packaging/ComponentsIgnore.cmake @@ -1,2 +1,2 @@ # Components ignored in both ZIP and WIX generators -set(COMPONENTS_IGNORE Debuginfo Server_Scripts SqlBench Test Embedded hashicorp-key-management plugin-hashicorp-key-management) +set(COMPONENTS_IGNORE Debuginfo Server_Scripts SqlBench Test Embedded plugin-hashicorp-key-management) From 75e11ec52c0f0f66293f7fe3a490774c6ea8eae9 Mon Sep 17 00:00:00 2001 From: ParadoxV5 Date: Sun, 6 Apr 2025 15:55:12 -0600 Subject: [PATCH 055/125] MDEV-36238 11.4 follow-up: `Master_SSL_Verify_Server_Cert=0` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tweak `multi_source.master_info_file` re. MDEV-31857’s new default `Master_SSL_Verify_Server_Cert=0` was optional before that secure-by-default change. Now, passwordless connections must disable SSL certificate verification. Because the default unnamed connection cannot be deleted by RESET REPLICA ALL, it must be explicitly left passwordless and having `Master_SSL_Verify_Server_Cert=0`. The named connection is cleaned up by RESET REPLICA ALL and thus not affected. --- mysql-test/suite/multi_source/master_info_file.result | 2 +- mysql-test/suite/multi_source/master_info_file.test | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/multi_source/master_info_file.result b/mysql-test/suite/multi_source/master_info_file.result index 08fe34677be..9a0623369d7 100644 --- a/mysql-test/suite/multi_source/master_info_file.result +++ b/mysql-test/suite/multi_source/master_info_file.result @@ -1,4 +1,4 @@ -CHANGE MASTER TO master_host='127.0.0.1', master_user='root', master_port=SERVER_MYPORT_1; +CHANGE MASTER TO master_host='127.0.0.1', master_user='root', master_port=SERVER_MYPORT_1, master_ssl_verify_server_cert=0; CHANGE MASTER 'named' TO master_host='localhost', master_user='test', master_port=SERVER_MYPORT_2; --list_files @@datadir *.info relay-log-named.info diff --git a/mysql-test/suite/multi_source/master_info_file.test b/mysql-test/suite/multi_source/master_info_file.test index 812b22b5706..48478fa6e35 100644 --- a/mysql-test/suite/multi_source/master_info_file.test +++ b/mysql-test/suite/multi_source/master_info_file.test @@ -5,7 +5,7 @@ --source include/not_embedded.inc --replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1 ---eval CHANGE MASTER TO master_host='127.0.0.1', master_user='root', master_port=$SERVER_MYPORT_1 +--eval CHANGE MASTER TO master_host='127.0.0.1', master_user='root', master_port=$SERVER_MYPORT_1, master_ssl_verify_server_cert=0 --replace_result $SERVER_MYPORT_2 SERVER_MYPORT_2 --eval CHANGE MASTER 'named' TO master_host='localhost', master_user='test', master_port=$SERVER_MYPORT_2 From 0dad1458e7e0c0c16e5db9a6adaac2bdc1ce0476 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 9 Mar 2025 22:15:23 +0200 Subject: [PATCH 056/125] Add memory allocated by my_once_alloc() to memory status Global Memory_used and global Memory_used_initial now includes memory allocated by mysys libraries during server startup. --- include/my_sys.h | 1 + mysys/my_once.c | 7 +++++++ sql/mysqld.cc | 6 +++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/my_sys.h b/include/my_sys.h index 902a7b47351..30c843ede21 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -163,6 +163,7 @@ extern my_thread_id (*sf_malloc_dbug_id)(void); typedef void (*MALLOC_SIZE_CB) (long long size, my_bool is_thread_specific); extern void set_malloc_size_cb(MALLOC_SIZE_CB func); extern MALLOC_SIZE_CB update_malloc_size; +extern int64 my_malloc_init_memory_allocated; /* defines when allocating data */ extern void *my_malloc(PSI_memory_key key, size_t size, myf MyFlags); diff --git a/mysys/my_once.c b/mysys/my_once.c index e2bea0a4d1e..b4a30da95ea 100644 --- a/mysys/my_once.c +++ b/mysys/my_once.c @@ -19,6 +19,9 @@ #include "my_static.h" #include "mysys_err.h" #include +#include + +int64 my_malloc_init_memory_allocated= 0; /* Alloc for things we don't nend to free run-time (that only @@ -62,6 +65,9 @@ void* my_once_alloc(size_t Size, myf MyFlags) my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_FATAL), get_size); return((uchar*) 0); } + my_atomic_add64_explicit(&my_malloc_init_memory_allocated, + (int64) get_size, + MY_MEMORY_ORDER_RELAXED); DBUG_PRINT("test",("my_once_malloc %lu byte malloced", (ulong) get_size)); next->next= 0; next->size= get_size; @@ -114,6 +120,7 @@ void my_once_free(void) free((uchar*) old); } my_once_root_block=0; + my_malloc_init_memory_allocated= 0; DBUG_VOID_RETURN; } /* my_once_free */ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 22cc05de9df..38394d33675 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -6064,8 +6064,8 @@ int mysqld_main(int argc, char **argv) (void)MYSQL_SET_STAGE(0 ,__FILE__, __LINE__); /* Memory used when everything is setup */ - start_memory_used= global_status_var.global_memory_used; - + start_memory_used= (global_status_var.global_memory_used + + my_malloc_init_memory_allocated); run_main_loop(); /* Shutdown requested */ @@ -7344,7 +7344,7 @@ static int show_memory_used(THD *thd, SHOW_VAR *var, void *buff, if (scope == OPT_GLOBAL) { calc_sum_of_all_status_if_needed(status_var); - *(longlong*) buff= (status_var->global_memory_used + + *(longlong*) buff= (status_var->global_memory_used + my_malloc_init_memory_allocated + status_var->local_memory_used); } else From 10c063f9f07d6ab8f18b4b7b88da75d5bb2eae55 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Sat, 22 Mar 2025 12:45:13 +0400 Subject: [PATCH 057/125] MDEV-36213 Doubled memory usage (11.4.4 <-> 11.4.5) Fixing the code adding MySQL _0900_ collations as _uca1400_ aliases not to perform deep initialization of the corresponding _uca1400_ collations. Only basic initialization is now performed which allows to watch these collations (both _0900_ and _uca1400_) in queries to INFORMATION_SCHEMA tables COLLATIONS and COLLATION_CHARACTER_SET_APPLICABILITY, as well as in SHOW COLLATION statements. Deep initialization is now performed only when a collation (either the _0900_ alias or the corresponding _uca1400_ collation) is used for the very first time after the server startup. Refactoring was done to maintain the code easier: - most of the _uca1400_ code was moved from ctype-uca.c to a new file ctype-uca1400.c - most of the _0900_ code was moved from type-uca.c to a new file ctype-uca0900.c Change details: - The original function add_alias_for_collation() added by the patch for "MDEV-20912 Add support for utf8mb4_0900_* collations in MariaDB Server" was removed from mysys/charset.c, as it had two two problems: a. it forced deep initialization of the _uca1400_ collations when adding _0900_ aliases for them at the server startup (the main reported problem) b. the collation initialization code in add_alias_for_collation() was related more to collations rather than to memory management, so /strings should be a better place for it than /mysys. The code from add_alias_for_collation() was split into separate functions. Cyclic dependency was removed. `#include ` was removed from /strings/ctype-uca.c. Collations are now added using a callback function MY_CHARSET_LOADED::add_collation, like it is done for user collations defined in Index.xml. The code in /mysys sets MY_CHARSET_LOADED::add_collation to add_compiled_collation(). - The function compare_collations() was removed. A new virtual function was added into my_collation_handler_st instead: my_bool (*eq_collation)(CHARSET_INFO *self, CHARSET_INFO *other); because it is the collation handler who knows how to detect equal collations by comparing only some of CHARSET_INFO members without their deep initialization. Three implementations were added: - my_ci_eq_collation_uca() for UCA collations, it compares _0900_ collations as equal to their corresponding _uca1400_ collations. - my_ci_eq_collation_utf8mb4_bin(), it compares utf8mb4_nopad_bin and utf8mb4_0900_bin as equal. - my_ci_eq_collation_generic() - the default implementation, which compares all collations as not equal. A C++ wrapper CHARSET_INFO::eq_collations() was added. The code in /sql was changes to use the wrapper instead of the former calls for the removed function compare_collations(). - A part of add_alias_for_collation() was moved into a new function my_ci_alloc(). It allocates a memory for a new charset_info_st instance together with the collation name and the comment using a single MY_CHARSET_LOADER::once_alloc call, which points to my_once_alloc() in the server. - A part of add_alias_for_collation() was moved into a new function my_ci_make_comment_for_alias(). It makes an "Alias for xxx" string, e.g. "Alias for utf8mb4_uca1400_swedish_ai_ci" in case of utf8mb4_sv_0900_ai_ci. - A part of the code in create_tailoring() was moved to a new function my_uca1400_collation_get_initialized_shared_uca(), to reuse the code between _uca1400_ and _0900_ collations. - A new function my_collation_id_is_mysql_uca0900() was added in addition to my_collation_id_is_mysql_uca1400(). - Functions to build collation names were added: my_uca0900_collation_build_name() my_uca1400_collation_build_name() - A shared function function was added: my_bool my_uca1400_collation_alloc_and_init(MY_CHARSET_LOADER *loader, LEX_CSTRING name, LEX_CSTRING comment, const uca_collation_def_param_t *param, uint id) It's reused to add _uca1400_ and _0900_ collations, with basic initialization (without deep initialization). - The function add_compiled_collation() changed its return type from void to int, to make it compatible with MY_CHARSET_LOADER::add_collation. - Functions mysql_uca0900_collation_definition_add(), mysql_uca0900_utf8mb4_collation_definitions_add(), mysql_utf8mb4_0900_bin_add() were added into ctype-uca0900.c. They get MY_CHARSET_LOADER as a parameter. - Functions my_uca1400_collation_definition_add(), my_uca1400_collation_definitions_add() were moved from charset-def.c to strings/ctype-uca1400.c. The latter now accepts MY_CHARSET_LOADER as the first parameter instead of initializing a MY_CHARSET_LOADER inside. - init_compiled_charsets() now initializes a MY_CHARSET_LOADER variable and passes it to all functions adding collations: - mysql_utf8mb4_0900_collation_definitions_add() - mysql_uca0900_utf8mb4_collation_definitions_add() - mysql_utf8mb4_0900_bin_add() - A new structure was added into ctype-uca.h: typedef struct uca_collation_def_param { my_cs_encoding_t cs_id; uint tailoring_id; uint nopad_flags; uint level_flags; } uca_collation_def_param_t; It simplifies reusing the code for _uca1400_ and _0900_ collations. - The definition of MY_UCA1400_COLLATION_DEFINITION was moved from ctype-uca.c to ctype-uca1400.h, to reuse the code for _uca1400_ and _0900_ collations. - The definitions of "MY_UCA_INFO my_uca_v1400" and "MY_UCA_INFO my_uca1400_info_tailored[][]" were moved from ctype-uca.c to ctype-uca1400.c. - The definitions/declarations of: - mysql_0900_collation_start, - struct mysql_0900_to_mariadb_1400_mapping - mysql_0900_to_mariadb_1400_mapping - mysql_utf8mb4_0900_collation_definitions_add() were moved from ctype-uca.c to ctype-uca0900.c - Functions my_uca1400_make_builtin_collation_id() my_uca1400_collation_definition_init() my_uca1400_collation_id_uca400_compat() my_ci_get_collation_name_uca1400_context() were moved from ctype-uca.c to ctype-uca1400.c and ctype-uca1400.h - A part of my_uca1400_collation_definition_init() was moved into my_uca0520_builtin_collation_by_id(), to make functions smaller. --- include/m_ctype.h | 30 +- include/my_sys.h | 5 +- mysql-test/main/ctype_utf8mb4_0900.result | 147 ++++++ mysql-test/main/ctype_utf8mb4_0900.test | 26 ++ mysql-test/main/ctype_utf8mb4_0900_mem.opt | 1 + mysql-test/main/ctype_utf8mb4_0900_mem.result | 108 +++++ mysql-test/main/ctype_utf8mb4_0900_mem.test | 112 +++++ mysys/charset-def.c | 85 +--- mysys/charset.c | 114 +---- sql/field.cc | 18 +- sql/item.cc | 2 +- strings/CMakeLists.txt | 3 +- strings/ctype-big5.c | 12 +- strings/ctype-bin.c | 9 +- strings/ctype-cp932.c | 12 +- strings/ctype-czech.c | 3 +- strings/ctype-euc_kr.c | 12 +- strings/ctype-eucjpms.c | 12 +- strings/ctype-gb2312.c | 12 +- strings/ctype-gbk.c | 12 +- strings/ctype-latin1.c | 3 +- strings/ctype-simple.c | 6 +- strings/ctype-sjis.c | 12 +- strings/ctype-tis620.c | 6 +- strings/ctype-uca.c | 422 +++--------------- strings/ctype-uca.h | 11 +- strings/ctype-uca.inl | 12 +- strings/ctype-uca0520.h | 69 +++ strings/ctype-uca0900.c | 221 +++++++++ strings/ctype-uca0900.h | 47 ++ strings/ctype-uca1400.c | 363 +++++++++++++++ strings/ctype-uca1400.h | 79 +++- strings/ctype-ucs2.c | 51 ++- strings/ctype-ujis.c | 12 +- strings/ctype-utf8.c | 46 +- strings/ctype-win1250ch.c | 3 +- strings/ctype.c | 38 ++ strings/strings_def.h | 8 + 38 files changed, 1498 insertions(+), 646 deletions(-) create mode 100644 mysql-test/main/ctype_utf8mb4_0900_mem.opt create mode 100644 mysql-test/main/ctype_utf8mb4_0900_mem.result create mode 100644 mysql-test/main/ctype_utf8mb4_0900_mem.test create mode 100644 strings/ctype-uca0520.h create mode 100644 strings/ctype-uca0900.c create mode 100644 strings/ctype-uca0900.h create mode 100644 strings/ctype-uca1400.c diff --git a/include/m_ctype.h b/include/m_ctype.h index b7fc1ad9125..136dd48f5b2 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -568,6 +568,22 @@ struct my_collation_handler_st uint (*get_id)(CHARSET_INFO *cs, my_collation_id_type_t type); LEX_CSTRING (*get_collation_name)(CHARSET_INFO *cs, my_collation_name_mode_t mode); + /* + Check if two collations are equally defined, so DTCollation aggregation + code considers them as equal. This is useful for collation aliases, + e.g. for MySQL 0900 collation aliases for 1400 MariaDB collations. + For example, these queries work without raising "Illegal mix of collations": + + SELECT _utf8mb4'a' COLLATE utf8mb4_uca1400_nopad_ai_ci = + _utf8mb4'a' COLLATE utf8mb4_0900_ai_ci; + + SELECT ... WHERE column_with_collation_utf8mb4_uca1400_nopad_ai_ci = + column_with_collation_tf8mb4_0900_ai_ci; + + @return 0 Different + @return 1 Identical + */ + my_bool (*eq_collation)(CHARSET_INFO *self, CHARSET_INFO *other); }; @@ -1110,6 +1126,19 @@ struct charset_info_st { return (coll->get_collation_name)(this, mode); } + + /* + Check if two collations are equally defined. For details + see the definition of eq_collation() in my_collation_handler_st. + + @return 0 Different + @return 1 Identical + */ + my_bool eq_collation(CHARSET_INFO *rhs) const + { + return this == rhs || (coll->eq_collation)(this, rhs); + } + #endif /* __cplusplus */ }; @@ -1693,7 +1722,6 @@ my_bool my_propagate_complex(CHARSET_INFO *cs, const uchar *str, size_t len); uint my_ci_get_id_generic(CHARSET_INFO *cs, my_collation_id_type_t type); LEX_CSTRING my_ci_get_collation_name_generic(CHARSET_INFO *cs, my_collation_name_mode_t mode); -my_bool compare_collations(CHARSET_INFO *cs1, CHARSET_INFO *cs2); typedef struct { diff --git a/include/my_sys.h b/include/my_sys.h index 30c843ede21..bc439f86d01 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -1123,11 +1123,8 @@ static inline my_bool my_charset_same(CHARSET_INFO *cs1, CHARSET_INFO *cs2) return (cs1->cs_name.str == cs2->cs_name.str); } extern my_bool init_compiled_charsets(myf flags); -extern void add_compiled_collation(struct charset_info_st *cs); +extern int add_compiled_collation(struct charset_info_st *cs); extern void add_compiled_extra_collation(struct charset_info_st *cs); -extern my_bool add_alias_for_collation(LEX_CSTRING *collation_name, - uint collation_id, - LEX_CSTRING *alias, uint alias_id); extern size_t escape_string_for_mysql(CHARSET_INFO *charset_info, char *to, size_t to_length, const char *from, size_t length, diff --git a/mysql-test/main/ctype_utf8mb4_0900.result b/mysql-test/main/ctype_utf8mb4_0900.result index 6d0421900b9..5bb3123aab8 100644 --- a/mysql-test/main/ctype_utf8mb4_0900.result +++ b/mysql-test/main/ctype_utf8mb4_0900.result @@ -199,6 +199,153 @@ CREATE OR REPLACE TABLE t1 (p int primary key auto_increment, a VARCHAR(10), key alter table t1 modify a varchar(10) collate utf8mb4_uca1400_swedish_nopad_ai_ci, algorithm=nocopy; drop table t1; # +# Print protocol collation IDs for 0900 collations +# They should be known to libmariadb +# See libmariadb/libmariadb/ma_charset.c +# +FOR rec IN (SELECT COLLATION_NAME +FROM INFORMATION_SCHEMA.COLLATION_CHARACTER_SET_APPLICABILITY +WHERE CHARACTER_SET_NAME='utf8mb4' + AND COLLATION_NAME RLIKE '_0900_' + ORDER BY ID) +DO +EXECUTE IMMEDIATE CONCAT('SET NAMES utf8mb4 COLLATE ', rec.COLLATION_NAME); +SELECT rec.COLLATION_NAME; +END FOR; +$$ +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 18 Y 0 0 255 +utf8mb4_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 24 Y 0 0 256 +utf8mb4_de_pb_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 257 +utf8mb4_is_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 258 +utf8mb4_lv_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 259 +utf8mb4_ro_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 260 +utf8mb4_sl_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 261 +utf8mb4_pl_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 262 +utf8mb4_et_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 263 +utf8mb4_es_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 264 +utf8mb4_sv_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 265 +utf8mb4_tr_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 266 +utf8mb4_cs_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 267 +utf8mb4_da_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 268 +utf8mb4_lt_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 269 +utf8mb4_sk_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 26 Y 0 0 270 +utf8mb4_es_trad_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 271 +utf8mb4_la_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 273 +utf8mb4_eo_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 274 +utf8mb4_hu_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 275 +utf8mb4_hr_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 277 +utf8mb4_vi_0900_ai_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 18 Y 0 0 278 +utf8mb4_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 24 Y 0 0 279 +utf8mb4_de_pb_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 280 +utf8mb4_is_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 281 +utf8mb4_lv_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 282 +utf8mb4_ro_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 283 +utf8mb4_sl_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 284 +utf8mb4_pl_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 285 +utf8mb4_et_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 286 +utf8mb4_es_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 287 +utf8mb4_sv_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 288 +utf8mb4_tr_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 289 +utf8mb4_cs_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 290 +utf8mb4_da_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 291 +utf8mb4_lt_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 292 +utf8mb4_sk_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 26 Y 0 0 293 +utf8mb4_es_trad_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 294 +utf8mb4_la_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 296 +utf8mb4_eo_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 297 +utf8mb4_hu_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 298 +utf8mb4_hr_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 21 Y 0 0 300 +utf8mb4_vi_0900_as_cs +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 18 Y 0 0 305 +utf8mb4_0900_as_ci +Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr +def COLLATION_NAME rec.COLLATION_NAME 253 256 16 Y 0 0 309 +utf8mb4_0900_bin +# # MDEV-36361 Wrong utf8mb4_0900_bin alias for utf8mb4_bin (should be utf8mb4_nopad_bin) # SELECT collation_name, id, comment diff --git a/mysql-test/main/ctype_utf8mb4_0900.test b/mysql-test/main/ctype_utf8mb4_0900.test index 98c633a98ab..d8dbff07ca1 100644 --- a/mysql-test/main/ctype_utf8mb4_0900.test +++ b/mysql-test/main/ctype_utf8mb4_0900.test @@ -84,6 +84,32 @@ CREATE OR REPLACE TABLE t1 (p int primary key auto_increment, a VARCHAR(10), key alter table t1 modify a varchar(10) collate utf8mb4_uca1400_swedish_nopad_ai_ci, algorithm=nocopy; drop table t1; +--echo # +--echo # Print protocol collation IDs for 0900 collations +--echo # They should be known to libmariadb +--echo # See libmariadb/libmariadb/ma_charset.c +--echo # + +--disable_column_names +--disable_ps_protocol +--enable_metadata +DELIMITER $$; +FOR rec IN (SELECT COLLATION_NAME + FROM INFORMATION_SCHEMA.COLLATION_CHARACTER_SET_APPLICABILITY + WHERE CHARACTER_SET_NAME='utf8mb4' + AND COLLATION_NAME RLIKE '_0900_' + ORDER BY ID) +DO + EXECUTE IMMEDIATE CONCAT('SET NAMES utf8mb4 COLLATE ', rec.COLLATION_NAME); + SELECT rec.COLLATION_NAME; +END FOR; +$$ +DELIMITER ;$$ +--disable_metadata +--enable_ps_protocol +--enable_column_names + + --echo # --echo # MDEV-36361 Wrong utf8mb4_0900_bin alias for utf8mb4_bin (should be utf8mb4_nopad_bin) --echo # diff --git a/mysql-test/main/ctype_utf8mb4_0900_mem.opt b/mysql-test/main/ctype_utf8mb4_0900_mem.opt new file mode 100644 index 00000000000..e61509fb54b --- /dev/null +++ b/mysql-test/main/ctype_utf8mb4_0900_mem.opt @@ -0,0 +1 @@ +--init_connect="set @a='ctype_utf8mb4_0900_mem - run a dedicated mariadbd for this test'" diff --git a/mysql-test/main/ctype_utf8mb4_0900_mem.result b/mysql-test/main/ctype_utf8mb4_0900_mem.result new file mode 100644 index 00000000000..574789d0c6f --- /dev/null +++ b/mysql-test/main/ctype_utf8mb4_0900_mem.result @@ -0,0 +1,108 @@ +# +# MDEV-36213 Doubled memory usage (11.4.4 <-> 11.4.5) +# +SET NAMES utf8mb4; +CREATE FUNCTION memory_used() RETURNS BIGINT RETURN +(SELECT variable_value +FROM information_schema.global_status +WHERE variable_name='memory_used'); +CREATE PROCEDURE p1(cl VARCHAR(64)) +BEGIN +DECLARE mem_before BIGINT; +DECLARE mem_after BIGINT; +DECLARE query TEXT DEFAULT CONCAT('SET @a= _utf8mb4 0x20 COLLATE ', cl); +SET mem_before= memory_used(); +EXECUTE IMMEDIATE query; +SET mem_after= memory_used(); +SELECT +CASE +WHEN mem_after-mem_before >= 1024*1024 THEN '>=1M' + ELSE '<1M' + END AS diff, +CONCAT(query,';') AS query; +END; +/ +CREATE PROCEDURE p2(cl VARCHAR(64)) +BEGIN +DECLARE mem_before BIGINT; +DECLARE mem_after BIGINT; +DECLARE query TEXT DEFAULT CONCAT( +'SELECT id, full_collation_name' + ' FROM information_schema.collation_character_set_applicability' + ' WHERE full_collation_name LIKE ''PATTERN'' ORDER BY id'); +SET query= REPLACE(query, 'PATTERN', cl); +SELECT query; +SET mem_before= memory_used(); +EXECUTE IMMEDIATE query; +SET mem_after=memory_used(); +SELECT +CASE +WHEN mem_before-mem_after >= 1024*1024 THEN '>=1M' + ELSE '<1M' + END AS diff; +END; +/ +# +# Initialize spanish2 collations, an UCA-14.0.0 collation goes first +# +>=1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_spanish2_ai_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_spanish2_ai_cs; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_spanish2_as_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_spanish2_as_cs; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_spanish2_nopad_ai_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_spanish2_nopad_ai_cs; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_spanish2_nopad_as_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_spanish2_nopad_as_cs; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_es_trad_0900_ai_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_es_trad_0900_as_cs; +# +# I_S queries for initialized collations should not add memory +# +SELECT id, full_collation_name FROM information_schema.collation_character_set_applicability WHERE full_collation_name LIKE 'utf8mb4_uca1400_spanish2%' ORDER BY id +2416 utf8mb4_uca1400_spanish2_ai_ci +2417 utf8mb4_uca1400_spanish2_ai_cs +2418 utf8mb4_uca1400_spanish2_as_ci +2419 utf8mb4_uca1400_spanish2_as_cs +2420 utf8mb4_uca1400_spanish2_nopad_ai_ci +2421 utf8mb4_uca1400_spanish2_nopad_ai_cs +2422 utf8mb4_uca1400_spanish2_nopad_as_ci +2423 utf8mb4_uca1400_spanish2_nopad_as_cs +<1M +SELECT id, full_collation_name FROM information_schema.collation_character_set_applicability WHERE full_collation_name LIKE 'utf8mb4_%es_trad_0900%' ORDER BY id +270 utf8mb4_es_trad_0900_ai_ci +293 utf8mb4_es_trad_0900_as_cs +<1M +# +# I_S queries for not initialized collations should not add memory +# +SELECT id, full_collation_name FROM information_schema.collation_character_set_applicability WHERE full_collation_name LIKE 'utf8mb4_uca1400_german2%' ORDER BY id +2464 utf8mb4_uca1400_german2_ai_ci +2465 utf8mb4_uca1400_german2_ai_cs +2466 utf8mb4_uca1400_german2_as_ci +2467 utf8mb4_uca1400_german2_as_cs +2468 utf8mb4_uca1400_german2_nopad_ai_ci +2469 utf8mb4_uca1400_german2_nopad_ai_cs +2470 utf8mb4_uca1400_german2_nopad_as_ci +2471 utf8mb4_uca1400_german2_nopad_as_cs +<1M +SELECT id, full_collation_name FROM information_schema.collation_character_set_applicability WHERE full_collation_name LIKE 'utf8mb4_%de_pb_0900%' ORDER BY id +256 utf8mb4_de_pb_0900_ai_ci +279 utf8mb4_de_pb_0900_as_cs +<1M +# +# Initialize german2 collations, an UCA-9.0.0 alias goes first +# +>=1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_de_pb_0900_ai_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_de_pb_0900_as_cs; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_german2_ai_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_german2_ai_cs; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_german2_as_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_german2_as_cs; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_german2_nopad_ai_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_german2_nopad_ai_cs; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_german2_nopad_as_ci; +<1M SET @a= _utf8mb4 0x20 COLLATE utf8mb4_uca1400_german2_nopad_as_cs; +DROP PROCEDURE p2; +DROP PROCEDURE p1; +DROP FUNCTION memory_used; +# End of 11.4 tests diff --git a/mysql-test/main/ctype_utf8mb4_0900_mem.test b/mysql-test/main/ctype_utf8mb4_0900_mem.test new file mode 100644 index 00000000000..7a03376d47e --- /dev/null +++ b/mysql-test/main/ctype_utf8mb4_0900_mem.test @@ -0,0 +1,112 @@ +--echo # +--echo # MDEV-36213 Doubled memory usage (11.4.4 <-> 11.4.5) +--echo # + +SET NAMES utf8mb4; + +CREATE FUNCTION memory_used() RETURNS BIGINT RETURN +(SELECT variable_value + FROM information_schema.global_status + WHERE variable_name='memory_used'); + +DELIMITER /; + +CREATE PROCEDURE p1(cl VARCHAR(64)) +BEGIN + DECLARE mem_before BIGINT; + DECLARE mem_after BIGINT; + DECLARE query TEXT DEFAULT CONCAT('SET @a= _utf8mb4 0x20 COLLATE ', cl); + + SET mem_before= memory_used(); + EXECUTE IMMEDIATE query; + SET mem_after= memory_used(); + + SELECT + CASE + WHEN mem_after-mem_before >= 1024*1024 THEN '>=1M' + ELSE '<1M' + END AS diff, + CONCAT(query,';') AS query; +END; +/ + +CREATE PROCEDURE p2(cl VARCHAR(64)) +BEGIN + DECLARE mem_before BIGINT; + DECLARE mem_after BIGINT; + DECLARE query TEXT DEFAULT CONCAT( + 'SELECT id, full_collation_name' + ' FROM information_schema.collation_character_set_applicability' + ' WHERE full_collation_name LIKE ''PATTERN'' ORDER BY id'); + + SET query= REPLACE(query, 'PATTERN', cl); + SELECT query; + SET mem_before= memory_used(); + EXECUTE IMMEDIATE query; + SET mem_after=memory_used(); + + SELECT + CASE + WHEN mem_before-mem_after >= 1024*1024 THEN '>=1M' + ELSE '<1M' + END AS diff; +END; +/ +DELIMITER ;/ + + +--disable_column_names +--disable_query_log + +--echo # +--echo # Initialize spanish2 collations, an UCA-14.0.0 collation goes first +--echo # + +CALL p1('utf8mb4_uca1400_spanish2_ai_ci'); +CALL p1('utf8mb4_uca1400_spanish2_ai_cs'); +CALL p1('utf8mb4_uca1400_spanish2_as_ci'); +CALL p1('utf8mb4_uca1400_spanish2_as_cs'); +CALL p1('utf8mb4_uca1400_spanish2_nopad_ai_ci'); +CALL p1('utf8mb4_uca1400_spanish2_nopad_ai_cs'); +CALL p1('utf8mb4_uca1400_spanish2_nopad_as_ci'); +CALL p1('utf8mb4_uca1400_spanish2_nopad_as_cs'); +CALL p1('utf8mb4_es_trad_0900_ai_ci'); +CALL p1('utf8mb4_es_trad_0900_as_cs'); + +--echo # +--echo # I_S queries for initialized collations should not add memory +--echo # + +CALL p2('utf8mb4_uca1400_spanish2%'); +CALL p2('utf8mb4_%es_trad_0900%'); + +--echo # +--echo # I_S queries for not initialized collations should not add memory +--echo # + +CALL p2('utf8mb4_uca1400_german2%'); +CALL p2('utf8mb4_%de_pb_0900%'); + +--echo # +--echo # Initialize german2 collations, an UCA-9.0.0 alias goes first +--echo # + +CALL p1('utf8mb4_de_pb_0900_ai_ci'); +CALL p1('utf8mb4_de_pb_0900_as_cs'); +CALL p1('utf8mb4_uca1400_german2_ai_ci'); +CALL p1('utf8mb4_uca1400_german2_ai_cs'); +CALL p1('utf8mb4_uca1400_german2_as_ci'); +CALL p1('utf8mb4_uca1400_german2_as_cs'); +CALL p1('utf8mb4_uca1400_german2_nopad_ai_ci'); +CALL p1('utf8mb4_uca1400_german2_nopad_ai_cs'); +CALL p1('utf8mb4_uca1400_german2_nopad_as_ci'); +CALL p1('utf8mb4_uca1400_german2_nopad_as_cs'); +--enable_query_log +--enable_column_names + + +DROP PROCEDURE p2; +DROP PROCEDURE p1; +DROP FUNCTION memory_used; + +--echo # End of 11.4 tests diff --git a/mysys/charset-def.c b/mysys/charset-def.c index e2e5a747eb5..5c2fac91773 100644 --- a/mysys/charset-def.c +++ b/mysys/charset-def.c @@ -184,76 +184,10 @@ extern struct charset_info_st my_charset_utf8mb4_unicode_520_nopad_ci; #endif /* HAVE_UCA_COLLATIONS */ -static my_bool -my_uca1400_collation_definition_add(MY_CHARSET_LOADER *loader, - my_cs_encoding_t charset_id, - uint tailoring_id, - my_bool nopad, - my_bool secondary_level, - my_bool tertiary_level) -{ - struct charset_info_st *tmp; - uint collation_id= my_uca1400_make_builtin_collation_id(charset_id, - tailoring_id, - nopad, - secondary_level, - tertiary_level); - if (!collation_id) - return FALSE; - if (!(tmp= (struct charset_info_st*) - my_once_alloc(sizeof(CHARSET_INFO),MYF(0)))) - return TRUE; - if (my_uca1400_collation_definition_init(loader, tmp, collation_id)) - return TRUE; - add_compiled_collation(tmp); - return FALSE; -} - - -static my_bool -my_uca1400_collation_definitions_add() -{ - my_cs_encoding_t charset_id; - MY_CHARSET_LOADER loader; - my_charset_loader_init_mysys(&loader); - for (charset_id= (my_cs_encoding_t) 0; - charset_id <= (my_cs_encoding_t) MY_CS_ENCODING_LAST; - charset_id++) - { - uint tailoring_id; - for (tailoring_id= 0 ; - tailoring_id < MY_UCA1400_COLLATION_DEFINITION_COUNT; - tailoring_id++) - { - uint nopad; - for (nopad= 0; nopad < 2; nopad++) - { - uint secondary_level; - for (secondary_level= 0; secondary_level < 2; secondary_level++) - { - if (my_uca1400_collation_definition_add(&loader, - charset_id, tailoring_id, - (my_bool) nopad, - (my_bool) secondary_level, - FALSE)) - return TRUE; - if (my_uca1400_collation_definition_add(&loader, - charset_id, tailoring_id, - (my_bool) nopad, - (my_bool) secondary_level, - TRUE)) - return TRUE; - } - } - } - } - return FALSE; -} - - my_bool init_compiled_charsets(myf flags __attribute__((unused))) { CHARSET_INFO *cs; + MY_CHARSET_LOADER loader; add_compiled_collation(&my_charset_bin); add_compiled_collation(&my_charset_filename); @@ -541,9 +475,22 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) for (cs=compiled_charsets; cs->coll_name.str; cs++) add_compiled_extra_collation((struct charset_info_st *) cs); - if (my_uca1400_collation_definitions_add()) + /* + my_charset_loader_init_mysys() initializes + MY_CHARSET_LOADER::add_collation to the function + add_collation() defined in charset.c + Let's reset it to add_compiled_collation(). + */ + my_charset_loader_init_mysys(&loader); + loader.add_collation= add_compiled_collation; + + if (my_uca1400_collation_definitions_add(&loader)) return TRUE; - if (mysql_utf8mb4_0900_collation_definitions_add()) + + if (mysql_uca0900_utf8mb4_collation_definitions_add(&loader)) + return TRUE; + + if (mysql_utf8mb4_0900_bin_add(&loader)) return TRUE; return FALSE; diff --git a/mysys/charset.c b/mysys/charset.c index 8918bb1b2ac..aa5fe3e0b8f 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -597,7 +597,7 @@ CHARSET_INFO *default_charset_info = &my_charset_latin1; All related character sets should share same cname */ -void add_compiled_collation(struct charset_info_st *cs) +int add_compiled_collation(struct charset_info_st *cs) { DBUG_ASSERT(cs->number < array_elements(all_charsets)); all_charsets[cs->number]= cs; @@ -613,6 +613,7 @@ void add_compiled_collation(struct charset_info_st *cs) DBUG_ASSERT(org->cs_name.length == strlen(cs->cs_name.str)); #endif } + return 0; } @@ -640,69 +641,6 @@ void add_compiled_extra_collation(struct charset_info_st *cs) } -/* - Add an alias for a collation with an unique id - Used to add MySQL utf8mb4_0900 collations to MariaDB as an alias for the - corresponding utf8mb4_1400 collation -*/ - -my_bool add_alias_for_collation(LEX_CSTRING *collation_name, uint org_id, - LEX_CSTRING *alias, uint alias_id) -{ - char *coll_name, *comment; - struct charset_info_st *new_ci; - CHARSET_INFO *org; - MY_CHARSET_LOADER loader; - char comment_buff[64+15]; - size_t comment_length; - DBUG_ASSERT(all_charsets[org_id]); - - if (!(org= all_charsets[org_id])) - return 1; - - DBUG_ASSERT(!my_strcasecmp(&my_charset_latin1, org->coll_name.str, - collation_name->str)); -#ifdef DEBUG_PRINT_ALIAS - fprintf(stderr, "alias: %s collation: %s org_id: %u\n", - alias->str, collation_name->str, org_id); -#endif - - /* - We have to init the character set to ensure it is not changed after we copy - it. - */ - my_charset_loader_init_mysys(&loader); - if (my_ci_init_charset((struct charset_info_st*) org, &loader) || - my_ci_init_collation((struct charset_info_st*) org, &loader) || - (org->m_ctype && - init_state_maps((struct charset_info_st*) org))) - return 1; - ((struct charset_info_st*) org)->state|= MY_CS_READY; - - comment_length= strxnmov(comment_buff, sizeof(comment_buff)-1, - "Alias for ", collation_name->str, - NullS) - comment_buff; - - if (!(new_ci= ((struct charset_info_st*) - my_once_alloc(sizeof(CHARSET_INFO) + - alias->length + comment_length + 2, - MYF(MY_WME))))) - return 1; - - coll_name= (char*) (new_ci+1); - comment= coll_name + alias->length +1; - memcpy((void*) new_ci, org, sizeof(CHARSET_INFO)); - (new_ci->coll_name.str)= coll_name; - memcpy(coll_name, alias->str, alias->length+1); - memcpy(comment, comment_buff, comment_length+1); - new_ci->coll_name.length= alias->length; - new_ci->comment= comment; - new_ci->number= alias_id; - all_charsets[alias_id]= new_ci; - return 0; -} - - static my_pthread_once_t charsets_initialized= MY_PTHREAD_ONCE_INIT; static my_pthread_once_t charsets_template= MY_PTHREAD_ONCE_INIT; @@ -722,54 +660,6 @@ my_bool my_collation_is_known_id(uint id) } -/* - Compare if two collations are identical. - They are identical if all slots are identical except collation name and - number. Note that alias collations are made by memcpy(), which means that - also the also padding in the structures are identical. - - Note that this code assumes knowledge of the CHARSET_INFO structure. - Especially the place of number, cs_name, coll_name and tailoring. - - Other option would have been to add a new member 'alias_collation' - into CHARSET_INFO where all identical collations would point to, - but that would have changed the CHARSET_INFO structure which would - have required a lot more changes. - - @return 0 Identical - @return 1 Different -*/ - -my_bool compare_collations(CHARSET_INFO *cs1, CHARSET_INFO *cs2) -{ - size_t length; - - if (cs1 == cs2) - return 0; - - /* Quick check to detect different collation */ - if (cs1->cset != cs2->cset || cs1->coll != cs2->coll || - cs1->uca != cs2->uca) - goto diff; - - /* We don't compare character set number */ - if (cs1->primary_number != cs2->primary_number) - goto diff; - if (cs1->binary_number != cs2->binary_number) - goto diff; - if (cs1->state != cs2->state) - goto diff; - - /* Compare everything after comment_name */ - length= sizeof(CHARSET_INFO) - (((char*) &cs1->tailoring) - (char*) cs1); - - if (!memcmp(&cs1->tailoring, &cs2->tailoring, length)) - return 0; -diff: - return 1; -} - - /* Collation use statistics functions do not lock counters to avoid mutex contention. This can lose diff --git a/sql/field.cc b/sql/field.cc index 7358b142621..19fd3528bd0 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -2719,7 +2719,7 @@ bool Field_null::is_equal(const Column_definition &new_field) const { DBUG_ASSERT(!compression_method()); return (new_field.type_handler() == type_handler() && - !compare_collations(new_field.charset, field_charset()) && + new_field.charset->eq_collation(field_charset()) && new_field.length == max_display_length()); } @@ -7492,7 +7492,7 @@ bool Field_string::is_equal(const Column_definition &new_field) const DBUG_ASSERT(!compression_method()); return (new_field.type_handler() == type_handler() && new_field.char_length == char_length() && - !compare_collations(new_field.charset, field_charset()) && + new_field.charset->eq_collation(field_charset()) && new_field.length == max_display_length()); } @@ -7516,7 +7516,7 @@ Field_longstr::cmp_to_string_with_same_collation(const Item_bool_func *cond, { return (!cmp_is_done_using_type_handler_of_this(cond, item) ? Data_type_compatibility::INCOMPATIBLE_DATA_TYPE : - compare_collations(charset(), cond->compare_collation()) ? + !charset()->eq_collation(cond->compare_collation()) ? Data_type_compatibility::INCOMPATIBLE_COLLATION : Data_type_compatibility::OK); } @@ -7528,7 +7528,7 @@ Field_longstr::cmp_to_string_with_stricter_collation(const Item_bool_func *cond, { return (!cmp_is_done_using_type_handler_of_this(cond, item) ? Data_type_compatibility::INCOMPATIBLE_DATA_TYPE : - (compare_collations(charset(), cond->compare_collation()) && + (!charset()->eq_collation(cond->compare_collation()) && !(cond->compare_collation()->state & MY_CS_BINSORT) && !Utf8_narrow::should_do_narrowing(this, cond->compare_collation())) ? Data_type_compatibility::INCOMPATIBLE_COLLATION : @@ -8447,7 +8447,7 @@ bool Field_varstring::is_equal(const Column_definition &new_field) const new_field.length == field_length && new_field.char_length == char_length() && !new_field.compression_method() == !compression_method() && - !compare_collations(new_field.charset, field_charset())); + new_field.charset->eq_collation(field_charset())); } @@ -8714,7 +8714,7 @@ uint32 Field_blob::get_length(const uchar *pos, uint packlength_arg) const */ int Field_blob::copy_value(Field_blob *from) { - DBUG_ASSERT(!compare_collations(field_charset(), from->charset())); + DBUG_ASSERT(field_charset()->eq_collation(from->charset())); DBUG_ASSERT(!compression_method() == !from->compression_method()); int rc= 0; uint32 length= from->get_length(); @@ -9248,7 +9248,7 @@ bool Field_blob::is_equal(const Column_definition &new_field) const return (new_field.type_handler() == type_handler() && !new_field.compression_method() == !compression_method() && new_field.pack_length == pack_length() && - !compare_collations(new_field.charset, field_charset())); + new_field.charset->eq_collation(field_charset())); } @@ -9746,7 +9746,7 @@ bool Field_enum::is_equal(const Column_definition &new_field) const type, charset and have the same underlying length. */ if (new_field.type_handler() != type_handler() || - compare_collations(new_field.charset, field_charset()) || + !new_field.charset->eq_collation(field_charset()) || new_field.pack_length != pack_length()) return false; @@ -9853,7 +9853,7 @@ Field_enum::can_optimize_range_or_keypart_ref(const Item_bool_func *cond, case REAL_RESULT: return Data_type_compatibility::OK; case STRING_RESULT: - return (!compare_collations(charset(), cond->compare_collation()) ? + return (charset()->eq_collation(cond->compare_collation()) ? Data_type_compatibility::OK : Data_type_compatibility::INCOMPATIBLE_COLLATION); case ROW_RESULT: diff --git a/sql/item.cc b/sql/item.cc index 6748ce2123c..5f6bc3a8d57 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -2568,7 +2568,7 @@ bool DTCollation::aggregate(const DTCollation &dt, uint flags) } else { - if (!compare_collations(collation, dt.collation)) + if (collation->eq_collation(dt.collation)) { /* Do nothing */ } diff --git a/strings/CMakeLists.txt b/strings/CMakeLists.txt index 099f3c67660..241a9d62e2e 100644 --- a/strings/CMakeLists.txt +++ b/strings/CMakeLists.txt @@ -20,7 +20,8 @@ ${CMAKE_BINARY_DIR}/strings SET(STRINGS_SOURCES bchange.c bmove_upp.c ctype-big5.c ctype-bin.c ctype-cp932.c ctype-czech.c ctype-euc_kr.c ctype-eucjpms.c ctype-extra.c ctype-gb2312.c ctype-gbk.c - ctype-latin1.c ctype-mb.c ctype-simple.c ctype-sjis.c ctype-tis620.c ctype-uca.c + ctype-latin1.c ctype-mb.c ctype-simple.c ctype-sjis.c ctype-tis620.c + ctype-uca.c ctype-uca0900.c ctype-uca1400.c ctype-ucs2.c ctype-ujis.c ctype-utf8.c ctype-win1250ch.c ctype.c decimal.c dtoa.c int2str.c ctype-unidata.c is_prefix.c llstr.c longlong2str.c my_strtoll10.c my_vsnprintf.c diff --git a/strings/ctype-big5.c b/strings/ctype-big5.c index aaf6769989b..6bd5699339e 100644 --- a/strings/ctype-big5.c +++ b/strings/ctype-big5.c @@ -6730,7 +6730,8 @@ static MY_COLLATION_HANDLER my_collation_handler_big5_chinese_ci= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -6751,7 +6752,8 @@ static MY_COLLATION_HANDLER my_collation_handler_big5_bin= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -6772,7 +6774,8 @@ static MY_COLLATION_HANDLER my_collation_handler_big5_chinese_nopad_ci= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -6793,7 +6796,8 @@ static MY_COLLATION_HANDLER my_collation_handler_big5_nopad_bin= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-bin.c b/strings/ctype-bin.c index 084468790e5..31d5fc78f33 100644 --- a/strings/ctype-bin.c +++ b/strings/ctype-bin.c @@ -529,7 +529,8 @@ MY_COLLATION_HANDLER my_collation_8bit_bin_handler = my_min_str_8bit_simple, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -550,7 +551,8 @@ MY_COLLATION_HANDLER my_collation_8bit_nopad_bin_handler = my_min_str_8bit_simple_nopad, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -571,7 +573,8 @@ static MY_COLLATION_HANDLER my_collation_binary_handler = my_min_str_8bit_simple_nopad, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-cp932.c b/strings/ctype-cp932.c index f01909d4f65..ca48d1d7bd0 100644 --- a/strings/ctype-cp932.c +++ b/strings/ctype-cp932.c @@ -34687,7 +34687,8 @@ static MY_COLLATION_HANDLER my_collation_handler_cp932_japanese_ci= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -34708,7 +34709,8 @@ static MY_COLLATION_HANDLER my_collation_handler_cp932_bin= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -34729,7 +34731,8 @@ static MY_COLLATION_HANDLER my_collation_handler_cp932_japanese_nopad_ci= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -34750,7 +34753,8 @@ static MY_COLLATION_HANDLER my_collation_handler_cp932_nopad_bin= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-czech.c b/strings/ctype-czech.c index 64aff2e7be0..6e478a69b76 100644 --- a/strings/ctype-czech.c +++ b/strings/ctype-czech.c @@ -598,7 +598,8 @@ static MY_COLLATION_HANDLER my_collation_latin2_czech_cs_handler = my_min_str_8bit_simple, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; struct charset_info_st my_charset_latin2_czech_cs = diff --git a/strings/ctype-euc_kr.c b/strings/ctype-euc_kr.c index dd6ab60bd3a..cdfbf51fd3c 100644 --- a/strings/ctype-euc_kr.c +++ b/strings/ctype-euc_kr.c @@ -9977,7 +9977,8 @@ static MY_COLLATION_HANDLER my_collation_handler_euckr_korean_ci= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -9998,7 +9999,8 @@ static MY_COLLATION_HANDLER my_collation_handler_euckr_bin= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -10019,7 +10021,8 @@ static MY_COLLATION_HANDLER my_collation_handler_euckr_korean_nopad_ci= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -10040,7 +10043,8 @@ static MY_COLLATION_HANDLER my_collation_handler_euckr_nopad_bin= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-eucjpms.c b/strings/ctype-eucjpms.c index a5f727551aa..9ee9cb8c76e 100644 --- a/strings/ctype-eucjpms.c +++ b/strings/ctype-eucjpms.c @@ -67515,7 +67515,8 @@ static MY_COLLATION_HANDLER my_collation_eucjpms_japanese_ci_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -67536,7 +67537,8 @@ static MY_COLLATION_HANDLER my_collation_eucjpms_bin_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -67557,7 +67559,8 @@ static MY_COLLATION_HANDLER my_collation_eucjpms_japanese_nopad_ci_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -67578,7 +67581,8 @@ static MY_COLLATION_HANDLER my_collation_eucjpms_nopad_bin_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-gb2312.c b/strings/ctype-gb2312.c index f3e9c902389..f4c2838dc75 100644 --- a/strings/ctype-gb2312.c +++ b/strings/ctype-gb2312.c @@ -6381,7 +6381,8 @@ static MY_COLLATION_HANDLER my_collation_handler_gb2312_chinese_ci= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -6402,7 +6403,8 @@ static MY_COLLATION_HANDLER my_collation_handler_gb2312_bin= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -6423,7 +6425,8 @@ static MY_COLLATION_HANDLER my_collation_handler_gb2312_chinese_nopad_ci= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -6444,7 +6447,8 @@ static MY_COLLATION_HANDLER my_collation_handler_gb2312_nopad_bin= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-gbk.c b/strings/ctype-gbk.c index a22e6aa8d84..44501276c92 100644 --- a/strings/ctype-gbk.c +++ b/strings/ctype-gbk.c @@ -10663,7 +10663,8 @@ static MY_COLLATION_HANDLER my_collation_handler_gbk_chinese_ci= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -10684,7 +10685,8 @@ static MY_COLLATION_HANDLER my_collation_handler_gbk_bin= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -10705,7 +10707,8 @@ static MY_COLLATION_HANDLER my_collation_handler_gbk_chinese_nopad_ci= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -10726,7 +10729,8 @@ static MY_COLLATION_HANDLER my_collation_handler_gbk_nopad_bin= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; static MY_CHARSET_HANDLER my_charset_handler= diff --git a/strings/ctype-latin1.c b/strings/ctype-latin1.c index 1b8660ca08c..ac68c017308 100644 --- a/strings/ctype-latin1.c +++ b/strings/ctype-latin1.c @@ -742,7 +742,8 @@ static MY_COLLATION_HANDLER my_collation_german2_ci_handler= my_min_str_8bit_simple, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-simple.c b/strings/ctype-simple.c index dc2acc5a608..300cf56ef67 100644 --- a/strings/ctype-simple.c +++ b/strings/ctype-simple.c @@ -2192,7 +2192,8 @@ MY_COLLATION_HANDLER my_collation_8bit_simple_ci_handler = my_min_str_8bit_simple, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -2213,5 +2214,6 @@ MY_COLLATION_HANDLER my_collation_8bit_simple_nopad_ci_handler = my_min_str_8bit_simple_nopad, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-sjis.c b/strings/ctype-sjis.c index a5fed41b964..4ce618aba61 100644 --- a/strings/ctype-sjis.c +++ b/strings/ctype-sjis.c @@ -34075,7 +34075,8 @@ static MY_COLLATION_HANDLER my_collation_handler_sjis_japanese_ci= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -34096,7 +34097,8 @@ static MY_COLLATION_HANDLER my_collation_handler_sjis_bin= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -34117,7 +34119,8 @@ static MY_COLLATION_HANDLER my_collation_handler_sjis_japanese_nopad_ci= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -34138,7 +34141,8 @@ static MY_COLLATION_HANDLER my_collation_handler_sjis_nopad_bin= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-tis620.c b/strings/ctype-tis620.c index 0b728346f6a..d7996f71787 100644 --- a/strings/ctype-tis620.c +++ b/strings/ctype-tis620.c @@ -881,7 +881,8 @@ static MY_COLLATION_HANDLER my_collation_ci_handler = my_min_str_8bit_simple, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; static MY_COLLATION_HANDLER my_collation_nopad_ci_handler = @@ -901,7 +902,8 @@ static MY_COLLATION_HANDLER my_collation_nopad_ci_handler = my_min_str_8bit_simple_nopad, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; static MY_CHARSET_HANDLER my_charset_handler= diff --git a/strings/ctype-uca.c b/strings/ctype-uca.c index 3650eca61b9..bdac97ebb9e 100644 --- a/strings/ctype-uca.c +++ b/strings/ctype-uca.c @@ -34,8 +34,8 @@ #include "strings_def.h" #include -#include #include "ctype-uca.h" +#include "ctype-uca0520.h" #include "ctype-unidata.h" #include "my_bit.h" @@ -30214,78 +30214,6 @@ MY_UCA_INFO my_uca_v520= }; -#include "ctype-uca1400data.h" - -static MY_UCA_INFO my_uca_v1400= -{ - { - { - 0x10FFFF, /* maxchar */ - (uchar *) uca1400_length, - (uint16 **) uca1400_weight, - { /* Contractions: */ - array_elements(uca1400_contractions), /* nitems */ - uca1400_contractions, /* item */ - NULL /* flags */ - }, - 0, /* levelno */ - {0}, /* contraction_hash */ - NULL /* booster */ - }, - - { - 0x10FFFF, /* maxchar */ - (uchar *) uca1400_length_secondary, - (uint16 **) uca1400_weight_secondary, - { /* Contractions: */ - array_elements(uca1400_contractions_secondary), /* nitems */ - uca1400_contractions_secondary, /* item */ - NULL /* flags */ - }, - 1, /* levelno */ - {0}, /* contraction_hash */ - NULL /* booster */ - }, - - { - 0x10FFFF, /* maxchar */ - (uchar *) uca1400_length_tertiary, - (uint16 **) uca1400_weight_tertiary, - { /* Contractions: */ - array_elements(uca1400_contractions_tertiary), /* nitems */ - uca1400_contractions_tertiary, /* item */ - NULL /* flags */ - }, - 2, /* levelno */ - {0}, /* contraction_hash */ - NULL /* booster */ - } - - }, - - uca1400_non_ignorable_first, - uca1400_non_ignorable_last, - - uca1400_primary_ignorable_first, - uca1400_primary_ignorable_last, - - uca1400_secondary_ignorable_first, - uca1400_secondary_ignorable_last, - - uca1400_tertiary_ignorable_first, - uca1400_tertiary_ignorable_last, - - 0x0000, /* first_trailing */ - 0x0000, /* last_trailing */ - - uca1400_variable_first, - uca1400_variable_last, - - /* Misc */ - uca1400_version -}; - - /******************************************************/ /* @@ -31247,25 +31175,12 @@ static const char myanmar[]= "[shift-after-method expand]" ; -typedef struct my_uca1400_collation_definition_st -{ - const char * tailoring; - const char * name; - uint16 id_utf8mb3; - uint16 id_utf8mb4; - uint16 id_ucs2; - uint16 id_utf16; - uint16 id_utf32; -} MY_UCA1400_COLLATION_DEFINITION; - - - /* UCA1400 collation definitions in the order of their UCA400 counterparts, with IDs of their closest UCA1400 counterparts, for character sets utf8mb3, utf8mb4, ucs2, utf16, utf32. */ -static MY_UCA1400_COLLATION_DEFINITION +MY_UCA1400_COLLATION_DEFINITION my_uca1400_collation_definitions[MY_UCA1400_COLLATION_DEFINITION_COUNT]= { #define COLDEF(tl,name,id_utf8mb3,id_utf8mb4,id_ucs2,id_utf16,id_utf32) \ @@ -31309,9 +31224,17 @@ my_uca1400_collation_definitions[MY_UCA1400_COLLATION_DEFINITION_COUNT]= }; -static MY_UCA_INFO -my_uca1400_info_tailored[MY_CS_ENCODING_LAST+1] - [MY_UCA1400_COLLATION_DEFINITION_COUNT]; +static my_bool +my_ci_eq_collation_uca(CHARSET_INFO *a, CHARSET_INFO *b) +{ + return a->cset == b->cset && + a->coll == b->coll && + a->uca == b->uca && + a->casefold == b->casefold && + (a->state & MY_CS_NOPAD) == (b->state & MY_CS_NOPAD) && + a->levels_for_order == b->levels_for_order && + a->tailoring == b->tailoring; +} typedef struct my_uca_scanner_param_st @@ -34722,6 +34645,30 @@ my_uca_info_init(MY_CHARSET_LOADER *loader, } +/* + Initialize (if needed) an element of the array my_uca1400_info_tailored[]. + UCA1400 collations with equal character set and tailoring + (but with different level flags) share the same MY_UCA_INFO. +*/ +static MY_UCA_INFO * +my_uca1400_collation_get_initialized_shared_uca(MY_CHARSET_LOADER *loader, + struct charset_info_st *cs, + MY_COLL_RULES *rules, + const MY_UCA_INFO *src_uca, + uint id) +{ + my_cs_encoding_t enc= my_uca1400_collation_id_to_charset_id(id); + uint tailoring= my_uca1400_collation_id_to_tailoring_id(id); + MY_UCA_INFO *dst_uca= &my_uca1400_info_tailored[enc][tailoring]; + DBUG_ASSERT(my_collation_id_is_uca1400(id)); + if (!dst_uca->level[0].weights/*Check if already initialized*/ && + (my_uca_info_init(loader, dst_uca, rules, cs, src_uca, + (1<number)) + if (my_collation_id_is_mysql_uca0900(cs->number)) { - /* - UCA1400 collations with equal character set and tailoring - (but with different level flags) share the same MY_UCA_INFO. - */ - my_cs_encoding_t enc= my_uca1400_collation_id_to_charset_id(cs->number); - uint tailoring= my_uca1400_collation_id_to_tailoring_id(cs->number); - MY_UCA_INFO *dst_uca= &my_uca1400_info_tailored[enc][tailoring]; - if (!dst_uca->level[0].weights && - (rc= my_uca_info_init(loader, dst_uca, &rules, cs, src_uca, - (1<uca= dst_uca; + uint id1400= mysql_0900_mapping[cs->number - mysql_0900_collation_start]. + collation_id; + if (!(cs->uca= my_uca1400_collation_get_initialized_shared_uca(loader, cs, + &rules, + src_uca, + id1400))) + goto ex; + } + else if (my_collation_id_is_uca1400(cs->number)) + { + if (!(cs->uca= my_uca1400_collation_get_initialized_shared_uca(loader, cs, + &rules, + src_uca, + cs->number))) + goto ex; } else { @@ -39397,122 +39347,6 @@ struct charset_info_st my_charset_utf16_unicode_520_nopad_ci= #endif /* HAVE_CHARSET_utf16 */ -uint -my_uca1400_make_builtin_collation_id(my_cs_encoding_t charset_id, - uint tailoring_id, - my_bool nopad, - my_bool secondary_level, - my_bool tertiary_level) -{ - if (!my_uca1400_collation_definitions[tailoring_id].tailoring) - return 0; - return MY_UCA1400_COLLATION_ID_POSSIBLE_MIN + - (charset_id << 8) + - (tailoring_id << 3) + - (nopad << 2) + - (secondary_level << 1) + - (tertiary_level << 0); -} - - -my_bool -my_uca1400_collation_definition_init(MY_CHARSET_LOADER *loader, - struct charset_info_st *dst, - uint id) -{ - my_cs_encoding_t cs_id= my_uca1400_collation_id_to_charset_id(id); - uint tailoring_id= my_uca1400_collation_id_to_tailoring_id(id); - my_bool nopad= my_uca1400_collation_id_to_nopad_flag(id); - my_bool secondary_level= my_uca1400_collation_id_to_secondary_level_flag(id); - my_bool tertiary_level= my_uca1400_collation_id_to_tertiary_level_flag(id); - const MY_UCA1400_COLLATION_DEFINITION *def= - &my_uca1400_collation_definitions[tailoring_id]; - char tmp[128], *coll_name; - size_t length; - - switch (cs_id) { - case MY_CS_ENCODING_UTF8MB3: - *dst= nopad ? my_charset_utf8mb3_unicode_520_nopad_ci : - my_charset_utf8mb3_unicode_520_ci; - break; - case MY_CS_ENCODING_UTF8MB4: - *dst= nopad ? my_charset_utf8mb4_unicode_520_nopad_ci : - my_charset_utf8mb4_unicode_520_ci; - break; -#ifdef HAVE_CHARSET_ucs2 - case MY_CS_ENCODING_UCS2: - *dst= nopad ? my_charset_ucs2_unicode_520_nopad_ci : - my_charset_ucs2_unicode_520_ci; - break; -#endif -#ifdef HAVE_CHARSET_utf16 - case MY_CS_ENCODING_UTF16: - *dst= nopad ? my_charset_utf16_unicode_520_nopad_ci : - my_charset_utf16_unicode_520_ci; - break; -#endif -#ifdef HAVE_CHARSET_utf32 - case MY_CS_ENCODING_UTF32: - *dst= nopad ? my_charset_utf32_unicode_520_nopad_ci : - my_charset_utf32_unicode_520_ci; - break; -#endif - } - - dst->number= id; - dst->uca= &my_uca_v1400; - dst->tailoring= def->tailoring; - if (def->tailoring == turkish) - dst->casefold= &my_casefold_unicode1400tr; - else - dst->casefold= &my_casefold_unicode1400; - if (nopad) - dst->state|= MY_CS_NOPAD; - my_ci_set_level_flags(dst, (1 << MY_CS_LEVEL_BIT_PRIMARY) | - (secondary_level ? - 1 << MY_CS_LEVEL_BIT_SECONDARY : 0) | - (tertiary_level ? - 1 << MY_CS_LEVEL_BIT_TERTIARY : 0)); - - length= my_snprintf(tmp, sizeof(tmp), "%.*s_uca1400%s%s%s%s%s", - (int) dst->cs_name.length, dst->cs_name.str, - def->name[0] ? "_" : "", - def->name, - nopad ? "_nopad" : "", - secondary_level ? "_as" : "_ai", - tertiary_level ? "_cs" : "_ci"); - if (!(coll_name= loader->once_alloc(length + 1))) - return TRUE; - strcpy(coll_name, tmp); - dst->coll_name.str= coll_name; - dst->coll_name.length= length; - return FALSE; -} - - -/* - Return UCA-4.0.0 compatible ID, e.g. for use in the protocol - with the old clients. -*/ -static uint my_uca1400_collation_id_uca400_compat(uint id) -{ - uint tlid= my_uca1400_collation_id_to_tailoring_id(id); - my_cs_encoding_t csid= my_uca1400_collation_id_to_charset_id(id); - MY_UCA1400_COLLATION_DEFINITION *def; - DBUG_ASSERT(my_collation_id_is_uca1400(id)); - if (!(def= &my_uca1400_collation_definitions[tlid])->name) - return id; - switch (csid) { - case MY_CS_ENCODING_UTF8MB3: return def->id_utf8mb3; - case MY_CS_ENCODING_UTF8MB4: return def->id_utf8mb4; - case MY_CS_ENCODING_UCS2: return def->id_ucs2; - case MY_CS_ENCODING_UTF16: return def->id_utf16; - case MY_CS_ENCODING_UTF32: return def->id_utf32; - } - return id; -} - - uint my_ci_get_id_uca(CHARSET_INFO *cs, my_collation_id_type_t type) { switch (type) @@ -39532,23 +39366,6 @@ uint my_ci_get_id_uca(CHARSET_INFO *cs, my_collation_id_type_t type) } -LEX_CSTRING my_ci_get_collation_name_uca1400_context(CHARSET_INFO *cs) -{ - LEX_CSTRING res; - DBUG_ASSERT(my_collation_id_is_uca1400(cs->number)); - - if (cs->coll_name.length <= cs->cs_name.length || - cs->coll_name.str[cs->cs_name.length] != '_') - { - DBUG_ASSERT(0); - return cs->coll_name; - } - res.str= cs->coll_name.str + cs->cs_name.length + 1; - res.length= cs->coll_name.length - cs->cs_name.length - 1; - return res; -} - - LEX_CSTRING my_ci_get_collation_name_uca(CHARSET_INFO *cs, my_collation_name_mode_t mode) { @@ -39565,137 +39382,4 @@ LEX_CSTRING my_ci_get_collation_name_uca(CHARSET_INFO *cs, return cs->coll_name; } - -/* - Add support for MySQL 8.0 utf8mb4_0900_.. collations - - The collation id's where collected from fprintf() in add_alias_for_collation() -*/ - -#define mysql_0900_collation_start 255 - -struct mysql_0900_to_mariadb_1400_mapping -{ - const char *mysql_col_name, *mariadb_col_name, *case_sensitivity; - uint collation_id; -}; - -struct mysql_0900_to_mariadb_1400_mapping mysql_0900_mapping[]= -{ - /* 255 Ascent insensitive, Case insensitive 'ai_ci' */ - {"", "", "ai_ci", 2308}, - {"de_pb", "german2", "ai_ci", 2468}, - {"is", "icelandic", "ai_ci", 2316}, - {"lv", "latvian", "ai_ci", 2324}, - {"ro", "romanian", "ai_ci", 2332}, - {"sl", "slovenian", "ai_ci", 2340}, - {"pl", "polish", "ai_ci", 2348}, - {"et", "estonian", "ai_ci", 2356}, - {"es", "spanish", "ai_ci", 2364}, - {"sv", "swedish", "ai_ci", 2372}, - {"tr", "turkish", "ai_ci", 2380}, - {"cs", "czech", "ai_ci", 2388}, - {"da", "danish", "ai_ci", 2396}, - {"lt", "lithuanian", "ai_ci", 2404}, - {"sk", "slovak", "ai_ci", 2412}, - {"es_trad", "spanish2", "ai_ci", 2420}, - {"la", "roman", "ai_ci", 2428}, - {"fa", NullS, "ai_ci", 0}, // Disabled in MySQL - {"eo", "esperanto", "ai_ci", 2444}, - {"hu", "hungarian", "ai_ci", 2452}, - {"hr", "croatian", "ai_ci", 2500}, - {"si", NullS, "ai_ci", 0}, // Disabled in MySQL - {"vi", "vietnamese", "ai_ci", 2492}, - - /* 278 Ascent sensitive, Case sensitive 'as_cs' */ - {"","", "as_cs", 2311}, - {"de_pb", "german2", "as_cs", 2471}, - {"is", "icelandic", "as_cs", 2319}, - {"lv", "latvian", "as_cs", 2327}, - {"ro", "romanian", "as_cs", 2335}, - {"sl", "slovenian", "as_cs", 2343}, - {"pl", "polish", "as_cs", 2351}, - {"et", "estonian", "as_cs", 2359}, - {"es", "spanish", "as_cs", 2367}, - {"sv", "swedish", "as_cs", 2375}, - {"tr", "turkish", "as_cs", 2383}, - {"cs", "czech", "as_cs", 2391}, - {"da", "danish", "as_cs", 2399}, - {"lt", "lithuanian", "as_cs", 2407}, - {"sk", "slovak", "as_cs", 2415}, - {"es_trad", "spanish2", "as_cs", 2423}, - {"la", "roman", "as_cs", 2431}, - {"fa", NullS, "as_cs", 0}, // Disabled in MySQL - {"eo", "esperanto", "as_cs", 2447}, - {"hu", "hungarian", "as_cs", 2455}, - {"hr", "croatian", "as_cs", 2503}, - {"si", NullS, "as_cs", 0}, // Disabled in MySQL - {"vi", "vietnamese", "as_cs", 2495}, - - {"", NullS, "as_cs", 0}, // Missing - {"", NullS, "as_cs", 0}, // Missing - {"_ja_0900_as_cs", NullS, "as_cs", 0}, // Not supported - {"_ja_0900_as_cs_ks", NullS, "as_cs", 0}, // Not supported - - /* 305 Ascent-sensitive, Case insensitive 'as_ci' */ - {"","", "as_ci", 2310}, - {"ru", NullS, "ai_ci", 0}, // Not supported - {"ru", NullS, "as_cs", 0}, // Not supported - {"zh", NullS, "as_cs", 0}, // Not supported - {NullS, NullS, "", 0} -}; - - -static LEX_CSTRING - mysql_utf8mb4_0900_bin= {STRING_WITH_LEN("utf8mb4_0900_bin")}, - mariadb_utf8mb4_nopad_bin= {STRING_WITH_LEN("utf8mb4_nopad_bin")}; - -/* - Map mysql character sets to MariaDB using the same definition but with - with the MySQL collation name and id. -*/ - -my_bool mysql_utf8mb4_0900_collation_definitions_add() -{ - uint id= mysql_0900_collation_start; - struct mysql_0900_to_mariadb_1400_mapping *map; - - for (map= mysql_0900_mapping; map->mysql_col_name ; map++, id++) - { - if (map->mariadb_col_name) /* Supported collation */ - { - size_t org_length, ali_length; - char original[64], alias[64]; - LEX_CSTRING org_name, alias_name; - - org_length= (strxnmov(original, sizeof(original)-1, - "utf8mb4_uca1400_", - map->mariadb_col_name, - (map->mariadb_col_name[0] ? "_" : ""), - "nopad_", - map->case_sensitivity, - NullS) - original); - ali_length= (strxnmov(alias, sizeof(alias)-1, - "utf8mb4_", map->mysql_col_name, - (map->mysql_col_name[0] ? "_" : ""), - "0900_", - map->case_sensitivity, - NullS) - alias); - org_name.str= original; - org_name.length= org_length; - alias_name.str= alias; - alias_name.length= ali_length; - - if (add_alias_for_collation(&org_name, map->collation_id, &alias_name, - id)) - return 1; - } - } - - if (add_alias_for_collation(&mariadb_utf8mb4_nopad_bin, 1070, - &mysql_utf8mb4_0900_bin, 309)) - return 1; - return 0; -} - #endif /* HAVE_UCA_COLLATIONS */ diff --git a/strings/ctype-uca.h b/strings/ctype-uca.h index dd84f92faa6..8b6e3f71ef0 100644 --- a/strings/ctype-uca.h +++ b/strings/ctype-uca.h @@ -118,9 +118,18 @@ typedef enum my_cs_encoding_enum #define MY_CS_ENCODING_LAST MY_CS_ENCODING_UTF32 -#include "ctype-uca1400.h" +typedef struct uca_collation_def_param +{ + my_cs_encoding_t cs_id; + uint tailoring_id; + uint nopad_flags; + uint level_flags; +} uca_collation_def_param_t; +#include "ctype-uca1400.h" +#include "ctype-uca0900.h" + static inline MY_UCA_IMPLICIT_WEIGHT my_uca_implicit_weight_primary(uint version, my_wc_t code) { diff --git a/strings/ctype-uca.inl b/strings/ctype-uca.inl index ceef733a1e6..6be44e63e03 100644 --- a/strings/ctype-uca.inl +++ b/strings/ctype-uca.inl @@ -963,7 +963,8 @@ MY_COLLATION_HANDLER MY_FUNCTION_NAME(collation_handler)= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_uca, - my_ci_get_collation_name_uca + my_ci_get_collation_name_uca, + my_ci_eq_collation_uca }; @@ -989,7 +990,8 @@ MY_COLLATION_HANDLER MY_FUNCTION_NAME(collation_handler_nopad)= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_uca, - my_ci_get_collation_name_uca + my_ci_get_collation_name_uca, + my_ci_eq_collation_uca }; @@ -1013,7 +1015,8 @@ MY_COLLATION_HANDLER MY_FUNCTION_NAME(collation_handler_multilevel)= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_uca, - my_ci_get_collation_name_uca + my_ci_get_collation_name_uca, + my_ci_eq_collation_uca }; @@ -1037,7 +1040,8 @@ MY_COLLATION_HANDLER MY_FUNCTION_NAME(collation_handler_nopad_multilevel)= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_uca, - my_ci_get_collation_name_uca + my_ci_get_collation_name_uca, + my_ci_eq_collation_uca }; diff --git a/strings/ctype-uca0520.h b/strings/ctype-uca0520.h new file mode 100644 index 00000000000..302d776256a --- /dev/null +++ b/strings/ctype-uca0520.h @@ -0,0 +1,69 @@ +#ifndef CTYPE_UCA_0520_H +#define CTYPE_UCA_0520_H +/* Copyright (c) 2025, MariaDB Corporation + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; version 2 + of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + MA 02110-1335 USA */ + + +extern struct charset_info_st my_charset_utf8mb3_unicode_520_nopad_ci; +extern struct charset_info_st my_charset_utf8mb3_unicode_520_ci; +extern struct charset_info_st my_charset_utf8mb4_unicode_520_nopad_ci; +extern struct charset_info_st my_charset_utf8mb4_unicode_520_ci; +extern struct charset_info_st my_charset_ucs2_unicode_520_nopad_ci; +extern struct charset_info_st my_charset_ucs2_unicode_520_ci; +extern struct charset_info_st my_charset_utf16_unicode_520_nopad_ci; +extern struct charset_info_st my_charset_utf16_unicode_520_ci; +extern struct charset_info_st my_charset_utf32_unicode_520_nopad_ci; +extern struct charset_info_st my_charset_utf32_unicode_520_ci; +extern struct charset_info_st my_charset_utf8mb4_turkish_uca_ci; + + +/* + Get a UCA-5.2.0 CHARSET_INFO using its character set ID and PAD flags. + Used to initialize UCA-14.0.0 collations. +*/ +static inline +CHARSET_INFO *my_uca0520_builtin_collation_by_id(my_cs_encoding_t cs_id, + uint nopad_flags) +{ + switch (cs_id) { + case MY_CS_ENCODING_UTF8MB3: + return nopad_flags ? &my_charset_utf8mb3_unicode_520_nopad_ci : + &my_charset_utf8mb3_unicode_520_ci; + case MY_CS_ENCODING_UTF8MB4: + return nopad_flags ? &my_charset_utf8mb4_unicode_520_nopad_ci : + &my_charset_utf8mb4_unicode_520_ci; +#ifdef HAVE_CHARSET_ucs2 + case MY_CS_ENCODING_UCS2: + return nopad_flags ? &my_charset_ucs2_unicode_520_nopad_ci : + &my_charset_ucs2_unicode_520_ci; +#endif +#ifdef HAVE_CHARSET_utf16 + case MY_CS_ENCODING_UTF16: + return nopad_flags ? &my_charset_utf16_unicode_520_nopad_ci : + &my_charset_utf16_unicode_520_ci; +#endif +#ifdef HAVE_CHARSET_utf32 + case MY_CS_ENCODING_UTF32: + return nopad_flags ? &my_charset_utf32_unicode_520_nopad_ci : + &my_charset_utf32_unicode_520_ci; +#endif + } + return NULL; +} + + +#endif /* CTYPE_UCA_0520_H */ diff --git a/strings/ctype-uca0900.c b/strings/ctype-uca0900.c new file mode 100644 index 00000000000..e792349c4c0 --- /dev/null +++ b/strings/ctype-uca0900.c @@ -0,0 +1,221 @@ +/* Copyright (c) 2025, MariaDB Corporation + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; version 2 + of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + MA 02110-1335 USA */ + +#include "my_global.h" +#include "strings_def.h" +#include "ctype-uca.h" + +struct mysql_0900_to_mariadb_1400_mapping + mysql_0900_mapping[mysql_0900_collation_num]= +{ + /* 255 Ascent insensitive, Case insensitive 'ai_ci' */ + {"", "", "ai_ci", 2308}, + {"de_pb", "german2", "ai_ci", 2468}, + {"is", "icelandic", "ai_ci", 2316}, + {"lv", "latvian", "ai_ci", 2324}, + {"ro", "romanian", "ai_ci", 2332}, + {"sl", "slovenian", "ai_ci", 2340}, + {"pl", "polish", "ai_ci", 2348}, + {"et", "estonian", "ai_ci", 2356}, + {"es", "spanish", "ai_ci", 2364}, + {"sv", "swedish", "ai_ci", 2372}, + {"tr", "turkish", "ai_ci", 2380}, + {"cs", "czech", "ai_ci", 2388}, + {"da", "danish", "ai_ci", 2396}, + {"lt", "lithuanian", "ai_ci", 2404}, + {"sk", "slovak", "ai_ci", 2412}, + {"es_trad", "spanish2", "ai_ci", 2420}, + {"la", "roman", "ai_ci", 2428}, + {"fa", NullS, "ai_ci", 0}, // Disabled in MySQL + {"eo", "esperanto", "ai_ci", 2444}, + {"hu", "hungarian", "ai_ci", 2452}, + {"hr", "croatian", "ai_ci", 2500}, + {"si", NullS, "ai_ci", 0}, // Disabled in MySQL + {"vi", "vietnamese", "ai_ci", 2492}, + + /* 278 Ascent sensitive, Case sensitive 'as_cs' */ + {"","", "as_cs", 2311}, + {"de_pb", "german2", "as_cs", 2471}, + {"is", "icelandic", "as_cs", 2319}, + {"lv", "latvian", "as_cs", 2327}, + {"ro", "romanian", "as_cs", 2335}, + {"sl", "slovenian", "as_cs", 2343}, + {"pl", "polish", "as_cs", 2351}, + {"et", "estonian", "as_cs", 2359}, + {"es", "spanish", "as_cs", 2367}, + {"sv", "swedish", "as_cs", 2375}, + {"tr", "turkish", "as_cs", 2383}, + {"cs", "czech", "as_cs", 2391}, + {"da", "danish", "as_cs", 2399}, + {"lt", "lithuanian", "as_cs", 2407}, + {"sk", "slovak", "as_cs", 2415}, + {"es_trad", "spanish2", "as_cs", 2423}, + {"la", "roman", "as_cs", 2431}, + {"fa", NullS, "as_cs", 0}, // Disabled in MySQL + {"eo", "esperanto", "as_cs", 2447}, + {"hu", "hungarian", "as_cs", 2455}, + {"hr", "croatian", "as_cs", 2503}, + {"si", NullS, "as_cs", 0}, // Disabled in MySQL + {"vi", "vietnamese", "as_cs", 2495}, + + {"", NullS, "as_cs", 0}, // Missing + {"", NullS, "as_cs", 0}, // Missing + {"_ja_0900_as_cs", NullS, "as_cs", 0}, // Not supported + {"_ja_0900_as_cs_ks", NullS, "as_cs", 0}, // Not supported + + /* 305 Ascent-sensitive, Case insensitive 'as_ci' */ + {"","", "as_ci", 2310}, + {"ru", NullS, "ai_ci", 0}, // Not supported + {"ru", NullS, "as_cs", 0}, // Not supported + {"zh", NullS, "as_cs", 0}, // Not supported + {NullS, NullS, "", 0} +}; + + +static LEX_CSTRING +my_uca0900_collation_build_name(char *buffer, size_t buffer_size, + const char *cs_name, + const char *tailoring_name, + const char *sensitivity_suffix) +{ + LEX_CSTRING res; + DBUG_ASSERT(buffer_size > 1); + res.str= buffer; + res.length= (strxnmov(buffer, buffer_size - 1, + cs_name, "_", tailoring_name, + (tailoring_name[0] ? "_" : ""), + "0900_", + sensitivity_suffix, + NullS) - buffer); + return res; +} + + +static LEX_CSTRING +my_ci_make_comment_for_alias(char *buffer, size_t buffer_size, + const char *srcname) +{ + LEX_CSTRING res= {buffer, 0}; + DBUG_ASSERT(buffer_size > 0); + res.length= strxnmov(buffer, buffer_size - 1, "Alias for ", srcname, NullS) - + buffer; + return res; +} + + +/* + Add a MySQL UCA-0900 collation as an alias for a MariaDB UCA-1400 collation. +*/ +static my_bool +mysql_uca0900_collation_definition_add(MY_CHARSET_LOADER *loader, + const struct + mysql_0900_to_mariadb_1400_mapping *map, + uint alias_id) +{ + char comment_buffer[MY_CS_COLLATION_NAME_SIZE + 15]; + char alias_buffer[MY_CS_COLLATION_NAME_SIZE + 1]; + char name1400_buffer[MY_CS_COLLATION_NAME_SIZE + 1]; + LEX_CSTRING comment= {comment_buffer, 0}; + LEX_CSTRING alias_name= {alias_buffer, 0}; + LEX_CSTRING name1400= {name1400_buffer, 0}; + LEX_CSTRING utf8mb4= {STRING_WITH_LEN("utf8mb4")}; + uint id1400= map->collation_id; + uca_collation_def_param_t param= my_uca1400_collation_param_by_id(id1400); + const MY_UCA1400_COLLATION_DEFINITION *def1400= + &my_uca1400_collation_definitions[param.tailoring_id]; + + DBUG_ASSERT(my_collation_id_is_mysql_uca0900(alias_id)); + + alias_name= my_uca0900_collation_build_name(alias_buffer, + sizeof(alias_buffer), + "utf8mb4", + map->mysql_col_name, + map->case_sensitivity); + + name1400= my_uca1400_collation_build_name(name1400_buffer, + sizeof(name1400_buffer), + &utf8mb4, def1400->name, ¶m); + comment= my_ci_make_comment_for_alias(comment_buffer, sizeof(comment_buffer), + name1400.str); + +#ifdef DEBUG_PRINT_ALIAS + fprintf(stderr, "alias[%u] %-26s -> [%u] %s\n", + id, alias_name.str, id1400, name1400.str); +#endif + + return my_uca1400_collation_alloc_and_init(loader, alias_name, + comment, ¶m, alias_id); +} + + +/* + Add support for MySQL 8.0 utf8mb4_0900_.. UCA collations. + + The collation id's were collected from fprintf() + in mysql_uca0900_collation_definition_add(). + + Map mysql character sets to MariaDB using the same definition but + with the MySQL collation name and id. +*/ + +my_bool +mysql_uca0900_utf8mb4_collation_definitions_add(MY_CHARSET_LOADER *loader) +{ + uint alias_id= mysql_0900_collation_start; + struct mysql_0900_to_mariadb_1400_mapping *map; + + for (map= mysql_0900_mapping; map->mysql_col_name ; map++, alias_id++) + { + if (map->mariadb_col_name) /* Supported collation */ + { + if (mysql_uca0900_collation_definition_add(loader, map, alias_id)) + return TRUE; + } + } + + return FALSE; +} + + +/* + Add MySQL utf8mb4_0900_bin collation as + an alias for MariaDB utf8mb4_nopad_bin. +*/ +my_bool mysql_utf8mb4_0900_bin_add(MY_CHARSET_LOADER *loader) +{ + CHARSET_INFO *src= &my_charset_utf8mb4_nopad_bin; + LEX_CSTRING alias_name= {STRING_WITH_LEN("utf8mb4_0900_bin")}; + uint alias_id= 309; + char comment_buffer[MY_CS_COLLATION_NAME_SIZE+15]; + LEX_CSTRING comment= my_ci_make_comment_for_alias(comment_buffer, + sizeof(comment_buffer), + src->coll_name.str); + struct charset_info_st *dst= my_ci_alloc(loader, alias_name, &alias_name, + comment, &comment); + if (!dst) + return TRUE; + + *dst= *src; + + dst->number= alias_id; + dst->coll_name= alias_name; + dst->comment= comment.str; + + (loader->add_collation)(dst); + + return FALSE; +} diff --git a/strings/ctype-uca0900.h b/strings/ctype-uca0900.h new file mode 100644 index 00000000000..c1a533077b9 --- /dev/null +++ b/strings/ctype-uca0900.h @@ -0,0 +1,47 @@ +#ifndef CTYPE_UCA_0900_H +#define CTYPE_UCA_0900_H +/* Copyright (c) 2025, MariaDB Corporation + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; version 2 + of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + MA 02110-1335 USA */ + + +#define mysql_0900_collation_start 255 +#define mysql_0900_collation_end 308 +#define mysql_0900_collation_num \ + (mysql_0900_collation_end - mysql_0900_collation_start + 1 + 1/*End marker*/) + +struct mysql_0900_to_mariadb_1400_mapping +{ + const char *mysql_col_name, *mariadb_col_name, *case_sensitivity; + uint collation_id; +}; + +extern struct mysql_0900_to_mariadb_1400_mapping + mysql_0900_mapping[mysql_0900_collation_num]; + + +static inline +my_bool my_collation_id_is_mysql_uca0900(uint id) +{ + return id >= mysql_0900_collation_start && + id <= mysql_0900_collation_end; +} + +my_bool mysql_uca0900_utf8mb4_collation_definitions_add(MY_CHARSET_LOADER *ld); + +my_bool mysql_utf8mb4_0900_bin_add(MY_CHARSET_LOADER *loader); + +#endif /* CTYPE_UCA_0900_H */ diff --git a/strings/ctype-uca1400.c b/strings/ctype-uca1400.c new file mode 100644 index 00000000000..986ec669730 --- /dev/null +++ b/strings/ctype-uca1400.c @@ -0,0 +1,363 @@ +/* Copyright (c) 2025, MariaDB Corporation + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; version 2 + of the License. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not, write to the Free + Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + MA 02110-1335 USA */ + +#include "strings_def.h" +#include "m_ctype.h" +#include "ctype-uca.h" +#include "ctype-uca0520.h" +#include "ctype-unidata.h" +#include "ctype-uca1400data.h" + +/* + Return UCA-4.0.0 compatible ID (known since MySQL-4.1), + e.g. for use in the protocol with the old clients. +*/ +uint my_uca1400_collation_id_uca400_compat(uint id) +{ + uint tlid= my_uca1400_collation_id_to_tailoring_id(id); + my_cs_encoding_t csid= my_uca1400_collation_id_to_charset_id(id); + const MY_UCA1400_COLLATION_DEFINITION *def; + DBUG_ASSERT(my_collation_id_is_uca1400(id)); + if (!(def= &my_uca1400_collation_definitions[tlid])->name) + return id; + switch (csid) { + case MY_CS_ENCODING_UTF8MB3: return def->id_utf8mb3; + case MY_CS_ENCODING_UTF8MB4: return def->id_utf8mb4; + case MY_CS_ENCODING_UCS2: return def->id_ucs2; + case MY_CS_ENCODING_UTF16: return def->id_utf16; + case MY_CS_ENCODING_UTF32: return def->id_utf32; + } + return id; +} + + +/* + Get a short (without the character set prefix) collation name + of a UCA-14.0.0 collation, e.g. + utf8mb4_uca1400_swedish_ai_ci -> uca1400_swedish_ai_ci +*/ +LEX_CSTRING my_ci_get_collation_name_uca1400_context(CHARSET_INFO *cs) +{ + LEX_CSTRING res; + DBUG_ASSERT(my_collation_id_is_uca1400(cs->number)); + + if (cs->coll_name.length <= cs->cs_name.length || + cs->coll_name.str[cs->cs_name.length] != '_') + { + DBUG_ASSERT(0); + return cs->coll_name; /* Something went wrong, return the full name. */ + } + res.str= cs->coll_name.str + cs->cs_name.length + 1; + res.length= cs->coll_name.length - cs->cs_name.length - 1; + return res; +} + + +/* + A preliminary initialized data for a UCA-14.0.0 collation. + The goal is to have the "logical position" members initialized (see below). + Weight tables are initialized later, at create_tailoring() time. +*/ +MY_UCA_INFO my_uca_v1400= +{ + { + { + 0x10FFFF, /* maxchar */ + (uchar *) uca1400_length, + (uint16 **) uca1400_weight, + { /* Contractions: */ + array_elements(uca1400_contractions), /* nitems */ + uca1400_contractions, /* item */ + NULL /* flags */ + }, + 0, /* levelno */ + {0}, /* contraction_hash */ + NULL /* booster */ + }, + + { + 0x10FFFF, /* maxchar */ + (uchar *) uca1400_length_secondary, + (uint16 **) uca1400_weight_secondary, + { /* Contractions: */ + array_elements(uca1400_contractions_secondary), /* nitems */ + uca1400_contractions_secondary, /* item */ + NULL /* flags */ + }, + 1, /* levelno */ + {0}, /* contraction_hash */ + NULL /* booster */ + }, + + { + 0x10FFFF, /* maxchar */ + (uchar *) uca1400_length_tertiary, + (uint16 **) uca1400_weight_tertiary, + { /* Contractions: */ + array_elements(uca1400_contractions_tertiary), /* nitems */ + uca1400_contractions_tertiary, /* item */ + NULL /* flags */ + }, + 2, /* levelno */ + {0}, /* contraction_hash */ + NULL /* booster */ + } + + }, + + /* Logical positions */ + uca1400_non_ignorable_first, + uca1400_non_ignorable_last, + + uca1400_primary_ignorable_first, + uca1400_primary_ignorable_last, + + uca1400_secondary_ignorable_first, + uca1400_secondary_ignorable_last, + + uca1400_tertiary_ignorable_first, + uca1400_tertiary_ignorable_last, + + 0x0000, /* first_trailing */ + 0x0000, /* last_trailing */ + + uca1400_variable_first, + uca1400_variable_last, + + /* Misc */ + uca1400_version +}; + + +/* + An array of MY_UCA_INFO (sorting tables). + Collations having the same character set and tailoring + (but different pad and accent/case sensitivity flags) + share the same array element. Also, aliases for MySQL-8.0 + UCA-9.0.0 collations share the same array element with the + corresponding UCA-14.0.0 MariaDB collations. + + For example, all these collation share one element of the array: + - utf8mb4_uca1400_swedish_ai_ci + - utf8mb4_uca1400_swedish_ai_cs + - utf8mb4_uca1400_swedish_as_ci + - utf8mb4_uca1400_swedish_as_cs + - utf8mb4_uca1400_swedish_nopad_ai_ci + - utf8mb4_uca1400_swedish_nopad_ai_cs + - utf8mb4_uca1400_swedish_nopad_as_ci + - utf8mb4_uca1400_swedish_nopad_as_cs + - utf8mb4_sv_0900_ai_ci + - utf8mb4_sv_0900_as_cs +*/ +MY_UCA_INFO +my_uca1400_info_tailored[MY_CS_ENCODING_LAST+1] + [MY_UCA1400_COLLATION_DEFINITION_COUNT]; + + +/* + Make an UCA-14.0.0 collation ID using its properties. +*/ +uint my_uca1400_make_builtin_collation_id(my_cs_encoding_t charset_id, + uint tailoring_id, + my_bool nopad, + my_bool secondary_level, + my_bool tertiary_level) +{ + if (!my_uca1400_collation_definitions[tailoring_id].tailoring) + return 0; + return MY_UCA1400_COLLATION_ID_POSSIBLE_MIN + + (charset_id << 8) + + (tailoring_id << 3) + + (nopad << 2) + + (secondary_level << 1) + + (tertiary_level << 0); +} + + +/* + Make an UCA-14.0.0 full collation name as a concatenation of its + - Character set name + - UCA version + - Language rules (tailoring) + - pad characteristics + - accent sensitivity + - case sensitivity + e.g.: "utf8mb4" + "_uca1400" + "_swedish" + "_as" + "_ci" +*/ +LEX_CSTRING +my_uca1400_collation_build_name(char *buffer, size_t buffer_size, + const LEX_CSTRING *cs_name, + const char *tailoring_name, + const uca_collation_def_param_t *prm) +{ + LEX_CSTRING res; + res.str= buffer; + res.length= + my_snprintf(buffer, buffer_size, "%.*s_uca1400%s%s%s%s%s", + (int) cs_name->length, cs_name->str, + tailoring_name[0] ? "_" : "", + tailoring_name, + prm->nopad_flags ? "_nopad" : "", + prm->level_flags & (1<level_flags & (1<tailoring_id]; + + /* Copy the entire charset_info_st from an in-compiled one. */ + *dst= *my_uca0520_builtin_collation_by_id(param->cs_id, param->nopad_flags); + + /* Now replace some members according to param */ + DBUG_ASSERT((dst->state & uca1400_unexpected_flags()) == 0); + dst->uca= &my_uca_v1400; + dst->tailoring= def->tailoring; + if (def->tailoring == my_charset_utf8mb4_turkish_uca_ci.tailoring) + dst->casefold= &my_casefold_unicode1400tr; + else + dst->casefold= &my_casefold_unicode1400; + + dst->state|= param->nopad_flags; + my_ci_set_level_flags(dst, param->level_flags); +} + + +/* + Allocate memory for a new charset_info_st instance together + with its name and comment. + Perform preliminary initialization, then add to the list + of available collations using MY_CHARSET_LOADER::add_collation. +*/ +my_bool +my_uca1400_collation_alloc_and_init(MY_CHARSET_LOADER *loader, + LEX_CSTRING name, + LEX_CSTRING comment, + const uca_collation_def_param_t *param, + uint id) +{ + struct charset_info_st *dst; + + if (!(dst= my_ci_alloc(loader, name, &name, comment, &comment))) + return TRUE; + + my_uca1400_collation_definition_init(loader, dst, param); + + dst->number= id; + dst->coll_name= name; + dst->comment= comment.str; + + return (loader->add_collation)(dst) != 0; +} + + +/* + Make an UCA-14.0.0 full collation name using its id, + then allocate and add the collation. +*/ +static +my_bool my_uca1400_collation_definition_add(MY_CHARSET_LOADER *loader, uint id) +{ + char coll_name_buffer[MY_CS_COLLATION_NAME_SIZE + 1]; + LEX_CSTRING coll_name; + LEX_CSTRING comment= {"",0}; + uca_collation_def_param_t param= my_uca1400_collation_param_by_id(id); + CHARSET_INFO *src= my_uca0520_builtin_collation_by_id(param.cs_id, + param.nopad_flags); + const MY_UCA1400_COLLATION_DEFINITION *def= + &my_uca1400_collation_definitions[param.tailoring_id]; + + coll_name= my_uca1400_collation_build_name(coll_name_buffer, + sizeof(coll_name_buffer), + &src->cs_name, + def->name, + ¶m); + + return my_uca1400_collation_alloc_and_init(loader, coll_name, comment, + ¶m, id); +} + + +/* + Add UCA-14.0.0 collations for all combinations of: + - Unicode character sets (utf8mb3, utf8mb4, ucs2, utf16, utf32) + - language rules (tailorings) + - pad properties + - accent sensitivity + - case sensitivity +*/ +my_bool my_uca1400_collation_definitions_add(MY_CHARSET_LOADER *loader) +{ + my_cs_encoding_t charset_id; + for (charset_id= (my_cs_encoding_t) 0; + charset_id <= (my_cs_encoding_t) MY_CS_ENCODING_LAST; + charset_id++) + { + uint tailoring_id; + for (tailoring_id= 0 ; + tailoring_id < MY_UCA1400_COLLATION_DEFINITION_COUNT; + tailoring_id++) + { + my_bool nopad; /* PAD / NOPAD */ + for (nopad= 0; nopad < 2; nopad++) + { + my_bool secondary_level; /* ai / as */ + for (secondary_level= 0; secondary_level < 2; secondary_level++) + { + my_bool tertiary_level; /* ci / cs */ + for (tertiary_level= 0; tertiary_level < 2; tertiary_level++) + { + uint id= my_uca1400_make_builtin_collation_id(charset_id, + tailoring_id, + nopad, + secondary_level, + tertiary_level); + if (id && my_uca1400_collation_definition_add(loader, id)) + return TRUE; + } + } + } + } + } + return FALSE; +} diff --git a/strings/ctype-uca1400.h b/strings/ctype-uca1400.h index 1d923a27939..5b60dcd629a 100644 --- a/strings/ctype-uca1400.h +++ b/strings/ctype-uca1400.h @@ -187,6 +187,19 @@ my_collation_id_is_uca1400(uint id) id <= MY_UCA1400_COLLATION_ID_POSSIBLE_MAX); } + +typedef struct my_uca1400_collation_definition_st +{ + const char * tailoring; + const char * name; + uint16 id_utf8mb3; + uint16 id_utf8mb4; + uint16 id_ucs2; + uint16 id_utf16; + uint16 id_utf32; +} MY_UCA1400_COLLATION_DEFINITION; + + /* UCA1400 collation ID: @@ -204,6 +217,7 @@ my_collation_id_is_uca1400(uint id) static inline my_cs_encoding_t my_uca1400_collation_id_to_charset_id(uint id) { + DBUG_ASSERT(id); return (my_cs_encoding_t) ((id >> 8) & 0x07); } @@ -211,6 +225,7 @@ my_uca1400_collation_id_to_charset_id(uint id) static inline uint my_uca1400_collation_id_to_tailoring_id(uint id) { + DBUG_ASSERT(id); return (id >> 3) & 0x1F; } @@ -218,21 +233,52 @@ my_uca1400_collation_id_to_tailoring_id(uint id) static inline my_bool my_uca1400_collation_id_to_nopad_flag(uint id) { + DBUG_ASSERT(id); return (my_bool) ((id >> 2) & 0x01); } static inline my_bool my_uca1400_collation_id_to_secondary_level_flag(uint id) { + DBUG_ASSERT(id); return (my_bool) ((id >> 1) & 0x01); } static inline my_bool my_uca1400_collation_id_to_tertiary_level_flag(uint id) { + DBUG_ASSERT(id); return (my_bool) ((id >> 0) & 0x01); } +static inline uint +my_uca1400_collation_id_to_level_flags(uint id) +{ + my_bool secondary_level, tertiary_level; + DBUG_ASSERT(id); + secondary_level= my_uca1400_collation_id_to_secondary_level_flag(id); + tertiary_level= my_uca1400_collation_id_to_tertiary_level_flag(id); + return (1 << MY_CS_LEVEL_BIT_PRIMARY) | + (secondary_level ? 1 << MY_CS_LEVEL_BIT_SECONDARY : 0) | + (tertiary_level ? 1 << MY_CS_LEVEL_BIT_TERTIARY : 0); +} + + +/* + Return an UCA-14.0.0 collation properties using its ID. +*/ +static inline uca_collation_def_param_t +my_uca1400_collation_param_by_id(uint id) +{ + uca_collation_def_param_t res; + DBUG_ASSERT(id); + res.cs_id= my_uca1400_collation_id_to_charset_id(id); + res.tailoring_id= my_uca1400_collation_id_to_tailoring_id(id); + res.nopad_flags= my_uca1400_collation_id_to_nopad_flag(id); + res.level_flags= my_uca1400_collation_id_to_level_flags(id); + return res; +} + uint my_uca1400_make_builtin_collation_id(my_cs_encoding_t charset_id, @@ -241,13 +287,36 @@ my_uca1400_make_builtin_collation_id(my_cs_encoding_t charset_id, my_bool secondary_level, my_bool tertiary_level); -my_bool -my_uca1400_collation_definition_init(MY_CHARSET_LOADER *loader, - struct charset_info_st *dst, - uint collation_id); +LEX_CSTRING +my_uca1400_collation_build_name(char *buffer, size_t buffer_size, + const LEX_CSTRING *cs_name, + const char *tailoring_name, + const uca_collation_def_param_t *prm); +my_bool +my_uca1400_collation_alloc_and_init(MY_CHARSET_LOADER *loader, + LEX_CSTRING name, + LEX_CSTRING comment, + const uca_collation_def_param_t *param, + uint id); + +LEX_CSTRING my_ci_get_collation_name_uca1400_context(CHARSET_INFO *cs); + +uint my_uca1400_collation_id_uca400_compat(uint id); + +my_bool my_uca1400_collation_definitions_add(MY_CHARSET_LOADER *loader); + + +/* Exported data */ #define MY_UCA1400_COLLATION_DEFINITION_COUNT 26 -my_bool mysql_utf8mb4_0900_collation_definitions_add(); +extern MY_UCA1400_COLLATION_DEFINITION +my_uca1400_collation_definitions[MY_UCA1400_COLLATION_DEFINITION_COUNT]; + +extern MY_UCA_INFO my_uca_v1400; + + +extern MY_UCA_INFO my_uca1400_info_tailored[MY_CS_ENCODING_LAST+1] + [MY_UCA1400_COLLATION_DEFINITION_COUNT]; #endif /* CTYPE_UCA_1400_H */ diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c index beebbac1ada..b234bb3c357 100644 --- a/strings/ctype-ucs2.c +++ b/strings/ctype-ucs2.c @@ -1505,7 +1505,8 @@ static MY_COLLATION_HANDLER my_collation_utf16_general_ci_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1526,7 +1527,8 @@ static MY_COLLATION_HANDLER my_collation_utf16_bin_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1547,7 +1549,8 @@ static MY_COLLATION_HANDLER my_collation_utf16_general_nopad_ci_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1568,7 +1571,8 @@ static MY_COLLATION_HANDLER my_collation_utf16_nopad_bin_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1858,7 +1862,8 @@ static MY_COLLATION_HANDLER my_collation_utf16le_general_ci_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1879,7 +1884,8 @@ static MY_COLLATION_HANDLER my_collation_utf16le_bin_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1900,7 +1906,8 @@ static MY_COLLATION_HANDLER my_collation_utf16le_general_nopad_ci_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1921,7 +1928,8 @@ static MY_COLLATION_HANDLER my_collation_utf16le_nopad_bin_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -2663,7 +2671,8 @@ static MY_COLLATION_HANDLER my_collation_utf32_general_ci_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -2684,7 +2693,8 @@ static MY_COLLATION_HANDLER my_collation_utf32_bin_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -2705,7 +2715,8 @@ static MY_COLLATION_HANDLER my_collation_utf32_general_nopad_ci_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -2726,7 +2737,8 @@ static MY_COLLATION_HANDLER my_collation_utf32_nopad_bin_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -3263,7 +3275,8 @@ static MY_COLLATION_HANDLER my_collation_ucs2_general_ci_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -3284,7 +3297,8 @@ static MY_COLLATION_HANDLER my_collation_ucs2_general_mysql500_ci_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -3305,7 +3319,8 @@ static MY_COLLATION_HANDLER my_collation_ucs2_bin_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -3326,7 +3341,8 @@ static MY_COLLATION_HANDLER my_collation_ucs2_general_nopad_ci_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -3347,7 +3363,8 @@ static MY_COLLATION_HANDLER my_collation_ucs2_nopad_bin_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-ujis.c b/strings/ctype-ujis.c index 28bd11c1531..e6d23cb4f60 100644 --- a/strings/ctype-ujis.c +++ b/strings/ctype-ujis.c @@ -67261,7 +67261,8 @@ static MY_COLLATION_HANDLER my_collation_ujis_japanese_ci_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -67282,7 +67283,8 @@ static MY_COLLATION_HANDLER my_collation_ujis_bin_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -67303,7 +67305,8 @@ static MY_COLLATION_HANDLER my_collation_ujis_japanese_nopad_ci_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -67324,7 +67327,8 @@ static MY_COLLATION_HANDLER my_collation_ujis_nopad_bin_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c index be27a419697..2d271bba6ba 100644 --- a/strings/ctype-utf8.c +++ b/strings/ctype-utf8.c @@ -1119,7 +1119,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_general_ci_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1140,7 +1141,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_general_mysql500_ci_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1161,7 +1163,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_bin_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1182,7 +1185,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_general_nopad_ci_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1203,7 +1207,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_nopad_bin_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -1529,7 +1534,8 @@ static MY_COLLATION_HANDLER my_collation_cs_handler = my_hash_sort_utf8mb3, my_propagate_simple, my_min_str_mb_simple, - my_max_str_mb_simple + my_max_str_mb_simple, + my_ci_eq_collation_generic }; struct charset_info_st my_charset_utf8mb3_general_cs= @@ -2848,7 +2854,8 @@ static MY_COLLATION_HANDLER my_collation_filename_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; static MY_CHARSET_HANDLER my_charset_filename_handler= @@ -3393,6 +3400,19 @@ my_charlen_utf8mb4(CHARSET_INFO *cs __attribute__((unused)), } +static my_bool +my_ci_eq_collation_utf8mb4_bin(CHARSET_INFO *a, CHARSET_INFO *b) +{ + return a->cset == b->cset && + a->coll == b->coll && + a->uca == b->uca && a->uca == NULL && + a->casefold == b->casefold && + (a->state & MY_CS_NOPAD) == (b->state & MY_CS_NOPAD) && + a->levels_for_order == b->levels_for_order && + a->tailoring == b->tailoring && a->tailoring == NULL; +} + + #define MY_FUNCTION_NAME(x) my_ ## x ## _utf8mb4 #define CHARLEN(cs,str,end) my_charlen_utf8mb4(cs,str,end) #define DEFINE_WELL_FORMED_CHAR_LENGTH_USING_CHARLEN @@ -3475,7 +3495,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_general_ci_handler= my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -3496,7 +3517,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_bin_handler = my_min_str_mb_simple, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_utf8mb4_bin }; @@ -3517,7 +3539,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_general_nopad_ci_handler= my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; @@ -3538,7 +3561,8 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_nopad_bin_handler = my_min_str_mb_simple_nopad, my_max_str_mb_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_utf8mb4_bin }; diff --git a/strings/ctype-win1250ch.c b/strings/ctype-win1250ch.c index a908bdd2b95..d68fdd5410c 100644 --- a/strings/ctype-win1250ch.c +++ b/strings/ctype-win1250ch.c @@ -690,7 +690,8 @@ static MY_COLLATION_HANDLER my_collation_czech_cs_handler = my_min_str_8bit_simple, my_max_str_8bit_simple, my_ci_get_id_generic, - my_ci_get_collation_name_generic + my_ci_get_collation_name_generic, + my_ci_eq_collation_generic }; diff --git a/strings/ctype.c b/strings/ctype.c index 81b757310f8..629514e5e9c 100644 --- a/strings/ctype.c +++ b/strings/ctype.c @@ -1413,3 +1413,41 @@ uint my_casefold_multiply_2(CHARSET_INFO *cs) { return 2; } + + +my_bool my_ci_eq_collation_generic(CHARSET_INFO *self, CHARSET_INFO *other) +{ + return FALSE; +} + + +/* + Allocate a memory block for a new charset_info_st together with + its name and its comment in a single once_alloc() call. + Copy the name and the comment into the new block. +*/ +struct charset_info_st *my_ci_alloc(MY_CHARSET_LOADER *loader, + const LEX_CSTRING name, + LEX_CSTRING *out_name, + const LEX_CSTRING comment, + LEX_CSTRING *out_comment) +{ + size_t nbytes= sizeof(struct charset_info_st) + + name.length + comment.length + 2; + struct charset_info_st *csinfo; + char *dst; + if (!(csinfo= (struct charset_info_st*) (loader->once_alloc)(nbytes))) + return NULL; + dst= ((char*) csinfo) + sizeof(struct charset_info_st); + + memcpy(dst, name.str, name.length + 1); + out_name->str= dst; + out_name->length= name.length; + dst+= name.length + 1; + + memcpy(dst, comment.str, comment.length + 1); + out_comment->str= dst; + out_comment->length= comment.length; + + return csinfo; +} diff --git a/strings/strings_def.h b/strings/strings_def.h index 2c226e890c7..1958eb4577d 100644 --- a/strings/strings_def.h +++ b/strings/strings_def.h @@ -20,6 +20,7 @@ #undef DBUG_ASSERT_AS_PRINTF #include /* Define standard vars */ #include "m_string.h" /* Exernal definitions of string functions */ +#include "m_ctype.h" /* We can't use the original DBUG_ASSERT() (which includes _db_flush()) @@ -148,6 +149,13 @@ void my_ci_set_level_flags(struct charset_info_st *cs, uint flags); uint my_casefold_multiply_1(CHARSET_INFO *cs); uint my_casefold_multiply_2(CHARSET_INFO *cs); +my_bool my_ci_eq_collation_generic(CHARSET_INFO *self, CHARSET_INFO *other); + +struct charset_info_st *my_ci_alloc(MY_CHARSET_LOADER *loader, + const LEX_CSTRING name, + LEX_CSTRING *out_name, + const LEX_CSTRING comment, + LEX_CSTRING *out_comment); /* Some common character set names */ extern const char charset_name_latin2[]; From b6391d4e03b15db1f9f1f6284648ff2aa6208324 Mon Sep 17 00:00:00 2001 From: bsrikanth-mariadb Date: Mon, 14 Apr 2025 23:07:07 -0400 Subject: [PATCH 058/125] MDEV-36463 Change expression_cache name to subquery_cache, and make appropriate changes to the test files --- mysql-test/main/analyze_format_json.result | 4 ++-- mysql-test/main/explain_json.result | 6 +++--- mysql-test/main/order_by.result | 2 +- mysql-test/main/show_analyze_json.result | 12 ++++++------ mysql-test/main/show_explain_json.result | 18 +++++++++--------- mysql-test/main/subselect4.result | 2 +- mysql-test/main/subselect_cache.result | 12 ++++++------ sql/sql_explain.cc | 2 +- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/mysql-test/main/analyze_format_json.result b/mysql-test/main/analyze_format_json.result index 1c76fa4971c..8a967186180 100644 --- a/mysql-test/main/analyze_format_json.result +++ b/mysql-test/main/analyze_format_json.result @@ -1000,7 +1000,7 @@ ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "state": "uninitialized", "r_loops": 0, "query_block": { @@ -1098,7 +1098,7 @@ ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "state": "uninitialized", "r_loops": 0, "query_block": { diff --git a/mysql-test/main/explain_json.result b/mysql-test/main/explain_json.result index 195528577a3..9260b80f916 100644 --- a/mysql-test/main/explain_json.result +++ b/mysql-test/main/explain_json.result @@ -404,7 +404,7 @@ EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "state": "uninitialized", "query_block": { "select_id": 2, @@ -451,7 +451,7 @@ EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "state": "uninitialized", "query_block": { "select_id": 2, @@ -2112,7 +2112,7 @@ EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "state": "uninitialized", "query_block": { "select_id": 2, diff --git a/mysql-test/main/order_by.result b/mysql-test/main/order_by.result index ba96d3c0974..b8e125ad731 100644 --- a/mysql-test/main/order_by.result +++ b/mysql-test/main/order_by.result @@ -4523,7 +4523,7 @@ ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "r_loops": 50, "r_hit_ratio": 0, "query_block": { diff --git a/mysql-test/main/show_analyze_json.result b/mysql-test/main/show_analyze_json.result index 66b94a6b383..d528f6086c0 100644 --- a/mysql-test/main/show_analyze_json.result +++ b/mysql-test/main/show_analyze_json.result @@ -638,7 +638,7 @@ SHOW ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "state": "uninitialized", "r_loops": 0, "query_block": { @@ -707,7 +707,7 @@ SHOW ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "r_loops": 2, "r_hit_ratio": 0, "query_block": { @@ -781,7 +781,7 @@ SHOW ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "r_loops": 2, "r_hit_ratio": 0, "query_block": { @@ -1077,7 +1077,7 @@ SHOW ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "r_loops": 1, "r_hit_ratio": 0, "query_block": { @@ -1135,7 +1135,7 @@ SHOW ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "r_loops": 2, "r_hit_ratio": 0, "query_block": { @@ -1194,7 +1194,7 @@ SHOW ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "r_loops": 3, "r_hit_ratio": 0, "query_block": { diff --git a/mysql-test/main/show_explain_json.result b/mysql-test/main/show_explain_json.result index 1768406c371..35ba28b5e3f 100644 --- a/mysql-test/main/show_explain_json.result +++ b/mysql-test/main/show_explain_json.result @@ -421,7 +421,7 @@ SHOW EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "query_block": { "select_id": 2, "cost": "COST_REPLACED", @@ -476,7 +476,7 @@ SHOW EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "query_block": { "select_id": 2, "cost": "COST_REPLACED", @@ -531,7 +531,7 @@ SHOW EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "query_block": { "select_id": 2, "cost": "COST_REPLACED", @@ -942,7 +942,7 @@ SHOW EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "query_block": { "select_id": 2, "cost": "COST_REPLACED", @@ -987,7 +987,7 @@ SHOW EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "query_block": { "select_id": 2, "cost": "COST_REPLACED", @@ -1032,7 +1032,7 @@ SHOW EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "query_block": { "select_id": 2, "cost": "COST_REPLACED", @@ -1092,7 +1092,7 @@ SHOW EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "query_block": { "select_id": 2, "cost": "COST_REPLACED", @@ -1137,7 +1137,7 @@ SHOW EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "query_block": { "select_id": 2, "cost": "COST_REPLACED", @@ -1182,7 +1182,7 @@ SHOW EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "query_block": { "select_id": 2, "cost": "COST_REPLACED", diff --git a/mysql-test/main/subselect4.result b/mysql-test/main/subselect4.result index 6924b1973f7..371091d4dfe 100644 --- a/mysql-test/main/subselect4.result +++ b/mysql-test/main/subselect4.result @@ -2911,7 +2911,7 @@ ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "state": "disabled", "r_loops": 0, "query_block": { diff --git a/mysql-test/main/subselect_cache.result b/mysql-test/main/subselect_cache.result index 97c4ef04a11..701a3dca3ad 100644 --- a/mysql-test/main/subselect_cache.result +++ b/mysql-test/main/subselect_cache.result @@ -72,7 +72,7 @@ ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "r_loops": 10, "r_hit_ratio": 60, "query_block": { @@ -137,7 +137,7 @@ ANALYZE ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "r_loops": 10, "r_hit_ratio": 60, "query_block": { @@ -189,7 +189,7 @@ ANALYZE } }, { - "expression_cache": { + "subquery_cache": { "r_loops": 10, "r_hit_ratio": 60, "query_block": { @@ -243,7 +243,7 @@ EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "state": "uninitialized", "query_block": { "select_id": 2, @@ -288,7 +288,7 @@ EXPLAIN ], "subqueries": [ { - "expression_cache": { + "subquery_cache": { "state": "uninitialized", "query_block": { "union_result": { @@ -329,7 +329,7 @@ EXPLAIN } }, { - "expression_cache": { + "subquery_cache": { "state": "uninitialized", "query_block": { "select_id": 2, diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc index 5eb0030603f..aafccdac447 100644 --- a/sql/sql_explain.cc +++ b/sql/sql_explain.cc @@ -967,7 +967,7 @@ bool Explain_node::print_explain_json_cache(Json_writer *writer, if (cache_tracker) { cache_tracker->fetch_current_stats(); - writer->add_member("expression_cache").start_object(); + writer->add_member("subquery_cache").start_object(); if (cache_tracker->state != Expression_cache_tracker::OK) { writer->add_member("state"). From 1c9f64e54ffb109bb6cf6a189e863bfa54e46510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 16 Apr 2025 15:55:45 +0300 Subject: [PATCH 059/125] MDEV-36613 Incorrect undo logging for indexes on virtual columns Starting with mysql/mysql-server@02f8eaa9988dadb73dd68630dd82393cfa19bfb8 and commit 2e814d4702d71a04388386a9f591d14a35980bfe the index ID of indexes on virtual columns was being encoded insufficiently in InnoDB undo log records. Only the least significant 32 bits were being written. This could lead to some corruption of the affected indexes on ROLLBACK, as well as to missed chances to remove some history from such indexes when purging the history of committed transactions that included DELETE or an UPDATE in the indexes. dict_hdr_create(): In debug instrumented builds, initialize the DICT_HDR_INDEX_ID close to the 32-bit barrier, instead of initializing it to DICT_HDR_FIRST_ID (10). This will allow the changed code to be exercised while running ./mtr --suite=gcol,vcol. trx_undo_log_v_idx(): Encode large index->id in a similar way as mysql/mysql-server@e00328b4d068c7485ac2ffe27207ed1f462c718d but using a different implementation. trx_undo_read_v_idx_low(): Decode large index->id in a similar way as mach_u64_read_much_compressed(). Reviewed by: Debarun Banerjee --- .../suite/gcol/r/innodb_virtual_basic.result | 2 + .../suite/gcol/t/innodb_virtual_basic.test | 37 ++++++++++++++++++- storage/innobase/trx/trx0rec.cc | 26 ++++++++++--- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/gcol/r/innodb_virtual_basic.result b/mysql-test/suite/gcol/r/innodb_virtual_basic.result index 3823887186b..35534d68e63 100644 --- a/mysql-test/suite/gcol/r/innodb_virtual_basic.result +++ b/mysql-test/suite/gcol/r/innodb_virtual_basic.result @@ -86,6 +86,8 @@ delete from t where a =13; DROP INDEX idx1 ON t; DROP INDEX idx2 ON t; DROP TABLE t; +# restart +set default_storage_engine=innodb; /* Test large BLOB data */ CREATE TABLE `t` ( `a` BLOB, diff --git a/mysql-test/suite/gcol/t/innodb_virtual_basic.test b/mysql-test/suite/gcol/t/innodb_virtual_basic.test index b64daa2bcdb..69f9f89ccee 100644 --- a/mysql-test/suite/gcol/t/innodb_virtual_basic.test +++ b/mysql-test/suite/gcol/t/innodb_virtual_basic.test @@ -1,6 +1,6 @@ --source include/have_innodb.inc --source include/have_partition.inc ---source include/big_test.inc +--source include/not_embedded.inc call mtr.add_suppression("\\[Warning\\] InnoDB: Compute virtual"); @@ -66,6 +66,41 @@ DROP INDEX idx1 ON t; DROP INDEX idx2 ON t; DROP TABLE t; +let MYSQLD_DATADIR=`select @@datadir`; +let PAGE_SIZE=`select @@innodb_page_size`; +--source include/shutdown_mysqld.inc +perl; +do "$ENV{MTR_SUITE_DIR}/../innodb/include/crc32.pl"; +my $file = "$ENV{MYSQLD_DATADIR}/ibdata1"; +open(FILE, "+<$file") || die "Unable to open $file"; +binmode FILE; +my $ps= $ENV{PAGE_SIZE}; +my $page; +die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps; +my $full_crc32 = unpack("N",substr($page,54,4)) & 0x10; # FIL_SPACE_FLAGS +sysseek(FILE, 7*$ps, 0) || die "Unable to seek $file\n"; +die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps; +substr($page,54,4)=pack("N",0xc001cafe); # 32 MSB of 64-bit DICT_HDR_INDEX_ID +my $polynomial = 0x82f63b78; # CRC-32C +if ($full_crc32) +{ + my $ck = mycrc32(substr($page, 0, $ps-4), 0, $polynomial); + substr($page, $ps-4, 4) = pack("N", $ck); +} +else +{ + my $ck= pack("N",mycrc32(substr($page, 4, 22), 0, $polynomial) ^ + mycrc32(substr($page, 38, $ps - 38 - 8), 0, $polynomial)); + substr($page,0,4)=$ck; + substr($page,$ps-8,4)=$ck; +} +sysseek(FILE, 7*$ps, 0) || die "Unable to rewind $file\n"; +syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n"; +close(FILE) || die "Unable to close $file"; +EOF +--source include/start_mysqld.inc +set default_storage_engine=innodb; + /* Test large BLOB data */ CREATE TABLE `t` ( `a` BLOB, diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index 33a3962047f..d815f180aba 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -148,7 +148,9 @@ trx_undo_log_v_idx( ulint n_idx = 0; for (const auto& v_index : vcol->v_indexes) { n_idx++; - /* FIXME: index->id is 64 bits! */ + if (uint32_t hi= uint32_t(v_index.index->id >> 32)) { + size += 1 + mach_get_compressed_size(hi); + } size += mach_get_compressed_size(uint32_t(v_index.index->id)); size += mach_get_compressed_size(v_index.nth_field); } @@ -175,10 +177,14 @@ trx_undo_log_v_idx( ptr += mach_write_compressed(ptr, n_idx); for (const auto& v_index : vcol->v_indexes) { - ptr += mach_write_compressed( - /* FIXME: index->id is 64 bits! */ - ptr, uint32_t(v_index.index->id)); - + /* This is compatible with + ptr += mach_u64_write_much_compressed(ptr, v_index.index-id) + (the added "if" statement is fixing an old regression). */ + if (uint32_t hi= uint32_t(v_index.index->id >> 32)) { + *ptr++ = 0xff; + ptr += mach_write_compressed(ptr, hi); + } + ptr += mach_write_compressed(ptr, uint32_t(v_index.index->id)); ptr += mach_write_compressed(ptr, v_index.nth_field); } @@ -217,7 +223,15 @@ trx_undo_read_v_idx_low( dict_index_t* clust_index = dict_table_get_first_index(table); for (ulint i = 0; i < num_idx; i++) { - index_id_t id = mach_read_next_compressed(&ptr); + index_id_t id = 0; + /* This is like mach_u64_read_much_compressed(), + but advancing ptr to the next field. */ + if (*ptr == 0xff) { + ptr++; + id = mach_read_next_compressed(&ptr); + id <<= 32; + } + id |= mach_read_next_compressed(&ptr); ulint pos = mach_read_next_compressed(&ptr); dict_index_t* index = dict_table_get_next_index(clust_index); From 51179067fcfb1a8b892992a1b46513968bcd2654 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 16 Apr 2025 10:09:57 +1000 Subject: [PATCH 060/125] MDEV-36182: Revert "MDEV-26674 workaround for mariadb-backup" Remove version check on the kernel as it now corresponds to a working RHEL9 kernel and the problem was only there in pre-release kernels that shouldn't have been used in production. This reverts commit 3dc0d884ecec0149e35493b3450f610d8ff5d742. --- extra/mariabackup/xtrabackup.cc | 20 +++----------------- storage/innobase/handler/ha_innodb.cc | 7 +++---- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index ab255411ddf..33d172de82e 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1581,11 +1581,6 @@ uint xb_client_options_count = array_elements(xb_client_options); static const char *dbug_option; #endif -#ifdef HAVE_URING -extern const char *io_uring_may_be_unsafe; -bool innodb_use_native_aio_default(); -#endif - struct my_option xb_server_options[] = { {"datadir", 'h', "Path to the database root.", (G_PTR*) &mysql_data_home, @@ -1705,12 +1700,7 @@ struct my_option xb_server_options[] = "Use native AIO if supported on this platform.", (G_PTR*) &srv_use_native_aio, (G_PTR*) &srv_use_native_aio, 0, GET_BOOL, NO_ARG, -#ifdef HAVE_URING - innodb_use_native_aio_default(), -#else - TRUE, -#endif - 0, 0, 0, 0, 0}, + TRUE, 0, 0, 0, 0, 0}, {"innodb_page_size", OPT_INNODB_PAGE_SIZE, "The universal page size of the database.", (G_PTR*) &innobase_page_size, (G_PTR*) &innobase_page_size, 0, @@ -2302,12 +2292,8 @@ static bool innodb_init_param() msg("InnoDB: Using Linux native AIO"); } #elif defined(HAVE_URING) - if (!srv_use_native_aio) { - } else if (io_uring_may_be_unsafe) { - msg("InnoDB: Using liburing on this kernel %s may cause hangs;" - " see https://jira.mariadb.org/browse/MDEV-26674", - io_uring_may_be_unsafe); - } else { + + if (srv_use_native_aio) { msg("InnoDB: Using liburing"); } #else diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index ca1ba831677..b6154b15adf 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -151,7 +151,7 @@ void close_thread_tables(THD* thd); #ifdef HAVE_URING /** The Linux kernel version if io_uring() is considered unsafe */ -const char *io_uring_may_be_unsafe; +static const char *io_uring_may_be_unsafe; #endif #define INSIDE_HA_INNOBASE_CC @@ -19502,10 +19502,9 @@ static MYSQL_SYSVAR_STR(version, innodb_version_str, #ifdef HAVE_URING # include static utsname uname_for_io_uring; -#else -static #endif -bool innodb_use_native_aio_default() + +static bool innodb_use_native_aio_default() { #ifdef HAVE_URING utsname &u= uname_for_io_uring; From 1055bc957e0aa269be0f3b593cf214805c605566 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 16 Apr 2025 10:12:49 +1000 Subject: [PATCH 061/125] MDEV-36182: Revert "MDEV-26674: Set innodb_use_native_aio=OFF ...when using io_uring on a potentially affected kernel" Remove version check on the kernel as it now corresponds to a working RHEL9 kernel and the problem was only there in pre-release kernels that shouldn't have been used in production. This reverts commit 1193a793c40b806c6f1f007bbd87f4d9a73e686d. --- storage/innobase/handler/ha_innodb.cc | 41 +-------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index b6154b15adf..562362db16e 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -149,11 +149,6 @@ void close_thread_tables(THD* thd); #include "wsrep_sst.h" #endif /* WITH_WSREP */ -#ifdef HAVE_URING -/** The Linux kernel version if io_uring() is considered unsafe */ -static const char *io_uring_may_be_unsafe; -#endif - #define INSIDE_HA_INNOBASE_CC #define EQ_CURRENT_THD(thd) ((thd) == current_thd) @@ -4189,14 +4184,6 @@ static int innodb_init_params() cases, we ignore the setting of innodb_use_native_aio. */ srv_use_native_aio = FALSE; #endif -#ifdef HAVE_URING - if (srv_use_native_aio && io_uring_may_be_unsafe) { - sql_print_warning("innodb_use_native_aio may cause " - "hangs with this kernel %s; see " - "https://jira.mariadb.org/browse/MDEV-26674", - io_uring_may_be_unsafe); - } -#endif #ifdef _WIN32 switch (srv_file_flush_method) { @@ -19499,36 +19486,10 @@ static MYSQL_SYSVAR_STR(version, innodb_version_str, PLUGIN_VAR_NOCMDOPT | PLUGIN_VAR_READONLY, "InnoDB version", NULL, NULL, INNODB_VERSION_STR); -#ifdef HAVE_URING -# include -static utsname uname_for_io_uring; -#endif - -static bool innodb_use_native_aio_default() -{ -#ifdef HAVE_URING - utsname &u= uname_for_io_uring; - if (!uname(&u) && u.release[0] == '5' && u.release[1] == '.' && - u.release[2] == '1' && u.release[3] >= '1' && u.release[3] <= '5' && - u.release[4] == '.') - { - if (u.release[3] == '5') { - const char *s= strstr(u.version, "5.15."); - if (s || (s= strstr(u.release, "5.15."))) - if ((s[5] >= '3' || s[6] >= '0')) - return true; /* 5.15.3 and later should be fine */ - } - io_uring_may_be_unsafe= u.release; - return false; /* working around io_uring hangs (MDEV-26674) */ - } -#endif - return true; -} - static MYSQL_SYSVAR_BOOL(use_native_aio, srv_use_native_aio, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, "Use native AIO if supported on this platform.", - NULL, NULL, innodb_use_native_aio_default()); + NULL, NULL, TRUE); #ifdef HAVE_LIBNUMA static MYSQL_SYSVAR_BOOL(numa_interleave, srv_numa_interleave, From 73cdeda347d5eddd8dae699f9b5ed22e59894eca Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 16 Apr 2025 10:24:33 +1000 Subject: [PATCH 062/125] tpool: remove m_group_enqueued (unused) Noted in MDEV-36542 that it wasn't used and the compile warning -Wunused-private-field is legitimate. Review: Vladislav Vaintroub --- tpool/tpool_generic.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/tpool/tpool_generic.cc b/tpool/tpool_generic.cc index c86fcf277cf..8d7e6a754bc 100644 --- a/tpool/tpool_generic.cc +++ b/tpool/tpool_generic.cc @@ -218,7 +218,6 @@ class thread_pool_generic : public thread_pool /** Overall number of enqueues*/ unsigned long long m_tasks_enqueued; - unsigned long long m_group_enqueued; /** Overall number of dequeued tasks. */ unsigned long long m_tasks_dequeued; From f89f8aa313cb891b488a0911d9f7e90ad27d0c02 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 15 Apr 2025 15:49:25 +1000 Subject: [PATCH 063/125] MDEV-36357 MDEV-35452 Temporarily disable view protocol for a query in spider/bg.basic_sql It should be re-enabled once MDEV-36357 is fixed. Also added some documentation to spider result operations. --- storage/spider/ha_spider.cc | 4 ---- .../spider/mysql-test/spider/bg/t/basic_sql.test | 3 +++ storage/spider/spd_db_conn.cc | 14 +++++++++++++- storage/spider/spd_db_include.h | 4 ++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/storage/spider/ha_spider.cc b/storage/spider/ha_spider.cc index 0089b1fa0f6..cff2d580f39 100644 --- a/storage/spider/ha_spider.cc +++ b/storage/spider/ha_spider.cc @@ -4882,10 +4882,6 @@ int ha_spider::rnd_init( } } pushed_pos = NULL; -/* - if (wide_handler->external_lock_type == F_WRLCK) - check_and_start_bulk_update(SPD_BU_START_BY_INDEX_OR_RND_INIT); -*/ rnd_scan_and_first = scan; if ( scan && diff --git a/storage/spider/mysql-test/spider/bg/t/basic_sql.test b/storage/spider/mysql-test/spider/bg/t/basic_sql.test index 6421198fa95..3aae2e34455 100644 --- a/storage/spider/mysql-test/spider/bg/t/basic_sql.test +++ b/storage/spider/mysql-test/spider/bg/t/basic_sql.test @@ -1039,8 +1039,11 @@ if ($USE_CHILD_GROUP2) } } --connection master_1 +# MDEV-36357 +--disable_view_protocol SELECT a.a, a.b, date_format(a.c, '%Y-%m-%d %H:%i:%s') FROM tb_l a WHERE EXISTS (SELECT * FROM ta_l b WHERE b.b = a.b) ORDER BY a.a; +--enable_view_protocol if ($USE_CHILD_GROUP2) { if (!$OUTPUT_CHILD_GROUP2) diff --git a/storage/spider/spd_db_conn.cc b/storage/spider/spd_db_conn.cc index d33115ca1a0..46d9757cefb 100644 --- a/storage/spider/spd_db_conn.cc +++ b/storage/spider/spd_db_conn.cc @@ -3553,6 +3553,10 @@ int spider_db_store_result( db_conn = conn->db_conn; if (!result_list->current) { + /* + Point ->current and ->bgs_current to ->first (create ->first + if needed) + */ if (!result_list->first) { if (!(result_list->first = (SPIDER_RESULT *) @@ -3577,13 +3581,17 @@ int spider_db_store_result( } result_list->bgs_current = result_list->current; current = (SPIDER_RESULT*) result_list->current; - } else { + } else { /* result_list->current != NULL */ if ( #ifndef WITHOUT_SPIDER_BG_SEARCH result_list->bgs_phase > 0 || #endif result_list->quick_phase > 0 ) { + /* + Advance bgs_current to the next result. Create a new result + if needed + */ if (result_list->bgs_current == result_list->last) { if (!(result_list->last = (SPIDER_RESULT *) @@ -3628,6 +3636,10 @@ int spider_db_store_result( } current = (SPIDER_RESULT*) result_list->bgs_current; } else { + /* + Advance current to the next result. Create a new result if + needed + */ if (result_list->current == result_list->last) { if (!(result_list->last = (SPIDER_RESULT *) diff --git a/storage/spider/spd_db_include.h b/storage/spider/spd_db_include.h index bcb16ab391b..26909be1d99 100644 --- a/storage/spider/spd_db_include.h +++ b/storage/spider/spd_db_include.h @@ -1731,7 +1731,7 @@ typedef struct st_spider_result st_spider_result *next; SPIDER_POSITION *first_position; /* for quick mode */ int pos_page_size; /* for quick mode */ - longlong record_num; + longlong record_num; /* number of rows */ bool finish_flg; bool use_position; bool first_pos_use_position; @@ -1783,7 +1783,7 @@ typedef struct st_spider_result_list bool sorted; bool desc_flg; longlong current_row_num; - longlong record_num; + longlong record_num; /* number of rows */ bool finish_flg; longlong limit_num; longlong internal_offset; From 839e8bfe9f900fb3de034a74d133286417928e06 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 16 Apr 2025 10:36:33 +1000 Subject: [PATCH 064/125] MDEV-36182: liburing - incorrect error handing. As noted by Jens Axobe, errno isn't set, it is returned by the io_uring_queue_init function. As such users where getting the following as the common case when EPERM was actually intended to be returned. "mariadbd: io_uring_queue_init() failed with errno 0". Add to error message the correct relevant information around EPERM. --- tpool/aio_liburing.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tpool/aio_liburing.cc b/tpool/aio_liburing.cc index 447c2335c74..acc1c6174e2 100644 --- a/tpool/aio_liburing.cc +++ b/tpool/aio_liburing.cc @@ -34,9 +34,9 @@ class aio_uring final : public tpool::aio public: aio_uring(tpool::thread_pool *tpool, int max_aio) : tpool_(tpool) { - if (io_uring_queue_init(max_aio, &uring_, 0) != 0) + if (const auto e= io_uring_queue_init(max_aio, &uring_, 0)) { - switch (const auto e= errno) { + switch (-e) { case ENOMEM: my_printf_error(ER_UNKNOWN_ERROR, "io_uring_queue_init() failed with ENOMEM:" @@ -57,6 +57,12 @@ public: "(newer than 5.1 required)", ME_ERROR_LOG | ME_WARNING); break; + case EPERM: + my_printf_error(ER_UNKNOWN_ERROR, + "io_uring_queue_init() failed with EPERM:" + " sysctl kernel.io_uring_disabled has the value 2, or 1 and the user of the process is not a member of sysctl kernel.io_uring_group. (see man 2 io_uring_setup).", + ME_ERROR_LOG | ME_WARNING); + break; default: my_printf_error(ER_UNKNOWN_ERROR, "io_uring_queue_init() failed with errno %d", From f388222d495aa8b445da2653343dfb8233b218a8 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 17 Apr 2025 10:28:17 +0530 Subject: [PATCH 065/125] MDEV-36504 Memory leak after CREATE TABLE..SELECT Problem: ======== - After commit cc8eefb0dca1372378905fbae11044f20364c42d (MDEV-33087), InnoDB does use bulk insert operation for ALTER TABLE.. ALGORITHM=COPY and CREATE TABLE..SELECT as well. InnoDB fails to clear the bulk buffer when it encounters error during CREATE..SELECT. Problem is that while transaction cleanup, InnoDB fails to identify the bulk insert for DDL operation. Fix: ==== - Represent bulk_insert in trx by 2 bits. By doing that, InnoDB can distinguish between TRX_DML_BULK, TRX_DDL_BULK. During DDL, set bulk insert value for transaction to TRX_DDL_BULK. - Introduce a parameter HA_EXTRA_ABORT_ALTER_COPY which rollbacks only TRX_DDL_BULK transaction. - bulk_insert_apply() happens for TRX_DDL_BULK transaction happens only during HA_EXTRA_END_ALTER_COPY extra() call. --- include/my_base.h | 5 +++- .../suite/innodb/r/alter_copy_bulk.result | 21 +++++++++++++++ .../suite/innodb/t/alter_copy_bulk.test | 21 +++++++++++++++ sql/ha_partition.cc | 5 +++- sql/sql_insert.cc | 2 +- sql/sql_table.cc | 1 + storage/innobase/handler/ha_innodb.cc | 22 +++++++++++----- storage/innobase/handler/handler0alter.cc | 1 + storage/innobase/include/trx0trx.h | 26 ++++++++++++++++--- storage/innobase/include/trx0types.h | 9 +++++++ storage/innobase/row/row0ins.cc | 8 +++--- storage/innobase/row/row0mysql.cc | 8 ++++-- storage/innobase/trx/trx0trx.cc | 3 +-- storage/mroonga/ha_mroonga.cpp | 8 ++++++ 14 files changed, 118 insertions(+), 22 deletions(-) diff --git a/include/my_base.h b/include/my_base.h index 93fe1d1165a..0e4dab9da5a 100644 --- a/include/my_base.h +++ b/include/my_base.h @@ -219,7 +219,10 @@ enum ha_extra_function { /** Start writing rows during ALTER TABLE...ALGORITHM=COPY. */ HA_EXTRA_BEGIN_ALTER_COPY, /** Finish writing rows during ALTER TABLE...ALGORITHM=COPY. */ - HA_EXTRA_END_ALTER_COPY + HA_EXTRA_END_ALTER_COPY, + /** Abort of writing rows during ALTER TABLE..ALGORITHM=COPY or + CREATE..SELCT */ + HA_EXTRA_ABORT_ALTER_COPY }; /* Compatible option, to be deleted in 6.0 */ diff --git a/mysql-test/suite/innodb/r/alter_copy_bulk.result b/mysql-test/suite/innodb/r/alter_copy_bulk.result index cd57d95af96..ea9d8e2787c 100644 --- a/mysql-test/suite/innodb/r/alter_copy_bulk.result +++ b/mysql-test/suite/innodb/r/alter_copy_bulk.result @@ -91,3 +91,24 @@ INSERT INTO t1 VALUES ALTER TABLE t1 FORCE, ALGORITHM=COPY; DROP TABLE t1; SET GLOBAL innodb_stats_persistent=@default_stats_persistent; +# +# MDEV-36504 Memory leak after insert into empty table +# +CREATE TABLE t1 (k INT PRIMARY KEY)ENGINE=InnoDB; +INSERT INTO t1 SET k= 1; +START TRANSACTION; +INSERT INTO t1 SET k= 2; +SELECT COUNT(*) > 0 FROM mysql.innodb_index_stats LOCK IN SHARE MODE; +COUNT(*) > 0 +1 +connect con1,localhost,root,,,; +SET innodb_lock_wait_timeout=0; +CREATE TABLE t2(f1 INT DEFAULT 1 PRIMARY KEY) +STATS_PERSISTENT= 1 ENGINE=InnoDB as SELECT k FROM t1; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +disconnect con1; +connection default; +SET innodb_lock_wait_timeout=default; +DROP TABLE t1; +DROP TABLE IF EXISTS t2; +# restart diff --git a/mysql-test/suite/innodb/t/alter_copy_bulk.test b/mysql-test/suite/innodb/t/alter_copy_bulk.test index bc19aca00da..acca4c2d9f0 100644 --- a/mysql-test/suite/innodb/t/alter_copy_bulk.test +++ b/mysql-test/suite/innodb/t/alter_copy_bulk.test @@ -109,3 +109,24 @@ INSERT INTO t1 VALUES ALTER TABLE t1 FORCE, ALGORITHM=COPY; DROP TABLE t1; SET GLOBAL innodb_stats_persistent=@default_stats_persistent; + +--echo # +--echo # MDEV-36504 Memory leak after insert into empty table +--echo # +CREATE TABLE t1 (k INT PRIMARY KEY)ENGINE=InnoDB; +INSERT INTO t1 SET k= 1; +START TRANSACTION; +INSERT INTO t1 SET k= 2; +SELECT COUNT(*) > 0 FROM mysql.innodb_index_stats LOCK IN SHARE MODE; + +connect(con1,localhost,root,,,); +SET innodb_lock_wait_timeout=0; +--error ER_LOCK_WAIT_TIMEOUT +CREATE TABLE t2(f1 INT DEFAULT 1 PRIMARY KEY) + STATS_PERSISTENT= 1 ENGINE=InnoDB as SELECT k FROM t1; +disconnect con1; +connection default; +SET innodb_lock_wait_timeout=default; +DROP TABLE t1; +DROP TABLE IF EXISTS t2; +--source include/restart_mysqld.inc diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 3e1240d7b43..d1f623d566a 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -2141,7 +2141,9 @@ int ha_partition::change_partitions(HA_CREATE_INFO *create_info, m_added_file[i]->extra(HA_EXTRA_BEGIN_ALTER_COPY); error= copy_partitions(copied, deleted); for (i= 0; i < part_count; i++) - m_added_file[i]->extra(HA_EXTRA_END_ALTER_COPY); + m_added_file[i]->extra(error + ? HA_EXTRA_ABORT_ALTER_COPY + : HA_EXTRA_END_ALTER_COPY); if (unlikely(error)) { /* @@ -9467,6 +9469,7 @@ int ha_partition::extra(enum ha_extra_function operation) case HA_EXTRA_STARTING_ORDERED_INDEX_SCAN: case HA_EXTRA_BEGIN_ALTER_COPY: case HA_EXTRA_END_ALTER_COPY: + case HA_EXTRA_ABORT_ALTER_COPY: DBUG_RETURN(loop_partitions(extra_cb, &operation)); default: { diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index f456bea19ad..1b3a59281b7 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -4532,7 +4532,7 @@ void select_insert::abort_result_set() table->file->ha_rnd_end(); table->file->extra(HA_EXTRA_NO_IGNORE_DUP_KEY); table->file->extra(HA_EXTRA_WRITE_CANNOT_REPLACE); - + table->file->extra(HA_EXTRA_ABORT_ALTER_COPY); /* If at least one row has been inserted/modified and will stay in the table (the table doesn't have transactions) we must write to diff --git a/sql/sql_table.cc b/sql/sql_table.cc index ba6c981518e..89095125fe3 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -12378,6 +12378,7 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, bool ignore, if (alt_error > 0) { error= alt_error; + to->file->extra(HA_EXTRA_ABORT_ALTER_COPY); copy_data_error_ignore(error, false, to, thd, alter_ctx); } } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index a2b75f43291..d6d9414ed43 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3652,7 +3652,7 @@ ha_innobase::init_table_handle_for_HANDLER(void) m_prebuilt->used_in_HANDLER = TRUE; reset_template(); - m_prebuilt->trx->bulk_insert = false; + m_prebuilt->trx->bulk_insert &= TRX_DDL_BULK; } /*********************************************************************//** @@ -4501,7 +4501,7 @@ static bool end_of_statement(trx_t *trx) noexcept undo_no_t savept= 0; trx->rollback(&savept); /* MariaDB will roll back the entire transaction. */ - trx->bulk_insert= false; + trx->bulk_insert&= TRX_DDL_BULK; trx->last_stmt_start= 0; return true; } @@ -15875,7 +15875,7 @@ ha_innobase::extra( stmt_boundary: trx->bulk_insert_apply(); trx->end_bulk_insert(*m_prebuilt->table); - trx->bulk_insert = false; + trx->bulk_insert &= TRX_DDL_BULK; break; case HA_EXTRA_NO_KEYREAD: (void)check_trx_exists(ha_thd()); @@ -15941,7 +15941,7 @@ ha_innobase::extra( break; } m_prebuilt->table->skip_alter_undo = 0; - if (dberr_t err= trx->bulk_insert_apply()) { + if (dberr_t err= trx->bulk_insert_apply()) { m_prebuilt->table->skip_alter_undo = 0; return convert_error_code_to_mysql( err, m_prebuilt->table->flags, @@ -15949,7 +15949,7 @@ ha_innobase::extra( } trx->end_bulk_insert(*m_prebuilt->table); - trx->bulk_insert = false; + trx->bulk_insert &= TRX_DDL_BULK; if (!m_prebuilt->table->is_temporary() && !high_level_read_only) { /* During copy_data_between_tables(), InnoDB only @@ -15968,6 +15968,13 @@ ha_innobase::extra( log_buffer_flush_to_disk(); } break; + case HA_EXTRA_ABORT_ALTER_COPY: + if (m_prebuilt->table->skip_alter_undo) { + trx = check_trx_exists(ha_thd()); + m_prebuilt->table->skip_alter_undo = 0; + trx->rollback(); + } + break; default:/* Do nothing */ ; } @@ -16062,7 +16069,8 @@ ha_innobase::start_stmt( break; } - trx->bulk_insert = false; + ut_ad(trx->bulk_insert != TRX_DDL_BULK); + trx->bulk_insert = TRX_NO_BULK; trx->last_stmt_start = trx->undo_no; } @@ -16270,7 +16278,7 @@ ha_innobase::external_lock( if (!trx->bulk_insert) { break; } - trx->bulk_insert = false; + trx->bulk_insert &= TRX_DDL_BULK; trx->last_stmt_start = trx->undo_no; } diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 7b60f7102b9..98dcd9f4521 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -5544,6 +5544,7 @@ static bool innodb_insert_sys_columns( DBUG_EXECUTE_IF("instant_insert_fail", my_error(ER_INTERNAL_ERROR, MYF(0), "InnoDB: Insert into SYS_COLUMNS failed"); + mem_heap_free(info->heap); return true;); if (DB_SUCCESS != que_eval_sql( diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 7e36806c569..946785cc0b0 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -809,8 +809,13 @@ public: /** normally set; "SET unique_checks=0, foreign_key_checks=0" enables bulk insert into an empty table */ unsigned check_unique_secondary:1; - /** whether an insert into an empty table is active */ - unsigned bulk_insert:1; + /** whether an insert into an empty table is active + Possible states are + TRX_NO_BULK + TRX_DML_BULK + TRX_DDL_BULK + @see trx_bulk_insert in trx0types.h */ + unsigned bulk_insert:2; /*------------------------------*/ /* MySQL has a transaction coordinator to coordinate two phase commit between multiple storage engines and the binary log. When @@ -1117,6 +1122,7 @@ public: ut_ad(!is_not_inheriting_locks()); ut_ad(check_foreigns); ut_ad(check_unique_secondary); + ut_ad(bulk_insert == TRX_NO_BULK); } /** This has to be invoked on SAVEPOINT or at the end of a statement. @@ -1142,6 +1148,8 @@ public: rollback to the start of a statement will work. */ void end_bulk_insert() { + if (bulk_insert == TRX_DDL_BULK) + return; for (auto& t : mod_tables) t.second.end_bulk_insert(); } @@ -1149,7 +1157,15 @@ public: /** @return whether a bulk insert into empty table is in progress */ bool is_bulk_insert() const { - if (!bulk_insert || check_unique_secondary || check_foreigns) + switch (bulk_insert) { + case TRX_NO_BULK: + return false; + case TRX_DDL_BULK: + return true; + default: + ut_ad(bulk_insert == TRX_DML_BULK); + } + if (check_unique_secondary || check_foreigns) return false; for (const auto& t : mod_tables) if (t.second.is_bulk_insert()) @@ -1179,9 +1195,11 @@ public: /** Do the bulk insert for the buffered insert operation for the transaction. @return DB_SUCCESS or error code */ + template dberr_t bulk_insert_apply() { - return UNIV_UNLIKELY(bulk_insert) ? bulk_insert_apply_low(): DB_SUCCESS; + static_assert(type != TRX_NO_BULK, ""); + return bulk_insert == type ? bulk_insert_apply_low(): DB_SUCCESS; } private: diff --git a/storage/innobase/include/trx0types.h b/storage/innobase/include/trx0types.h index ecb8ecc023b..2da83a226d0 100644 --- a/storage/innobase/include/trx0types.h +++ b/storage/innobase/include/trx0types.h @@ -65,6 +65,15 @@ enum trx_state_t { TRX_STATE_COMMITTED_IN_MEMORY }; +/** Transaction bulk insert operation @see trx_t::bulk_insert */ +enum trx_bulk_insert { + TRX_NO_BULK, + /** bulk insert is being executed during DML */ + TRX_DML_BULK, + /** bulk insert is being executed in copy_data_between_tables() */ + TRX_DDL_BULK +}; + /** Memory objects */ /* @{ */ /** Transaction */ diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index d355b674c49..134a0815e6a 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2775,16 +2775,16 @@ err_exit: && !index->table->has_spatial_index()) { ut_ad(!index->table->skip_alter_undo); - trx->bulk_insert = true; + trx->bulk_insert = TRX_DML_BULK; err = lock_table(index->table, NULL, LOCK_X, thr); if (err != DB_SUCCESS) { trx->error_state = err; - trx->bulk_insert = false; + trx->bulk_insert = TRX_NO_BULK; goto err_exit; } if (index->table->n_rec_locks) { avoid_bulk: - trx->bulk_insert = false; + trx->bulk_insert = TRX_NO_BULK; goto row_level_insert; } #ifdef WITH_WSREP @@ -2844,7 +2844,7 @@ avoid_bulk: bulk buffer and doesn't check for constraint validity of foreign key relationship. */ trx_start_if_not_started(trx, true); - trx->bulk_insert = true; + trx->bulk_insert = TRX_DDL_BULK; auto m = trx->mod_tables.emplace(index->table, 0); m.first->second.start_bulk_insert(index->table, true); err = m.first->second.bulk_insert_buffered( diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index cb06ec9f52a..f93ad6eb3ee 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -687,8 +687,12 @@ handle_new_error: /* MariaDB will roll back the latest SQL statement */ break; } - /* MariaDB will roll back the entire transaction. */ - trx->bulk_insert = false; + /* For DML, InnoDB does partial rollback and clear + bulk buffer in row_mysql_handle_errors(). + For ALTER TABLE ALGORITHM=COPY & CREATE TABLE...SELECT, + the bulk insert transaction will be rolled back inside + ha_innobase::extra(HA_EXTRA_ABORT_ALTER_COPY) */ + trx->bulk_insert &= TRX_DDL_BULK; trx->last_stmt_start = 0; break; case DB_LOCK_WAIT: diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 53a66ced689..28b7fed90cd 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -134,8 +134,6 @@ trx_init( trx->will_lock = false; - trx->bulk_insert = false; - trx->apply_online_log = false; ut_d(trx->start_file = 0); @@ -1513,6 +1511,7 @@ bool trx_t::commit_cleanup() noexcept *detailed_error= '\0'; mod_tables.clear(); + bulk_insert= TRX_NO_BULK; check_foreigns= true; check_unique_secondary= true; assert_freed(); diff --git a/storage/mroonga/ha_mroonga.cpp b/storage/mroonga/ha_mroonga.cpp index 4a6fee624ba..2dbc3f351cc 100644 --- a/storage/mroonga/ha_mroonga.cpp +++ b/storage/mroonga/ha_mroonga.cpp @@ -558,6 +558,9 @@ static const char *mrn_inspect_extra_function(enum ha_extra_function operation) case HA_EXTRA_END_ALTER_COPY: inspected = "HA_EXTRA_END_ALTER_COPY"; break; + case HA_EXTRA_ABORT_ALTER_COPY: + inspected = "HA_EXTRA_ABORT_ALTER_COPY"; + break; #ifdef MRN_HAVE_HA_EXTRA_EXPORT case HA_EXTRA_EXPORT: inspected = "HA_EXTRA_EXPORT"; @@ -593,6 +596,11 @@ static const char *mrn_inspect_extra_function(enum ha_extra_function operation) inspected = "HA_EXTRA_END_ALTER_COPY"; break; #endif +#ifdef MRN_HAVE_HA_EXTRA_ABORT_ALTER_COPY + case HA_EXTRA_ABORT_ALTER_COPY: + inspected = "HA_EXTRA_ABORT_ALTER_COPY"; + break; +#endif #ifdef MRN_HAVE_HA_EXTRA_NO_AUTOINC_LOCKING case HA_EXTRA_NO_AUTOINC_LOCKING: inspected = "HA_EXTRA_NO_AUTOINC_LOCKING"; From 23cc3eb1f79caf40e736cc4ea96fabddb3e21cc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 17 Apr 2025 15:11:21 +0300 Subject: [PATCH 066/125] MDEV-36257: Fix a debug assertion ha_innobase::info_low(): Assert that dict_table_t::stat_initialized() only within the critical section. Changes of this field should be protected by dict_table_t::lock_latch. --- storage/innobase/handler/ha_innodb.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d6d9414ed43..b5e013d8a6b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -14867,13 +14867,13 @@ stats_fetch: ulint stat_clustered_index_size; ulint stat_sum_of_other_index_sizes; - ut_ad(ib_table->stat_initialized()); - #if !defined NO_ELISION && !defined SUX_LOCK_GENERIC if (xbegin()) { if (ib_table->stats_mutex_is_locked()) xabort(); + ut_ad(ib_table->stat_initialized()); + n_rows = ib_table->stat_n_rows; stat_clustered_index_size @@ -14888,6 +14888,8 @@ stats_fetch: { ib_table->stats_shared_lock(); + ut_ad(ib_table->stat_initialized()); + n_rows = ib_table->stat_n_rows; stat_clustered_index_size From f99586668a79dd571a7dee7bce916d6b894d072a Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 11 Apr 2025 08:28:42 +0200 Subject: [PATCH 067/125] MDEV-36380 User has unauthorized access to a sequence through a view with security invoker check sequence privileges in Item_func_nextval::fix_fields(), just like column privileges are checked in Item_field::fix_fields() remove sequence specific hacks that kinda made sequence privilege checks works, but not in all cases. And they were too lax, didn't requre SELECT privilege for NEXTVAL. Also INSERT privilege looks wrong here, UPDATE would've been more appropriate, but won't change that for compatibility reasons. also fixes MDEV-36413 User without any privileges to a sequence can read from it and modify it via column default --- mysql-test/main/view_grant.result | 46 +++++++++++++++++ mysql-test/main/view_grant.test | 47 +++++++++++++++++ mysql-test/suite/sql_sequence/grant.result | 47 ++++++++++++++++- mysql-test/suite/sql_sequence/grant.test | 50 ++++++++++++++++++- mysql-test/suite/sql_sequence/gtid.result | 2 +- .../suite/sql_sequence/replication.result | 2 +- mysql-test/suite/sql_sequence/view.test | 1 - sql/item_func.cc | 10 ++++ sql/item_func.h | 7 +++ sql/sql_acl.cc | 14 ++---- sql/sql_parse.cc | 11 +--- 11 files changed, 211 insertions(+), 26 deletions(-) diff --git a/mysql-test/main/view_grant.result b/mysql-test/main/view_grant.result index f1bebf98ecd..57c0a0fcc55 100644 --- a/mysql-test/main/view_grant.result +++ b/mysql-test/main/view_grant.result @@ -1985,4 +1985,50 @@ connection default; DROP VIEW v1; DROP USER foo; DROP USER FOO; +# +# MDEV-36380: User has unauthorized access to a sequence through +# a view with security invoker +# +create database db; +use db; +create sequence s; +create sql security invoker view vin as select nextval(s); +create sql security definer view vdn as select nextval(s); +create sql security invoker view vil as select lastval(s); +create sql security definer view vdl as select lastval(s); +create sql security invoker view vis as select setval(s,20); +create sql security definer view vds as select setval(s,30); +create user u@localhost; +grant select on db.vin to u@localhost; +grant select on db.vdn to u@localhost; +grant select on db.vil to u@localhost; +grant select on db.vdl to u@localhost; +grant select on db.vis to u@localhost; +grant select on db.vds to u@localhost; +connect con1,localhost,u,,db; +select nextval(s); +ERROR 42000: SELECT, INSERT command denied to user 'u'@'localhost' for table `db`.`s` +select * from vin; +ERROR HY000: View 'db.vin' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +select * from vdn; +nextval(s) +1 +select lastval(s); +ERROR 42000: SELECT command denied to user 'u'@'localhost' for table `db`.`s` +select * from vil; +ERROR HY000: View 'db.vil' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +select * from vdl; +lastval(s) +1 +select setval(s,10); +ERROR 42000: INSERT command denied to user 'u'@'localhost' for table `db`.`s` +select * from vis; +ERROR HY000: View 'db.vis' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them +select * from vds; +setval(s,30) +30 +disconnect con1; +connection default; +drop database db; +drop user u@localhost; # End of 10.5 tests diff --git a/mysql-test/main/view_grant.test b/mysql-test/main/view_grant.test index fc0521933cc..d61362b6cc3 100644 --- a/mysql-test/main/view_grant.test +++ b/mysql-test/main/view_grant.test @@ -2239,4 +2239,51 @@ DROP VIEW v1; DROP USER foo; DROP USER FOO; +--echo # +--echo # MDEV-36380: User has unauthorized access to a sequence through +--echo # a view with security invoker +--echo # +create database db; +use db; +create sequence s; +create sql security invoker view vin as select nextval(s); +create sql security definer view vdn as select nextval(s); +create sql security invoker view vil as select lastval(s); +create sql security definer view vdl as select lastval(s); +create sql security invoker view vis as select setval(s,20); +create sql security definer view vds as select setval(s,30); +create user u@localhost; +grant select on db.vin to u@localhost; +grant select on db.vdn to u@localhost; +grant select on db.vil to u@localhost; +grant select on db.vdl to u@localhost; +grant select on db.vis to u@localhost; +grant select on db.vds to u@localhost; + +--connect (con1,localhost,u,,db) +--error ER_TABLEACCESS_DENIED_ERROR +select nextval(s); +--error ER_VIEW_INVALID +select * from vin; +--disable_ps2_protocol +select * from vdn; +--enable_ps2_protocol + +--error ER_TABLEACCESS_DENIED_ERROR +select lastval(s); +--error ER_VIEW_INVALID +select * from vil; +select * from vdl; + +--error ER_TABLEACCESS_DENIED_ERROR +select setval(s,10); +--error ER_VIEW_INVALID +select * from vis; +select * from vds; + +--disconnect con1 +--connection default +drop database db; +drop user u@localhost; + --echo # End of 10.5 tests diff --git a/mysql-test/suite/sql_sequence/grant.result b/mysql-test/suite/sql_sequence/grant.result index 0a69d69fc74..fc3421efcb6 100644 --- a/mysql-test/suite/sql_sequence/grant.result +++ b/mysql-test/suite/sql_sequence/grant.result @@ -47,14 +47,57 @@ next_not_cached_value minimum_value maximum_value start_value increment cache_si 11 1 9223372036854775806 1 1 1000 0 0 connection only_alter; select next value for s1; -ERROR 42000: INSERT command denied to user 'only_alter'@'localhost' for table `mysqltest_1`.`s1` +ERROR 42000: SELECT, INSERT command denied to user 'only_alter'@'localhost' for table `mysqltest_1`.`s1` alter sequence s1 restart= 11; select * from s1; ERROR 42000: SELECT command denied to user 'only_alter'@'localhost' for table `mysqltest_1`.`s1` connection default; -drop database mysqltest_1; drop user 'normal'@'%'; drop user 'read_only'@'%'; drop user 'read_write'@'%'; drop user 'alter'@'%'; drop user 'only_alter'@'%'; +drop sequence s1; +# +# MDEV-36413 User without any privileges to a sequence can read from +# it and modify it via column default +# +create sequence s1; +create sequence s2; +select * from s2; +next_not_cached_value minimum_value maximum_value start_value increment cache_size cycle_option cycle_count +1 1 9223372036854775806 1 1 1000 0 0 +create table t2 (a int not null default(nextval(s1))); +insert into t2 values(); +create user u; +grant create, insert, select, drop on mysqltest_1.t1 to u; +grant insert, select on mysqltest_1.s1 to u; +grant select on mysqltest_1.t2 to u; +connect con1,localhost,u,,mysqltest_1; +select nextval(s2); +ERROR 42000: SELECT, INSERT command denied to user 'u'@'localhost' for table `mysqltest_1`.`s2` +show create sequence s2; +ERROR 42000: SHOW command denied to user 'u'@'localhost' for table `mysqltest_1`.`s2` +create table t1 (a int not null default(nextval(s1))); +drop table t1; +create table t1 (a int not null default(nextval(s1))) select a from t2; +insert into t1 values(); +select * from t1; +a +1 +2 +drop table t1; +create table t1 (a int not null default(nextval(s1))) select a from (select t2.a from t2,t2 as t3 where t2.a=t3.a) as t4; +drop table t1; +create table t1 (a int not null default(nextval(s2))); +ERROR 42000: SELECT, INSERT command denied to user 'u'@'localhost' for table `mysqltest_1`.`s2` +create table t1 (a int not null default(nextval(s1)), +b int not null default(nextval(s2))); +ERROR 42000: SELECT, INSERT command denied to user 'u'@'localhost' for table `mysqltest_1`.`s2` +disconnect con1; +connection default; +drop user u; +drop database mysqltest_1; +# +# End of 10.11 tests +# diff --git a/mysql-test/suite/sql_sequence/grant.test b/mysql-test/suite/sql_sequence/grant.test index fb8a9f813a6..c205bd34223 100644 --- a/mysql-test/suite/sql_sequence/grant.test +++ b/mysql-test/suite/sql_sequence/grant.test @@ -60,10 +60,58 @@ select * from s1; # connection default; -drop database mysqltest_1; drop user 'normal'@'%'; drop user 'read_only'@'%'; drop user 'read_write'@'%'; drop user 'alter'@'%'; drop user 'only_alter'@'%'; +drop sequence s1; +--echo # +--echo # MDEV-36413 User without any privileges to a sequence can read from +--echo # it and modify it via column default +--echo # + +create sequence s1; +create sequence s2; +select * from s2; +create table t2 (a int not null default(nextval(s1))); +insert into t2 values(); + +create user u; +grant create, insert, select, drop on mysqltest_1.t1 to u; +grant insert, select on mysqltest_1.s1 to u; +grant select on mysqltest_1.t2 to u; + +--connect(con1,localhost,u,,mysqltest_1) +--error ER_TABLEACCESS_DENIED_ERROR +select nextval(s2); +--error ER_TABLEACCESS_DENIED_ERROR +show create sequence s2; + +create table t1 (a int not null default(nextval(s1))); +drop table t1; +create table t1 (a int not null default(nextval(s1))) select a from t2; +insert into t1 values(); +select * from t1; +drop table t1; +create table t1 (a int not null default(nextval(s1))) select a from (select t2.a from t2,t2 as t3 where t2.a=t3.a) as t4; +drop table t1; +--error ER_TABLEACCESS_DENIED_ERROR +create table t1 (a int not null default(nextval(s2))); +--error ER_TABLEACCESS_DENIED_ERROR +create table t1 (a int not null default(nextval(s1)), + b int not null default(nextval(s2))); +--disconnect con1 +--connection default +drop user u; + +# +# Cleanup +# + +drop database mysqltest_1; + +--echo # +--echo # End of 10.11 tests +--echo # diff --git a/mysql-test/suite/sql_sequence/gtid.result b/mysql-test/suite/sql_sequence/gtid.result index 5c0003d4ea3..f59dc5595ee 100644 --- a/mysql-test/suite/sql_sequence/gtid.result +++ b/mysql-test/suite/sql_sequence/gtid.result @@ -174,7 +174,7 @@ create sequence s_db.s2; drop sequence s_db.s2; connection m_normal_2; select next value for s_db.s1; -ERROR 42000: INSERT command denied to user 'normal_2'@'localhost' for table `s_db`.`s1` +ERROR 42000: SELECT, INSERT command denied to user 'normal_2'@'localhost' for table `s_db`.`s1` create sequence s_db.s2; ERROR 42000: CREATE command denied to user 'normal_2'@'localhost' for table `s_db`.`s2` connection m_normal_1; diff --git a/mysql-test/suite/sql_sequence/replication.result b/mysql-test/suite/sql_sequence/replication.result index 762c332dbd6..0e4ca1c939e 100644 --- a/mysql-test/suite/sql_sequence/replication.result +++ b/mysql-test/suite/sql_sequence/replication.result @@ -285,7 +285,7 @@ create sequence s_db.s2; drop sequence s_db.s2; connection m_normal_2; select NEXT VALUE for s_db.s1; -ERROR 42000: INSERT command denied to user 'normal_2'@'localhost' for table `s_db`.`s1` +ERROR 42000: SELECT, INSERT command denied to user 'normal_2'@'localhost' for table `s_db`.`s1` create sequence s_db.s2; ERROR 42000: CREATE command denied to user 'normal_2'@'localhost' for table `s_db`.`s2` connection m_normal_1; diff --git a/mysql-test/suite/sql_sequence/view.test b/mysql-test/suite/sql_sequence/view.test index affac002878..7e2cd712325 100644 --- a/mysql-test/suite/sql_sequence/view.test +++ b/mysql-test/suite/sql_sequence/view.test @@ -1,5 +1,4 @@ --source include/have_sequence.inc ---source include/have_innodb.inc # # Test sequences with views diff --git a/sql/item_func.cc b/sql/item_func.cc index 6a7b12ae4a0..3ca3d5af3a3 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -7092,6 +7092,16 @@ longlong Item_func_cursor_rowcount::val_int() /***************************************************************************** SEQUENCE functions *****************************************************************************/ +bool Item_func_nextval::check_access_and_fix_fields(THD *thd, Item **ref, + privilege_t want_access) +{ + table_list->sequence= false; + bool error= check_single_table_access(thd, want_access, table_list, false); + table_list->sequence= true; + if (error && table_list->belong_to_view) + table_list->hide_view_error(thd); + return error || Item_longlong_func::fix_fields(thd, ref); +} longlong Item_func_nextval::val_int() { diff --git a/sql/item_func.h b/sql/item_func.h index f4f0558a77b..68b3669c3b4 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -3774,11 +3774,14 @@ class Item_func_nextval :public Item_longlong_func protected: TABLE_LIST *table_list; TABLE *table; + bool check_access_and_fix_fields(THD *, Item **ref, privilege_t); public: Item_func_nextval(THD *thd, TABLE_LIST *table_list_arg): Item_longlong_func(thd), table_list(table_list_arg) {} longlong val_int() override; const char *func_name() const override { return "nextval"; } + bool fix_fields(THD *thd, Item **ref) override + { return check_access_and_fix_fields(thd, ref, INSERT_ACL | SELECT_ACL); } bool fix_length_and_dec() override { unsigned_flag= 0; @@ -3820,6 +3823,8 @@ class Item_func_lastval :public Item_func_nextval public: Item_func_lastval(THD *thd, TABLE_LIST *table_list_arg): Item_func_nextval(thd, table_list_arg) {} + bool fix_fields(THD *thd, Item **ref) override + { return check_access_and_fix_fields(thd, ref, SELECT_ACL); } longlong val_int() override; const char *func_name() const override { return "lastval"; } Item *do_get_copy(THD *thd) const override @@ -3840,6 +3845,8 @@ public: : Item_func_nextval(thd, table_list_arg), nextval(nextval_arg), round(round_arg), is_used(is_used_arg) {} + bool fix_fields(THD *thd, Item **ref) override + { return check_access_and_fix_fields(thd, ref, INSERT_ACL); } longlong val_int() override; const char *func_name() const override { return "setval"; } void print(String *str, enum_query_type query_type) override; diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 385d3da18e2..939061fdf8e 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -8304,19 +8304,13 @@ bool check_grant(THD *thd, privilege_t want_access, TABLE_LIST *tables, /* If sequence is used as part of NEXT VALUE, PREVIOUS VALUE or SELECT, - we need to modify the requested access rights depending on how the - sequence is used. + the privilege will be checked in ::fix_fields(). + Direct SELECT of a sequence table doesn't set t_ref->sequence, so + privileges will be checked normally, as for any table. */ if (t_ref->sequence && !(want_access & ~(SELECT_ACL | INSERT_ACL | UPDATE_ACL | DELETE_ACL))) - { - /* - We want to have either SELECT or INSERT rights to sequences depending - on how they are accessed - */ - orig_want_access= ((t_ref->lock_type == TL_WRITE_ALLOW_WRITE) ? - INSERT_ACL : SELECT_ACL); - } + continue; const ACL_internal_table_access *access= get_cached_table_access(&t_ref->grant.m_internal, diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 705dc35cdb7..b017cba3c36 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -7332,18 +7332,9 @@ check_table_access(THD *thd, privilege_t requirements, TABLE_LIST *tables, DBUG_PRINT("info", ("derived: %d view: %d", table_ref->derived != 0, table_ref->view != 0)); - if (table_ref->is_anonymous_derived_table()) + if (table_ref->is_anonymous_derived_table() || table_ref->sequence) continue; - if (table_ref->sequence) - { - /* We want to have either SELECT or INSERT rights to sequences depending - on how they are accessed - */ - want_access= ((table_ref->lock_type == TL_WRITE_ALLOW_WRITE) ? - INSERT_ACL : SELECT_ACL); - } - if (check_access(thd, want_access, table_ref->get_db_name().str, &table_ref->grant.privilege, &table_ref->grant.m_internal, From 7f1492d0bc01fe24a55a3c5f9a9314b5c91d79d6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 17 Apr 2025 17:08:04 +0200 Subject: [PATCH 068/125] cleanup: rename hide_view_error->replace_view_error_with_generic as requested by Monty --- sql/item.cc | 6 +++--- sql/item_func.cc | 2 +- sql/sql_base.cc | 2 +- sql/sql_update.cc | 2 +- sql/sql_view.cc | 2 +- sql/table.cc | 4 ++-- sql/table.h | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sql/item.cc b/sql/item.cc index 9498d816bb0..466ba63b1ce 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -10930,8 +10930,8 @@ void dummy_error_processor(THD *thd, void *data) {} /** - Wrapper of hide_view_error call for Name_resolution_context error - processor. + Wrapper of replace_view_error_with_generic call for Name_resolution_context + error processor. @note hide view underlying tables details in error messages @@ -10939,7 +10939,7 @@ void dummy_error_processor(THD *thd, void *data) void view_error_processor(THD *thd, void *data) { - ((TABLE_LIST *)data)->hide_view_error(thd); + ((TABLE_LIST *)data)->replace_view_error_with_generic(thd); } /** diff --git a/sql/item_func.cc b/sql/item_func.cc index 3ca3d5af3a3..5bea3f440b4 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -7099,7 +7099,7 @@ bool Item_func_nextval::check_access_and_fix_fields(THD *thd, Item **ref, bool error= check_single_table_access(thd, want_access, table_list, false); table_list->sequence= true; if (error && table_list->belong_to_view) - table_list->hide_view_error(thd); + table_list->replace_view_error_with_generic(thd); return error || Item_longlong_func::fix_fields(thd, ref); } diff --git a/sql/sql_base.cc b/sql/sql_base.cc index ac4bce1dce0..ed23208e358 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -8009,7 +8009,7 @@ bool setup_tables_and_check_access(THD *thd, Name_resolution_context *context, if (table_list->belong_to_view && !table_list->view && check_single_table_access(thd, access, table_list, FALSE)) { - tables->hide_view_error(thd); + tables->replace_view_error_with_generic(thd); DBUG_RETURN(TRUE); } access= want_access; diff --git a/sql/sql_update.cc b/sql/sql_update.cc index f2cea510ce7..1327bd786c6 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -1684,7 +1684,7 @@ static bool multi_update_check_table_access(THD *thd, TABLE_LIST *table, if (multi_update_check_table_access(thd, tbl, tables_for_update, &updated)) { - tbl->hide_view_error(thd); + tbl->replace_view_error_with_generic(thd); return true; } } diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 89d0704f6df..9df5c89cf37 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -337,7 +337,7 @@ bool create_view_precheck(THD *thd, TABLE_LIST *tables, TABLE_LIST *view, { if (check_single_table_access(thd, SELECT_ACL, tbl, FALSE)) { - tbl->hide_view_error(thd); + tbl->replace_view_error_with_generic(thd); goto err; } } diff --git a/sql/table.cc b/sql/table.cc index 978dbee254e..1b5d6741b45 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -6065,9 +6065,9 @@ bool TABLE_LIST::prep_check_option(THD *thd, uint8 check_opt_type) @pre This method can be called only if there is an error. */ -void TABLE_LIST::hide_view_error(THD *thd) +void TABLE_LIST::replace_view_error_with_generic(THD *thd) { - if ((thd->killed && !thd->is_error())|| thd->get_internal_handler()) + if ((thd->killed && !thd->is_error()) || thd->get_internal_handler()) return; /* Hide "Unknown column" or "Unknown function" error */ DBUG_ASSERT(thd->is_error()); diff --git a/sql/table.h b/sql/table.h index 981fc5d23c5..eeec8b05022 100644 --- a/sql/table.h +++ b/sql/table.h @@ -2776,7 +2776,7 @@ struct TABLE_LIST bool check_single_table(TABLE_LIST **table, table_map map, TABLE_LIST *view); bool set_insert_values(MEM_ROOT *mem_root); - void hide_view_error(THD *thd); + void replace_view_error_with_generic(THD *thd); TABLE_LIST *find_underlying_table(TABLE *table); TABLE_LIST *first_leaf_for_name_resolution(); TABLE_LIST *last_leaf_for_name_resolution(); From c25237c28d90f002e3f01453d06fd6a83307160a Mon Sep 17 00:00:00 2001 From: Dave Gosselin Date: Fri, 4 Apr 2025 16:07:27 -0400 Subject: [PATCH 069/125] MDEV-36211 Incorrect query result for binary_column NOT LIKE binary_column During optimize_cond, we incorrectly removed the NOT LIKE condition when attempting to remove any equality conditions. Item_func_like's override of eq_cmp_result() returns COND_TRUE when its collation is the binary collation. Item_bool_func2's implementation of remove_eq_conds would then attempt to detect if both arguments were equal to one another and return a NULL condition to optimize_conds. This removes the condition from ever being evaluated (and Item_func_like::val_bool is never called in this case), rendering the incorrect result. Fix this by checking the negated condition during eq_cmp_result() to return either COND_FALSE in the negated==true case, or COND_TRUE in the negated==false case which has the effect of not removing the NOT LIKE/LIKE condition for the query. --- mysql-test/main/func_like.result | 19 +++++++++++++++++++ mysql-test/main/func_like.test | 15 +++++++++++++++ sql/item.h | 13 +++++++++++++ sql/item_cmpfunc.h | 13 +++++++++++-- 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/func_like.result b/mysql-test/main/func_like.result index ba053eac877..4cdb5de37fb 100644 --- a/mysql-test/main/func_like.result +++ b/mysql-test/main/func_like.result @@ -424,3 +424,22 @@ Warnings: Note 1003 select 1 like `test`.`t1`.`c1` | `test`.`t1`.`c2` AS `1 LIKE c1|c2`,1 like `test`.`t1`.`c1` & `test`.`t1`.`c2` AS `1 LIKE c1&c2`,1 like `test`.`t1`.`c2` >> `test`.`t1`.`c1` AS `1 LIKE c2>>c1`,2 like `test`.`t1`.`c2` << `test`.`t1`.`c1` AS `2 LIKE c2< 0 AS `1 LIKE c1||c2`,2 like `test`.`t1`.`c1` + `test`.`t1`.`c2` AS `2 LIKE c1+c2`,-1 like `test`.`t1`.`c1` - `test`.`t1`.`c2` AS `-1 LIKE c1-c2`,2 like `test`.`t1`.`c1` * `test`.`t1`.`c2` AS `2 LIKE c1*c2`,0.5000 like `test`.`t1`.`c1` / `test`.`t1`.`c2` AS `0.5000 LIKE c1/c2`,0 like `test`.`t1`.`c1` DIV `test`.`t1`.`c2` AS `0 LIKE c1 DIV c2`,0 like `test`.`t1`.`c1` MOD `test`.`t1`.`c2` AS `0 LIKE c1 MOD c2` from `test`.`t1` order by `test`.`t1`.`c2` DROP VIEW v1; DROP TABLE t1; +# +# MDEV-36211 Incorrect query result for binary_column NOT LIKE binary_column +# +CREATE TABLE t1 (c1 BLOB NOT NULL); +INSERT INTO t1 (c1) VALUES (1); +SELECT c1 FROM t1 WHERE c1 NOT LIKE c1; +c1 +SELECT c1 FROM t1 WHERE c1 LIKE c1; +c1 +1 +DROP TABLE t1; +CREATE TABLE t1 (c1 BLOB); +INSERT INTO t1 (c1) VALUES (1); +SELECT c1 FROM t1 WHERE c1 NOT LIKE c1; +c1 +SELECT c1 FROM t1 WHERE c1 LIKE c1; +c1 +1 +DROP TABLE t1; diff --git a/mysql-test/main/func_like.test b/mysql-test/main/func_like.test index 751e9c19e40..fc22ba3a2ee 100644 --- a/mysql-test/main/func_like.test +++ b/mysql-test/main/func_like.test @@ -291,3 +291,18 @@ SELECT * FROM v1; EXPLAIN EXTENDED SELECT * FROM v1; DROP VIEW v1; DROP TABLE t1; + +--echo # +--echo # MDEV-36211 Incorrect query result for binary_column NOT LIKE binary_column +--echo # +CREATE TABLE t1 (c1 BLOB NOT NULL); +INSERT INTO t1 (c1) VALUES (1); +SELECT c1 FROM t1 WHERE c1 NOT LIKE c1; +SELECT c1 FROM t1 WHERE c1 LIKE c1; +DROP TABLE t1; + +CREATE TABLE t1 (c1 BLOB); +INSERT INTO t1 (c1) VALUES (1); +SELECT c1 FROM t1 WHERE c1 NOT LIKE c1; +SELECT c1 FROM t1 WHERE c1 LIKE c1; +DROP TABLE t1; diff --git a/sql/item.h b/sql/item.h index cd2f157750e..ceb9a7fdc98 100644 --- a/sql/item.h +++ b/sql/item.h @@ -1954,6 +1954,19 @@ public: */ virtual Item *clone_item(THD *thd) const { return nullptr; } + /* + @detail + The meaning of this function seems to be: + Check what the item would return if it was provided with two identical + non-NULL arguments. + It is not clear why it is defined for generic class Item or what its other + uses are. + + @return + COND_TRUE Would return true + COND_FALSE Would return false + COND_OK May return either, depending on the argument type. + */ virtual cond_result eq_cmp_result() const { return COND_OK; } inline uint float_length(uint decimals_par) const { return decimals < FLOATING_POINT_DECIMALS ? (DBL_DIG+2+decimals_par) : DBL_DIG+8;} diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index 158ad8273e8..f9d9b18a70d 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -2945,9 +2945,18 @@ public: TODO: We could still replace "expr1" to "const" in "expr1 LIKE expr2" in case of a "PAD SPACE" collation, but only if "expr2" has '%' - at the end. + at the end. */ - return compare_collation() == &my_charset_bin ? COND_TRUE : COND_OK; + if (compare_collation() == &my_charset_bin) + { + /* + 'foo' NOT LIKE 'foo' is false, + 'foo' LIKE 'foo' is true. + */ + return negated? COND_FALSE : COND_TRUE; + } + + return COND_OK; } void add_key_fields(JOIN *join, KEY_FIELD **key_fields, uint *and_level, table_map usable_tables, SARGABLE_PARAM **sargables) From 07b442aa688835610e1a3174154dc53cf273744a Mon Sep 17 00:00:00 2001 From: Rex Date: Wed, 16 Apr 2025 13:36:05 +1100 Subject: [PATCH 070/125] MDEV-36607 find_order_in_list mismatch when order item needs fixing() This bug is exposed by MDEV-30073, causing bogus warning messages to be pushed by find_order_in_list(), but which is otherwise benign. An existing test case in show_explain.test, MDEV-238 can be used together with an assert to find a query which exposes the issue. if (resolution == RESOLVED_BEHIND_ALIAS && order_item->fix_fields_if_needed_for_order_by(thd, order->item)) return TRUE; /* Lookup the current GROUP field in the FROM clause. */ order_item_type= order_item->type(); + DBUG_ASSERT( order_item_type == (*order->item)->type() ); This will fail here CREATE TABLE t2 ( a INT ); INSERT INTO t2 VALUES (1),(2),(1),(4),(2); explain SELECT alias.a FROM t2, ( SELECT * FROM t2 ) AS alias GROUP BY alias.a; This assert makes little sense after the patch. DaveGosselin-MariaDB approved these changes Apr 18, 2025 --- sql/sql_select.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index dac97711c17..b4edf8a3050 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -27157,9 +27157,13 @@ find_order_in_list(THD *thd, Ref_ptr_array ref_pointer_array, original field name, we should additionally check if we have conflict for this name (in case if we would perform lookup in all tables). */ - if (resolution == RESOLVED_BEHIND_ALIAS && - order_item->fix_fields_if_needed_for_order_by(thd, order->item)) - return TRUE; + if (resolution == RESOLVED_BEHIND_ALIAS) + { + if (order_item->fix_fields_if_needed_for_order_by(thd, order->item)) + return TRUE; + // fix_fields may have replaced order->item, reset local variable. + order_item= *order->item; + } /* Lookup the current GROUP field in the FROM clause. */ order_item_type= order_item->type(); From 5b1bdf60766003dbf9dccb86ca5114bff04053f5 Mon Sep 17 00:00:00 2001 From: ParadoxV5 Date: Mon, 14 Apr 2025 16:37:30 -0600 Subject: [PATCH 071/125] MDEV-36359: Patch NULL deref after disabling Semi-Sync primary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under `@@rpl_semi_sync_master_wait_no_slave = 0`, when `rpl_semi_sync_master_clients` decrements to zero, the primary reverts to async replication. This code did not check whether Semi- Sync is still globally enabled or not as it didn’t matter before. However, after MDEV-33551 (#3089) split the transactions’ACK condition variables to per-transaction, this function now needs Semi-Sync’s transaction tracker to unblock these condition variables in batch, but this tracker is `NULL` when Semi-Sync Primary is disabled. Co-authored-by: Kristian Nielsen --- ...semi_sync_master_disable_with_slave.result | 41 +++++++++++ ...l_semi_sync_master_disable_with_slave.test | 68 +++++++++++++++++++ sql/semisync_master.cc | 4 +- 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/rpl/r/rpl_semi_sync_master_disable_with_slave.result create mode 100644 mysql-test/suite/rpl/t/rpl_semi_sync_master_disable_with_slave.test diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_master_disable_with_slave.result b/mysql-test/suite/rpl/r/rpl_semi_sync_master_disable_with_slave.result new file mode 100644 index 00000000000..3b0686c351e --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_semi_sync_master_disable_with_slave.result @@ -0,0 +1,41 @@ +# Set up Semi-Sync with rpl_semi_sync_master_wait_no_slave=0 +include/master-slave.inc +[connection master] +SET @@GLOBAL.rpl_semi_sync_master_enabled= 1; +SET @@GLOBAL.rpl_semi_sync_master_wait_no_slave= 0; +connection slave; +SET @@GLOBAL.rpl_semi_sync_slave_enabled= 1; +include/start_slave.inc +connection master; +connection slave; +connection master; +SELECT ID INTO @binlog_dump_tid +FROM information_schema.PROCESSLIST WHERE COMMAND = 'Binlog Dump'; +# Control State +SELECT STATE FROM information_schema.PROCESSLIST WHERE ID = @binlog_dump_tid; +STATE +Master has sent all binlog to slave; waiting for more updates +SHOW STATUS LIKE 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +# Disable Semi-Sync while the dump thread is still connected to its slave +SET @@GLOBAL.rpl_semi_sync_master_enabled = 0; +SELECT STATE FROM information_schema.PROCESSLIST WHERE ID = @binlog_dump_tid; +STATE +Master has sent all binlog to slave; waiting for more updates +SHOW STATUS LIKE 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +# Disconnect the slave and wait until the master's dump thread is gone +connection slave; +STOP SLAVE; +connection master; +SHOW STATUS LIKE 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 0 +# Cleanup +SET @@GLOBAL.rpl_semi_sync_master_enabled= 0; +SET @@GLOBAL.rpl_semi_sync_master_wait_no_slave= 1; +connection slave; +SET @@GLOBAL.rpl_semi_sync_slave_enabled= 0; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_master_disable_with_slave.test b/mysql-test/suite/rpl/t/rpl_semi_sync_master_disable_with_slave.test new file mode 100644 index 00000000000..28d8e7fcf7b --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_master_disable_with_slave.test @@ -0,0 +1,68 @@ +# MDEV-36359: Master crashes when reverting to async after Semi-Sync disabled. +# +# Assert behavior of turning Semi-Sync off on +# the master when still connected to a slave + +--source include/have_binlog_format_mixed.inc # format-agnostic + +--echo # Set up Semi-Sync with rpl_semi_sync_master_wait_no_slave=0 +--let $rpl_skip_start_slave= 1 +--source include/master-slave.inc + +--let $orig_master_enabled=`SELECT @@GLOBAL.rpl_semi_sync_master_enabled` +SET @@GLOBAL.rpl_semi_sync_master_enabled= 1; +--let $orig_wait_no_slave=`SELECT @@GLOBAL.rpl_semi_sync_master_wait_no_slave` +SET @@GLOBAL.rpl_semi_sync_master_wait_no_slave= 0; + +--connection slave +--let $orig_slave_enabled=`SELECT @@GLOBAL.rpl_semi_sync_slave_enabled` +SET @@GLOBAL.rpl_semi_sync_slave_enabled= 1; +--source include/start_slave.inc + +--connection master +# Make sure Semi-Sync is active +--let $status_var= Rpl_semi_sync_master_status +--let $status_var_value= ON +--source include/wait_for_status_var.inc + +--sync_slave_with_master +--connection master + +--disable_cursor_protocol +SELECT ID INTO @binlog_dump_tid + FROM information_schema.PROCESSLIST WHERE COMMAND = 'Binlog Dump'; +--enable_cursor_protocol + +--echo # Control State +SELECT STATE FROM information_schema.PROCESSLIST WHERE ID = @binlog_dump_tid; +SHOW STATUS LIKE 'Rpl_semi_sync_master_clients'; + +--echo # Disable Semi-Sync while the dump thread is still connected to its slave +SET @@GLOBAL.rpl_semi_sync_master_enabled = 0; +--let $status_var_value= OFF +--source include/wait_for_status_var.inc + +SELECT STATE FROM information_schema.PROCESSLIST WHERE ID = @binlog_dump_tid; +SHOW STATUS LIKE 'Rpl_semi_sync_master_clients'; + +--echo # Disconnect the slave and wait until the master's dump thread is gone +--connection slave +STOP SLAVE; +# Starting with MDEV-13073, +# Semi-Sync STOP SLAVE also terminates its dump thread on the master. +--connection master + +# MDEV-36359: The disconnection would crash the master and leave the wait with +# error 2013 'Lost connection to server during query' +--let $wait_condition= SELECT COUNT(*)=0 FROM information_schema.PROCESSLIST WHERE ID = @binlog_dump_tid +--source include/wait_condition.inc +SHOW STATUS LIKE 'Rpl_semi_sync_master_clients'; + +--echo # Cleanup +--eval SET @@GLOBAL.rpl_semi_sync_master_enabled= $orig_master_enabled +--eval SET @@GLOBAL.rpl_semi_sync_master_wait_no_slave= $orig_wait_no_slave +--connection slave +--eval SET @@GLOBAL.rpl_semi_sync_slave_enabled= $orig_slave_enabled + +--let $rpl_only_running_threads= 1 +--source include/rpl_end.inc diff --git a/sql/semisync_master.cc b/sql/semisync_master.cc index 3e33cff06f6..43afb8f996e 100644 --- a/sql/semisync_master.cc +++ b/sql/semisync_master.cc @@ -565,12 +565,14 @@ void Repl_semi_sync_master::remove_slave() { lock(); DBUG_ASSERT(rpl_semi_sync_master_clients > 0); - if (!(--rpl_semi_sync_master_clients) && !rpl_semi_sync_master_wait_no_slave) + if (!(--rpl_semi_sync_master_clients) && !rpl_semi_sync_master_wait_no_slave + && get_master_enabled()) { /* Signal transactions waiting in commit_trx() that they do not have to wait anymore. */ + DBUG_ASSERT(m_active_tranxs); m_active_tranxs->clear_active_tranx_nodes(NULL, 0, signal_waiting_transaction); } From 7b3e02e1aaeba150b096b3516a44066bf65c1495 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 15 Apr 2025 19:30:44 +0400 Subject: [PATCH 072/125] MDEV-36565 Assertion `src != ((void *)0)' failed in my_casedn_8bit The crash happened when ExtractValue() returning an empty string as a result was passed to LCASE() or UCASE(). Item_func_xml_extractvalue::val_str() could return a String {Ptr=0,str_value=0} in some cases, to mean an empty retult. But virtual my_charset_handler_st functions caseup() and casedn() do not expect {src=nullptr,srclen=0} as input and: - raise a DBUG_ASSERT() in debug builds, or - raise a "applying zero offset to null pointer" warning in UBSAN builds Fixing Item_func_xml_extractvalue::val_str() to return a String {Ptr="",str_length=0} instead of {Ptr=0,str_value=0}. A similar fix was done earlier in Field_set::val_str(). See c69fb1a6273. --- mysql-test/main/xml.result | 14 +++++++++++++- mysql-test/main/xml.test | 14 +++++++++++++- sql/item_xmlfunc.cc | 4 ++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/mysql-test/main/xml.result b/mysql-test/main/xml.result index d0b1d1782a1..a960d8cc316 100644 --- a/mysql-test/main/xml.result +++ b/mysql-test/main/xml.result @@ -1320,5 +1320,17 @@ f foo # -# Start of 10.5 tests +# End of 10.5 tests # +# Start of 11.4 tests +# +# MDEV-36565 Assertion `src != ((void *)0)' failed in my_casedn_8bit +# +SET NAMES latin1; +SELECT lcase((extractvalue('a', 'a'))) a FROM (select 1) dt; +a + +SELECT ucase((extractvalue('a', 'a'))) a FROM (select 1) dt; +a + +# End of 11.4 tests diff --git a/mysql-test/main/xml.test b/mysql-test/main/xml.test index 2d0dd9907bb..e0954c5f6bb 100644 --- a/mysql-test/main/xml.test +++ b/mysql-test/main/xml.test @@ -822,5 +822,17 @@ DROP TABLE t1; SELECT 'foo' AS f UNION SELECT BINARY( UpdateXML('', '/a', '')) AS f; --echo # ---echo # Start of 10.5 tests +--echo # End of 10.5 tests --echo # + +--echo # Start of 11.4 tests + +--echo # +--echo # MDEV-36565 Assertion `src != ((void *)0)' failed in my_casedn_8bit +--echo # + +SET NAMES latin1; +SELECT lcase((extractvalue('a', 'a'))) a FROM (select 1) dt; +SELECT ucase((extractvalue('a', 'a'))) a FROM (select 1) dt; + +--echo # End of 11.4 tests diff --git a/sql/item_xmlfunc.cc b/sql/item_xmlfunc.cc index 02c37b45d8f..86c4e423947 100644 --- a/sql/item_xmlfunc.cc +++ b/sql/item_xmlfunc.cc @@ -176,8 +176,8 @@ public: } } - str->length(0); - str->set_charset(collation.collation); + // Make sure we never return {Ptr=nullptr, str_length=0} + str->copy("", 0, collation.collation); for (uint i=0 ; i < numnodes; i++) { if(active[i]) From 15fd232da47d7ec550caced84741bbc1348da04c Mon Sep 17 00:00:00 2001 From: Dave Gosselin Date: Wed, 9 Apr 2025 13:57:09 -0400 Subject: [PATCH 073/125] MDEV-36235 Incorrect result for BETWEEN over unique blob prefix Disallow range optimization for BETWEEN when casting one of the arguments from STRING to a numeric type would be required to construct a range for the query. Adds a new method on Item_func_between called can_optimize_range_const which allows range optimization when the types of the arguments to BETWEEN would permit it. --- mysql-test/main/type_binary.result | 58 ++++++ mysql-test/main/type_binary.test | 11 + mysql-test/main/type_blob.result | 190 ++++++++++++++++++ mysql-test/main/type_blob.test | 45 +++++ mysql-test/main/type_varbinary.result | 42 ++++ mysql-test/main/type_varbinary.test | 10 + .../mysql-test/type_inet/type_inet6.result | 23 +++ .../mysql-test/type_inet/type_inet6.test | 12 ++ sql/item_cmpfunc.h | 17 ++ sql/opt_range.cc | 67 +++--- 10 files changed, 450 insertions(+), 25 deletions(-) create mode 100644 mysql-test/main/type_varbinary.result create mode 100644 mysql-test/main/type_varbinary.test diff --git a/mysql-test/main/type_binary.result b/mysql-test/main/type_binary.result index d1aa4ada5af..32593cb728b 100644 --- a/mysql-test/main/type_binary.result +++ b/mysql-test/main/type_binary.result @@ -397,3 +397,61 @@ indexed_col not_indexed_col DROP TABLE t2; DROP TABLE t1; SET note_verbosity=DEFAULT; +# +# MDEV-36235 Incorrect result for BETWEEN over unique blob prefix +# +CREATE TABLE t1 (c1 BINARY(16), UNIQUE (c1)); +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2); +SELECT HEX(c1) FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +HEX(c1) +31000000000000000000000000000000 +32000000000000000000000000000000 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: '-1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: '-2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: '1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: '2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +SELECT HEX(c1) FROM t1 IGNORE KEY(c1) WHERE 'a' BETWEEN 0 AND (c1); +HEX(c1) +31000000000000000000000000000000 +32000000000000000000000000000000 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: '-2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: '-1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: '1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: '2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +SELECT HEX(c1) FROM t1 WHERE '#' BETWEEN c1 AND 0; +HEX(c1) +2D310000000000000000000000000000 +2D320000000000000000000000000000 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '-1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '-2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +SELECT HEX(c1) FROM t1 IGNORE KEY(c1) WHERE '#' BETWEEN c1 AND 0; +HEX(c1) +2D320000000000000000000000000000 +2D310000000000000000000000000000 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '-2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '-1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '2\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +DROP TABLE t1; diff --git a/mysql-test/main/type_binary.test b/mysql-test/main/type_binary.test index 495c776cc75..916d6402b63 100644 --- a/mysql-test/main/type_binary.test +++ b/mysql-test/main/type_binary.test @@ -178,3 +178,14 @@ DELIMITER ;$$ --source unusable_keys_joins.inc DROP TABLE t1; SET note_verbosity=DEFAULT; + +--echo # +--echo # MDEV-36235 Incorrect result for BETWEEN over unique blob prefix +--echo # +CREATE TABLE t1 (c1 BINARY(16), UNIQUE (c1)); +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2); +SELECT HEX(c1) FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +SELECT HEX(c1) FROM t1 IGNORE KEY(c1) WHERE 'a' BETWEEN 0 AND (c1); +SELECT HEX(c1) FROM t1 WHERE '#' BETWEEN c1 AND 0; +SELECT HEX(c1) FROM t1 IGNORE KEY(c1) WHERE '#' BETWEEN c1 AND 0; +DROP TABLE t1; diff --git a/mysql-test/main/type_blob.result b/mysql-test/main/type_blob.result index b9d1f2cae31..14685bcf149 100644 --- a/mysql-test/main/type_blob.result +++ b/mysql-test/main/type_blob.result @@ -1419,3 +1419,193 @@ Note 1105 Cannot use key parts with `test`.`t1`.`indexed_col` in the rewritten c DROP TABLE t2; DROP TABLE t1; SET note_verbosity=DEFAULT; +# +# MDEV-36235 Incorrect result for BETWEEN over unique blob prefix +# +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1(2))) engine=myisam; +INSERT INTO t1 (c1) VALUES (1); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +c1 +1 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +DROP TABLE t1; +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1(2))); +INSERT INTO t1 (c1) VALUES (1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +c1 +1 +2 +3 +4 +5 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +SELECT c1 FROM t1 WHERE 3 BETWEEN 10*POW(-1,c1) AND (c1); +c1 +3 +5 +SELECT c1 FROM t1 WHERE 'a' BETWEEN 10*POW(-1,c1) AND (c1); +c1 +1 +3 +5 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +DROP TABLE t1; +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1(2))) engine=myisam; +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +c1 +1 +2 +3 +4 +5 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +c1 +-2 +-1 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +DROP TABLE t1; +CREATE TABLE t1 (c1 TINYBLOB NOT NULL); +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +c1 +1 +2 +3 +4 +5 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +c1 +-2 +-1 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +DROP TABLE t1; +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1(2))) engine=innodb; +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +c1 +1 +2 +3 +4 +5 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +c1 +-2 +-1 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +ALTER TABLE t1 engine=myisam; +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +c1 +1 +2 +3 +4 +5 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +c1 +-2 +-1 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +DROP TABLE t1; +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1)) engine=innodb; +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +c1 +1 +2 +3 +4 +5 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +c1 +-2 +-1 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +DROP TABLE t1; diff --git a/mysql-test/main/type_blob.test b/mysql-test/main/type_blob.test index ad58946afea..bb90e7434d6 100644 --- a/mysql-test/main/type_blob.test +++ b/mysql-test/main/type_blob.test @@ -808,3 +808,48 @@ DELIMITER ;$$ --source unusable_keys_joins.inc DROP TABLE t1; SET note_verbosity=DEFAULT; + +--echo # +--echo # MDEV-36235 Incorrect result for BETWEEN over unique blob prefix +--echo # +# myisam has a special optimization for tables with one row +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1(2))) engine=myisam; +INSERT INTO t1 (c1) VALUES (1); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +DROP TABLE t1; + +# This case shows that we don't transform the entire WHERE clause +# into a range condition. +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1(2))); +INSERT INTO t1 (c1) VALUES (1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +SELECT c1 FROM t1 WHERE 3 BETWEEN 10*POW(-1,c1) AND (c1); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 10*POW(-1,c1) AND (c1); +DROP TABLE t1; + +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1(2))) engine=myisam; +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +DROP TABLE t1; + +CREATE TABLE t1 (c1 TINYBLOB NOT NULL); +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +DROP TABLE t1; + +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1(2))) engine=innodb; +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +ALTER TABLE t1 engine=myisam; +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +DROP TABLE t1; + +CREATE TABLE t1 (c1 TINYBLOB, UNIQUE (c1)) engine=innodb; +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2),(3),(4),(5); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +DROP TABLE t1; diff --git a/mysql-test/main/type_varbinary.result b/mysql-test/main/type_varbinary.result new file mode 100644 index 00000000000..41f7403a24f --- /dev/null +++ b/mysql-test/main/type_varbinary.result @@ -0,0 +1,42 @@ +# +# MDEV-36235 Incorrect result for BETWEEN over unique blob prefix +# +CREATE TABLE t1 (c1 VARBINARY(10), UNIQUE (c1)); +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +c1 +1 +2 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +SELECT c1 FROM t1 IGNORE KEY(c1) WHERE 'a' BETWEEN 0 AND (c1); +c1 +1 +2 +Warnings: +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +Warning 1292 Truncated incorrect DOUBLE value: 'a' +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +c1 +-1 +-2 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +SELECT c1 FROM t1 IGNORE KEY(c1) WHERE '#' BETWEEN c1 AND 0; +c1 +-2 +-1 +Warnings: +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +Warning 1292 Truncated incorrect DECIMAL value: '#' +DROP TABLE t1; diff --git a/mysql-test/main/type_varbinary.test b/mysql-test/main/type_varbinary.test new file mode 100644 index 00000000000..2bb35dfd195 --- /dev/null +++ b/mysql-test/main/type_varbinary.test @@ -0,0 +1,10 @@ +--echo # +--echo # MDEV-36235 Incorrect result for BETWEEN over unique blob prefix +--echo # +CREATE TABLE t1 (c1 VARBINARY(10), UNIQUE (c1)); +INSERT INTO t1 (c1) VALUES (-2),(-1),(1),(2); +SELECT c1 FROM t1 WHERE 'a' BETWEEN 0 AND (c1); +SELECT c1 FROM t1 IGNORE KEY(c1) WHERE 'a' BETWEEN 0 AND (c1); +SELECT c1 FROM t1 WHERE '#' BETWEEN c1 AND 0; +SELECT c1 FROM t1 IGNORE KEY(c1) WHERE '#' BETWEEN c1 AND 0; +DROP TABLE t1; diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6.result b/plugin/type_inet/mysql-test/type_inet/type_inet6.result index d43352dfff2..c8ebcf5be98 100644 --- a/plugin/type_inet/mysql-test/type_inet/type_inet6.result +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6.result @@ -2407,3 +2407,26 @@ f::f DROP TABLE t1; SET max_sort_length=DEFAULT; # End of 10.8 tests +# +# MDEV-36235 Incorrect result for BETWEEN over unique blob prefix +# +CREATE OR REPLACE TABLE t1 (c1 BINARY(16), UNIQUE (c1)); +INSERT INTO t1 (c1) VALUES (0x00000000000000000000000000000001); +INSERT INTO t1 (c1) VALUES (0x00000000000000000000000000000002); +SELECT CAST(c1 AS INET6) FROM t1 WHERE '::1' BETWEEN CAST('::1' AS INET6) AND c1; +CAST(c1 AS INET6) +::1 +::2 +SELECT CAST(c1 AS INET6) FROM t1 IGNORE KEY(c1) WHERE '::1' BETWEEN CAST('::1' AS INET6) AND c1; +CAST(c1 AS INET6) +::1 +::2 +SELECT CAST(c1 AS INET6) FROM t1 WHERE '::2' BETWEEN c1 AND CAST('::2' AS INET6); +CAST(c1 AS INET6) +::1 +::2 +SELECT CAST(c1 AS INET6) FROM t1 IGNORE KEY(c1) WHERE '::2' BETWEEN c1 AND CAST('::2' AS INET6); +CAST(c1 AS INET6) +::1 +::2 +DROP TABLE t1; diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6.test b/plugin/type_inet/mysql-test/type_inet/type_inet6.test index c3128140b71..1fd5feea004 100644 --- a/plugin/type_inet/mysql-test/type_inet/type_inet6.test +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6.test @@ -1741,3 +1741,15 @@ DROP TABLE t1; SET max_sort_length=DEFAULT; --echo # End of 10.8 tests + +--echo # +--echo # MDEV-36235 Incorrect result for BETWEEN over unique blob prefix +--echo # +CREATE OR REPLACE TABLE t1 (c1 BINARY(16), UNIQUE (c1)); +INSERT INTO t1 (c1) VALUES (0x00000000000000000000000000000001); +INSERT INTO t1 (c1) VALUES (0x00000000000000000000000000000002); +SELECT CAST(c1 AS INET6) FROM t1 WHERE '::1' BETWEEN CAST('::1' AS INET6) AND c1; +SELECT CAST(c1 AS INET6) FROM t1 IGNORE KEY(c1) WHERE '::1' BETWEEN CAST('::1' AS INET6) AND c1; +SELECT CAST(c1 AS INET6) FROM t1 WHERE '::2' BETWEEN c1 AND CAST('::2' AS INET6); +SELECT CAST(c1 AS INET6) FROM t1 IGNORE KEY(c1) WHERE '::2' BETWEEN c1 AND CAST('::2' AS INET6); +DROP TABLE t1; diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h index f9d9b18a70d..a896e461bf5 100644 --- a/sql/item_cmpfunc.h +++ b/sql/item_cmpfunc.h @@ -1003,6 +1003,23 @@ public: class Item_func_between :public Item_func_opt_neg { + /* + If the types of the arguments to BETWEEN permit, then: + + WHERE const1 BETWEEN expr2 AND field1 + can be optimized as if it was just: + WHERE const1 <= field1 + + as expr2 could be an arbitrary expression. More generally, + this optimization is permitted if aggregation for comparison + for three expressions (const1,const2,field1) and for two + expressions (const1,field1) return the same type handler. + + @param [IN] field_item - This is a field from the right side + of the BETWEEN operator. + */ + bool can_optimize_range_const(Item_field *field_item) const; + protected: SEL_TREE *get_func_mm_tree(RANGE_OPT_PARAM *param, Field *field, Item *value) override; diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 3f9c5663b26..7f3fece735f 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -8351,56 +8351,58 @@ SEL_TREE *Item_func_in::get_func_row_mm_tree(RANGE_OPT_PARAM *param, /* Build conjunction of all SEL_TREEs for a simple predicate applying equalities - + SYNOPSIS get_full_func_mm_tree() param PARAM from SQL_SELECT::test_quick_select field_item field in the predicate - value constant in the predicate (or a field already read from + value constant in the predicate (or a field already read from a table in the case of dynamic range access) (for BETWEEN it contains the number of the field argument, - for IN it's always 0) + for IN it's always 0) inv TRUE <> NOT cond_func is considered (makes sense only when cond_func is BETWEEN or IN) DESCRIPTION - For a simple SARGable predicate of the form (f op c), where f is a field and - c is a constant, the function builds a conjunction of all SEL_TREES that can - be obtained by the substitution of f for all different fields equal to f. + For a simple SARGable predicate of the form (f op c), where f is a field + and c is a constant, the function builds a conjunction of all SEL_TREES that + can be obtained by the substitution of f for all different fields equal to f. - NOTES + NOTES If the WHERE condition contains a predicate (fi op c), then not only SELL_TREE for this predicate is built, but the trees for the results of substitution of fi for each fj belonging to the same multiple equality as fi are built as well. - E.g. for WHERE t1.a=t2.a AND t2.a > 10 + E.g. for WHERE t1.a=t2.a AND t2.a > 10 a SEL_TREE for t2.a > 10 will be built for quick select from t2 - and + and a SEL_TREE for t1.a > 10 will be built for quick select from t1. - A BETWEEN predicate of the form (fi [NOT] BETWEEN c1 AND c2) is treated - in a similar way: we build a conjuction of trees for the results - of all substitutions of fi for equal fj. + A BETWEEN predicate of the form (fi [NOT] BETWEEN c1 AND c2), where fi + is some field, is treated in a similar way: we build a conjuction of + trees for the results of all substitutions of fi equal fj. + Yet a predicate of the form (c BETWEEN f1i AND f2i) is processed differently. It is considered as a conjuction of two SARGable - predicates (f1i <= c) and (f2i <=c) and the function get_full_func_mm_tree - is called for each of them separately producing trees for - AND j (f1j <=c ) and AND j (f2j <= c) + predicates (f1i <= c) and (c <= f2i) and the function get_full_func_mm_tree + is called for each of them separately producing trees for + AND j (f1j <= c) and AND j (c <= f2j) After this these two trees are united in one conjunctive tree. It's easy to see that the same tree is obtained for - AND j,k (f1j <=c AND f2k<=c) - which is equivalent to + AND j,k (f1j <= c AND c <= f2k) + which is equivalent to AND j,k (c BETWEEN f1j AND f2k). + The validity of the processing of the predicate (c NOT BETWEEN f1i AND f2i) which equivalent to (f1i > c OR f2i < c) is not so obvious. Here the - function get_full_func_mm_tree is called for (f1i > c) and (f2i < c) - producing trees for AND j (f1j > c) and AND j (f2j < c). Then this two - trees are united in one OR-tree. The expression + function get_full_func_mm_tree is called for (f1i > c) and called for + (f2i < c) producing trees for AND j (f1j > c) and AND j (f2j < c). Then + this two trees are united in one OR-tree. The expression (AND j (f1j > c) OR AND j (f2j < c) is equivalent to the expression - AND j,k (f1j > c OR f2k < c) - which is just a translation of + AND j,k (f1j > c OR f2k < c) + which is just a translation of AND j,k (c NOT BETWEEN f1j AND f2k) In the cases when one of the items f1, f2 is a constant c1 we do not create @@ -8413,9 +8415,9 @@ SEL_TREE *Item_func_in::get_func_row_mm_tree(RANGE_OPT_PARAM *param, As to IN predicates only ones of the form (f IN (c1,...,cn)), where f1 is a field and c1,...,cn are constant, are considered as SARGable. We never try to narrow the index scan using predicates of - the form (c IN (c1,...,f,...,cn)). - - RETURN + the form (c IN (c1,...,f,...,cn)). + + RETURN Pointer to the tree representing the built conjunction of SEL_TREEs */ @@ -8623,6 +8625,19 @@ SEL_TREE *Item::get_mm_tree(RANGE_OPT_PARAM *param, Item **cond_ptr) } +bool +Item_func_between::can_optimize_range_const(Item_field *field_item) const +{ + const Type_handler *fi_handler= field_item->type_handler_for_comparison(); + Type_handler_hybrid_field_type cmp(fi_handler); + if (cmp.aggregate_for_comparison(args[0]->type_handler_for_comparison()) || + cmp.type_handler() != m_comparator.type_handler()) + return false; // Cannot optimize range because of type mismatch. + + return true; +} + + SEL_TREE * Item_func_between::get_mm_tree(RANGE_OPT_PARAM *param, Item **cond_ptr) { @@ -8648,6 +8663,8 @@ Item_func_between::get_mm_tree(RANGE_OPT_PARAM *param, Item **cond_ptr) if (arguments()[i]->real_item()->type() == Item::FIELD_ITEM) { Item_field *field_item= (Item_field*) (arguments()[i]->real_item()); + if (!can_optimize_range_const(field_item)) + continue; SEL_TREE *tmp= get_full_func_mm_tree(param, field_item, (Item*)(intptr) i); if (negated) From 459dfe99d148271834a0fcbc895fc7b9508d1451 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 18 Apr 2025 19:53:55 +0200 Subject: [PATCH 074/125] Fix appveyor config Sometimes pull request commit would be incorrectly skipped. --- appveyor.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 65ba8305323..f7c58c2d09f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,8 +18,9 @@ before_build: return $null } } - + Get-ChildItem Env: | Where-Object { $_.Name -like 'APPVEYOR*COMMIT' } | ForEach-Object { "$($_.Name)=$($_.Value)" } $commit = $env:APPVEYOR_REPO_COMMIT + $commit2 = $env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT $branch = $env:APPVEYOR_REPO_BRANCH $latest = $null $mainBranch = $branch -match '^(main|\d+\.\d+)$' @@ -32,8 +33,8 @@ before_build: $mainBranch = $False "Pull Request build detected" } - if ($latest -and ($commit -ne $latest) -and (-not $mainBranch)) { - "Skipping outdated commit $commit (latest is $latest)" + if ($latest -and ($commit -ne $latest) -and ($commit2 -ne $latest) -and (-not $mainBranch)) { + "Skipping outdated commit (latest is $latest)" Exit-AppVeyorBuild } From f699010c0fc570786f6fe271f4dc3b2c84f8521d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2024 14:48:09 +1100 Subject: [PATCH 075/125] json_unescape: don't fill unconverted characters with ? Return an error, and handle let the caller handler the SQL error. --- strings/json_lib.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/strings/json_lib.c b/strings/json_lib.c index 31d5807418a..4d9a9aac594 100644 --- a/strings/json_lib.c +++ b/strings/json_lib.c @@ -1656,15 +1656,7 @@ int json_unescape(CHARSET_INFO *json_cs, } if (c_len == MY_CS_ILUNI) { - /* - Result charset doesn't support the json's character. - Let's replace it with the '?' symbol. - */ - if ((c_len= my_ci_wc_mb(res_cs, '?', res, res_end)) > 0) - { - res+= c_len; - continue; - } + return -1; } /* Result buffer is too small. */ return -1; From ea20948b8a9b007734645f2fe16f712f39762140 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2024 14:49:20 +1100 Subject: [PATCH 076/125] json: escaping/unescaping errors should be handled. Add the attribute warn_unused_result on the functions to ensure they are handled. --- include/json_lib.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/json_lib.h b/include/json_lib.h index 6342e36ab85..7de7f282751 100644 --- a/include/json_lib.h +++ b/include/json_lib.h @@ -387,7 +387,7 @@ int json_find_paths_next(json_engine_t *je, json_find_paths_t *state); Returns negative integer in the case of an error, the length of the result otherwise. */ -int json_unescape(CHARSET_INFO *json_cs, +int __attribute__((warn_unused_result)) json_unescape(CHARSET_INFO *json_cs, const uchar *json_str, const uchar *json_end, CHARSET_INFO *res_cs, uchar *res, uchar *res_end); @@ -401,7 +401,8 @@ int json_unescape(CHARSET_INFO *json_cs, JSON_ERROR_OUT_OF_SPACE Not enough space in the provided buffer JSON_ERROR_ILLEGAL_SYMBOL Source symbol cannot be represented in JSON */ -int json_escape(CHARSET_INFO *str_cs, const uchar *str, const uchar *str_end, +int __attribute__((warn_unused_result)) json_escape(CHARSET_INFO *str_cs, + const uchar *str, const uchar *str_end, CHARSET_INFO *json_cs, uchar *json, uchar *json_end); From 5a536adb03029a1ccdde1fa0b100cd0356929337 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2024 14:58:59 +1100 Subject: [PATCH 077/125] Arg_comparator::compare_{e_,}json_str_basic unescaping warnings Push a warning if the unescaping failed to resolve into the target character set. This uses the ER_JSON_BAD_CHAR, which is normally around functions, but we can't add new error codes so use this as is. Use same args for the error as JSON functions would for this error code. --- sql/item_jsonfunc.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index b2d0646c844..7408c840dd0 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -4123,7 +4123,13 @@ int Arg_comparator::compare_json_str_basic(Item *j, Item *s) &my_charset_utf8mb3_general_ci, (uchar *) value2.ptr(), (uchar *) (value2.ptr() + je.value_len))) < 0) + { + if (current_thd) + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_JSON_BAD_CHR, ER_THD(current_thd, ER_JSON_BAD_CHR), + 0, "comparison", (int)((const char *) je.s.c_str - js->ptr())); goto error; + } value2.length(c_len); js= &value2; @@ -4172,7 +4178,13 @@ int Arg_comparator::compare_e_json_str_basic(Item *j, Item *s) &my_charset_utf8mb3_general_ci, (uchar *) value1.ptr(), (uchar *) (value1.ptr() + value_len))) < 0) + { + if (current_thd) + push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN, + ER_JSON_BAD_CHR, ER_THD(current_thd, ER_JSON_BAD_CHR), + 0, "equality comparison", 0); return 1; + } value1.length(c_len); res1= &value1; } From ccbcafc22e27e8267243cdd6e7b113ff34ce6563 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2024 15:02:46 +1100 Subject: [PATCH 078/125] MDEV-35614: JSON_UNQUOTE doesn't work with emojis emojis are a 4 byte utf sequence. Fix the conversion in JSON_UNQUOTE to utf8mb4_bin by default. --- mysql-test/main/func_json.result | 37 ++++++++++++++++++++ mysql-test/main/func_json.test | 22 ++++++++++++ mysql-test/suite/json/r/json_no_table.result | 2 +- sql/item_jsonfunc.cc | 6 ++-- 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/mysql-test/main/func_json.result b/mysql-test/main/func_json.result index 5b3909b961e..df4685ba9ee 100644 --- a/mysql-test/main/func_json.result +++ b/mysql-test/main/func_json.result @@ -1766,6 +1766,43 @@ FROM JSON_TABLE (@data, '$[*]' COLUMNS (data text PATH '$.Data')) AS t; data # +# MDEV-35614 JSON_UNQUOTE doesn't work with emojis +# +SELECT HEX(JSON_UNQUOTE('"\\ud83d\\ude0a"')) as hex_smiley; +hex_smiley +F09F988A +set names utf8mb4; +SELECT JSON_UNQUOTE('"\\ud83d\\ude0a"') as smiley; +smiley +😊 +SELECT JSON_UNQUOTE('"\\ud83d\\ude0a"') = JSON_UNQUOTE('"\\ud83d\\ude0a"') as equal_smileys; +equal_smileys +1 +SELECT JSON_UNQUOTE('"\\ud83d\\ude0a"') <= JSON_UNQUOTE('"\\ud83d\\ude0a"') as less_or_equal_smileys; +less_or_equal_smileys +1 +set @v='{ "color":"😊" }'; +select @v as v, collation(@v) as collation_v; +v collation_v +{ "color":"😊" } utf8mb4_general_ci +select json_valid(@v) as valid; +valid +1 +select json_extract(@v,'$.color') as color_extraction, collation(json_extract(@v,'$.color')) as color_extraction_collation; +color_extraction color_extraction_collation +"😊" utf8mb4_general_ci +select json_unquote(json_extract(@v,'$.color')) as unquoted, collation(json_unquote(json_extract(@v,'$.color'))) as unquoted_collation; +unquoted unquoted_collation +😊 utf8mb4_bin +SELECT JSON_UNQUOTE('"\\uc080\\ude0a"') as invalid_utf8mb4; +invalid_utf8mb4 +"\uc080\ude0a" +Warnings: +Warning 4035 Broken JSON string in argument 1 to function 'json_unquote' at position 13 +show warnings; +Level Code Message +Warning 4035 Broken JSON string in argument 1 to function 'json_unquote' at position 13 +# # End of 10.6 tests # # diff --git a/mysql-test/main/func_json.test b/mysql-test/main/func_json.test index 8f7a0e1aa66..bdb53be509f 100644 --- a/mysql-test/main/func_json.test +++ b/mysql-test/main/func_json.test @@ -1194,6 +1194,7 @@ SELECT JSON_EXTRACT('{"a": 1,"b": 2}','$.a'); SET @@collation_connection= @save_collation_connection; + --echo # --echo # End of 10.5 tests --echo # @@ -1231,6 +1232,27 @@ SELECT data FROM JSON_TABLE (@data, '$[*]' COLUMNS (data text PATH '$.Data')) AS t; + +--echo # +--echo # MDEV-35614 JSON_UNQUOTE doesn't work with emojis +--echo # + +SELECT HEX(JSON_UNQUOTE('"\\ud83d\\ude0a"')) as hex_smiley; +set names utf8mb4; +SELECT JSON_UNQUOTE('"\\ud83d\\ude0a"') as smiley; + +SELECT JSON_UNQUOTE('"\\ud83d\\ude0a"') = JSON_UNQUOTE('"\\ud83d\\ude0a"') as equal_smileys; +SELECT JSON_UNQUOTE('"\\ud83d\\ude0a"') <= JSON_UNQUOTE('"\\ud83d\\ude0a"') as less_or_equal_smileys; + +set @v='{ "color":"😊" }'; +select @v as v, collation(@v) as collation_v; +select json_valid(@v) as valid; +select json_extract(@v,'$.color') as color_extraction, collation(json_extract(@v,'$.color')) as color_extraction_collation; +select json_unquote(json_extract(@v,'$.color')) as unquoted, collation(json_unquote(json_extract(@v,'$.color'))) as unquoted_collation; + +SELECT JSON_UNQUOTE('"\\uc080\\ude0a"') as invalid_utf8mb4; +show warnings; + --echo # --echo # End of 10.6 tests --echo # diff --git a/mysql-test/suite/json/r/json_no_table.result b/mysql-test/suite/json/r/json_no_table.result index 5819a10ebfa..ab32364b163 100644 --- a/mysql-test/suite/json/r/json_no_table.result +++ b/mysql-test/suite/json/r/json_no_table.result @@ -2886,7 +2886,7 @@ json_unquote(json_compact('["a", "b", "c"]')) ["a", "b", "c"] select charset(json_unquote('"abc"')); charset(json_unquote('"abc"')) -utf8mb3 +utf8mb4 select json_quote(convert(X'e68891' using utf8)); json_quote(convert(X'e68891' using utf8)) "我" diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 7408c840dd0..9fa0d66c47e 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -851,7 +851,7 @@ String *Item_func_json_quote::val_str(String *str) bool Item_func_json_unquote::fix_length_and_dec(THD *thd) { - collation.set(&my_charset_utf8mb3_general_ci, + collation.set(&my_charset_utf8mb4_bin, DERIVATION_COERCIBLE, MY_REPERTOIRE_ASCII); max_length= args[0]->max_char_length() * collation.collation->mbmaxlen; set_maybe_null(); @@ -894,12 +894,12 @@ String *Item_func_json_unquote::val_str(String *str) return js; str->length(0); - str->set_charset(&my_charset_utf8mb3_general_ci); + str->set_charset(&my_charset_utf8mb4_bin); if (str->realloc_with_extra_if_needed(je.value_len) || (c_len= json_unescape(js->charset(), je.value, je.value + je.value_len, - &my_charset_utf8mb3_general_ci, + &my_charset_utf8mb4_bin, (uchar *) str->ptr(), (uchar *) (str->ptr() + je.value_len))) < 0) goto error; From ca144971e1345ed051165ca4c4761d729a166077 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2024 15:14:23 +1100 Subject: [PATCH 079/125] MDEV-35614: json_unescape for comparison uses utf8mb4_bin --- sql/item_jsonfunc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 9fa0d66c47e..b64c502201c 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -4120,7 +4120,7 @@ int Arg_comparator::compare_json_str_basic(Item *j, Item *s) if (value2.realloc_with_extra_if_needed(je.value_len) || (c_len= json_unescape(js->charset(), je.value, je.value + je.value_len, - &my_charset_utf8mb3_general_ci, + &my_charset_utf8mb4_bin, (uchar *) value2.ptr(), (uchar *) (value2.ptr() + je.value_len))) < 0) { @@ -4175,7 +4175,7 @@ int Arg_comparator::compare_e_json_str_basic(Item *j, Item *s) if (value1.realloc_with_extra_if_needed(value_len) || (c_len= json_unescape(value1.charset(), (uchar *) value, (uchar *) value+value_len, - &my_charset_utf8mb3_general_ci, + &my_charset_utf8mb4_bin, (uchar *) value1.ptr(), (uchar *) (value1.ptr() + value_len))) < 0) { From 7d9660ed93c6bcf56a6dafd0872b27ac1709778a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 12 Dec 2024 14:55:56 +1100 Subject: [PATCH 080/125] item_json*: handle memory allocations JSON functions append in multiple ways, however there isn't always error handling, and many time it doesn't make it to the end user. Made the appending string functions withing item_jsonfunc warn if their true/false result (did an error occur) isn't handled. Add error handling to many json functions. realloc_with_extra_if_needed was also previously lacking OOM handing. --- sql/item_jsonfunc.cc | 92 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 18 deletions(-) diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index b64c502201c..998bc9d8262 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -74,7 +74,8 @@ static bool eq_ascii_string(const CHARSET_INFO *cs, } -static bool append_simple(String *s, const char *a, size_t a_len) +static bool __attribute__((warn_unused_result)) +append_simple(String *s, const char *a, size_t a_len) { if (!s->realloc_with_extra_if_needed(s->length() + a_len)) { @@ -86,7 +87,8 @@ static bool append_simple(String *s, const char *a, size_t a_len) } -static inline bool append_simple(String *s, const uchar *a, size_t a_len) +static inline bool __attribute__((warn_unused_result)) +append_simple(String *s, const uchar *a, size_t a_len) { return append_simple(s, (const char *) a, a_len); } @@ -300,8 +302,10 @@ static int json_nice(json_engine_t *je, String *nice_js, nice_js->length(0); nice_js->set_charset(je->s.cs); - nice_js->alloc(je->s.str_end - je->s.c_str + 32); + if (nice_js->alloc(je->s.str_end - je->s.c_str + 32)) + goto error; + DBUG_ASSERT(mode != Item_func_json_format::DETAILED || (tab_size >= 0 && tab_size <= TAB_SIZE_LIMIT)); @@ -347,7 +351,8 @@ static int json_nice(json_engine_t *je, String *nice_js, goto error; nice_js->append('"'); - append_simple(nice_js, key_start, key_end - key_start); + if (append_simple(nice_js, key_start, key_end - key_start)) + goto error; nice_js->append(colon, colon_len); } /* now we have key value to handle, so no 'break'. */ @@ -2248,24 +2253,67 @@ String *Item_func_json_array_insert::val_str(String *str) str->set_charset(js->charset()); if (item_pos) { - if (append_simple(str, js->ptr(), item_pos - js->ptr()) || - (n_item > 0 && str->append(" ", 1)) || - append_json_value(str, args[n_arg+1], &tmp_val) || - str->append(",", 1) || - (n_item == 0 && str->append(" ", 1)) || - append_simple(str, item_pos, js->end() - item_pos)) + my_ptrdiff_t size= item_pos - js->ptr(); + if (append_simple(str, js->ptr(), size)) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int) size); goto return_null; /* Out of memory. */ + } + if (n_item > 0 && str->append(" ", 1)) + { + my_error(ER_OUTOFMEMORY, MYF(0), 1); + goto return_null; /* Out of memory. */ + } + if (append_json_value(str, args[n_arg+1], &tmp_val)) + { + my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); + goto return_null; /* Out of memory. */ + } + if (str->append(",", 1)) + { + my_error(ER_OUTOFMEMORY, MYF(0), 1); + goto return_null; /* Out of memory. */ + } + if (n_item == 0 && str->append(" ", 1)) + { + my_error(ER_OUTOFMEMORY, MYF(0), 1); + goto return_null; /* Out of memory. */ + } + size= js->end() - item_pos; + if (append_simple(str, item_pos, size)) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int) size); + goto return_null; /* Out of memory. */ + } } else { + my_ptrdiff_t size; /* Insert position wasn't found - append to the array. */ DBUG_ASSERT(je.state == JST_ARRAY_END); item_pos= (const char *) (je.s.c_str - je.sav_c_len); - if (append_simple(str, js->ptr(), item_pos - js->ptr()) || - (n_item > 0 && str->append(", ", 2)) || - append_json_value(str, args[n_arg+1], &tmp_val) || - append_simple(str, item_pos, js->end() - item_pos)) + size= item_pos - js->ptr(); + if (append_simple(str, js->ptr(), size)) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int) size); goto return_null; /* Out of memory. */ + } + if (n_item > 0 && str->append(", ", 2)) + { + my_error(ER_OUTOFMEMORY, MYF(0), 2); + goto return_null; /* Out of memory. */ + } + if (append_json_value(str, args[n_arg+1], &tmp_val)) + { + my_error(ER_OUTOFMEMORY, MYF(0), tmp_val.length()); + goto return_null; /* Out of memory. */ + } + size= js->end() - item_pos; + if (append_simple(str, item_pos, size)) + { + my_error(ER_OUTOFMEMORY, MYF(0), (int) size); + goto return_null; /* Out of memory. */ + } } { @@ -4117,8 +4165,12 @@ int Arg_comparator::compare_json_str_basic(Item *j, Item *s) goto error; if (je.value_type == JSON_VALUE_STRING) { - if (value2.realloc_with_extra_if_needed(je.value_len) || - (c_len= json_unescape(js->charset(), je.value, + if (value2.realloc_with_extra_if_needed(je.value_len)) + { + my_error(ER_OUTOFMEMORY, MYF(0), je.value_len); + goto error; + } + if ((c_len= json_unescape(js->charset(), je.value, je.value + je.value_len, &my_charset_utf8mb4_bin, (uchar *) value2.ptr(), @@ -4172,8 +4224,12 @@ int Arg_comparator::compare_e_json_str_basic(Item *j, Item *s) if (type == JSON_VALUE_STRING) { - if (value1.realloc_with_extra_if_needed(value_len) || - (c_len= json_unescape(value1.charset(), (uchar *) value, + if (value1.realloc_with_extra_if_needed(value_len)) + { + my_error(ER_OUTOFMEMORY, MYF(0), value_len); + return 1; + } + if ((c_len= json_unescape(value1.charset(), (uchar *) value, (uchar *) value+value_len, &my_charset_utf8mb4_bin, (uchar *) value1.ptr(), From 8c6b0d092ad2215ef0a1ac17f0ac7e3d087556d3 Mon Sep 17 00:00:00 2001 From: Tony Chen Date: Thu, 6 Mar 2025 05:27:48 +0000 Subject: [PATCH 081/125] MDEV-36245 Long server_audit_file_path causes buffer overflow Limit size of server_audit_file_path value Currently, the length of this value is not checked and can cause a buffer overflow if given a long file path specifying a directory. In file_logger:logger_open(), there is a check: ``` if (new_log.path_len+n_dig(rotations)+1 > FN_REFLEN) // handle error ``` As n_dig(rotations) may return up to 3, this inherently limits the file path to FN_REFLEN - 4 characters. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc. --- plugin/server_audit/server_audit.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index 8b49f354f47..183295773f5 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -2836,6 +2836,14 @@ static void update_file_path(MYSQL_THD thd, { char *new_name= (*(char **) save) ? *(char **) save : empty_str; + if (strlen(new_name) + 4 > FN_REFLEN) + { + error_header(); + fprintf(stderr, "server_audit_file_path can't exceed %d characters.\n", FN_REFLEN - 4); + CLIENT_ERROR(1, "server_audit_file_path can't exceed %d characters.\n", MYF(ME_WARNING), FN_REFLEN - 4); + return; + } + ADD_ATOMIC(internal_stop_logging, 1); error_header(); fprintf(stderr, "Log file name was changed to '%s'.\n", new_name); From 51c5b753356d31bceadbf1025117278946011635 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 13 Mar 2025 17:29:22 +0200 Subject: [PATCH 082/125] Always call mysql_cond_broadcast(&rli->data_cond) under data_lock This is a safetly fix to try to fix random failures in parallel_backup_xa_debug reported as: sync_slave_with_master failed: 'select master_pos_wait('master-bin.000001', 1034, 300, '')' returned -1 One possible reason could be lost signals, which this patch fixes. --- sql/rpl_parallel.cc | 2 +- sql/slave.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 502a0501553..167d3dc19a0 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -124,8 +124,8 @@ handle_queued_pos_update(THD *thd, rpl_parallel_thread::queued_event *qev) else if (cmp == 0 && rli->group_master_log_pos < qev->future_event_master_log_pos) rli->group_master_log_pos= qev->future_event_master_log_pos; - mysql_mutex_unlock(&rli->data_lock); mysql_cond_broadcast(&rli->data_cond); + mysql_mutex_unlock(&rli->data_lock); } diff --git a/sql/slave.cc b/sql/slave.cc index 7f7db020aa1..de1476e5925 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -5897,9 +5897,9 @@ err_during_init: rli->relay_log.description_event_for_exec= 0; rli->reset_inuse_relaylog(); /* Wake up master_pos_wait() */ - mysql_mutex_unlock(&rli->data_lock); DBUG_PRINT("info",("Signaling possibly waiting master_pos_wait() functions")); mysql_cond_broadcast(&rli->data_cond); + mysql_mutex_unlock(&rli->data_lock); rli->ignore_log_space_limit= 0; /* don't need any lock */ /* we die so won't remember charset - re-update them on next thread start */ thd->system_thread_info.rpl_sql_info->cached_charset_invalidate(); From 35c25cd10729631caccb07ed20ef9b5ba08a45bb Mon Sep 17 00:00:00 2001 From: Luis Contreras Date: Thu, 3 Apr 2025 18:56:25 +0300 Subject: [PATCH 083/125] MDEV-36412 Concerns compilation issue on community edition for x86_64 with X32 ABI --- include/my_stack_alloc.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/my_stack_alloc.h b/include/my_stack_alloc.h index 796bb69f877..59d9c0da57f 100644 --- a/include/my_stack_alloc.h +++ b/include/my_stack_alloc.h @@ -38,6 +38,8 @@ static inline void *my_get_stack_pointer(void *default_stack) #if defined(__GNUC__) || defined(__clang__) /* GCC and Clang compilers */ #if defined(__i386__) /* Intel x86 (32-bit) */ __asm__ volatile ("movl %%esp, %0" : "=r" (stack_ptr)); +#elif defined(__x86_64__) && defined (__ILP32__) /* Intel x86-64 (64-bit), X32 ABI */ + __asm__ volatile ("movl %%esp, %0" : "=r" (stack_ptr)); #elif defined(__x86_64__) /* Intel x86-64 (64-bit) */ __asm__ volatile ("movq %%rsp, %0" : "=r" (stack_ptr)); #elif defined(__powerpc__) /* PowerPC (32-bit) */ From fbec528cbb577744a4341602926113c3fe48c080 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Thu, 6 Mar 2025 05:27:48 +0000 Subject: [PATCH 084/125] MDEV-36245 review changes Closes #3874 --- mysql-test/suite/plugins/r/server_audit.result | 3 +++ mysql-test/suite/plugins/t/server_audit.test | 4 ++++ plugin/server_audit/server_audit.c | 8 ++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/plugins/r/server_audit.result b/mysql-test/suite/plugins/r/server_audit.result index fc9d98f5d0a..06b60448942 100644 --- a/mysql-test/suite/plugins/r/server_audit.result +++ b/mysql-test/suite/plugins/r/server_audit.result @@ -20,6 +20,9 @@ set global server_audit_file_path=null; set global server_audit_incl_users=null; set global server_audit_file_path='server_audit.log'; set global server_audit_output_type=file; +set global server_audit_file_path=REPEAT(REPEAT('new_file_name', 50), 50); +Warnings: +Warning 1 server_audit_file_path can't exceed FN_LEN characters. set global server_audit_logging=on; set global server_audit_incl_users= repeat("'root',", 10000); ERROR 42000: Variable 'server_audit_incl_users' can't be set to the value of ''root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','root','...' diff --git a/mysql-test/suite/plugins/t/server_audit.test b/mysql-test/suite/plugins/t/server_audit.test index 47d6fdacdcf..673bc7ff221 100644 --- a/mysql-test/suite/plugins/t/server_audit.test +++ b/mysql-test/suite/plugins/t/server_audit.test @@ -20,6 +20,10 @@ set global server_audit_file_path=null; set global server_audit_incl_users=null; set global server_audit_file_path='server_audit.log'; set global server_audit_output_type=file; + +--replace_regex /[1-9][0-9][0-9]+/FN_LEN/ +set global server_audit_file_path=REPEAT(REPEAT('new_file_name', 50), 50); + set global server_audit_logging=on; --error ER_WRONG_VALUE_FOR_VAR diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index 183295773f5..5c2c65c4161 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -2839,8 +2839,12 @@ static void update_file_path(MYSQL_THD thd, if (strlen(new_name) + 4 > FN_REFLEN) { error_header(); - fprintf(stderr, "server_audit_file_path can't exceed %d characters.\n", FN_REFLEN - 4); - CLIENT_ERROR(1, "server_audit_file_path can't exceed %d characters.\n", MYF(ME_WARNING), FN_REFLEN - 4); + fprintf(stderr, + "server_audit_file_path can't exceed %d characters.\n", + FN_REFLEN - 4); + fprintf(stderr, "Log filename remains unchanged '%s'.\n", file_path); + CLIENT_ERROR(1, "server_audit_file_path can't exceed %d characters.", + MYF(ME_WARNING), FN_REFLEN - 4); return; } From c76d17a917fee7b10cfe801670eb1a5ef785f8ce Mon Sep 17 00:00:00 2001 From: Dmitry Shulga Date: Thu, 27 Mar 2025 18:02:37 +0700 Subject: [PATCH 085/125] MDEV-36390: Minor refactoring of the method get_expr_query at the classes sp_instr_cpush/sp_instr_cursor_copy_struct Duplicated code from methods sp_instr_cpush::get_expr_query sp_instr_cursor_copy_struct::get_expr_query were extracted to the standalone function get_cursor_query Additionally, added correct parsing of a cursor definition with a new line or TAB instead of the space immediately after FOR/IS. --- mysql-test/main/sp_validation.result | 92 ++++++++++++++++++++++++++++ mysql-test/main/sp_validation.test | 91 +++++++++++++++++++++++++++ sql/sp_instr.h | 48 +++++++++------ 3 files changed, 211 insertions(+), 20 deletions(-) diff --git a/mysql-test/main/sp_validation.result b/mysql-test/main/sp_validation.result index 9ef644482d5..0808fd7cd78 100644 --- a/mysql-test/main/sp_validation.result +++ b/mysql-test/main/sp_validation.result @@ -1993,4 +1993,96 @@ f1() # Clean up DROP FUNCTION f1; DROP TABLE t1; +# +# MDEV-36390: Minor refactoring of the method get_expr_query at the classes sp_instr_cpush/sp_instr_cursor_copy_struct +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1); +CREATE OR REPLACE PROCEDURE p1() +BEGIN +DECLARE va INT; +# Check that the TAB character after the clause FOR is skipped and +# the body of cursor is remembered correctly for subsequent re-parsing +DECLARE cur CURSOR FOR SELECT a FROM t1; +OPEN cur; +FETCH cur INTO va; +SELECT va; +CLOSE cur; +END; +$ +CREATE OR REPLACE PROCEDURE p2() +BEGIN +DECLARE va INT; +# Check that the newline character after the clause FOR is skipped and +# the body of cursor is remembered correctly for subsequent re-parsing +DECLARE cur CURSOR FOR +SELECT a FROM t1; +OPEN cur; +FETCH cur INTO va; +SELECT va; +CLOSE cur; +END; +$ +CREATE OR REPLACE PROCEDURE p3() +BEGIN +DECLARE va INT; +# Check that C-style comment and the newline character after +# the clause FOR is skipped and the body of cursor is remembered +# correctly for subsequent re-parsing +DECLARE cur CURSOR FOR /* Explicit comment */ +SELECT a FROM t1; +OPEN cur; +FETCH cur INTO va; +SELECT va; +CLOSE cur; +END; +$ +CREATE OR REPLACE PROCEDURE p4() +BEGIN +DECLARE va INT; +# Check that SQL-style comment and the newline character after +# the clause FOR is skipped and the body of cursor is remembered +# correctly for subsequent re-parsing +DECLARE cur CURSOR FOR -- Explicit comment +SELECT a FROM t1; +OPEN cur; +FETCH cur INTO va; +SELECT va; +CLOSE cur; +END; +$ +CALL p1(); +va +1 +CALL p2(); +va +1 +CALL p3(); +va +1 +CALL p4(); +va +1 +ALTER TABLE t1 COMMENT 'The Comment 1'; +# The following statements will run re-parsing of +# cursor declaration statements inside the stored +# procedures p1, p2, p3, p4. +CALL p1(); +va +1 +CALL p2(); +va +1 +CALL p3(); +va +1 +CALL p4(); +va +1 +# Clean up +DROP PROCEDURE p1; +DROP PROCEDURE p2; +DROP PROCEDURE p3; +DROP PROCEDURE p4; +DROP TABLE t1; SET sql_mode = default; diff --git a/mysql-test/main/sp_validation.test b/mysql-test/main/sp_validation.test index 6f095710e2c..bf1f147deec 100644 --- a/mysql-test/main/sp_validation.test +++ b/mysql-test/main/sp_validation.test @@ -2795,5 +2795,96 @@ SELECT f1(); DROP FUNCTION f1; DROP TABLE t1; +--echo # +--echo # MDEV-36390: Minor refactoring of the method get_expr_query at the classes sp_instr_cpush/sp_instr_cursor_copy_struct +--echo # +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1); + +--delimiter $ + +CREATE OR REPLACE PROCEDURE p1() +BEGIN + DECLARE va INT; + # Check that the TAB character after the clause FOR is skipped and + # the body of cursor is remembered correctly for subsequent re-parsing + DECLARE cur CURSOR FOR SELECT a FROM t1; + + OPEN cur; + FETCH cur INTO va; + SELECT va; + CLOSE cur; +END; +$ + +CREATE OR REPLACE PROCEDURE p2() +BEGIN + DECLARE va INT; + # Check that the newline character after the clause FOR is skipped and + # the body of cursor is remembered correctly for subsequent re-parsing + DECLARE cur CURSOR FOR + SELECT a FROM t1; + + OPEN cur; + FETCH cur INTO va; + SELECT va; + CLOSE cur; +END; +$ + +CREATE OR REPLACE PROCEDURE p3() +BEGIN + DECLARE va INT; + # Check that C-style comment and the newline character after + # the clause FOR is skipped and the body of cursor is remembered + # correctly for subsequent re-parsing + DECLARE cur CURSOR FOR /* Explicit comment */ + SELECT a FROM t1; + + OPEN cur; + FETCH cur INTO va; + SELECT va; + CLOSE cur; +END; +$ + +CREATE OR REPLACE PROCEDURE p4() +BEGIN + DECLARE va INT; + # Check that SQL-style comment and the newline character after + # the clause FOR is skipped and the body of cursor is remembered + # correctly for subsequent re-parsing + DECLARE cur CURSOR FOR -- Explicit comment + SELECT a FROM t1; + + OPEN cur; + FETCH cur INTO va; + SELECT va; + CLOSE cur; +END; +$ + +--delimiter ; + +CALL p1(); +CALL p2(); +CALL p3(); +CALL p4(); +ALTER TABLE t1 COMMENT 'The Comment 1'; +--echo # The following statements will run re-parsing of +--echo # cursor declaration statements inside the stored +--echo # procedures p1, p2, p3, p4. +CALL p1(); +CALL p2(); +CALL p3(); +CALL p4(); + +--echo # Clean up +DROP PROCEDURE p1; +DROP PROCEDURE p2; +DROP PROCEDURE p3; +DROP PROCEDURE p4; +DROP TABLE t1; + SET sql_mode = default; --enable_ps2_protocol diff --git a/sql/sp_instr.h b/sql/sp_instr.h index 5d5af0a18f0..3bd2963648a 100644 --- a/sql/sp_instr.h +++ b/sql/sp_instr.h @@ -1227,6 +1227,32 @@ public: }; // class sp_instr_hreturn : public sp_instr_jump +/** + Get a query text associated with the cursor. +*/ + +static inline LEX_CSTRING get_cursor_query(const LEX_CSTRING &cursor_stmt) +{ + /* + Lexer on processing the clause CURSOR FOR / CURSOR IS doesn't + move a pointer on cpp_buf after the token FOR/IS so skip it explicitly + in order to get correct value of cursor's query string. + */ + + if (strncasecmp(cursor_stmt.str, "FOR", 3) == 0 && + my_isspace(current_thd->variables.character_set_client, + cursor_stmt.str[3])) + return LEX_CSTRING{cursor_stmt.str + 4, cursor_stmt.length - 4}; + + if (strncasecmp(cursor_stmt.str, "IS", 2) == 0 && + my_isspace(current_thd->variables.character_set_client, + cursor_stmt.str[2])) + return LEX_CSTRING{cursor_stmt.str + 3, cursor_stmt.length - 3}; + + return cursor_stmt; +} + + /** This is DECLARE CURSOR */ @@ -1287,16 +1313,7 @@ public: protected: LEX_CSTRING get_expr_query() const override { - /* - Lexer on processing the clause CURSOR FOR / CURSOR IS doesn't - move a pointer on cpp_buf after the token FOR/IS so skip it explicitly - in order to get correct value of cursor's query string. - */ - if (strncasecmp(m_cursor_stmt.str, "FOR ", 4) == 0) - return LEX_CSTRING{m_cursor_stmt.str + 4, m_cursor_stmt.length - 4}; - if (strncasecmp(m_cursor_stmt.str, "IS ", 3) == 0) - return LEX_CSTRING{m_cursor_stmt.str + 3, m_cursor_stmt.length - 3}; - return m_cursor_stmt; + return get_cursor_query(m_cursor_stmt); } bool on_after_expr_parsing(THD *) override @@ -1428,16 +1445,7 @@ public: protected: LEX_CSTRING get_expr_query() const override { - /* - Lexer on processing the clause CURSOR FOR / CURSOR IS doesn't - move a pointer on cpp_buf after the token FOR/IS so skip it explicitly - in order to get correct value of cursor's query string. - */ - if (strncasecmp(m_cursor_stmt.str, "FOR ", 4) == 0) - return LEX_CSTRING{m_cursor_stmt.str + 4, m_cursor_stmt.length - 4}; - if (strncasecmp(m_cursor_stmt.str, "IS ", 3) == 0) - return LEX_CSTRING{m_cursor_stmt.str + 3, m_cursor_stmt.length - 3}; - return m_cursor_stmt; + return get_cursor_query(m_cursor_stmt); } bool on_after_expr_parsing(THD *) override From 5f6b92a738a4380a46494206bf1ceec2999053c5 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Sun, 20 Apr 2025 10:01:52 +0200 Subject: [PATCH 086/125] New CC --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index 7d930974c0c..867f0d18253 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit 7d930974c0c5588a8872f30b83d6bc429106f825 +Subproject commit 867f0d18253f48eb5bead9e6636ec06824017aeb From f844f65492c2298bd63067a5af73cba45c5e0761 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 20 Apr 2025 12:56:15 +0300 Subject: [PATCH 087/125] mtr backup tests should not read local .my.cnf files Haviong loose-skip-ssl in ~/.my.cnf caused tests to fail --- mysql-test/suite/mariabackup/backup_ssl.test | 4 ++-- mysql-test/suite/mariabackup/backup_ssl_system_ca.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/mariabackup/backup_ssl.test b/mysql-test/suite/mariabackup/backup_ssl.test index c38efc885c6..1ae0b287a88 100644 --- a/mysql-test/suite/mariabackup/backup_ssl.test +++ b/mysql-test/suite/mariabackup/backup_ssl.test @@ -20,12 +20,12 @@ echo #; # fails to connect, passwordless root echo # tcp ssl ssl-verify-server-cert; error 1; -exec $XTRABACKUP --protocol=tcp --user=root --port=$MASTER_MYPORT --backup --target-dir=$targetdir; +exec $XTRABACKUP --no-defaults --protocol=tcp --user=root --port=$MASTER_MYPORT --backup --target-dir=$targetdir; --echo # --echo # MDEV-32473 --disable-ssl doesn't disable it --echo # # connects fine echo # tcp skip-ssl; -exec $XTRABACKUP --protocol=tcp --user=root --skip-ssl --port=$MASTER_MYPORT --backup --target-dir=$targetdir; +exec $XTRABACKUP --no-defaults --protocol=tcp --user=root --skip-ssl --port=$MASTER_MYPORT --backup --target-dir=$targetdir; rmdir $targetdir; diff --git a/mysql-test/suite/mariabackup/backup_ssl_system_ca.test b/mysql-test/suite/mariabackup/backup_ssl_system_ca.test index 18db74fe431..b1604c762d7 100644 --- a/mysql-test/suite/mariabackup/backup_ssl_system_ca.test +++ b/mysql-test/suite/mariabackup/backup_ssl_system_ca.test @@ -20,11 +20,11 @@ rmdir $targetdir; echo # tcp, not self-signed cert with a wrong hostname: fails; error 1; -exec $XTRABACKUP --protocol=tcp --user=root --port=$MASTER_MYPORT --backup --target-dir=$targetdir; +exec $XTRABACKUP --no-defaults --protocol=tcp --user=root --port=$MASTER_MYPORT --backup --target-dir=$targetdir; echo # tcp, not self-signed cert with a wrong hostname: fails even with a password (no auto-verification); error 1; -exec $XTRABACKUP --protocol=tcp --user=backup_user --password=x --port=$MASTER_MYPORT --backup --target-dir=$targetdir; +exec $XTRABACKUP --no-defaults --protocol=tcp --user=backup_user --password=x --port=$MASTER_MYPORT --backup --target-dir=$targetdir; remove_file $MYSQL_TMP_DIR/ed1f42db.0; DROP USER backup_user; From 952ffb55f93bf6a6c5d1c9617e5a2a56207ee674 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Sun, 20 Apr 2025 19:01:49 +0200 Subject: [PATCH 088/125] Fix valgrind detection --- mysql-test/main/sp-no-valgrind.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/main/sp-no-valgrind.test b/mysql-test/main/sp-no-valgrind.test index 10a238f9871..72ef9fb36a9 100644 --- a/mysql-test/main/sp-no-valgrind.test +++ b/mysql-test/main/sp-no-valgrind.test @@ -1,5 +1,5 @@ --source include/not_msan.inc ---source include/not_valgrind_build.inc +--source include/not_valgrind.inc --echo # MDEV-20699 do not cache SP in SHOW CREATE --echo # Warmup round, this might allocate some memory for session variable From 236dec69b71c49d72b6aacf978f9728b2aed72a2 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Sun, 20 Apr 2025 20:48:21 +0200 Subject: [PATCH 089/125] new CC --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index a7ad25b01bd..55abb320382 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit a7ad25b01bd7716d24181657abd4fb203a883371 +Subproject commit 55abb3203826a7b3593f0728d6d077d4e0f19259 From 616fb6831df1d78016b8a38e575ac594c4477a8e Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 16 Apr 2025 09:09:43 +0200 Subject: [PATCH 090/125] new libfmt --- cmake/libfmt.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/libfmt.cmake b/cmake/libfmt.cmake index 7fded49b916..47f2e00bae1 100644 --- a/cmake/libfmt.cmake +++ b/cmake/libfmt.cmake @@ -15,8 +15,8 @@ MACRO(BUNDLE_LIBFMT) ExternalProject_Add( libfmt PREFIX "${dir}" - URL "https://github.com/fmtlib/fmt/releases/download/11.0.2/fmt-11.0.2.zip" - URL_MD5 c622dca45ec3fc95254c48370a9f7a1d + URL "https://github.com/fmtlib/fmt/releases/download/11.1.4/fmt-11.1.4.zip" + URL_MD5 ad6a56b15cddf4aad2a234e7cfc9e8c9 INSTALL_COMMAND "" CONFIGURE_COMMAND "" BUILD_COMMAND "" From 820114bd2515759224f925528c8201e6155334fb Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Sun, 20 Apr 2025 11:06:52 +0200 Subject: [PATCH 091/125] new CC --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index ba712ddf961..b5a2c9f3c27 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit ba712ddf9611304e8dfc12e829ad899222d1b4cb +Subproject commit b5a2c9f3c275861447ca21ee1f01560135ec6c2f From a96c094d1ba59f793ac50dbcc4b1c37c3923f3d3 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Mon, 14 Apr 2025 18:49:39 +0300 Subject: [PATCH 092/125] MDEV-25012 Server crash in find_field_in_tables, Assertion `name' failed in find_field_in_table_ref The main crash with segfault in find_field_in_tables() was fixed by 6aa47fae304 (MDEV-35276). This fix is for debug assertion. Since Item_default_value is also Item_field there is nothing to be done except adding DEFAULT_VALUE_ITEM type check. --- mysql-test/main/derived_cond_pushdown.result | 21 +++++++++++++++++++ mysql-test/main/derived_cond_pushdown.test | 22 ++++++++++++++++++++ sql/sql_lex.cc | 3 ++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/derived_cond_pushdown.result b/mysql-test/main/derived_cond_pushdown.result index 63d57f2d1a1..37b7cdc7404 100644 --- a/mysql-test/main/derived_cond_pushdown.result +++ b/mysql-test/main/derived_cond_pushdown.result @@ -19132,4 +19132,25 @@ FROM cte2 GROUP BY 1 ; ( SELECT 1 FROM ( SELECT 1 FROM cte1) dt GROUP BY x HAVING x= 1 ) 1 +create table t1 (f int); +create view v1 as select f, count(*) c from t1 group by f; +# +# MDEV-25012 Server crash in find_field_in_tables, Assertion `name' failed in find_field_in_table_ref +# +select * from v1 where export_set(1, default(f), 'x', aes_decrypt('secret', f)); +f c +show warnings; +Level Code Message +drop view v1; +drop table t1; +create table t(c3 longtext) ; +with cte1 as +( +select default(c3) as a +from t group by 1 +) +select * from cte1 +where cte1.a >= 1; +a +drop table t; # End of 10.5 tests diff --git a/mysql-test/main/derived_cond_pushdown.test b/mysql-test/main/derived_cond_pushdown.test index 9a09e398b97..3312ee73aba 100644 --- a/mysql-test/main/derived_cond_pushdown.test +++ b/mysql-test/main/derived_cond_pushdown.test @@ -4270,4 +4270,26 @@ SELECT FROM cte2 GROUP BY 1 ; +create table t1 (f int); +create view v1 as select f, count(*) c from t1 group by f; + +--echo # +--echo # MDEV-25012 Server crash in find_field_in_tables, Assertion `name' failed in find_field_in_table_ref +--echo # +select * from v1 where export_set(1, default(f), 'x', aes_decrypt('secret', f)); +show warnings; +# cleanup +drop view v1; +drop table t1; + +create table t(c3 longtext) ; +with cte1 as +( + select default(c3) as a + from t group by 1 +) +select * from cte1 +where cte1.a >= 1; +drop table t; + --echo # End of 10.5 tests diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 023da3567e8..eb57cb8050b 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -11064,7 +11064,8 @@ st_select_lex::build_pushable_cond_for_having_pushdown(THD *thd, Item *cond) Field_pair *get_corresponding_field_pair(Item *item, List pair_list) { - DBUG_ASSERT(item->type() == Item::FIELD_ITEM || + DBUG_ASSERT(item->type() == Item::DEFAULT_VALUE_ITEM || + item->type() == Item::FIELD_ITEM || (item->type() == Item::REF_ITEM && ((((Item_ref *) item)->ref_type() == Item_ref::VIEW_REF) || (((Item_ref *) item)->ref_type() == Item_ref::REF)))); From 837fad6e41722d9b2b16fa263a2f2c08daea1118 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Mon, 14 Apr 2025 15:50:40 +0300 Subject: [PATCH 093/125] MDEV-31647 Stack looping and SIGSEGV in Item_args::walk_args on UPDATE The problem is with window function which requires its own sorting but it is impossible as table is updated while select is progressing. That's it, multi-update queries do not use temporary tables to store updated rows and substitute them at the end of the query. Instead, updates are performed directly on the target table, row by row, during query execution. MariaDB processes updates in a way that ensures each row is updated only once, even if it matches multiple conditions in the query. This behavior avoids redundant updates and does not require intermediate storage in a temporary table. The detailed cause of the loop invoked by window function was explained by Yuchen Pei in MDEV-31647 comments. The fix disables window functions for multi-update. Note that MySQL throws ER_UPDATE_TABLE_USED in that case which is the result of check_unique_table(). But this function is not used for multi-update in MariaDB and this check cannot be done because some level of SELECT expressions is allowed (MDEV-13911). --- mysql-test/main/multi_update.result | 20 ++++++++++++++++++++ mysql-test/main/multi_update.test | 28 ++++++++++++++++++++++++++++ sql/sql_update.cc | 3 ++- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/multi_update.result b/mysql-test/main/multi_update.result index 222c592cbce..f0fe16ee38a 100644 --- a/mysql-test/main/multi_update.result +++ b/mysql-test/main/multi_update.result @@ -1371,3 +1371,23 @@ c1 c2 c3 12 5 8 drop table t1,t2,t3,t; # End of 10.4 tests +# +# MDEV-31647 Stack looping and SIGSEGV in Item_args::walk_args on UPDATE +# +create table t1 (c int, c2 int) engine=innodb; +update t1 set c=0 where c=( +select 1 from (select 1 as v1) as v2 +natural join t1) order by last_value (c2) over (order by c2); +ERROR HY000: Invalid use of group function +update t1 set c=0 where c=( +select 1 from (select 1 as v1) as v2 +natural join t1) order by last_value (c2) over (); +ERROR HY000: Invalid use of group function +update t1 set c=0 where c=( +select 1 from (select 1 as v1) as v2 +natural join t1) order by c2; +select 1 from (select 1 as v1) as v2 +natural join t1 order by last_value (c2) over (order by c2); +1 +drop table t1; +# End of 10.5 tests diff --git a/mysql-test/main/multi_update.test b/mysql-test/main/multi_update.test index 329394e8468..3392e076cb2 100644 --- a/mysql-test/main/multi_update.test +++ b/mysql-test/main/multi_update.test @@ -1200,3 +1200,31 @@ select * from t1; drop table t1,t2,t3,t; --echo # End of 10.4 tests + +--echo # +--echo # MDEV-31647 Stack looping and SIGSEGV in Item_args::walk_args on UPDATE +--echo # +--source include/have_innodb.inc +create table t1 (c int, c2 int) engine=innodb; + +--error ER_INVALID_GROUP_FUNC_USE +update t1 set c=0 where c=( + select 1 from (select 1 as v1) as v2 + natural join t1) order by last_value (c2) over (order by c2); + +--error ER_INVALID_GROUP_FUNC_USE +update t1 set c=0 where c=( + select 1 from (select 1 as v1) as v2 + natural join t1) order by last_value (c2) over (); + +update t1 set c=0 where c=( + select 1 from (select 1 as v1) as v2 + natural join t1) order by c2; + +select 1 from (select 1 as v1) as v2 + natural join t1 order by last_value (c2) over (order by c2); + + +drop table t1; + +--echo # End of 10.5 tests diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 1327bd786c6..fe5e24b6f90 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -2334,7 +2334,8 @@ multi_update::initialize_tables(JOIN *join) if (unlikely((thd->variables.option_bits & OPTION_SAFE_UPDATES) && error_if_full_join(join))) DBUG_RETURN(1); - if (join->implicit_grouping) + if (join->implicit_grouping || + join->select_lex->have_window_funcs()) { my_error(ER_INVALID_GROUP_FUNC_USE, MYF(0)); DBUG_RETURN(1); From 588f7a5af7e25bb4e2a00bea22b67308ad1ef1a6 Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Mon, 13 Jan 2025 11:17:55 -0700 Subject: [PATCH 094/125] MDEV-35694: Mysqlbinlog --stop-position should warn if EOF not reached with --read-from-remote-server MDEV-27037 added functionality to warn users that a specified stop-position or stop-datetime was never reached. It only worked for local files though. The patch in MDEV-35528 changed the implementation for stop-datetime to work to provide the warning also when using --read-from-remote-server. The PR for that MDEV (#3670) was limited to only the stop-datetime field. This patch updates the --stop-position warning to also work with --read-from-remote-server. Reviewed By: ============ Andrei Elkin Jimmy Hu --- client/mysqlbinlog.cc | 61 ++++++++++++++----- ...nlog_mysqlbinlog_warn_stop_position.result | 45 ++++++++++++++ ...binlog_mysqlbinlog_warn_stop_position.test | 13 ++-- 3 files changed, 97 insertions(+), 22 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 77f74e0d42c..b06b8fb74e5 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -160,7 +160,13 @@ static Server_gtid_event_filter *server_id_gtid_filter= NULL; static char *start_datetime_str, *stop_datetime_str; static my_time_t start_datetime= 0, stop_datetime= MY_TIME_T_MAX; -static my_time_t last_processed_datetime= MY_TIME_T_MAX; + +typedef struct _last_processed_ev_t +{ + ulonglong position; + my_time_t datetime; +} last_processed_ev_t; +static last_processed_ev_t last_processed_ev= {0, MY_TIME_T_MAX}; static ulonglong rec_count= 0; static MYSQL* mysql = NULL; @@ -1611,7 +1617,19 @@ err: end: rec_count++; end_skip_count: - last_processed_datetime= ev_when; + /* + Update the last_processed_ev, unless the event is a fake event (i.e. format + description (ev pointer is reset to 0) or rotate event (ev->when is 0)), or + the event is encrypted (i.e. type is Unknown). + */ + if (ev && + !(ev_type == UNKNOWN_EVENT && + ((Unknown_log_event *) ev)->what == Unknown_log_event::ENCRYPTED) && + !(ev_type == ROTATE_EVENT && !ev->when)) + { + last_processed_ev.position= pos + ev->data_written; + last_processed_ev.datetime= ev_when; + } DBUG_PRINT("info", ("end event processing")); /* @@ -2925,6 +2943,9 @@ static Exit_status handle_event_text_mode(PRINT_EVENT_INFO *print_event_info, if (old_off != BIN_LOG_HEADER_SIZE) *len= 1; // fake event, don't increment old_off } + DBUG_ASSERT(old_off + ev->data_written == old_off + (*len - 1) || + (*len == 1 && + (type == ROTATE_EVENT || type == FORMAT_DESCRIPTION_EVENT))); Exit_status retval= process_event(print_event_info, ev, old_off, logname); if (retval != OK_CONTINUE) DBUG_RETURN(retval); @@ -2943,6 +2964,9 @@ static Exit_status handle_event_text_mode(PRINT_EVENT_INFO *print_event_info, DBUG_RETURN(ERROR_STOP); } + DBUG_ASSERT(old_off + ev->data_written == old_off + (*len - 1) || + (*len == 1 && + (type == ROTATE_EVENT || type == FORMAT_DESCRIPTION_EVENT))); retval= process_event(print_event_info, ev, old_off, logname); if (retval != OK_CONTINUE) { @@ -3342,6 +3366,8 @@ static Exit_status check_header(IO_CACHE* file, the new one, so we should not do it ourselves in this case. */ + DBUG_ASSERT(tmp_pos + new_description_event->data_written == + my_b_tell(file)); Exit_status retval= process_event(print_event_info, new_description_event, tmp_pos, logname); @@ -3495,20 +3521,17 @@ static Exit_status dump_local_log_entries(PRINT_EVENT_INFO *print_event_info, } // else read_error == 0 means EOF, that's OK, we break in this case - /* - Emit a warning in the event that we finished processing input - before reaching the boundary indicated by --stop-position. - */ - if (((longlong)stop_position != stop_position_default) && - stop_position > my_b_tell(file)) - { - retval = OK_STOP; - warning("Did not reach stop position %llu before " - "end of input", stop_position); - } - goto end; } + + /* + The real location that we have read up to in the file should align with + the size of the event, unless the event is encrypted. + */ + DBUG_ASSERT( + ((ev->get_type_code() == UNKNOWN_EVENT && + ((Unknown_log_event *) ev)->what == Unknown_log_event::ENCRYPTED)) || + old_off + ev->data_written == my_b_tell(file)); if ((retval= process_event(print_event_info, ev, old_off, logname)) != OK_CONTINUE) goto end; @@ -3687,10 +3710,18 @@ int main(int argc, char** argv) start_position= BIN_LOG_HEADER_SIZE; } + /* + Emit a warning if we finished processing input before reaching the stop + boundaries indicated by --stop-datetime or --stop-position. + */ if (stop_datetime != MY_TIME_T_MAX && - stop_datetime > last_processed_datetime) + stop_datetime > last_processed_ev.datetime) warning("Did not reach stop datetime '%s' before end of input", stop_datetime_str); + if ((static_cast(stop_position) != stop_position_default) && + stop_position > last_processed_ev.position) + warning("Did not reach stop position %llu before end of input", + stop_position); /* If enable flashback, need to print the events from the end to the diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result index 1517fd765ac..40fb791cf02 100644 --- a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_warn_stop_position.result @@ -18,6 +18,51 @@ drop table t1; # Ensuring file offset of binlog_f2_mid < binlog_f1_end # # +# Test using --read-from-remote-server +# +connection default; +# +# --stop-position tests +# +# Case 1.a) With one binlog file, a --stop-position before the end of +# the file should not result in a warning +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f1_pre_rotate binlog_f1_full --result-file=tmp/warn_position_test_file.out 2>&1 +# +# Case 1.b) With one binlog file, a --stop-position at the exact end of +# the file should not result in a warning +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f1_end binlog_f1_full --result-file=tmp/warn_position_test_file.out 2>&1 +# +# Case 1.c) With one binlog file, a --stop-position past the end of the +# file should(!) result in a warning +# MYSQL_BINLOG --read-from-remote-server --short-form --stop-position=binlog_f1_over_eof binlog_f1_full --result-file=tmp/warn_position_test_file.out 2>&1 +WARNING: Did not reach stop position before end of input +# +# Case 2.a) With two binlog files, a --stop-position targeting b2 which +# exists in the size of b1 should: +# 1) not provide any warnings +# 2) not prevent b2 from outputting its desired events before the +# stop position +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f2_mid binlog_f1_full binlog_f2_full --result-file=tmp/warn_position_test_file.out 2>&1 +include/assert_grep.inc [Ensure all intended GTIDs are present] +include/assert_grep.inc [Ensure the next GTID binlogged is _not_ present] +# +# Case 2.b) With two binlog files, a --stop-position targeting the end +# of binlog 2 should: +# 1) not provide any warnings +# 2) not prevent b2 from outputting its entire binary log +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f2_end binlog_f1_full binlog_f2_full --result-file=tmp/warn_position_test_file.out 2>&1 +include/assert_grep.inc [Ensure a GTID exists for each transaction] +include/assert_grep.inc [Ensure the last GTID binlogged is present] +# +# Case 2.c) With two binlog files, a --stop-position targeting beyond +# the eof of binlog 2 should: +# 1) provide a warning that the stop position was not reached +# 2) not prevent b2 from outputting its entire binary log +# MYSQL_BINLOG --read-from-remote-server --stop-position=binlog_f2_over_eof binlog_f1_full binlog_f2_full --result-file=tmp/warn_position_test_file.out 2>&1 +WARNING: Did not reach stop position before end of input +include/assert_grep.inc [Ensure a GTID exists for each transaction] +# +# # Test using local binlog files # connection default; diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test index a9179297220..472e208229c 100644 --- a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_warn_stop_position.test @@ -64,13 +64,12 @@ if ($binlog_f2_mid > $binlog_f1_end) --die Mid point chosen to end in binlog 2 does not exist in earlier binlog } -#--echo # -#--echo # -#--echo # Test using --read-from-remote-server -#--echo # -#--let $read_from_remote_server= 1 -#--emit warning is not supported by --read-from-remote-server now -#--source binlog_mysqlbinlog_warn_stop_position.inc +--echo # +--echo # +--echo # Test using --read-from-remote-server +--echo # +--let $read_from_remote_server= 1 +--source binlog_mysqlbinlog_warn_stop_position.inc --echo # --echo # From 3393d1064f31538b88ea150a2ca1bde83a390836 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Tue, 15 Apr 2025 21:49:41 +0300 Subject: [PATCH 095/125] MDEV-34075 corruption when query cache cannot allocate block When allocate_block() have failed Query_cache::insert_table() misses to init node->parent which is then accessed with outdated data. There is check in Query_cache::register_all_tables(): if (block_table->parent) unlink_table(block_table); So unlink_table() is avoided when parent is 0. --- mysql-test/main/query_cache.result | 23 +++++++++++++++++++++++ mysql-test/main/query_cache.test | 22 ++++++++++++++++++++++ sql/sql_cache.cc | 2 ++ 3 files changed, 47 insertions(+) diff --git a/mysql-test/main/query_cache.result b/mysql-test/main/query_cache.result index 504aabafda0..db5dc23ef2c 100644 --- a/mysql-test/main/query_cache.result +++ b/mysql-test/main/query_cache.result @@ -2239,6 +2239,29 @@ drop table t1; # set global Query_cache_size=18446744073709547520; # +# MDEV-34075 corruption when query cache cannot allocate block +# +set global query_cache_type=1; +create table t1 (c1 smallint null, c2 binary (25) not null, c3 tinyint(4) null, c4 binary (15) not null primary key, c5 smallint not null unique key,c6 decimal(10,8) not null default 3.141592) engine=innodb; +set global query_cache_size=81920; +select * from t1 where b=1 and c=1; +ERROR 42S22: Unknown column 'b' in 'where clause' +set session query_cache_type=1; +drop table t1; +create table t1 (c1 int not null, c2 char(5)) engine=innodb partition by linear key(c1) partitions 99; +select * from t1 where c1 <='1998-12-29 00:00:00' order by c1,c2; +c1 c2 +select group_concat(a separator '###') as names from t1 having left(names, 1)='j'; +ERROR 42S22: Unknown column 'a' in 'field list' +select * from t1; +c1 c2 +select count(*) from t1; +count(*) +0 +select G.a, c.a from t1 c, t1 G; +ERROR 42S22: Unknown column 'G.a' in 'field list' +drop table t1; +# # End of 10.5 tests # restore defaults diff --git a/mysql-test/main/query_cache.test b/mysql-test/main/query_cache.test index 2a9f34c7cf1..6cd8b886b29 100644 --- a/mysql-test/main/query_cache.test +++ b/mysql-test/main/query_cache.test @@ -2,6 +2,8 @@ -- source include/long_test.inc -- source include/no_valgrind_without_big.inc -- source include/no_view_protocol.inc +-- source include/have_partition.inc +-- source include/have_innodb.inc --disable_ps2_protocol set @save_query_cache_size=@@query_cache_size; @@ -1846,6 +1848,26 @@ drop table t1; set global Query_cache_size=18446744073709547520; --enable_warnings +--echo # +--echo # MDEV-34075 corruption when query cache cannot allocate block +--echo # +set global query_cache_type=1; +create table t1 (c1 smallint null, c2 binary (25) not null, c3 tinyint(4) null, c4 binary (15) not null primary key, c5 smallint not null unique key,c6 decimal(10,8) not null default 3.141592) engine=innodb; +set global query_cache_size=81920; +--error ER_BAD_FIELD_ERROR +select * from t1 where b=1 and c=1; +set session query_cache_type=1; +drop table t1; +create table t1 (c1 int not null, c2 char(5)) engine=innodb partition by linear key(c1) partitions 99; +select * from t1 where c1 <='1998-12-29 00:00:00' order by c1,c2; +--error ER_BAD_FIELD_ERROR +select group_concat(a separator '###') as names from t1 having left(names, 1)='j'; +select * from t1; +select count(*) from t1; +--error ER_BAD_FIELD_ERROR +select G.a, c.a from t1 c, t1 G; +drop table t1; + --echo # --echo # End of 10.5 tests --echo # diff --git a/sql/sql_cache.cc b/sql/sql_cache.cc index a54d1174e35..e04767d215d 100644 --- a/sql/sql_cache.cc +++ b/sql/sql_cache.cc @@ -3552,6 +3552,7 @@ Query_cache::insert_table(THD *thd, size_t key_len, const char *key, if (table_block == 0) { DBUG_PRINT("qcache", ("Can't write table name to cache")); + node->parent= NULL; DBUG_RETURN(0); } Query_cache_table *header= table_block->table(); @@ -3575,6 +3576,7 @@ Query_cache::insert_table(THD *thd, size_t key_len, const char *key, DBUG_PRINT("qcache", ("Can't insert table to hash")); // write_block_data return locked block free_memory_block(table_block); + node->parent= NULL; DBUG_RETURN(0); } char *db= header->db(); From dac3d702f77f6d53a94cfc377342f5f2e148a8d7 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 21 Apr 2025 13:45:38 +0530 Subject: [PATCH 096/125] MDEV-36649 dict_acquire_mdl_shared() aborts when table mode is DICT_TABLE_OP_OPEN_ONLY_IF_CACHED - InnoDB fails to check the table is being dropped or evicted while acquiring the MDL for the table when table open operation mode is DICT_TABLE_OP_OPEN_ONLY_IF_CACHED. This is caused by the commit 337bf8ac4bb9a89f51cd35ec867c74fbb8d23faa (MDEV-36122) Fix: === dict_acquire_mdl_shared(): If the table is evicted or dropped when table operation mode is DICT_TABLE_OP_OPEN_IF_CACHED then return nullptr --- mysql-test/suite/innodb/r/stats_persistent.result | 10 ++++++++++ mysql-test/suite/innodb/t/stats_persistent.test | 12 ++++++++++++ storage/innobase/dict/dict0dict.cc | 2 ++ 3 files changed, 24 insertions(+) diff --git a/mysql-test/suite/innodb/r/stats_persistent.result b/mysql-test/suite/innodb/r/stats_persistent.result index 7e9c038d6f7..077c4de71a0 100644 --- a/mysql-test/suite/innodb/r/stats_persistent.result +++ b/mysql-test/suite/innodb/r/stats_persistent.result @@ -17,3 +17,13 @@ test.t1 analyze status Engine-independent statistics collected test.t1 analyze status OK SET DEBUG_SYNC= 'RESET'; DROP TABLE t1; +# +# MDEV-36649 dict_acquire_mdl_shared() aborts when table +# mode is DICT_TABLE_OP_OPEN_ONLY_IF_CACHED +# +set @old_defragment_stats_accuracy= @@innodb_defragment_stats_accuracy; +SET GLOBAL innodb_defragment_stats_accuracy=1; +CREATE TABLE t (a INT ) ENGINE=INNODB; +INSERT INTO t SELECT * FROM seq_1_to_1000; +DROP TABLE t; +set global innodb_defragment_stats_accuracy= @old_defragment_stats_accuracy; diff --git a/mysql-test/suite/innodb/t/stats_persistent.test b/mysql-test/suite/innodb/t/stats_persistent.test index 8561298c4d3..4d11aef9d2d 100644 --- a/mysql-test/suite/innodb/t/stats_persistent.test +++ b/mysql-test/suite/innodb/t/stats_persistent.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_sequence.inc --source include/have_debug.inc --source include/have_debug_sync.inc --source include/count_sessions.inc @@ -26,3 +27,14 @@ SET DEBUG_SYNC= 'RESET'; DROP TABLE t1; --source include/wait_until_count_sessions.inc + +--echo # +--echo # MDEV-36649 dict_acquire_mdl_shared() aborts when table +--echo # mode is DICT_TABLE_OP_OPEN_ONLY_IF_CACHED +--echo # +set @old_defragment_stats_accuracy= @@innodb_defragment_stats_accuracy; +SET GLOBAL innodb_defragment_stats_accuracy=1; +CREATE TABLE t (a INT ) ENGINE=INNODB; +INSERT INTO t SELECT * FROM seq_1_to_1000; +DROP TABLE t; +set global innodb_defragment_stats_accuracy= @old_defragment_stats_accuracy; diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 4f293c41854..ef4c24f6fb5 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -795,6 +795,8 @@ lookup: dict_sys.freeze(SRW_LOCK_CALL); goto return_without_mdl; } + else + goto return_without_mdl; if (*mdl) { From 63a69ab9363a749dbbfeef23b911b195885364b2 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 21 Apr 2025 15:03:01 +0200 Subject: [PATCH 097/125] cleanup: remote automatic conversion char* -> Lex_ident considered harmful, see e.g. changes in check_period_fields() --- sql/handler.cc | 20 +++++--------------- sql/handler.h | 2 -- sql/lex_string.h | 2 +- sql/sql_table.cc | 6 +++--- sql/table.h | 4 ++-- sql/vers_string.h | 2 +- 6 files changed, 12 insertions(+), 24 deletions(-) diff --git a/sql/handler.cc b/sql/handler.cc index 923696fe99d..85358db6f15 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -8253,16 +8253,6 @@ int del_global_index_stat(THD *thd, TABLE* table, KEY* key_info) VERSIONING functions ******************************************************************************/ -bool Vers_parse_info::is_start(const char *name) const -{ - DBUG_ASSERT(name); - return as_row.start && as_row.start.streq(name); -} -bool Vers_parse_info::is_end(const char *name) const -{ - DBUG_ASSERT(name); - return as_row.end && as_row.end.streq(name); -} bool Vers_parse_info::is_start(const Create_field &f) const { return f.flags & VERS_ROW_START; @@ -8317,8 +8307,8 @@ bool Vers_parse_info::create_sys_field(THD *thd, const char *field_name, return false; } -const Lex_ident Vers_parse_info::default_start= "row_start"; -const Lex_ident Vers_parse_info::default_end= "row_end"; +const Lex_ident Vers_parse_info::default_start= { STRING_WITH_LEN("row_start")}; +const Lex_ident Vers_parse_info::default_end= { STRING_WITH_LEN("row_end")}; bool Vers_parse_info::fix_implicit(THD *thd, Alter_info *alter_info) { @@ -8577,7 +8567,7 @@ bool Vers_parse_info::fix_alter_info(THD *thd, Alter_info *alter_info, if (alter_info->flags & ALTER_ADD_SYSTEM_VERSIONING) { - if (check_sys_fields(table_name, share->db, alter_info)) + if (check_sys_fields(share->table_name, share->db, alter_info)) return true; } @@ -8883,8 +8873,8 @@ bool Table_scope_and_contents_source_st::check_period_fields( } } - bool res= period_info.check_field(row_start, period.start.str) - || period_info.check_field(row_end, period.end.str); + bool res= period_info.check_field(row_start, period.start) + || period_info.check_field(row_end, period.end); if (res) return true; diff --git a/sql/handler.h b/sql/handler.h index 617050b9a6f..51c4de0003a 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -2117,8 +2117,6 @@ struct Vers_parse_info: public Table_period_info } protected: - bool is_start(const char *name) const; - bool is_end(const char *name) const; bool is_start(const Create_field &f) const; bool is_end(const Create_field &f) const; bool fix_implicit(THD *thd, Alter_info *alter_info); diff --git a/sql/lex_string.h b/sql/lex_string.h index 56f37706aa8..5e048b18873 100644 --- a/sql/lex_string.h +++ b/sql/lex_string.h @@ -110,7 +110,7 @@ class Lex_cstring : public LEX_CSTRING class Lex_cstring_strlen: public Lex_cstring { public: - Lex_cstring_strlen(const char *from) + explicit Lex_cstring_strlen(const char *from) :Lex_cstring(from, from ? strlen(from) : 0) { } }; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 89095125fe3..06501449ab7 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -6235,7 +6235,7 @@ drop_create_field: } else if (drop->type == Alter_drop::PERIOD) { - if (table->s->period.name.streq(drop->name)) + if (table->s->period.name.streq(Lex_ident(drop->name))) remove_drop= FALSE; } else /* Alter_drop::KEY and Alter_drop::FOREIGN_KEY */ @@ -9227,7 +9227,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, for (bool found= false; !found && (drop= drop_it++); ) { found= drop->type == Alter_drop::PERIOD && - table->s->period.name.streq(drop->name); + table->s->period.name.streq(Lex_ident(drop->name)); } if (drop) @@ -9270,7 +9270,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, } } - if (share->period.constr_name.streq(check->name.str)) + if (share->period.constr_name.streq(check->name)) { if (!drop_period && !keep) { diff --git a/sql/table.h b/sql/table.h index 1aedaec8b52..137cb537587 100644 --- a/sql/table.h +++ b/sql/table.h @@ -2192,7 +2192,7 @@ struct vers_select_conds_t void init(vers_system_time_t _type, Vers_history_point _start= Vers_history_point(), Vers_history_point _end= Vers_history_point(), - Lex_ident _name= "SYSTEM_TIME") + Lex_ident _name= { STRING_WITH_LEN("SYSTEM_TIME") }) { type= _type; orig_type= _type; @@ -2207,7 +2207,7 @@ struct vers_select_conds_t void set_all() { type= SYSTEM_TIME_ALL; - name= "SYSTEM_TIME"; + name= { STRING_WITH_LEN("SYSTEM_TIME") }; } void print(String *str, enum_query_type query_type) const; diff --git a/sql/vers_string.h b/sql/vers_string.h index c5be9c359e3..9c1730fad81 100644 --- a/sql/vers_string.h +++ b/sql/vers_string.h @@ -62,7 +62,7 @@ public: { } Lex_cstring_with_compare(const LEX_CSTRING src) : Lex_cstring(src.str, src.length) { } - Lex_cstring_with_compare(const char *_str) : Lex_cstring(_str, strlen(_str)) + explicit Lex_cstring_with_compare(const char *_str) : Lex_cstring(_str, strlen(_str)) { } bool streq(const Lex_cstring_with_compare& b) const { From ab71860161d1a492165defcc7afdaa96a4ab7cc7 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 21 Apr 2025 15:03:55 +0200 Subject: [PATCH 098/125] cleanup: check_column_name(const Lex_ident &name) --- sql/sql_lex.h | 2 +- sql/sql_select.cc | 2 +- sql/sql_table.cc | 4 ++-- sql/sql_view.cc | 2 +- sql/sql_yacc.yy | 2 +- sql/table.cc | 13 +++---------- sql/table.h | 3 +-- 7 files changed, 10 insertions(+), 18 deletions(-) diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 6312102e076..d53dd8057a5 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -4681,7 +4681,7 @@ public: int add_period(Lex_ident name, Lex_ident_sys_st start, Lex_ident_sys_st end) { - if (check_period_name(name.str)) { + if (check_column_name(name)) { my_error(ER_WRONG_COLUMN_NAME, MYF(0), name.str); return 1; } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b4edf8a3050..5a98220fab3 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -30629,7 +30629,7 @@ void st_select_lex::print_item_list(THD *thd, String *str, */ if (top_level || item->is_explicit_name() || - !check_column_name(item->name.str)) + !check_column_name(item->name)) item->print_item_w_name(str, query_type); else item->print(str, query_type); diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 06501449ab7..d9ba09fd2a2 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -3107,7 +3107,7 @@ static bool mysql_prepare_create_table_stage1(THD *thd, DBUG_ASSERT(sql_field->charset); - if (check_column_name(sql_field->field_name.str)) + if (check_column_name(sql_field->field_name)) { my_error(ER_WRONG_COLUMN_NAME, MYF(0), sql_field->field_name.str); DBUG_RETURN(TRUE); @@ -3745,7 +3745,7 @@ mysql_prepare_create_table_finalize(THD *thd, HA_CREATE_INFO *create_info, key_part_info++; } - if (!key_info->name.str || check_column_name(key_info->name.str)) + if (!key_info->name.str || check_column_name(key_info->name)) { my_error(ER_WRONG_NAME_FOR_INDEX, MYF(0), key_info->name.str); DBUG_RETURN(TRUE); diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 316e4242553..58da2833ec1 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -183,7 +183,7 @@ void make_valid_column_names(THD *thd, List &item_list) for (uint column_no= 1; (item= it++); column_no++) { - if (item->is_explicit_name() || !check_column_name(item->name.str)) + if (item->is_explicit_name() || !check_column_name(item->name)) continue; name_len= my_snprintf(buff, NAME_LEN, "Name_exp_%u", column_no); item->orig_name= item->name.str; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 2abd0a529af..be7e918706a 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -9107,7 +9107,7 @@ select_item: if ($4.str) { if (unlikely(Lex->sql_command == SQLCOM_CREATE_VIEW && - check_column_name($4.str))) + check_column_name($4))) my_yyabort_error((ER_WRONG_COLUMN_NAME, MYF(0), $4.str)); $2->base_flags|= item_base_t::IS_EXPLICIT_NAME; $2->set_name(thd, $4); diff --git a/sql/table.cc b/sql/table.cc index 3fbac639ecb..3a51228d1f3 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -5320,9 +5320,10 @@ bool check_table_name(const char *name, size_t length, bool check_for_path_chars } -bool check_column_name(const char *name) +bool check_column_name(const Lex_ident &ident) { // name length in symbols + const char *name= ident.str, *end= ident.str + ident.length; size_t name_length= 0; bool last_char_is_space= TRUE; @@ -5332,9 +5333,7 @@ bool check_column_name(const char *name) last_char_is_space= my_isspace(system_charset_info, *name); if (system_charset_info->use_mb()) { - int len=my_ismbchar(system_charset_info, name, - name+system_charset_info->mbmaxlen); - if (len) + if (int len= my_ismbchar(system_charset_info, name, end)) { name += len; name_length++; @@ -5354,12 +5353,6 @@ bool check_column_name(const char *name) } -bool check_period_name(const char *name) -{ - return check_column_name(name); -} - - /** Checks whether a table is intact. Should be done *just* after the table has been opened. diff --git a/sql/table.h b/sql/table.h index 137cb537587..d249a84d8cc 100644 --- a/sql/table.h +++ b/sql/table.h @@ -3388,8 +3388,7 @@ void open_table_error(TABLE_SHARE *share, enum open_frm_error error, int db_errno); void update_create_info_from_table(HA_CREATE_INFO *info, TABLE *form); bool check_db_name(LEX_STRING *db); -bool check_column_name(const char *name); -bool check_period_name(const char *name); +bool check_column_name(const Lex_ident &name); bool check_table_name(const char *name, size_t length, bool check_for_path_chars); int rename_file_ext(const char * from,const char * to,const char * ext); char *get_field(MEM_ROOT *mem, Field *field); From 7b0820b8b716cb1c586c8a837c565c89ff5db0ab Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 21 Apr 2025 15:06:30 +0200 Subject: [PATCH 099/125] cleanup: redundant my_casedn_str() hashing/comparison uses case-insensitive collation anyway --- .../suite/rpl/r/rpl_master_pos_wait.result | 3 +++ .../suite/rpl/t/rpl_master_pos_wait.test | 1 + sql/rpl_mi.cc | 18 ++++++------------ 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_master_pos_wait.result b/mysql-test/suite/rpl/r/rpl_master_pos_wait.result index aae3418b546..a954e6eb138 100644 --- a/mysql-test/suite/rpl/r/rpl_master_pos_wait.result +++ b/mysql-test/suite/rpl/r/rpl_master_pos_wait.result @@ -43,6 +43,9 @@ NULL select master_pos_wait('master-bin.000001',1000000,1,"my_slave"); master_pos_wait('master-bin.000001',1000000,1,"my_slave") -1 +select master_pos_wait('master-bin.000001',1000000,1,"MY_SLAVE"); +master_pos_wait('master-bin.000001',1000000,1,"MY_SLAVE") +-1 STOP SLAVE 'my_slave'; RESET SLAVE 'my_slave' ALL; change master to master_port=MASTER_MYPORT, master_host='127.0.0.1', master_user='root'; diff --git a/mysql-test/suite/rpl/t/rpl_master_pos_wait.test b/mysql-test/suite/rpl/t/rpl_master_pos_wait.test index 437d8412086..0044e6e544e 100644 --- a/mysql-test/suite/rpl/t/rpl_master_pos_wait.test +++ b/mysql-test/suite/rpl/t/rpl_master_pos_wait.test @@ -48,6 +48,7 @@ select master_pos_wait('master-bin.000001',1000000,1); --echo # Call with a valid connection name -- hangs before MDEV-7130 fix (expected -1) select master_pos_wait('master-bin.000001',1000000,1,"my_slave"); +select master_pos_wait('master-bin.000001',1000000,1,"MY_SLAVE"); STOP SLAVE 'my_slave'; RESET SLAVE 'my_slave' ALL; diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index 6e1ec2c494f..2183137c2f5 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -1369,27 +1369,21 @@ Master_info_index::get_master_info(const LEX_CSTRING *connection_name, Sql_condition::enum_warning_level warning) { Master_info *mi; - char buff[MAX_CONNECTION_NAME+1], *res; - size_t buff_length; DBUG_ENTER("get_master_info"); DBUG_PRINT("enter", ("connection_name: '%.*s'", (int) connection_name->length, connection_name->str)); - /* Make name lower case for comparison */ - res= strmake(buff, connection_name->str, connection_name->length); - my_casedn_str(system_charset_info, buff); - buff_length= (size_t) (res-buff); - + if (!connection_name->str) + connection_name= &empty_clex_str; mi= (Master_info*) my_hash_search(&master_info_hash, - (uchar*) buff, buff_length); + (uchar*) connection_name->str, + connection_name->length); if (!mi && warning != Sql_condition::WARN_LEVEL_NOTE) { my_error(WARN_NO_MASTER_INFO, - MYF(warning == Sql_condition::WARN_LEVEL_WARN ? ME_WARNING : - 0), - (int) connection_name->length, - connection_name->str); + MYF(warning == Sql_condition::WARN_LEVEL_WARN ? ME_WARNING : 0), + (int) connection_name->length, connection_name->str); } DBUG_RETURN(mi); } From 8c7c144f1994b3a53772fed10fd6df6bff02c3ed Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 14 Apr 2025 16:49:25 +0300 Subject: [PATCH 100/125] MDEV-36592: In JOIN ... USING(columns), query plan depends on join order "t1 JOIN t2 USING(col1,...)" calls mark_common_columns() to mark the listed columns as used in both used tables, t1 and t2. Due to a typo bug, it would mark the wrong column in the second table (t2): instead of t2.col1 it would mark the last column in t2. The harmful effects included JOIN_TAB(t2)->covering_keys not being set correctly. This changed the cost to access the table and then caused different query plans depending on which table was the second in the JOIN ... USING syntax. --- mysql-test/main/join.result | 29 +++++++++++++++++++++++++++++ mysql-test/main/join.test | 25 +++++++++++++++++++++++++ sql/sql_base.cc | 3 +++ 3 files changed, 57 insertions(+) diff --git a/mysql-test/main/join.result b/mysql-test/main/join.result index ecee13990d9..058478ef373 100644 --- a/mysql-test/main/join.result +++ b/mysql-test/main/join.result @@ -3611,3 +3611,32 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 100 Using where 1 SIMPLE t2 ref kp1 kp1 5 test.t1.a 1 Using index condition drop table t1,t2; +# +# MDEV-36592: If the join_condition is specified via USING (column_list), the query plan depends ... +# +CREATE TABLE t1 ( +id int(11), +f1 char(255), +PRIMARY KEY (id) +); +INSERT INTO t1 (id) VALUES (1),(2),(3); +UPDATE t1 SET f1=REPEAT('a',250); +CREATE TABLE t2 (id int(11), f2 INT NOT NULL); +INSERT INTO t2 select seq, seq from seq_1_to_20; +ANALYZE TABLE t1, t2; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +test.t2 analyze status Engine-independent statistics collected +test.t2 analyze status OK +# In both queries, t1 should use type=index, not type=ALL: +EXPLAIN SELECT count(*) FROM t2 JOIN t1 USING (id); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index PRIMARY PRIMARY 4 NULL 3 Using index +1 SIMPLE t2 ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) +EXPLAIN SELECT count(*) FROM t1 JOIN t2 USING (id); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index PRIMARY PRIMARY 4 NULL 3 Using index +1 SIMPLE t2 ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) +DROP TABLE t1,t2; +# End of 10.11 tests diff --git a/mysql-test/main/join.test b/mysql-test/main/join.test index 4106b07a16b..57a9ccd97d0 100644 --- a/mysql-test/main/join.test +++ b/mysql-test/main/join.test @@ -2015,3 +2015,28 @@ where t2.kp1=t1.a and t2.kp1<=100 and t2.kp2<=20; drop table t1,t2; + +--echo # +--echo # MDEV-36592: If the join_condition is specified via USING (column_list), the query plan depends ... +--echo # +CREATE TABLE t1 ( + id int(11), + f1 char(255), + PRIMARY KEY (id) +); +INSERT INTO t1 (id) VALUES (1),(2),(3); +UPDATE t1 SET f1=REPEAT('a',250); + +CREATE TABLE t2 (id int(11), f2 INT NOT NULL); +INSERT INTO t2 select seq, seq from seq_1_to_20; + +ANALYZE TABLE t1, t2; + +--echo # In both queries, t1 should use type=index, not type=ALL: +EXPLAIN SELECT count(*) FROM t2 JOIN t1 USING (id); +EXPLAIN SELECT count(*) FROM t1 JOIN t2 USING (id); + +DROP TABLE t1,t2; + +--echo # End of 10.11 tests + diff --git a/sql/sql_base.cc b/sql/sql_base.cc index d669defa992..0a3134f3709 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -7414,6 +7414,9 @@ mark_common_columns(THD *thd, TABLE_LIST *table_ref_1, TABLE_LIST *table_ref_2, if (!found) continue; // No matching field + /* Restore field_2 to point to the field which was a match for field_1. */ + field_2= nj_col_2->field(); + /* field_1 and field_2 have the same names. Check if they are in the USING clause (if present), mark them as common fields, and add a new From 4bedb222a807a4cc3f424e0f57b1921ab0d3e253 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 22 Apr 2025 17:11:56 +0530 Subject: [PATCH 101/125] MDEV-36304 InnoDB: Missing FILE_CREATE, FILE_DELETE or FILE_MODIFY error during mariabackup --prepare Reason: ====== During --prepare of partial backup, if InnoDB encounters the redo log for the excluded tablespace then InnoDB stores the space id in dirty tablespace list during recovery, anticipates that it may encounter FILE_* redo log records in the future. Even though we encounter FILE_* record for the partial excluded tablespace then we fail to replace the name in dirty tablespace list. This lead to missing of FILE_* redo log records error. Solution: ======== fil_name_process(): Rename the file name from "" to name encountered during FILE_* record recv_init_missing_space(): Correct the condition to print the warning message of missing tablespace during mariabackup restore process. --- storage/innobase/log/log0recv.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 9a71521d707..004dc65c752 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1307,6 +1307,13 @@ got_deleted: } else if (p.second // the first FILE_MODIFY or FILE_RENAME || f.name != fname.name) { reload: + if (f.name.size() == 0) { + /* Augment the recv_spaces.emplace_hint() for the + FILE_MODIFY record that had been added by + recv_sys_t::parse() */ + f.name = fname.name; + } + fil_space_t* space; /* Check if the tablespace file exists and contains @@ -2577,8 +2584,8 @@ page_id_corrupted: if (i != recv_spaces.end() && i->first == space_id); else if (recovered_lsn < mlog_checkpoint_lsn) /* We have not seen all records between the checkpoint and - FILE_CHECKPOINT. There should be a FILE_DELETE for this - tablespace later. */ + FILE_CHECKPOINT. There should be a FILE_DELETE or FILE_MODIFY + for this tablespace later, to be handled in fil_name_process() */ recv_spaces.emplace_hint(i, space_id, file_name_t("", false)); else { @@ -4158,7 +4165,7 @@ recv_init_missing_space(dberr_t err, const recv_spaces_t::const_iterator& i) break; case SRV_OPERATION_RESTORE: case SRV_OPERATION_RESTORE_EXPORT: - if (i->second.name.find("/#sql") != std::string::npos) { + if (i->second.name.find("/#sql") == std::string::npos) { ib::warn() << "Tablespace " << i->first << " was not" " found at " << i->second.name << " when" " restoring a (partial?) backup. All redo log" From 47e687b109e465a31ec029969e302ca1a73208a3 Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Tue, 22 Apr 2025 15:49:53 +0300 Subject: [PATCH 102/125] MDEV-36639 innodb_snapshot_isolation=1 gives error for not committed row changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set solution is to check if transaction, which modified a record, is still active in lock_clust_rec_read_check_and_lock(). if yes, then just request a lock. If no, then, depending on if the current transaction read view can see the changes, return eighter DB_RECORD_CHANGED or request a lock. We can do the check in lock_clust_rec_read_check_and_lock() because transaction tries to set a lock on the record which cursor points to after transaction resuming and cursor position restoring. If the lock already exists, then we don't request the lock again. But for the current commit it's important that lock_clust_rec_read_check_and_lock() will be invoked again for the same record, so we can do the check again after transaction, which modified a record, was committed or rolled back. MDEV-33802(4aa9291) is partially reverted. If some transaction holds implicit lock on some record and transaction with snapshot isolation level requests conflicting lock on the same record, it should be blocked instead of returning DB_RECORD_CHANGED to have ability to continue execution when implicit lock owner is rolled back. The construction -------------------------------------------------------------------------- let $wait_condition= select count(*) = 1 from information_schema.processlist where state = 'Updating' and info = 'UPDATE t SET b = 2 WHERE a'; --source include/wait_condition.inc -------------------------------------------------------------------------- is not reliable enought to make sure transaction is blocked in test case, the test failed sporadically with -------------------------------------------------------------------------- ./mtr --max-test-fail=1 --parallel=96 lock_isolation{,,,,,,,}{,,,}{,,} \ --repeat=500 -------------------------------------------------------------------------- command. That's why it was replaced with debug sync-points. Reviewed by: Marko Mäkelä --- .../suite/innodb/r/lock_isolation.result | 90 +++++++++++- mysql-test/suite/innodb/t/lock_isolation.test | 136 +++++++++++++----- storage/innobase/lock/lock0lock.cc | 63 +++----- 3 files changed, 207 insertions(+), 82 deletions(-) diff --git a/mysql-test/suite/innodb/r/lock_isolation.result b/mysql-test/suite/innodb/r/lock_isolation.result index 5975b531451..1e1625ae458 100644 --- a/mysql-test/suite/innodb/r/lock_isolation.result +++ b/mysql-test/suite/innodb/r/lock_isolation.result @@ -1,3 +1,6 @@ +connect disable_purging,localhost,root; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; # # MDEV-26642 Weird SELECT view when a record is # modified to the same value by two transactions @@ -52,15 +55,17 @@ DROP TABLE t; # MDEV-26643 Inconsistent behaviors of UPDATE under # READ UNCOMMITTED and READ COMMITTED isolation level # -CREATE TABLE t(a INT, b INT) ENGINE=InnoDB; +CREATE TABLE t(a INT, b INT) ENGINE=InnoDB STATS_PERSISTENT=0; INSERT INTO t VALUES(NULL, 1), (2, 2); SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; BEGIN; UPDATE t SET a = 10; connection consistent; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; UPDATE t SET b = 20 WHERE a; connection default; +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; COMMIT; connection consistent; SELECT * FROM t; @@ -74,8 +79,10 @@ BEGIN; UPDATE t SET a = 10; connection consistent; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; UPDATE t SET b = 20 WHERE a; connection default; +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; COMMIT; connection consistent; SELECT * FROM t; @@ -89,8 +96,10 @@ BEGIN; UPDATE t SET a = 10; connection con_weird; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; UPDATE t SET b = 20 WHERE a; connection default; +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; SELECT * FROM t; a b 10 1 @@ -113,8 +122,10 @@ UPDATE t SET b = 3; connection consistent; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; BEGIN; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; UPDATE t SET b = 2 WHERE a; connection default; +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; UPDATE t SET a = 1; COMMIT; connection consistent; @@ -128,20 +139,25 @@ DROP TABLE t; # # MDEV-33802 Weird read view after ROLLBACK of other transactions # -CREATE TABLE t(a INT PRIMARY KEY, b INT UNIQUE) ENGINE=InnoDB; -INSERT INTO t SET a=1; -BEGIN; -INSERT INTO t SET a=2; +CREATE TABLE t(a INT PRIMARY KEY, b INT UNIQUE) ENGINE=InnoDB STATS_PERSISTENT=0; connection consistent; START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; +INSERT INTO t SET a=1; +connection consistent; SAVEPOINT sp1; SELECT * FROM t FORCE INDEX (b) FOR UPDATE; ERROR HY000: Record has changed since last read in table 't' SAVEPOINT sp1; +connection default; +BEGIN; +INSERT INTO t SET a=2; connection con_weird; START TRANSACTION WITH CONSISTENT SNAPSHOT; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; SELECT * FROM t FORCE INDEX (b) FOR UPDATE; connection default; +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; ROLLBACK; connection con_weird; a b @@ -149,12 +165,74 @@ a b SELECT * FROM t FORCE INDEX (b) FOR UPDATE; a b 1 NULL +COMMIT; disconnect con_weird; connection consistent; SELECT * FROM t FORCE INDEX (b) FOR UPDATE; a b 1 NULL -disconnect consistent; +COMMIT; connection default; +TRUNCATE TABLE t; +# +# MDEV-36639 innodb_snapshot_isolation=1 gives error for not comitted row changes +# +INSERT INTO t VALUES (1,1),(2,2); +connection default; +# Case 1: Transaction A modifies a record, transaction B with snapshot +# isolation level is blocked by A, then A is committed. +# Expected behaviour: B gets ER_CHECKREAD. +BEGIN; +UPDATE t SET b=3 WHERE a = 1; +connection consistent; +SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; +BEGIN; +SELECT * FROM t; +a b +1 1 +2 2 +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; +SELECT * FROM t WHERE a=1 FOR UPDATE; +connection default; +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; +COMMIT; +connection consistent; +ERROR HY000: Record has changed since last read in table 't' +# Case 2: Transaction A modifies a record, transaction B with snapshot +# isolation level is blocked by A, then A is rolled back. +# Expected behaviour: B continues execution. +connection default; +BEGIN; +UPDATE t SET b=4 WHERE a=1; +connection consistent; +BEGIN; +SELECT * FROM t; +a b +2 2 +1 3 +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; +SELECT * FROM t WHERE a=1 FOR UPDATE; +connection default; +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; +ROLLBACK; +connection consistent; +a b +1 3 +ROLLBACK; +# Case 3: Transaction B with snapshot isolation level started with +# consistent snapshot. Transaction A modifies a record and is committed. +# Both B tries to read modified by A record. +# Expected behavior: B gets ER_CHECKREAD. +connection consistent; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; +UPDATE t SET b=4 WHERE a=1; +connection consistent; +SELECT * FROM t WHERE a=1 FOR UPDATE; +ERROR HY000: Record has changed since last read in table 't' +disconnect consistent; +disconnect disable_purging; +connection default; +SET DEBUG_SYNC="RESET"; DROP TABLE t; # End of 10.6 tests diff --git a/mysql-test/suite/innodb/t/lock_isolation.test b/mysql-test/suite/innodb/t/lock_isolation.test index bac034a50e4..b332f2e867a 100644 --- a/mysql-test/suite/innodb/t/lock_isolation.test +++ b/mysql-test/suite/innodb/t/lock_isolation.test @@ -1,9 +1,16 @@ --source include/have_innodb.inc +--source include/count_sessions.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc --disable_query_log call mtr.add_suppression("InnoDB: Transaction was aborted due to "); --enable_query_log +--connect disable_purging,localhost,root +START TRANSACTION WITH CONSISTENT SNAPSHOT; + +--connection default --echo # --echo # MDEV-26642 Weird SELECT view when a record is --echo # modified to the same value by two transactions @@ -41,22 +48,18 @@ DROP TABLE t; --echo # READ UNCOMMITTED and READ COMMITTED isolation level --echo # -CREATE TABLE t(a INT, b INT) ENGINE=InnoDB; +CREATE TABLE t(a INT, b INT) ENGINE=InnoDB STATS_PERSISTENT=0; INSERT INTO t VALUES(NULL, 1), (2, 2); SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; BEGIN; UPDATE t SET a = 10; --connection consistent SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; --send UPDATE t SET b = 20 WHERE a --connection default -let $wait_condition= - select count(*) = 1 from information_schema.processlist - where state = 'Updating' - and info = 'UPDATE t SET b = 20 WHERE a'; ---source include/wait_condition.inc - +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; COMMIT; --connection consistent @@ -70,14 +73,11 @@ BEGIN; UPDATE t SET a = 10; --connection consistent SET TRANSACTION ISOLATION LEVEL READ COMMITTED; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; --send UPDATE t SET b = 20 WHERE a --connection default -let $wait_condition= - select count(*) = 1 from information_schema.processlist - where info = 'UPDATE t SET b = 20 WHERE a'; ---source include/wait_condition.inc - +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; COMMIT; --connection consistent @@ -91,15 +91,11 @@ BEGIN; UPDATE t SET a = 10; --connection con_weird SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; send UPDATE t SET b = 20 WHERE a; --connection default -let $wait_condition= - select count(*) = 1 from information_schema.processlist - where state = 'Updating' - and info = 'UPDATE t SET b = 20 WHERE a'; ---source include/wait_condition.inc - +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; SELECT * FROM t; COMMIT; @@ -123,14 +119,11 @@ SET TRANSACTION ISOLATION LEVEL READ COMMITTED; BEGIN; # As semi-consistent read is disabled for innodb_snapshot_isolation=ON, the # following UPDATE must be blocked on the first record. +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; --send UPDATE t SET b = 2 WHERE a --connection default -let $wait_condition= - select count(*) = 1 from information_schema.processlist - where state = 'Updating' and info = 'UPDATE t SET b = 2 WHERE a'; ---source include/wait_condition.inc - +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; UPDATE t SET a = 1; COMMIT; --connection consistent @@ -149,13 +142,15 @@ DROP TABLE t; --echo # MDEV-33802 Weird read view after ROLLBACK of other transactions --echo # -CREATE TABLE t(a INT PRIMARY KEY, b INT UNIQUE) ENGINE=InnoDB; -INSERT INTO t SET a=1; - -BEGIN; INSERT INTO t SET a=2; +CREATE TABLE t(a INT PRIMARY KEY, b INT UNIQUE) ENGINE=InnoDB STATS_PERSISTENT=0; --connection consistent START TRANSACTION WITH CONSISTENT SNAPSHOT; + +--connection default +INSERT INTO t SET a=1; + +--connection consistent SAVEPOINT sp1; --disable_ps2_protocol --error ER_CHECKREAD @@ -163,29 +158,100 @@ SELECT * FROM t FORCE INDEX (b) FOR UPDATE; --enable_ps2_protocol SAVEPOINT sp1; +--connection default +BEGIN; INSERT INTO t SET a=2; + --connection con_weird START TRANSACTION WITH CONSISTENT SNAPSHOT; -send -SELECT * FROM t FORCE INDEX (b) FOR UPDATE; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; +--send SELECT * FROM t FORCE INDEX (b) FOR UPDATE --connection default -let $wait_condition= - select count(*) = 1 from information_schema.processlist - where state = 'Sending data' - and info LIKE 'SELECT * FROM t %'; ---source include/wait_condition.inc +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; ROLLBACK; --connection con_weird --reap SELECT * FROM t FORCE INDEX (b) FOR UPDATE; +COMMIT; --disconnect con_weird --connection consistent SELECT * FROM t FORCE INDEX (b) FOR UPDATE; ---disconnect consistent +COMMIT; --connection default +TRUNCATE TABLE t; + +--echo # +--echo # MDEV-36639 innodb_snapshot_isolation=1 gives error for not comitted row changes +--echo # +INSERT INTO t VALUES (1,1),(2,2); + +--connection default +--echo # Case 1: Transaction A modifies a record, transaction B with snapshot +--echo # isolation level is blocked by A, then A is committed. +--echo # Expected behaviour: B gets ER_CHECKREAD. +BEGIN; +UPDATE t SET b=3 WHERE a = 1; + +--connection consistent +SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; +BEGIN; +SELECT * FROM t; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; +--send SELECT * FROM t WHERE a=1 FOR UPDATE + +--connection default +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; +COMMIT; + +--connection consistent +--error ER_CHECKREAD +--reap + +--echo # Case 2: Transaction A modifies a record, transaction B with snapshot +--echo # isolation level is blocked by A, then A is rolled back. +--echo # Expected behaviour: B continues execution. + +--connection default +BEGIN; +UPDATE t SET b=4 WHERE a=1; + +--connection consistent +BEGIN; +SELECT * FROM t; +SET DEBUG_SYNC="lock_wait_before_suspend SIGNAL select_blocked"; +--send SELECT * FROM t WHERE a=1 FOR UPDATE + +--connection default +SET DEBUG_SYNC="now WAIT_FOR select_blocked"; +ROLLBACK; + +--connection consistent +--reap +ROLLBACK; + +--echo # Case 3: Transaction B with snapshot isolation level started with +--echo # consistent snapshot. Transaction A modifies a record and is committed. +--echo # Both B tries to read modified by A record. +--echo # Expected behavior: B gets ER_CHECKREAD. + +--connection consistent +START TRANSACTION WITH CONSISTENT SNAPSHOT; + +--connection default +UPDATE t SET b=4 WHERE a=1; + +--connection consistent +--error ER_CHECKREAD +SELECT * FROM t WHERE a=1 FOR UPDATE; +--disconnect consistent +--disconnect disable_purging + +--connection default +SET DEBUG_SYNC="RESET"; DROP TABLE t; +--source include/wait_until_count_sessions.inc --echo # End of 10.6 tests diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 7ecdc034bbc..3e4e619b3a0 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -6052,17 +6052,10 @@ lock_clust_rec_modify_check_and_lock( for it */ trx_t *trx = thr_get_trx(thr); - if (const trx_t *owner = - lock_rec_convert_impl_to_expl(trx, *block, - rec, index, offsets)) { - if (owner == trx) { - /* We already hold an exclusive lock. */ - return DB_SUCCESS; - } - - if (trx->snapshot_isolation && trx->read_view.is_open()) { - return DB_RECORD_CHANGED; - } + if (lock_rec_convert_impl_to_expl(trx, *block, + rec, index, offsets) == trx) { + /* We already hold an exclusive lock. */ + return DB_SUCCESS; } err = lock_rec_lock(true, LOCK_X | LOCK_REC_NOT_GAP, @@ -6224,19 +6217,11 @@ lock_sec_rec_read_check_and_lock( return DB_SUCCESS; } - if (page_rec_is_supremum(rec)) { - } else if (const trx_t *owner = - lock_rec_convert_impl_to_expl(trx, *block, - rec, index, offsets)) { - if (owner == trx) { - if (gap_mode == LOCK_REC_NOT_GAP) { - /* We already hold an exclusive lock. */ - return DB_SUCCESS; - } - } else if (trx->snapshot_isolation - && trx->read_view.is_open()) { - return DB_RECORD_CHANGED; - } + if (!page_rec_is_supremum(rec) + && lock_rec_convert_impl_to_expl(trx, *block, rec, index, + offsets) == trx + && gap_mode == LOCK_REC_NOT_GAP) { + return DB_SUCCESS; } #ifdef WITH_WSREP @@ -6316,28 +6301,24 @@ lock_clust_rec_read_check_and_lock( trx_t *trx = thr_get_trx(thr); if (lock_table_has(trx, index->table, LOCK_X) || heap_no == PAGE_HEAP_NO_SUPREMUM) { - } else if (const trx_t *owner = - lock_rec_convert_impl_to_expl(trx, *block, - rec, index, offsets)) { - if (owner == trx) { - if (gap_mode == LOCK_REC_NOT_GAP) { - /* We already hold an exclusive lock. */ - return DB_SUCCESS; - } - } else if (trx->snapshot_isolation - && trx->read_view.is_open()) { - return DB_RECORD_CHANGED; - } + } else if (lock_rec_convert_impl_to_expl(trx, *block, rec, index, + offsets) == trx + && gap_mode == LOCK_REC_NOT_GAP) { + /* We already hold an exclusive lock. */ + return DB_SUCCESS; } if (heap_no > PAGE_HEAP_NO_SUPREMUM && gap_mode != LOCK_GAP && trx->snapshot_isolation - && trx->read_view.is_open() - && !trx->read_view.changes_visible( - trx_read_trx_id(rec + row_trx_id_offset(rec, index))) - && IF_WSREP(!(trx->is_wsrep() + && trx->read_view.is_open()) { + trx_id_t trx_id= trx_read_trx_id(rec + + row_trx_id_offset(rec, index)); + if (!trx_sys.is_registered(trx, trx_id) + && !trx->read_view.changes_visible(trx_id) + && IF_WSREP(!(trx->is_wsrep() && wsrep_thd_skip_locking(trx->mysql_thd)), true)) { - return DB_RECORD_CHANGED; + return DB_RECORD_CHANGED; + } } dberr_t err = lock_rec_lock(false, gap_mode | mode, From 26754e23b2e992876138cd31f2afc5c5057001bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Fri, 12 Jul 2024 14:04:48 +0300 Subject: [PATCH 103/125] Typo: HAVE_valgrind should be used, not HAVE_VALGRIND This caused the MariaDB Server version to not have the -valgrind flag when compiled with valgrind support. --- sql/mysqld.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 7a7c1f0491a..9cdbd5e7319 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -8783,7 +8783,7 @@ char *set_server_version(char *buf, size_t size) bool is_log= opt_log || global_system_variables.sql_log_slow || opt_bin_log; bool is_debug= IF_DBUG(!strstr(MYSQL_SERVER_SUFFIX_STR, "-debug"), 0); const char *is_valgrind= -#ifdef HAVE_VALGRIND +#ifdef HAVE_valgrind !strstr(MYSQL_SERVER_SUFFIX_STR, "-valgrind") ? "-valgrind" : #endif ""; From 1697717f9b62eac0db9fea97bff392fd9a70f1eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 22 Apr 2025 18:44:46 +0300 Subject: [PATCH 104/125] Add -asan to server version suffix if -fsanitize is used This is to allow mtr tests using --include not_valgrind_build.inc to also work. Reviewed-by: monty@mariadb.com --- sql/mysqld.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 9cdbd5e7319..3f10a84fd83 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -8787,11 +8787,18 @@ char *set_server_version(char *buf, size_t size) !strstr(MYSQL_SERVER_SUFFIX_STR, "-valgrind") ? "-valgrind" : #endif ""; + const char *is_asan= +#ifdef __SANITIZE_ADDRESS__ + !strstr(MYSQL_SERVER_SUFFIX_STR, "-asan") ? "-asan" : +#endif + ""; + return strxnmov(buf, size - 1, MYSQL_SERVER_VERSION, MYSQL_SERVER_SUFFIX_STR, IF_EMBEDDED("-embedded", ""), is_valgrind, + is_asan, is_debug ? "-debug" : "", is_log ? "-log" : "", NullS); From 1ae8c63ba6a62cccac28a67f558863d6cfbd648d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= Date: Tue, 22 Apr 2025 20:21:53 +0300 Subject: [PATCH 105/125] Revert "Fix valgrind detection" This reverts commit 952ffb55f93bf6a6c5d1c9617e5a2a56207ee674. --- mysql-test/main/sp-no-valgrind.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/main/sp-no-valgrind.test b/mysql-test/main/sp-no-valgrind.test index 72ef9fb36a9..10a238f9871 100644 --- a/mysql-test/main/sp-no-valgrind.test +++ b/mysql-test/main/sp-no-valgrind.test @@ -1,5 +1,5 @@ --source include/not_msan.inc ---source include/not_valgrind.inc +--source include/not_valgrind_build.inc --echo # MDEV-20699 do not cache SP in SHOW CREATE --echo # Warmup round, this might allocate some memory for session variable From b1eec9d8af2c12b7afe6447edfb9c021c209c3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 17 Apr 2025 09:59:00 +0300 Subject: [PATCH 106/125] MDEV-36516 : galera_3nodes.galera_gtid_2_cluster test failed on 10.5 Add wait-conditions to verify that INSERTs are replicated before checking GTIDs. Signed-off-by: Julius Goryavsky --- .../t/galera_gtid_2_cluster.test | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test index 750f840c470..fe71303277c 100644 --- a/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test +++ b/mysql-test/suite/galera_3nodes/t/galera_gtid_2_cluster.test @@ -77,6 +77,8 @@ select @@gtid_binlog_state; --echo cluster 2 node 1 --connection node_4 +--let $wait_condition = SELECT COUNT(*) = 1 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 21, 1); select @@gtid_binlog_state; @@ -85,11 +87,16 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 2 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 1 node 2 --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 2 FROM test.t1; +--source include/wait_condition.inc + select @@gtid_binlog_state; insert into t1 values (1, 12, 3); select @@gtid_binlog_state; @@ -99,10 +106,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 3 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 1 node 3 --connection node_3 +--let $wait_condition = SELECT COUNT(*) = 3 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (1, 13, 4); select @@gtid_binlog_state; @@ -112,10 +123,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 2 node 2 --connection node_5 +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 22, 2); select @@gtid_binlog_state; @@ -125,37 +140,55 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 5 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 2 node 3 --connection node_6 +--let $wait_condition = SELECT COUNT(*) = 5 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 23, 3); select @@gtid_binlog_state; --echo #wait for sync cluster 2 and 1 --connection node_4 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo # check other nodes are consistent --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_3 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_5 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_6 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --echo cluster 1 node 1 --connection node_1 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; drop table t1; stop slave; @@ -250,6 +283,8 @@ select @@gtid_binlog_state; --sleep 2 --echo cluster 2 node 1 --connection node_4 +--let $wait_condition = SELECT COUNT(*) = 1 FROM test.t1; +--source include/wait_condition.inc insert into t1 values (2, 21, 1); select @@gtid_binlog_state; @@ -258,11 +293,16 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 2 FROM test.t1; +--source include/wait_condition.inc + select * from t1 order by 1, 2, 3; --echo cluster 1 node 2 --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 2 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (1, 12, 3); select @@gtid_binlog_state; @@ -272,10 +312,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 3 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 1 node 3 --connection node_3 +--let $wait_condition = SELECT COUNT(*) = 3 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (1, 13, 4); select @@gtid_binlog_state; @@ -285,10 +329,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_4 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 2 node 2 --connection node_5 +--let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 22, 2); select @@gtid_binlog_state; @@ -298,10 +346,14 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 5 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo cluster 2 node 3 --connection node_6 +--let $wait_condition = SELECT COUNT(*) = 5 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; insert into t1 values (2, 23, 3); select @@gtid_binlog_state; @@ -311,24 +363,36 @@ select @@gtid_binlog_state; --source include/save_master_gtid.inc --connection node_1 --source include/sync_with_master_gtid.inc +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select * from t1 order by 1, 2, 3; --echo # check other nodes are consistent --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_3 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_5 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --connection node_6 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; select * from t1 order by 1, 2, 3; --echo cluster 1 node 1 --connection node_1 +--let $wait_condition = SELECT COUNT(*) = 6 FROM test.t1; +--source include/wait_condition.inc select @@gtid_binlog_state; drop table t1; stop slave; From 67b6f9f2851a547dcfa8de64270ce032ecf89afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 17 Apr 2025 12:32:32 +0300 Subject: [PATCH 107/125] MDEV-36618 : galera.galera_as_slave_nonprim test: result content mismatch Remove unnecessary sleeps and use wait_for_slave_to_stop instead. Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera/r/galera_as_slave_nonprim.result | 1 - mysql-test/suite/galera/t/galera_as_slave_nonprim.test | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_as_slave_nonprim.result b/mysql-test/suite/galera/r/galera_as_slave_nonprim.result index 969e844577e..3f625989fa1 100644 --- a/mysql-test/suite/galera/r/galera_as_slave_nonprim.result +++ b/mysql-test/suite/galera/r/galera_as_slave_nonprim.result @@ -12,7 +12,6 @@ connection node_1; connection node_4; INSERT INTO t1 VALUES (1),(2),(3),(4),(5); connection node_2; -connection node_1; expected_error 1 connection node_2; diff --git a/mysql-test/suite/galera/t/galera_as_slave_nonprim.test b/mysql-test/suite/galera/t/galera_as_slave_nonprim.test index a45d3fd8b6f..cf1b9a9ddaa 100644 --- a/mysql-test/suite/galera/t/galera_as_slave_nonprim.test +++ b/mysql-test/suite/galera/t/galera_as_slave_nonprim.test @@ -2,7 +2,7 @@ # Test the behavior of a Galera async slave if it goes non-prim. Async replication # should abort with an error but it should be possible to restart it. # -# The galera/galera_2node_slave.cnf describes the setup of the nodes +# The galera_3nodes_as_slave.cnf describes the setup of the nodes # --source include/have_innodb.inc @@ -45,9 +45,8 @@ SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1'; INSERT INTO t1 VALUES (1),(2),(3),(4),(5); --connection node_2 ---sleep 5 +wait_for_slave_to_stop; --let $value = query_get_value(SHOW SLAVE STATUS, Last_SQL_Error, 1) ---connection node_1 --disable_query_log --eval SELECT "$value" IN ("Error 'Unknown command' on query. Default database: 'test'. Query: 'BEGIN'", "Node has dropped from cluster") AS expected_error --enable_query_log @@ -75,7 +74,6 @@ START SLAVE; --connection node_4 DROP TABLE t1; ---sleep 2 --connection node_2 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc From fe25a30a92ffca11b2d10fec8a65937def50acbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 22 Apr 2025 08:57:16 +0300 Subject: [PATCH 108/125] MDEV-36620 : galera_toi_ddl_nonconflicting test failure Test changes only. Idea of the test is to test two concurrent nonconflicting DDL-clauses. Therefore, use debug sync to really execute two DDL-clauses as concurrently as Galera allows. Signed-off-by: Julius Goryavsky --- .../r/galera_toi_ddl_nonconflicting.result | 80 ++++++++++++++----- .../t/galera_toi_ddl_nonconflicting.test | 68 ++++++++++++---- 2 files changed, 113 insertions(+), 35 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_toi_ddl_nonconflicting.result b/mysql-test/suite/galera/r/galera_toi_ddl_nonconflicting.result index 5412cd3faee..abdf46f84ba 100644 --- a/mysql-test/suite/galera/r/galera_toi_ddl_nonconflicting.result +++ b/mysql-test/suite/galera/r/galera_toi_ddl_nonconflicting.result @@ -1,29 +1,69 @@ connection node_2; connection node_1; +connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; +connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; +connection node_1; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 INTEGER); +INSERT INTO t1(f2) SELECT seq FROM seq_1_to_1000; +connection node_2a; +SET SESSION wsrep_sync_wait=0; +connection node_1a; +# Block the applier on node_1 and issue a ddl from node_2 +SET SESSION wsrep_sync_wait=0; +SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync'; connection node_2; -ALTER TABLE t1 ADD COLUMN f3 INTEGER; INSERT INTO t1 (f1, f2) VALUES (DEFAULT, 123);; +# DDL 1 +ALTER TABLE t1 ADD COLUMN f3 INTEGER; INSERT INTO t1 VALUES (NULL, 10000, 10000);; +connection node_1a; +SET SESSION wsrep_on = 0; +SET SESSION wsrep_on = 1; +SET GLOBAL wsrep_provider_options = 'dbug='; +# This will block on acquiring total order isolation connection node_1; +# DDL 2 CREATE UNIQUE INDEX i1 ON t1(f2);; +connection node_1a; +# Signal DDL 1 +SET GLOBAL wsrep_provider_options = 'dbug='; +SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync'; connection node_2; -INSERT INTO t1 (f1, f2) VALUES (DEFAULT, 234); -SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; -COUNT(*) = 3 -1 -SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; -COUNT(*) = 2 -1 -SELECT COUNT(*) = 2 FROM t1; -COUNT(*) = 2 -1 connection node_1; -SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; -COUNT(*) = 3 -1 -SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; -COUNT(*) = 2 -1 -SELECT COUNT(*) = 2 FROM t1; -COUNT(*) = 2 -1 +connection node_2; +SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +EXPECT_3 +3 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; +EXPECT_2 +2 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `f1` int(11) NOT NULL AUTO_INCREMENT, + `f2` int(11) DEFAULT NULL, + `f3` int(11) DEFAULT NULL, + PRIMARY KEY (`f1`), + UNIQUE KEY `i1` (`f2`) +) ENGINE=InnoDB AUTO_INCREMENT=2002 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT COUNT(*) AS EXPECT_1001 FROM t1; +EXPECT_1001 +1001 +connection node_1; +SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +EXPECT_3 +3 +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; +EXPECT_2 +2 +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `f1` int(11) NOT NULL AUTO_INCREMENT, + `f2` int(11) DEFAULT NULL, + `f3` int(11) DEFAULT NULL, + PRIMARY KEY (`f1`), + UNIQUE KEY `i1` (`f2`) +) ENGINE=InnoDB AUTO_INCREMENT=2047 DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT COUNT(*) AS EXPECT_1001 FROM t1; +EXPECT_1001 +1001 DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_toi_ddl_nonconflicting.test b/mysql-test/suite/galera/t/galera_toi_ddl_nonconflicting.test index d431fc0b9ed..9836aaa57ef 100644 --- a/mysql-test/suite/galera/t/galera_toi_ddl_nonconflicting.test +++ b/mysql-test/suite/galera/t/galera_toi_ddl_nonconflicting.test @@ -1,43 +1,81 @@ --source include/galera_cluster.inc --source include/have_innodb.inc +--source include/have_sequence.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/galera_have_debug_sync.inc # # In this test, we simultaneously send two non-conflicting ALTER TABLE statements # +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2 +--connection node_1 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 INTEGER); +INSERT INTO t1(f2) SELECT seq FROM seq_1_to_1000; ---connection node_2 +--connection node_2a +SET SESSION wsrep_sync_wait=0; --let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; --source include/wait_condition.inc ---send ALTER TABLE t1 ADD COLUMN f3 INTEGER; INSERT INTO t1 (f1, f2) VALUES (DEFAULT, 123); +--let $wait_condition = SELECT COUNT(*) = 1000 FROM t1; +--source include/wait_condition.inc +--connection node_1a +--echo # Block the applier on node_1 and issue a ddl from node_2 +SET SESSION wsrep_sync_wait=0; +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_set_sync_point.inc + +--connection node_2 +--echo # DDL 1 +--send ALTER TABLE t1 ADD COLUMN f3 INTEGER; INSERT INTO t1 VALUES (NULL, 10000, 10000); + +--connection node_1a +--source include/galera_wait_sync_point.inc +--source include/galera_clear_sync_point.inc + +--echo # This will block on acquiring total order isolation --connection node_1 +--echo # DDL 2 --send CREATE UNIQUE INDEX i1 ON t1(f2); +--connection node_1a +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE LIKE 'acquiring total order%' or STATE LIKE 'Waiting for table metadata%' +--source include/wait_condition.inc + +--echo # Signal DDL 1 +--source include/galera_clear_sync_point.inc +--let $galera_sync_point = apply_monitor_slave_enter_sync +--source include/galera_signal_sync_point.inc + --connection node_2 --reap -INSERT INTO t1 (f1, f2) VALUES (DEFAULT, 234); - ---let $wait_condition = SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; ---source include/wait_condition.inc ---let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; ---source include/wait_condition.inc - -SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; -SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; -SELECT COUNT(*) = 2 FROM t1; --connection node_1 --reap +--connection node_2 --let $wait_condition = SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc --let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; --source include/wait_condition.inc -SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; -SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; -SELECT COUNT(*) = 2 FROM t1; +SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; +SHOW CREATE TABLE t1; +SELECT COUNT(*) AS EXPECT_1001 FROM t1; + +--connection node_1 +--let $wait_condition = SELECT COUNT(*) = 3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc + +SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +SELECT COUNT(*) AS EXPECT_2 FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 't1'; +SHOW CREATE TABLE t1; +SELECT COUNT(*) AS EXPECT_1001 FROM t1; DROP TABLE t1; From 1a044437a3944887bcbcb4eddd6ea6fd56f65c06 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 21 Mar 2025 11:43:31 +0200 Subject: [PATCH 109/125] MDEV-16523 update RocksDB to 6.29fb --- storage/rocksdb/build_rocksdb.cmake | 112 +++++++++++++----- storage/rocksdb/ha_rocksdb.cc | 9 +- .../r/corrupted_data_reads_debug.result | 10 +- .../mysql-test/rocksdb/r/drop_table3.result | 10 +- .../mysql-test/rocksdb/r/rocksdb.result | 2 +- .../rocksdb/r/rocksdb_datadir.result | 2 +- .../rocksdb/r/truncate_table3.result | 10 +- .../mysql-test/rocksdb/t/drop_table3.inc | 15 ++- storage/rocksdb/rdb_i_s.cc | 5 +- storage/rocksdb/rocksdb | 2 +- 10 files changed, 110 insertions(+), 67 deletions(-) diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index e89dbfc0627..3bb05c06caf 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -176,35 +176,53 @@ set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) # - *_test.cc # - *_bench.cc set(ROCKSDB_SOURCES + cache/cache.cc + cache/cache_entry_roles.cc + cache/cache_key.cc + cache/cache_reservation_manager.cc cache/clock_cache.cc cache/lru_cache.cc cache/sharded_cache.cc db/arena_wrapped_db_iter.cc + db/blob/blob_fetcher.cc + db/blob/blob_file_addition.cc + db/blob/blob_file_builder.cc + db/blob/blob_file_builder.cc + db/blob/blob_file_cache.cc + db/blob/blob_file_garbage.cc + db/blob/blob_file_meta.cc + db/blob/blob_file_reader.cc + db/blob/blob_garbage_meter.cc + db/blob/blob_log_format.cc + db/blob/blob_log_sequential_reader.cc + db/blob/blob_log_writer.cc + db/blob/prefetch_buffer_collection.cc db/builder.cc db/c.cc db/column_family.cc - db/compacted_db_impl.cc db/compaction/compaction.cc db/compaction/compaction_iterator.cc - db/compaction/compaction_picker.cc db/compaction/compaction_job.cc + db/compaction/compaction_picker.cc db/compaction/compaction_picker_fifo.cc db/compaction/compaction_picker_level.cc db/compaction/compaction_picker_universal.cc + db/compaction/sst_partitioner.cc db/convenience.cc db/db_filesnapshot.cc + db/dbformat.cc + db/db_impl/compacted_db_impl.cc db/db_impl/db_impl.cc - db/db_impl/db_impl_write.cc db/db_impl/db_impl_compaction_flush.cc - db/db_impl/db_impl_files.cc - db/db_impl/db_impl_open.cc db/db_impl/db_impl_debug.cc db/db_impl/db_impl_experimental.cc + db/db_impl/db_impl_files.cc + db/db_impl/db_impl_open.cc db/db_impl/db_impl_readonly.cc db/db_impl/db_impl_secondary.cc + db/db_impl/db_impl_write.cc db/db_info_dumper.cc db/db_iter.cc - db/dbformat.cc db/error_handler.cc db/event_helpers.cc db/experimental.cc @@ -215,14 +233,16 @@ set(ROCKSDB_SOURCES db/forward_iterator.cc db/import_column_family_job.cc db/internal_stats.cc - db/logs_with_prep_tracker.cc db/log_reader.cc + db/logs_with_prep_tracker.cc db/log_writer.cc db/malloc_stats.cc db/memtable.cc db/memtable_list.cc db/merge_helper.cc db/merge_operator.cc + db/output_validator.cc + db/periodic_work_scheduler.cc db/range_del_aggregator.cc db/range_tombstone_fragmenter.cc db/repair.cc @@ -233,25 +253,32 @@ set(ROCKSDB_SOURCES db/trim_history_scheduler.cc db/version_builder.cc db/version_edit.cc + db/version_edit_handler.cc db/version_set.cc + db/wal_edit.cc db/wal_manager.cc - db/write_batch.cc db/write_batch_base.cc + db/write_batch.cc db/write_controller.cc db/write_thread.cc + env/composite_env.cc env/env.cc env/env_chroot.cc env/env_encryption.cc env/env_hdfs.cc env/file_system.cc + env/file_system_tracer.cc + env/fs_remap.cc env/mock_env.cc + env/unique_id_gen.cc file/delete_scheduler.cc + file/filename.cc file/file_prefetch_buffer.cc file/file_util.cc - file/filename.cc + file/line_file_reader.cc file/random_access_file_reader.cc - file/read_write_util.cc file/readahead_raf.cc + file/read_write_util.cc file/sequence_file_reader.cc file/sst_file_manager_impl.cc file/writable_file_writer.cc @@ -281,29 +308,38 @@ set(ROCKSDB_SOURCES monitoring/thread_status_util.cc monitoring/thread_status_util_debug.cc options/cf_options.cc + options/configurable.cc + options/customizable.cc options/db_options.cc options/options.cc options/options_helper.cc options/options_parser.cc - options/options_sanity_check.cc port/stack_trace.cc table/adaptive/adaptive_table_factory.cc - table/block_based/block.cc + table/block_based/binary_search_index_reader.cc table/block_based/block_based_filter_block.cc table/block_based/block_based_table_builder.cc table/block_based/block_based_table_factory.cc + table/block_based/block_based_table_iterator.cc table/block_based/block_based_table_reader.cc table/block_based/block_builder.cc + table/block_based/block.cc + table/block_based/block_prefetcher.cc table/block_based/block_prefix_index.cc - table/block_based/data_block_hash_index.cc table/block_based/data_block_footer.cc + table/block_based/data_block_hash_index.cc table/block_based/filter_block_reader_common.cc table/block_based/filter_policy.cc table/block_based/flush_block_policy.cc table/block_based/full_filter_block.cc + table/block_based/hash_index_reader.cc table/block_based/index_builder.cc + table/block_based/index_reader_common.cc table/block_based/parsed_full_filter_block.cc table/block_based/partitioned_filter_block.cc + table/block_based/partitioned_index_iterator.cc + table/block_based/partitioned_index_reader.cc + table/block_based/reader_common.cc table/block_based/uncompression_dict_reader.cc table/block_fetcher.cc table/cuckoo/cuckoo_table_builder.cc @@ -321,10 +357,13 @@ set(ROCKSDB_SOURCES table/plain/plain_table_index.cc table/plain/plain_table_key_coding.cc table/plain/plain_table_reader.cc + table/sst_file_dumper.cc table/sst_file_reader.cc table/sst_file_writer.cc + table/table_factory.cc table/table_properties.cc table/two_level_iterator.cc + table/unique_id.cc test_util/sync_point.cc test_util/sync_point_impl.cc test_util/testutil.cc @@ -335,8 +374,12 @@ set(ROCKSDB_SOURCES tools/ldb_tool.cc tools/sst_dump_tool.cc tools/trace_analyzer_tool.cc - trace_replay/trace_replay.cc trace_replay/block_cache_tracer.cc + trace_replay/io_tracer.cc + trace_replay/trace_record.cc + trace_replay/trace_record_handler.cc + trace_replay/trace_record_result.cc + trace_replay/trace_replay.cc util/coding.cc util/compaction_job_stats_impl.cc util/comparator.cc @@ -344,17 +387,8 @@ set(ROCKSDB_SOURCES util/concurrent_task_limiter_impl.cc util/crc32c.cc util/dynamic_bloom.cc - util/hash.cc - util/murmurhash.cc - util/random.cc - util/rate_limiter.cc - util/slice.cc util/file_checksum_helper.cc - util/status.cc - util/string_util.cc - util/thread_local.cc - util/threadpool_imp.cc - util/xxhash.cc + util/hash.cc utilities/backupable/backupable_db.cc utilities/blob_db/blob_compaction_filter.cc utilities/blob_db/blob_db.cc @@ -362,10 +396,8 @@ set(ROCKSDB_SOURCES utilities/blob_db/blob_db_impl_filesnapshot.cc utilities/blob_db/blob_dump_tool.cc utilities/blob_db/blob_file.cc - utilities/blob_db/blob_log_reader.cc - utilities/blob_db/blob_log_writer.cc - utilities/blob_db/blob_log_format.cc utilities/checkpoint/checkpoint_impl.cc + utilities/compaction_filters.cc utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc utilities/debug.cc utilities/env_mirror.cc @@ -373,11 +405,12 @@ set(ROCKSDB_SOURCES utilities/leveldb_options/leveldb_options.cc utilities/memory/memory_util.cc utilities/merge_operators/bytesxor.cc + utilities/merge_operators.cc utilities/merge_operators/max.cc utilities/merge_operators/put.cc utilities/merge_operators/sortlist.cc - utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/string_append/stringappend2.cc + utilities/merge_operators/string_append/stringappend.cc utilities/merge_operators/uint64add.cc utilities/object_registry.cc utilities/option_change_migration/option_change_migration.cc @@ -391,22 +424,37 @@ set(ROCKSDB_SOURCES utilities/simulator_cache/sim_cache.cc utilities/table_properties_collectors/compact_on_deletion_collector.cc utilities/trace/file_trace_reader_writer.cc - utilities/transactions/optimistic_transaction_db_impl.cc + utilities/trace/replayer_impl.cc + utilities/transactions/lock/lock_manager.cc + utilities/transactions/lock/point/point_lock_manager.cc + utilities/transactions/lock/point/point_lock_tracker.cc utilities/transactions/optimistic_transaction.cc + utilities/transactions/optimistic_transaction_db_impl.cc utilities/transactions/pessimistic_transaction.cc utilities/transactions/pessimistic_transaction_db.cc utilities/transactions/snapshot_checker.cc utilities/transactions/transaction_base.cc utilities/transactions/transaction_db_mutex_impl.cc - utilities/transactions/transaction_lock_mgr.cc utilities/transactions/transaction_util.cc utilities/transactions/write_prepared_txn.cc utilities/transactions/write_prepared_txn_db.cc utilities/transactions/write_unprepared_txn.cc utilities/transactions/write_unprepared_txn_db.cc utilities/ttl/db_ttl_impl.cc + utilities/wal_filter.cc utilities/write_batch_with_index/write_batch_with_index.cc utilities/write_batch_with_index/write_batch_with_index_internal.cc + util/murmurhash.cc + util/random.cc + util/rate_limiter.cc + util/regex.cc + util/ribbon_config.cc + util/slice.cc + util/status.cc + util/string_util.cc + util/thread_local.cc + util/threadpool_imp.cc + util/xxhash.cc ) @@ -484,8 +532,10 @@ IF(CMAKE_VERSION VERSION_GREATER "2.8.10") STRING(TIMESTAMP GIT_DATE_TIME "%Y-%m-%d %H:%M:%S") ENDIF() +# psergey-added: +SET(GIT_MOD 0) CONFIGURE_FILE(${ROCKSDB_SOURCE_DIR}/util/build_version.cc.in build_version.cc @ONLY) -INCLUDE_DIRECTORIES(${ROCKSDB_SOURCE_DIR}/util) + list(APPEND SOURCES ${CMAKE_CURRENT_BINARY_DIR}/build_version.cc) ADD_CONVENIENCE_LIBRARY(rocksdblib ${SOURCES}) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index c72f0c73de0..a050d0a3fe6 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -1596,7 +1596,7 @@ static MYSQL_SYSVAR_BOOL( "BlockBasedTableOptions::no_block_cache for RocksDB", nullptr, nullptr, rocksdb_tbl_options->no_block_cache); -static MYSQL_SYSVAR_SIZE_T(block_size, rocksdb_tbl_options->block_size, +static MYSQL_SYSVAR_UINT64_T(block_size, rocksdb_tbl_options->block_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "BlockBasedTableOptions::block_size for RocksDB", nullptr, nullptr, rocksdb_tbl_options->block_size, @@ -3992,7 +3992,7 @@ static int rocksdb_commit_by_xid(handlerton *const hton, XID *const xid) { DBUG_ASSERT(xid != nullptr); DBUG_ASSERT(commit_latency_stats != nullptr); - rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true); + rocksdb::StopWatchNano timer(rocksdb::SystemClock::Default().get(), true); const auto name = rdb_xid_to_string(*xid); DBUG_ASSERT(!name.empty()); @@ -4187,7 +4187,7 @@ static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx) DBUG_ASSERT(thd != nullptr); DBUG_ASSERT(commit_latency_stats != nullptr); - rocksdb::StopWatchNano timer(rocksdb::Env::Default(), true); + rocksdb::StopWatchNano timer(rocksdb::SystemClock::Default().get(), true); /* note: h->external_lock(F_UNLCK) is called after this function is called) */ Rdb_transaction *tx = get_tx_from_thd(thd); @@ -4732,8 +4732,7 @@ static bool rocksdb_show_status(handlerton *const hton, THD *const thd, if (tf_name.find("BlockBasedTable") != std::string::npos) { const rocksdb::BlockBasedTableOptions *const bbt_opt = - reinterpret_cast( - table_factory->GetOptions()); + table_factory->GetOptions(); if (bbt_opt != nullptr) { if (bbt_opt->block_cache.get() != nullptr) { diff --git a/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result b/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result index 9a12f51e3b5..7fb80995ece 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result @@ -20,7 +20,7 @@ set @tmp1=@@rocksdb_verify_row_debug_checksums; set rocksdb_verify_row_debug_checksums=1; set session debug_dbug= "+d,myrocks_simulate_bad_row_read1"; select * from t1 where pk=1; -ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_row_read1"; set rocksdb_verify_row_debug_checksums=@tmp1; select * from t1 where pk=1; @@ -28,11 +28,11 @@ pk col1 1 1 set session debug_dbug= "+d,myrocks_simulate_bad_row_read2"; select * from t1 where pk=1; -ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_row_read2"; set session debug_dbug= "+d,myrocks_simulate_bad_row_read3"; select * from t1 where pk=1; -ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_row_read3"; insert into t1 values(4,'0123456789'); select * from t1; @@ -56,7 +56,7 @@ pk col1 ABCD 1 set session debug_dbug= "+d,myrocks_simulate_bad_pk_read1"; select * from t2; -ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_pk_read1"; drop table t2; create table t2 ( @@ -69,6 +69,6 @@ pk col1 ABCD 1 set session debug_dbug= "+d,myrocks_simulate_bad_pk_read1"; select * from t2; -ERROR HY000: Got error 205 'Found data corruption.' from ROCKSDB +ERROR HY000: Got error 206 'Found data corruption.' from ROCKSDB set session debug_dbug= "-d,myrocks_simulate_bad_pk_read1"; drop table t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/drop_table3.result b/storage/rocksdb/mysql-test/rocksdb/r/drop_table3.result index 954e6079bba..de8cbcbcd02 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/drop_table3.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/drop_table3.result @@ -1,12 +1,6 @@ -call mtr.add_suppression("Column family 'cf1' not found"); -call mtr.add_suppression("Column family 'rev:cf2' not found"); DROP TABLE IF EXISTS t1; call mtr.add_suppression("Column family 'cf1' not found"); call mtr.add_suppression("Column family 'rev:cf2' not found"); -set global rocksdb_compact_cf = 'cf1'; -set global rocksdb_compact_cf = 'rev:cf2'; -set global rocksdb_signal_drop_index_thread = 1; -# restart CREATE TABLE t1 ( a int not null, b int not null, @@ -15,6 +9,10 @@ primary key (a,b) comment 'cf1', key (b) comment 'rev:cf2' ) ENGINE=RocksDB; DELETE FROM t1; +set global rocksdb_compact_cf = 'cf1'; +set global rocksdb_compact_cf = 'rev:cf2'; +set global rocksdb_signal_drop_index_thread = 1; +# restart select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; drop table t1; select case when variable_value-@a < 500000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result index 989d28e773d..683beacf0b6 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result @@ -982,7 +982,7 @@ rocksdb_skip_fill_cache OFF rocksdb_skip_unique_check_tables .* rocksdb_sst_mgr_rate_bytes_per_sec 0 rocksdb_stats_dump_period_sec 600 -rocksdb_stats_level 0 +rocksdb_stats_level 1 rocksdb_stats_recalc_rate 0 rocksdb_store_row_debug_checksums OFF rocksdb_strict_collation_check OFF diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_datadir.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_datadir.result index 40c53f6fd8a..51f4ecb9684 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_datadir.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_datadir.result @@ -1,2 +1,2 @@ Check for MANIFEST files -MANIFEST-000006 +MANIFEST-000004 diff --git a/storage/rocksdb/mysql-test/rocksdb/r/truncate_table3.result b/storage/rocksdb/mysql-test/rocksdb/r/truncate_table3.result index 7c5631a2c97..359124e83ac 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/truncate_table3.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/truncate_table3.result @@ -1,12 +1,6 @@ -call mtr.add_suppression("Column family 'cf1' not found"); -call mtr.add_suppression("Column family 'rev:cf2' not found"); DROP TABLE IF EXISTS t1; call mtr.add_suppression("Column family 'cf1' not found"); call mtr.add_suppression("Column family 'rev:cf2' not found"); -set global rocksdb_compact_cf = 'cf1'; -set global rocksdb_compact_cf = 'rev:cf2'; -set global rocksdb_signal_drop_index_thread = 1; -# restart CREATE TABLE t1 ( a int not null, b int not null, @@ -15,6 +9,10 @@ primary key (a,b) comment 'cf1', key (b) comment 'rev:cf2' ) ENGINE=RocksDB; DELETE FROM t1; +set global rocksdb_compact_cf = 'cf1'; +set global rocksdb_compact_cf = 'rev:cf2'; +set global rocksdb_signal_drop_index_thread = 1; +# restart select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; truncate table t1; select case when variable_value-@a < 500000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/drop_table3.inc b/storage/rocksdb/mysql-test/rocksdb/t/drop_table3.inc index dd9a19a29cb..033166ae946 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/drop_table3.inc +++ b/storage/rocksdb/mysql-test/rocksdb/t/drop_table3.inc @@ -1,8 +1,5 @@ --source include/have_rocksdb.inc -call mtr.add_suppression("Column family 'cf1' not found"); -call mtr.add_suppression("Column family 'rev:cf2' not found"); - --disable_warnings DROP TABLE IF EXISTS t1; --enable_warnings @@ -10,11 +7,6 @@ DROP TABLE IF EXISTS t1; call mtr.add_suppression("Column family 'cf1' not found"); call mtr.add_suppression("Column family 'rev:cf2' not found"); -# Start from clean slate -set global rocksdb_compact_cf = 'cf1'; -set global rocksdb_compact_cf = 'rev:cf2'; -set global rocksdb_signal_drop_index_thread = 1; ---source include/restart_mysqld.inc CREATE TABLE t1 ( a int not null, @@ -29,6 +21,12 @@ let $max = 50000; let $table = t1; --source drop_table3_repopulate_table.inc +# Start from clean slate +set global rocksdb_compact_cf = 'cf1'; +set global rocksdb_compact_cf = 'rev:cf2'; +set global rocksdb_signal_drop_index_thread = 1; +--source include/restart_mysqld.inc + --disable_cursor_protocol select variable_value into @a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; --enable_cursor_protocol @@ -49,6 +47,7 @@ let $wait_condition = select count(*) = 0 --source include/wait_condition.inc select case when variable_value-@a < 500000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; +#select variable_value-@a from information_schema.global_status where variable_name='rocksdb_compact_read_bytes'; # Cleanup DROP TABLE IF EXISTS t1; diff --git a/storage/rocksdb/rdb_i_s.cc b/storage/rocksdb/rdb_i_s.cc index c830c59a38b..6906da34e30 100644 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@ -587,8 +587,7 @@ static int rdb_i_s_cfoptions_fill_table( cf_option_types.push_back( {"PREFIX_EXTRACTOR", opts.prefix_extractor == nullptr ? "NULL" - : std::string(opts.prefix_extractor->Name())}); - + : std::string(opts.prefix_extractor->AsString())}); // get COMPACTION_STYLE option switch (opts.compaction_style) { case rocksdb::kCompactionStyleLevel: @@ -646,7 +645,7 @@ static int rdb_i_s_cfoptions_fill_table( // get table related options std::vector table_options = - split_into_vector(opts.table_factory->GetPrintableTableOptions(), '\n'); + split_into_vector(opts.table_factory->GetPrintableOptions(), '\n'); for (auto option : table_options) { option.erase(std::remove(option.begin(), option.end(), ' '), diff --git a/storage/rocksdb/rocksdb b/storage/rocksdb/rocksdb index bba5e7bc210..79f08d7ffa6 160000 --- a/storage/rocksdb/rocksdb +++ b/storage/rocksdb/rocksdb @@ -1 +1 @@ -Subproject commit bba5e7bc21093d7cfa765e1280a7c4fdcd284288 +Subproject commit 79f08d7ffa6d34d9ca3357777bcb335884a56cfb From f5405ef5114ff946d01abac9ea93a59b732cca1a Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Tue, 22 Apr 2025 10:53:35 +0200 Subject: [PATCH 110/125] RPM fixes for centos centos can go up to "centosstream10" now --- cmake/cpack_rpm.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 50a08f475f7..4147c91e3b7 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -270,7 +270,7 @@ ELSEIF(RPM MATCHES "fedora" OR RPM MATCHES "(rhel|centos)7") ALTERNATIVE_NAME("server" "mariadb-server") ALTERNATIVE_NAME("server" "mysql-compat-server") ALTERNATIVE_NAME("test" "mariadb-test") -ELSEIF(RPM MATCHES "(rhel|centos|rocky)[89]") +ELSEIF(RPM MATCHES "(rhel|centos|rocky)") SET(epoch 3:) ALTERNATIVE_NAME("backup" "mariadb-backup") ALTERNATIVE_NAME("client" "mariadb") From 8925877dc86bdcff0170e1b0ec21483cc57a211a Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Wed, 16 Apr 2025 08:06:15 +1000 Subject: [PATCH 111/125] MDEV-36591: RHEL8(+compat)/Ubuntu 20.04 cannot start systemd servce (EXIT_CAPABILTIES/218) Combined AmbientCapabilities and CapabilityBoundingSet configuration within a service file we have found by testing aren't supported in the systemd v245 (Ubuntu 20.04) and v239 (RHEL8) for non-root users. This resulted in a service start error EXIT_CAPABILITIES, a systemd limitation of the version that we cannot work around consequences. The systemd version 247 these combined capabilities have been tested to work on Debian 11. No other supported major distros run systemd version 246, and if they did, the missing capability of CAP_IPC_LOCK won't be noticed as it was a convenience for --memlock users. As such we disable the AmbientCapabilites for CAP_IPC_LOCK rather that disabling the CapabilityBoundingSet, because doing the later will disable authentication for MariaDB users that have configured PAM with MariaDB. Should a user require CAP_IPC_LOCK they can append in their own systemd overlay file this configuration in the CapabilityBoundingSet and configure the capability file attributes on the mariadbd executable to have the IPC_LOCK capability. This isn't configured by default as the presence of a capability in the MariaDB Server is detected by openssl libraries as "insecure" which will then ignore any user configured TLS configuration file passed though by the OPENSSL_CONF environment variable. --- cmake/systemd.cmake | 9 +++++++++ support-files/mariadb.service.in | 5 +---- support-files/mariadb@.service.in | 5 +---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cmake/systemd.cmake b/cmake/systemd.cmake index a093b89f3c8..404441be747 100644 --- a/cmake/systemd.cmake +++ b/cmake/systemd.cmake @@ -57,6 +57,15 @@ MACRO(CHECK_SYSTEMD) # ProtectSystem=full prevents it ReadWritePaths=-${MYSQL_DATADIR}\n") ENDIF() + # systemd version 245 (Ubuntu 20.04) and less cannot + # handle ambient capbilities on non-root processes + # 247 (Debian 11) is a version afterwards that is known to work. + IF(LIBSYSTEMD_VERSION VERSION_GREATER_EQUAL 247) + SET(SYSTEMD_AMBIENT_CAPABILITIES +"# CAP_IPC_LOCK To allow --memlock to be used as non-root user +AmbientCapabilities=CAP_IPC_LOCK +") + ENDIF() MESSAGE_ONCE(systemd "Systemd features enabled") ELSE() diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in index f53a0b8ceda..7ae0278dc44 100644 --- a/support-files/mariadb.service.in +++ b/support-files/mariadb.service.in @@ -47,10 +47,7 @@ PrivateNetwork=false User=mysql Group=mysql -# CAP_IPC_LOCK To allow memlock to be used as non-root user -# These are enabled by default -AmbientCapabilities=CAP_IPC_LOCK - +@SYSTEMD_AMBIENT_CAPABILITIES@ # CAP_DAC_OVERRIDE To allow auth_pam_tool (which is SUID root) to read /etc/shadow when it's chmod 0 # does nothing for non-root, not needed if /etc/shadow is u+r # CAP_AUDIT_WRITE auth_pam_tool needs it on Debian for whatever reason diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index cb5e885e1aa..4095f04b800 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -177,10 +177,7 @@ PrivateNetwork=false ## Package maintainers ## -# CAP_IPC_LOCK To allow memlock to be used as non-root user -# These are enabled by default -AmbientCapabilities=CAP_IPC_LOCK - +@SYSTEMD_AMBIENT_CAPABILITIES@ # CAP_DAC_OVERRIDE To allow auth_pam_tool (which is SUID root) to read /etc/shadow when it's chmod 0 # does nothing for non-root, not needed if /etc/shadow is u+r # CAP_AUDIT_WRITE auth_pam_tool needs it on Debian for whatever reason From 2fa50befbd0ddc8046c9de78983670c27ecc7586 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 23 Apr 2025 10:50:39 +0200 Subject: [PATCH 112/125] rpm: restore MariaDB-test dependency on MariaDB-common after 41b036bff0bc --- cmake/cpack_rpm.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 4147c91e3b7..ad58e5ea151 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -223,7 +223,7 @@ IF(WITH_WSREP) "galera-4" "rsync" "grep" "gawk" "iproute" "coreutils" "findutils" "tar") SETA(CPACK_RPM_server_PACKAGE_RECOMMENDS "lsof" "socat") - SETA(CPACK_RPM_test_PACKAGE_REQUIRES "socat") + SETA(CPACK_RPM_test_PACKAGE_REQUIRES "${CPACK_RPM_PACKAGE_REQUIRES}" "socat") ENDIF() SET(CPACK_RPM_server_PRE_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/support-files/rpm/server-prein.sh) From f1a8b7fe95399ebe2a1c4a370e332d61dbf6891a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 23 Apr 2025 15:42:12 +0300 Subject: [PATCH 113/125] MDEV-36646: innodb_buffer_pool_size change aborted A statement SET GLOBAL innodb_buffer_pool_size=... could fail for no good reason when the buffer pool contains many pages that can actually be evicted. buf_flush_LRU_list_batch(): Keep evicting as long as the buffer pool is being shrunk, for at most innodb_lru_scan_depth extra blocks. Disregard the flush limit for pages that are marked as freed in files. buf_flush_LRU_to_withdraw(): Update the to_withdraw target during buf_flush_LRU_list_batch(). buf_pool_t::will_be_withdrawn(): Allow also ptr=nullptr (the condition will not hold for it). This fixes a regression that was introduced in commit b6923420f326ac030e4f3ef89a2acddb45eccb30 (MDEV-29445) and caught by the test innodb.temp_truncate_freed in MariaDB Server 11.4. Tested by: Thirunarayanan Balathandayuthapani Reviewed by: Thirunarayanan Balathandayuthapani --- storage/innobase/buf/buf0buf.cc | 3 +-- storage/innobase/buf/buf0flu.cc | 39 ++++++++++++++++++++++++++---- storage/innobase/include/buf0buf.h | 2 +- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index c3da36ede3a..261a796141d 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1696,8 +1696,7 @@ ATTRIBUTE_COLD buf_pool_t::shrink_status buf_pool_t::shrink(size_t size) continue; } - if (UNIV_LIKELY_NULL(b->zip.data) && - will_be_withdrawn(b->zip.data, size)) + if (UNIV_UNLIKELY(will_be_withdrawn(b->zip.data, size))) { block= buf_buddy_shrink(b, block); ut_ad(mach_read_from_4(b->zip.data + FIL_PAGE_OFFSET) == id.page_no()); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 13eabd92f3b..ad026abf64f 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1212,6 +1212,21 @@ static void buf_flush_discard_page(buf_page_t *bpage) noexcept buf_LRU_free_page(bpage, true); } +/** Adjust to_withdraw during buf_pool_t::shrink() */ +ATTRIBUTE_COLD static size_t buf_flush_LRU_to_withdraw(size_t to_withdraw, + const buf_page_t &bpage) + noexcept +{ + mysql_mutex_assert_owner(&buf_pool.mutex); + if (!buf_pool.is_shrinking()) + return 0; + const size_t size{buf_pool.size_in_bytes_requested}; + if (buf_pool.will_be_withdrawn(bpage.frame, size) || + buf_pool.will_be_withdrawn(bpage.zip.data, size)) + to_withdraw--; + return to_withdraw; +} + /** Flush dirty blocks from the end buf_pool.LRU, and move clean blocks to buf_pool.free. @param max maximum number of blocks to flush @@ -1222,7 +1237,9 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, { size_t scanned= 0; mysql_mutex_assert_owner(&buf_pool.mutex); - size_t free_limit{buf_pool.LRU_scan_depth + to_withdraw}; + size_t free_limit{buf_pool.LRU_scan_depth}; + if (UNIV_UNLIKELY(to_withdraw > free_limit)) + to_withdraw= free_limit; const auto neighbors= UT_LIST_GET_LEN(buf_pool.LRU) < BUF_LRU_OLD_MIN_LEN ? 0 : buf_pool.flush_neighbors; fil_space_t *space= nullptr; @@ -1246,6 +1263,7 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, bpage && ((UT_LIST_GET_LEN(buf_pool.LRU) > buf_lru_min_len && UT_LIST_GET_LEN(buf_pool.free) < free_limit) || + to_withdraw || recv_recovery_is_on()); ++scanned, bpage= buf_pool.lru_hp.get()) { @@ -1261,6 +1279,8 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, if (state != buf_page_t::FREED && (state >= buf_page_t::READ_FIX || (~buf_page_t::LRU_MASK & state))) continue; + if (UNIV_UNLIKELY(to_withdraw != 0)) + to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); buf_LRU_free_page(bpage, true); ++n->evicted; if (UNIV_LIKELY(scanned & 31)) @@ -1332,23 +1352,32 @@ static void buf_flush_LRU_list_batch(ulint max, flush_counters_t *n, continue; } + if (state < buf_page_t::UNFIXED) + goto flush; + if (n->flushed >= max && !recv_recovery_is_on()) { bpage->lock.u_unlock(true); break; } - if (neighbors && space->is_rotational() && + if (neighbors && space->is_rotational() && UNIV_LIKELY(!to_withdraw) && /* Skip neighbourhood flush from LRU list if we haven't yet reached half of the free page target. */ UT_LIST_GET_LEN(buf_pool.free) * 2 >= free_limit) n->flushed+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, n->flushed, max); - else if (bpage->flush(space)) - ++n->flushed; else - continue; + { + flush: + if (UNIV_UNLIKELY(to_withdraw != 0)) + to_withdraw= buf_flush_LRU_to_withdraw(to_withdraw, *bpage); + if (bpage->flush(space)) + ++n->flushed; + else + continue; + } goto reacquire_mutex; } diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index caa9093c6d1..202771fcfe1 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1288,7 +1288,7 @@ public: bool will_be_withdrawn(const byte *ptr, size_t size) const noexcept { const char *p= reinterpret_cast(ptr); - ut_ad(p >= memory); + ut_ad(!p || p >= memory); ut_ad(p < memory + size_in_bytes_max); return p >= memory + size; } From 76938eda9de68f7facf70a33f165c98aa1a2c9b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 23 Apr 2025 17:06:33 +0300 Subject: [PATCH 114/125] MDEV-36625 : galera.galera_var_replicate_myisam_on test failure Test case changes only. Add proper wait_conditions to wait expected database state. Signed-off-by: Julius Goryavsky --- .../r/galera_var_replicate_myisam_on.result | 6 ++ .../t/galera_var_replicate_myisam_on.test | 64 ++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/r/galera_var_replicate_myisam_on.result b/mysql-test/suite/galera/r/galera_var_replicate_myisam_on.result index 3e616552368..f3b0947ed9a 100644 --- a/mysql-test/suite/galera/r/galera_var_replicate_myisam_on.result +++ b/mysql-test/suite/galera/r/galera_var_replicate_myisam_on.result @@ -202,6 +202,9 @@ id b 3 200 4 5 connection node_2; +SELECT COUNT(*) FROM t1; +COUNT(*) +10 SELECT * FROM t1 ORDER BY id; id b 1 1 @@ -227,6 +230,9 @@ DROP TABLE t1, t2; CREATE TABLE t1 (a INT, b INT, UNIQUE(a)) ENGINE=MyISAM; CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW SET NEW.a=1; INSERT INTO t1 (a,b) VALUES (10,20); +SELECT * from t1; +a b +1 20 connection node_2; SELECT * from t1; a b diff --git a/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test b/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test index ad0a7a37d63..023b32a203b 100644 --- a/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test +++ b/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test @@ -23,6 +23,11 @@ INSERT INTO t1 VALUES (2), (3); INSERT INTO t1 SELECT 4 FROM DUAL UNION ALL SELECT 5 FROM DUAL; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 5 FROM t1; +--source include/wait_condition.inc + SELECT COUNT(*) AS EXPECT_5 FROM t1; DROP TABLE t1; @@ -38,6 +43,13 @@ REPLACE INTO t1 VALUES (1, 'klm'), (2,'xyz'); REPLACE INTO t1 SELECT 3, 'yyy' FROM DUAL; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 3 FROM t1; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1 WHERE f1 = 3 AND f2 = 'yyy'; +--source include/wait_condition.inc + SELECT COUNT(*) AS EXPECT_3 FROM t1; SELECT COUNT(*) AS EXPECT_1 FROM t1 WHERE f1 = 1 AND f2 = 'klm'; SELECT COUNT(*) AS EXPECT_1 FROM t1 WHERE f1 = 2 AND f2 = 'xyz'; @@ -51,6 +63,9 @@ SELECT COUNT(*) AS EXPECT_1 FROM t1 WHERE f1 = 3 AND f2 = 'yyy'; UPDATE t1 SET f2 = 'zzz' WHERE f2 = 'yyy'; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1 WHERE f2 = 'zzz'; +--source include/wait_condition.inc + SELECT COUNT(*) AS EXPECT_1 FROM t1 WHERE f2 = 'zzz'; # @@ -61,6 +76,9 @@ SELECT COUNT(*) AS EXPECT_1 FROM t1 WHERE f2 = 'zzz'; DELETE FROM t1 WHERE f2 = 'zzz'; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM t1 WHERE f2 = 'zzz'; +--source include/wait_condition.inc + SELECT COUNT(*) AS EXPECT_0 FROM t1 WHERE f2 = 'zzz'; # @@ -71,6 +89,9 @@ SELECT COUNT(*) AS EXPECT_0 FROM t1 WHERE f2 = 'zzz'; TRUNCATE TABLE t1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM t1; +--source include/wait_condition.inc + SELECT COUNT(*) AS EXPECT_0 FROM t1; DROP TABLE t1; @@ -88,6 +109,15 @@ INSERT INTO t2 VALUES (1); COMMIT; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM t2; +--source include/wait_condition.inc + SELECT COUNT(*) AS EXPECT_1 FROM t1; SELECT COUNT(*) AS EXPECT_1 FROM t2; @@ -102,6 +132,11 @@ INSERT INTO t2 VALUES (2); ROLLBACK; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 2 FROM t1; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM t2; +--source include/wait_condition.inc + SELECT COUNT(*) AS EXPECT_2 FROM t1; SELECT COUNT(*) AS EXPECT_1 FROM t2; @@ -120,7 +155,13 @@ INSERT INTO t1 VALUES (1); INSERT INTO t2 VALUES (1); --connection node_2 -# The MyISAM update is replicated immediately, so a duplicate key error happens even before the COMMIT +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc +# The MyISAM update is replicated when executed, so a duplicate key error happens even before the COMMIT --error ER_DUP_ENTRY INSERT INTO t1 VALUES (1); @@ -149,6 +190,10 @@ EXECUTE rep; SELECT * FROM t1 ORDER BY id; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 11 FROM t1; +--source include/wait_condition.inc SELECT * FROM t1 ORDER BY id; DROP TABLE t1; @@ -175,6 +220,10 @@ CALL proc(); SELECT * FROM t1 ORDER BY id; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 11 FROM t1; +--source include/wait_condition.inc SELECT * FROM t1 ORDER BY id; DROP PROCEDURE proc; @@ -198,6 +247,13 @@ SELECT * FROM t1 ORDER BY id; SELECT * FROM t2 ORDER BY id; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2'; +--source include/wait_condition.inc +SELECT COUNT(*) FROM t1; +--let $wait_condition = SELECT COUNT(*) = 10 FROM t1; +--source include/wait_condition.inc SELECT * FROM t1 ORDER BY id; SELECT * FROM t2 ORDER BY id; DROP TRIGGER tr1; @@ -208,8 +264,14 @@ DROP TABLE t1, t2; CREATE TABLE t1 (a INT, b INT, UNIQUE(a)) ENGINE=MyISAM; CREATE TRIGGER tr1 BEFORE INSERT ON t1 FOR EACH ROW SET NEW.a=1; INSERT INTO t1 (a,b) VALUES (10,20); +SELECT * from t1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc + SELECT * from t1; --connection node_1 DROP TABLE t1; From 2f5c260f55d9c7a8bf61ee78ee228ac212a1518d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 23 Apr 2025 15:32:51 +0300 Subject: [PATCH 115/125] MDEV-36514 : galera_var_ignore_apply_errors test failure: table doesn't exist Test changes only. Added proper wait_conditions to wait for expected replication state. Signed-off-by: Julius Goryavsky --- .../t/galera_var_ignore_apply_errors.test | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.test b/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.test index feec79196c3..9364b9b52d6 100644 --- a/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.test +++ b/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.test @@ -22,6 +22,8 @@ SET GLOBAL wsrep_on = ON; DROP TABLE t1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc SHOW TABLES; # Drop schema that does not exist @@ -33,6 +35,8 @@ SET GLOBAL wsrep_on = ON; DROP SCHEMA s1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME LIKE 's1'; +--source include/wait_condition.inc SHOW SCHEMAS; # Drop index that does not exist using DROP INDEX @@ -45,6 +49,10 @@ SET GLOBAL wsrep_on = ON; DROP INDEX idx1 ON t1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_SYS_INDEXES WHERE NAME LIKE 'idx1'; +--source include/wait_condition.inc SHOW CREATE TABLE t1; DROP TABLE t1; @@ -58,6 +66,10 @@ SET GLOBAL wsrep_on = ON; ALTER TABLE t1 DROP INDEX idx1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_SYS_INDEXES WHERE NAME LIKE 'idx1'; +--source include/wait_condition.inc SHOW CREATE TABLE t1; DROP TABLE t1; @@ -71,6 +83,11 @@ SET GLOBAL wsrep_on = ON; ALTER TABLE t1 DROP COLUMN f2; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_SYS_COLUMNS WHERE NAME LIKE 'f2'; +--source include/wait_condition.inc + SHOW CREATE TABLE t1; DROP TABLE t1; @@ -93,6 +110,10 @@ DELETE FROM t1 WHERE f1 = 1; SELECT COUNT(*) AS expect_0 FROM t1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 0 FROM t1; +--source include/wait_condition.inc SELECT COUNT(*) AS expect_0 FROM t1; DROP TABLE t1; @@ -112,6 +133,10 @@ COMMIT; SELECT COUNT(*) AS expect_1 FROM t1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM t1; +--source include/wait_condition.inc SELECT COUNT(*) AS expect_1 FROM t1; DROP TABLE t1; @@ -136,6 +161,8 @@ DELETE FROM t1; SELECT COUNT(*) AS expect_0 FROM t1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc --let $wait_condition = SELECT COUNT(*) = 0 FROM t1; --source include/wait_condition.inc SELECT VARIABLE_VALUE expect_Primary FROM performance_schema.global_status WHERE VARIABLE_NAME = 'wsrep_cluster_status'; @@ -171,6 +198,8 @@ SET AUTOCOMMIT=ON; SELECT COUNT(*) AS expect_0 FROM t1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc --let $wait_condition = SELECT COUNT(*) = 0 FROM t1; --source include/wait_condition.inc SELECT VARIABLE_VALUE expect_Primary FROM performance_schema.global_status WHERE VARIABLE_NAME = 'wsrep_cluster_status'; @@ -202,6 +231,8 @@ DELETE t1, t2 FROM t1 JOIN t2 WHERE t1.f1 = t2.f1; SELECT COUNT(*) expect_0 FROM t1; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/t1'; +--source include/wait_condition.inc --let $wait_condition = SELECT COUNT(*) = 0 FROM t1; --source include/wait_condition.inc SELECT VARIABLE_VALUE = 'Primary' FROM performance_schema.global_status WHERE VARIABLE_NAME = 'wsrep_cluster_status'; @@ -219,6 +250,10 @@ CREATE TABLE child (id INT, parent_id INT, INDEX par_ind (parent_id), FOREIGN KE INSERT INTO child VALUES (1,1),(2,2),(3,3); --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/parent'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/child'; +--source include/wait_condition.inc --let $wait_condition = SELECT COUNT(*) = 3 FROM child; --source include/wait_condition.inc @@ -233,6 +268,10 @@ SELECT COUNT(*) AS expect_0 FROM parent; SELECT COUNT(*) AS expect_0 FROM child; --connection node_2 +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/parent'; +--source include/wait_condition.inc +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES WHERE NAME LIKE 'test/child'; +--source include/wait_condition.inc --let $wait_condition = SELECT COUNT(*) = 0 FROM child; --source include/wait_condition.inc SELECT VARIABLE_VALUE = 'Primary' FROM performance_schema.global_status WHERE VARIABLE_NAME = 'wsrep_cluster_status'; From 1d5557d9c0baef51bea6ffd312720fc08f1dd203 Mon Sep 17 00:00:00 2001 From: ParadoxV5 Date: Tue, 22 Apr 2025 20:15:08 -0600 Subject: [PATCH 116/125] MDEV-36663 Semi-sync Replica Can't Kill Dump Thread When Using SSL When a replica stops an established semi-sync connection, it is supposed to kill the corresponding binlog dump thread on the primary server. However, when connections are configured to use SSL, this new connection created by the replica to kill the dump thread doesn't have any logic to configure SSL options, and thereby the connection can't be made, and the dump thread will never be killed. This patch adds logic to configure the semi-sync kill connection with SSL. The exising logic to set up the connection options for the regular connection was extracted into a function that the semi-sync kill connection invokes. Co-author: Brandon Nesterenko --- .../suite/rpl/r/rpl_semi_sync_ssl_stop.result | 53 ++++++++++ .../suite/rpl/t/rpl_semi_sync_ssl_stop.test | 100 ++++++++++++++++++ sql/rpl_mi.cc | 49 +++++++++ sql/rpl_mi.h | 11 ++ sql/semisync_slave.cc | 9 +- sql/semisync_slave.h | 2 +- sql/slave.cc | 42 +------- 7 files changed, 220 insertions(+), 46 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_semi_sync_ssl_stop.result create mode 100644 mysql-test/suite/rpl/t/rpl_semi_sync_ssl_stop.test diff --git a/mysql-test/suite/rpl/r/rpl_semi_sync_ssl_stop.result b/mysql-test/suite/rpl/r/rpl_semi_sync_ssl_stop.result new file mode 100644 index 00000000000..8bf8f27c676 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_semi_sync_ssl_stop.result @@ -0,0 +1,53 @@ +# Skip starting the slave because we manually start with SSL later +include/master-slave.inc +[connection master] +# +# Setup +connection master; +CREATE USER replssl@localhost; +GRANT REPLICATION SLAVE on *.* to replssl@localhost REQUIRE SSL; +set @orig_master_enabled= @@GLOBAL.rpl_semi_sync_master_enabled; +SET @@GLOBAL.rpl_semi_sync_master_enabled= 1; +connection slave; +CHANGE MASTER TO +master_user='replssl', +master_password='', +master_ssl=1, +master_ssl_ca='MYSQL_TEST_DIR/std_data/cacert.pem', +master_ssl_cert='MYSQL_TEST_DIR/std_data/client-cert.pem', +master_ssl_key='MYSQL_TEST_DIR/std_data/client-key.pem'; +set @orig_slave_enabled= @@GLOBAL.rpl_semi_sync_slave_enabled; +SET @@GLOBAL.rpl_semi_sync_slave_enabled= 1; +include/start_slave.inc +connection master; +# Verify Semi-Sync is active +SHOW STATUS LIKE 'Rpl_semi_sync_master_clients'; +Variable_name Value +Rpl_semi_sync_master_clients 1 +# Create some table so slave can be seen as up-to-date and working +connection master; +CREATE TABLE t1 (a INT); +connection slave; +# Disconnect the slave and wait until the master's dump thread is gone +connection slave; +STOP SLAVE; +connection master; +# MDEV-36663: Verifying dump thread connection is killed.. +# ..done +# Cleanup +connection master; +SET @@GLOBAL.rpl_semi_sync_master_enabled= @orig_master_enabled; +DROP USER replssl@localhost; +DROP TABLE t1; +connection slave; +SET @@GLOBAL.rpl_semi_sync_slave_enabled= @orig_slave_enabled; +CHANGE MASTER TO +master_user='root', +master_ssl=0, +master_ssl_ca='', +master_ssl_cert='', +master_ssl_key=''; +connection slave; +include/start_slave.inc +include/rpl_end.inc +# End of rpl_semi_sync_ssl_stop.inc diff --git a/mysql-test/suite/rpl/t/rpl_semi_sync_ssl_stop.test b/mysql-test/suite/rpl/t/rpl_semi_sync_ssl_stop.test new file mode 100644 index 00000000000..cab9caf8ac4 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_semi_sync_ssl_stop.test @@ -0,0 +1,100 @@ +# +# This test verifies that semi-sync setups configured to use SSL can kill +# the replication connection when the IO thread is stopped (e.g. from +# STOP SLAVE). The way it should happen, is that the IO thread creates a new +# connection to the primary which issues KILL on the connection id of the +# replication connection. MDEV-36663 reported an issue where this new +# kill-oriented connection could not connect to a primary when it requires +# connections to use SSL. +# +# This test sets up a semi-sync SSL master-slave topology, and stops the +# slave IO thread. It then validates that the connection was killed by using +# the wait_condition.inc utility to wait for the binlog dump thread to die, +# and also validates that the status variable Rpl_semi_sync_master_clients +# reports as 0. +# +# References: +# MDEV-36663: Semi-sync Replica Can't Kill Dump Thread When Using SSL +# +--source include/have_binlog_format_mixed.inc # format-agnostic +--source include/have_ssl_communication.inc + +--echo # Skip starting the slave because we manually start with SSL later +--let $rpl_skip_start_slave= 1 +--source include/master-slave.inc + +--echo # +--echo # Setup +--connection master +CREATE USER replssl@localhost; +GRANT REPLICATION SLAVE on *.* to replssl@localhost REQUIRE SSL; + +set @orig_master_enabled= @@GLOBAL.rpl_semi_sync_master_enabled; +SET @@GLOBAL.rpl_semi_sync_master_enabled= 1; + +--connection slave +--replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR +eval CHANGE MASTER TO + master_user='replssl', + master_password='', + master_ssl=1, + master_ssl_ca='$MYSQL_TEST_DIR/std_data/cacert.pem', + master_ssl_cert='$MYSQL_TEST_DIR/std_data/client-cert.pem', + master_ssl_key='$MYSQL_TEST_DIR/std_data/client-key.pem'; + +set @orig_slave_enabled= @@GLOBAL.rpl_semi_sync_slave_enabled; +SET @@GLOBAL.rpl_semi_sync_slave_enabled= 1; + +--source include/start_slave.inc + +--connection master +--echo # Verify Semi-Sync is active +--let $status_var= Rpl_semi_sync_master_status +--let $status_var_value= ON +--source include/wait_for_status_var.inc +SHOW STATUS LIKE 'Rpl_semi_sync_master_clients'; + +--echo # Create some table so slave can be seen as up-to-date and working +--connection master +CREATE TABLE t1 (a INT); +--sync_slave_with_master + +--echo # Disconnect the slave and wait until the master's dump thread is gone +--connection slave +STOP SLAVE; +--connection master + +--echo # MDEV-36663: Verifying dump thread connection is killed.. +# Prior to MDEV-36663 fixes, this would time out and +# Rpl_semi_sync_master_clients would remain 1. +--let $wait_condition= SELECT COUNT(*)=0 FROM information_schema.PROCESSLIST WHERE COMMAND = 'Binlog Dump' +--source include/wait_condition.inc + +--let $n_master_clients= query_get_value(SHOW STATUS LIKE 'Rpl_semi_sync_master_clients', Value, 1) +if ($n_master_clients) +{ + --echo # Rpl_semi_sync_master_clients: $n_master_clients + --die Semi-sync dump thread connection not killed +} +--echo # ..done + +--echo # Cleanup +--connection master +SET @@GLOBAL.rpl_semi_sync_master_enabled= @orig_master_enabled; +DROP USER replssl@localhost; +DROP TABLE t1; + +--connection slave +SET @@GLOBAL.rpl_semi_sync_slave_enabled= @orig_slave_enabled; +CHANGE MASTER TO + master_user='root', + master_ssl=0, + master_ssl_ca='', + master_ssl_cert='', + master_ssl_key=''; + +--connection slave +--source include/start_slave.inc + +--source include/rpl_end.inc +--echo # End of rpl_semi_sync_ssl_stop.inc diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index 2183137c2f5..b19a585f47e 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -21,6 +21,7 @@ #include "slave.h" #include "strfunc.h" #include "sql_repl.h" +#include #ifdef HAVE_REPLICATION @@ -2068,4 +2069,52 @@ bool Master_info_index::flush_all_relay_logs() DBUG_RETURN(result); } +void setup_mysql_connection_for_master(MYSQL *mysql, Master_info *mi, + uint timeout) +{ + DBUG_ASSERT(mi); + DBUG_ASSERT(mi->mysql); + mysql_options(mysql, MYSQL_OPT_CONNECT_TIMEOUT, (char *) &timeout); + mysql_options(mysql, MYSQL_OPT_READ_TIMEOUT, (char *) &timeout); + +#ifdef HAVE_OPENSSL + if (mi->ssl) + { + mysql_ssl_set(mysql, + mi->ssl_key[0]?mi->ssl_key:0, + mi->ssl_cert[0]?mi->ssl_cert:0, + mi->ssl_ca[0]?mi->ssl_ca:0, + mi->ssl_capath[0]?mi->ssl_capath:0, + mi->ssl_cipher[0]?mi->ssl_cipher:0); + mysql_options(mysql, MYSQL_OPT_SSL_CRL, + mi->ssl_crl[0] ? mi->ssl_crl : 0); + mysql_options(mysql, MYSQL_OPT_SSL_CRLPATH, + mi->ssl_crlpath[0] ? mi->ssl_crlpath : 0); + mysql_options(mysql, MYSQL_OPT_SSL_VERIFY_SERVER_CERT, + &mi->ssl_verify_server_cert); + } +#endif + + /* + If server's default charset is not supported (like utf16, utf32) as client + charset, then set client charset to 'latin1' (default client charset). + */ + if (is_supported_parser_charset(default_charset_info)) + mysql_options(mysql, MYSQL_SET_CHARSET_NAME, default_charset_info->cs_name.str); + else + { + sql_print_information("'%s' can not be used as client character set. " + "'%s' will be used as default client character set " + "while connecting to master.", + default_charset_info->cs_name.str, + default_client_charset_info->cs_name.str); + mysql_options(mysql, MYSQL_SET_CHARSET_NAME, + default_client_charset_info->cs_name.str); + } + + /* Set MYSQL_PLUGIN_DIR in case master asks for an external authentication plugin */ + if (opt_plugin_dir_ptr && *opt_plugin_dir_ptr) + mysql_options(mysql, MYSQL_PLUGIN_DIR, opt_plugin_dir_ptr); +} + #endif /* HAVE_REPLICATION */ diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index 159e099ff32..9f6e1066c55 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -487,5 +487,16 @@ void free_key_master_info(Master_info *mi); uint any_slave_sql_running(bool already_locked); bool give_error_if_slave_running(bool already_lock); +/* + Sets up the basic options for a MYSQL connection, mysql, to connect to the + primary server described by the Master_info parameter, mi. The timeout must + be passed explicitly, as different types of connections created by the slave + will use different values. + + Assumes mysql_init() has already been called on the mysql connection object. +*/ +void setup_mysql_connection_for_master(MYSQL *mysql, Master_info *mi, + uint timeout); + #endif /* HAVE_REPLICATION */ #endif /* RPL_MI_H */ diff --git a/sql/semisync_slave.cc b/sql/semisync_slave.cc index d10754ad374..32e53dab758 100644 --- a/sql/semisync_slave.cc +++ b/sql/semisync_slave.cc @@ -141,7 +141,7 @@ void Repl_semi_sync_slave::slave_stop(Master_info *mi) DBUG_ASSERT(!debug_sync_set_action(mi->io_thd, STRING_WITH_LEN(act))); };); #endif - kill_connection(mi->mysql); + kill_connection(mi); } set_slave_enabled(0); @@ -158,8 +158,9 @@ void Repl_semi_sync_slave::slave_reconnect(Master_info *mi) } -void Repl_semi_sync_slave::kill_connection(MYSQL *mysql) +void Repl_semi_sync_slave::kill_connection(Master_info *mi) { + MYSQL *mysql= mi->mysql; if (!mysql) return; @@ -168,8 +169,8 @@ void Repl_semi_sync_slave::kill_connection(MYSQL *mysql) size_t kill_buffer_length; kill_mysql = mysql_init(kill_mysql); - mysql_options(kill_mysql, MYSQL_OPT_CONNECT_TIMEOUT, &m_kill_conn_timeout); - mysql_options(kill_mysql, MYSQL_OPT_READ_TIMEOUT, &m_kill_conn_timeout); + + setup_mysql_connection_for_master(kill_mysql, mi, m_kill_conn_timeout); mysql_options(kill_mysql, MYSQL_OPT_WRITE_TIMEOUT, &m_kill_conn_timeout); bool ret= (!mysql_real_connect(kill_mysql, mysql->host, diff --git a/sql/semisync_slave.h b/sql/semisync_slave.h index 6811584c9c8..79949067de8 100644 --- a/sql/semisync_slave.h +++ b/sql/semisync_slave.h @@ -92,7 +92,7 @@ public: void slave_stop(Master_info *mi); void slave_reconnect(Master_info *mi); int request_transmit(Master_info *mi); - void kill_connection(MYSQL *mysql); + void kill_connection(Master_info *mi); private: /* True when init_object has been called */ diff --git a/sql/slave.cc b/sql/slave.cc index 038010c4be1..23851fdaf5c 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -7618,50 +7618,10 @@ static int connect_to_master(THD* thd, MYSQL* mysql, Master_info* mi, if (opt_slave_compressed_protocol) client_flag|= CLIENT_COMPRESS; /* We will use compression */ - mysql_options(mysql, MYSQL_OPT_CONNECT_TIMEOUT, (char *) &slave_net_timeout); - mysql_options(mysql, MYSQL_OPT_READ_TIMEOUT, (char *) &slave_net_timeout); + setup_mysql_connection_for_master(mi->mysql, mi, slave_net_timeout); mysql_options(mysql, MYSQL_OPT_USE_THREAD_SPECIFIC_MEMORY, (char*) &my_true); -#ifdef HAVE_OPENSSL - if (mi->ssl) - { - mysql_ssl_set(mysql, - mi->ssl_key[0]?mi->ssl_key:0, - mi->ssl_cert[0]?mi->ssl_cert:0, - mi->ssl_ca[0]?mi->ssl_ca:0, - mi->ssl_capath[0]?mi->ssl_capath:0, - mi->ssl_cipher[0]?mi->ssl_cipher:0); - mysql_options(mysql, MYSQL_OPT_SSL_CRL, - mi->ssl_crl[0] ? mi->ssl_crl : 0); - mysql_options(mysql, MYSQL_OPT_SSL_CRLPATH, - mi->ssl_crlpath[0] ? mi->ssl_crlpath : 0); - mysql_options(mysql, MYSQL_OPT_SSL_VERIFY_SERVER_CERT, - &mi->ssl_verify_server_cert); - } -#endif - - /* - If server's default charset is not supported (like utf16, utf32) as client - charset, then set client charset to 'latin1' (default client charset). - */ - if (is_supported_parser_charset(default_charset_info)) - mysql_options(mysql, MYSQL_SET_CHARSET_NAME, default_charset_info->cs_name.str); - else - { - sql_print_information("'%s' can not be used as client character set. " - "'%s' will be used as default client character set " - "while connecting to master.", - default_charset_info->cs_name.str, - default_client_charset_info->cs_name.str); - mysql_options(mysql, MYSQL_SET_CHARSET_NAME, - default_client_charset_info->cs_name.str); - } - - /* Set MYSQL_PLUGIN_DIR in case master asks for an external authentication plugin */ - if (opt_plugin_dir_ptr && *opt_plugin_dir_ptr) - mysql_options(mysql, MYSQL_PLUGIN_DIR, opt_plugin_dir_ptr); - /* we disallow empty users */ if (mi->user[0] == 0) { From 1c09b5eadb40a2d0bc95c4c658455179549f950f Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 24 Apr 2025 13:07:12 +1000 Subject: [PATCH 117/125] packaging prep for Ubuntu Plucky Plucky Puffin as Ubuntu 25.04 released. Also prepare, ahead of time, Ubuntu Questing Quokka for 25.10 Also prepare for Debian 14 - Forky. --- debian/autobake-deb.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 1a8233bec3f..e23101d1eee 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -110,7 +110,7 @@ in replace_uring_with_aio fi ;& - "trixie"|"sid") + "trixie"|"forky"|"sid") # The default packaging should always target Debian Sid, so in this case # there is intentionally no customizations whatsoever. ;; @@ -129,7 +129,7 @@ in replace_uring_with_aio fi ;& - "noble"|"oracular") + "noble"|"oracular"|"plucky"|"questing") # mariadb-plugin-rocksdb s390x not supported by us (yet) # ubuntu doesn't support mips64el yet, so keep this just # in case something changes. From 5f69e1815206620de4a0a739e195c0aed620c842 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 24 Apr 2025 13:46:43 +0530 Subject: [PATCH 118/125] MDEV-36682 Conflict between MDEV-36504 and MDEV-16329 Reason: ======= - MDEV-16239 does apply the DML logs after bulk insert for ALTER TABLE..ALGORITHM=COPY, but InnoDB fails to reset the bulk_insert in ha_innobase::extra(HA_EXTRA_END_ALTER_COPY). This leads to crash while applying DML logs. Solution: ======= ha_innobase::extra(HA_EXTRA_END_ALTER_COPY): Reset TRX_DDL_BULK at the end of bulk insert operation --- storage/innobase/handler/ha_innodb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 1fc94e6f174..f1c48ccfcb7 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -15939,7 +15939,7 @@ ha_innobase::extra( } trx->end_bulk_insert(*m_prebuilt->table); - trx->bulk_insert &= TRX_DDL_BULK; + trx->bulk_insert &= TRX_DML_BULK; if (!m_prebuilt->table->is_temporary() && !high_level_read_only) { /* During copy_data_between_tables(), InnoDB only From 05813f54cf1f3044e20d9bd7b2ca1379dadcf6ff Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Thu, 24 Apr 2025 00:23:48 +0400 Subject: [PATCH 119/125] MDEV-36648 - main.mdl_sync extra test.t2 row in I_S.metadata_lock_info SELECT FROM information_schema.metadata_lock_info may sporadically return MDL_SHARED lock acquired by InnoDB purge thread. Fix proposed in a7bf0a42d0c wasn't enough to solve this problem: apparently it did protect against purging old rows, but not against locking the table. Reverted a7bf0a42d0c in favour of solution proposed in e996f77cd87. That is use wait_all_purged.inc to make sure purge is completed. --- mysql-test/main/mdl_sync.result | 5 +---- mysql-test/main/mdl_sync.test | 8 +------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/mysql-test/main/mdl_sync.result b/mysql-test/main/mdl_sync.result index c6b80c50b40..600bfdd19d2 100644 --- a/mysql-test/main/mdl_sync.result +++ b/mysql-test/main/mdl_sync.result @@ -2431,9 +2431,6 @@ create table t1 (a int) engine=myisam; create table t2 (a int) stats_persistent=0, engine=innodb; insert into t1 values (1); insert into t2 values (1); -connect con1, localhost, root; -start transaction with consistent snapshot; -connection default; SET DEBUG_SYNC= 'after_open_table_mdl_shared SIGNAL table_opened WAIT_FOR grlwait execute 2'; update t1,t2 set t1.a=2,t2.a=3; connection con2; @@ -2456,6 +2453,7 @@ SET DEBUG_SYNC= 'now SIGNAL grlwait'; SET DEBUG_SYNC= 'now WAIT_FOR table_opened'; SET DEBUG_SYNC= 'mdl_acquire_lock_wait SIGNAL grlwait'; FLUSH TABLES WITH READ LOCK; +InnoDB 0 transactions not purged SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME MDL_BACKUP_FTWRL2 Backup lock @@ -2465,7 +2463,6 @@ connection default; SET DEBUG_SYNC= 'RESET'; drop table t1,t2; disconnect con2; -disconnect con1; # # Bug#50786 Assertion `thd->mdl_context.trans_sentinel() == __null' # failed in open_ltable() diff --git a/mysql-test/main/mdl_sync.test b/mysql-test/main/mdl_sync.test index 29d95460d16..5495ed0b478 100644 --- a/mysql-test/main/mdl_sync.test +++ b/mysql-test/main/mdl_sync.test @@ -3115,12 +3115,6 @@ create table t2 (a int) stats_persistent=0, engine=innodb; insert into t1 values (1); insert into t2 values (1); -connect (con1, localhost, root); -# disable innodb purge thread, otherwise it might start purging t2, -# and will take an mdl, affecting metadata_lock_info output. -start transaction with consistent snapshot; -connection default; - SET DEBUG_SYNC= 'after_open_table_mdl_shared SIGNAL table_opened WAIT_FOR grlwait execute 2'; --send update t1,t2 set t1.a=2,t2.a=3 @@ -3156,6 +3150,7 @@ FLUSH TABLES WITH READ LOCK; let $wait_condition= SELECT COUNT(*)=1 FROM information_schema.metadata_lock_info; --source include/wait_condition.inc +--source ../suite/innodb/include/wait_all_purged.inc SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; unlock tables; @@ -3166,7 +3161,6 @@ connection default; SET DEBUG_SYNC= 'RESET'; drop table t1,t2; disconnect con2; -disconnect con1; --echo # --echo # Bug#50786 Assertion `thd->mdl_context.trans_sentinel() == __null' From ddd5ba3a00ebf164c169efb312ee5f4a624f575c Mon Sep 17 00:00:00 2001 From: Mohanad Date: Fri, 28 Feb 2025 14:43:20 +0200 Subject: [PATCH 120/125] MDEV-14432 mysqldump does not preserve case of table names in generated sql inside the get_lookup_field_values when lower_case_table_names's value is 2 this should reserve the case for the table's and database's names so this commit changes the condition to lowercase only when the lower_case_table_name's value is 1 not just 1 and 2 "any value not equal 0" --- mysql-test/main/lowercase_table2.result | 2 +- sql/sql_show.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/lowercase_table2.result b/mysql-test/main/lowercase_table2.result index 3a4c46eb776..2b6fb358d39 100644 --- a/mysql-test/main/lowercase_table2.result +++ b/mysql-test/main/lowercase_table2.result @@ -185,7 +185,7 @@ create table myUC (i int); select TABLE_SCHEMA,TABLE_NAME FROM information_schema.TABLES where TABLE_SCHEMA ='mysqltest_LC2'; TABLE_SCHEMA TABLE_NAME -mysqltest_lc2 myUC +mysqltest_LC2 myUC use test; drop database mysqltest_LC2; # diff --git a/sql/sql_show.cc b/sql/sql_show.cc index d70e20fe0b3..78ed924d313 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -4348,7 +4348,7 @@ bool get_lookup_field_values(THD *thd, COND *cond, bool fix_table_name_case, break; } - if (lower_case_table_names && !rc) + if (lower_case_table_names == 1 && !rc) { /* We can safely do in-place upgrades here since all of the above cases From 9b0294cd126d2211d6f9357ae9273ce223579e70 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Fri, 25 Apr 2025 01:06:07 +0400 Subject: [PATCH 121/125] MDEV-36666 - atomic.alter_table still times out often According to buildbot cross-reference atomic.alter_table is failing ~10 times a day due to 900 seconds test case timeout. Split it into two tests: atomic.alter_table_myisam and atomic.alter_table_innodb. It should reduce failure frequency down to once a day or so, similarly to atomic.alter_table_aria test. Diabling InnoDB for MyISAM/Aria/RocksDB tests makes them 20-35% faster. --- mysql-test/suite/atomic/README.txt | 2 +- .../{alter_table.test => alter_table.inc} | 1 - mysql-test/suite/atomic/alter_table_aria.test | 2 +- ...alter_table.opt => alter_table_innodb.opt} | 0 .../suite/atomic/alter_table_innodb.result | 1396 +++++++++++++++++ .../suite/atomic/alter_table_innodb.test | 7 + ...table.result => alter_table_myisam.result} | 1394 ---------------- .../suite/atomic/alter_table_myisam.test | 6 + .../suite/atomic/alter_table_rocksdb.test | 2 +- .../suite/atomic/alter_table_trigger.test | 2 +- 10 files changed, 1413 insertions(+), 1399 deletions(-) rename mysql-test/suite/atomic/{alter_table.test => alter_table.inc} (99%) rename mysql-test/suite/atomic/{alter_table.opt => alter_table_innodb.opt} (100%) create mode 100644 mysql-test/suite/atomic/alter_table_innodb.result create mode 100644 mysql-test/suite/atomic/alter_table_innodb.test rename mysql-test/suite/atomic/{alter_table.result => alter_table_myisam.result} (54%) create mode 100644 mysql-test/suite/atomic/alter_table_myisam.test diff --git a/mysql-test/suite/atomic/README.txt b/mysql-test/suite/atomic/README.txt index b1453d8a4f8..160bd1decd6 100644 --- a/mysql-test/suite/atomic/README.txt +++ b/mysql-test/suite/atomic/README.txt @@ -3,7 +3,7 @@ the following: - Add # before --exec echo "restart" ... - Force $e (engine), $c (crash point) and $r (crash position) to the values - where things goes wrong. See comments in alter_table.test for how to do this. + where things goes wrong. See comments in alter_table.inc for how to do this. - start mariadbd in a debugger run the following in the debugger diff --git a/mysql-test/suite/atomic/alter_table.test b/mysql-test/suite/atomic/alter_table.inc similarity index 99% rename from mysql-test/suite/atomic/alter_table.test rename to mysql-test/suite/atomic/alter_table.inc index bad3bbad2d5..40acfcf7576 100644 --- a/mysql-test/suite/atomic/alter_table.test +++ b/mysql-test/suite/atomic/alter_table.inc @@ -1,6 +1,5 @@ --source include/long_test.inc --source include/have_debug.inc ---source include/have_innodb.inc --source include/have_log_bin.inc --source include/not_valgrind.inc diff --git a/mysql-test/suite/atomic/alter_table_aria.test b/mysql-test/suite/atomic/alter_table_aria.test index 6bf44b0463d..a83137956b1 100644 --- a/mysql-test/suite/atomic/alter_table_aria.test +++ b/mysql-test/suite/atomic/alter_table_aria.test @@ -4,4 +4,4 @@ let $engine_count=1; let $engines='aria'; let $extra_engine=myisam; ---source alter_table.test +--source alter_table.inc diff --git a/mysql-test/suite/atomic/alter_table.opt b/mysql-test/suite/atomic/alter_table_innodb.opt similarity index 100% rename from mysql-test/suite/atomic/alter_table.opt rename to mysql-test/suite/atomic/alter_table_innodb.opt diff --git a/mysql-test/suite/atomic/alter_table_innodb.result b/mysql-test/suite/atomic/alter_table_innodb.result new file mode 100644 index 00000000000..258895a0cd6 --- /dev/null +++ b/mysql-test/suite/atomic/alter_table_innodb.result @@ -0,0 +1,1396 @@ +create database test2; +RESET MASTER; + +engine: innodb + + +query: ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new" + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new" +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 COMMENT "new" + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new" +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 change column a c int COMMENT "new" + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new" +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2 + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t2.frm +t2.ibd +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2 +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +t2.frm +t2.ibd +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2 +crash point: ddl_log_alter_after_rename_triggers +t2.frm +t2.ibd +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2 +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 disable keys + +crash point: ddl_log_alter_after_create_frm +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_create_table +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_copy +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_log +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_rename_frm +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_rename_triggers +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_delete_backup +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +Warnings: +Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option +"No crash!" + +query: ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_prepare_inplace +"No crash!" +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" +crash point: ddl_log_alter_after_rename_to_backup +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" +crash point: ddl_log_alter_after_rename_to_backup_log +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" +crash point: ddl_log_alter_before_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_delete_backup +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" +crash point: ddl_log_alter_after_drop_original_table +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" + +query: ALTER TABLE t1 rename t2 + +crash point: ddl_log_alter_after_create_frm +"No crash!" +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +"No crash!" +crash point: ddl_log_alter_after_copy +"No crash!" +crash point: ddl_log_alter_after_log +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_rename_triggers +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 COMMENT "new", rename t2 + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t2.frm +t2.ibd +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename t2 +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +t2.frm +t2.ibd +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename t2 +crash point: ddl_log_alter_after_rename_triggers +t2.frm +t2.ibd +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename t2 +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 change column a c int COMMENT "new", rename t2 + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t2.frm +t2.ibd +Table Create Table +t2 CREATE TABLE `t2` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", rename t2 +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +t2.frm +t2.ibd +Table Create Table +t2 CREATE TABLE `t2` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", rename t2 +crash point: ddl_log_alter_after_rename_triggers +t2.frm +t2.ibd +Table Create Table +t2 CREATE TABLE `t2` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", rename t2 +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 ENGINE=aria, COMMENT "new" + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_prepare_inplace +"No crash!" +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" +crash point: ddl_log_alter_after_rename_to_backup +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" +crash point: ddl_log_alter_after_rename_to_backup_log +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" +crash point: ddl_log_alter_before_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_delete_backup +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" +crash point: ddl_log_alter_after_drop_original_table +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" + +query: ALTER TABLE t1 change column a c int COMMENT "new", engine=aria + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_prepare_inplace +"No crash!" +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria +crash point: ddl_log_alter_after_rename_to_backup +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria +crash point: ddl_log_alter_after_rename_to_backup_log +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria +crash point: ddl_log_alter_before_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_delete_backup +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria +crash point: ddl_log_alter_after_drop_original_table +t1.MAD +t1.MAI +t1.frm +Table Create Table +t1 CREATE TABLE `t1` ( + `c` int(11) DEFAULT NULL COMMENT 'new', + `b` int(11) DEFAULT NULL, + KEY `a` (`c`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria + +query: ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_prepare_inplace +"No crash!" +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t2.MAD +t2.MAI +t2.frm +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria +crash point: ddl_log_alter_after_rename_to_backup +t2.MAD +t2.MAI +t2.frm +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria +crash point: ddl_log_alter_after_rename_to_backup_log +t2.MAD +t2.MAI +t2.frm +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +t2.MAD +t2.MAI +t2.frm +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria +crash point: ddl_log_alter_before_rename_triggers +t2.MAD +t2.MAI +t2.frm +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria +crash point: ddl_log_alter_after_rename_triggers +t2.MAD +t2.MAI +t2.frm +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria +crash point: ddl_log_alter_after_delete_backup +t2.MAD +t2.MAI +t2.frm +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria +crash point: ddl_log_alter_after_drop_original_table +t2.MAD +t2.MAI +t2.frm +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria + +query: ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename test2.t2 + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename test2.t2 +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +t2.frm +t2.ibd +"Table is in test2" +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename test2.t2 +crash point: ddl_log_alter_after_rename_triggers +t2.frm +t2.ibd +"Table is in test2" +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + `c` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename test2.t2 +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 COMMENT "new", rename test2.t2 + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename test2.t2 +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +t2.frm +t2.ibd +"Table is in test2" +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename test2.t2 +crash point: ddl_log_alter_after_rename_triggers +t2.frm +t2.ibd +"Table is in test2" +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename test2.t2 +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 ADD key(b), COMMENT "new" + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`), + KEY `b` (`b`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD key(b), COMMENT "new" +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" + +query: ALTER TABLE t1 DROP INDEX a + +crash point: ddl_log_alter_after_create_frm +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_create_table +"No crash!" +crash point: ddl_log_alter_after_prepare_inplace +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_copy +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `a` (`a`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +crash point: ddl_log_alter_after_log +t1.frm +t1.ibd +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +count(*) +2 +master-bin.000002 # Query # # use `test`; ALTER TABLE t1 DROP INDEX a +crash point: ddl_log_alter_after_rename_to_backup +"No crash!" +crash point: ddl_log_alter_after_rename_to_backup_log +"No crash!" +crash point: ddl_log_alter_rename_frm +"No crash!" +crash point: ddl_log_alter_after_rename_to_original +"No crash!" +crash point: ddl_log_alter_before_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_rename_triggers +"No crash!" +crash point: ddl_log_alter_after_delete_backup +"No crash!" +crash point: ddl_log_alter_after_drop_original_table +"No crash!" diff --git a/mysql-test/suite/atomic/alter_table_innodb.test b/mysql-test/suite/atomic/alter_table_innodb.test new file mode 100644 index 00000000000..a70d05eba20 --- /dev/null +++ b/mysql-test/suite/atomic/alter_table_innodb.test @@ -0,0 +1,7 @@ +# +# Test atomic alter table with InnoDB + +--source include/have_innodb.inc +let $engine_count=1; +let $engines='innodb'; +--source alter_table.inc diff --git a/mysql-test/suite/atomic/alter_table.result b/mysql-test/suite/atomic/alter_table_myisam.result similarity index 54% rename from mysql-test/suite/atomic/alter_table.result rename to mysql-test/suite/atomic/alter_table_myisam.result index 8475f3c8c76..c072401359c 100644 --- a/mysql-test/suite/atomic/alter_table.result +++ b/mysql-test/suite/atomic/alter_table_myisam.result @@ -1739,1397 +1739,3 @@ t1 CREATE TABLE `t1` ( count(*) 2 master-bin.000002 # Query # # use `test`; ALTER TABLE t1 DROP INDEX a - -engine: innodb - - -query: ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new" - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new" -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 COMMENT "new" - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new" -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 change column a c int COMMENT "new" - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new" -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2 - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t2.frm -t2.ibd -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2 -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -t2.frm -t2.ibd -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2 -crash point: ddl_log_alter_after_rename_triggers -t2.frm -t2.ibd -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2 -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 disable keys - -crash point: ddl_log_alter_after_create_frm -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_create_table -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_copy -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_log -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_rename_frm -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_rename_triggers -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_delete_backup -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -Warnings: -Note 1031 Storage engine InnoDB of the table `test`.`t1` doesn't have this option -"No crash!" - -query: ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_prepare_inplace -"No crash!" -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" -crash point: ddl_log_alter_after_rename_to_backup -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" -crash point: ddl_log_alter_after_rename_to_backup_log -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" -crash point: ddl_log_alter_before_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_delete_backup -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" -crash point: ddl_log_alter_after_drop_original_table -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, ALGORITHM=copy, COMMENT "new" - -query: ALTER TABLE t1 rename t2 - -crash point: ddl_log_alter_after_create_frm -"No crash!" -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -"No crash!" -crash point: ddl_log_alter_after_copy -"No crash!" -crash point: ddl_log_alter_after_log -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_rename_triggers -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 COMMENT "new", rename t2 - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t2.frm -t2.ibd -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename t2 -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -t2.frm -t2.ibd -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename t2 -crash point: ddl_log_alter_after_rename_triggers -t2.frm -t2.ibd -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename t2 -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 change column a c int COMMENT "new", rename t2 - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t2.frm -t2.ibd -Table Create Table -t2 CREATE TABLE `t2` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", rename t2 -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -t2.frm -t2.ibd -Table Create Table -t2 CREATE TABLE `t2` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", rename t2 -crash point: ddl_log_alter_after_rename_triggers -t2.frm -t2.ibd -Table Create Table -t2 CREATE TABLE `t2` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", rename t2 -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 ENGINE=aria, COMMENT "new" - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_prepare_inplace -"No crash!" -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" -crash point: ddl_log_alter_after_rename_to_backup -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" -crash point: ddl_log_alter_after_rename_to_backup_log -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" -crash point: ddl_log_alter_before_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_delete_backup -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" -crash point: ddl_log_alter_after_drop_original_table -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ENGINE=aria, COMMENT "new" - -query: ALTER TABLE t1 change column a c int COMMENT "new", engine=aria - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_prepare_inplace -"No crash!" -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria -crash point: ddl_log_alter_after_rename_to_backup -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria -crash point: ddl_log_alter_after_rename_to_backup_log -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria -crash point: ddl_log_alter_before_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_delete_backup -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria -crash point: ddl_log_alter_after_drop_original_table -t1.MAD -t1.MAI -t1.frm -Table Create Table -t1 CREATE TABLE `t1` ( - `c` int(11) DEFAULT NULL COMMENT 'new', - `b` int(11) DEFAULT NULL, - KEY `a` (`c`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 change column a c int COMMENT "new", engine=aria - -query: ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_prepare_inplace -"No crash!" -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t2.MAD -t2.MAI -t2.frm -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria -crash point: ddl_log_alter_after_rename_to_backup -t2.MAD -t2.MAI -t2.frm -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria -crash point: ddl_log_alter_after_rename_to_backup_log -t2.MAD -t2.MAI -t2.frm -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -t2.MAD -t2.MAI -t2.frm -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria -crash point: ddl_log_alter_before_rename_triggers -t2.MAD -t2.MAI -t2.frm -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria -crash point: ddl_log_alter_after_rename_triggers -t2.MAD -t2.MAI -t2.frm -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria -crash point: ddl_log_alter_after_delete_backup -t2.MAD -t2.MAI -t2.frm -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria -crash point: ddl_log_alter_after_drop_original_table -t2.MAD -t2.MAI -t2.frm -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci PAGE_CHECKSUM=1 COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename t2, engine=aria - -query: ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename test2.t2 - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename test2.t2 -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -t2.frm -t2.ibd -"Table is in test2" -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename test2.t2 -crash point: ddl_log_alter_after_rename_triggers -t2.frm -t2.ibd -"Table is in test2" -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - `c` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN c INT, COMMENT "new", rename test2.t2 -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 COMMENT "new", rename test2.t2 - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename test2.t2 -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -t2.frm -t2.ibd -"Table is in test2" -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename test2.t2 -crash point: ddl_log_alter_after_rename_triggers -t2.frm -t2.ibd -"Table is in test2" -Table Create Table -t2 CREATE TABLE `t2` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 COMMENT "new", rename test2.t2 -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 ADD key(b), COMMENT "new" - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`), - KEY `b` (`b`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='new' -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 ADD key(b), COMMENT "new" -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" - -query: ALTER TABLE t1 DROP INDEX a - -crash point: ddl_log_alter_after_create_frm -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_create_table -"No crash!" -crash point: ddl_log_alter_after_prepare_inplace -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_copy -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `a` (`a`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -crash point: ddl_log_alter_after_log -t1.frm -t1.ibd -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -count(*) -2 -master-bin.000002 # Query # # use `test`; ALTER TABLE t1 DROP INDEX a -crash point: ddl_log_alter_after_rename_to_backup -"No crash!" -crash point: ddl_log_alter_after_rename_to_backup_log -"No crash!" -crash point: ddl_log_alter_rename_frm -"No crash!" -crash point: ddl_log_alter_after_rename_to_original -"No crash!" -crash point: ddl_log_alter_before_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_rename_triggers -"No crash!" -crash point: ddl_log_alter_after_delete_backup -"No crash!" -crash point: ddl_log_alter_after_drop_original_table -"No crash!" diff --git a/mysql-test/suite/atomic/alter_table_myisam.test b/mysql-test/suite/atomic/alter_table_myisam.test new file mode 100644 index 00000000000..917438f464c --- /dev/null +++ b/mysql-test/suite/atomic/alter_table_myisam.test @@ -0,0 +1,6 @@ +# +# Test atomic alter table with MyISAM + +let $engine_count=1; +let $engines='myisam'; +--source alter_table.inc diff --git a/mysql-test/suite/atomic/alter_table_rocksdb.test b/mysql-test/suite/atomic/alter_table_rocksdb.test index fa34008dd58..3e51fc87f1c 100644 --- a/mysql-test/suite/atomic/alter_table_rocksdb.test +++ b/mysql-test/suite/atomic/alter_table_rocksdb.test @@ -3,4 +3,4 @@ let $engine_count=1; let $engines='rocksdb'; set global rocksdb_flush_log_at_trx_commit=1; ---source alter_table.test +--source alter_table.inc diff --git a/mysql-test/suite/atomic/alter_table_trigger.test b/mysql-test/suite/atomic/alter_table_trigger.test index 1d6d5c224c7..1c06fc49483 100644 --- a/mysql-test/suite/atomic/alter_table_trigger.test +++ b/mysql-test/suite/atomic/alter_table_trigger.test @@ -7,7 +7,7 @@ # # Testing of atomic create table with crashes in a lot of different places # -# This is very similar to the alter_table.test, but includes testing of +# This is very similar to the alter_table.inc, but includes testing of # triggers in with ALTER TABLE .. RENAME. # From 9b313d2de1df65626abb3b1d6c973f74addb12fb Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Tue, 1 Apr 2025 20:57:29 +0200 Subject: [PATCH 122/125] MDEV-32086 Server crash when inserting from derived table containing insert target table Use original solution for INSERT ... SELECT - select result buferisation. Also fix MDEV-36447 and MDEV-33139 --- mysql-test/main/derived_cond_pushdown.result | 125 ++++++++---------- mysql-test/main/derived_view.result | 2 + mysql-test/main/insert_select.result | 129 ++++++++++++++++++- mysql-test/main/insert_select.test | 56 +++++++- sql/sql_base.cc | 15 ++- sql/sql_base.h | 1 + sql/sql_insert.cc | 11 +- 7 files changed, 260 insertions(+), 79 deletions(-) diff --git a/mysql-test/main/derived_cond_pushdown.result b/mysql-test/main/derived_cond_pushdown.result index 37b7cdc7404..234810f56c3 100644 --- a/mysql-test/main/derived_cond_pushdown.result +++ b/mysql-test/main/derived_cond_pushdown.result @@ -10249,9 +10249,8 @@ SELECT * FROM ( SELECT t1.f FROM v1 JOIN t1 ) AS t WHERE f IS NOT NULL; EXPLAIN INSERT INTO t1 SELECT * FROM ( SELECT t1.f FROM v1 JOIN t1 ) AS t WHERE f IS NOT NULL; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY ALL NULL NULL NULL NULL 144 Using where -2 DERIVED ALL NULL NULL NULL NULL 12 -2 DERIVED t1 ALL NULL NULL NULL NULL 12 Using where; Using join buffer (flat, BNL join) +1 PRIMARY ALL NULL NULL NULL NULL 12 Using temporary +1 PRIMARY t1 ALL NULL NULL NULL NULL 12 Using where; Using join buffer (flat, BNL join) 4 DERIVED t1 ALL NULL NULL NULL NULL 12 EXPLAIN FORMAT=JSON INSERT INTO t1 SELECT * FROM ( SELECT t1.f FROM v1 JOIN t1 ) AS t WHERE f IS NOT NULL; @@ -10259,45 +10258,35 @@ EXPLAIN { "query_block": { "select_id": 1, - "table": { - "table_name": "", - "access_type": "ALL", - "rows": 144, - "filtered": 100, - "attached_condition": "t.f is not null", - "materialized": { - "query_block": { - "select_id": 2, - "table": { - "table_name": "", - "access_type": "ALL", - "rows": 12, - "filtered": 100, - "materialized": { - "query_block": { - "select_id": 4, - "table": { - "table_name": "t1", - "access_type": "ALL", - "rows": 12, - "filtered": 100 - } - } - } - }, - "block-nl-join": { + "temporary_table": { + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 12, + "filtered": 100, + "materialized": { + "query_block": { + "select_id": 4, "table": { "table_name": "t1", "access_type": "ALL", "rows": 12, - "filtered": 100, - "attached_condition": "t1.f is not null" - }, - "buffer_type": "flat", - "buffer_size": "64", - "join_type": "BNL" + "filtered": 100 + } } } + }, + "block-nl-join": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 12, + "filtered": 100, + "attached_condition": "t1.f is not null" + }, + "buffer_type": "flat", + "buffer_size": "64", + "join_type": "BNL" } } } @@ -10328,43 +10317,33 @@ EXPLAIN { "query_block": { "select_id": 1, - "table": { - "table_name": "", - "access_type": "ALL", - "rows": 16, - "filtered": 100, - "attached_condition": "t.f is not null", - "materialized": { - "query_block": { - "select_id": 2, - "table": { - "table_name": "t1", - "access_type": "ALL", - "rows": 8, - "filtered": 100, - "attached_condition": "t1.f is not null" - }, - "table": { - "table_name": "", - "access_type": "ref", - "possible_keys": ["key0"], - "key": "key0", - "key_length": "4", - "used_key_parts": ["f"], - "ref": ["test.t1.f"], - "rows": 2, - "filtered": 100, - "materialized": { - "query_block": { - "select_id": 4, - "table": { - "table_name": "t1", - "access_type": "ALL", - "rows": 8, - "filtered": 100, - "attached_condition": "t1.f is not null" - } - } + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 8, + "filtered": 100, + "attached_condition": "t1.f is not null" + }, + "table": { + "table_name": "", + "access_type": "ref", + "possible_keys": ["key0"], + "key": "key0", + "key_length": "4", + "used_key_parts": ["f"], + "ref": ["test.t1.f"], + "rows": 2, + "filtered": 100, + "materialized": { + "query_block": { + "select_id": 4, + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 8, + "filtered": 100, + "attached_condition": "t1.f is not null" } } } diff --git a/mysql-test/main/derived_view.result b/mysql-test/main/derived_view.result index 64113cbe403..b33eba2d8de 100644 --- a/mysql-test/main/derived_view.result +++ b/mysql-test/main/derived_view.result @@ -2383,6 +2383,8 @@ SELECT * FROM t1; a 1 1 +1 +1 drop table t1,t2; set optimizer_switch=@save968720_optimizer_switch; # diff --git a/mysql-test/main/insert_select.result b/mysql-test/main/insert_select.result index 35c7880fa5f..1088a04ba50 100644 --- a/mysql-test/main/insert_select.result +++ b/mysql-test/main/insert_select.result @@ -1030,6 +1030,133 @@ a 3 DROP VIEW v1; DROP TABLE t1; +create table t1 (pk int, id int); +insert into t1 values (2,2), (3,3), (4,4); +insert into t1 +select 1,10 +from +( +select dt2.id from (select id from t1) dt2, t1 t where t.id=dt2.id +) dt +where dt.id=3; +select * from t1; +pk id +2 2 +3 3 +4 4 +1 10 +explain insert into t1 +select 1,10 +from +( +select dt2.id from (select id from t1) dt2, t1 t where t.id=dt2.id +) dt +where dt.id=3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 Using where; Using temporary +1 SIMPLE t ALL NULL NULL NULL NULL 4 Using where; Using join buffer (flat, BNL join) +explain format=json insert into t1 +select 1,10 +from +( +select dt2.id from (select id from t1) dt2, t1 t where t.id=dt2.id +) dt +where dt.id=3; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "temporary_table": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 4, + "filtered": 100, + "attached_condition": "t1.`id` = 3" + }, + "block-nl-join": { + "table": { + "table_name": "t", + "access_type": "ALL", + "rows": 4, + "filtered": 100, + "attached_condition": "t.`id` = 3" + }, + "buffer_type": "flat", + "buffer_size": "65", + "join_type": "BNL" + } + } + } +} +prepare stmt from "insert into t1 +select 1,10 +from +( +select dt2.id from (select id from t1) dt2, t1 t where t.id=dt2.id +) dt +where dt.id=3"; +execute stmt; +select * from t1; +pk id +2 2 +3 3 +4 4 +1 10 +1 10 +execute stmt; +select * from t1; +pk id +2 2 +3 3 +4 4 +1 10 +1 10 +1 10 +deallocate prepare stmt; +create procedure p() insert into t1 +select 1,10 +from +( +select dt2.id from (select id from t1) dt2, t1 t where t.id=dt2.id +) dt +where dt.id=3; +call p(); +select * from t1; +pk id +2 2 +3 3 +4 4 +1 10 +1 10 +1 10 +1 10 +call p(); +select * from t1; +pk id +2 2 +3 3 +4 4 +1 10 +1 10 +1 10 +1 10 +1 10 +drop procedure p; +drop table t1; # -# End of 10.5 test +# MDEV-33139: Crash of INSERT SELECT when preparing structures for +# split optimization # +CREATE TABLE v0 ( v1 INT UNIQUE ) ; +INSERT INTO v0 ( v1 ) VALUES +( ( SELECT 1 +FROM +( SELECT v1 +FROM v0 GROUP BY v1 ) AS v6 NATURAL JOIN +v0 AS v2 NATURAL JOIN +v0 AS v4 NATURAL JOIN +v0 AS v3 NATURAL JOIN +( SELECT v1 FROM v0 ) AS v7 ) ) ; +DROP TABLE v0; +# End of 10.5 tests diff --git a/mysql-test/main/insert_select.test b/mysql-test/main/insert_select.test index 0e9bd05a93e..5c2691c9dae 100644 --- a/mysql-test/main/insert_select.test +++ b/mysql-test/main/insert_select.test @@ -591,6 +591,60 @@ SELECT * FROM t1; DROP VIEW v1; DROP TABLE t1; +# +# MDEV-32086: condition pushdown into two mergeable derived tables, +# one containing the other, when they are forced to be +# materialized in INSERT +# +create table t1 (pk int, id int); +insert into t1 values (2,2), (3,3), (4,4); + +let $q= +insert into t1 + select 1,10 + from + ( + select dt2.id from (select id from t1) dt2, t1 t where t.id=dt2.id + ) dt + where dt.id=3; + +eval $q; +select * from t1; + +eval explain $q; +eval explain format=json $q; + +eval prepare stmt from "$q"; +execute stmt; +select * from t1; +execute stmt; +select * from t1; +deallocate prepare stmt; + +eval create procedure p() $q; +call p(); +select * from t1; +call p(); +select * from t1; +drop procedure p; + +drop table t1; + --echo # ---echo # End of 10.5 test +--echo # MDEV-33139: Crash of INSERT SELECT when preparing structures for +--echo # split optimization --echo # + +CREATE TABLE v0 ( v1 INT UNIQUE ) ; +INSERT INTO v0 ( v1 ) VALUES + ( ( SELECT 1 + FROM + ( SELECT v1 + FROM v0 GROUP BY v1 ) AS v6 NATURAL JOIN + v0 AS v2 NATURAL JOIN + v0 AS v4 NATURAL JOIN + v0 AS v3 NATURAL JOIN + ( SELECT v1 FROM v0 ) AS v7 ) ) ; +DROP TABLE v0; + +--echo # End of 10.5 tests diff --git a/sql/sql_base.cc b/sql/sql_base.cc index ed23208e358..8feb5cf57da 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -1184,11 +1184,20 @@ retry: DBUG_PRINT("info", ("found same copy of table or table which we should skip")); } - if (res && res->belong_to_derived) + /* + If we've found a duplicate in a derived table, try to work around that. + + For INSERT...SELECT, do not do any workarounds, return the duplicate. The + caller will enable buffering to handle this. + */ + if (res && res->belong_to_derived && + !(check_flag & CHECK_DUP_FOR_INSERT_SELECT)) { /* - We come here for queries of type: - INSERT INTO t1 (SELECT tmp.a FROM (select * FROM t1) as tmp); + We come here for queries like this: + + INSERT INTO t1 VALUES ((SELECT tmp.a FROM (select * FROM t1))); + DELETE FROM t1 WHERE ( ... (SELECT ... FROM t1) ) ; Try to fix by materializing the derived table */ diff --git a/sql/sql_base.h b/sql/sql_base.h index 90c47e69d94..9ec464df19a 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -72,6 +72,7 @@ enum find_item_error_report_type {REPORT_ALL_ERRORS, REPORT_EXCEPT_NOT_FOUND, #define CHECK_DUP_ALLOW_DIFFERENT_ALIAS 1 #define CHECK_DUP_FOR_CREATE 2 #define CHECK_DUP_SKIP_TEMP_TABLE 4 +#define CHECK_DUP_FOR_INSERT_SELECT 8 uint get_table_def_key(const TABLE_LIST *table_list, const char **key); TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type update, diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 34b71fcabe3..8f52924a4d5 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1710,6 +1710,14 @@ int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, DBUG_RETURN(1); } + /* + Check if we read from the same table we're inserting into. + Queries like INSERT INTO t1 VALUES ((SELECT ... FROM t1...)) are not + allowed. + + INSERT...SELECT is an exception: it will detect this case and use + buffering to handle it correctly. + */ if (!select_insert) { Item *fake_conds= 0; @@ -4042,7 +4050,8 @@ select_insert::prepare(List &values, SELECT_LEX_UNIT *u) Is table which we are changing used somewhere in other parts of query */ - if (unique_table(thd, table_list, table_list->next_global, 0)) + if (unique_table(thd, table_list, table_list->next_global, + CHECK_DUP_FOR_INSERT_SELECT)) { /* Using same table for INSERT and SELECT */ lex->current_select->options|= OPTION_BUFFER_RESULT; From 4fc9dc84b017cf9f30585bcdef0663f9425fe460 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Fri, 18 Apr 2025 12:14:23 +0200 Subject: [PATCH 123/125] MDEV-32086 (part 2) Server crash when inserting from derived table containing insert target table Get rid of need of matherialization for usual INSERT (cache results in Item_cache* if needed) - subqueries in VALUE do not see new records in the table we are inserting to - subqueries in RETIRNING prohibited to use the table we are inserting to --- mysql-test/main/insert.result | 72 +++++++- mysql-test/main/insert.test | 56 +++++- mysql-test/main/insert_returning.result | 2 + mysql-test/main/insert_returning.test | 2 + mysql-test/main/lowercase_view.result | 12 -- mysql-test/main/lowercase_view.test | 12 -- mysql-test/main/merge.result | 17 +- mysql-test/main/merge.test | 17 +- mysql-test/main/subselect.result | 20 ++- mysql-test/main/subselect.test | 10 +- mysql-test/main/subselect_elimination.result | 12 +- mysql-test/main/subselect_elimination.test | 7 +- .../main/subselect_no_exists_to_in.result | 20 ++- mysql-test/main/subselect_no_mat.result | 20 ++- mysql-test/main/subselect_no_opts.result | 20 ++- mysql-test/main/subselect_no_scache.result | 20 ++- mysql-test/main/subselect_no_semijoin.result | 20 ++- mysql-test/main/view.result | 49 ++++-- mysql-test/main/view.test | 30 ++-- mysql-test/suite/sql_sequence/other.result | 1 - mysql-test/suite/sql_sequence/other.test | 1 - sql/item.h | 12 ++ sql/item_subselect.cc | 24 +++ sql/item_subselect.h | 1 + sql/sql_base.cc | 95 +++++++---- sql/sql_base.h | 3 +- sql/sql_insert.cc | 102 +++++++++-- sql/sql_insert.h | 2 +- sql/sql_lex.cc | 42 +++++ sql/sql_lex.h | 4 + sql/sql_prepare.cc | 4 +- sql/table.h | 2 + tests/mysql_client_test.c | 159 ++++++++++++++++++ 33 files changed, 705 insertions(+), 165 deletions(-) diff --git a/mysql-test/main/insert.result b/mysql-test/main/insert.result index 586dbbff153..49ebd7406d2 100644 --- a/mysql-test/main/insert.result +++ b/mysql-test/main/insert.result @@ -806,5 +806,75 @@ a 8 drop table t1; # -# End of 10.5 tests +# MDEV-32086 Server crash when inserting from derived table containing insert target table +# (part 2) # +create table t1 (pk int, id int); +insert into t1 values (2,2), (3,3), (4,4); +select * from t1; +pk id +2 2 +3 3 +4 4 +select 101+count(*) +from +( +select dt2.id +from (select id from t1) dt2, t1 t where t.id=dt2.id +) dt +where dt.id<1000; +101+count(*) +104 +prepare s from ' +insert into t1 values( + (select 101+count(*) + from + ( + select dt2.id + from (select id from t1) dt2, t1 t where t.id=dt2.id + ) dt + where dt.id<1000 + ), 123 +) +'; +execute s; +select * from t1; +pk id +2 2 +3 3 +4 4 +104 123 +select 101+count(*) +from +( +select dt2.id +from (select id from t1) dt2, t1 t where t.id=dt2.id +) dt +where dt.id<1000; +101+count(*) +105 +execute s; +select * from t1; +pk id +2 2 +3 3 +4 4 +104 123 +105 123 +drop table t1; +# +# Try this: INSERT INTO t1 VALUES ... reference to t1 +# RETURNING (subquery not touching t1) +create table t1 (a int, b int); +create table t2 (a int, b int); +# This is accepted: +insert into t1 (a) values +(3), +((select max(a) from t1)) +returning +a, b, (select max(a) from t2); +a b (select max(a) from t2) +3 NULL NULL +NULL NULL NULL +drop table t1,t2; +# End of 10.5 tests diff --git a/mysql-test/main/insert.test b/mysql-test/main/insert.test index 4b5a5181f09..ec100cb114e 100644 --- a/mysql-test/main/insert.test +++ b/mysql-test/main/insert.test @@ -675,5 +675,59 @@ select * from t1; drop table t1; --echo # ---echo # End of 10.5 tests +--echo # MDEV-32086 Server crash when inserting from derived table containing insert target table +--echo # (part 2) --echo # + +create table t1 (pk int, id int); +insert into t1 values (2,2), (3,3), (4,4); +select * from t1; +select 101+count(*) + from + ( + select dt2.id + from (select id from t1) dt2, t1 t where t.id=dt2.id + ) dt + where dt.id<1000; +prepare s from ' +insert into t1 values( + (select 101+count(*) + from + ( + select dt2.id + from (select id from t1) dt2, t1 t where t.id=dt2.id + ) dt + where dt.id<1000 + ), 123 +) +'; +execute s; +select * from t1; +select 101+count(*) + from + ( + select dt2.id + from (select id from t1) dt2, t1 t where t.id=dt2.id + ) dt + where dt.id<1000; +execute s; +select * from t1; + +drop table t1; + +--echo # +--echo # Try this: INSERT INTO t1 VALUES ... reference to t1 +--echo # RETURNING (subquery not touching t1) +create table t1 (a int, b int); +create table t2 (a int, b int); + +--echo # This is accepted: +insert into t1 (a) values + (3), + ((select max(a) from t1)) +returning + a, b, (select max(a) from t2); + +drop table t1,t2; + +--echo # End of 10.5 tests diff --git a/mysql-test/main/insert_returning.result b/mysql-test/main/insert_returning.result index b2ed9c90e51..35c096f865e 100644 --- a/mysql-test/main/insert_returning.result +++ b/mysql-test/main/insert_returning.result @@ -498,6 +498,8 @@ t1 WHERE id1=1) 5 6 INSERT INTO t2(id2,val2) VALUES(5,'f') RETURNING (SELECT id2 FROM t2); ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +INSERT INTO t2(id2,val2) VALUES(5,'f') RETURNING (SELECT 1 UNION SELECT id2 FROM t2); +ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data INSERT INTO t2 (id2, val2) VALUES (6,'f') RETURNING t1.*; ERROR 42S02: Unknown table 'test.t1' # diff --git a/mysql-test/main/insert_returning.test b/mysql-test/main/insert_returning.test index 837d61d2c16..c2ad613aed0 100644 --- a/mysql-test/main/insert_returning.test +++ b/mysql-test/main/insert_returning.test @@ -199,6 +199,8 @@ INSERT INTO t2(id2,val2) VALUES(5,'e') RETURNING id2, (SELECT id1+id2 FROM t1 WHERE id1=1); --error ER_UPDATE_TABLE_USED INSERT INTO t2(id2,val2) VALUES(5,'f') RETURNING (SELECT id2 FROM t2); +--error ER_UPDATE_TABLE_USED +INSERT INTO t2(id2,val2) VALUES(5,'f') RETURNING (SELECT 1 UNION SELECT id2 FROM t2); --error ER_BAD_TABLE_ERROR INSERT INTO t2 (id2, val2) VALUES (6,'f') RETURNING t1.*; diff --git a/mysql-test/main/lowercase_view.result b/mysql-test/main/lowercase_view.result index af53f67869d..845df4d647d 100644 --- a/mysql-test/main/lowercase_view.result +++ b/mysql-test/main/lowercase_view.result @@ -16,29 +16,17 @@ create view v1Aa as select * from t1aA; create view v2aA as select * from v1aA; create view v3Aa as select v2Aa.col1 from v2aA,t2Aa where v2Aa.col1 = t2aA.col1; insert into v2Aa values ((select max(col1) from v1aA)); -ERROR HY000: The definition of table 'v1aA' prevents operation INSERT on table 'v2Aa' insert into t1aA values ((select max(col1) from v1Aa)); -ERROR HY000: The definition of table 'v1Aa' prevents operation INSERT on table 't1aA' insert into v2aA values ((select max(col1) from v1aA)); -ERROR HY000: The definition of table 'v1aA' prevents operation INSERT on table 'v2aA' insert into v2Aa values ((select max(col1) from t1Aa)); -ERROR HY000: The definition of table 'v2Aa' prevents operation INSERT on table 'v2Aa' insert into t1aA values ((select max(col1) from t1Aa)); -ERROR HY000: Table 't1aA' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into v2aA values ((select max(col1) from t1aA)); -ERROR HY000: The definition of table 'v2aA' prevents operation INSERT on table 'v2aA' insert into v2Aa values ((select max(col1) from v2aA)); -ERROR HY000: Table 'v2Aa' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into t1Aa values ((select max(col1) from v2Aa)); -ERROR HY000: The definition of table 'v2Aa' prevents operation INSERT on table 't1Aa' insert into v2aA values ((select max(col1) from v2Aa)); -ERROR HY000: Table 'v2aA' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into v3Aa (col1) values ((select max(col1) from v1Aa)); -ERROR HY000: The definition of table 'v1Aa' prevents operation INSERT on table 'v3Aa' insert into v3aA (col1) values ((select max(col1) from t1aA)); -ERROR HY000: The definition of table 'v3aA' prevents operation INSERT on table 'v3aA' insert into v3Aa (col1) values ((select max(col1) from v2aA)); -ERROR HY000: The definition of table 'v2aA' prevents operation INSERT on table 'v3Aa' drop view v3aA,v2Aa,v1aA; drop table t1Aa,t2Aa; create table t1Aa (col1 int); diff --git a/mysql-test/main/lowercase_view.test b/mysql-test/main/lowercase_view.test index cdd0256d639..52aae7b2b3c 100644 --- a/mysql-test/main/lowercase_view.test +++ b/mysql-test/main/lowercase_view.test @@ -23,29 +23,17 @@ create table t2aA (col1 int); create view v1Aa as select * from t1aA; create view v2aA as select * from v1aA; create view v3Aa as select v2Aa.col1 from v2aA,t2Aa where v2Aa.col1 = t2aA.col1; --- error 1443 insert into v2Aa values ((select max(col1) from v1aA)); --- error 1443 insert into t1aA values ((select max(col1) from v1Aa)); --- error 1443 insert into v2aA values ((select max(col1) from v1aA)); --- error 1443 insert into v2Aa values ((select max(col1) from t1Aa)); --- error 1093 insert into t1aA values ((select max(col1) from t1Aa)); --- error 1443 insert into v2aA values ((select max(col1) from t1aA)); --- error 1093 insert into v2Aa values ((select max(col1) from v2aA)); --- error 1443 insert into t1Aa values ((select max(col1) from v2Aa)); --- error 1093 insert into v2aA values ((select max(col1) from v2Aa)); --- error 1443 insert into v3Aa (col1) values ((select max(col1) from v1Aa)); --- error 1443 insert into v3aA (col1) values ((select max(col1) from t1aA)); --- error 1443 insert into v3Aa (col1) values ((select max(col1) from v2aA)); drop view v3aA,v2Aa,v1aA; drop table t1Aa,t2Aa; diff --git a/mysql-test/main/merge.result b/mysql-test/main/merge.result index 6722dd38b9c..e0ed3a31004 100644 --- a/mysql-test/main/merge.result +++ b/mysql-test/main/merge.result @@ -3689,33 +3689,22 @@ insert into tmp (b) values (1); insert into t1 (a) values (1); insert into t3 (b) values (1); insert into m1 (a) values ((select max(a) from m1)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from m2)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from t1)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from t2)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from t3, m1)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from t3, m2)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from t3, t1)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from t3, t2)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from tmp, m1)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from tmp, m2)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from tmp, t1)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from tmp, t2)); -ERROR HY000: Table 'm1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into m1 (a) values ((select max(a) from v1)); -ERROR HY000: The definition of table 'v1' prevents operation INSERT on table 'm1' insert into m1 (a) values ((select max(a) from tmp, v1)); -ERROR HY000: The definition of table 'v1' prevents operation INSERT on table 'm1' +select count(*) from m1; +count(*) +15 drop view v1; drop temporary table tmp; drop table t1, t2, t3, m1, m2; diff --git a/mysql-test/main/merge.test b/mysql-test/main/merge.test index f64f0c1d099..ad6268667e0 100644 --- a/mysql-test/main/merge.test +++ b/mysql-test/main/merge.test @@ -2704,37 +2704,24 @@ insert into tmp (b) values (1); insert into t1 (a) values (1); insert into t3 (b) values (1); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from m1)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from m2)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from t1)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from t2)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from t3, m1)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from t3, m2)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from t3, t1)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from t3, t2)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from tmp, m1)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from tmp, m2)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from tmp, t1)); ---error ER_UPDATE_TABLE_USED insert into m1 (a) values ((select max(a) from tmp, t2)); - ---error ER_VIEW_PREVENT_UPDATE + insert into m1 (a) values ((select max(a) from v1)); ---error ER_VIEW_PREVENT_UPDATE insert into m1 (a) values ((select max(a) from tmp, v1)); +select count(*) from m1; drop view v1; diff --git a/mysql-test/main/subselect.result b/mysql-test/main/subselect.result index 6f110d1d603..e8b2f7eea31 100644 --- a/mysql-test/main/subselect.result +++ b/mysql-test/main/subselect.result @@ -679,22 +679,24 @@ create table t3 (b int); insert into t2 values (1); insert into t3 values (1),(2); INSERT INTO t1 (x) VALUES ((SELECT x FROM t1)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data INSERT INTO t1 (x) VALUES ((SELECT b FROM t3)); ERROR 21000: Subquery returns more than 1 row INSERT INTO t1 (x) VALUES ((SELECT a FROM t2)); select * from t1; x +NULL 1 insert into t2 values (1); INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 INSERT INTO t1 (x) select (SELECT SUM(a)+1 FROM t2) FROM t2; select * from t1; x +NULL 1 2 3 @@ -702,6 +704,7 @@ x INSERT INTO t1 (x) select (SELECT SUM(x)+2 FROM t1) FROM t2; select * from t1; x +NULL 1 2 3 @@ -711,6 +714,7 @@ x INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 3 @@ -727,7 +731,7 @@ insert into t3 values (1),(2); select * from t1; x y replace into t1 (x, y) VALUES ((SELECT x FROM t1), (SELECT a+1 FROM t2)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 23000: Column 'x' cannot be null replace into t1 (x, y) VALUES ((SELECT a FROM t3), (SELECT a+1 FROM t2)); ERROR 21000: Subquery returns more than 1 row replace into t1 (x, y) VALUES ((SELECT a FROM t2), (SELECT a+1 FROM t2)); @@ -795,13 +799,21 @@ SELECT * FROM t2 WHERE id IN (SELECT 5 UNION SELECT 2); id 2 INSERT INTO t2 VALUES ((SELECT * FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row INSERT INTO t2 VALUES ((SELECT id FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row +select * from t2; +id +1 +2 +INSERT INTO t2 VALUES ((SELECT count(*) FROM t2)); +INSERT INTO t2 VALUES ((SELECT max(id) FROM t2)); SELECT * FROM t2; id 1 2 +2 +2 CREATE TABLE t1 (id int(11) default NULL, KEY id (id)) ENGINE=MyISAM CHARSET=latin1; INSERT INTO t1 values (1),(1); UPDATE t2 SET id=(SELECT * FROM t1); diff --git a/mysql-test/main/subselect.test b/mysql-test/main/subselect.test index 89e7d18b2b2..9c4f69177e7 100644 --- a/mysql-test/main/subselect.test +++ b/mysql-test/main/subselect.test @@ -415,7 +415,6 @@ create table t2 (a int) ENGINE=MyISAM; create table t3 (b int); insert into t2 values (1); insert into t3 values (1),(2); --- error ER_UPDATE_TABLE_USED INSERT INTO t1 (x) VALUES ((SELECT x FROM t1)); -- error ER_SUBQUERY_NO_1_ROW INSERT INTO t1 (x) VALUES ((SELECT b FROM t3)); @@ -450,7 +449,7 @@ create table t3 (a int); insert into t2 values (1); insert into t3 values (1),(2); select * from t1; --- error ER_UPDATE_TABLE_USED +-- error ER_BAD_NULL_ERROR replace into t1 (x, y) VALUES ((SELECT x FROM t1), (SELECT a+1 FROM t2)); -- error ER_SUBQUERY_NO_1_ROW replace into t1 (x, y) VALUES ((SELECT a FROM t3), (SELECT a+1 FROM t2)); @@ -486,10 +485,13 @@ EXPLAIN EXTENDED SELECT * FROM t2 WHERE id IN (SELECT 1+(select 1)); EXPLAIN EXTENDED SELECT * FROM t2 WHERE id IN (SELECT 1 UNION SELECT 3); SELECT * FROM t2 WHERE id IN (SELECT 5 UNION SELECT 3); SELECT * FROM t2 WHERE id IN (SELECT 5 UNION SELECT 2); --- error ER_UPDATE_TABLE_USED +-- error ER_SUBQUERY_NO_1_ROW INSERT INTO t2 VALUES ((SELECT * FROM t2)); --- error ER_UPDATE_TABLE_USED +-- error ER_SUBQUERY_NO_1_ROW INSERT INTO t2 VALUES ((SELECT id FROM t2)); +select * from t2; +INSERT INTO t2 VALUES ((SELECT count(*) FROM t2)); +INSERT INTO t2 VALUES ((SELECT max(id) FROM t2)); SELECT * FROM t2; CREATE TABLE t1 (id int(11) default NULL, KEY id (id)) ENGINE=MyISAM CHARSET=latin1; INSERT INTO t1 values (1),(1); diff --git a/mysql-test/main/subselect_elimination.result b/mysql-test/main/subselect_elimination.result index 17400e490bc..e2bbbe1b6f2 100644 --- a/mysql-test/main/subselect_elimination.result +++ b/mysql-test/main/subselect_elimination.result @@ -136,12 +136,22 @@ DROP TABLE t1; # access within null pointer CREATE TABLE x (x INT) ENGINE=InnoDB; INSERT INTO x (x) VALUES (0); +select NULL IN (SELECT (SELECT x FROM (SELECT x FROM +(SELECT 0 IN (SELECT x=0 FROM (SELECT x FROM (SELECT (SELECT (SELECT (SELECT +(SELECT 0 AS x) FROM x AS x) IN (SELECT 0 AS x) AS x) FROM x AS x) IN +(SELECT x WHERE x=0) AS x FROM x AS x) AS x) AS x GROUP BY x) AS x FROM x) AS x) +AS x) IN (SELECT 0 AS x) AS x FROM x) as exp; +exp +NULL INSERT INTO x (x) VALUES (x IN (SELECT (SELECT x FROM (SELECT x FROM (SELECT 0 IN (SELECT x=0 FROM (SELECT x FROM (SELECT (SELECT (SELECT (SELECT (SELECT 0 AS x) FROM x AS x) IN (SELECT 0 AS x) AS x) FROM x AS x) IN (SELECT x WHERE x=0) AS x FROM x AS x) AS x) AS x GROUP BY x) AS x FROM x) AS x) AS x) IN (SELECT 0 AS x) AS x FROM x)); -ERROR HY000: Table 'x' is specified twice, both as a target for 'INSERT' and as a separate source for data +select * from x; +x +0 +NULL DROP TABLE x; # MDEV-28622: Item_subselect eliminated flag set but Item still # evaluated/used. diff --git a/mysql-test/main/subselect_elimination.test b/mysql-test/main/subselect_elimination.test index 9d973477b28..e2b5b50f36d 100644 --- a/mysql-test/main/subselect_elimination.test +++ b/mysql-test/main/subselect_elimination.test @@ -133,12 +133,17 @@ DROP TABLE t1; CREATE TABLE x (x INT) ENGINE=InnoDB; INSERT INTO x (x) VALUES (0); ---error ER_UPDATE_TABLE_USED +select NULL IN (SELECT (SELECT x FROM (SELECT x FROM +(SELECT 0 IN (SELECT x=0 FROM (SELECT x FROM (SELECT (SELECT (SELECT (SELECT +(SELECT 0 AS x) FROM x AS x) IN (SELECT 0 AS x) AS x) FROM x AS x) IN +(SELECT x WHERE x=0) AS x FROM x AS x) AS x) AS x GROUP BY x) AS x FROM x) AS x) +AS x) IN (SELECT 0 AS x) AS x FROM x) as exp; INSERT INTO x (x) VALUES (x IN (SELECT (SELECT x FROM (SELECT x FROM (SELECT 0 IN (SELECT x=0 FROM (SELECT x FROM (SELECT (SELECT (SELECT (SELECT (SELECT 0 AS x) FROM x AS x) IN (SELECT 0 AS x) AS x) FROM x AS x) IN (SELECT x WHERE x=0) AS x FROM x AS x) AS x) AS x GROUP BY x) AS x FROM x) AS x) AS x) IN (SELECT 0 AS x) AS x FROM x)); +select * from x; DROP TABLE x; --echo # MDEV-28622: Item_subselect eliminated flag set but Item still diff --git a/mysql-test/main/subselect_no_exists_to_in.result b/mysql-test/main/subselect_no_exists_to_in.result index 860cc252073..3df77e3ba9a 100644 --- a/mysql-test/main/subselect_no_exists_to_in.result +++ b/mysql-test/main/subselect_no_exists_to_in.result @@ -683,22 +683,24 @@ create table t3 (b int); insert into t2 values (1); insert into t3 values (1),(2); INSERT INTO t1 (x) VALUES ((SELECT x FROM t1)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data INSERT INTO t1 (x) VALUES ((SELECT b FROM t3)); ERROR 21000: Subquery returns more than 1 row INSERT INTO t1 (x) VALUES ((SELECT a FROM t2)); select * from t1; x +NULL 1 insert into t2 values (1); INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 INSERT INTO t1 (x) select (SELECT SUM(a)+1 FROM t2) FROM t2; select * from t1; x +NULL 1 2 3 @@ -706,6 +708,7 @@ x INSERT INTO t1 (x) select (SELECT SUM(x)+2 FROM t1) FROM t2; select * from t1; x +NULL 1 2 3 @@ -715,6 +718,7 @@ x INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 3 @@ -731,7 +735,7 @@ insert into t3 values (1),(2); select * from t1; x y replace into t1 (x, y) VALUES ((SELECT x FROM t1), (SELECT a+1 FROM t2)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 23000: Column 'x' cannot be null replace into t1 (x, y) VALUES ((SELECT a FROM t3), (SELECT a+1 FROM t2)); ERROR 21000: Subquery returns more than 1 row replace into t1 (x, y) VALUES ((SELECT a FROM t2), (SELECT a+1 FROM t2)); @@ -799,13 +803,21 @@ SELECT * FROM t2 WHERE id IN (SELECT 5 UNION SELECT 2); id 2 INSERT INTO t2 VALUES ((SELECT * FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row INSERT INTO t2 VALUES ((SELECT id FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row +select * from t2; +id +1 +2 +INSERT INTO t2 VALUES ((SELECT count(*) FROM t2)); +INSERT INTO t2 VALUES ((SELECT max(id) FROM t2)); SELECT * FROM t2; id 1 2 +2 +2 CREATE TABLE t1 (id int(11) default NULL, KEY id (id)) ENGINE=MyISAM CHARSET=latin1; INSERT INTO t1 values (1),(1); UPDATE t2 SET id=(SELECT * FROM t1); diff --git a/mysql-test/main/subselect_no_mat.result b/mysql-test/main/subselect_no_mat.result index c4b306c6f06..721d4f5fdf9 100644 --- a/mysql-test/main/subselect_no_mat.result +++ b/mysql-test/main/subselect_no_mat.result @@ -686,22 +686,24 @@ create table t3 (b int); insert into t2 values (1); insert into t3 values (1),(2); INSERT INTO t1 (x) VALUES ((SELECT x FROM t1)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data INSERT INTO t1 (x) VALUES ((SELECT b FROM t3)); ERROR 21000: Subquery returns more than 1 row INSERT INTO t1 (x) VALUES ((SELECT a FROM t2)); select * from t1; x +NULL 1 insert into t2 values (1); INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 INSERT INTO t1 (x) select (SELECT SUM(a)+1 FROM t2) FROM t2; select * from t1; x +NULL 1 2 3 @@ -709,6 +711,7 @@ x INSERT INTO t1 (x) select (SELECT SUM(x)+2 FROM t1) FROM t2; select * from t1; x +NULL 1 2 3 @@ -718,6 +721,7 @@ x INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 3 @@ -734,7 +738,7 @@ insert into t3 values (1),(2); select * from t1; x y replace into t1 (x, y) VALUES ((SELECT x FROM t1), (SELECT a+1 FROM t2)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 23000: Column 'x' cannot be null replace into t1 (x, y) VALUES ((SELECT a FROM t3), (SELECT a+1 FROM t2)); ERROR 21000: Subquery returns more than 1 row replace into t1 (x, y) VALUES ((SELECT a FROM t2), (SELECT a+1 FROM t2)); @@ -802,13 +806,21 @@ SELECT * FROM t2 WHERE id IN (SELECT 5 UNION SELECT 2); id 2 INSERT INTO t2 VALUES ((SELECT * FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row INSERT INTO t2 VALUES ((SELECT id FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row +select * from t2; +id +1 +2 +INSERT INTO t2 VALUES ((SELECT count(*) FROM t2)); +INSERT INTO t2 VALUES ((SELECT max(id) FROM t2)); SELECT * FROM t2; id 1 2 +2 +2 CREATE TABLE t1 (id int(11) default NULL, KEY id (id)) ENGINE=MyISAM CHARSET=latin1; INSERT INTO t1 values (1),(1); UPDATE t2 SET id=(SELECT * FROM t1); diff --git a/mysql-test/main/subselect_no_opts.result b/mysql-test/main/subselect_no_opts.result index d286d7ed015..6afdeb6fc8f 100644 --- a/mysql-test/main/subselect_no_opts.result +++ b/mysql-test/main/subselect_no_opts.result @@ -682,22 +682,24 @@ create table t3 (b int); insert into t2 values (1); insert into t3 values (1),(2); INSERT INTO t1 (x) VALUES ((SELECT x FROM t1)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data INSERT INTO t1 (x) VALUES ((SELECT b FROM t3)); ERROR 21000: Subquery returns more than 1 row INSERT INTO t1 (x) VALUES ((SELECT a FROM t2)); select * from t1; x +NULL 1 insert into t2 values (1); INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 INSERT INTO t1 (x) select (SELECT SUM(a)+1 FROM t2) FROM t2; select * from t1; x +NULL 1 2 3 @@ -705,6 +707,7 @@ x INSERT INTO t1 (x) select (SELECT SUM(x)+2 FROM t1) FROM t2; select * from t1; x +NULL 1 2 3 @@ -714,6 +717,7 @@ x INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 3 @@ -730,7 +734,7 @@ insert into t3 values (1),(2); select * from t1; x y replace into t1 (x, y) VALUES ((SELECT x FROM t1), (SELECT a+1 FROM t2)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 23000: Column 'x' cannot be null replace into t1 (x, y) VALUES ((SELECT a FROM t3), (SELECT a+1 FROM t2)); ERROR 21000: Subquery returns more than 1 row replace into t1 (x, y) VALUES ((SELECT a FROM t2), (SELECT a+1 FROM t2)); @@ -798,13 +802,21 @@ SELECT * FROM t2 WHERE id IN (SELECT 5 UNION SELECT 2); id 2 INSERT INTO t2 VALUES ((SELECT * FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row INSERT INTO t2 VALUES ((SELECT id FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row +select * from t2; +id +1 +2 +INSERT INTO t2 VALUES ((SELECT count(*) FROM t2)); +INSERT INTO t2 VALUES ((SELECT max(id) FROM t2)); SELECT * FROM t2; id 1 2 +2 +2 CREATE TABLE t1 (id int(11) default NULL, KEY id (id)) ENGINE=MyISAM CHARSET=latin1; INSERT INTO t1 values (1),(1); UPDATE t2 SET id=(SELECT * FROM t1); diff --git a/mysql-test/main/subselect_no_scache.result b/mysql-test/main/subselect_no_scache.result index 3975ccc5193..80e9a003b5e 100644 --- a/mysql-test/main/subselect_no_scache.result +++ b/mysql-test/main/subselect_no_scache.result @@ -685,22 +685,24 @@ create table t3 (b int); insert into t2 values (1); insert into t3 values (1),(2); INSERT INTO t1 (x) VALUES ((SELECT x FROM t1)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data INSERT INTO t1 (x) VALUES ((SELECT b FROM t3)); ERROR 21000: Subquery returns more than 1 row INSERT INTO t1 (x) VALUES ((SELECT a FROM t2)); select * from t1; x +NULL 1 insert into t2 values (1); INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 INSERT INTO t1 (x) select (SELECT SUM(a)+1 FROM t2) FROM t2; select * from t1; x +NULL 1 2 3 @@ -708,6 +710,7 @@ x INSERT INTO t1 (x) select (SELECT SUM(x)+2 FROM t1) FROM t2; select * from t1; x +NULL 1 2 3 @@ -717,6 +720,7 @@ x INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 3 @@ -733,7 +737,7 @@ insert into t3 values (1),(2); select * from t1; x y replace into t1 (x, y) VALUES ((SELECT x FROM t1), (SELECT a+1 FROM t2)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 23000: Column 'x' cannot be null replace into t1 (x, y) VALUES ((SELECT a FROM t3), (SELECT a+1 FROM t2)); ERROR 21000: Subquery returns more than 1 row replace into t1 (x, y) VALUES ((SELECT a FROM t2), (SELECT a+1 FROM t2)); @@ -801,13 +805,21 @@ SELECT * FROM t2 WHERE id IN (SELECT 5 UNION SELECT 2); id 2 INSERT INTO t2 VALUES ((SELECT * FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row INSERT INTO t2 VALUES ((SELECT id FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row +select * from t2; +id +1 +2 +INSERT INTO t2 VALUES ((SELECT count(*) FROM t2)); +INSERT INTO t2 VALUES ((SELECT max(id) FROM t2)); SELECT * FROM t2; id 1 2 +2 +2 CREATE TABLE t1 (id int(11) default NULL, KEY id (id)) ENGINE=MyISAM CHARSET=latin1; INSERT INTO t1 values (1),(1); UPDATE t2 SET id=(SELECT * FROM t1); diff --git a/mysql-test/main/subselect_no_semijoin.result b/mysql-test/main/subselect_no_semijoin.result index fab8ad0e2cc..8f05b95cb8c 100644 --- a/mysql-test/main/subselect_no_semijoin.result +++ b/mysql-test/main/subselect_no_semijoin.result @@ -682,22 +682,24 @@ create table t3 (b int); insert into t2 values (1); insert into t3 values (1),(2); INSERT INTO t1 (x) VALUES ((SELECT x FROM t1)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data INSERT INTO t1 (x) VALUES ((SELECT b FROM t3)); ERROR 21000: Subquery returns more than 1 row INSERT INTO t1 (x) VALUES ((SELECT a FROM t2)); select * from t1; x +NULL 1 insert into t2 values (1); INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 INSERT INTO t1 (x) select (SELECT SUM(a)+1 FROM t2) FROM t2; select * from t1; x +NULL 1 2 3 @@ -705,6 +707,7 @@ x INSERT INTO t1 (x) select (SELECT SUM(x)+2 FROM t1) FROM t2; select * from t1; x +NULL 1 2 3 @@ -714,6 +717,7 @@ x INSERT DELAYED INTO t1 (x) VALUES ((SELECT SUM(a) FROM t2)); select * from t1; x +NULL 1 2 3 @@ -730,7 +734,7 @@ insert into t3 values (1),(2); select * from t1; x y replace into t1 (x, y) VALUES ((SELECT x FROM t1), (SELECT a+1 FROM t2)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 23000: Column 'x' cannot be null replace into t1 (x, y) VALUES ((SELECT a FROM t3), (SELECT a+1 FROM t2)); ERROR 21000: Subquery returns more than 1 row replace into t1 (x, y) VALUES ((SELECT a FROM t2), (SELECT a+1 FROM t2)); @@ -798,13 +802,21 @@ SELECT * FROM t2 WHERE id IN (SELECT 5 UNION SELECT 2); id 2 INSERT INTO t2 VALUES ((SELECT * FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row INSERT INTO t2 VALUES ((SELECT id FROM t2)); -ERROR HY000: Table 't2' is specified twice, both as a target for 'INSERT' and as a separate source for data +ERROR 21000: Subquery returns more than 1 row +select * from t2; +id +1 +2 +INSERT INTO t2 VALUES ((SELECT count(*) FROM t2)); +INSERT INTO t2 VALUES ((SELECT max(id) FROM t2)); SELECT * FROM t2; id 1 2 +2 +2 CREATE TABLE t1 (id int(11) default NULL, KEY id (id)) ENGINE=MyISAM CHARSET=latin1; INSERT INTO t1 values (1),(1); UPDATE t2 SET id=(SELECT * FROM t1); diff --git a/mysql-test/main/view.result b/mysql-test/main/view.result index 1686a6c0026..a1b78dcb551 100644 --- a/mysql-test/main/view.result +++ b/mysql-test/main/view.result @@ -944,31 +944,19 @@ create view v1 as select * from t1; create view v2 as select * from v1; create view v3 as select v2.col1 from v2,t2 where v2.col1 = t2.col1; insert into v2 values ((select max(col1) from v1)); -ERROR HY000: The definition of table 'v1' prevents operation INSERT on table 'v2' insert into t1 values ((select max(col1) from v1)); -ERROR HY000: The definition of table 'v1' prevents operation INSERT on table 't1' insert into v2 values ((select max(col1) from v1)); -ERROR HY000: The definition of table 'v1' prevents operation INSERT on table 'v2' insert into v2 values ((select max(col1) from t1)); -ERROR HY000: The definition of table 'v2' prevents operation INSERT on table 'v2' insert into t1 values ((select max(col1) from t1)); -ERROR HY000: Table 't1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into v2 values ((select max(col1) from t1)); -ERROR HY000: The definition of table 'v2' prevents operation INSERT on table 'v2' insert into v2 values ((select max(col1) from v2)); -ERROR HY000: Table 'v2' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into t1 values ((select max(col1) from v2)); -ERROR HY000: The definition of table 'v2' prevents operation INSERT on table 't1' insert into v2 values ((select max(col1) from v2)); -ERROR HY000: Table 'v2' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into v3 (col1) values ((select max(col1) from v1)); -ERROR HY000: The definition of table 'v1' prevents operation INSERT on table 'v3' insert into v3 (col1) values ((select max(col1) from t1)); -ERROR HY000: The definition of table 'v3' prevents operation INSERT on table 'v3' insert into v3 (col1) values ((select max(col1) from v2)); -ERROR HY000: The definition of table 'v2' prevents operation INSERT on table 'v3' -insert into v3 (col1) values ((select CONVERT_TZ('20050101000000','UTC','MET') from v2)); -ERROR HY000: The definition of table 'v2' prevents operation INSERT on table 'v3' +insert into v3 (col1) values ((select CONVERT_TZ('20050101000000','UTC','MET') from v2 LIMIT 1)); +ERROR 22003: Out of range value for column 'col1' at row 2 insert into v3 (col1) values ((select CONVERT_TZ('20050101000000','UTC','MET') from t2)); insert into t3 values ((select CONVERT_TZ('20050101000000','UTC','MET') from t2)); ERROR 23000: Column 'col1' cannot be null @@ -978,6 +966,18 @@ insert into t1 (col1) values ((select max(col1) from v4)); select * from t1; col1 NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL 1 2 3 @@ -1332,9 +1332,26 @@ create view v3 as select * from t1 where 20 < (select (s1) from v2); insert into v3 values (30); ERROR HY000: The target table v3 of the INSERT is not insertable-into create view v4 as select * from v2 where 20 < (select (s1) from t1); +select * from t1; +s1 insert into v4 values (30); -ERROR HY000: The target table v4 of the INSERT is not insertable-into -drop view v4, v3, v2, v1; +select * from t1; +s1 +30 +create view v5 as select * from v2 where s1 < (select min(s1) from t1) WITH CHECK OPTION; +# can't insert only less then minimum +insert into v5 values (40); +ERROR 44000: CHECK OPTION failed `test`.`v5` +# allow insert the new minimum +insert into v5 values (10); +# always emply view (can't be something less than minimum) +select * from v5; +s1 +select * from t1; +s1 +30 +10 +drop view v5, v4, v3, v2, v1; drop table t1; create table t1 (a int); create view v1 as select * from t1; diff --git a/mysql-test/main/view.test b/mysql-test/main/view.test index 3c6117c0e36..37c5783c347 100644 --- a/mysql-test/main/view.test +++ b/mysql-test/main/view.test @@ -865,33 +865,21 @@ create table t3 (col1 datetime not null); create view v1 as select * from t1; create view v2 as select * from v1; create view v3 as select v2.col1 from v2,t2 where v2.col1 = t2.col1; --- error ER_VIEW_PREVENT_UPDATE insert into v2 values ((select max(col1) from v1)); --- error ER_VIEW_PREVENT_UPDATE insert into t1 values ((select max(col1) from v1)); --- error ER_VIEW_PREVENT_UPDATE insert into v2 values ((select max(col1) from v1)); --- error ER_VIEW_PREVENT_UPDATE insert into v2 values ((select max(col1) from t1)); --- error ER_UPDATE_TABLE_USED insert into t1 values ((select max(col1) from t1)); --- error ER_VIEW_PREVENT_UPDATE insert into v2 values ((select max(col1) from t1)); --- error ER_UPDATE_TABLE_USED insert into v2 values ((select max(col1) from v2)); --- error ER_VIEW_PREVENT_UPDATE insert into t1 values ((select max(col1) from v2)); --- error ER_UPDATE_TABLE_USED insert into v2 values ((select max(col1) from v2)); --- error ER_VIEW_PREVENT_UPDATE insert into v3 (col1) values ((select max(col1) from v1)); --- error ER_VIEW_PREVENT_UPDATE insert into v3 (col1) values ((select max(col1) from t1)); --- error ER_VIEW_PREVENT_UPDATE insert into v3 (col1) values ((select max(col1) from v2)); # check with TZ tables in list --- error ER_VIEW_PREVENT_UPDATE -insert into v3 (col1) values ((select CONVERT_TZ('20050101000000','UTC','MET') from v2)); +--error ER_WARN_DATA_OUT_OF_RANGE +insert into v3 (col1) values ((select CONVERT_TZ('20050101000000','UTC','MET') from v2 LIMIT 1)); insert into v3 (col1) values ((select CONVERT_TZ('20050101000000','UTC','MET') from t2)); -- error ER_BAD_NULL_ERROR insert into t3 values ((select CONVERT_TZ('20050101000000','UTC','MET') from t2)); @@ -1209,9 +1197,19 @@ create view v3 as select * from t1 where 20 < (select (s1) from v2); -- error ER_NON_INSERTABLE_TABLE insert into v3 values (30); create view v4 as select * from v2 where 20 < (select (s1) from t1); --- error ER_NON_INSERTABLE_TABLE +select * from t1; insert into v4 values (30); -drop view v4, v3, v2, v1; +select * from t1; +create view v5 as select * from v2 where s1 < (select min(s1) from t1) WITH CHECK OPTION; +--echo # can't insert only less then minimum +--error ER_VIEW_CHECK_FAILED +insert into v5 values (40); +--echo # allow insert the new minimum +insert into v5 values (10); +--echo # always emply view (can't be something less than minimum) +select * from v5; +select * from t1; +drop view v5, v4, v3, v2, v1; drop table t1; # diff --git a/mysql-test/suite/sql_sequence/other.result b/mysql-test/suite/sql_sequence/other.result index b950a7d8b13..247eb5b30a8 100644 --- a/mysql-test/suite/sql_sequence/other.result +++ b/mysql-test/suite/sql_sequence/other.result @@ -48,7 +48,6 @@ create sequence s2; insert into s1 (next_not_cached_value, minimum_value) values (100,1000); ERROR HY000: Field 'maximum_value' doesn't have a default value insert into s1 values (next value for s1, 1,9223372036854775806,1,1,1000,0,0); -ERROR HY000: Table 's1' is specified twice, both as a target for 'INSERT' and as a separate source for data insert into s1 values(1000,9223372036854775806,1,1,1,1000,0,0); ERROR HY000: Sequence 'test.s1' has out of range value for options insert into s1 values(0,9223372036854775806,1,1,1,1000,0,0); diff --git a/mysql-test/suite/sql_sequence/other.test b/mysql-test/suite/sql_sequence/other.test index 9761e4268a8..f727f4856fb 100644 --- a/mysql-test/suite/sql_sequence/other.test +++ b/mysql-test/suite/sql_sequence/other.test @@ -38,7 +38,6 @@ create sequence s1; create sequence s2; --error ER_NO_DEFAULT_FOR_FIELD insert into s1 (next_not_cached_value, minimum_value) values (100,1000); ---error ER_UPDATE_TABLE_USED insert into s1 values (next value for s1, 1,9223372036854775806,1,1,1000,0,0); --error ER_SEQUENCE_INVALID_DATA insert into s1 values(1000,9223372036854775806,1,1,1,1000,0,0); diff --git a/sql/item.h b/sql/item.h index d60d5c01985..ac351e94dfb 100644 --- a/sql/item.h +++ b/sql/item.h @@ -723,6 +723,17 @@ public: virtual const String *const_ptr_string() const { return NULL; } }; +struct subselect_table_finder_param +{ + THD *thd; + /* + We're searching for different TABLE_LIST objects referring to the same + table as this one + */ + const TABLE_LIST *find; + /* NUL - not found, ERROR_TABLE - search error, or the found table reference */ + TABLE_LIST *dup; +}; /****************************************************************************/ @@ -2068,6 +2079,7 @@ public: set_extraction_flag(*(int*)arg); return 0; } + virtual bool subselect_table_finder_processor(void *arg) { return 0; }; /* TRUE if the expression depends only on the table indicated by tab_map diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc index 05e98071947..05ea91c3cb1 100644 --- a/sql/item_subselect.cc +++ b/sql/item_subselect.cc @@ -7078,3 +7078,27 @@ void Subq_materialization_tracker::report_partial_merge_keys( for (uint i= 0; i < merge_keys_count; i++) partial_match_array_sizes[i]= merge_keys[i]->get_key_buff_elements(); } + + +/* + Check if somewhere inside this subselect we read the table. This means a + full read "(SELECT ... FROM tbl)", outside reference to tbl.column does not + count +*/ + +bool +Item_subselect::subselect_table_finder_processor(void *arg) +{ + subselect_table_finder_param *param= (subselect_table_finder_param *)arg; + for (SELECT_LEX *sl= unit->first_select(); sl; sl= sl->next_select()) + { + TABLE_LIST *dup; + if ((dup= sl->find_table(param->thd, ¶m->find->db, + ¶m->find->table_name))) + { + param->dup= dup; + return TRUE; + } + } + return FALSE; +}; diff --git a/sql/item_subselect.h b/sql/item_subselect.h index 7c4ca91bfac..7eb3b8b17b8 100644 --- a/sql/item_subselect.h +++ b/sql/item_subselect.h @@ -277,6 +277,7 @@ public: { return TRUE; } + bool subselect_table_finder_processor(void *arg) override; void register_as_with_rec_ref(With_element *with_elem); void init_expr_cache_tracker(THD *thd); diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 8feb5cf57da..53663a6b90e 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -19,6 +19,7 @@ #include "mariadb.h" #include "sql_base.h" // setup_table_map +#include "sql_list.h" #include "sql_priv.h" #include "unireg.h" #include "debug_sync.h" @@ -1122,7 +1123,6 @@ TABLE_LIST* find_dup_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list, t_name= &table->table_name; t_alias= &table->alias; -retry: DBUG_PRINT("info", ("real table: %s.%s", d_name->str, t_name->str)); for (TABLE_LIST *tl= table_list; tl ; tl= tl->next_global, res= 0) { @@ -1184,37 +1184,53 @@ retry: DBUG_PRINT("info", ("found same copy of table or table which we should skip")); } - /* - If we've found a duplicate in a derived table, try to work around that. - - For INSERT...SELECT, do not do any workarounds, return the duplicate. The - caller will enable buffering to handle this. - */ - if (res && res->belong_to_derived && - !(check_flag & CHECK_DUP_FOR_INSERT_SELECT)) - { - /* - We come here for queries like this: - - INSERT INTO t1 VALUES ((SELECT tmp.a FROM (select * FROM t1))); - DELETE FROM t1 WHERE ( ... (SELECT ... FROM t1) ) ; - - Try to fix by materializing the derived table - */ - TABLE_LIST *derived= res->belong_to_derived; - if (derived->is_merged_derived() && !derived->derived->is_excluded()) - { - DBUG_PRINT("info", - ("convert merged to materialization to resolve the conflict")); - derived->change_refs_to_fields(); - derived->set_materialized_derived(); - goto retry; - } - } DBUG_RETURN(res); } +TABLE_LIST* unique_table_in_select_list(THD *thd, TABLE_LIST *table, SELECT_LEX *sel) +{ + subselect_table_finder_param param= {thd, table, NULL}; + List_iterator_fast it(sel->item_list); + Item *item; + while ((item= it++)) + { + if (item->walk(&Item::subselect_table_finder_processor, FALSE, ¶m)) + { + if (param.dup == NULL) + return ERROR_TABLE; + return param.dup; + } + DBUG_ASSERT(param.dup == NULL); + } + return NULL; +} + + +typedef TABLE_LIST* (*find_table_callback)(THD *thd, TABLE_LIST *table, + TABLE_LIST *table_list, + uint check_flag, SELECT_LEX *sel); + +static +TABLE_LIST* +find_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list, + uint check_flag, SELECT_LEX *sel, find_table_callback callback ); + +TABLE_LIST* unique_table_callback(THD *thd, TABLE_LIST *table, + TABLE_LIST *table_list, + uint check_flag, SELECT_LEX *sel) +{ + return find_dup_table(thd, table, table_list, check_flag); +} + + +TABLE_LIST* unique_in_sel_table_callback(THD *thd, TABLE_LIST *table, + TABLE_LIST *table_list, + uint check_flag, SELECT_LEX *sel) +{ + return unique_table_in_select_list(thd, table, sel); +} + /** Test that the subject table of INSERT/UPDATE/DELETE/CREATE or (in case of MyISAMMRG) one of its children are not used later @@ -1233,6 +1249,25 @@ retry: TABLE_LIST* unique_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list, uint check_flag) +{ + return find_table(thd, table, table_list, check_flag, NULL, + &unique_table_callback); +} + + +TABLE_LIST* +unique_table_in_insert_returning_subselect(THD *thd, TABLE_LIST *table, SELECT_LEX *sel) +{ + return find_table(thd, table, NULL, 0, sel, + &unique_in_sel_table_callback); + +} + + +static +TABLE_LIST* +find_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list, + uint check_flag, SELECT_LEX *sel, find_table_callback callback ) { TABLE_LIST *dup; @@ -1264,12 +1299,12 @@ unique_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list, if (!tmp_parent) break; - if ((dup= find_dup_table(thd, child, child->next_global, check_flag))) + if ((dup= (*callback)(thd, child, child->next_global, check_flag, sel))) break; } } else - dup= find_dup_table(thd, table, table_list, check_flag); + dup= (*callback)(thd, table, table_list, check_flag, sel); return dup; } diff --git a/sql/sql_base.h b/sql/sql_base.h index 9ec464df19a..e07600d88ba 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -72,7 +72,6 @@ enum find_item_error_report_type {REPORT_ALL_ERRORS, REPORT_EXCEPT_NOT_FOUND, #define CHECK_DUP_ALLOW_DIFFERENT_ALIAS 1 #define CHECK_DUP_FOR_CREATE 2 #define CHECK_DUP_SKIP_TEMP_TABLE 4 -#define CHECK_DUP_FOR_INSERT_SELECT 8 uint get_table_def_key(const TABLE_LIST *table_list, const char **key); TABLE *open_ltable(THD *thd, TABLE_LIST *table_list, thr_lock_type update, @@ -290,6 +289,8 @@ bool open_and_lock_internal_tables(TABLE *table, bool lock); bool lock_tables(THD *thd, TABLE_LIST *tables, uint counter, uint flags); int decide_logging_format(THD *thd, TABLE_LIST *tables); void close_thread_table(THD *thd, TABLE **table_ptr); +TABLE_LIST* +unique_table_in_insert_returning_subselect(THD *thd, TABLE_LIST *table, SELECT_LEX *sel); TABLE_LIST *unique_table(THD *thd, TABLE_LIST *table, TABLE_LIST *table_list, uint check_flag); bool is_equal(const LEX_CSTRING *a, const LEX_CSTRING *b); diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 8f52924a4d5..db02fb4875d 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -57,6 +57,7 @@ */ #include "mariadb.h" /* NO_EMBEDDED_ACCESS_CHECKS */ +#include "sql_list.h" #include "sql_priv.h" #include "sql_insert.h" #include "sql_update.h" // compare_record @@ -714,6 +715,8 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, Name_resolution_context_state ctx_state; SELECT_LEX *returning= thd->lex->has_returning() ? thd->lex->returning() : 0; unsigned char *readbuff= NULL; + List insert_values_cache; + bool cache_insert_values= FALSE; #ifndef EMBEDDED_LIBRARY char *query= thd->query(); @@ -771,7 +774,7 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, if ((res= mysql_prepare_insert(thd, table_list, fields, values, update_fields, update_values, duplic, - &unused_conds, FALSE))) + &unused_conds, FALSE, &cache_insert_values))) { retval= thd->is_error(); if (res < 0) @@ -1000,8 +1003,42 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, restore_record(table,s->default_values); // Get empty record thd->reconsider_logging_format_for_iodup(table); } + + if (cache_insert_values) + { + insert_values_cache.empty(); + while ((values= its++)) + { + List *caches= new (thd->mem_root) List_item; + List_iterator_fast iv(*values); + Item *item; + if (caches == 0) + { + error= 1; + goto values_loop_end; + } + caches->empty(); + while((item= iv++)) + { + Item_cache *cache= item->get_cache(thd); + if (!cache) + { + error= 1; + goto values_loop_end; + } + cache->setup(thd, item); + caches->push_back(cache); + } + insert_values_cache.push_back(caches); + } + its.rewind(); + } + do { + List_iterator_fast itc(insert_values_cache); + List_iterator_fast *itr; + DBUG_PRINT("info", ("iteration %llu", iteration)); if (iteration && bulk_parameters_set(thd)) { @@ -1009,7 +1046,24 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, goto values_loop_end; } - while ((values= its++)) + if (cache_insert_values) + { + List_item *caches; + while ((caches= itc++)) + { + List_iterator_fast ic(*caches); + Item_cache *cache; + while((cache= (Item_cache*) ic++)) + { + cache->cache_value(); + } + } + itc.rewind(); + itr= &itc; + } + else + itr= &its; + while ((values= (*itr)++)) { if (fields.elements || !value_count) { @@ -1112,7 +1166,7 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, break; thd->get_stmt_da()->inc_current_row_for_warning(); } - its.rewind(); + itr->rewind(); iteration++; } while (bulk_parameters_iterations(thd)); @@ -1601,6 +1655,7 @@ static void prepare_for_positional_update(TABLE *table, TABLE_LIST *tables) table_list Global/local table list where Where clause (for insert ... select) select_insert TRUE if INSERT ... SELECT statement + cache_insert_values insert's VALUES(...) has to be pre-computed TODO (in far future) In cases of: @@ -1622,7 +1677,7 @@ int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, List &fields, List_item *values, List &update_fields, List &update_values, enum_duplicates duplic, COND **where, - bool select_insert) + bool select_insert, bool * const cache_insert_values) { SELECT_LEX *select_lex= thd->lex->first_select_lex(); Name_resolution_context *context= &select_lex->context; @@ -1712,11 +1767,12 @@ int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, /* Check if we read from the same table we're inserting into. - Queries like INSERT INTO t1 VALUES ((SELECT ... FROM t1...)) are not - allowed. + Queries like INSERT INTO t1 VALUES ((SELECT ... FROM t1...)) have + to pre-compute the VALUES part. + Reading from the same table in the RETURNING clause is not allowed. - INSERT...SELECT is an exception: it will detect this case and use - buffering to handle it correctly. + INSERT...SELECT detects this case in select_insert::prepare and also + uses buffering to handle it correcly. */ if (!select_insert) { @@ -1725,10 +1781,30 @@ int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, if ((duplicate= unique_table(thd, table_list, table_list->next_global, CHECK_DUP_ALLOW_DIFFERENT_ALIAS))) { - update_non_unique_table_error(table_list, "INSERT", duplicate); - DBUG_RETURN(1); + /* + This is INSERT INTO ... VALUES (...) and it must pre-compute the + values to be inserted. + */ + (*cache_insert_values)= true; } + else + (*cache_insert_values)= false; + select_lex->fix_prepare_information(thd, &fake_conds, &fake_conds); + + if ((*cache_insert_values) && thd->lex->has_returning()) + { + // Check if the table we're inserting into is also in RETURNING clause + TABLE_LIST *dup= + unique_table_in_insert_returning_subselect(thd, table_list, + thd->lex->returning()); + if (dup) + { + if (dup != ERROR_TABLE) + update_non_unique_table_error(table_list, "INSERT", duplicate); + DBUG_RETURN(1); + } + } } /* Only call prepare_for_posistion() if we are not performing a DELAYED @@ -3857,6 +3933,7 @@ int mysql_insert_select_prepare(THD *thd, select_result *sel_res) int res; LEX *lex= thd->lex; SELECT_LEX *select_lex= lex->first_select_lex(); + bool cache_insert_values= false; DBUG_ENTER("mysql_insert_select_prepare"); /* @@ -3867,7 +3944,7 @@ int mysql_insert_select_prepare(THD *thd, select_result *sel_res) if ((res= mysql_prepare_insert(thd, lex->query_tables, lex->field_list, 0, lex->update_list, lex->value_list, lex->duplicates, - &select_lex->where, TRUE))) + &select_lex->where, TRUE, &cache_insert_values))) DBUG_RETURN(res); /* @@ -4050,8 +4127,7 @@ select_insert::prepare(List &values, SELECT_LEX_UNIT *u) Is table which we are changing used somewhere in other parts of query */ - if (unique_table(thd, table_list, table_list->next_global, - CHECK_DUP_FOR_INSERT_SELECT)) + if (unique_table(thd, table_list, table_list->next_global, 0)) { /* Using same table for INSERT and SELECT */ lex->current_select->options|= OPTION_BUFFER_RESULT; diff --git a/sql/sql_insert.h b/sql/sql_insert.h index 9aa234b715e..e6dc13d928c 100644 --- a/sql/sql_insert.h +++ b/sql/sql_insert.h @@ -27,7 +27,7 @@ int mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, List &fields, List_item *values, List &update_fields, List &update_values, enum_duplicates duplic, - COND **where, bool select_insert); + COND **where, bool select_insert, bool * const cache_results); bool mysql_insert(THD *thd,TABLE_LIST *table,List &fields, List &values, List &update_fields, List &update_values, enum_duplicates flag, diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index eb57cb8050b..b53988706ae 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -12047,3 +12047,45 @@ bool SELECT_LEX_UNIT::explainable() const derived->is_materialized_derived() : // (3) false; } + +/** + Find the real table in prepared SELECT tree + + NOTE: all SELECT must be prepared (to have leaf table list). + + NOTE: it looks only for real tables (not view or derived) + + @param thd the current thread handle + @param db_name name of db of the table to look for + @param db_name name of db of the table to look for + + @return first found table, NULL or ERROR_TABLE +*/ + +TABLE_LIST *SELECT_LEX::find_table(THD *thd, + const LEX_CSTRING *db_name, + const LEX_CSTRING *table_name) +{ + uchar buff[STACK_BUFF_ALLOC]; // Max argument in function + if (check_stack_overrun(thd, STACK_MIN_SIZE, buff)) + return NULL; + + List_iterator_fast ti(leaf_tables); + TABLE_LIST *table; + while ((table= ti++)) + { + if (cmp(&table->db, db_name) == 0 && + cmp(&table->table_name, table_name) == 0) + return table; + } + + for (SELECT_LEX_UNIT *u= first_inner_unit(); u; u= u->next_unit()) + { + for (st_select_lex *sl= u->first_select(); sl; sl=sl->next_select()) + { + if ((table= sl->find_table(thd, db_name, table_name))) + return table; + } + } + return NULL; +} diff --git a/sql/sql_lex.h b/sql/sql_lex.h index f1c3e041c7a..0f493a345e4 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -1666,6 +1666,10 @@ public: void lex_start(LEX *plex); bool is_unit_nest() { return (nest_flags & UNIT_NEST_FL); } void mark_as_unit_nest() { nest_flags= UNIT_NEST_FL; } + + TABLE_LIST *find_table(THD *thd, + const LEX_CSTRING *db_name, + const LEX_CSTRING *table_name); }; typedef class st_select_lex SELECT_LEX; diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index c102e2f734e..28e734105cd 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -1287,6 +1287,7 @@ static bool mysql_test_insert(Prepared_statement *stmt, THD *thd= stmt->thd; List_iterator_fast its(values_list); List_item *values; + bool cache_results= FALSE; DBUG_ENTER("mysql_test_insert"); /* @@ -1330,7 +1331,8 @@ static bool mysql_test_insert(Prepared_statement *stmt, } if (mysql_prepare_insert(thd, table_list, fields, values, update_fields, - update_values, duplic, &unused_conds, FALSE)) + update_values, duplic, &unused_conds, FALSE, + &cache_results)) goto error; value_count= values->elements; diff --git a/sql/table.h b/sql/table.h index eeec8b05022..07715363e0a 100644 --- a/sql/table.h +++ b/sql/table.h @@ -3023,6 +3023,8 @@ private: ulong m_table_ref_version; }; +#define ERROR_TABLE ((TABLE_LIST*) 0x1) + class Item; /* diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index f06566cf208..5fabbd96a48 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -22626,6 +22626,164 @@ static void test_mdev_34958() rc= mysql_query(mysql, "DROP TABLE t1, t2"); myquery(rc); } + +/* Server crash when inserting from derived table containing insert target table */ +static void test_mdev_32086() +{ + int rc; + MYSQL_STMT *stmt_insert; + MYSQL_BIND bind[2]; + MYSQL_RES *result; + MYSQL_ROW row; + unsigned int vals[] = { 123, 124}; + unsigned int vals_array_len = 2; + const char *insert_stmt= "\ +insert into t1 values(\ + (select 101+count(*)\ + from\ + (\ + select dt2.id\ + from (select id from t1) dt2, t1 t where t.id=dt2.id\ + ) dt\ + where dt.id<1000\ + ), ?\ +)"; + + /* Set up test's environment */ + + + rc= mysql_query(mysql, "create table t1 (pk int, id int);"); + myquery(rc); + + rc= mysql_query(mysql, "insert into t1 values (2,2), (3,3), (4,4);"); + myquery(rc); + + stmt_insert = mysql_stmt_init(mysql); + if (!stmt_insert) + { + fprintf(stderr, "mysql_stmt_init failed: Error: %s\n", + mysql_error(mysql)); + exit(1); + } + + rc= mysql_stmt_prepare(stmt_insert, insert_stmt, strlen(insert_stmt)); + if (rc) + { + fprintf(stderr, "mysql_stmt_prepare failed: %s\n", + mysql_stmt_error(stmt_insert)); + exit(1); + } + + memset(&bind[0], 0, sizeof(MYSQL_BIND)); + + bind[0].buffer_type= MYSQL_TYPE_LONG; + bind[0].buffer= vals; + + rc= mysql_stmt_attr_set(stmt_insert, STMT_ATTR_ARRAY_SIZE, &vals_array_len); + if (rc) + { + fprintf(stderr, "mysql_stmt_prepare failed: %s\n", + mysql_stmt_error(stmt_insert)); + exit(1); + } + + rc= mysql_stmt_bind_param(stmt_insert, bind); + if (rc) + { + fprintf(stderr, "mysql_stmt_bind_param failed: %s\n", + mysql_stmt_error(stmt_insert)); + exit(1); + } + + rc= mysql_stmt_execute(stmt_insert); + if (rc) + { + fprintf(stderr, "mysql_stmt_execute failed: %s\n", + mysql_stmt_error(stmt_insert)); + exit(1); + } + + /* + pk id + 2 2 + 3 3 + 4 4 + 104 123 + 104 124 + */ + rc= mysql_query(mysql, "select * from t1"); + if (rc) + { + fprintf(stderr, "Query failed: %s\n", mysql_error(mysql)); + } + result= mysql_store_result(mysql); + row= mysql_fetch_row(result); + DIE_UNLESS(atoi(row[0]) == 2 && atoi(row[1]) == 2); + row= mysql_fetch_row(result); + DIE_UNLESS(atoi(row[0]) == 3 && atoi(row[1]) == 3); + row= mysql_fetch_row(result); + DIE_UNLESS(atoi(row[0]) == 4 && atoi(row[1]) == 4); + row= mysql_fetch_row(result); + printf("\n %d, %d \n", atoi(row[0]), atoi(row[1])); + DIE_UNLESS(atoi(row[0]) == 104 && atoi(row[1]) == 123); + row= mysql_fetch_row(result); + printf("\n %d, %d \n", atoi(row[0]), atoi(row[1])); + DIE_UNLESS(atoi(row[0]) == 104 && atoi(row[1]) == 124); + row= mysql_fetch_row(result); + DIE_UNLESS(row == NULL); + + mysql_free_result(result); + + rc= mysql_stmt_execute(stmt_insert); + if (rc) + { + fprintf(stderr, "mysql_stmt_execute failed: %s\n", + mysql_stmt_error(stmt_insert)); + exit(1); + } + /* + pk id + 2 2 + 3 3 + 4 4 + 104 123 + 104 124 + 106 123 + 106 124 + */ + rc= mysql_query(mysql, "select * from t1"); + if (rc) + { + fprintf(stderr, "Query failed: %s\n", mysql_error(mysql)); + } + result= mysql_store_result(mysql); + row= mysql_fetch_row(result); + DIE_UNLESS(atoi(row[0]) == 2 && atoi(row[1]) == 2); + row= mysql_fetch_row(result); + DIE_UNLESS(atoi(row[0]) == 3 && atoi(row[1]) == 3); + row= mysql_fetch_row(result); + DIE_UNLESS(atoi(row[0]) == 4 && atoi(row[1]) == 4); + row= mysql_fetch_row(result); + DIE_UNLESS(atoi(row[0]) == 104 && atoi(row[1]) == 123); + row= mysql_fetch_row(result); + DIE_UNLESS(atoi(row[0]) == 104 && atoi(row[1]) == 124); + row= mysql_fetch_row(result); + printf("\n %d, %d \n", atoi(row[0]), atoi(row[1])); + DIE_UNLESS(atoi(row[0]) == 106 && atoi(row[1]) == 123); + row= mysql_fetch_row(result); + printf("\n %d, %d \n", atoi(row[0]), atoi(row[1])); + DIE_UNLESS(atoi(row[0]) == 106 && atoi(row[1]) == 124); + row= mysql_fetch_row(result); + DIE_UNLESS(row == NULL); + + mysql_free_result(result); + + mysql_stmt_close(stmt_insert); + + /* Clean up */ + rc= mysql_query(mysql, "DROP TABLE t1"); + myquery(rc); +} #endif // EMBEDDED_LIBRARY /* @@ -22980,6 +23138,7 @@ static struct my_tests_st my_tests[]= { { "test_mdev_34718_bd", test_mdev_34718_bd }, { "test_mdev_34718_ad", test_mdev_34718_ad }, { "test_mdev_34958", test_mdev_34958 }, + { "test_mdev_32086", test_mdev_32086 }, #endif { 0, 0 } }; From 9579ee4fa22b4b0e579e08cce6ea231ac77181fe Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 25 Apr 2025 15:16:47 +0200 Subject: [PATCH 124/125] Revert "MDEV-36591: RHEL8(+compat)/Ubuntu 20.04 cannot start systemd servce (EXIT_CAPABILTIES/218)" This reverts commit 8925877dc86bdcff0170e1b0ec21483cc57a211a. --- cmake/systemd.cmake | 9 --------- support-files/mariadb.service.in | 5 ++++- support-files/mariadb@.service.in | 5 ++++- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/cmake/systemd.cmake b/cmake/systemd.cmake index 404441be747..a093b89f3c8 100644 --- a/cmake/systemd.cmake +++ b/cmake/systemd.cmake @@ -57,15 +57,6 @@ MACRO(CHECK_SYSTEMD) # ProtectSystem=full prevents it ReadWritePaths=-${MYSQL_DATADIR}\n") ENDIF() - # systemd version 245 (Ubuntu 20.04) and less cannot - # handle ambient capbilities on non-root processes - # 247 (Debian 11) is a version afterwards that is known to work. - IF(LIBSYSTEMD_VERSION VERSION_GREATER_EQUAL 247) - SET(SYSTEMD_AMBIENT_CAPABILITIES -"# CAP_IPC_LOCK To allow --memlock to be used as non-root user -AmbientCapabilities=CAP_IPC_LOCK -") - ENDIF() MESSAGE_ONCE(systemd "Systemd features enabled") ELSE() diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in index 7ae0278dc44..f53a0b8ceda 100644 --- a/support-files/mariadb.service.in +++ b/support-files/mariadb.service.in @@ -47,7 +47,10 @@ PrivateNetwork=false User=mysql Group=mysql -@SYSTEMD_AMBIENT_CAPABILITIES@ +# CAP_IPC_LOCK To allow memlock to be used as non-root user +# These are enabled by default +AmbientCapabilities=CAP_IPC_LOCK + # CAP_DAC_OVERRIDE To allow auth_pam_tool (which is SUID root) to read /etc/shadow when it's chmod 0 # does nothing for non-root, not needed if /etc/shadow is u+r # CAP_AUDIT_WRITE auth_pam_tool needs it on Debian for whatever reason diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index 4095f04b800..cb5e885e1aa 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -177,7 +177,10 @@ PrivateNetwork=false ## Package maintainers ## -@SYSTEMD_AMBIENT_CAPABILITIES@ +# CAP_IPC_LOCK To allow memlock to be used as non-root user +# These are enabled by default +AmbientCapabilities=CAP_IPC_LOCK + # CAP_DAC_OVERRIDE To allow auth_pam_tool (which is SUID root) to read /etc/shadow when it's chmod 0 # does nothing for non-root, not needed if /etc/shadow is u+r # CAP_AUDIT_WRITE auth_pam_tool needs it on Debian for whatever reason From c461188ca6ad6ec3a54201eb87ebd75797d296df Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 25 Apr 2025 15:35:41 +0200 Subject: [PATCH 125/125] MDEV-36681 Remove systemd CapabilityBoundingSet as unnecessary Hopefully, this ends the long story of CapabilityBoundingSet in mariadb.service. Started from MDEV-9095 (27e6fd9a5968) which was supposed to let --memlock work without root, but instead of adding the necessary capability (CAP_IPC_LOCK) by putting it into AmbientCapabilities it removed all other capabilities, by putting CAP_IPC_LOCK into CapabilityBoundingSet (which is the mask of allowed capabilities). This broke pam plugin, which needed CAP_DAC_OVERRIDE, it was fixed in MDEV-19878 (dd93028dae22) by appending CAP_DAC_OVERRIDE to CapabilityBoundingSet. Obviously, memlock still didn't work, this was fixed in MDEV-33301 (76a27155b4cd) by moving CAP_IPC_LOCK to AmbientCapabilities. Unfortunately, it moved too much (everything), so MDEV-36229 (85ecb80fa319) fixed it moving CAP_DAC_OVERRIDE back to CapabilityBoundingSet. This caused MDEV-36591 (8925877dc8) triggering a bug in old systemd versions. And it broke pam plugin on CentOS Stream 10, where CAP_DAC_OVERRIDE alone was apparently not enough. Let's finally fix this by removing CapabilityBoundingSet completely and keeping CAP_IPC_LOCK in AmbientCapabilities, which should've been the correct fix for MDEV-9095 from the start. --- support-files/mariadb.service.in | 5 ----- support-files/mariadb@.service.in | 5 ----- 2 files changed, 10 deletions(-) diff --git a/support-files/mariadb.service.in b/support-files/mariadb.service.in index f53a0b8ceda..6d0141a1b35 100644 --- a/support-files/mariadb.service.in +++ b/support-files/mariadb.service.in @@ -51,11 +51,6 @@ Group=mysql # These are enabled by default AmbientCapabilities=CAP_IPC_LOCK -# CAP_DAC_OVERRIDE To allow auth_pam_tool (which is SUID root) to read /etc/shadow when it's chmod 0 -# does nothing for non-root, not needed if /etc/shadow is u+r -# CAP_AUDIT_WRITE auth_pam_tool needs it on Debian for whatever reason -CapabilityBoundingSet=CAP_DAC_OVERRIDE CAP_AUDIT_WRITE - # PrivateDevices=true implies NoNewPrivileges=true and # SUID auth_pam_tool suddenly doesn't do setuid anymore PrivateDevices=false diff --git a/support-files/mariadb@.service.in b/support-files/mariadb@.service.in index cb5e885e1aa..f2fc8b72cf8 100644 --- a/support-files/mariadb@.service.in +++ b/support-files/mariadb@.service.in @@ -181,11 +181,6 @@ PrivateNetwork=false # These are enabled by default AmbientCapabilities=CAP_IPC_LOCK -# CAP_DAC_OVERRIDE To allow auth_pam_tool (which is SUID root) to read /etc/shadow when it's chmod 0 -# does nothing for non-root, not needed if /etc/shadow is u+r -# CAP_AUDIT_WRITE auth_pam_tool needs it on Debian for whatever reason -CapabilityBoundingSet=CAP_DAC_OVERRIDE CAP_AUDIT_WRITE - # PrivateDevices=true implies NoNewPrivileges=true and # SUID auth_pam_tool suddenly doesn't do setuid anymore PrivateDevices=false