From 47908d3f7762d334803045c2d66501daefd3bd9e Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Wed, 29 Jan 2025 00:43:40 +0400 Subject: [PATCH 01/47] MDEV-32619 Fix setting SRID with ST_*FromWKB(). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit second argument is handled in Item_func_geometry_from_wkb::val_str. contributed by Maximilian Krög --- mysql-test/main/gis.result | 27 +++++++++++++++++++++++++++ mysql-test/main/gis.test | 26 ++++++++++++++++++++++++++ sql/item_geofunc.cc | 9 +++++++++ 3 files changed, 62 insertions(+) diff --git a/mysql-test/main/gis.result b/mysql-test/main/gis.result index 196758b1e94..5fa310e93ed 100644 --- a/mysql-test/main/gis.result +++ b/mysql-test/main/gis.result @@ -5474,4 +5474,31 @@ INSERT INTO t VALUES (1,POINT(0,0)),(2,POINT(0,0)); SELECT NTH_VALUE(a,b) OVER () FROM t; ERROR HY000: Illegal parameter data types point and bigint for operation '-' DROP TABLE t; +# +# MDEV-32619 Settng SRID on geometry with ST_*FromWKKB(g, srid) +# +SELECT +ST_SRID(g1), +ST_SRID(ST_GeomFromWKB(g1, 4326)), +ST_SRID(ST_GeomFromWKB(g1)), +ST_AsText(g1), +ST_SRID(ST_PointFromWKB(g2, 4326)), +ST_SRID(g2), +ST_SRID(ST_LineStringFromWKB(g3, 3)), +ST_SRID(ST_PolygonFromWKB(g4, 4)), +ST_SRID(ST_MultiPointFromWKB(g5, 5)), +ST_SRID(ST_MultiLineStringFromWKB(g6, 6)), +ST_SRID(ST_MultiPolygonFromWKB(g7, 7)) +FROM ( +SELECT +POINT(1, 2) AS g1, +POINT(4, 3) AS g2, +LINESTRING(POINT(4, 3), POINT(4, 4)) AS g3, +POLYGON(LINESTRING(POINT(4, 3), POINT(4, 4), POINT(3, 4), POINT(4, 3))) AS g4, +MULTIPOINT(POINT(4, 3)) AS g5, +MULTILINESTRING(LINESTRING(POINT(4, 3), POINT(4, 4))) AS g6, +MULTIPOLYGON(POLYGON(LINESTRING(POINT(4, 3), POINT(4, 4), POINT(3, 4), POINT(4, 3)))) AS g7 +) AS t; +ST_SRID(g1) ST_SRID(ST_GeomFromWKB(g1, 4326)) ST_SRID(ST_GeomFromWKB(g1)) ST_AsText(g1) ST_SRID(ST_PointFromWKB(g2, 4326)) ST_SRID(g2) ST_SRID(ST_LineStringFromWKB(g3, 3)) ST_SRID(ST_PolygonFromWKB(g4, 4)) ST_SRID(ST_MultiPointFromWKB(g5, 5)) ST_SRID(ST_MultiLineStringFromWKB(g6, 6)) ST_SRID(ST_MultiPolygonFromWKB(g7, 7)) +0 4326 0 POINT(1 2) 4326 0 3 4 5 6 7 # End of 10.5 tests diff --git a/mysql-test/main/gis.test b/mysql-test/main/gis.test index 6dc932b4f8d..03c1df94d1f 100644 --- a/mysql-test/main/gis.test +++ b/mysql-test/main/gis.test @@ -3482,4 +3482,30 @@ INSERT INTO t VALUES (1,POINT(0,0)),(2,POINT(0,0)); SELECT NTH_VALUE(a,b) OVER () FROM t; DROP TABLE t; +--echo # +--echo # MDEV-32619 Settng SRID on geometry with ST_*FromWKKB(g, srid) +--echo # +SELECT + ST_SRID(g1), + ST_SRID(ST_GeomFromWKB(g1, 4326)), + ST_SRID(ST_GeomFromWKB(g1)), + ST_AsText(g1), + ST_SRID(ST_PointFromWKB(g2, 4326)), + ST_SRID(g2), + ST_SRID(ST_LineStringFromWKB(g3, 3)), + ST_SRID(ST_PolygonFromWKB(g4, 4)), + ST_SRID(ST_MultiPointFromWKB(g5, 5)), + ST_SRID(ST_MultiLineStringFromWKB(g6, 6)), + ST_SRID(ST_MultiPolygonFromWKB(g7, 7)) +FROM ( + SELECT + POINT(1, 2) AS g1, + POINT(4, 3) AS g2, + LINESTRING(POINT(4, 3), POINT(4, 4)) AS g3, + POLYGON(LINESTRING(POINT(4, 3), POINT(4, 4), POINT(3, 4), POINT(4, 3))) AS g4, + MULTIPOINT(POINT(4, 3)) AS g5, + MULTILINESTRING(LINESTRING(POINT(4, 3), POINT(4, 4))) AS g6, + MULTIPOLYGON(POLYGON(LINESTRING(POINT(4, 3), POINT(4, 4), POINT(3, 4), POINT(4, 3)))) AS g7 +) AS t; + --echo # End of 10.5 tests diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc index 1c364f7b31d..81cd1268f64 100644 --- a/sql/item_geofunc.cc +++ b/sql/item_geofunc.cc @@ -91,6 +91,15 @@ String *Item_func_geometry_from_wkb::val_str(String *str) { String *str_ret= args[0]->val_str(str); null_value= args[0]->null_value; + if (!null_value && arg_count == 2 && !args[1]->null_value) { + srid= (uint32)args[1]->val_int(); + + if (str->copy(*str_ret)) + return 0; + + int4store(str->ptr(), srid); + return str; + } return str_ret; } From f1276aa1bc8ede06ffea973d186d3b0630ffe36a Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Tue, 9 Jul 2024 08:59:59 -0600 Subject: [PATCH 02/47] MDEV-26652: xa transactions binlogged in wrong order Disclaimer: This report was fixed in a previous commit with MDEV-21117, this patch only adds a test to show the presence of the fix. Prior to MDEV-21117, the ordering of the handlers in a transaction's ha_info list solely determined the order in which the handlertons commit. The binlog is supposed to commit first, and is normally placed first in the ha_list to do so; however, in multi-engine 2-phase XA transactions, the binlog can be placed second. This allowed a race-condition for other concurrent transactions to commit and binlog before the prior XA COMMIT would be written to binlog. MDEV-21117 fixed this so the binlog is specially considered to commit first, before traversing the ha_list (see commit_one_phase_2() in sql/hander.cc for this specific change in 6c39eaeb126). --- .../rpl/r/rpl_xa_2pc_multi_engine.result | 26 ++++++++ .../suite/rpl/t/rpl_xa_2pc_multi_engine.test | 63 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 mysql-test/suite/rpl/r/rpl_xa_2pc_multi_engine.result create mode 100644 mysql-test/suite/rpl/t/rpl_xa_2pc_multi_engine.test diff --git a/mysql-test/suite/rpl/r/rpl_xa_2pc_multi_engine.result b/mysql-test/suite/rpl/r/rpl_xa_2pc_multi_engine.result new file mode 100644 index 00000000000..9541107e62b --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_xa_2pc_multi_engine.result @@ -0,0 +1,26 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1 (a int primary key, b int) engine=innodb; +insert t1 values (1,1),(3,3),(5,5),(7,7); +create table t2 (m int) engine=aria; +# Create multi-engine, two-phase XA transaction (T1) +xa start '1'; +insert t2 values (1); +update t1 set b=50 where b=5; +xa end '1'; +xa prepare '1'; +# Create T2 +connection server_1; +update t1 set b=10 where a=5; +connection master; +xa commit '1'; +connection server_1; +include/save_master_gtid.inc +# This would hang prior to MDEV-21117 +connection slave; +include/sync_with_master_gtid.inc +connection master; +drop table t1, t2; +include/rpl_end.inc +# End of rpl_xa_2pc_multi_engine.test diff --git a/mysql-test/suite/rpl/t/rpl_xa_2pc_multi_engine.test b/mysql-test/suite/rpl/t/rpl_xa_2pc_multi_engine.test new file mode 100644 index 00000000000..854935ae6a5 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_xa_2pc_multi_engine.test @@ -0,0 +1,63 @@ +# +# This test ensures binlog order is correct for multi-engine, two-phase XA +# transactions. MDEV-26652 exposed a race condition which would allow +# concurrent transactions which modify the same table record to binlog in +# the "opposite" order, i.e. what _should_ be: +# T1 XA PREPARE +# T1 XA COMMIT +# T2 +# +# was binlogged as +# T1 XA PREPARE +# T2 +# T1 XA COMMIT +# +# which would break replication. +# +# Note that the actual fix for this issue was done with MDEV-21117. +# +# References: +# MDEV-26652: xa transactions binlogged in wrong order +# MDEV-21117: refine the server binlog-based recovery for semisync +# +source include/have_binlog_format_row.inc; +source include/have_innodb.inc; +source include/master-slave.inc; + +--connection master +create table t1 (a int primary key, b int) engine=innodb; +insert t1 values (1,1),(3,3),(5,5),(7,7); +create table t2 (m int) engine=aria; + + +--echo # Create multi-engine, two-phase XA transaction (T1) +xa start '1'; +insert t2 values (1); +update t1 set b=50 where b=5; +xa end '1'; + +# Aria doesn't support XA PREPARE, so disable warnings +--disable_warnings +xa prepare '1'; +--enable_warnings + +--echo # Create T2 +--connection server_1 +--send update t1 set b=10 where a=5 + +--connection master +xa commit '1'; + +--connection server_1 +--reap +--source include/save_master_gtid.inc + +--echo # This would hang prior to MDEV-21117 +--connection slave +--source include/sync_with_master_gtid.inc + +--connection master +drop table t1, t2; + +--source include/rpl_end.inc +--echo # End of rpl_xa_2pc_multi_engine.test From 5f68fd52a944897a8e6e5e5aa782b9edce8d4a22 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Thu, 30 Jan 2025 16:30:56 +0200 Subject: [PATCH 03/47] MDEV-35955 Wrong result for UPDATE ... ORDER BY LIMIT which uses tmp.table (Variant 2) Multi-table UPDATE ... ORDER BY ... LIMIT could update the wrong rows when ORDER BY was resolved by Using temporary + Using filesort. == Background: ref_pointer_array == join->order[->next*]->item point into join->ref_pointer_array, which has pointers to the used Item objects. This indirection is employed so that we can switch the ORDER BY expressions from using the original Items to using the values of their "image" fields in the temporary table. The variant of ref_pointer_array that has pointers to temp table fields is created when JOIN::make_aggr_tables_info() calls change_refs_to_tmp_fields(). == The problem == The created array didn't match element-by-element the original ref_pointer_array. When arrays were switched, ORDER BY elements started to point to the wrong temp.table fields, causing the wrong sorting. == The cause == The cause is JOIN::add_fields_for_current_rowid(). This function is called for UPDATE statements to make the rowids of rows in the original tables to be saved in the temporary tables. It adds extra columns to the select list in table_fields argument. However, select lists are organized in a way that extra elements must be added *to the front* of the list, and then change_refs_to_tmp_fields() will add extra fields *to the end* of ref_pointer_array. So, add_fields_for_current_rowid() adds new fields to the back of table_fields list. This caused change_refs_to_tmp_fields() to produce ref_pointer_array slice with extra elements in the front, causing any references through ref_pointer_array to come to the wrong values. == The fix == Make JOIN::add_fields_for_current_rowid() add fields to the front of the select list. --- mysql-test/main/update.result | 80 +++++++++++++++++++++++++++++++++++ mysql-test/main/update.test | 40 ++++++++++++++++++ sql/sql_select.cc | 9 +++- 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/update.result b/mysql-test/main/update.result index da1964d70c4..5f91415f32e 100644 --- a/mysql-test/main/update.result +++ b/mysql-test/main/update.result @@ -765,3 +765,83 @@ ccc yyy u xxb drop table t1; # End of MariaDB 10.4 tests +# +# MDEV-35955 Wrong result for UPDATE ... ORDER BY LIMIT which uses tmp.table +# +create table t1 (id int primary key, v int); +create table t2 (id int primary key, v int); +insert into t1 (id, v) values (2,3),(1,4); +insert into t2 (id, v) values (5,5),(6,6); +select t1.*, t2.* from t1, t2 order by t1.id, t2.id limit 2; +id v id v +1 4 5 5 +1 4 6 6 +UPDATE t1, t2 SET t1.v=-1, t2.v=-1 ORDER BY t1.id, t2.id LIMIT 2; +select * from t1; +id v +2 3 +1 -1 +select * from t2; +id v +5 -1 +6 -1 +drop table t1, t2; +create table t1 (id int primary key, v text) engine=myisam; +create table t2 (id int primary key, v text) engine=myisam; +insert into t1 (id, v) values (1,'b'),(2,'fo'),(3,'bar'),(4,'barr'),(5,'bazzz'); +insert into t2 (id, v) values (6,'quxqux'),(7,'foofoof'),(8,'barbarba'),(9,'quxquxqux'),(10,'bazbazbazb'); +select t1.*, t2.* from t1, t2 order by t1.id, t2.id limit 2; +id v id v +1 b 6 quxqux +1 b 7 foofoof +update t1, t2 set t1.v='DELETED', t2.v='DELETED' order by t1.id, t2.id limit 2; +select * from t1; +id v +1 DELETED +2 fo +3 bar +4 barr +5 bazzz +select * from t2; +id v +6 DELETED +7 DELETED +8 barbarba +9 quxquxqux +10 bazbazbazb +drop table t1, t2; +create table t1 (id int primary key, v int); +create table t2 (id int primary key, v int); +create table t3 (id int primary key, v int); +insert into t1 (id, v) values (1, 1000), (2, 2000), (3, 3000), (4, 4000), (5, 5000); +insert into t2 (id, v) values (10, 100), (20, 200), (30, 300), (40, 400), (50, 500); +insert into t3 (id, v) values (11, 111), (22, 222), (33, 333), (44, 444), (55, 555); +select t1.*, t2.*, t3.* from t1, t2, t3 order by t1.id, t2.id, t3.id limit 3; +id v id v id v +1 1000 10 100 11 111 +1 1000 10 100 22 222 +1 1000 10 100 33 333 +UPDATE t1, t2, t3 SET t1.v=-1, t2.v=-2, t3.v=-3 ORDER BY t1.id, t2.id, t3.id LIMIT 3; +select * from t1; +id v +1 -1 +2 2000 +3 3000 +4 4000 +5 5000 +select * from t2; +id v +10 -2 +20 200 +30 300 +40 400 +50 500 +select * from t3; +id v +11 -3 +22 -3 +33 -3 +44 444 +55 555 +drop table t1, t2, t3; +# End of MariaDB 10.11 tests diff --git a/mysql-test/main/update.test b/mysql-test/main/update.test index 9b329cce3c9..fe938dd35f0 100644 --- a/mysql-test/main/update.test +++ b/mysql-test/main/update.test @@ -707,3 +707,43 @@ select * from t1; drop table t1; --echo # End of MariaDB 10.4 tests + +--echo # +--echo # MDEV-35955 Wrong result for UPDATE ... ORDER BY LIMIT which uses tmp.table +--echo # + +create table t1 (id int primary key, v int); +create table t2 (id int primary key, v int); +insert into t1 (id, v) values (2,3),(1,4); +insert into t2 (id, v) values (5,5),(6,6); +select t1.*, t2.* from t1, t2 order by t1.id, t2.id limit 2; +UPDATE t1, t2 SET t1.v=-1, t2.v=-1 ORDER BY t1.id, t2.id LIMIT 2; +select * from t1; +select * from t2; + +drop table t1, t2; +create table t1 (id int primary key, v text) engine=myisam; +create table t2 (id int primary key, v text) engine=myisam; +insert into t1 (id, v) values (1,'b'),(2,'fo'),(3,'bar'),(4,'barr'),(5,'bazzz'); +insert into t2 (id, v) values (6,'quxqux'),(7,'foofoof'),(8,'barbarba'),(9,'quxquxqux'),(10,'bazbazbazb'); +select t1.*, t2.* from t1, t2 order by t1.id, t2.id limit 2; +update t1, t2 set t1.v='DELETED', t2.v='DELETED' order by t1.id, t2.id limit 2; +select * from t1; +select * from t2; + +drop table t1, t2; +create table t1 (id int primary key, v int); +create table t2 (id int primary key, v int); +create table t3 (id int primary key, v int); +insert into t1 (id, v) values (1, 1000), (2, 2000), (3, 3000), (4, 4000), (5, 5000); +insert into t2 (id, v) values (10, 100), (20, 200), (30, 300), (40, 400), (50, 500); +insert into t3 (id, v) values (11, 111), (22, 222), (33, 333), (44, 444), (55, 555); +select t1.*, t2.*, t3.* from t1, t2, t3 order by t1.id, t2.id, t3.id limit 3; +UPDATE t1, t2, t3 SET t1.v=-1, t2.v=-2, t3.v=-3 ORDER BY t1.id, t2.id, t3.id LIMIT 3; +select * from t1; +select * from t2; +select * from t3; + +drop table t1, t2, t3; + +--echo # End of MariaDB 10.11 tests diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b88e8b4c5d5..572a59f1989 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -3581,7 +3581,14 @@ bool JOIN::add_fields_for_current_rowid(JOIN_TAB *cur, List *table_fields) continue; Item *item= new (thd->mem_root) Item_temptable_rowid(tab->table); item->fix_fields(thd, 0); - table_fields->push_back(item, thd->mem_root); + /* + table_fields points to JOIN::all_fields or JOIN::tmp_all_fields_*. + These lists start with "added" fields and then their suffix is shared + with JOIN::fields_list or JOIN::tmp_fields_list*. + Because of that, new elements can only be added to the front of the list, + not to the back. + */ + table_fields->push_front(item, thd->mem_root); cur->tmp_table_param->func_count++; } return 0; From 3f5b6a9837413d4a6a8526a076be5eba88a56286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 14 Jan 2025 14:29:29 +0200 Subject: [PATCH 04/47] MDEV-35748 : Attempting to create a CONNECT engine Table results in non-InnoDB sequences in Galera cluster error Problem was incorrect condition on wsrep_check_sequence when ENGINE!=InnoDB. Fix is not use DB_TYPE_XXX because it is not correct on dynamic storage engines. Instead used storage engine name is looked from thd->lex->m_sql_cmd->option_storage_engine_name. For CREATE TABLE allow anyting except ENGINE=SEQUENCE. For CREATE SEQUENCE only ENGINE=InnoDB is supported. For ALTER TABLE if original table contains sequence information only ENGINE=InnoDB is supported. Signed-off-by: Julius Goryavsky --- mysql-test/include/have_rocksdb.inc | 4 +++ mysql-test/suite/galera/r/MDEV-35748.result | 31 +++++++++++++++++++++ mysql-test/suite/galera/t/MDEV-35748.opt | 1 + mysql-test/suite/galera/t/MDEV-35748.test | 22 +++++++++++++++ sql/sql_cmd.h | 1 + sql/sql_table.cc | 23 +++++++++++++-- 6 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 mysql-test/include/have_rocksdb.inc create mode 100644 mysql-test/suite/galera/r/MDEV-35748.result create mode 100644 mysql-test/suite/galera/t/MDEV-35748.opt create mode 100644 mysql-test/suite/galera/t/MDEV-35748.test diff --git a/mysql-test/include/have_rocksdb.inc b/mysql-test/include/have_rocksdb.inc new file mode 100644 index 00000000000..bc6ec648605 --- /dev/null +++ b/mysql-test/include/have_rocksdb.inc @@ -0,0 +1,4 @@ +if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'ROCKSDB' AND support IN ('YES', 'DEFAULT', 'ENABLED')`) +{ + --skip Test requires MyRocks engine +} diff --git a/mysql-test/suite/galera/r/MDEV-35748.result b/mysql-test/suite/galera/r/MDEV-35748.result new file mode 100644 index 00000000000..fca2ab3ad40 --- /dev/null +++ b/mysql-test/suite/galera/r/MDEV-35748.result @@ -0,0 +1,31 @@ +connection node_2; +connection node_1; +connection node_1; +INSTALL PLUGIN IF NOT EXISTS connect SONAME 'ha_connect'; +CREATE TABLE t1 (f INT) ENGINE=CONNECT; +Warnings: +Warning 1105 No table_type. Will be set to DOS +Warning 1105 No file name. Table will use t1.dos +CREATE TABLE t2 (f INT) ENGINE=ROCKSDB; +CREATE TABLE t3 (f INT) ENGINE=SEQUENCE; +ERROR 42000: This version of MariaDB doesn't yet support 'non-InnoDB sequences in Galera cluster' +show warnings; +Level Code Message +Error 1235 This version of MariaDB doesn't yet support 'non-InnoDB sequences in Galera cluster' +Note 1235 ENGINE=SEQUENCE not supported by Galera +connection node_2; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `f` int(11) DEFAULT NULL +) ENGINE=CONNECT DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `f` int(11) DEFAULT NULL +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +show create table t3; +ERROR 42S02: Table 'test.t3' doesn't exist +connection node_1; +DROP TABLE t1, t2; +UNINSTALL PLUGIN IF EXISTS connect; diff --git a/mysql-test/suite/galera/t/MDEV-35748.opt b/mysql-test/suite/galera/t/MDEV-35748.opt new file mode 100644 index 00000000000..df675545bf9 --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-35748.opt @@ -0,0 +1 @@ +--plugin-load=$HA_ROCKSDB_SO diff --git a/mysql-test/suite/galera/t/MDEV-35748.test b/mysql-test/suite/galera/t/MDEV-35748.test new file mode 100644 index 00000000000..0c0e83dcff7 --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-35748.test @@ -0,0 +1,22 @@ +--source include/galera_cluster.inc +--source include/have_sequence.inc +--source include/have_rocksdb.inc + +--connection node_1 +INSTALL PLUGIN IF NOT EXISTS connect SONAME 'ha_connect'; + +CREATE TABLE t1 (f INT) ENGINE=CONNECT; +CREATE TABLE t2 (f INT) ENGINE=ROCKSDB; +--error ER_NOT_SUPPORTED_YET +CREATE TABLE t3 (f INT) ENGINE=SEQUENCE; +show warnings; + +--connection node_2 +show create table t1; +show create table t2; +--error ER_NO_SUCH_TABLE +show create table t3; + +--connection node_1 +DROP TABLE t1, t2; +UNINSTALL PLUGIN IF EXISTS connect; diff --git a/sql/sql_cmd.h b/sql/sql_cmd.h index ea518375226..e1ce96efb29 100644 --- a/sql/sql_cmd.h +++ b/sql/sql_cmd.h @@ -141,6 +141,7 @@ public: handlerton **ha, bool tmp_table); bool is_set() { return m_storage_engine_name.str != NULL; } + const LEX_CSTRING *name() const { return &m_storage_engine_name; } }; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 428b406c58c..6f5c1839e1f 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -5708,9 +5708,26 @@ bool wsrep_check_sequence(THD* thd, // In Galera cluster we support only InnoDB sequences if (db_type != DB_TYPE_INNODB) { - my_error(ER_NOT_SUPPORTED_YET, MYF(0), - "non-InnoDB sequences in Galera cluster"); - return(true); + // Currently any dynamic storage engine is not possible to identify + // using DB_TYPE_XXXX and ENGINE=SEQUENCE is one of them. + // Therefore, we get storage engine name from lex. + const LEX_CSTRING *tb_name= thd->lex->m_sql_cmd->option_storage_engine_name()->name(); + // (1) CREATE TABLE ... ENGINE=SEQUENCE OR + // (2) ALTER TABLE ... ENGINE= OR + // Note in ALTER TABLE table->s->sequence != nullptr + // (3) CREATE SEQUENCE ... ENGINE= + if ((thd->lex->sql_command == SQLCOM_CREATE_TABLE && + lex_string_eq(tb_name, STRING_WITH_LEN("SEQUENCE"))) || + (thd->lex->sql_command == SQLCOM_ALTER_TABLE) || + (thd->lex->sql_command == SQLCOM_CREATE_SEQUENCE)) + { + my_error(ER_NOT_SUPPORTED_YET, MYF(0), + "non-InnoDB sequences in Galera cluster"); + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, + ER_NOT_SUPPORTED_YET, + "ENGINE=%s not supported by Galera", tb_name->str); + return(true); + } } // In Galera cluster it is best to use INCREMENT BY 0 with CACHE From 7d69902d834a1c9c96df6305bcfd53336d471c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 13 Apr 2023 13:45:00 +0300 Subject: [PATCH 05/47] MDEV-29775 : Assertion `0' failed in void Protocol::end_statement() when adding data to the MyISAM table after setting wsrep_mode=replicate_myisam If wsrep_replicate_myisam=ON we allow wsrep_forced_binlog_format to be [DEFAULT|ROW]. Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera/r/mdev-29775.result | 44 ++++++++++++++++++ mysql-test/suite/galera/t/mdev-29775.test | 45 ++++++++++++++++++ .../r/wsrep_forced_binlog_format.result} | 0 .../wsrep/t/wsrep_forced_binlog_format.cnf | 7 +++ .../t/wsrep_forced_binlog_format.test} | 5 +- sql/sys_vars.cc | 8 +++- sql/wsrep_var.cc | 46 ++++++++++++++++++- sql/wsrep_var.h | 5 +- 8 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 mysql-test/suite/galera/r/mdev-29775.result create mode 100644 mysql-test/suite/galera/t/mdev-29775.test rename mysql-test/suite/{sys_vars/r/wsrep_forced_binlog_format_basic.result => wsrep/r/wsrep_forced_binlog_format.result} (100%) create mode 100644 mysql-test/suite/wsrep/t/wsrep_forced_binlog_format.cnf rename mysql-test/suite/{sys_vars/t/wsrep_forced_binlog_format_basic.test => wsrep/t/wsrep_forced_binlog_format.test} (91%) diff --git a/mysql-test/suite/galera/r/mdev-29775.result b/mysql-test/suite/galera/r/mdev-29775.result new file mode 100644 index 00000000000..dc38f18b433 --- /dev/null +++ b/mysql-test/suite/galera/r/mdev-29775.result @@ -0,0 +1,44 @@ +connection node_2; +connection node_1; +SET GLOBAL wsrep_replicate_myisam=ON; +CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM; +INSERT INTO t VALUES(); +SELECT * FROM t; +f0 +NULL +connection node_2; +SELECT * FROM t; +f0 +NULL +DROP TABLE t; +connection node_1; +SET GLOBAL wsrep_replicate_myisam=ON; +SET GLOBAL wsrep_forced_binlog_format=ROW; +CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM; +INSERT INTO t VALUES(); +SELECT * FROM t; +f0 +NULL +connection node_2; +SELECT * FROM t; +f0 +NULL +DROP TABLE t; +connection node_1; +SET GLOBAL wsrep_forced_binlog_format=DEFAULT; +SET GLOBAL wsrep_replicate_myisam=OFF; +SET GLOBAL wsrep_forced_binlog_format=MIXED; +SET GLOBAL wsrep_replicate_myisam=ON; +ERROR HY000: wsrep_replicate_myisam=ON can't be enabled if wsrep_forced_binlog != [NONE|ROW] +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; +SET GLOBAL wsrep_replicate_myisam=ON; +ERROR HY000: wsrep_replicate_myisam=ON can't be enabled if wsrep_forced_binlog != [NONE|ROW] +SET GLOBAL wsrep_forced_binlog_format=NONE; +SET GLOBAL wsrep_replicate_myisam=OFF; +SET GLOBAL wsrep_replicate_myisam=ON; +SET GLOBAL wsrep_forced_binlog_format=MIXED; +ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_replicate_myisam=ON +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; +ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_replicate_myisam=ON +SET GLOBAL wsrep_forced_binlog_format=NONE; +SET GLOBAL wsrep_replicate_myisam=OFF; diff --git a/mysql-test/suite/galera/t/mdev-29775.test b/mysql-test/suite/galera/t/mdev-29775.test new file mode 100644 index 00000000000..5c6725d780d --- /dev/null +++ b/mysql-test/suite/galera/t/mdev-29775.test @@ -0,0 +1,45 @@ +--source include/galera_cluster.inc + +# +# MDEV-29775 : Assertion `0' failed in void Protocol::end_statement() when adding data to the MyISAM table after setting wsrep_mode=replicate_myisam +# +SET GLOBAL wsrep_replicate_myisam=ON; +CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM; +INSERT INTO t VALUES(); +SELECT * FROM t; +--connection node_2 +SELECT * FROM t; +DROP TABLE t; + +--connection node_1 +SET GLOBAL wsrep_replicate_myisam=ON; +SET GLOBAL wsrep_forced_binlog_format=ROW; +CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM; +INSERT INTO t VALUES(); +SELECT * FROM t; +--connection node_2 +SELECT * FROM t; +DROP TABLE t; + +--connection node_1 +SET GLOBAL wsrep_forced_binlog_format=DEFAULT; +SET GLOBAL wsrep_replicate_myisam=OFF; + +SET GLOBAL wsrep_forced_binlog_format=MIXED; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_replicate_myisam=ON; +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_replicate_myisam=ON; + +SET GLOBAL wsrep_forced_binlog_format=NONE; +SET GLOBAL wsrep_replicate_myisam=OFF; + +SET GLOBAL wsrep_replicate_myisam=ON; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_forced_binlog_format=MIXED; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; + +SET GLOBAL wsrep_forced_binlog_format=NONE; +SET GLOBAL wsrep_replicate_myisam=OFF; diff --git a/mysql-test/suite/sys_vars/r/wsrep_forced_binlog_format_basic.result b/mysql-test/suite/wsrep/r/wsrep_forced_binlog_format.result similarity index 100% rename from mysql-test/suite/sys_vars/r/wsrep_forced_binlog_format_basic.result rename to mysql-test/suite/wsrep/r/wsrep_forced_binlog_format.result diff --git a/mysql-test/suite/wsrep/t/wsrep_forced_binlog_format.cnf b/mysql-test/suite/wsrep/t/wsrep_forced_binlog_format.cnf new file mode 100644 index 00000000000..b1c96d2614d --- /dev/null +++ b/mysql-test/suite/wsrep/t/wsrep_forced_binlog_format.cnf @@ -0,0 +1,7 @@ +!include ../my.cnf + +[mysqld.1] +wsrep-on=ON +wsrep-cluster-address=gcomm:// +wsrep-provider=@ENV.WSREP_PROVIDER +binlog-format=ROW diff --git a/mysql-test/suite/sys_vars/t/wsrep_forced_binlog_format_basic.test b/mysql-test/suite/wsrep/t/wsrep_forced_binlog_format.test similarity index 91% rename from mysql-test/suite/sys_vars/t/wsrep_forced_binlog_format_basic.test rename to mysql-test/suite/wsrep/t/wsrep_forced_binlog_format.test index 455034bb623..37e9c6c718e 100644 --- a/mysql-test/suite/sys_vars/t/wsrep_forced_binlog_format_basic.test +++ b/mysql-test/suite/wsrep/t/wsrep_forced_binlog_format.test @@ -1,4 +1,7 @@ ---source include/have_wsrep.inc +--source include/have_innodb.inc +--source include/have_wsrep_provider.inc +--source include/have_binlog_format_row.inc + --echo # --echo # wsrep_forced_binlog_format diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index a2058dee224..fa258ef5de2 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -6035,7 +6035,9 @@ static const char *wsrep_binlog_format_names[]= static Sys_var_enum Sys_wsrep_forced_binlog_format( "wsrep_forced_binlog_format", "binlog format to take effect over user's choice", GLOBAL_VAR(wsrep_forced_binlog_format), CMD_LINE(REQUIRED_ARG), - wsrep_binlog_format_names, DEFAULT(BINLOG_FORMAT_UNSPEC)); + wsrep_binlog_format_names, DEFAULT(BINLOG_FORMAT_UNSPEC), + NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(wsrep_forced_binlog_format_check)); static Sys_var_mybool Sys_wsrep_recover_datadir( "wsrep_recover", "Recover database state after crash and exit", @@ -6044,7 +6046,9 @@ static Sys_var_mybool Sys_wsrep_recover_datadir( static Sys_var_mybool Sys_wsrep_replicate_myisam( "wsrep_replicate_myisam", "To enable myisam replication", - GLOBAL_VAR(wsrep_replicate_myisam), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); + GLOBAL_VAR(wsrep_replicate_myisam), CMD_LINE(OPT_ARG), DEFAULT(FALSE), + NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(wsrep_replicate_myisam_check)); static Sys_var_mybool Sys_wsrep_log_conflicts( "wsrep_log_conflicts", "To log multi-master conflicts", diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 0db2e15d071..61241702ef8 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -1,4 +1,4 @@ -/* Copyright 2008-2022 Codership Oy +/* Copyright 2008-2023 Codership Oy This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1124,3 +1124,47 @@ bool wsrep_gtid_domain_id_update(sys_var* self, THD *thd, enum_var_type) wsrep_gtid_server.domain_id= wsrep_gtid_domain_id; return false; } + +bool wsrep_replicate_myisam_check(sys_var *self, THD* thd, set_var* var) +{ + bool new_replicate_myisam= (bool)var->save_result.ulonglong_value; + + if (new_replicate_myisam && !WSREP_ON_) + { + my_message(ER_WRONG_ARGUMENTS, "wsrep_replicate_myisam=ON can't be enabled " + "if wsrep_on=OFF", MYF(0)); + return true; + } + if (new_replicate_myisam && + !(wsrep_forced_binlog_format == BINLOG_FORMAT_UNSPEC || + wsrep_forced_binlog_format == BINLOG_FORMAT_ROW)) + { + my_message(ER_WRONG_ARGUMENTS, "wsrep_replicate_myisam=ON can't be enabled " + "if wsrep_forced_binlog != [NONE|ROW]", MYF(0)); + return true; + } + return false; +} + +bool wsrep_forced_binlog_format_check(sys_var *self, THD* thd, set_var* var) +{ + ulonglong new_forced_binlog_format= var->save_result.ulonglong_value; + if (new_forced_binlog_format != BINLOG_FORMAT_UNSPEC && !WSREP_ON) + { + my_message(ER_WRONG_ARGUMENTS, "wsrep_forced_binlog_format can't be set " + "if wsrep_on=OFF", MYF(0)); + return true; + } + if (!(new_forced_binlog_format == BINLOG_FORMAT_UNSPEC || + new_forced_binlog_format == BINLOG_FORMAT_ROW)) + { + if (wsrep_replicate_myisam) + { + my_message(ER_WRONG_ARGUMENTS, "wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set " + "if wsrep_replicate_myisam=ON", MYF(0)); + return true; + } + } + + return false; +} diff --git a/sql/wsrep_var.h b/sql/wsrep_var.h index 997784674dd..89053991cb9 100644 --- a/sql/wsrep_var.h +++ b/sql/wsrep_var.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2013 Codership Oy +/* Copyright (C) 2013-2023 Codership Oy This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -107,6 +107,9 @@ extern bool wsrep_debug_update UPDATE_ARGS; extern bool wsrep_gtid_seq_no_check CHECK_ARGS; extern bool wsrep_gtid_domain_id_update UPDATE_ARGS; + +extern bool wsrep_replicate_myisam_check CHECK_ARGS; +extern bool wsrep_forced_binlog_format_check CHECK_ARGS; #else /* WITH_WSREP */ #define wsrep_provider_init(X) From de216618e2cec3fe7f36166de472397fe3e19c85 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 26 Sep 2024 05:06:40 +0200 Subject: [PATCH 06/47] MDEV-29775 addendum: Relaxation of unnecessary restrictions + cosmetic corrections taken from the 10.6+ branch of the fix (to simplify merge). --- .../t/galera_var_replicate_myisam_off.test | 1 - .../t/galera_var_replicate_myisam_on.test | 1 - sql/wsrep_var.cc | 24 ++++--------------- 3 files changed, 5 insertions(+), 21 deletions(-) diff --git a/mysql-test/suite/galera/t/galera_var_replicate_myisam_off.test b/mysql-test/suite/galera/t/galera_var_replicate_myisam_off.test index a9811283918..1261963520c 100644 --- a/mysql-test/suite/galera/t/galera_var_replicate_myisam_off.test +++ b/mysql-test/suite/galera/t/galera_var_replicate_myisam_off.test @@ -3,7 +3,6 @@ # --source include/galera_cluster.inc ---source include/have_innodb.inc --let $wsrep_replicate_myisam_orig = `SELECT @@wsrep_replicate_myisam` diff --git a/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test b/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test index acebe4cccdc..0a364452576 100644 --- a/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test +++ b/mysql-test/suite/galera/t/galera_var_replicate_myisam_on.test @@ -236,4 +236,3 @@ DROP TABLE t1; --disable_query_log --eval SET GLOBAL wsrep_replicate_myisam = $wsrep_replicate_myisam_orig --enable_query_log - diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 61241702ef8..1949c15f75c 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -365,14 +365,12 @@ bool wsrep_start_position_check (sys_var *self, THD* thd, set_var* var) var->save_result.string_value.length); start_pos_buf[var->save_result.string_value.length]= 0; - WSREP_DEBUG("SST wsrep_start_position check for new position %s old %s", - start_pos_buf, wsrep_start_position); + start_pos_buf, wsrep_start_position); // Verify the format. if (wsrep_start_position_verify(start_pos_buf)) return true; - // Give error if position is updated when wsrep is not enabled or // provider is not loaded. if ((!WSREP_ON || !Wsrep_server_state::instance().is_provider_loaded()) @@ -679,7 +677,7 @@ bool wsrep_cluster_address_update (sys_var *self, THD* thd, enum_var_type type) { wsrep_create_rollbacker(); WSREP_DEBUG("Cluster address update creating %ld applier threads running %lu", - wsrep_slave_threads, wsrep_running_applier_threads); + wsrep_slave_threads, wsrep_running_applier_threads); wsrep_create_appliers(wsrep_slave_threads); } mysql_mutex_unlock(&LOCK_wsrep_cluster_config); @@ -783,7 +781,7 @@ static void wsrep_slave_count_change_update () { wsrep_slave_count_change = (wsrep_slave_threads - wsrep_running_applier_threads); WSREP_DEBUG("Change on slave threads: New %ld old %lu difference %d", - wsrep_slave_threads, wsrep_running_applier_threads, wsrep_slave_count_change); + wsrep_slave_threads, wsrep_running_applier_threads, wsrep_slave_count_change); } bool wsrep_slave_threads_update (sys_var *self, THD* thd, enum_var_type type) @@ -808,9 +806,9 @@ bool wsrep_slave_threads_update (sys_var *self, THD* thd, enum_var_type type) // Thread creation and execution is asyncronous, therefore we need // wait them to be started or error produced while (wsrep_running_applier_threads != (ulong)wsrep_slave_threads && - !wsrep_thread_create_failed.load(std::memory_order_relaxed)) + !wsrep_thread_create_failed.load(std::memory_order_relaxed)) { - my_sleep(1000); + my_sleep(1000); } mysql_mutex_lock(&LOCK_global_system_variables); @@ -1129,12 +1127,6 @@ bool wsrep_replicate_myisam_check(sys_var *self, THD* thd, set_var* var) { bool new_replicate_myisam= (bool)var->save_result.ulonglong_value; - if (new_replicate_myisam && !WSREP_ON_) - { - my_message(ER_WRONG_ARGUMENTS, "wsrep_replicate_myisam=ON can't be enabled " - "if wsrep_on=OFF", MYF(0)); - return true; - } if (new_replicate_myisam && !(wsrep_forced_binlog_format == BINLOG_FORMAT_UNSPEC || wsrep_forced_binlog_format == BINLOG_FORMAT_ROW)) @@ -1149,12 +1141,6 @@ bool wsrep_replicate_myisam_check(sys_var *self, THD* thd, set_var* var) bool wsrep_forced_binlog_format_check(sys_var *self, THD* thd, set_var* var) { ulonglong new_forced_binlog_format= var->save_result.ulonglong_value; - if (new_forced_binlog_format != BINLOG_FORMAT_UNSPEC && !WSREP_ON) - { - my_message(ER_WRONG_ARGUMENTS, "wsrep_forced_binlog_format can't be set " - "if wsrep_on=OFF", MYF(0)); - return true; - } if (!(new_forced_binlog_format == BINLOG_FORMAT_UNSPEC || new_forced_binlog_format == BINLOG_FORMAT_ROW)) { From 22414d2ed0f1f8be26fb5e82e0129b629e5dbd20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 17 Jul 2023 17:29:20 +0300 Subject: [PATCH 07/47] MDEV-27861: Creating partitioned tables should not be allowed with wsrep_osu_method=TOI and wsrep_strict_ddl=ON Problem was incorrect handling of partitioned tables, because db_type == DB_TYPE_PARTITION_DB wsrep_should_replicate_ddl incorrectly marked DDL as not replicatable. However, in partitioned tables we should check implementing storage engine from table->file->partition_ht() if available because if partition handler is InnoDB all DDL should be allowed even with wsrep_strict_ddl. For other storage engines DDL should not be allowed and error should be issued. This is 10.5 version of the fix. Signed-off-by: Julius Goryavsky --- .../galera/r/galera_partitioned_tables.result | 167 ++++++++++++++++++ .../suite/galera/r/wsrep_strict_ddl.result | 6 +- .../galera/t/galera_partitioned_tables.test | 131 ++++++++++++++ sql/sql_rename.cc | 2 +- sql/sql_table.cc | 34 +++- sql/sql_trigger.cc | 6 +- sql/sql_truncate.cc | 29 ++- sql/sql_view.cc | 11 +- sql/wsrep_mysqld.cc | 80 +++++---- sql/wsrep_mysqld.h | 2 +- 10 files changed, 416 insertions(+), 52 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_partitioned_tables.result create mode 100644 mysql-test/suite/galera/t/galera_partitioned_tables.test diff --git a/mysql-test/suite/galera/r/galera_partitioned_tables.result b/mysql-test/suite/galera/r/galera_partitioned_tables.result new file mode 100644 index 00000000000..4b5c9e74c54 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_partitioned_tables.result @@ -0,0 +1,167 @@ +connection node_2; +connection node_1; +# wsrep-strict_ddl = OFF +SET GLOBAL wsrep_strict_ddl = OFF; +SELECT @@wsrep_strict_ddl; +@@wsrep_strict_ddl +0 +CREATE OR REPLACE TABLE t1 (v1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB +PARTITION BY KEY (v1) +PARTITIONS 2; +CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM +PARTITION BY KEY (v1) +PARTITIONS 2; +ALTER TABLE t1 ADD COLUMN v2 int; +ALTER TABLE t2 ADD COLUMN v2 int; +INSERT INTO t1 VALUES (1,1),(2,2); +INSERT INTO t2 VALUES (1,1),(2,2); +ALTER TABLE t1 ADD COLUMN v3 int, ENGINE=MyISAM; +ALTER TABLE t2 ADD COLUMN v3 int, ENGINE=Aria; +UPDATE t1 SET v3 = 3; +UPDATE t2 SET v3 = 3; +CREATE INDEX xx1 ON t1(v2); +CREATE INDEX xx2 ON t2(v2); +DROP INDEX xx1 ON t1; +DROP INDEX xx2 ON t2; +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +RENAME TABLE t1 TO t1_v2; +RENAME TABLE t2 TO t2_v2; +CREATE VIEW x1 AS SELECT * FROM t1_v2; +CREATE VIEW x2 AS SELECT * FROM t2_v2; +CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t1 +AFTER INSERT ON t1_v2 FOR EACH ROW +UPDATE t1_v2 SET t1_v2.v3 = t1_v2.v3+1; +CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t2 +AFTER INSERT ON t2_v2 FOR EACH ROW +UPDATE t2_v2 SET t2_v2.v3 = t2_v2.v3+1; +connection node_2; +SHOW CREATE TABLE t1_v2; +Table Create Table +t1_v2 CREATE TABLE `t1_v2` ( + `v1` int(11) NOT NULL, + `v2` int(11) DEFAULT NULL, + `v3` int(11) DEFAULT NULL, + PRIMARY KEY (`v1`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY KEY (`v1`) +PARTITIONS 2 +SHOW CREATE TABLE t2_v2; +Table Create Table +t2_v2 CREATE TABLE `t2_v2` ( + `v1` int(11) NOT NULL, + `v2` int(11) DEFAULT NULL, + `v3` int(11) DEFAULT NULL, + PRIMARY KEY (`v1`) +) ENGINE=Aria DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY KEY (`v1`) +PARTITIONS 2 +SHOW CREATE VIEW x1; +View Create View character_set_client collation_connection +x1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `x1` AS select `t1_v2`.`v1` AS `v1`,`t1_v2`.`v2` AS `v2`,`t1_v2`.`v3` AS `v3` from `t1_v2` latin1 latin1_swedish_ci +SHOW CREATE VIEW x2; +View Create View character_set_client collation_connection +x2 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `x2` AS select `t2_v2`.`v1` AS `v1`,`t2_v2`.`v2` AS `v2`,`t2_v2`.`v3` AS `v3` from `t2_v2` latin1 latin1_swedish_ci +SELECT * FROM t1_v2; +v1 v2 v3 +SELECT * FROM t2_v2; +v1 v2 v3 +connection node_1; +DROP VIEW x1; +DROP VIEW x2; +DROP TRIGGER increment_before_t1; +DROP TRIGGER increment_before_t2; +DROP TABLE t1_v2; +DROP TABLE t2_v2; +SET GLOBAL wsrep_strict_ddl = OFF; +CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM +PARTITION BY KEY (v1) +PARTITIONS 2; +# wsrep-strict_ddl = ON +SET GLOBAL wsrep_strict_ddl = ON; +SELECT @@wsrep_strict_ddl; +@@wsrep_strict_ddl +1 +CREATE OR REPLACE TABLE t1 (v1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB +PARTITION BY KEY (v1) +PARTITIONS 2; +CREATE OR REPLACE TABLE t3 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM +PARTITION BY KEY (v1) +PARTITIONS 2; +ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ALTER TABLE t1 ADD COLUMN v2 int; +ALTER TABLE t2 ADD COLUMN v2 int; +ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +INSERT INTO t1 VALUES (1,1),(2,2); +INSERT INTO t2 VALUES (1),(2); +ALTER TABLE t1 ADD COLUMN v3 int, ENGINE=MyISAM; +ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ALTER TABLE t2 ADD COLUMN v3 int, ENGINE=Aria; +ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +UPDATE t1 SET v2 = v2 + 3; +UPDATE t2 SET v1 = v1 + 3; +CREATE INDEX xx1 ON t1(v2); +CREATE INDEX xx2 ON t2(v2); +ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +DROP INDEX xx1 ON t1; +DROP INDEX xx2 on t2; +ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +RENAME TABLE t1 TO t1_v2; +RENAME TABLE t2 TO t2_v2; +RENAME TABLE t2_v2 TO t2; +CREATE VIEW x1 AS SELECT * FROM t1_v2; +CREATE VIEW x2 AS SELECT * FROM t2; +ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t1 +AFTER INSERT ON t1_v2 FOR EACH ROW +UPDATE t1_v2 SET t1_v2.v2 = t1_v2.v2+1; +CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t2 +AFTER INSERT ON t2 FOR EACH ROW +UPDATE t2 SET t2.v1 = t2.v1+1; +ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +connection node_2; +SHOW CREATE TABLE t1_v2; +Table Create Table +t1_v2 CREATE TABLE `t1_v2` ( + `v1` int(11) NOT NULL, + `v2` int(11) DEFAULT NULL, + PRIMARY KEY (`v1`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY KEY (`v1`) +PARTITIONS 2 +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `v1` int(11) NOT NULL, + `v2` int(11) DEFAULT NULL, + PRIMARY KEY (`v1`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci + PARTITION BY KEY (`v1`) +PARTITIONS 2 +SHOW CREATE VIEW x1; +View Create View character_set_client collation_connection +x1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `x1` AS select `t1_v2`.`v1` AS `v1`,`t1_v2`.`v2` AS `v2` from `t1_v2` latin1 latin1_swedish_ci +SELECT * FROM t1_v2; +v1 v2 +SELECT * FROM t2; +v1 v2 +connection node_1; +DROP VIEW x1; +DROP TRIGGER increment_before_t1; +DROP TABLE t1_v2; +DROP TABLE t2; +SET GLOBAL wsrep_strict_ddl=OFF; +CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM +PARTITION BY KEY (v1) +PARTITIONS 2; +# wsrep-strict_ddl = ON +SET GLOBAL wsrep_strict_ddl = ON; +SELECT @@wsrep_strict_ddl; +@@wsrep_strict_ddl +1 +ALTER TABLE t2 ENGINE=InnoDB; +DROP TABLE t2; +SET GLOBAL wsrep_strict_ddl = DEFAULT; diff --git a/mysql-test/suite/galera/r/wsrep_strict_ddl.result b/mysql-test/suite/galera/r/wsrep_strict_ddl.result index 450c11be22d..7b166f00bc3 100644 --- a/mysql-test/suite/galera/r/wsrep_strict_ddl.result +++ b/mysql-test/suite/galera/r/wsrep_strict_ddl.result @@ -20,7 +20,7 @@ ERROR HY000: DDL-statement is forbidden as table storage engine does not support SHOW WARNINGS; Level Code Message Error 4165 DDL-statement is forbidden as table storage engine does not support Galera replication -Warning 1031 WSREP: wsrep_strict_ddl=true and storage engine does not support Galera replication. +Warning 1031 WSREP: wsrep_strict_dll enabled. Storage engine Aria not supported. connection node_2; SHOW CREATE TABLE t1; ERROR 42S02: Table 'test.t1' doesn't exist @@ -31,7 +31,9 @@ ERROR HY000: DDL-statement is forbidden as table storage engine does not support SHOW WARNINGS; Level Code Message Error 4165 DDL-statement is forbidden as table storage engine does not support Galera replication -Warning 1031 WSREP: wsrep_strict_ddl=true and storage engine does not support Galera replication. +Warning 1031 WSREP: wsrep_strict_dll enabled. Storage engine MyISAM not supported. +Error 4165 DDL-statement is forbidden as table storage engine does not support Galera replication +Warning 1031 WSREP: wsrep_strict_dll enabled. Storage engine MyISAM not supported. SHOW CREATE TABLE t2; Table Create Table t2 CREATE TABLE `t2` ( diff --git a/mysql-test/suite/galera/t/galera_partitioned_tables.test b/mysql-test/suite/galera/t/galera_partitioned_tables.test new file mode 100644 index 00000000000..d65ee6c1a49 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_partitioned_tables.test @@ -0,0 +1,131 @@ +--source include/galera_cluster.inc +--source include/have_partition.inc +--source include/have_innodb.inc +--source include/have_aria.inc + +--echo # wsrep-strict_ddl = OFF +SET GLOBAL wsrep_strict_ddl = OFF; +SELECT @@wsrep_strict_ddl; +CREATE OR REPLACE TABLE t1 (v1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB + PARTITION BY KEY (v1) + PARTITIONS 2; +CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM + PARTITION BY KEY (v1) + PARTITIONS 2; +ALTER TABLE t1 ADD COLUMN v2 int; +ALTER TABLE t2 ADD COLUMN v2 int; +INSERT INTO t1 VALUES (1,1),(2,2); +INSERT INTO t2 VALUES (1,1),(2,2); +ALTER TABLE t1 ADD COLUMN v3 int, ENGINE=MyISAM; +ALTER TABLE t2 ADD COLUMN v3 int, ENGINE=Aria; +UPDATE t1 SET v3 = 3; +UPDATE t2 SET v3 = 3; +CREATE INDEX xx1 ON t1(v2); +CREATE INDEX xx2 ON t2(v2); +DROP INDEX xx1 ON t1; +DROP INDEX xx2 ON t2; +TRUNCATE TABLE t1; +TRUNCATE TABLE t2; +RENAME TABLE t1 TO t1_v2; +RENAME TABLE t2 TO t2_v2; +CREATE VIEW x1 AS SELECT * FROM t1_v2; +CREATE VIEW x2 AS SELECT * FROM t2_v2; +CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t1 + AFTER INSERT ON t1_v2 FOR EACH ROW + UPDATE t1_v2 SET t1_v2.v3 = t1_v2.v3+1; +CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t2 + AFTER INSERT ON t2_v2 FOR EACH ROW + UPDATE t2_v2 SET t2_v2.v3 = t2_v2.v3+1; + +--connection node_2 +SHOW CREATE TABLE t1_v2; +SHOW CREATE TABLE t2_v2; +SHOW CREATE VIEW x1; +SHOW CREATE VIEW x2; + +SELECT * FROM t1_v2; +SELECT * FROM t2_v2; + +--connection node_1 +DROP VIEW x1; +DROP VIEW x2; +DROP TRIGGER increment_before_t1; +DROP TRIGGER increment_before_t2; +DROP TABLE t1_v2; +DROP TABLE t2_v2; + +SET GLOBAL wsrep_strict_ddl = OFF; +CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM + PARTITION BY KEY (v1) + PARTITIONS 2; +--echo # wsrep-strict_ddl = ON +SET GLOBAL wsrep_strict_ddl = ON; +SELECT @@wsrep_strict_ddl; +CREATE OR REPLACE TABLE t1 (v1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB + PARTITION BY KEY (v1) + PARTITIONS 2; +--error ER_GALERA_REPLICATION_NOT_SUPPORTED +CREATE OR REPLACE TABLE t3 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM + PARTITION BY KEY (v1) + PARTITIONS 2; +ALTER TABLE t1 ADD COLUMN v2 int; +--error ER_GALERA_REPLICATION_NOT_SUPPORTED +ALTER TABLE t2 ADD COLUMN v2 int; +INSERT INTO t1 VALUES (1,1),(2,2); +INSERT INTO t2 VALUES (1),(2); +--error ER_GALERA_REPLICATION_NOT_SUPPORTED +ALTER TABLE t1 ADD COLUMN v3 int, ENGINE=MyISAM; +--error ER_GALERA_REPLICATION_NOT_SUPPORTED +ALTER TABLE t2 ADD COLUMN v3 int, ENGINE=Aria; +UPDATE t1 SET v2 = v2 + 3; +UPDATE t2 SET v1 = v1 + 3; +CREATE INDEX xx1 ON t1(v2); +--error ER_GALERA_REPLICATION_NOT_SUPPORTED +CREATE INDEX xx2 ON t2(v2); +DROP INDEX xx1 ON t1; +--error ER_GALERA_REPLICATION_NOT_SUPPORTED +DROP INDEX xx2 on t2; +TRUNCATE TABLE t1; +--error ER_GALERA_REPLICATION_NOT_SUPPORTED +TRUNCATE TABLE t2; +# At the moment can't restrict rename +RENAME TABLE t1 TO t1_v2; +RENAME TABLE t2 TO t2_v2; +RENAME TABLE t2_v2 TO t2; +CREATE VIEW x1 AS SELECT * FROM t1_v2; +--error ER_GALERA_REPLICATION_NOT_SUPPORTED +CREATE VIEW x2 AS SELECT * FROM t2; +CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t1 + AFTER INSERT ON t1_v2 FOR EACH ROW + UPDATE t1_v2 SET t1_v2.v2 = t1_v2.v2+1; +--error ER_GALERA_REPLICATION_NOT_SUPPORTED +CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t2 + AFTER INSERT ON t2 FOR EACH ROW + UPDATE t2 SET t2.v1 = t2.v1+1; + +--connection node_2 +SHOW CREATE TABLE t1_v2; +SHOW CREATE TABLE t2; +SHOW CREATE VIEW x1; + +SELECT * FROM t1_v2; +SELECT * FROM t2; + +--connection node_1 +DROP VIEW x1; +DROP TRIGGER increment_before_t1; +DROP TABLE t1_v2; +# We allow dropping table +DROP TABLE t2; +SET GLOBAL wsrep_strict_ddl=OFF; + +CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM + PARTITION BY KEY (v1) + PARTITIONS 2; +--echo # wsrep-strict_ddl = ON +SET GLOBAL wsrep_strict_ddl = ON; +SELECT @@wsrep_strict_ddl; +ALTER TABLE t2 ENGINE=InnoDB; +DROP TABLE t2; + +SET GLOBAL wsrep_strict_ddl = DEFAULT; diff --git a/sql/sql_rename.cc b/sql/sql_rename.cc index 063645a4dce..5b4b572dba2 100644 --- a/sql/sql_rename.cc +++ b/sql/sql_rename.cc @@ -326,7 +326,7 @@ do_rename(THD *thd, TABLE_LIST *ren_table, const LEX_CSTRING *new_db, #ifdef WITH_WSREP if (WSREP(thd) && hton && hton != view_pseudo_hton && - !wsrep_should_replicate_ddl(thd, hton->db_type)) + !wsrep_should_replicate_ddl(thd, hton)) DBUG_RETURN(1); #endif diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 6f5c1839e1f..2b46fae9781 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2495,12 +2495,19 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, else { #ifdef WITH_WSREP - if (WSREP(thd) && hton && !wsrep_should_replicate_ddl(thd, hton->db_type)) + if (WSREP(thd) && hton) { - error= 1; - goto err; + handlerton *ht= hton; + // For partitioned tables resolve underlying handlerton + if (table->table && table->table->file->partition_ht()) + ht= table->table->file->partition_ht(); + if (!wsrep_should_replicate_ddl(thd, ht)) + { + error= 1; + goto err; + } } -#endif +#endif /* WITH_WSREP */ if (thd->locked_tables_mode == LTM_LOCK_TABLES || thd->locked_tables_mode == LTM_PRELOCKED_UNDER_LOCK_TABLES) @@ -10564,10 +10571,21 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, if (WSREP(thd) && table && (thd->lex->sql_command == SQLCOM_ALTER_TABLE || thd->lex->sql_command == SQLCOM_CREATE_INDEX || - thd->lex->sql_command == SQLCOM_DROP_INDEX) && - !wsrep_should_replicate_ddl(thd, table->s->db_type()->db_type)) - DBUG_RETURN(true); -#endif /* WITH_WSREP */ + thd->lex->sql_command == SQLCOM_DROP_INDEX)) + { + handlerton *ht= table->s->db_type(); + + // If alter used ENGINE= we use that + if (create_info->used_fields & HA_CREATE_USED_ENGINE) + ht= create_info->db_type; + // For partitioned tables resolve underlying handlerton + else if (table->file->partition_ht()) + ht= table->file->partition_ht(); + + if (!wsrep_should_replicate_ddl(thd, ht)) + DBUG_RETURN(true); + } +#endif DEBUG_SYNC(thd, "alter_table_after_open_tables"); diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index f85faff37ac..b8e4b741599 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -588,8 +588,12 @@ bool mysql_create_or_drop_trigger(THD *thd, TABLE_LIST *tables, bool create) table= tables->table; #ifdef WITH_WSREP + /* Resolve should we replicate creation of the trigger. + It should be replicated if storage engine(s) associated + to trigger are replicated by Galera. + */ if (WSREP(thd) && - !wsrep_should_replicate_ddl(thd, table->s->db_type()->db_type)) + !wsrep_should_replicate_ddl_iterate(thd, tables)) goto end; #endif diff --git a/sql/sql_truncate.cc b/sql/sql_truncate.cc index cb0849400ee..e86325d31a5 100644 --- a/sql/sql_truncate.cc +++ b/sql/sql_truncate.cc @@ -304,8 +304,15 @@ bool Sql_cmd_truncate_table::lock_table(THD *thd, TABLE_LIST *table_ref, versioned= table->versioned(); hton= table->file->ht; #ifdef WITH_WSREP + /* Resolve should we replicate truncate. It should + be replicated if storage engine(s) associated + are replicated by Galera. If this is partitioned + table we need to find out default partition + handlerton. + */ if (WSREP(thd) && - !wsrep_should_replicate_ddl(thd, hton->db_type)) + !wsrep_should_replicate_ddl(thd, table->file->partition_ht() ? + table->file->partition_ht() : hton)) DBUG_RETURN(TRUE); #endif @@ -327,12 +334,22 @@ bool Sql_cmd_truncate_table::lock_table(THD *thd, TABLE_LIST *table_ref, sequence= share->table_type == TABLE_TYPE_SEQUENCE; hton= share->db_type(); #ifdef WITH_WSREP - if (WSREP(thd) && - hton != view_pseudo_hton && - !wsrep_should_replicate_ddl(thd, hton->db_type)) + if (WSREP(thd) && hton != view_pseudo_hton) { - tdc_release_share(share); - DBUG_RETURN(TRUE); + /* Resolve should we replicate truncate. It should + be replicated if storage engine(s) associated + are replicated by Galera. If this is partitioned + table we need to find out default partition + handlerton. + */ + const handlerton *ht= share->default_part_plugin ? + plugin_hton(share->default_part_plugin) : hton; + + if (ht && !wsrep_should_replicate_ddl(thd, ht)) + { + tdc_release_share(share); + DBUG_RETURN(TRUE); + } } #endif diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 660e1b35cf9..89d0704f6df 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -443,8 +443,6 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; - WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); - /* ignore lock specs for CREATE statement */ @@ -462,13 +460,20 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, } #ifdef WITH_WSREP - if(!wsrep_should_replicate_ddl_iterate(thd, static_cast(tables))) + /* Resolve should we replicate creation of the view. + It should be replicated if storage engine(s) associated + to view are replicated by Galera. + */ + if (WSREP(thd) && + !wsrep_should_replicate_ddl_iterate(thd, tables)) { res= TRUE; goto err_no_relink; } #endif + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL); + view= lex->unlink_first_table(&link_to_local); if (check_db_dir_existence(view->db.str)) diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index ae52e319e3f..c905883e2fb 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -1,5 +1,5 @@ -/* Copyright (c) 2008, 2023 Codership Oy - Copyright (c) 2020, 2022, MariaDB +/* Copyright (c) 2008, 2024, Codership Oy + Copyright (c) 2020, 2024, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -2073,27 +2073,24 @@ static int wsrep_drop_table_query(THD* thd, uchar** buf, size_t* buf_len) /* Forward declarations. */ int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len); -bool wsrep_should_replicate_ddl_iterate(THD* thd, const TABLE_LIST* table_list) -{ - if (WSREP(thd)) - { - for (const TABLE_LIST* it= table_list; it; it= it->next_global) - { - if (it->table && - !wsrep_should_replicate_ddl(thd, it->table->s->db_type()->db_type)) - return false; - } - } - return true; -} +/*! Should DDL be replicated by Galera + * + * @param thd thread handle + * @param hton real storage engine handlerton + * + * @retval true if we should replicate DDL, false if not */ -bool wsrep_should_replicate_ddl(THD* thd, - const enum legacy_db_type db_type) +bool wsrep_should_replicate_ddl(THD* thd, const handlerton *hton) { if (!wsrep_strict_ddl) return true; + + if (!hton) + return true; - switch (db_type) + DBUG_ASSERT(hton != nullptr); + + switch (hton->db_type) { case DB_TYPE_INNODB: return true; @@ -2104,6 +2101,11 @@ bool wsrep_should_replicate_ddl(THD* thd, else WSREP_DEBUG("wsrep OSU failed for %s", wsrep_thd_query(thd)); break; + case DB_TYPE_PARTITION_DB: + /* In most cases this means we could not find out + table->file->partition_ht() */ + return true; + break; case DB_TYPE_ARIA: /* if (wsrep_replicate_aria) */ /* fallthrough */ @@ -2115,10 +2117,33 @@ bool wsrep_should_replicate_ddl(THD* thd, /* STRICT, treat as error */ my_error(ER_GALERA_REPLICATION_NOT_SUPPORTED, MYF(0)); push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_ILLEGAL_HA, - "WSREP: wsrep_strict_ddl=true and storage engine does not support Galera replication."); + ER_ILLEGAL_HA, + "WSREP: wsrep_strict_dll enabled. " + "Storage engine %s not supported.", + ha_resolve_storage_engine_name(hton)); return false; } + +bool wsrep_should_replicate_ddl_iterate(THD* thd, const TABLE_LIST* table_list) +{ + for (const TABLE_LIST* it= table_list; it; it= it->next_global) + { + if (it->table) + { + /* If this is partitioned table we need to find out + implementing storage engine handlerton. + */ + const handlerton *ht= it->table->file->partition_ht() ? + it->table->file->partition_ht() : + it->table->s->db_type(); + + if (!wsrep_should_replicate_ddl(thd, ht)) + return false; + } + } + return true; +} + /* Decide if statement should run in TOI. @@ -2147,7 +2172,7 @@ bool wsrep_can_run_in_toi(THD *thd, const char *db, const char *table, { return false; } - if (!wsrep_should_replicate_ddl(thd, create_info->db_type->db_type)) + if (!wsrep_should_replicate_ddl(thd, create_info->db_type)) { return false; } @@ -2227,16 +2252,11 @@ bool wsrep_can_run_in_toi(THD *thd, const char *db, const char *table, { if (create_info) { - enum legacy_db_type db_type; + const handlerton *hton= create_info->db_type; - if (create_info->db_type) - db_type= create_info->db_type->db_type; - else - { - const handlerton *hton= ha_default_handlerton(thd); - db_type= hton->db_type; - } - if (!wsrep_should_replicate_ddl(thd, db_type)) + if (!hton) + hton= ha_default_handlerton(thd); + if (!wsrep_should_replicate_ddl(thd, hton)) return false; } } diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 02d1d7fd248..1c8220fd035 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -339,7 +339,7 @@ int wsrep_to_isolation_begin(THD *thd, const char *db_, const char *table_, const wsrep::key_array *fk_tables= nullptr, const HA_CREATE_INFO* create_info= nullptr); -bool wsrep_should_replicate_ddl(THD* thd, const enum legacy_db_type db_type); +bool wsrep_should_replicate_ddl(THD* thd, const handlerton *hton); bool wsrep_should_replicate_ddl_iterate(THD* thd, const TABLE_LIST* table_list); void wsrep_to_isolation_end(THD *thd); From 0784dd32b1ed7348f8d6680b2f4f6e06a6e71256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 13 Apr 2023 13:45:00 +0300 Subject: [PATCH 08/47] MDEV-29775 : Assertion `0' failed in void Protocol::end_statement() when adding data to the MyISAM table after setting wsrep_mode=replicate_myisam If wsrep_mode=BINLOG_ROW_FORMAT_ONLY wsrep_forced_binlog_format can be DEFAULT (UNSPECIFIED) or ROW. Finally, if wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] or wsrep_replicate_myisam=ON we allow wsrep_forced_binlog_format to be [DEFAULT|ROW]. Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera/r/mdev-29775.result | 68 ++++++++++++++++----- mysql-test/suite/galera/t/mdev-29775.test | 66 +++++++++++++++----- sql/wsrep_var.cc | 37 ++++++++++- 3 files changed, 139 insertions(+), 32 deletions(-) diff --git a/mysql-test/suite/galera/r/mdev-29775.result b/mysql-test/suite/galera/r/mdev-29775.result index dc38f18b433..3dba185d5b0 100644 --- a/mysql-test/suite/galera/r/mdev-29775.result +++ b/mysql-test/suite/galera/r/mdev-29775.result @@ -1,6 +1,6 @@ connection node_2; connection node_1; -SET GLOBAL wsrep_replicate_myisam=ON; +SET GLOBAL wsrep_mode=REPLICATE_MYISAM; CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM; INSERT INTO t VALUES(); SELECT * FROM t; @@ -12,7 +12,7 @@ f0 NULL DROP TABLE t; connection node_1; -SET GLOBAL wsrep_replicate_myisam=ON; +SET GLOBAL wsrep_mode=REPLICATE_MYISAM; SET GLOBAL wsrep_forced_binlog_format=ROW; CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM; INSERT INTO t VALUES(); @@ -25,20 +25,60 @@ f0 NULL DROP TABLE t; connection node_1; +SET GLOBAL wsrep_mode=REPLICATE_ARIA; +CREATE TABLE t (f0 CHAR(0)) ENGINE=Aria; +INSERT INTO t VALUES(); +SELECT * FROM t; +f0 +NULL +connection node_2; +SELECT * FROM t; +f0 +NULL +DROP TABLE t; +connection node_1; +SET GLOBAL wsrep_mode=REPLICATE_ARIA; +SET GLOBAL wsrep_forced_binlog_format=ROW; +CREATE TABLE t (f0 CHAR(0)) ENGINE=Aria; +INSERT INTO t VALUES(); +SELECT * FROM t; +f0 +NULL +connection node_2; +SELECT * FROM t; +f0 +NULL +DROP TABLE t; +connection node_1; +SET GLOBAL wsrep_mode=REPLICATE_MYISAM; +SET GLOBAL wsrep_forced_binlog_format=MIXED; +ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; +ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] +SET GLOBAL wsrep_mode=REPLICATE_ARIA; +SET GLOBAL wsrep_forced_binlog_format=MIXED; +ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; +ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] +SET GLOBAL wsrep_mode=DEFAULT; +SET GLOBAL wsrep_forced_binlog_format=MIXED; +SET GLOBAL wsrep_mode = REPLICATE_MYISAM; +ERROR HY000: wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] can't be enabled if wsrep_forced_binlog != [NONE|ROW] +SET GLOBAL wsrep_mode = REPLICATE_ARIA; +ERROR HY000: wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] can't be enabled if wsrep_forced_binlog != [NONE|ROW] +SET GLOBAL wsrep_mode=DEFAULT; +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; +SET GLOBAL wsrep_mode = REPLICATE_MYISAM; +ERROR HY000: wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] can't be enabled if wsrep_forced_binlog != [NONE|ROW] +SET GLOBAL wsrep_mode = REPLICATE_ARIA; +ERROR HY000: wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] can't be enabled if wsrep_forced_binlog != [NONE|ROW] SET GLOBAL wsrep_forced_binlog_format=DEFAULT; -SET GLOBAL wsrep_replicate_myisam=OFF; +SET GLOBAL wsrep_mode=DEFAULT; SET GLOBAL wsrep_forced_binlog_format=MIXED; SET GLOBAL wsrep_replicate_myisam=ON; -ERROR HY000: wsrep_replicate_myisam=ON can't be enabled if wsrep_forced_binlog != [NONE|ROW] +ERROR HY000: wsrep_mode=REPLICATE_MYISAM can't be enabled if wsrep_forced_binlog != [NONE|ROW] SET GLOBAL wsrep_forced_binlog_format=STATEMENT; SET GLOBAL wsrep_replicate_myisam=ON; -ERROR HY000: wsrep_replicate_myisam=ON can't be enabled if wsrep_forced_binlog != [NONE|ROW] -SET GLOBAL wsrep_forced_binlog_format=NONE; -SET GLOBAL wsrep_replicate_myisam=OFF; -SET GLOBAL wsrep_replicate_myisam=ON; -SET GLOBAL wsrep_forced_binlog_format=MIXED; -ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_replicate_myisam=ON -SET GLOBAL wsrep_forced_binlog_format=STATEMENT; -ERROR HY000: wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set if wsrep_replicate_myisam=ON -SET GLOBAL wsrep_forced_binlog_format=NONE; -SET GLOBAL wsrep_replicate_myisam=OFF; +ERROR HY000: wsrep_mode=REPLICATE_MYISAM can't be enabled if wsrep_forced_binlog != [NONE|ROW] +SET GLOBAL wsrep_forced_binlog_format=DEFAULT; +SET GLOBAL wsrep_mode=DEFAULT; diff --git a/mysql-test/suite/galera/t/mdev-29775.test b/mysql-test/suite/galera/t/mdev-29775.test index 5c6725d780d..ec227a8972c 100644 --- a/mysql-test/suite/galera/t/mdev-29775.test +++ b/mysql-test/suite/galera/t/mdev-29775.test @@ -1,9 +1,10 @@ --source include/galera_cluster.inc +--source include/have_aria.inc # # MDEV-29775 : Assertion `0' failed in void Protocol::end_statement() when adding data to the MyISAM table after setting wsrep_mode=replicate_myisam # -SET GLOBAL wsrep_replicate_myisam=ON; +SET GLOBAL wsrep_mode=REPLICATE_MYISAM; CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM; INSERT INTO t VALUES(); SELECT * FROM t; @@ -12,7 +13,7 @@ SELECT * FROM t; DROP TABLE t; --connection node_1 -SET GLOBAL wsrep_replicate_myisam=ON; +SET GLOBAL wsrep_mode=REPLICATE_MYISAM; SET GLOBAL wsrep_forced_binlog_format=ROW; CREATE TABLE t (f0 CHAR(0)) ENGINE=MyISAM; INSERT INTO t VALUES(); @@ -22,9 +23,53 @@ SELECT * FROM t; DROP TABLE t; --connection node_1 +SET GLOBAL wsrep_mode=REPLICATE_ARIA; +CREATE TABLE t (f0 CHAR(0)) ENGINE=Aria; +INSERT INTO t VALUES(); +SELECT * FROM t; +--connection node_2 +SELECT * FROM t; +DROP TABLE t; + +--connection node_1 +SET GLOBAL wsrep_mode=REPLICATE_ARIA; +SET GLOBAL wsrep_forced_binlog_format=ROW; +CREATE TABLE t (f0 CHAR(0)) ENGINE=Aria; +INSERT INTO t VALUES(); +SELECT * FROM t; +--connection node_2 +SELECT * FROM t; +DROP TABLE t; + +--connection node_1 +SET GLOBAL wsrep_mode=REPLICATE_MYISAM; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_forced_binlog_format=MIXED; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; + +SET GLOBAL wsrep_mode=REPLICATE_ARIA; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_forced_binlog_format=MIXED; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; + +SET GLOBAL wsrep_mode=DEFAULT; +SET GLOBAL wsrep_forced_binlog_format=MIXED; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_mode = REPLICATE_MYISAM; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_mode = REPLICATE_ARIA; + +SET GLOBAL wsrep_mode=DEFAULT; +SET GLOBAL wsrep_forced_binlog_format=STATEMENT; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_mode = REPLICATE_MYISAM; +--error ER_WRONG_ARGUMENTS +SET GLOBAL wsrep_mode = REPLICATE_ARIA; + SET GLOBAL wsrep_forced_binlog_format=DEFAULT; -SET GLOBAL wsrep_replicate_myisam=OFF; - +SET GLOBAL wsrep_mode=DEFAULT; SET GLOBAL wsrep_forced_binlog_format=MIXED; --error ER_WRONG_ARGUMENTS SET GLOBAL wsrep_replicate_myisam=ON; @@ -32,14 +77,5 @@ SET GLOBAL wsrep_forced_binlog_format=STATEMENT; --error ER_WRONG_ARGUMENTS SET GLOBAL wsrep_replicate_myisam=ON; -SET GLOBAL wsrep_forced_binlog_format=NONE; -SET GLOBAL wsrep_replicate_myisam=OFF; - -SET GLOBAL wsrep_replicate_myisam=ON; ---error ER_WRONG_ARGUMENTS -SET GLOBAL wsrep_forced_binlog_format=MIXED; ---error ER_WRONG_ARGUMENTS -SET GLOBAL wsrep_forced_binlog_format=STATEMENT; - -SET GLOBAL wsrep_forced_binlog_format=NONE; -SET GLOBAL wsrep_replicate_myisam=OFF; +SET GLOBAL wsrep_forced_binlog_format=DEFAULT; +SET GLOBAL wsrep_mode=DEFAULT; diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 1c5e2311221..ce679d245ba 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -984,6 +984,28 @@ bool wsrep_max_ws_size_update(sys_var *self, THD *thd, enum_var_type) bool wsrep_mode_check(sys_var *self, THD* thd, set_var* var) { + ulonglong new_wsrep_mode= var->save_result.ulonglong_value; + if (new_wsrep_mode && !WSREP_ON) + { + my_message(ER_WRONG_ARGUMENTS, "wsrep_mode can't be set " + "if wsrep_on=OFF", MYF(0)); + return true; + } + ulonglong old_wsrep_mode= wsrep_mode; + wsrep_mode= new_wsrep_mode; + if (wsrep_check_mode(WSREP_MODE_REPLICATE_MYISAM) || + wsrep_check_mode(WSREP_MODE_REPLICATE_ARIA)) + { + if (!(wsrep_forced_binlog_format == BINLOG_FORMAT_UNSPEC || + wsrep_forced_binlog_format == BINLOG_FORMAT_ROW)) + { + my_message(ER_WRONG_ARGUMENTS, "wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA] " + "can't be enabled if wsrep_forced_binlog != [NONE|ROW]", MYF(0)); + wsrep_mode= old_wsrep_mode; + return true; + } + } + wsrep_mode= old_wsrep_mode; return false; } @@ -1157,7 +1179,7 @@ bool wsrep_replicate_myisam_check(sys_var *self, THD* thd, set_var* var) !(wsrep_forced_binlog_format == BINLOG_FORMAT_UNSPEC || wsrep_forced_binlog_format == BINLOG_FORMAT_ROW)) { - my_message(ER_WRONG_ARGUMENTS, "wsrep_replicate_myisam=ON can't be enabled " + my_message(ER_WRONG_ARGUMENTS, "wsrep_mode=REPLICATE_MYISAM can't be enabled " "if wsrep_forced_binlog != [NONE|ROW]", MYF(0)); return true; } @@ -1167,13 +1189,22 @@ bool wsrep_replicate_myisam_check(sys_var *self, THD* thd, set_var* var) bool wsrep_forced_binlog_format_check(sys_var *self, THD* thd, set_var* var) { ulonglong new_forced_binlog_format= var->save_result.ulonglong_value; + if (!(new_forced_binlog_format == BINLOG_FORMAT_UNSPEC || new_forced_binlog_format == BINLOG_FORMAT_ROW)) { - if (wsrep_replicate_myisam) + if (wsrep_check_mode(WSREP_MODE_BINLOG_ROW_FORMAT_ONLY)) { my_message(ER_WRONG_ARGUMENTS, "wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set " - "if wsrep_replicate_myisam=ON", MYF(0)); + "if wsrep_mode=BINLOG_ROW_FORMAT_ONLY", MYF(0)); + return true; + } + + if (wsrep_check_mode(WSREP_MODE_REPLICATE_MYISAM) || + wsrep_check_mode(WSREP_MODE_REPLICATE_ARIA)) + { + my_message(ER_WRONG_ARGUMENTS, "wsrep_forced_binlog_format=[MIXED|STATEMENT] can't be set " + "if wsrep_mode=[REPLICATE_MYISAM|REPLICATE_ARIA]", MYF(0)); return true; } } From b3925982a064d79d6f2cf57f532f143d36cfc004 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 2 May 2023 04:57:30 +0200 Subject: [PATCH 09/47] MDEV-29755 post-merge for 10.6+ 1) remove unnecessary restriction on changing mode; 2) adding lost wsrep_replicate_myisam_basic test. --- .../r/wsrep_replicate_myisam_basic.result | 15 +++++++++++++++ .../t/wsrep_replicate_myisam_basic.test | 19 +++++++++++++++++++ sql/wsrep_var.cc | 6 ------ 3 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 mysql-test/suite/sys_vars/r/wsrep_replicate_myisam_basic.result create mode 100644 mysql-test/suite/sys_vars/t/wsrep_replicate_myisam_basic.test diff --git a/mysql-test/suite/sys_vars/r/wsrep_replicate_myisam_basic.result b/mysql-test/suite/sys_vars/r/wsrep_replicate_myisam_basic.result new file mode 100644 index 00000000000..c8effa8c0f3 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/wsrep_replicate_myisam_basic.result @@ -0,0 +1,15 @@ +# +# wsrep_replicate_myisam +# +# save the initial value +SET @wsrep_mode_saved = @@global.wsrep_mode; + +# scope and valid values +SET @@global.wsrep_mode=REPLICATE_MYISAM; +SELECT @@global.wsrep_mode; +@@global.wsrep_mode +REPLICATE_MYISAM + +# restore the initial value +SET @@global.wsrep_mode = @wsrep_mode_saved; +# End of test diff --git a/mysql-test/suite/sys_vars/t/wsrep_replicate_myisam_basic.test b/mysql-test/suite/sys_vars/t/wsrep_replicate_myisam_basic.test new file mode 100644 index 00000000000..49ccdec047a --- /dev/null +++ b/mysql-test/suite/sys_vars/t/wsrep_replicate_myisam_basic.test @@ -0,0 +1,19 @@ +--source include/have_wsrep.inc + +--echo # +--echo # wsrep_replicate_myisam +--echo # + +--echo # save the initial value +SET @wsrep_mode_saved = @@global.wsrep_mode; + +--echo +--echo # scope and valid values +SET @@global.wsrep_mode=REPLICATE_MYISAM; +SELECT @@global.wsrep_mode; + +--echo +--echo # restore the initial value +SET @@global.wsrep_mode = @wsrep_mode_saved; + +--echo # End of test diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index ce679d245ba..3d381afa704 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -985,12 +985,6 @@ bool wsrep_max_ws_size_update(sys_var *self, THD *thd, enum_var_type) bool wsrep_mode_check(sys_var *self, THD* thd, set_var* var) { ulonglong new_wsrep_mode= var->save_result.ulonglong_value; - if (new_wsrep_mode && !WSREP_ON) - { - my_message(ER_WRONG_ARGUMENTS, "wsrep_mode can't be set " - "if wsrep_on=OFF", MYF(0)); - return true; - } ulonglong old_wsrep_mode= wsrep_mode; wsrep_mode= new_wsrep_mode; if (wsrep_check_mode(WSREP_MODE_REPLICATE_MYISAM) || From 9389428380afcef4f0a148e4e2e3327c1584d265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 17 Jul 2023 17:29:20 +0300 Subject: [PATCH 10/47] MDEV-27861: Creating partitioned tables should not be allowed with wsrep_osu_method=TOI and wsrep_strict_ddl=ON Problem was incorrect handling of partitioned tables, because db_type == DB_TYPE_PARTITION_DB wsrep_should_replicate_ddl incorrectly marked DDL as not replicatable. However, in partitioned tables we should check implementing storage engine from table->file->partition_ht() if available because if partition handler is InnoDB all DDL should be allowed even with wsrep_strict_ddl. For other storage engines DDL should not be allowed and error should be issued. This is 10.6 version of the fix. Signed-off-by: Julius Goryavsky --- .../galera/r/galera_partitioned_tables.result | 63 +++++++++++-------- .../r/wsrep_mode_strict_replication.result | 12 ++-- .../galera/t/galera_partitioned_tables.test | 26 ++++---- sql/wsrep_mysqld.h | 1 - 4 files changed, 56 insertions(+), 46 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_partitioned_tables.result b/mysql-test/suite/galera/r/galera_partitioned_tables.result index 4b5c9e74c54..3f24cae1495 100644 --- a/mysql-test/suite/galera/r/galera_partitioned_tables.result +++ b/mysql-test/suite/galera/r/galera_partitioned_tables.result @@ -1,10 +1,11 @@ connection node_2; connection node_1; -# wsrep-strict_ddl = OFF -SET GLOBAL wsrep_strict_ddl = OFF; -SELECT @@wsrep_strict_ddl; -@@wsrep_strict_ddl -0 +call mtr.add_suppression("WSREP: wsrep_mode = STRICT_REPLICATION enabled. Storage engine partition for table.*"); +# wsrep-mode= DEFAULT +SET GLOBAL wsrep_mode = ""; +SELECT @@wsrep_mode; +@@wsrep_mode + CREATE OR REPLACE TABLE t1 (v1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB PARTITION BY KEY (v1) PARTITIONS 2; @@ -73,55 +74,63 @@ DROP TRIGGER increment_before_t1; DROP TRIGGER increment_before_t2; DROP TABLE t1_v2; DROP TABLE t2_v2; -SET GLOBAL wsrep_strict_ddl = OFF; +SET GLOBAL wsrep_mode = ""; CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM PARTITION BY KEY (v1) PARTITIONS 2; -# wsrep-strict_ddl = ON -SET GLOBAL wsrep_strict_ddl = ON; -SELECT @@wsrep_strict_ddl; -@@wsrep_strict_ddl -1 +# wsrep-mode= STRICT_REPLICATION +SET GLOBAL wsrep_mode = "STRICT_REPLICATION"; +SELECT @@wsrep_mode; +@@wsrep_mode +STRICT_REPLICATION CREATE OR REPLACE TABLE t1 (v1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB PARTITION BY KEY (v1) PARTITIONS 2; CREATE OR REPLACE TABLE t3 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM PARTITION BY KEY (v1) PARTITIONS 2; -ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ERROR HY000: Galera replication not supported ALTER TABLE t1 ADD COLUMN v2 int; ALTER TABLE t2 ADD COLUMN v2 int; -ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ERROR HY000: Galera replication not supported INSERT INTO t1 VALUES (1,1),(2,2); +Warnings: +Warning 1290 WSREP: wsrep_mode = STRICT_REPLICATION enabled. Storage engine partition for table 'test'.'t1' is not supported in Galera INSERT INTO t2 VALUES (1),(2); +Warnings: +Warning 1290 WSREP: wsrep_mode = STRICT_REPLICATION enabled. Storage engine partition for table 'test'.'t2' is not supported in Galera ALTER TABLE t1 ADD COLUMN v3 int, ENGINE=MyISAM; -ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ERROR HY000: Galera replication not supported ALTER TABLE t2 ADD COLUMN v3 int, ENGINE=Aria; -ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ERROR HY000: Galera replication not supported UPDATE t1 SET v2 = v2 + 3; +Warnings: +Warning 1290 WSREP: wsrep_mode = STRICT_REPLICATION enabled. Storage engine partition for table 'test'.'t1' is not supported in Galera UPDATE t2 SET v1 = v1 + 3; +Warnings: +Warning 1290 WSREP: wsrep_mode = STRICT_REPLICATION enabled. Storage engine partition for table 'test'.'t2' is not supported in Galera CREATE INDEX xx1 ON t1(v2); CREATE INDEX xx2 ON t2(v2); -ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ERROR HY000: Galera replication not supported DROP INDEX xx1 ON t1; DROP INDEX xx2 on t2; -ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ERROR HY000: Galera replication not supported TRUNCATE TABLE t1; TRUNCATE TABLE t2; -ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ERROR HY000: Galera replication not supported RENAME TABLE t1 TO t1_v2; RENAME TABLE t2 TO t2_v2; RENAME TABLE t2_v2 TO t2; CREATE VIEW x1 AS SELECT * FROM t1_v2; CREATE VIEW x2 AS SELECT * FROM t2; -ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ERROR HY000: Galera replication not supported CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t1 AFTER INSERT ON t1_v2 FOR EACH ROW UPDATE t1_v2 SET t1_v2.v2 = t1_v2.v2+1; CREATE DEFINER=`root`@`localhost` TRIGGER increment_before_t2 AFTER INSERT ON t2 FOR EACH ROW UPDATE t2 SET t2.v1 = t2.v1+1; -ERROR HY000: DDL-statement is forbidden as table storage engine does not support Galera replication +ERROR HY000: Galera replication not supported connection node_2; SHOW CREATE TABLE t1_v2; Table Create Table @@ -153,15 +162,15 @@ DROP VIEW x1; DROP TRIGGER increment_before_t1; DROP TABLE t1_v2; DROP TABLE t2; -SET GLOBAL wsrep_strict_ddl=OFF; +SET GLOBAL wsrep_mode = ""; CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM PARTITION BY KEY (v1) PARTITIONS 2; -# wsrep-strict_ddl = ON -SET GLOBAL wsrep_strict_ddl = ON; -SELECT @@wsrep_strict_ddl; -@@wsrep_strict_ddl -1 +# wsrep-mode= STRICT_REPLICATION +SET GLOBAL wsrep_mode = "STRICT_REPLICATION"; +SELECT @@wsrep_mode; +@@wsrep_mode +STRICT_REPLICATION ALTER TABLE t2 ENGINE=InnoDB; DROP TABLE t2; -SET GLOBAL wsrep_strict_ddl = DEFAULT; +SET GLOBAL wsrep_mode = DEFAULT; diff --git a/mysql-test/suite/galera/r/wsrep_mode_strict_replication.result b/mysql-test/suite/galera/r/wsrep_mode_strict_replication.result index 66962be3391..678f337d79e 100644 --- a/mysql-test/suite/galera/r/wsrep_mode_strict_replication.result +++ b/mysql-test/suite/galera/r/wsrep_mode_strict_replication.result @@ -19,8 +19,8 @@ CREATE TABLE t1(a int) engine=Aria; ERROR HY000: Galera replication not supported SHOW WARNINGS; Level Code Message -Error 4165 DDL-statement is forbidden as table storage engine does not support Galera replication -Warning 1031 WSREP: wsrep_strict_dll enabled. Storage engine Aria not supported. +Error 4165 Galera replication not supported +Warning 1031 WSREP: wsrep_mode = STRICT_REPLICATION enabled. Storage engine Aria not supported. connection node_2; SHOW CREATE TABLE t1; ERROR 42S02: Table 'test.t1' doesn't exist @@ -30,10 +30,10 @@ ALTER TABLE t2 engine=MyISAM; ERROR HY000: Galera replication not supported SHOW WARNINGS; Level Code Message -Error 4165 DDL-statement is forbidden as table storage engine does not support Galera replication -Warning 1031 WSREP: wsrep_strict_dll enabled. Storage engine MyISAM not supported. -Error 4165 DDL-statement is forbidden as table storage engine does not support Galera replication -Warning 1031 WSREP: wsrep_strict_dll enabled. Storage engine MyISAM not supported. +Error 4165 Galera replication not supported +Warning 1031 WSREP: wsrep_mode = STRICT_REPLICATION enabled. Storage engine MyISAM not supported. +Error 4165 Galera replication not supported +Warning 1031 WSREP: wsrep_mode = STRICT_REPLICATION enabled. Storage engine MyISAM not supported. SHOW CREATE TABLE t2; Table Create Table t2 CREATE TABLE `t2` ( diff --git a/mysql-test/suite/galera/t/galera_partitioned_tables.test b/mysql-test/suite/galera/t/galera_partitioned_tables.test index d65ee6c1a49..f29adcf1501 100644 --- a/mysql-test/suite/galera/t/galera_partitioned_tables.test +++ b/mysql-test/suite/galera/t/galera_partitioned_tables.test @@ -3,9 +3,11 @@ --source include/have_innodb.inc --source include/have_aria.inc ---echo # wsrep-strict_ddl = OFF -SET GLOBAL wsrep_strict_ddl = OFF; -SELECT @@wsrep_strict_ddl; +call mtr.add_suppression("WSREP: wsrep_mode = STRICT_REPLICATION enabled. Storage engine partition for table.*"); + +--echo # wsrep-mode= DEFAULT +SET GLOBAL wsrep_mode = ""; +SELECT @@wsrep_mode; CREATE OR REPLACE TABLE t1 (v1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB PARTITION BY KEY (v1) PARTITIONS 2; @@ -54,13 +56,13 @@ DROP TRIGGER increment_before_t2; DROP TABLE t1_v2; DROP TABLE t2_v2; -SET GLOBAL wsrep_strict_ddl = OFF; +SET GLOBAL wsrep_mode = ""; CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM PARTITION BY KEY (v1) PARTITIONS 2; ---echo # wsrep-strict_ddl = ON -SET GLOBAL wsrep_strict_ddl = ON; -SELECT @@wsrep_strict_ddl; +--echo # wsrep-mode= STRICT_REPLICATION +SET GLOBAL wsrep_mode = "STRICT_REPLICATION"; +SELECT @@wsrep_mode; CREATE OR REPLACE TABLE t1 (v1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB PARTITION BY KEY (v1) PARTITIONS 2; @@ -117,15 +119,15 @@ DROP TRIGGER increment_before_t1; DROP TABLE t1_v2; # We allow dropping table DROP TABLE t2; -SET GLOBAL wsrep_strict_ddl=OFF; +SET GLOBAL wsrep_mode = ""; CREATE OR REPLACE TABLE t2 (v1 INT NOT NULL PRIMARY KEY) ENGINE=MyISAM PARTITION BY KEY (v1) PARTITIONS 2; ---echo # wsrep-strict_ddl = ON -SET GLOBAL wsrep_strict_ddl = ON; -SELECT @@wsrep_strict_ddl; +--echo # wsrep-mode= STRICT_REPLICATION +SET GLOBAL wsrep_mode = "STRICT_REPLICATION"; +SELECT @@wsrep_mode; ALTER TABLE t2 ENGINE=InnoDB; DROP TABLE t2; -SET GLOBAL wsrep_strict_ddl = DEFAULT; +SET GLOBAL wsrep_mode = DEFAULT; diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index 1c3ac610cf2..02df7bf5ed5 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -614,7 +614,6 @@ bool wsrep_table_list_has_non_temp_tables(THD *thd, TABLE_LIST *tables); #define wsrep_thr_deinit() do {} while(0) #define wsrep_init_globals() do {} while(0) #define wsrep_create_appliers(X) do {} while(0) -#define wsrep_should_replicate_ddl(X,Y) (1) #define wsrep_cluster_address_exists() (false) #define WSREP_MYSQL_DB (0) #define WSREP_TO_ISOLATION_BEGIN(db_, table_, table_list_) do { } while(0) From a1d2dfa6567d7673381e4210252bfafd33608d1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 29 Jan 2025 13:04:29 +0200 Subject: [PATCH 11/47] MDEV-35941 : galera_bf_abort_lock_table fails with wait for metadata lock Test fixes only. Signed-off-by: Julius Goryavsky --- .../wait_condition_with_debug_and_kill.inc | 35 +++++++++++++++++++ .../r/galera_bf_abort_lock_table.result | 1 + .../galera/t/galera_bf_abort_lock_table.cnf | 5 +++ .../galera/t/galera_bf_abort_lock_table.test | 8 +++-- mysql-test/suite/galera/t/mysql-wsrep#198.cnf | 7 ++-- .../suite/galera/t/mysql-wsrep#198.test | 3 +- 6 files changed, 51 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/galera/include/wait_condition_with_debug_and_kill.inc create mode 100644 mysql-test/suite/galera/t/galera_bf_abort_lock_table.cnf diff --git a/mysql-test/suite/galera/include/wait_condition_with_debug_and_kill.inc b/mysql-test/suite/galera/include/wait_condition_with_debug_and_kill.inc new file mode 100644 index 00000000000..184f75eb57c --- /dev/null +++ b/mysql-test/suite/galera/include/wait_condition_with_debug_and_kill.inc @@ -0,0 +1,35 @@ +# include/wait_condition_with_debug_and_kill.inc +# +# SUMMARY +# +# Waits until the passed statement returns true, or the operation +# times out. If the operation times out, the additional error +# statement will be executed and server is killed. +# +# USAGE +# +# let $wait_condition= +# SELECT c = 3 FROM t; +# let $wait_condition_on_error_output= select count(*) from t; +# [let $explicit_default_wait_timeout= N] # to override the default reset +# --source include/wait_condition_with_debug_and_kill.inc +# +# OR +# +# let $wait_timeout= 60; # Override default 30 seconds with 60. +# let $wait_condition= +# SELECT c = 3 FROM t; +# let $wait_condition_on_error_output= select count(*) from t; +# --source include/wait_condition_with_debug_and_kill.inc +# --echo Executed the test condition $wait_condition_reps times +# +# +# EXAMPLE +# events_bugs.test, events_time_zone.test +# + +--source include/wait_condition_with_debug.inc +if (!$success) +{ + --source include/kill_galera.inc +} diff --git a/mysql-test/suite/galera/r/galera_bf_abort_lock_table.result b/mysql-test/suite/galera/r/galera_bf_abort_lock_table.result index e7882e43b6a..6b6c45cbd29 100644 --- a/mysql-test/suite/galera/r/galera_bf_abort_lock_table.result +++ b/mysql-test/suite/galera/r/galera_bf_abort_lock_table.result @@ -7,6 +7,7 @@ LOCK TABLE t1 WRITE; connection node_1; INSERT INTO t1 VALUES (2); connection node_2; +SET SESSION wsrep_sync_wait = 0; UNLOCK TABLES; COMMIT; SELECT COUNT(*) = 1 FROM t1; diff --git a/mysql-test/suite/galera/t/galera_bf_abort_lock_table.cnf b/mysql-test/suite/galera/t/galera_bf_abort_lock_table.cnf new file mode 100644 index 00000000000..033e6f8b99a --- /dev/null +++ b/mysql-test/suite/galera/t/galera_bf_abort_lock_table.cnf @@ -0,0 +1,5 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +wsrep-debug=1 +loose-galera-bf-abort-lock-table=1 diff --git a/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test b/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test index 4582f3f972d..06009712c7b 100644 --- a/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test +++ b/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test @@ -1,5 +1,6 @@ --source include/galera_cluster.inc --source include/have_innodb.inc +--source include/force_restart.inc # # Test that a local LOCK TABLE will NOT be broken by an incoming remote transaction against that table @@ -16,13 +17,16 @@ LOCK TABLE t1 WRITE; INSERT INTO t1 VALUES (2); --connection node_2 +SET SESSION wsrep_sync_wait = 0; --let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'Waiting for table metadata lock' ---source include/wait_condition.inc +--let $wait_condition_on_error_output = SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST +--source include/wait_condition_with_debug_and_kill.inc UNLOCK TABLES; --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'Waiting for table metadata lock' ---source include/wait_condition.inc +--let $wait_condition_on_error_output = SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST +--source include/wait_condition_with_debug_and_kill.inc COMMIT; SELECT COUNT(*) = 1 FROM t1; diff --git a/mysql-test/suite/galera/t/mysql-wsrep#198.cnf b/mysql-test/suite/galera/t/mysql-wsrep#198.cnf index d599f3940e4..bbeb0e31c31 100644 --- a/mysql-test/suite/galera/t/mysql-wsrep#198.cnf +++ b/mysql-test/suite/galera/t/mysql-wsrep#198.cnf @@ -1,9 +1,6 @@ !include ../galera_2nodes.cnf -[mysqld.1] -log-bin -wsrep-debug=1 - -[mysqld.1] +[mysqld] log-bin wsrep-debug=1 +loose-mysql-wsrep198=1 diff --git a/mysql-test/suite/galera/t/mysql-wsrep#198.test b/mysql-test/suite/galera/t/mysql-wsrep#198.test index a8190f2ae61..2c89f859328 100644 --- a/mysql-test/suite/galera/t/mysql-wsrep#198.test +++ b/mysql-test/suite/galera/t/mysql-wsrep#198.test @@ -22,7 +22,8 @@ LOCK TABLE t2 WRITE; --connection node_2 SET SESSION wsrep_sync_wait = 0; --let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE = 'Waiting for table metadata lock' ---source include/wait_condition.inc +--let $wait_condition_on_error_output = SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST +--source include/wait_condition_with_debug_and_kill.inc --connection node_1 INSERT INTO t2 VALUES (1); From 81e507718574fa8683a796628156283892d3bb87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 13 Nov 2024 12:58:18 +0200 Subject: [PATCH 12/47] MDEV-34738 : Upgrade 10.11 -> 11.4 fails when wsrep_provider_options socket.ssl_cipher is set Problem was in Galera library where wsrep_provider socket.ssl_cipher parameter was defined as type_bool when it should have been string type. Therefore, this test requires Galera library version 26.4.21 where fix is. Here we just verify that we can upgrade ssl parameters so that socket.cipher = AES256-SHA and we use OpenSSL. Signed-off-by: Julius Goryavsky --- .../suite/galera/r/galera_ssl_cipher.result | 30 +++++++ .../suite/galera/t/galera_ssl_cipher.cnf | 11 +++ .../suite/galera/t/galera_ssl_cipher.test | 82 +++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 mysql-test/suite/galera/r/galera_ssl_cipher.result create mode 100644 mysql-test/suite/galera/t/galera_ssl_cipher.cnf create mode 100644 mysql-test/suite/galera/t/galera_ssl_cipher.test diff --git a/mysql-test/suite/galera/r/galera_ssl_cipher.result b/mysql-test/suite/galera/r/galera_ssl_cipher.result new file mode 100644 index 00000000000..76c459b9546 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_ssl_cipher.result @@ -0,0 +1,30 @@ +connection node_2; +connection node_1; +# Correct Galera library found +connection node_1; +connection node_2; +connection node_1; +connection node_2; +SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'; +VARIABLE_VALUE = 'Synced' +1 +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +VARIABLE_VALUE = 2 +1 +connection node_1; +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +VARIABLE_VALUE = 2 +1 +connection node_2; +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +VARIABLE_VALUE = 2 +1 +connection node_1; +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +VARIABLE_VALUE = 2 +1 +connection node_2; +connection node_1; +call mtr.add_suppression("WSREP: write_handler\\(\\)"); +connection node_2; +call mtr.add_suppression("WSREP: write_handler\\(\\)"); diff --git a/mysql-test/suite/galera/t/galera_ssl_cipher.cnf b/mysql-test/suite/galera/t/galera_ssl_cipher.cnf new file mode 100644 index 00000000000..dff9c96afbb --- /dev/null +++ b/mysql-test/suite/galera/t/galera_ssl_cipher.cnf @@ -0,0 +1,11 @@ +!include ../galera_2nodes.cnf + +[mysqld] +loose-galera-ssl-cipher=1 +wsrep-debug=1 + +[mysqld.1] +wsrep_provider_options='base_port=@mysqld.1.#galera_port;socket.ssl=yes;socket.ssl_cert=@ENV.MYSQL_TEST_DIR/std_data/galera-cert.pem;socket.ssl_key=@ENV.MYSQL_TEST_DIR/std_data/galera-key.pem;cert.log_conflicts=YES' + +[mysqld.2] +wsrep_provider_options='base_port=@mysqld.2.#galera_port;socket.ssl=yes;socket.ssl_cert=@ENV.MYSQL_TEST_DIR/std_data/galera-cert.pem;socket.ssl_key=@ENV.MYSQL_TEST_DIR/std_data/galera-key.pem;cert.log_conflicts=YES' diff --git a/mysql-test/suite/galera/t/galera_ssl_cipher.test b/mysql-test/suite/galera/t/galera_ssl_cipher.test new file mode 100644 index 00000000000..b2cd6188079 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_ssl_cipher.test @@ -0,0 +1,82 @@ +# +# Test upgrading the SSL chipher +# + +--source include/galera_cluster.inc +--source include/have_ssl_communication.inc +--source include/have_openssl.inc +--source include/force_restart.inc + +# +# Lowest supported Galera library version +# +--let $galera_version=26.4.21 +source ../wsrep/include/check_galera_version.inc; + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +# Setup galera ports +--connection node_1 +--source suite/galera/include/galera_base_port.inc +--let $NODE_GALERAPORT_1 = $_NODE_GALERAPORT + +--connection node_2 +--source suite/galera/include/galera_base_port.inc +--let $NODE_GALERAPORT_2 = $_NODE_GALERAPORT + +SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'; +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; + +# 2. Restart node #1 with a socket.ssl_cipher + +--connection node_1 +--source include/shutdown_mysqld.inc +--let $restart_noprint = 1 +--let $start_mysqld_params = --wsrep-cluster-address=gcomm://127.0.0.1:$NODE_GALERAPORT_2 --wsrep_provider_options=base_port=$NODE_GALERAPORT_1;socket.ssl=yes;socket.ssl_ca=$MYSQL_TEST_DIR/std_data/galera-upgrade-ca-cert.pem;socket.ssl_cert=$MYSQL_TEST_DIR/std_data/galera-cert.pem;socket.ssl_key=$MYSQL_TEST_DIR/std_data/galera-key.pem;socket.ssl_cipher=AES256-SHA +--source include/start_mysqld.inc +--source include/wait_until_connected_again.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'; +--source include/wait_condition.inc +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; + +# 3. Restart node #2 with the new socket.ssl_ca , socket.ssl_cert, socket.ssl_key and socket.ssl_cipher + +--connection node_2 +--source include/shutdown_mysqld.inc +--let $start_mysqld_params = --wsrep_provider_options=base_port=$NODE_GALERAPORT_2;socket.ssl=yes;socket.ssl_ca=$MYSQL_TEST_DIR/std_data/galera-upgrade-ca-cert.pem;socket.ssl_cert=$MYSQL_TEST_DIR/std_data/galera-upgrade-server-cert.pem;socket.ssl_key=$MYSQL_TEST_DIR/std_data/galera-upgrade-server-key.pem;socket.ssl_cipher=AES256-SHA +--source include/start_mysqld.inc +--source include/wait_until_connected_again.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'; +--source include/wait_condition.inc +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; + +# 4. Restart node #1 with the new socket.ssl_ca , socket.ssl_cert, socket.ssl_key and socket.ssl_cipher + +--connection node_1 +--source include/shutdown_mysqld.inc +--let $start_mysqld_params = --wsrep-cluster-address=gcomm://127.0.0.1:$NODE_GALERAPORT_2 --wsrep_provider_options=base_port=$NODE_GALERAPORT_1;socket.ssl=yes;socket.ssl_ca=$MYSQL_TEST_DIR/std_data/galera-upgrade-ca-cert.pem;socket.ssl_cert=$MYSQL_TEST_DIR/std_data/galera-upgrade-server-cert.pem;socket.ssl_key=$MYSQL_TEST_DIR/std_data/galera-upgrade-server-key.pem;socket.ssl_cipher=AES256-SHA +--source include/start_mysqld.inc +--source include/wait_until_connected_again.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'; +--source include/wait_condition.inc +SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; + +# 5. Make sure node_2 is ready as well +--connection node_2 +--source include/galera_wait_ready.inc + +# Upgrade complete. Both nodes now use the new key and certificate + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc + +--connection node_1 +call mtr.add_suppression("WSREP: write_handler\\(\\)"); +--connection node_2 +call mtr.add_suppression("WSREP: write_handler\\(\\)"); From 900bbbe4a89cf7fd2f443bc13ce6af2291f8ac21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 3 Feb 2025 08:11:43 +0200 Subject: [PATCH 13/47] MDEV-33295 innodb.doublewrite occasionally fails When the first attempt of XA ROLLBACK is expected to fail, some recovered changes could be written back through the doublewrite buffer. Should that happen, the next recovery attempt (after mangling the data file t1.ibd further) could fail because no copy of the affected pages would be available in the doublewrite buffer. To prevent this from happening, ensure that the doublewrite buffer will not be used and no log checkpoint occurs during the previous failed recovery attempt. Also, let a successful XA ROLLBACK serve the additional purpose of freeing a BLOB page and therefore rewriting page 0, which we must then be able to recover despite induced corruption. In the last restart step, we will tolerate an unexpected checkpoint, because one is frequently occurring on FreeBSD and AIX, despite our efforts to force a buffer pool flush before each "no checkpoint" section. --- mysql-test/suite/innodb/r/doublewrite.result | 18 +++----- .../suite/innodb/t/doublewrite.combinations | 2 + mysql-test/suite/innodb/t/doublewrite.test | 46 ++++++++++++------- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result index 667c65d7b7b..d381c6850c3 100644 --- a/mysql-test/suite/innodb/r/doublewrite.result +++ b/mysql-test/suite/innodb/r/doublewrite.result @@ -11,9 +11,11 @@ insert into t1 values(5, repeat('.',12)); commit work; SET GLOBAL innodb_fast_shutdown = 0; # restart +SET GLOBAL innodb_max_dirty_pages_pct_lwm=0,innodb_max_dirty_pages_pct=0; +SET GLOBAL innodb_max_dirty_pages_pct=99; connect dml,localhost,root,,; XA START 'x'; -insert into t1 values (6, repeat('%', @@innodb_page_size/2)); +insert into t1 values(6, repeat('%', @@innodb_page_size/2)); XA END 'x'; XA PREPARE 'x'; disconnect dml; @@ -23,7 +25,6 @@ flush table t1 for export; # restart FOUND 1 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=0\]/ in mysqld.1.err # restart -XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text test.t1 check status OK @@ -34,18 +35,13 @@ f1 f2 3 //////////// 4 ------------ 5 ............ -connect dml,localhost,root,,; -XA START 'x'; -insert into t1 values (6, repeat('%', @@innodb_page_size/2)); -XA END 'x'; -XA PREPARE 'x'; -disconnect dml; -connection default; -flush table t1 for export; +SET GLOBAL innodb_max_dirty_pages_pct_lwm=0,innodb_max_dirty_pages_pct=0; +SET GLOBAL innodb_max_dirty_pages_pct=99; +XA ROLLBACK 'x'; +FLUSH TABLE t1 FOR EXPORT; # Kill the server # restart FOUND 4 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=[03]\]/ in mysqld.1.err -XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb/t/doublewrite.combinations b/mysql-test/suite/innodb/t/doublewrite.combinations index 4f52013f6fc..797b7158778 100644 --- a/mysql-test/suite/innodb/t/doublewrite.combinations +++ b/mysql-test/suite/innodb/t/doublewrite.combinations @@ -1,7 +1,9 @@ [strict_crc32] --innodb-checksum-algorithm=strict_crc32 --innodb-use-atomic-writes=0 +--innodb-undo-tablespaces=0 [strict_full_crc32] --innodb-checksum-algorithm=strict_full_crc32 --innodb-use-atomic-writes=0 +--innodb-undo-tablespaces=0 diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index faafbb7cbac..a9c81901c47 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -42,13 +42,17 @@ commit work; SET GLOBAL innodb_fast_shutdown = 0; let $shutdown_timeout=; --source include/restart_mysqld.inc -# Ensure that buf_flush_page_cleaner() has woken up from its -# first my_cond_timedwait() and gone idle. -sleep 1; +SET GLOBAL innodb_max_dirty_pages_pct_lwm=0,innodb_max_dirty_pages_pct=0; +let $wait_condition = +SELECT variable_value = 0 +FROM information_schema.global_status +WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +--source include/wait_condition.inc +SET GLOBAL innodb_max_dirty_pages_pct=99; --source ../include/no_checkpoint_start.inc connect (dml,localhost,root,,); XA START 'x'; -insert into t1 values (6, repeat('%', @@innodb_page_size/2)); +insert into t1 values(6, repeat('%', @@innodb_page_size/2)); XA END 'x'; XA PREPARE 'x'; disconnect dml; @@ -56,10 +60,12 @@ connection default; flush table t1 for export; -let $restart_parameters=; --let CLEANUP_IF_CHECKPOINT=drop table t1, unexpected_checkpoint; --source ../include/no_checkpoint_end.inc +--copy_file $MYSQLD_DATADIR/ibdata1 $MYSQLD_DATADIR/ibdata1.bak +--copy_file $MYSQLD_DATADIR/ib_logfile0 $MYSQLD_DATADIR/ib_logfile0.bak + perl; use IO::Handle; do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; @@ -148,6 +154,12 @@ let $shutdown_timeout=0; --source include/shutdown_mysqld.inc let $shutdown_timeout=; # Corrupt the file in a better way. + +--remove_file $MYSQLD_DATADIR/ibdata1 +--remove_file $MYSQLD_DATADIR/ib_logfile0 +--move_file $MYSQLD_DATADIR/ibdata1.bak $MYSQLD_DATADIR/ibdata1 +--move_file $MYSQLD_DATADIR/ib_logfile0.bak $MYSQLD_DATADIR/ib_logfile0 + perl; use IO::Handle; my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; @@ -160,22 +172,23 @@ syswrite(FILE, chr(0) x ($page_size/2)); close FILE; EOF --source include/start_mysqld.inc -XA ROLLBACK 'x'; check table t1; select f1, f2 from t1; +SET GLOBAL innodb_max_dirty_pages_pct_lwm=0,innodb_max_dirty_pages_pct=0; +let $wait_condition = +SELECT variable_value = 0 +FROM information_schema.global_status +WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'; +--source include/wait_condition.inc +SET GLOBAL innodb_max_dirty_pages_pct=99; --source ../include/no_checkpoint_start.inc -connect (dml,localhost,root,,); -XA START 'x'; -insert into t1 values (6, repeat('%', @@innodb_page_size/2)); -XA END 'x'; -XA PREPARE 'x'; -disconnect dml; -connection default; +XA ROLLBACK 'x'; +FLUSH TABLE t1 FOR EXPORT; -flush table t1 for export; - -let $restart_parameters=; +# If we are skipping the test at this point due to an unexpected +# checkpoint, we will already have tested a part of this functionality. +--let CLEANUP_IF_CHECKPOINT=drop table t1; --source ../include/no_checkpoint_end.inc # Zero out the first page in file and try to recover from dblwr @@ -189,7 +202,6 @@ EOF --source include/start_mysqld.inc let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=[1-9][0-9]*, page number=[03]\\]; --source include/search_pattern_in_file.inc -XA ROLLBACK 'x'; check table t1; select f1, f2 from t1; drop table t1; From f6fd591a0896b54bc32409486420778d05947a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 3 Feb 2025 08:28:01 +0200 Subject: [PATCH 14/47] mtr: Globally suppress some rare warnings But do not suppress InnoDB errors that should be impossible starting with MariaDB Server 10.6. --- mysql-test/mariadb-test-run.pl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 3cfd2db2e03..a47be67a050 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -4458,14 +4458,12 @@ sub extract_warning_lines ($$) { ( @global_suppressions, qr/error .*connecting to master/, - qr/InnoDB: Error: in ALTER TABLE `test`.`t[12]`/, - qr/InnoDB: Error: table `test`.`t[12]` .*does not exist in the InnoDB internal/, - qr/InnoDB: Warning: a long semaphore wait:/, qr/InnoDB: Dumping buffer pool.*/, qr/InnoDB: Buffer pool.*/, qr/InnoDB: Could not free any blocks in the buffer pool!/, - qr/InnoDB: Warning: Writer thread is waiting this semaphore:/, qr/InnoDB: innodb_open_files .* should not be greater than/, + qr/InnoDB: Trying to delete tablespace.*but there are.*pending/, + qr/InnoDB: Tablespace 1[0-9]* was not found at .*, and innodb_force_recovery was set/, qr/Slave: Unknown table 't1' .* 1051/, qr/Slave SQL:.*(Internal MariaDB error code: [[:digit:]]+|Query:.*)/, qr/slave SQL thread aborted/, From 75b24a002ff5b95f2d6ae28231227c2588e7d8e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 3 Feb 2025 08:29:52 +0200 Subject: [PATCH 15/47] Suppress processist_state='buffer pool load' Sometimes, thread/innodb/thread_pool_thread may display processlist_state='buffer pool load' instead of NULL when the test is executed soon enough after server startup. Let us suppress that information to avoid spurious failures. --- mysql-test/suite/perfschema/r/threads_innodb.result | 10 +++++----- mysql-test/suite/perfschema/t/threads_innodb.test | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/perfschema/r/threads_innodb.result b/mysql-test/suite/perfschema/r/threads_innodb.result index abfedb57644..bd81c5991f3 100644 --- a/mysql-test/suite/perfschema/r/threads_innodb.result +++ b/mysql-test/suite/perfschema/r/threads_innodb.result @@ -1,10 +1,10 @@ SELECT name, type, processlist_user, processlist_host, processlist_db, -processlist_command, processlist_time, processlist_state, processlist_info, +processlist_command, processlist_time, processlist_info, parent_thread_id, role, instrumented FROM performance_schema.threads WHERE name LIKE 'thread/innodb/%' GROUP BY name; -name type processlist_user processlist_host processlist_db processlist_command processlist_time processlist_state processlist_info parent_thread_id role instrumented -thread/innodb/page_cleaner_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL NULL YES -thread/innodb/page_encrypt_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL NULL YES -thread/innodb/thread_pool_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL NULL YES +name type processlist_user processlist_host processlist_db processlist_command processlist_time processlist_info parent_thread_id role instrumented +thread/innodb/page_cleaner_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL YES +thread/innodb/page_encrypt_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL YES +thread/innodb/thread_pool_thread BACKGROUND NULL NULL NULL NULL NULL NULL NULL NULL YES diff --git a/mysql-test/suite/perfschema/t/threads_innodb.test b/mysql-test/suite/perfschema/t/threads_innodb.test index 366c3b82ce8..e791ded2597 100644 --- a/mysql-test/suite/perfschema/t/threads_innodb.test +++ b/mysql-test/suite/perfschema/t/threads_innodb.test @@ -14,7 +14,7 @@ # We suppress here duplicates rows with the goal to avoid that the test fails # in case some defaults are changed. SELECT name, type, processlist_user, processlist_host, processlist_db, - processlist_command, processlist_time, processlist_state, processlist_info, + processlist_command, processlist_time, processlist_info, parent_thread_id, role, instrumented FROM performance_schema.threads WHERE name LIKE 'thread/innodb/%' From 10fd2c207a8d79e038fba752a72129a3a0e94b6b Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Thu, 30 Jan 2025 10:10:39 +0100 Subject: [PATCH 16/47] MDEV-35946 Assertion `thd->is_error()' failed in Sql_cmd_dml::prepare Fix a regression that caused assertion thd->is_error() after sync wait failures. If wsrep_sync_wait() fails make sure a appropriate error is set. Partially revert 75dd0246f8b0b7c1d7d7762415089a74c433cb1d. Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera/r/MDEV-35946.result | 16 ++++++++ mysql-test/suite/galera/t/MDEV-35946.test | 41 +++++++++++++++++++++ sql/wsrep_mysqld.cc | 7 +++- 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/galera/r/MDEV-35946.result create mode 100644 mysql-test/suite/galera/t/MDEV-35946.test diff --git a/mysql-test/suite/galera/r/MDEV-35946.result b/mysql-test/suite/galera/r/MDEV-35946.result new file mode 100644 index 00000000000..1ebc88c1a48 --- /dev/null +++ b/mysql-test/suite/galera/r/MDEV-35946.result @@ -0,0 +1,16 @@ +connection node_2; +connection node_1; +connection node_1; +connection node_2; +SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1'; +SET SESSION wsrep_sync_wait=0; +SET SESSION wsrep_sync_wait=DEFAULT; +DELETE FROM mysql.wsrep_streaming_log; +ERROR HY000: Lock wait timeout exceeded; try restarting transaction +SET SESSION wsrep_sync_wait=0; +SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0'; +SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; +VARIABLE_VALUE +Primary +SET SESSION wsrep_sync_wait=DEFAULT; +CALL mtr.add_suppression("WSREP: Protocol violation\\. JOIN message sender (.*) is not in state transfer \\(SYNCED\\)\\. Message ignored\\."); diff --git a/mysql-test/suite/galera/t/MDEV-35946.test b/mysql-test/suite/galera/t/MDEV-35946.test new file mode 100644 index 00000000000..23c3d463a48 --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-35946.test @@ -0,0 +1,41 @@ +# +# MDEV-35946: Assertion `thd->is_error()' failed in Sql_cmd_dml::prepare +# +--source include/have_innodb.inc +--source include/galera_cluster.inc + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +# +# Disconnect from the cluster +# +SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1'; +SET SESSION wsrep_sync_wait=0; +--let $wait_condition = SELECT VARIABLE_VALUE = 'non-Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; +--source include/wait_condition.inc +SET SESSION wsrep_sync_wait=DEFAULT; + +# +# If bug is present, assertion will fire +# during the execution of the following DELETE +# +--error ER_LOCK_WAIT_TIMEOUT +DELETE FROM mysql.wsrep_streaming_log; + + +# +# Reconnect to the cluster +# +SET SESSION wsrep_sync_wait=0; +SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0'; +--let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; +--source include/wait_condition.inc +SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status'; +SET SESSION wsrep_sync_wait=DEFAULT; + + +--source include/auto_increment_offset_restore.inc +CALL mtr.add_suppression("WSREP: Protocol violation\\. JOIN message sender (.*) is not in state transfer \\(SYNCED\\)\\. Message ignored\\."); diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index c905883e2fb..974ddfc1d4c 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -1231,7 +1231,12 @@ bool wsrep_sync_wait (THD* thd, uint mask) This allows autocommit SELECTs and a first SELECT after SET AUTOCOMMIT=0 TODO: modify to check if thd has locked any rows. */ - return thd->wsrep_cs().sync_wait(-1); + if (thd->wsrep_cs().sync_wait(-1)) + { + wsrep_override_error(thd, thd->wsrep_cs().current_error(), + thd->wsrep_cs().current_error_status()); + return true; + } } return false; From 1f93aece3d187e870bc5810107f393c42daa0cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 7 Nov 2024 14:41:02 +0200 Subject: [PATCH 17/47] MDEV-24485 : galera.galera_bf_kill_debug MTR failed: A long semaphore wait Test sends a signal to debug_sync and in next command resets debug_sync signals. There is small possibility that sended signal is not yet handled in receiving thread and reset will destroy it causing sync wait timeout. Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera/r/galera_bf_kill_debug.result | 5 +++-- mysql-test/suite/galera/t/galera_bf_kill_debug.test | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_bf_kill_debug.result b/mysql-test/suite/galera/r/galera_bf_kill_debug.result index 52bd1b0e370..1f5be6c4e0e 100644 --- a/mysql-test/suite/galera/r/galera_bf_kill_debug.result +++ b/mysql-test/suite/galera/r/galera_bf_kill_debug.result @@ -40,18 +40,19 @@ drop table t1; disconnect node_2a; connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2; connection node_2a; -CREATE TABLE t1 (i int primary key); +CREATE TABLE t1 (i int primary key) engine=innodb; SET DEBUG_SYNC = "before_wsrep_ordered_commit SIGNAL bwoc_reached WAIT_FOR bwoc_continue"; INSERT INTO t1 VALUES (1); connection node_2; SET DEBUG_SYNC = "now WAIT_FOR bwoc_reached"; SET DEBUG_SYNC = "now SIGNAL bwoc_continue"; -SET DEBUG_SYNC='RESET'; connection node_2a; connection node_2; +SET DEBUG_SYNC='RESET'; select * from t1; i 1 disconnect node_2a; +disconnect node_2b; connection node_1; drop table t1; diff --git a/mysql-test/suite/galera/t/galera_bf_kill_debug.test b/mysql-test/suite/galera/t/galera_bf_kill_debug.test index d24a6dd19ef..a89bd1d06b2 100644 --- a/mysql-test/suite/galera/t/galera_bf_kill_debug.test +++ b/mysql-test/suite/galera/t/galera_bf_kill_debug.test @@ -110,7 +110,7 @@ drop table t1; --connection node_2a --let $connection_id = `SELECT CONNECTION_ID()` -CREATE TABLE t1 (i int primary key); +CREATE TABLE t1 (i int primary key) engine=innodb; # Set up sync point SET DEBUG_SYNC = "before_wsrep_ordered_commit SIGNAL bwoc_reached WAIT_FOR bwoc_continue"; @@ -129,17 +129,17 @@ SET DEBUG_SYNC = "now WAIT_FOR bwoc_reached"; --enable_query_log SET DEBUG_SYNC = "now SIGNAL bwoc_continue"; -SET DEBUG_SYNC='RESET'; --connection node_2a --error 0,1213 --reap --connection node_2 +SET DEBUG_SYNC='RESET'; # victim was able to complete the INSERT select * from t1; --disconnect node_2a +--disconnect node_2b --connection node_1 drop table t1; - From 64b964e757d9871b54c434631bf55608fbbf119c Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 4 Feb 2025 10:11:58 -0500 Subject: [PATCH 18/47] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 84827b2ea99..42ffde83307 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=6 -MYSQL_VERSION_PATCH=21 +MYSQL_VERSION_PATCH=22 SERVER_MATURITY=stable From 6e6fcf4d43e9f5812e1870821968a77c9f826b62 Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Fri, 31 Jan 2025 14:32:22 +0300 Subject: [PATCH 19/47] MDEV-34489 innodb.innodb_row_lock_time_ms fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test fails trying to compare (innodb/lock)_row_lock_time_avg with some limit. We can't predict (innodb/lock)_row_lock_time_avg value, because it's counted as the whole waiting time divided by the amount of waits. Both waiting time and amount of waits depend on the previous tests execution. The corresponding counters in lock_sys can't be reset with any query. Remove (innodb/lock)_row_lock_time_avg comparision from the test. information_schema.global_status.innodb_row_lock_time can't be reset, compare its difference instead of absolute value. Reviewed by: Marko Mäkelä --- .../innodb/r/innodb_row_lock_time_ms.result | 40 ++++++++++------- .../innodb/t/innodb_row_lock_time_ms.test | 45 +++++++++++++------ 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_row_lock_time_ms.result b/mysql-test/suite/innodb/r/innodb_row_lock_time_ms.result index 984d789e058..bba1342f394 100644 --- a/mysql-test/suite/innodb/r/innodb_row_lock_time_ms.result +++ b/mysql-test/suite/innodb/r/innodb_row_lock_time_ms.result @@ -1,10 +1,18 @@ CREATE TABLE `t`(`id` INT, PRIMARY KEY(`id`)) ENGINE=InnoDB STATS_PERSISTENT=0; INSERT INTO t VALUES (1); -SET GLOBAL innodb_monitor_reset = "module_innodb"; +SET GLOBAL innodb_monitor_disable="lock_row_lock_time"; +SET GLOBAL innodb_monitor_disable="lock_row_lock_time_max"; +SET GLOBAL innodb_monitor_reset_all='lock_row_lock_time'; +SET GLOBAL innodb_monitor_reset_all='lock_row_lock_time_max'; +SET GLOBAL innodb_monitor_enable="lock_row_lock_time"; +SET GLOBAL innodb_monitor_enable="lock_row_lock_time_max"; BEGIN; SELECT * FROM t FOR UPDATE; id 1 +SELECT @innodb_row_lock_time_before := variable_value +FROM information_schema.global_status +WHERE LOWER(variable_name) = 'innodb_row_lock_time'; connect con1,localhost,root,,; SET innodb_lock_wait_timeout = 1; SELECT * FROM t FOR UPDATE; @@ -12,29 +20,27 @@ ERROR HY000: Lock wait timeout exceeded; try restarting transaction disconnect con1; connection default; COMMIT; -SELECT variable_value > 100 FROM information_schema.global_status +SELECT variable_value - @innodb_row_lock_time_before > 100 +FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_row_lock_time'; -variable_value > 100 +variable_value - @innodb_row_lock_time_before > 100 1 -SELECT variable_value > 100 FROM information_schema.global_status +SELECT variable_value > 100 +FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_row_lock_time_max'; variable_value > 100 1 -SELECT variable_value > 100 FROM information_schema.global_status -WHERE LOWER(variable_name) = 'innodb_row_lock_time_avg'; -variable_value > 100 -1 -SELECT count_reset > 100 FROM INFORMATION_SCHEMA.INNODB_METRICS -WHERE NAME="lock_row_lock_time"; +SELECT count_reset > 100 +FROM INFORMATION_SCHEMA.INNODB_METRICS +WHERE NAME='lock_row_lock_time'; count_reset > 100 1 -SELECT count_reset > 100 FROM INFORMATION_SCHEMA.INNODB_METRICS -WHERE NAME="lock_row_lock_time_max"; -count_reset > 100 -1 -SELECT count_reset > 100 FROM INFORMATION_SCHEMA.INNODB_METRICS -WHERE NAME="lock_row_lock_time_avg"; +SELECT count_reset > 100 +FROM INFORMATION_SCHEMA.INNODB_METRICS +WHERE NAME='lock_row_lock_time_max'; count_reset > 100 1 DROP TABLE t; -SET GLOBAL innodb_monitor_reset=default; +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; diff --git a/mysql-test/suite/innodb/t/innodb_row_lock_time_ms.test b/mysql-test/suite/innodb/t/innodb_row_lock_time_ms.test index 4a100821819..1cd43dbca7f 100644 --- a/mysql-test/suite/innodb/t/innodb_row_lock_time_ms.test +++ b/mysql-test/suite/innodb/t/innodb_row_lock_time_ms.test @@ -5,11 +5,26 @@ CREATE TABLE `t`(`id` INT, PRIMARY KEY(`id`)) ENGINE=InnoDB STATS_PERSISTENT=0; INSERT INTO t VALUES (1); -SET GLOBAL innodb_monitor_reset = "module_innodb"; +SET GLOBAL innodb_monitor_disable="lock_row_lock_time"; +SET GLOBAL innodb_monitor_disable="lock_row_lock_time_max"; +SET GLOBAL innodb_monitor_reset_all='lock_row_lock_time'; +SET GLOBAL innodb_monitor_reset_all='lock_row_lock_time_max'; +SET GLOBAL innodb_monitor_enable="lock_row_lock_time"; +SET GLOBAL innodb_monitor_enable="lock_row_lock_time_max"; BEGIN; SELECT * FROM t FOR UPDATE; +# We can't predict (innodb/lock)_row_lock_time_avg value, because it's counted +# as the whole waiting time divided by the amount of waits. The +# corresponding counters in lock_sys can't be reset with any query. + +--disable_result_log +SELECT @innodb_row_lock_time_before := variable_value + FROM information_schema.global_status + WHERE LOWER(variable_name) = 'innodb_row_lock_time'; +--enable_result_log + --connect(con1,localhost,root,,) SET innodb_lock_wait_timeout = 1; --error ER_LOCK_WAIT_TIMEOUT @@ -19,24 +34,28 @@ SELECT * FROM t FOR UPDATE; --connection default COMMIT; -SELECT variable_value > 100 FROM information_schema.global_status +SELECT variable_value - @innodb_row_lock_time_before > 100 + FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_row_lock_time'; -SELECT variable_value > 100 FROM information_schema.global_status +# We can't use 'variable_value - @innodb_row_lock_time_max_before' trick for +# innodb_row_lock_time_max, because we can't reset it, and we don't know the +# initial value at the moment of the test execution. +SELECT variable_value > 100 + FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_row_lock_time_max'; -SELECT variable_value > 100 FROM information_schema.global_status - WHERE LOWER(variable_name) = 'innodb_row_lock_time_avg'; - -SELECT count_reset > 100 FROM INFORMATION_SCHEMA.INNODB_METRICS - WHERE NAME="lock_row_lock_time"; -SELECT count_reset > 100 FROM INFORMATION_SCHEMA.INNODB_METRICS - WHERE NAME="lock_row_lock_time_max"; -SELECT count_reset > 100 FROM INFORMATION_SCHEMA.INNODB_METRICS - WHERE NAME="lock_row_lock_time_avg"; +SELECT count_reset > 100 + FROM INFORMATION_SCHEMA.INNODB_METRICS + WHERE NAME='lock_row_lock_time'; +SELECT count_reset > 100 + FROM INFORMATION_SCHEMA.INNODB_METRICS + WHERE NAME='lock_row_lock_time_max'; DROP TABLE t; --disable_warnings -SET GLOBAL innodb_monitor_reset=default; +SET GLOBAL innodb_monitor_enable=default; +SET GLOBAL innodb_monitor_disable=default; +SET GLOBAL innodb_monitor_reset_all=default; --enable_warnings --source include/wait_until_count_sessions.inc From 0e80d3bba8a0ddb0f7241b4df57029d76ee9e989 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 4 Feb 2025 11:18:48 -0500 Subject: [PATCH 20/47] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index eb45175874a..283afc0001c 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=11 -MYSQL_VERSION_PATCH=11 +MYSQL_VERSION_PATCH=12 SERVER_MATURITY=stable From 6be0940f104e81404604aed0850e21223341adee Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Tue, 4 Feb 2025 12:36:00 -0500 Subject: [PATCH 21/47] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7a561f98fa8..0a37eafcf46 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=11 MYSQL_VERSION_MINOR=4 -MYSQL_VERSION_PATCH=5 +MYSQL_VERSION_PATCH=6 SERVER_MATURITY=stable From 91de54e098b179f033deb1ee61fc3d7d5997f588 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 4 Feb 2025 21:46:42 +0200 Subject: [PATCH 22/47] Remove redundant if-statement in Index_prefix_calc::get_avg_frequency the code went like this: for ( ... ; i < prefixes ; ...) { if (i < prefixes) { ... --- sql/sql_statistics.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 51a13fc54a7..163cb8da9fc 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -2077,12 +2077,9 @@ public: for (i= 0, state= calc_state; i < prefixes; i++, state++) { - if (i < prefixes) - { - double val= state->prefix_count == 0 ? - 0 : (double) state->entry_count / state->prefix_count; - index_info->collected_stats->set_avg_frequency(i, val); - } + double val= state->prefix_count == 0 ? + 0 : (double) state->entry_count / state->prefix_count; + index_info->collected_stats->set_avg_frequency(i, val); } } }; From b7d67ceb5f80cea3d45df7251bbcb128fd5ddd1e Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Sat, 8 Feb 2025 17:39:27 +0400 Subject: [PATCH 23/47] MDEV-36047 Package body variables are not allowed as FETCH targets It was not possible to use a package body variable as a fetch target: CREATE PACKAGE BODY pkg AS vc INT := 0; FUNCTION f1 RETURN INT AS CURSOR cur IS SELECT 1 AS c FROM DUAL; BEGIN OPEN cur; FETCH cur INTO vc; -- this returned "Undeclared variable: vc" error. CLOSE cur; RETURN vc; END; END; FETCH assumed that all fetch targets reside of the same sp_rcontext instance with the cursor. This patch fixes the problem. Now a cursor and its fetch target can reside in different sp_rcontext instances. Details: - Adding a helper class sp_rcontext_addr (a combination of Sp_rcontext_handler pointer and an offset in the rcontext) - Adding a new class sp_fetch_target deriving from sp_rcontext_addr. Fetch targets in "FETCH cur INTO target1, target2 ..." are now collected into this structure instead of sp_variable. sp_variable cannot be used any more to store fetch targets, because it does not have a pointer to Sp_rcontext_handler (it only has the current rcontext offset). - Removing members sp_instr_set members m_rcontext_handler and m_offset. Deriving sp_instr_set from sp_rcontext_addr instead. - Renaming sp_instr_cfetch member "List m_varlist" to "List m_fetch_target_list". - Fixing LEX::sp_add_cfetch() to return the pointer to the created sp_fetch_target instance (instead of returning bool). This helps to make the grammar in sql_yacc.c simpler - Renaming LEX::sp_add_cfetch() to LEX::sp_add_instr_cfetch(), as `if(sp_add_cfetch())` changed its meaning to the opposite, to avoid automatic wrong merge from earlier versions. - Chaning the "List *vars" parameter to sp_cursor::fetch to have the data type "List *". - Changing the data type of "List &vars" in sp_cursor::Select_fetch_into_spvars::send_data_to_variable_list() to "List &". - Adding THD helper methods get_rcontext() and get_variable(). - Moving the code from sql_yacc.yy into a new LEX method LEX::make_fetch_target(). - Simplifying the grammar in sql_yacc.yy using the new LEX method. Changing the data type of the bison rule sp_fetch_list from "void" to "List *". --- mysql-test/include/sp-cursor-pkg-01.inc | 24 ++++ mysql-test/include/sp-cursor-pkg-02.inc | 20 ++++ mysql-test/include/sp-cursor-pkg-03.inc | 19 ++++ mysql-test/main/sp-cursor.result | 72 ++++++++++++ mysql-test/main/sp-cursor.test | 20 ++++ mysql-test/main/sp-package-code.result | 102 +++++++++++++++++ mysql-test/main/sp-package-code.test | 23 ++++ .../suite/compat/oracle/r/sp-cursor.result | 74 +++++++++++++ .../compat/oracle/r/sp-package-code.result | 104 ++++++++++++++++++ .../compat/oracle/t/sp-cursor-pkg-01.inc | 26 +++++ .../compat/oracle/t/sp-cursor-pkg-02.inc | 20 ++++ .../compat/oracle/t/sp-cursor-pkg-03.inc | 19 ++++ .../suite/compat/oracle/t/sp-cursor.test | 21 ++++ .../compat/oracle/t/sp-package-code.test | 25 +++++ sql/sp_head.cc | 6 +- sql/sp_instr.cc | 12 +- sql/sp_instr.h | 21 ++-- sql/sp_pcontext.h | 21 ++++ sql/sp_rcontext.cc | 33 +++--- sql/sql_class.cc | 12 ++ sql/sql_class.h | 17 ++- sql/sql_lex.cc | 30 ++++- sql/sql_lex.h | 5 +- sql/sql_yacc.yy | 52 ++++----- sql/structs.h | 25 +++++ 25 files changed, 731 insertions(+), 72 deletions(-) create mode 100644 mysql-test/include/sp-cursor-pkg-01.inc create mode 100644 mysql-test/include/sp-cursor-pkg-02.inc create mode 100644 mysql-test/include/sp-cursor-pkg-03.inc create mode 100644 mysql-test/suite/compat/oracle/t/sp-cursor-pkg-01.inc create mode 100644 mysql-test/suite/compat/oracle/t/sp-cursor-pkg-02.inc create mode 100644 mysql-test/suite/compat/oracle/t/sp-cursor-pkg-03.inc diff --git a/mysql-test/include/sp-cursor-pkg-01.inc b/mysql-test/include/sp-cursor-pkg-01.inc new file mode 100644 index 00000000000..78f2848e221 --- /dev/null +++ b/mysql-test/include/sp-cursor-pkg-01.inc @@ -0,0 +1,24 @@ +DELIMITER $$; +CREATE PACKAGE pkg + FUNCTION f1() RETURNS INT; +END; +$$ +CREATE PACKAGE BODY pkg + DECLARE vc INT DEFAULT 0; + FUNCTION f1() RETURNS INT + BEGIN + DECLARE cur CURSOR FOR SELECT 1 AS c FROM DUAL; + OPEN cur; + FETCH cur INTO vc; -- SHOW CODE should display vc with a "PACKAGE_BODY" prefix + CLOSE cur; + RETURN vc; + END; + BEGIN + DECLARE cur CURSOR FOR SELECT 1 AS c FROM DUAL; + OPEN cur; + FETCH cur INTO vc; -- SHOW CODE should display vc without a prefix + CLOSE cur; + END; +END; +$$ +DELIMITER ;$$ diff --git a/mysql-test/include/sp-cursor-pkg-02.inc b/mysql-test/include/sp-cursor-pkg-02.inc new file mode 100644 index 00000000000..d84bbac9209 --- /dev/null +++ b/mysql-test/include/sp-cursor-pkg-02.inc @@ -0,0 +1,20 @@ +# Mixing a package body variable and a local variable in the same FETCH. +DELIMITER $$; +CREATE PACKAGE pkg + FUNCTION f1() RETURNS TEXT; +END; +$$ +CREATE PACKAGE BODY pkg + DECLARE vc1 INT DEFAULT 0; + FUNCTION f1() RETURNS TEXT + BEGIN + DECLARE vc2 INT DEFAULT 0; + DECLARE cur CURSOR FOR SELECT 1 AS c1, 2 AS c2 FROM DUAL; + OPEN cur; + FETCH cur INTO vc1, vc2; + CLOSE cur; + RETURN CONCAT(vc1, ' ', vc2); + END; +END; +$$ +DELIMITER ;$$ diff --git a/mysql-test/include/sp-cursor-pkg-03.inc b/mysql-test/include/sp-cursor-pkg-03.inc new file mode 100644 index 00000000000..900b515ad5f --- /dev/null +++ b/mysql-test/include/sp-cursor-pkg-03.inc @@ -0,0 +1,19 @@ +# Fetching into a PACKAGE BODY variable of the ROW type +DELIMITER $$; +CREATE PACKAGE pkg + FUNCTION f1() RETURNS TEXT; +END; +$$ +CREATE PACKAGE BODY pkg + DECLARE vc ROW(p1 INT, p2 INT); + FUNCTION f1() RETURNS TEXT + BEGIN + DECLARE cur CURSOR FOR SELECT 1 AS c1, 2 AS c2 FROM DUAL; + OPEN cur; + FETCH cur INTO vc; + CLOSE cur; + RETURN CONCAT(vc.p1, ' ', vc.p2); + END; +END; +$$ +DELIMITER ;$$ diff --git a/mysql-test/main/sp-cursor.result b/mysql-test/main/sp-cursor.result index 83b05264339..7afd4c33aef 100644 --- a/mysql-test/main/sp-cursor.result +++ b/mysql-test/main/sp-cursor.result @@ -852,3 +852,75 @@ ERROR 42000: This version of MariaDB doesn't yet support 'OUT/INOUT cursor param # # End of 10.8 tests # +# Start of 11.4 tests +# +# MDEV-36047 Package body variables are not allowed as FETCH targets +# +CREATE PACKAGE pkg +FUNCTION f1() RETURNS INT; +END; +$$ +CREATE PACKAGE BODY pkg +DECLARE vc INT DEFAULT 0; +FUNCTION f1() RETURNS INT +BEGIN +DECLARE cur CURSOR FOR SELECT 1 AS c FROM DUAL; +OPEN cur; +FETCH cur INTO vc; -- SHOW CODE should display vc with a "PACKAGE_BODY" prefix +CLOSE cur; +RETURN vc; +END; +BEGIN +DECLARE cur CURSOR FOR SELECT 1 AS c FROM DUAL; +OPEN cur; +FETCH cur INTO vc; -- SHOW CODE should display vc without a prefix +CLOSE cur; +END; +END; +$$ +SELECT pkg.f1(); +pkg.f1() +1 +DROP PACKAGE pkg; +CREATE PACKAGE pkg +FUNCTION f1() RETURNS TEXT; +END; +$$ +CREATE PACKAGE BODY pkg +DECLARE vc1 INT DEFAULT 0; +FUNCTION f1() RETURNS TEXT +BEGIN +DECLARE vc2 INT DEFAULT 0; +DECLARE cur CURSOR FOR SELECT 1 AS c1, 2 AS c2 FROM DUAL; +OPEN cur; +FETCH cur INTO vc1, vc2; +CLOSE cur; +RETURN CONCAT(vc1, ' ', vc2); +END; +END; +$$ +SELECT pkg.f1(); +pkg.f1() +1 2 +DROP PACKAGE pkg; +CREATE PACKAGE pkg +FUNCTION f1() RETURNS TEXT; +END; +$$ +CREATE PACKAGE BODY pkg +DECLARE vc ROW(p1 INT, p2 INT); +FUNCTION f1() RETURNS TEXT +BEGIN +DECLARE cur CURSOR FOR SELECT 1 AS c1, 2 AS c2 FROM DUAL; +OPEN cur; +FETCH cur INTO vc; +CLOSE cur; +RETURN CONCAT(vc.p1, ' ', vc.p2); +END; +END; +$$ +SELECT pkg.f1(); +pkg.f1() +1 2 +DROP PACKAGE pkg; +# End of 11.4 tests diff --git a/mysql-test/main/sp-cursor.test b/mysql-test/main/sp-cursor.test index feb681201a2..d15a0c80a6f 100644 --- a/mysql-test/main/sp-cursor.test +++ b/mysql-test/main/sp-cursor.test @@ -872,3 +872,23 @@ DELIMITER ;$$ --echo # --echo # End of 10.8 tests --echo # + +--echo # Start of 11.4 tests + +--echo # +--echo # MDEV-36047 Package body variables are not allowed as FETCH targets +--echo # + +--source include/sp-cursor-pkg-01.inc +SELECT pkg.f1(); +DROP PACKAGE pkg; + +--source include/sp-cursor-pkg-02.inc +SELECT pkg.f1(); +DROP PACKAGE pkg; + +--source include/sp-cursor-pkg-03.inc +SELECT pkg.f1(); +DROP PACKAGE pkg; + +--echo # End of 11.4 tests diff --git a/mysql-test/main/sp-package-code.result b/mysql-test/main/sp-package-code.result index f69cb7ec671..290c939749f 100644 --- a/mysql-test/main/sp-package-code.result +++ b/mysql-test/main/sp-package-code.result @@ -258,3 +258,105 @@ Pos Instruction 6 set a.a@0["a"] b.a@1["a"] + 1 DROP PACKAGE pkg1; DROP TABLE t1; +# Start of 11.4 tests +# +# MDEV-36047 Package body variables are not allowed as FETCH targets +# +CREATE PACKAGE pkg +FUNCTION f1() RETURNS INT; +END; +$$ +CREATE PACKAGE BODY pkg +DECLARE vc INT DEFAULT 0; +FUNCTION f1() RETURNS INT +BEGIN +DECLARE cur CURSOR FOR SELECT 1 AS c FROM DUAL; +OPEN cur; +FETCH cur INTO vc; -- SHOW CODE should display vc with a "PACKAGE_BODY" prefix +CLOSE cur; +RETURN vc; +END; +BEGIN +DECLARE cur CURSOR FOR SELECT 1 AS c FROM DUAL; +OPEN cur; +FETCH cur INTO vc; -- SHOW CODE should display vc without a prefix +CLOSE cur; +END; +END; +$$ +SELECT pkg.f1() FROM DUAL; +pkg.f1() +1 +SHOW FUNCTION CODE pkg.f1; +Pos Instruction +0 cpush cur@0 +1 copen cur@0 +2 cfetch cur@0 PACKAGE_BODY.vc@0 +3 cclose cur@0 +4 freturn int PACKAGE_BODY.vc@0 +SHOW PACKAGE BODY CODE pkg; +Pos Instruction +0 set vc@0 0 +1 cpush cur@0 +2 copen cur@0 +3 cfetch cur@0 vc@0 +4 cclose cur@0 +5 cpop 1 +DROP PACKAGE pkg; +CREATE PACKAGE pkg +FUNCTION f1() RETURNS TEXT; +END; +$$ +CREATE PACKAGE BODY pkg +DECLARE vc1 INT DEFAULT 0; +FUNCTION f1() RETURNS TEXT +BEGIN +DECLARE vc2 INT DEFAULT 0; +DECLARE cur CURSOR FOR SELECT 1 AS c1, 2 AS c2 FROM DUAL; +OPEN cur; +FETCH cur INTO vc1, vc2; +CLOSE cur; +RETURN CONCAT(vc1, ' ', vc2); +END; +END; +$$ +SELECT pkg.f1() FROM DUAL; +pkg.f1() +1 2 +SHOW FUNCTION CODE pkg.f1; +Pos Instruction +0 set vc2@0 0 +1 cpush cur@0 +2 copen cur@0 +3 cfetch cur@0 PACKAGE_BODY.vc1@0 vc2@0 +4 cclose cur@0 +5 freturn blob concat(PACKAGE_BODY.vc1@0,' ',vc2@0) +DROP PACKAGE pkg; +CREATE PACKAGE pkg +FUNCTION f1() RETURNS TEXT; +END; +$$ +CREATE PACKAGE BODY pkg +DECLARE vc ROW(p1 INT, p2 INT); +FUNCTION f1() RETURNS TEXT +BEGIN +DECLARE cur CURSOR FOR SELECT 1 AS c1, 2 AS c2 FROM DUAL; +OPEN cur; +FETCH cur INTO vc; +CLOSE cur; +RETURN CONCAT(vc.p1, ' ', vc.p2); +END; +END; +$$ +SELECT pkg.f1() FROM DUAL; +pkg.f1() +1 2 +SHOW FUNCTION CODE pkg.f1; +Pos Instruction +0 cpush cur@0 +1 copen cur@0 +2 cfetch cur@0 PACKAGE_BODY.vc@0 +3 cclose cur@0 +4 freturn blob concat(PACKAGE_BODY.vc.p1@0[0],' ',PACKAGE_BODY.vc.p2@0[1]) +DROP PACKAGE pkg; +# End of 11.4 tests diff --git a/mysql-test/main/sp-package-code.test b/mysql-test/main/sp-package-code.test index e99c25e7aa2..740a3eeb8af 100644 --- a/mysql-test/main/sp-package-code.test +++ b/mysql-test/main/sp-package-code.test @@ -198,3 +198,26 @@ DROP PACKAGE pkg1; DROP TABLE t1; +--echo # Start of 11.4 tests + +--echo # +--echo # MDEV-36047 Package body variables are not allowed as FETCH targets +--echo # + +--source include/sp-cursor-pkg-01.inc +SELECT pkg.f1() FROM DUAL; +SHOW FUNCTION CODE pkg.f1; +SHOW PACKAGE BODY CODE pkg; +DROP PACKAGE pkg; + +--source include/sp-cursor-pkg-02.inc +SELECT pkg.f1() FROM DUAL; +SHOW FUNCTION CODE pkg.f1; +DROP PACKAGE pkg; + +--source include/sp-cursor-pkg-03.inc +SELECT pkg.f1() FROM DUAL; +SHOW FUNCTION CODE pkg.f1; +DROP PACKAGE pkg; + +--echo # End of 11.4 tests diff --git a/mysql-test/suite/compat/oracle/r/sp-cursor.result b/mysql-test/suite/compat/oracle/r/sp-cursor.result index aa9c5de8bc9..285934a3e30 100644 --- a/mysql-test/suite/compat/oracle/r/sp-cursor.result +++ b/mysql-test/suite/compat/oracle/r/sp-cursor.result @@ -1020,3 +1020,77 @@ ERROR 42000: This version of MariaDB doesn't yet support 'OUT/INOUT cursor param # # End of 10.8 tests # +# Start of 11.4 tests +# +# MDEV-36047 Package body variables are not allowed as FETCH targets +# +CREATE PACKAGE pkg AS +FUNCTION f1 RETURN INT; +END; +$$ +CREATE PACKAGE BODY pkg AS +vc INT := 0; +FUNCTION f1 RETURN INT AS +CURSOR cur IS SELECT 1 AS c FROM DUAL; +BEGIN +OPEN cur; +FETCH cur INTO vc; -- SHOW CODE should display vc with a "PACKAGE_BODY" prefix +CLOSE cur; +RETURN vc; +END; +BEGIN +DECLARE +CURSOR cur IS SELECT 1 AS c FROM DUAL; +BEGIN +OPEN cur; +FETCH cur INTO vc; -- SHOW CODE should display vc without a prefix +CLOSE cur; +END; +END; +$$ +SELECT pkg.f1() FROM DUAL; +pkg.f1() +1 +DROP PACKAGE pkg; +CREATE PACKAGE pkg AS +FUNCTION f1 RETURN TEXT; +END; +$$ +CREATE PACKAGE BODY pkg AS +vc1 INT := 0; +FUNCTION f1 RETURN TEXT AS +CURSOR cur IS SELECT 1 AS c1, 2 AS c2 FROM DUAL; +vc2 INT := 0; +BEGIN +OPEN cur; +FETCH cur INTO vc1, vc2; +CLOSE cur; +RETURN vc1 || ' ' || vc2; +END; +END; +$$ +SELECT pkg.f1() FROM DUAL; +pkg.f1() +1 2 +DROP PACKAGE pkg; +CREATE PACKAGE pkg AS +FUNCTION f1 RETURN TEXT; +END; +$$ +CREATE PACKAGE BODY pkg AS +vc ROW(p1 INT, p2 INT); +FUNCTION f1 RETURN TEXT AS +CURSOR cur IS SELECT 1 AS c1, 2 AS c2 FROM DUAL; +BEGIN +OPEN cur; +FETCH cur INTO vc; +CLOSE cur; +RETURN vc.p1 || ' ' || vc.p2; +END; +END; +$$ +SELECT pkg.f1() FROM DUAL; +pkg.f1() +1 2 +DROP PACKAGE pkg; +# End of 11.4 tests diff --git a/mysql-test/suite/compat/oracle/r/sp-package-code.result b/mysql-test/suite/compat/oracle/r/sp-package-code.result index 0dea72dba89..52b41a3c64e 100644 --- a/mysql-test/suite/compat/oracle/r/sp-package-code.result +++ b/mysql-test/suite/compat/oracle/r/sp-package-code.result @@ -243,3 +243,107 @@ Pos Instruction 7 jump 11 DROP PACKAGE pkg1; DROP TABLE t1; +# Start of 11.4 tests +# +# MDEV-36047 Package body variables are not allowed as FETCH targets +# +CREATE PACKAGE pkg AS +FUNCTION f1 RETURN INT; +END; +$$ +CREATE PACKAGE BODY pkg AS +vc INT := 0; +FUNCTION f1 RETURN INT AS +CURSOR cur IS SELECT 1 AS c FROM DUAL; +BEGIN +OPEN cur; +FETCH cur INTO vc; -- SHOW CODE should display vc with a "PACKAGE_BODY" prefix +CLOSE cur; +RETURN vc; +END; +BEGIN +DECLARE +CURSOR cur IS SELECT 1 AS c FROM DUAL; +BEGIN +OPEN cur; +FETCH cur INTO vc; -- SHOW CODE should display vc without a prefix +CLOSE cur; +END; +END; +$$ +SELECT pkg.f1() FROM DUAL; +pkg.f1() +1 +SHOW FUNCTION CODE pkg.f1; +Pos Instruction +0 cpush cur@0 +1 copen cur@0 +2 cfetch cur@0 PACKAGE_BODY.vc@0 +3 cclose cur@0 +4 freturn int PACKAGE_BODY.vc@0 +SHOW PACKAGE BODY CODE pkg; +Pos Instruction +0 set vc@0 0 +1 cpush cur@0 +2 copen cur@0 +3 cfetch cur@0 vc@0 +4 cclose cur@0 +5 cpop 1 +DROP PACKAGE pkg; +CREATE PACKAGE pkg AS +FUNCTION f1 RETURN TEXT; +END; +$$ +CREATE PACKAGE BODY pkg AS +vc1 INT := 0; +FUNCTION f1 RETURN TEXT AS +CURSOR cur IS SELECT 1 AS c1, 2 AS c2 FROM DUAL; +vc2 INT := 0; +BEGIN +OPEN cur; +FETCH cur INTO vc1, vc2; +CLOSE cur; +RETURN vc1 || ' ' || vc2; +END; +END; +$$ +SELECT pkg.f1() FROM DUAL; +pkg.f1() +1 2 +SHOW FUNCTION CODE pkg.f1; +Pos Instruction +0 set vc2@0 0 +1 cpush cur@0 +2 copen cur@0 +3 cfetch cur@0 PACKAGE_BODY.vc1@0 vc2@0 +4 cclose cur@0 +5 freturn blob concat(concat(PACKAGE_BODY.vc1@0,' '),vc2@0) +DROP PACKAGE pkg; +CREATE PACKAGE pkg AS +FUNCTION f1 RETURN TEXT; +END; +$$ +CREATE PACKAGE BODY pkg AS +vc ROW(p1 INT, p2 INT); +FUNCTION f1 RETURN TEXT AS +CURSOR cur IS SELECT 1 AS c1, 2 AS c2 FROM DUAL; +BEGIN +OPEN cur; +FETCH cur INTO vc; +CLOSE cur; +RETURN vc.p1 || ' ' || vc.p2; +END; +END; +$$ +SELECT pkg.f1() FROM DUAL; +pkg.f1() +1 2 +SHOW FUNCTION CODE pkg.f1; +Pos Instruction +0 cpush cur@0 +1 copen cur@0 +2 cfetch cur@0 PACKAGE_BODY.vc@0 +3 cclose cur@0 +4 freturn blob concat(concat(PACKAGE_BODY.vc.p1@0[0],' '),PACKAGE_BODY.vc.p2@0[1]) +DROP PACKAGE pkg; +# End of 11.4 tests diff --git a/mysql-test/suite/compat/oracle/t/sp-cursor-pkg-01.inc b/mysql-test/suite/compat/oracle/t/sp-cursor-pkg-01.inc new file mode 100644 index 00000000000..6e5aac0b254 --- /dev/null +++ b/mysql-test/suite/compat/oracle/t/sp-cursor-pkg-01.inc @@ -0,0 +1,26 @@ +DELIMITER $$; +CREATE PACKAGE pkg AS + FUNCTION f1 RETURN INT; +END; +$$ +CREATE PACKAGE BODY pkg AS + vc INT := 0; + FUNCTION f1 RETURN INT AS + CURSOR cur IS SELECT 1 AS c FROM DUAL; + BEGIN + OPEN cur; + FETCH cur INTO vc; -- SHOW CODE should display vc with a "PACKAGE_BODY" prefix + CLOSE cur; + RETURN vc; + END; +BEGIN + DECLARE + CURSOR cur IS SELECT 1 AS c FROM DUAL; + BEGIN + OPEN cur; + FETCH cur INTO vc; -- SHOW CODE should display vc without a prefix + CLOSE cur; + END; +END; +$$ +DELIMITER ;$$ diff --git a/mysql-test/suite/compat/oracle/t/sp-cursor-pkg-02.inc b/mysql-test/suite/compat/oracle/t/sp-cursor-pkg-02.inc new file mode 100644 index 00000000000..7a1eacee037 --- /dev/null +++ b/mysql-test/suite/compat/oracle/t/sp-cursor-pkg-02.inc @@ -0,0 +1,20 @@ +# Mixing a package body variable and a local variable in the same FETCH. +DELIMITER $$; +CREATE PACKAGE pkg AS + FUNCTION f1 RETURN TEXT; +END; +$$ +CREATE PACKAGE BODY pkg AS + vc1 INT := 0; + FUNCTION f1 RETURN TEXT AS + CURSOR cur IS SELECT 1 AS c1, 2 AS c2 FROM DUAL; + vc2 INT := 0; + BEGIN + OPEN cur; + FETCH cur INTO vc1, vc2; + CLOSE cur; + RETURN vc1 || ' ' || vc2; + END; +END; +$$ +DELIMITER ;$$ diff --git a/mysql-test/suite/compat/oracle/t/sp-cursor-pkg-03.inc b/mysql-test/suite/compat/oracle/t/sp-cursor-pkg-03.inc new file mode 100644 index 00000000000..e1c9fd6e8c5 --- /dev/null +++ b/mysql-test/suite/compat/oracle/t/sp-cursor-pkg-03.inc @@ -0,0 +1,19 @@ +# Fetching into a PACKAGE BODY variable of the ROW type +DELIMITER $$; +CREATE PACKAGE pkg AS + FUNCTION f1 RETURN TEXT; +END; +$$ +CREATE PACKAGE BODY pkg AS + vc ROW(p1 INT, p2 INT); + FUNCTION f1 RETURN TEXT AS + CURSOR cur IS SELECT 1 AS c1, 2 AS c2 FROM DUAL; + BEGIN + OPEN cur; + FETCH cur INTO vc; + CLOSE cur; + RETURN vc.p1 || ' ' || vc.p2; + END; +END; +$$ +DELIMITER ;$$ diff --git a/mysql-test/suite/compat/oracle/t/sp-cursor.test b/mysql-test/suite/compat/oracle/t/sp-cursor.test index d7e2a5dee68..eba00d24b8e 100644 --- a/mysql-test/suite/compat/oracle/t/sp-cursor.test +++ b/mysql-test/suite/compat/oracle/t/sp-cursor.test @@ -1042,3 +1042,24 @@ DELIMITER ;$$ --echo # --echo # End of 10.8 tests --echo # + + +--echo # Start of 11.4 tests + +--echo # +--echo # MDEV-36047 Package body variables are not allowed as FETCH targets +--echo # + +--source sp-cursor-pkg-01.inc +SELECT pkg.f1() FROM DUAL; +DROP PACKAGE pkg; + +--source sp-cursor-pkg-02.inc +SELECT pkg.f1() FROM DUAL; +DROP PACKAGE pkg; + +--source sp-cursor-pkg-03.inc +SELECT pkg.f1() FROM DUAL; +DROP PACKAGE pkg; + +--echo # End of 11.4 tests diff --git a/mysql-test/suite/compat/oracle/t/sp-package-code.test b/mysql-test/suite/compat/oracle/t/sp-package-code.test index 9cca53964ec..1aa2f7cba8b 100644 --- a/mysql-test/suite/compat/oracle/t/sp-package-code.test +++ b/mysql-test/suite/compat/oracle/t/sp-package-code.test @@ -180,3 +180,28 @@ SHOW PROCEDURE CODE pkg1.p1; SHOW PACKAGE BODY CODE pkg1; DROP PACKAGE pkg1; DROP TABLE t1; + + +--echo # Start of 11.4 tests + +--echo # +--echo # MDEV-36047 Package body variables are not allowed as FETCH targets +--echo # + +--source sp-cursor-pkg-01.inc +SELECT pkg.f1() FROM DUAL; +SHOW FUNCTION CODE pkg.f1; +SHOW PACKAGE BODY CODE pkg; +DROP PACKAGE pkg; + +--source sp-cursor-pkg-02.inc +SELECT pkg.f1() FROM DUAL; +SHOW FUNCTION CODE pkg.f1; +DROP PACKAGE pkg; + +--source sp-cursor-pkg-03.inc +SELECT pkg.f1() FROM DUAL; +SHOW FUNCTION CODE pkg.f1; +DROP PACKAGE pkg; + +--echo # End of 11.4 tests diff --git a/sql/sp_head.cc b/sql/sp_head.cc index aa51d212a8c..abbca16e9c7 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -3858,8 +3858,10 @@ bool sp_head::add_for_loop_open_cursor(THD *thd, sp_pcontext *spcont, spcont, coffset, false); if (instr_cfetch == NULL || add_instr(instr_cfetch)) return true; - instr_cfetch->add_to_varlist(index); - return false; + const sp_rcontext_addr raddr(&sp_rcontext_handler_local, index->offset); + sp_fetch_target *target= + new (thd->mem_root) sp_fetch_target(index->name, raddr); + return !target || instr_cfetch->add_to_fetch_target_list(target); } diff --git a/sql/sp_instr.cc b/sql/sp_instr.cc index 6b70199f513..7e403a24281 100644 --- a/sql/sp_instr.cc +++ b/sql/sp_instr.cc @@ -1923,7 +1923,7 @@ sp_instr_cfetch::execute(THD *thd, uint *nextp) Query_arena backup_arena; DBUG_ENTER("sp_instr_cfetch::execute"); - res= c ? c->fetch(thd, &m_varlist, m_error_on_no_data) : -1; + res= c ? c->fetch(thd, &m_fetch_target_list, m_error_on_no_data) : -1; *nextp= m_ip+1; DBUG_RETURN(res); @@ -1933,8 +1933,8 @@ sp_instr_cfetch::execute(THD *thd, uint *nextp) void sp_instr_cfetch::print(String *str) { - List_iterator_fast li(m_varlist); - sp_variable *pv; + List_iterator_fast li(m_fetch_target_list); + sp_fetch_target *pv; const LEX_CSTRING *cursor_name= m_ctx->find_cursor(m_cursor); /* cfetch name@offset vars... */ @@ -1953,12 +1953,14 @@ sp_instr_cfetch::print(String *str) str->qs_append(m_cursor); while ((pv= li++)) { - if (str->reserve(pv->name.length+SP_INSTR_UINT_MAXLEN+2)) + const LEX_CSTRING *prefix= pv->rcontext_handler()->get_name_prefix(); + if (str->reserve(pv->name.length+prefix->length+SP_INSTR_UINT_MAXLEN+2)) return; str->qs_append(' '); + str->qs_append(prefix); str->qs_append(&pv->name); str->qs_append('@'); - str->qs_append(pv->offset); + str->qs_append(pv->offset()); } } diff --git a/sql/sp_instr.h b/sql/sp_instr.h index 2717bc85c3b..a93f681e5d1 100644 --- a/sql/sp_instr.h +++ b/sql/sp_instr.h @@ -591,7 +591,8 @@ public: }; // class sp_instr_stmt : public sp_lex_instr -class sp_instr_set : public sp_lex_instr +class sp_instr_set : public sp_lex_instr, + public sp_rcontext_addr { sp_instr_set(const sp_instr_set &); /**< Prevent use of these */ void operator=(sp_instr_set &); @@ -603,8 +604,7 @@ public: LEX *lex, bool lex_resp, const LEX_CSTRING &expr_str) : sp_lex_instr(ip, ctx, lex, lex_resp), - m_rcontext_handler(rh), - m_offset(offset), + sp_rcontext_addr(rh, offset), m_value(val), m_expr_str(expr_str) {} @@ -651,8 +651,6 @@ protected: } sp_rcontext *get_rcontext(THD *thd) const; - const Sp_rcontext_handler *m_rcontext_handler; - uint m_offset; ///< Frame offset Item *m_value; private: @@ -1490,7 +1488,7 @@ public: m_cursor(c), m_error_on_no_data(error_on_no_data) { - m_varlist.empty(); + m_fetch_target_list.empty(); } virtual ~sp_instr_cfetch() = default; @@ -1499,14 +1497,19 @@ public: void print(String *str) override; - void add_to_varlist(sp_variable *var) + bool add_to_fetch_target_list(sp_fetch_target *target) { - m_varlist.push_back(var); + return m_fetch_target_list.push_back(target); + } + + void set_fetch_target_list(List *list) + { + m_fetch_target_list= *list; } private: uint m_cursor; - List m_varlist; + List m_fetch_target_list; bool m_error_on_no_data; public: diff --git a/sql/sp_pcontext.h b/sql/sp_pcontext.h index 71846ad4620..bf57e6f5b0e 100644 --- a/sql/sp_pcontext.h +++ b/sql/sp_pcontext.h @@ -87,6 +87,27 @@ public: uint *row_field_offset); }; + +/* + This class stores FETCH statement target variables: + FETCH cur INTO t1, t2, t2; + Targets can be: + - Local SP variables + - PACKAGE BODY variables +*/ +class sp_fetch_target: public Sql_alloc, + public sp_rcontext_addr +{ +public: + LEX_CSTRING name; + + sp_fetch_target(const LEX_CSTRING &name_arg, const sp_rcontext_addr &addr) + :sp_rcontext_addr(addr), + name(name_arg) + { } +}; + + /////////////////////////////////////////////////////////////////////////// /// This class represents an SQL/PSM label. Can refer to the identifier diff --git a/sql/sp_rcontext.cc b/sql/sp_rcontext.cc index 6c8111a134d..30dcab67600 100644 --- a/sql/sp_rcontext.cc +++ b/sql/sp_rcontext.cc @@ -788,7 +788,8 @@ void sp_cursor::destroy() } -int sp_cursor::fetch(THD *thd, List *vars, bool error_on_no_data) +int sp_cursor::fetch(THD *thd, List *vars, + bool error_on_no_data) { if (! server_side_cursor) { @@ -798,8 +799,7 @@ int sp_cursor::fetch(THD *thd, List *vars, bool error_on_no_data) } if (vars->elements != result.get_field_count() && (vars->elements != 1 || - result.get_field_count() != - thd->spcont->get_variable(vars->head()->offset)->cols())) + result.get_field_count() != thd->get_variable(*vars->head())->cols())) { my_message(ER_SP_WRONG_NO_OF_FETCH_ARGS, ER_THD(thd, ER_SP_WRONG_NO_OF_FETCH_ARGS), MYF(0)); @@ -866,11 +866,12 @@ int sp_cursor::Select_fetch_into_spvars::prepare(List &fields, bool sp_cursor::Select_fetch_into_spvars:: - send_data_to_variable_list(List &vars, List &items) + send_data_to_variable_list(List &vars, + List &items) { - List_iterator_fast spvar_iter(vars); + List_iterator_fast spvar_iter(vars); List_iterator_fast item_iter(items); - sp_variable *spvar; + sp_fetch_target *spvar; Item *item; /* Must be ensured by the caller */ @@ -882,7 +883,7 @@ bool sp_cursor::Select_fetch_into_spvars:: */ for (; spvar= spvar_iter++, item= item_iter++; ) { - if (thd->spcont->set_variable(thd, spvar->offset, &item)) + if (thd->get_rcontext(*spvar)->set_variable(thd, spvar->offset(), &item)) return true; } return false; @@ -891,7 +892,6 @@ bool sp_cursor::Select_fetch_into_spvars:: int sp_cursor::Select_fetch_into_spvars::send_data(List &items) { - Item *item; /* If we have only one variable in spvar_list, and this is a ROW variable, and the number of fields in the ROW variable matches the number of @@ -902,10 +902,15 @@ int sp_cursor::Select_fetch_into_spvars::send_data(List &items) we go through send_data_to_variable_list(). It will report an error on attempt to assign a scalar value to a ROW variable. */ - return spvar_list->elements == 1 && - (item= thd->spcont->get_variable(spvar_list->head()->offset)) && - item->type_handler() == &type_handler_row && - item->cols() == items.elements ? - thd->spcont->set_variable_row(thd, spvar_list->head()->offset, items) : - send_data_to_variable_list(*spvar_list, items); + if (m_fetch_target_list->elements == 1) + { + const sp_fetch_target *target= m_fetch_target_list->head(); + sp_rcontext *rctx= thd->get_rcontext(*target); + Item *item; + if ((item= rctx->get_variable(target->offset())) && + item->type_handler() == &type_handler_row && + item->cols() == items.elements) + return rctx->set_variable_row(thd, target->offset(), items); + } + return send_data_to_variable_list(*m_fetch_target_list, items); } diff --git a/sql/sql_class.cc b/sql/sql_class.cc index d429b66ba08..ab6bfd9cc3d 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -4435,6 +4435,18 @@ bool my_var_sp_row_field::set(THD *thd, Item *item) } +sp_rcontext *THD::get_rcontext(const sp_rcontext_addr &addr) +{ + return addr.rcontext_handler()->get_rcontext(spcont); +} + + +Item_field *THD::get_variable(const sp_rcontext_addr &addr) +{ + return get_rcontext(addr)->get_variable(addr.offset()); +} + + bool select_dumpvar::send_data_to_var_list(List &items) { DBUG_ENTER("select_dumpvar::send_data_to_var_list"); diff --git a/sql/sql_class.h b/sql/sql_class.h index d78f8a3e8a3..8cd757cfdaf 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -3985,6 +3985,9 @@ public: sp_rcontext *spcont; // SP runtime context + sp_rcontext *get_rcontext(const sp_rcontext_addr &addr); + Item_field *get_variable(const sp_rcontext_addr &addr); + /** number of name_const() substitutions, see sp_head.cc:subst_spvars() */ uint query_name_consts; @@ -6330,10 +6333,11 @@ private: /// FETCH INTO . class Select_fetch_into_spvars: public select_result_interceptor { - List *spvar_list; + List *m_fetch_target_list; uint field_count; bool m_view_structure_only; - bool send_data_to_variable_list(List &vars, List &items); + bool send_data_to_variable_list(List &vars, + List &items); public: Select_fetch_into_spvars(THD *thd_arg, bool view_structure_only) :select_result_interceptor(thd_arg), @@ -6342,11 +6346,14 @@ private: void reset(THD *thd_arg) { select_result_interceptor::reinit(thd_arg); - spvar_list= NULL; + m_fetch_target_list= NULL; field_count= 0; } uint get_field_count() { return field_count; } - void set_spvar_list(List *vars) { spvar_list= vars; } + void set_spvar_list(List *vars) + { + m_fetch_target_list= vars; + } bool send_eof() override { return FALSE; } int send_data(List &items) override; @@ -6376,7 +6383,7 @@ public: my_bool is_open() { return MY_TEST(server_side_cursor); } - int fetch(THD *, List *vars, bool error_on_no_data); + int fetch(THD *, List *vars, bool error_on_no_data); bool export_structure(THD *thd, Row_definition_list *list); diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 0131068a530..935fbdd2c4c 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -6556,6 +6556,21 @@ LEX::find_variable(const LEX_CSTRING *name, } +sp_fetch_target *LEX::make_fetch_target(THD *thd, const Lex_ident_sys_st &name) +{ + sp_pcontext *spc; + const Sp_rcontext_handler *rha; + sp_variable *spv= find_variable(&name, &spc, &rha); + if (unlikely(!spv)) + { + my_error(ER_SP_UNDECLARED_VAR, MYF(0), name.str); + return nullptr; + } + return new (thd->mem_root) sp_fetch_target(name, + sp_rcontext_addr(rha, spv->offset)); +} + + static bool is_new(const char *str) { return (str[0] == 'n' || str[0] == 'N') && @@ -7234,8 +7249,11 @@ bool LEX::sp_for_loop_cursor_iterate(THD *thd, const Lex_for_loop_st &loop) spcont, loop.m_cursor_offset, false); if (unlikely(instr == NULL) || unlikely(sphead->add_instr(instr))) return true; - instr->add_to_varlist(loop.m_index); - return false; + const sp_rcontext_addr raddr(&sp_rcontext_handler_local, + loop.m_index->offset); + sp_fetch_target *trg= + new (thd->mem_root) sp_fetch_target(loop.m_index->name, raddr); + return !trg || instr->add_to_fetch_target_list(trg); } @@ -9321,7 +9339,7 @@ int set_statement_var_if_exists(THD *thd, const char *var_name, } -bool LEX::sp_add_cfetch(THD *thd, const LEX_CSTRING *name) +sp_instr_cfetch *LEX::sp_add_instr_cfetch(THD *thd, const LEX_CSTRING *name) { uint offset; sp_instr_cfetch *i; @@ -9329,14 +9347,14 @@ bool LEX::sp_add_cfetch(THD *thd, const LEX_CSTRING *name) if (!spcont->find_cursor(name, &offset, false)) { my_error(ER_SP_CURSOR_MISMATCH, MYF(0), name->str); - return true; + return nullptr; } i= new (thd->mem_root) sp_instr_cfetch(sphead->instructions(), spcont, offset, !(thd->variables.sql_mode & MODE_ORACLE)); if (unlikely(i == NULL) || unlikely(sphead->add_instr(i))) - return true; - return false; + return nullptr; + return i; } diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 4d8fb5eceb6..2979d976d8e 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -294,8 +294,10 @@ class LEX_COLUMN; class sp_head; class sp_name; class sp_instr; +class sp_instr_cfetch; class sp_pcontext; class sp_variable; +class sp_fetch_target; class sp_expr_lex; class sp_assignment_lex; class partition_info; @@ -3994,6 +3996,7 @@ public: sp_pcontext *not_used_ctx; return find_variable(name, ¬_used_ctx, rh); } + sp_fetch_target *make_fetch_target(THD *thd, const Lex_ident_sys_st &name); bool set_variable(const Lex_ident_sys_st *name, Item *item, const LEX_CSTRING &expr_str); bool set_variable(const Lex_ident_sys_st *name1, @@ -4611,7 +4614,7 @@ public: create_info.add(options); return check_create_options(create_info); } - bool sp_add_cfetch(THD *thd, const LEX_CSTRING *name); + sp_instr_cfetch *sp_add_instr_cfetch(THD *thd, const LEX_CSTRING *name); bool sp_add_agg_cfetch(); bool set_command_with_check(enum_sql_command command, diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 1f868846f00..d83cd8d2a41 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -253,6 +253,7 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() Item_basic_constant *item_basic_constant; Key_part_spec *key_part; LEX *lex; + sp_instr_cfetch *instr_cfetch; sp_expr_lex *expr_lex; sp_assignment_lex *assignment_lex; class sp_lex_cursor *sp_cursor_stmt; @@ -266,6 +267,7 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() List *stmt_info_list; List *string_list; List *ident_sys_list; + List *fetch_target_list; Statement_information_item *stmt_info_item; String *string; TABLE_LIST *table_list; @@ -1566,6 +1568,12 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); sp_cursor_stmt_lex sp_cursor_stmt +%type + sp_proc_stmt_fetch_head + +%type + sp_fetch_list + %type expr_lex @@ -4146,23 +4154,26 @@ sp_proc_stmt_open: sp_proc_stmt_fetch_head: FETCH_SYM ident INTO { - if (unlikely(Lex->sp_add_cfetch(thd, &$2))) + if (unlikely(!($$= Lex->sp_add_instr_cfetch(thd, &$2)))) MYSQL_YYABORT; } | FETCH_SYM FROM ident INTO { - if (unlikely(Lex->sp_add_cfetch(thd, &$3))) + if (unlikely(!($$= Lex->sp_add_instr_cfetch(thd, &$3)))) MYSQL_YYABORT; } | FETCH_SYM NEXT_SYM FROM ident INTO { - if (unlikely(Lex->sp_add_cfetch(thd, &$4))) + if (unlikely(!($$= Lex->sp_add_instr_cfetch(thd, &$4)))) MYSQL_YYABORT; } ; sp_proc_stmt_fetch: - sp_proc_stmt_fetch_head sp_fetch_list { } + sp_proc_stmt_fetch_head sp_fetch_list + { + $1->set_fetch_target_list($2); + } | FETCH_SYM GROUP_SYM NEXT_SYM ROW_SYM { if (unlikely(Lex->sp_add_agg_cfetch())) @@ -4191,35 +4202,16 @@ sp_proc_stmt_close: sp_fetch_list: ident { - LEX *lex= Lex; - sp_head *sp= lex->sphead; - sp_pcontext *spc= lex->spcont; - sp_variable *spv= likely(spc != NULL) - ? spc->find_variable(&$1, false) - : NULL; - - if (unlikely(!spv)) - my_yyabort_error((ER_SP_UNDECLARED_VAR, MYF(0), $1.str)); - - /* An SP local variable */ - sp_instr_cfetch *i= (sp_instr_cfetch *)sp->last_instruction(); - i->add_to_varlist(spv); + sp_fetch_target *target= Lex->make_fetch_target(thd, $1); + if (!target || + !($$= List::make(thd->mem_root, target))) + MYSQL_YYABORT; } | sp_fetch_list ',' ident { - LEX *lex= Lex; - sp_head *sp= lex->sphead; - sp_pcontext *spc= lex->spcont; - sp_variable *spv= likely(spc != NULL) - ? spc->find_variable(&$3, false) - : NULL; - - if (unlikely(!spv)) - my_yyabort_error((ER_SP_UNDECLARED_VAR, MYF(0), $3.str)); - - /* An SP local variable */ - sp_instr_cfetch *i= (sp_instr_cfetch *)sp->last_instruction(); - i->add_to_varlist(spv); + sp_fetch_target *target= Lex->make_fetch_target(thd, $3); + if (!target || $1->push_back(target, thd->mem_root)) + MYSQL_YYABORT; } ; diff --git a/sql/structs.h b/sql/structs.h index 8da5d2507f5..ada7a5b77c2 100644 --- a/sql/structs.h +++ b/sql/structs.h @@ -1060,4 +1060,29 @@ public: }; +/* + A run-time address of an SP variable. Consists of: + - The rcontext type (LOCAL, PACKAGE BODY), + controlled by m_rcontext_handler + - The frame offset +*/ +class sp_rcontext_addr +{ +public: + sp_rcontext_addr(const class Sp_rcontext_handler *h, uint offset) + :m_rcontext_handler(h), m_offset(offset) + { } + const Sp_rcontext_handler *rcontext_handler() const + { + return m_rcontext_handler; + } + uint offset() const + { + return m_offset; + } +protected: + const class Sp_rcontext_handler *m_rcontext_handler; + uint m_offset; ///< Frame offset +}; + #endif /* STRUCTS_INCLUDED */ From cd03bf5c532e7276c22827ead69dd5bcfd1242a0 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 4 Feb 2025 21:35:55 +0200 Subject: [PATCH 24/47] Fixed costs in JOIN_TAB::estimate_scan_time() and HEAP MDEV-35958 Cost estimates for materialized derived tables are poor (Backport 11.8->11.4, the same patch) Estimate_scan_time() calculates the cost of scanning a derivied table. The old code did not take into account that the temporary table heap table may be converted to Aria. Things fixed: - Added checking if the temporary tables data will fit in the heap. If not, then calculate the cost based on the designated internal temporary table engine (Aria). - Removed MY_MAX(records, 1000) and instead trust the optimizer's estimate of records. This reduces the cost of temporary tables a bit for small tables, which caused a few changes in mtr results. - Fixed cost calculation for HEAP. - HEAP costs->row_next_find_cost was not set. This does not affect old costs calculation as this cost slot was not used anywhere. Now HEAP cost->row_next_find_cost is set, which allowed me to remove some duplicated computation in ha_heap::scan_time() Reviewed by: Sergei Petrunia --- mysql-test/main/cte_recursive.result | 31 +- mysql-test/main/derived_cond_pushdown.result | 335 +++++++++---------- mysql-test/main/opt_trace.result | 100 ++++-- mysql-test/main/opt_tvc.result | 6 +- mysql-test/main/optimizer_costs.result | 2 +- mysql-test/main/sp.result | 8 +- mysql-test/main/subselect_sj.result | 6 +- mysql-test/main/subselect_sj_jcl6.result | 8 +- sql/sql_select.cc | 54 ++- storage/heap/ha_heap.cc | 6 +- 10 files changed, 317 insertions(+), 239 deletions(-) diff --git a/mysql-test/main/cte_recursive.result b/mysql-test/main/cte_recursive.result index dcd36362245..ab23d9574d7 100644 --- a/mysql-test/main/cte_recursive.result +++ b/mysql-test/main/cte_recursive.result @@ -3913,8 +3913,8 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 3 Using where 1 PRIMARY ref key0 key0 23 test.t1.a1 1 FirstMatch(t1) 3 DERIVED t2 const PRIMARY PRIMARY 22 const 1 Using index -4 RECURSIVE UNION tt2 ALL b1 NULL NULL NULL 14 Using where -4 RECURSIVE UNION ref key0 key0 23 test.tt2.b1 1 +4 RECURSIVE UNION ALL NULL NULL NULL NULL 2 Using where +4 RECURSIVE UNION tt2 ref b1 b1 23 cte.a2 1 NULL UNION RESULT ALL NULL NULL NULL NULL NULL analyze format=json select fv from (select t1.a1, f1(t1.a2) fv from t1) dt @@ -4013,38 +4013,37 @@ ANALYZE "nested_loop": [ { "table": { - "table_name": "tt2", + "table_name": "", "access_type": "ALL", - "possible_keys": ["b1"], "loops": 1, "r_loops": 1, - "rows": 14, - "r_rows": 14, + "rows": 2, + "r_rows": 1, "cost": "REPLACED", "r_table_time_ms": "REPLACED", "r_other_time_ms": "REPLACED", - "r_engine_stats": REPLACED, "filtered": 100, "r_filtered": 100, - "attached_condition": "tt2.b1 is not null" + "attached_condition": "cte.a2 is not null" } }, { "table": { - "table_name": "", + "table_name": "tt2", "access_type": "ref", - "possible_keys": ["key0"], - "key": "key0", + "possible_keys": ["b1"], + "key": "b1", "key_length": "23", - "used_key_parts": ["a2"], - "ref": ["test.tt2.b1"], - "loops": 14, - "r_loops": 14, + "used_key_parts": ["b1"], + "ref": ["cte.a2"], + "loops": 2, + "r_loops": 1, "rows": 1, - "r_rows": 0.071428571, + "r_rows": 1, "cost": "REPLACED", "r_table_time_ms": "REPLACED", "r_other_time_ms": "REPLACED", + "r_engine_stats": REPLACED, "filtered": 100, "r_filtered": 100 } diff --git a/mysql-test/main/derived_cond_pushdown.result b/mysql-test/main/derived_cond_pushdown.result index c1e4c94f193..6877acba92d 100644 --- a/mysql-test/main/derived_cond_pushdown.result +++ b/mysql-test/main/derived_cond_pushdown.result @@ -3634,42 +3634,42 @@ set statement optimizer_switch='condition_pushdown_for_derived=off' for select * ((v1.a=v2.a) or (v1.a=t2.a)) and (t2.b<50) and (v1.b=v2.b); a b max_c avg_c a b max_c avg_c a b c d 6 20 315 279.3333 6 20 315 279.3333 2 3 207 207 -6 20 315 279.3333 6 20 315 279.3333 1 21 909 12 -6 20 315 279.3333 6 20 315 279.3333 7 13 312 406 -6 20 315 279.3333 6 20 315 279.3333 6 20 315 279 -6 20 315 279.3333 6 20 315 279.3333 1 19 203 107 -6 20 315 279.3333 6 20 315 279.3333 3 12 231 190 -6 20 315 279.3333 6 20 315 279.3333 6 23 303 909 8 33 404 213.6667 8 33 404 213.6667 2 3 207 207 +6 20 315 279.3333 6 20 315 279.3333 1 21 909 12 8 33 404 213.6667 8 33 404 213.6667 1 21 909 12 +6 20 315 279.3333 6 20 315 279.3333 7 13 312 406 8 33 404 213.6667 8 33 404 213.6667 7 13 312 406 +6 20 315 279.3333 6 20 315 279.3333 6 20 315 279 8 33 404 213.6667 8 33 404 213.6667 6 20 315 279 +6 20 315 279.3333 6 20 315 279.3333 1 19 203 107 8 33 404 213.6667 8 33 404 213.6667 1 19 203 107 +6 20 315 279.3333 6 20 315 279.3333 3 12 231 190 8 33 404 213.6667 8 33 404 213.6667 3 12 231 190 +6 20 315 279.3333 6 20 315 279.3333 6 23 303 909 8 33 404 213.6667 8 33 404 213.6667 6 23 303 909 select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and (t2.b<50) and (v1.b=v2.b); a b max_c avg_c a b max_c avg_c a b c d 6 20 315 279.3333 6 20 315 279.3333 2 3 207 207 -6 20 315 279.3333 6 20 315 279.3333 1 21 909 12 -6 20 315 279.3333 6 20 315 279.3333 7 13 312 406 -6 20 315 279.3333 6 20 315 279.3333 6 20 315 279 -6 20 315 279.3333 6 20 315 279.3333 1 19 203 107 -6 20 315 279.3333 6 20 315 279.3333 3 12 231 190 -6 20 315 279.3333 6 20 315 279.3333 6 23 303 909 8 33 404 213.6667 8 33 404 213.6667 2 3 207 207 +6 20 315 279.3333 6 20 315 279.3333 1 21 909 12 8 33 404 213.6667 8 33 404 213.6667 1 21 909 12 +6 20 315 279.3333 6 20 315 279.3333 7 13 312 406 8 33 404 213.6667 8 33 404 213.6667 7 13 312 406 +6 20 315 279.3333 6 20 315 279.3333 6 20 315 279 8 33 404 213.6667 8 33 404 213.6667 6 20 315 279 +6 20 315 279.3333 6 20 315 279.3333 1 19 203 107 8 33 404 213.6667 8 33 404 213.6667 1 19 203 107 +6 20 315 279.3333 6 20 315 279.3333 3 12 231 190 8 33 404 213.6667 8 33 404 213.6667 3 12 231 190 +6 20 315 279.3333 6 20 315 279.3333 6 23 303 909 8 33 404 213.6667 8 33 404 213.6667 6 23 303 909 explain select * from v1,v2,t2 where ((v1.a=v2.a) or (v1.a=t2.a)) and (t2.b<50) and (v1.b=v2.b); id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where -1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) -1 PRIMARY ref key0 key0 5 v1.b 2 Using where +1 PRIMARY ALL NULL NULL NULL NULL 20 Using where +1 PRIMARY ref key0 key0 5 v1.b 2 +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where; Using join buffer (flat, BNL join) 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort explain format=json select * from v1,v2,t2 where @@ -3682,28 +3682,12 @@ EXPLAIN "nested_loop": [ { "table": { - "table_name": "t2", + "table_name": "", "access_type": "ALL", "loops": 1, - "rows": 9, + "rows": 20, "cost": "COST_REPLACED", "filtered": 100, - "attached_condition": "t2.b < 50" - } - }, - { - "block-nl-join": { - "table": { - "table_name": "", - "access_type": "ALL", - "loops": 9, - "rows": 20, - "cost": "COST_REPLACED", - "filtered": 100 - }, - "buffer_type": "flat", - "buffer_size": "238", - "join_type": "BNL", "attached_condition": "v1.b is not null", "materialized": { "query_block": { @@ -3740,11 +3724,10 @@ EXPLAIN "key_length": "5", "used_key_parts": ["b"], "ref": ["v1.b"], - "loops": 180, + "loops": 20, "rows": 2, "cost": "COST_REPLACED", "filtered": 100, - "attached_condition": "v2.a = v1.a or v1.a = t2.a", "materialized": { "query_block": { "select_id": 3, @@ -3771,6 +3754,23 @@ EXPLAIN } } } + }, + { + "block-nl-join": { + "table": { + "table_name": "t2", + "access_type": "ALL", + "loops": 40, + "rows": 9, + "cost": "COST_REPLACED", + "filtered": 100, + "attached_condition": "t2.b < 50" + }, + "buffer_type": "flat", + "buffer_size": "1Kb", + "join_type": "BNL", + "attached_condition": "v2.a = v1.a or t2.a = v1.a" + } } ] } @@ -4158,9 +4158,9 @@ select * from v1,v2,t2 where (v1.a=1) and (v1.b>10) and (v1.b=v2.b); a b max_c avg_c a b max_c avg_c a b c d explain select * from v1,v2,t2 where (v1.a=1) and (v1.b>10) and (v1.b=v2.b); id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t2 ALL NULL NULL NULL NULL 9 -1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) +1 PRIMARY ALL NULL NULL NULL NULL 20 Using where 1 PRIMARY ref key0 key0 5 v1.b 2 +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using join buffer (flat, BNL join) 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort explain format=json select * from v1,v2,t2 where (v1.a=1) and (v1.b>10) and (v1.b=v2.b); @@ -4172,29 +4172,13 @@ EXPLAIN "nested_loop": [ { "table": { - "table_name": "t2", + "table_name": "", "access_type": "ALL", "loops": 1, - "rows": 9, + "rows": 20, "cost": "COST_REPLACED", - "filtered": 100 - } - }, - { - "block-nl-join": { - "table": { - "table_name": "", - "access_type": "ALL", - "loops": 9, - "rows": 20, - "cost": "COST_REPLACED", - "filtered": 100, - "attached_condition": "v1.a = 1 and v1.b > 10" - }, - "buffer_type": "flat", - "buffer_size": "238", - "join_type": "BNL", - "attached_condition": "v1.b is not null", + "filtered": 100, + "attached_condition": "v1.a = 1 and v1.b > 10 and v1.b is not null", "materialized": { "query_block": { "select_id": 2, @@ -4231,7 +4215,7 @@ EXPLAIN "key_length": "5", "used_key_parts": ["b"], "ref": ["v1.b"], - "loops": 180, + "loops": 20, "rows": 2, "cost": "COST_REPLACED", "filtered": 100, @@ -4261,6 +4245,21 @@ EXPLAIN } } } + }, + { + "block-nl-join": { + "table": { + "table_name": "t2", + "access_type": "ALL", + "loops": 40, + "rows": 9, + "cost": "COST_REPLACED", + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "1Kb", + "join_type": "BNL" + } } ] } @@ -4737,10 +4736,10 @@ where t1.a<8 group by a,b) v3, t2 where (v1.a=v2.a) and (v1.b=v3.b) and ((v3.avg_c>170) or (v3.a<5)) and ((v1.avg_c<400) or (v1.a>1)) and (v2.min_c<200); id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t2 ALL NULL NULL NULL NULL 9 -1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) +1 PRIMARY ALL NULL NULL NULL NULL 20 Using where 1 PRIMARY ref key0 key0 5 v1.a 2 Using where 1 PRIMARY ref key0 key0 5 v1.b 2 Using where +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using join buffer (flat, BNL join) 4 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort @@ -4761,28 +4760,12 @@ EXPLAIN "nested_loop": [ { "table": { - "table_name": "t2", + "table_name": "", "access_type": "ALL", "loops": 1, - "rows": 9, + "rows": 20, "cost": "COST_REPLACED", - "filtered": 100 - } - }, - { - "block-nl-join": { - "table": { - "table_name": "", - "access_type": "ALL", - "loops": 9, - "rows": 20, - "cost": "COST_REPLACED", - "filtered": 100, - "attached_condition": "v1.avg_c < 400 or v1.a > 1" - }, - "buffer_type": "flat", - "buffer_size": "238", - "join_type": "BNL", + "filtered": 100, "attached_condition": "(v1.avg_c < 400 or v1.a > 1) and v1.a is not null and v1.b is not null", "materialized": { "query_block": { @@ -4820,7 +4803,7 @@ EXPLAIN "key_length": "5", "used_key_parts": ["a"], "ref": ["v1.a"], - "loops": 180, + "loops": 20, "rows": 2, "cost": "COST_REPLACED", "filtered": 100, @@ -4861,7 +4844,7 @@ EXPLAIN "key_length": "5", "used_key_parts": ["b"], "ref": ["v1.b"], - "loops": 360, + "loops": 40, "rows": 2, "cost": "COST_REPLACED", "filtered": 100, @@ -4892,6 +4875,21 @@ EXPLAIN } } } + }, + { + "block-nl-join": { + "table": { + "table_name": "t2", + "access_type": "ALL", + "loops": 80, + "rows": 9, + "cost": "COST_REPLACED", + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "4Kb", + "join_type": "BNL" + } } ] } @@ -7796,104 +7794,104 @@ set statement optimizer_switch='condition_pushdown_for_derived=off' for select * (((v4.b>10) and (v4.a>1)) or (v4.b<20)) and (v1.max_c>200) and (v1.a=v4.a); a b min_c a b max_c avg_c a b c d 1 19 107 1 21 500 234.6000 2 3 207 207 -1 19 107 1 21 500 234.6000 1 21 909 12 -1 19 107 1 21 500 234.6000 7 13 312 406 -1 19 107 1 21 500 234.6000 8 64 248 107 -1 19 107 1 21 500 234.6000 6 20 315 279 -1 19 107 1 21 500 234.6000 1 19 203 107 -1 19 107 1 21 500 234.6000 8 80 800 314 -1 19 107 1 21 500 234.6000 3 12 231 190 -1 19 107 1 21 500 234.6000 6 23 303 909 5 16 207 5 16 207 207.0000 2 3 207 207 -5 16 207 5 16 207 207.0000 1 21 909 12 -5 16 207 5 16 207 207.0000 7 13 312 406 -5 16 207 5 16 207 207.0000 8 64 248 107 -5 16 207 5 16 207 207.0000 6 20 315 279 -5 16 207 5 16 207 207.0000 1 19 203 107 -5 16 207 5 16 207 207.0000 8 80 800 314 -5 16 207 5 16 207 207.0000 3 12 231 190 -5 16 207 5 16 207 207.0000 6 23 303 909 5 27 132 5 16 207 207.0000 2 3 207 207 -5 27 132 5 16 207 207.0000 1 21 909 12 -5 27 132 5 16 207 207.0000 7 13 312 406 -5 27 132 5 16 207 207.0000 8 64 248 107 -5 27 132 5 16 207 207.0000 6 20 315 279 -5 27 132 5 16 207 207.0000 1 19 203 107 -5 27 132 5 16 207 207.0000 8 80 800 314 -5 27 132 5 16 207 207.0000 3 12 231 190 -5 27 132 5 16 207 207.0000 6 23 303 909 6 20 315 6 20 315 279.3333 2 3 207 207 -6 20 315 6 20 315 279.3333 1 21 909 12 -6 20 315 6 20 315 279.3333 7 13 312 406 -6 20 315 6 20 315 279.3333 8 64 248 107 -6 20 315 6 20 315 279.3333 6 20 315 279 -6 20 315 6 20 315 279.3333 1 19 203 107 -6 20 315 6 20 315 279.3333 8 80 800 314 -6 20 315 6 20 315 279.3333 3 12 231 190 -6 20 315 6 20 315 279.3333 6 23 303 909 8 33 404 8 33 404 213.6667 2 3 207 207 +1 19 107 1 21 500 234.6000 1 21 909 12 +5 16 207 5 16 207 207.0000 1 21 909 12 +5 27 132 5 16 207 207.0000 1 21 909 12 +6 20 315 6 20 315 279.3333 1 21 909 12 8 33 404 8 33 404 213.6667 1 21 909 12 +1 19 107 1 21 500 234.6000 7 13 312 406 +5 16 207 5 16 207 207.0000 7 13 312 406 +5 27 132 5 16 207 207.0000 7 13 312 406 +6 20 315 6 20 315 279.3333 7 13 312 406 8 33 404 8 33 404 213.6667 7 13 312 406 +1 19 107 1 21 500 234.6000 8 64 248 107 +5 16 207 5 16 207 207.0000 8 64 248 107 +5 27 132 5 16 207 207.0000 8 64 248 107 +6 20 315 6 20 315 279.3333 8 64 248 107 8 33 404 8 33 404 213.6667 8 64 248 107 +1 19 107 1 21 500 234.6000 6 20 315 279 +5 16 207 5 16 207 207.0000 6 20 315 279 +5 27 132 5 16 207 207.0000 6 20 315 279 +6 20 315 6 20 315 279.3333 6 20 315 279 8 33 404 8 33 404 213.6667 6 20 315 279 +1 19 107 1 21 500 234.6000 1 19 203 107 +5 16 207 5 16 207 207.0000 1 19 203 107 +5 27 132 5 16 207 207.0000 1 19 203 107 +6 20 315 6 20 315 279.3333 1 19 203 107 8 33 404 8 33 404 213.6667 1 19 203 107 +1 19 107 1 21 500 234.6000 8 80 800 314 +5 16 207 5 16 207 207.0000 8 80 800 314 +5 27 132 5 16 207 207.0000 8 80 800 314 +6 20 315 6 20 315 279.3333 8 80 800 314 8 33 404 8 33 404 213.6667 8 80 800 314 +1 19 107 1 21 500 234.6000 3 12 231 190 +5 16 207 5 16 207 207.0000 3 12 231 190 +5 27 132 5 16 207 207.0000 3 12 231 190 +6 20 315 6 20 315 279.3333 3 12 231 190 8 33 404 8 33 404 213.6667 3 12 231 190 +1 19 107 1 21 500 234.6000 6 23 303 909 +5 16 207 5 16 207 207.0000 6 23 303 909 +5 27 132 5 16 207 207.0000 6 23 303 909 +6 20 315 6 20 315 279.3333 6 23 303 909 8 33 404 8 33 404 213.6667 6 23 303 909 select * from v4,v1,t2 where (((v4.b>10) and (v4.a>1)) or (v4.b<20)) and (v1.max_c>200) and (v1.a=v4.a); a b min_c a b max_c avg_c a b c d 1 19 107 1 21 500 234.6000 2 3 207 207 -1 19 107 1 21 500 234.6000 1 21 909 12 -1 19 107 1 21 500 234.6000 7 13 312 406 -1 19 107 1 21 500 234.6000 8 64 248 107 -1 19 107 1 21 500 234.6000 6 20 315 279 -1 19 107 1 21 500 234.6000 1 19 203 107 -1 19 107 1 21 500 234.6000 8 80 800 314 -1 19 107 1 21 500 234.6000 3 12 231 190 -1 19 107 1 21 500 234.6000 6 23 303 909 5 16 207 5 16 207 207.0000 2 3 207 207 -5 16 207 5 16 207 207.0000 1 21 909 12 -5 16 207 5 16 207 207.0000 7 13 312 406 -5 16 207 5 16 207 207.0000 8 64 248 107 -5 16 207 5 16 207 207.0000 6 20 315 279 -5 16 207 5 16 207 207.0000 1 19 203 107 -5 16 207 5 16 207 207.0000 8 80 800 314 -5 16 207 5 16 207 207.0000 3 12 231 190 -5 16 207 5 16 207 207.0000 6 23 303 909 5 27 132 5 16 207 207.0000 2 3 207 207 -5 27 132 5 16 207 207.0000 1 21 909 12 -5 27 132 5 16 207 207.0000 7 13 312 406 -5 27 132 5 16 207 207.0000 8 64 248 107 -5 27 132 5 16 207 207.0000 6 20 315 279 -5 27 132 5 16 207 207.0000 1 19 203 107 -5 27 132 5 16 207 207.0000 8 80 800 314 -5 27 132 5 16 207 207.0000 3 12 231 190 -5 27 132 5 16 207 207.0000 6 23 303 909 6 20 315 6 20 315 279.3333 2 3 207 207 -6 20 315 6 20 315 279.3333 1 21 909 12 -6 20 315 6 20 315 279.3333 7 13 312 406 -6 20 315 6 20 315 279.3333 8 64 248 107 -6 20 315 6 20 315 279.3333 6 20 315 279 -6 20 315 6 20 315 279.3333 1 19 203 107 -6 20 315 6 20 315 279.3333 8 80 800 314 -6 20 315 6 20 315 279.3333 3 12 231 190 -6 20 315 6 20 315 279.3333 6 23 303 909 8 33 404 8 33 404 213.6667 2 3 207 207 +1 19 107 1 21 500 234.6000 1 21 909 12 +5 16 207 5 16 207 207.0000 1 21 909 12 +5 27 132 5 16 207 207.0000 1 21 909 12 +6 20 315 6 20 315 279.3333 1 21 909 12 8 33 404 8 33 404 213.6667 1 21 909 12 +1 19 107 1 21 500 234.6000 7 13 312 406 +5 16 207 5 16 207 207.0000 7 13 312 406 +5 27 132 5 16 207 207.0000 7 13 312 406 +6 20 315 6 20 315 279.3333 7 13 312 406 8 33 404 8 33 404 213.6667 7 13 312 406 +1 19 107 1 21 500 234.6000 8 64 248 107 +5 16 207 5 16 207 207.0000 8 64 248 107 +5 27 132 5 16 207 207.0000 8 64 248 107 +6 20 315 6 20 315 279.3333 8 64 248 107 8 33 404 8 33 404 213.6667 8 64 248 107 +1 19 107 1 21 500 234.6000 6 20 315 279 +5 16 207 5 16 207 207.0000 6 20 315 279 +5 27 132 5 16 207 207.0000 6 20 315 279 +6 20 315 6 20 315 279.3333 6 20 315 279 8 33 404 8 33 404 213.6667 6 20 315 279 +1 19 107 1 21 500 234.6000 1 19 203 107 +5 16 207 5 16 207 207.0000 1 19 203 107 +5 27 132 5 16 207 207.0000 1 19 203 107 +6 20 315 6 20 315 279.3333 1 19 203 107 8 33 404 8 33 404 213.6667 1 19 203 107 +1 19 107 1 21 500 234.6000 8 80 800 314 +5 16 207 5 16 207 207.0000 8 80 800 314 +5 27 132 5 16 207 207.0000 8 80 800 314 +6 20 315 6 20 315 279.3333 8 80 800 314 8 33 404 8 33 404 213.6667 8 80 800 314 +1 19 107 1 21 500 234.6000 3 12 231 190 +5 16 207 5 16 207 207.0000 3 12 231 190 +5 27 132 5 16 207 207.0000 3 12 231 190 +6 20 315 6 20 315 279.3333 3 12 231 190 8 33 404 8 33 404 213.6667 3 12 231 190 +1 19 107 1 21 500 234.6000 6 23 303 909 +5 16 207 5 16 207 207.0000 6 23 303 909 +5 27 132 5 16 207 207.0000 6 23 303 909 +6 20 315 6 20 315 279.3333 6 23 303 909 8 33 404 8 33 404 213.6667 6 23 303 909 explain select * from v4,v1,t2 where (((v4.b>10) and (v4.a>1)) or (v4.b<20)) and (v1.max_c>200) and (v1.a=v4.a); id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t2 ALL NULL NULL NULL NULL 9 -1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) +1 PRIMARY ALL NULL NULL NULL NULL 20 Using where 1 PRIMARY ref key0 key0 5 v4.a 2 Using where +1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using join buffer (flat, BNL join) 4 DERIVED t1 ALL NULL NULL NULL NULL 20 Using temporary; Using filesort 2 DERIVED ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort 3 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort @@ -7907,28 +7905,12 @@ EXPLAIN "nested_loop": [ { "table": { - "table_name": "t2", + "table_name": "", "access_type": "ALL", "loops": 1, - "rows": 9, + "rows": 20, "cost": "COST_REPLACED", - "filtered": 100 - } - }, - { - "block-nl-join": { - "table": { - "table_name": "", - "access_type": "ALL", - "loops": 9, - "rows": 20, - "cost": "COST_REPLACED", - "filtered": 100, - "attached_condition": "v4.b > 10 and v4.a > 1 or v4.b < 20" - }, - "buffer_type": "flat", - "buffer_size": "238", - "join_type": "BNL", + "filtered": 100, "attached_condition": "(v4.b > 10 and v4.a > 1 or v4.b < 20) and v4.a is not null", "materialized": { "query_block": { @@ -7990,7 +7972,7 @@ EXPLAIN "key_length": "5", "used_key_parts": ["a"], "ref": ["v4.a"], - "loops": 180, + "loops": 20, "rows": 2, "cost": "COST_REPLACED", "filtered": 100, @@ -8020,6 +8002,21 @@ EXPLAIN } } } + }, + { + "block-nl-join": { + "table": { + "table_name": "t2", + "access_type": "ALL", + "loops": 40, + "rows": 9, + "cost": "COST_REPLACED", + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "1Kb", + "join_type": "BNL" + } } ] } @@ -22667,7 +22664,7 @@ EXPLAIN { "query_block": { "select_id": 1, - "cost": 0.012461052, + "cost": 0.010134054, "nested_loop": [ { "table": { @@ -22675,7 +22672,7 @@ EXPLAIN "access_type": "ALL", "loops": 1, "rows": 3, - "cost": 0.012461052, + "cost": 0.010134054, "filtered": 100, "attached_condition": "v1.r < 'x'", "materialized": { diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result index 44d5c2896cd..4c47741ef03 100644 --- a/mysql-test/main/opt_trace.result +++ b/mysql-test/main/opt_trace.result @@ -566,8 +566,8 @@ select * from v2 { "table": "", "table_scan": { "rows": 2, - "read_cost": 0.012350033, - "read_and_compare_cost": 0.012418701 + "read_cost": 0.010020701, + "read_and_compare_cost": 0.010089369 } } ] @@ -589,7 +589,7 @@ select * from v2 { "rows": 2, "rows_after_filter": 2, "rows_out": 2, - "cost": 0.012418701, + "cost": 0.010089369, "index_only": false, "chosen": true } @@ -598,7 +598,7 @@ select * from v2 { "type": "scan", "rows_read": 2, "rows_out": 2, - "cost": 0.012418701, + "cost": 0.010089369, "uses_join_buffering": false } } @@ -609,14 +609,14 @@ select * from v2 { "plan_prefix": "", "table": "", "rows_for_plan": 2, - "cost_for_plan": 0.012418701 + "cost_for_plan": 0.010089369 } ] }, { "best_join_order": [""], "rows": 2, - "cost": 0.012418701 + "cost": 0.010089369 }, { "attaching_conditions_to_tables": { @@ -954,8 +954,8 @@ explain select * from v1 { "table": "", "table_scan": { "rows": 10, - "read_cost": 0.012414166, - "read_and_compare_cost": 0.012757506 + "read_cost": 0.010103506, + "read_and_compare_cost": 0.010446846 } } ] @@ -977,7 +977,7 @@ explain select * from v1 { "rows": 10, "rows_after_filter": 10, "rows_out": 10, - "cost": 0.012757506, + "cost": 0.010446846, "index_only": false, "chosen": true } @@ -986,7 +986,7 @@ explain select * from v1 { "type": "scan", "rows_read": 10, "rows_out": 10, - "cost": 0.012757506, + "cost": 0.010446846, "uses_join_buffering": false } } @@ -997,14 +997,14 @@ explain select * from v1 { "plan_prefix": "", "table": "", "rows_for_plan": 10, - "cost_for_plan": 0.012757506 + "cost_for_plan": 0.010446846 } ] }, { "best_join_order": [""], "rows": 10, - "cost": 0.012757506 + "cost": 0.010446846 }, { "attaching_conditions_to_tables": { @@ -4974,8 +4974,8 @@ explain select * from (select rand() from t1)q { "table": "", "table_scan": { "rows": 3, - "read_cost": 0.01235805, - "read_and_compare_cost": 0.012461052 + "read_cost": 0.010031052, + "read_and_compare_cost": 0.010134054 } } ] @@ -4997,7 +4997,7 @@ explain select * from (select rand() from t1)q { "rows": 3, "rows_after_filter": 3, "rows_out": 3, - "cost": 0.012461052, + "cost": 0.010134054, "index_only": false, "chosen": true } @@ -5006,7 +5006,7 @@ explain select * from (select rand() from t1)q { "type": "scan", "rows_read": 3, "rows_out": 3, - "cost": 0.012461052, + "cost": 0.010134054, "uses_join_buffering": false } } @@ -5017,14 +5017,14 @@ explain select * from (select rand() from t1)q { "plan_prefix": "", "table": "", "rows_for_plan": 3, - "cost_for_plan": 0.012461052 + "cost_for_plan": 0.010134054 } ] }, { "best_join_order": [""], "rows": 3, - "cost": 0.012461052 + "cost": 0.010134054 }, { "attaching_conditions_to_tables": { @@ -11900,8 +11900,8 @@ UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a { "table": "", "table_scan": { "rows": 2, - "read_cost": 0.012350033, - "read_and_compare_cost": 0.012418701 + "read_cost": 0.010020701, + "read_and_compare_cost": 0.010089369 } } ] @@ -11949,7 +11949,7 @@ UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a { "rows": 2, "rows_after_filter": 2, "rows_out": 2, - "cost": 0.012418701, + "cost": 0.010089369, "index_only": false, "chosen": true } @@ -11958,7 +11958,7 @@ UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a { "type": "scan", "rows_read": 2, "rows_out": 2, - "cost": 0.012418701, + "cost": 0.010089369, "uses_join_buffering": false } } @@ -11986,8 +11986,8 @@ UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a { "rows": 2, "rows_after_filter": 2, "rows_out": 2, - "cost": 0.012911897, - "cost_without_join_buffer": 0.024837402, + "cost": 0.010582565, + "cost_without_join_buffer": 0.020178738, "index_only": false, "chosen": true } @@ -11996,7 +11996,7 @@ UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a { "type": "scan", "rows_read": 2, "rows_out": 2, - "cost": 0.012911897, + "cost": 0.010582565, "uses_join_buffering": true } } @@ -12007,7 +12007,7 @@ UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a { "plan_prefix": "t", "table": "", "rows_for_plan": 4, - "cost_for_plan": 0.023260307 + "cost_for_plan": 0.020930975 } ] }, @@ -12015,15 +12015,57 @@ UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a { "plan_prefix": "", "table": "", "rows_for_plan": 2, - "cost_for_plan": 0.012418701, - "pruned_by_heuristic": true + "cost_for_plan": 0.010089369, + "rest_of_plan": [ + { + "plan_prefix": "", + "get_costs_for_tables": [ + { + "best_access_path": { + "table": "t", + "plan_details": { + "record_count": 2 + }, + "considered_access_paths": [ + { + "access_type": "scan_with_join_cache", + "rows": 2, + "rows_after_filter": 2, + "rows_out": 2, + "cost": 0.010841606, + "cost_without_join_buffer": 0.02069682, + "index_only": false, + "chosen": true + } + ], + "chosen_access_method": { + "type": "scan", + "rows_read": 2, + "rows_out": 2, + "cost": 0.010841606, + "uses_join_buffering": true + } + } + } + ] + }, + { + "plan_prefix": "", + "table": "t", + "rows_for_plan": 4, + "cost_for_plan": 0.020930975, + "pruned_by_cost": true, + "current_cost": 0.020930975, + "best_cost": 0.020930975 + } + ] } ] }, { "best_join_order": ["t", ""], "rows": 4, - "cost": 0.023260307 + "cost": 0.020930975 }, { "substitute_best_equal": { diff --git a/mysql-test/main/opt_tvc.result b/mysql-test/main/opt_tvc.result index 15b6d9b4218..819d42b5392 100644 --- a/mysql-test/main/opt_tvc.result +++ b/mysql-test/main/opt_tvc.result @@ -530,11 +530,11 @@ a b 4 yq explain extended select * from t3 where a in (1,4); id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY t3 ALL idx NULL NULL NULL 28 100.00 Using where -1 PRIMARY eq_ref distinct_key distinct_key 4 test.t3.a 1 100.00 +1 PRIMARY ALL distinct_key NULL NULL NULL 2 100.00 +1 PRIMARY t3 ref idx idx 5 tvc_0._col_1 3 100.00 3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL No tables used Warnings: -Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from (values (1),(4)) `tvc_0` join `test`.`t3` where `tvc_0`.`_col_1` = `test`.`t3`.`a` +Note 1003 /* select#1 */ select `test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b` from (values (1),(4)) `tvc_0` join `test`.`t3` where `test`.`t3`.`a` = `tvc_0`.`_col_1` # use vectors in IN predeicate set @@in_predicate_conversion_threshold= 4; select * from t1 where (a,b) in ((1,2),(3,4)); diff --git a/mysql-test/main/optimizer_costs.result b/mysql-test/main/optimizer_costs.result index 797c6172b26..b7c3bbb5894 100644 --- a/mysql-test/main/optimizer_costs.result +++ b/mysql-test/main/optimizer_costs.result @@ -69,7 +69,7 @@ OPTIMIZER_KEY_NEXT_FIND_COST 0.000000 OPTIMIZER_DISK_READ_RATIO 0.000000 OPTIMIZER_ROW_COPY_COST 0.002334 OPTIMIZER_ROW_LOOKUP_COST 0.000000 -OPTIMIZER_ROW_NEXT_FIND_COST 0.000000 +OPTIMIZER_ROW_NEXT_FIND_COST 0.008017 OPTIMIZER_ROWID_COMPARE_COST 0.002653 OPTIMIZER_ROWID_COPY_COST 0.002653 show variables like "optimizer%cost"; diff --git a/mysql-test/main/sp.result b/mysql-test/main/sp.result index 9ffa477a20b..cfc718896f8 100644 --- a/mysql-test/main/sp.result +++ b/mysql-test/main/sp.result @@ -9031,7 +9031,7 @@ EXPLAIN { "query_block": { "select_id": 1, - "cost": 0.012588104, + "cost": 0.010268108, "nested_loop": [ { "table": { @@ -9039,7 +9039,7 @@ EXPLAIN "access_type": "ALL", "loops": 1, "rows": 6, - "cost": 0.012588104, + "cost": 0.010268108, "filtered": 100, "attached_condition": "dt.minB = (localB@1) and dt.a = (localA@0 + localB@1)", "materialized": { @@ -9073,7 +9073,7 @@ EXPLAIN { "query_block": { "select_id": 1, - "cost": 0.012588104, + "cost": 0.010268108, "nested_loop": [ { "table": { @@ -9081,7 +9081,7 @@ EXPLAIN "access_type": "ALL", "loops": 1, "rows": 6, - "cost": 0.012588104, + "cost": 0.010268108, "filtered": 100, "attached_condition": "dt.minB = (localB@1) and dt.a = (localA@0 + localB@1)", "materialized": { diff --git a/mysql-test/main/subselect_sj.result b/mysql-test/main/subselect_sj.result index 054c71351d0..e19c203397b 100644 --- a/mysql-test/main/subselect_sj.result +++ b/mysql-test/main/subselect_sj.result @@ -2260,10 +2260,10 @@ alias1.c IN (SELECT SQ3_alias1.b FROM t2 AS SQ3_alias1 STRAIGHT_JOIN t2 AS SQ3_alias2) LIMIT 100; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY alias2 ALL NULL NULL NULL NULL 20 +1 PRIMARY ALL NULL NULL NULL NULL 20 +1 PRIMARY alias2 ALL NULL NULL NULL NULL 20 Using join buffer (flat, BNL join) 1 PRIMARY t2 ALL NULL NULL NULL NULL 20 Using join buffer (flat, BNL join) -1 PRIMARY SQ3_alias1 ALL NULL NULL NULL NULL 20 Start temporary -1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (flat, BNL join) +1 PRIMARY SQ3_alias1 ALL NULL NULL NULL NULL 20 Using where; Start temporary 1 PRIMARY SQ3_alias2 index NULL PRIMARY 4 NULL 20 Using index; End temporary 2 DERIVED t2 ALL NULL NULL NULL NULL 20 create table t3 as diff --git a/mysql-test/main/subselect_sj_jcl6.result b/mysql-test/main/subselect_sj_jcl6.result index 6e397ddd754..7fc23e687a6 100644 --- a/mysql-test/main/subselect_sj_jcl6.result +++ b/mysql-test/main/subselect_sj_jcl6.result @@ -2266,10 +2266,10 @@ alias1.c IN (SELECT SQ3_alias1.b FROM t2 AS SQ3_alias1 STRAIGHT_JOIN t2 AS SQ3_alias2) LIMIT 100; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY alias2 ALL NULL NULL NULL NULL 20 -1 PRIMARY t2 ALL NULL NULL NULL NULL 20 Using join buffer (flat, BNL join) -1 PRIMARY SQ3_alias1 ALL NULL NULL NULL NULL 20 Start temporary; Using join buffer (incremental, BNL join) -1 PRIMARY ALL NULL NULL NULL NULL 20 Using where; Using join buffer (incremental, BNL join) +1 PRIMARY ALL NULL NULL NULL NULL 20 +1 PRIMARY alias2 ALL NULL NULL NULL NULL 20 Using join buffer (flat, BNL join) +1 PRIMARY t2 ALL NULL NULL NULL NULL 20 Using join buffer (incremental, BNL join) +1 PRIMARY SQ3_alias1 ALL NULL NULL NULL NULL 20 Using where; Start temporary; Using join buffer (incremental, BNL join) 1 PRIMARY SQ3_alias2 index NULL PRIMARY 4 NULL 20 Using index; End temporary; Using join buffer (incremental, BNL join) 2 DERIVED t2 ALL NULL NULL NULL NULL 20 create table t3 as diff --git a/sql/sql_select.cc b/sql/sql_select.cc index c3f50989e06..d9b90c49196 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -16567,24 +16567,62 @@ void JOIN_TAB::estimate_scan_time() } else { + bool using_heap= 0; + TABLE_SHARE *share= table->s; + handler *tmp_file= file; + records= table->stat_records(); + + if (share->db_type() == heap_hton) + { + /* Check that the rows will fit into the heap table */ + ha_rows max_rows; + max_rows= (ha_rows) (MY_MIN(thd->variables.tmp_memory_table_size, + thd->variables.max_heap_table_size) / + MY_ALIGN(share->reclength, sizeof(char*))); + if (records <= max_rows) + { + /* The rows will fit into the heap table */ + using_heap= 1; + } + else if (likely((tmp_file= get_new_handler(share, &table->mem_root, + TMP_ENGINE_HTON)))) + { + tmp_file->costs= &tmp_table_optimizer_costs; + } + else + tmp_file= file; // Fallback for OOM + } + /* The following is same as calling TABLE_SHARE::update_optimizer_costs, but without locks */ - if (table->s->db_type() == heap_hton) - memcpy(&table->s->optimizer_costs, &heap_optimizer_costs, + if (using_heap) + memcpy(&share->optimizer_costs, &heap_optimizer_costs, sizeof(heap_optimizer_costs)); else - memcpy(&table->s->optimizer_costs, &tmp_table_optimizer_costs, + { + memcpy(&share->optimizer_costs, &tmp_table_optimizer_costs, sizeof(tmp_table_optimizer_costs)); - file->set_optimizer_costs(thd); - table->s->optimizer_costs_inited=1; + /* Set data_file_length in case of Aria tmp table */ + tmp_file->stats.data_file_length= share->reclength * records; + } + + table->s->optimizer_costs_inited=1; + /* Add current WHERE and SCAN SETUP cost to tmp file */ + tmp_file->set_optimizer_costs(thd); - records= table->stat_records(); DBUG_ASSERT(table->opt_range_condition_rows == records); - cost->row_cost= table->file->ha_scan_time(MY_MAX(records, 1000)); - read_time= file->cost(cost->row_cost); + cost->row_cost= tmp_file->ha_scan_time(records); + tmp_file->stats.data_file_length= 0; + read_time= tmp_file->cost(cost->row_cost); row_copy_cost= table->s->optimizer_costs.row_copy_cost; + + if (file != tmp_file) + { + delete tmp_file; + file->set_optimizer_costs(thd); + } } found_records= records; diff --git a/storage/heap/ha_heap.cc b/storage/heap/ha_heap.cc index 1f37ad8b991..03748dcfccf 100644 --- a/storage/heap/ha_heap.cc +++ b/storage/heap/ha_heap.cc @@ -58,7 +58,7 @@ static void heap_update_optimizer_costs(OPTIMIZER_COSTS *costs) costs->key_copy_cost= 0; // Set in keyread_time() costs->row_copy_cost= 2.334e-06; // This is small as its just a memcpy costs->row_lookup_cost= 0; // Direct pointer - costs->row_next_find_cost= 0; + costs->row_next_find_cost= HEAP_ROW_NEXT_FIND_COST; costs->key_lookup_cost= 0; costs->key_next_find_cost= 0; costs->index_block_copy_cost= 0; @@ -272,7 +272,9 @@ IO_AND_CPU_COST ha_heap::keyread_time(uint index, ulong ranges, ha_rows rows, IO_AND_CPU_COST ha_heap::scan_time() { - return {0, (double) (stats.records+stats.deleted) * HEAP_ROW_NEXT_FIND_COST }; + /* The caller ha_scan_time() handles stats.records */ + + return {0, (double) stats.deleted * HEAP_ROW_NEXT_FIND_COST }; } From c9fe55ff7af8a46856eb3188f41f647cfa608e4e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 10 Feb 2025 14:32:47 +0100 Subject: [PATCH 25/47] MDEV-36056 Fix VS2019 compilation Fix casts introduced by dbfee9fc2bc8 in MDEV-34348 --- mysys/mf_keycache.c | 9 +++++---- storage/maria/ma_pagecache.c | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c index a563d52c0e9..2307341ddb2 100644 --- a/mysys/mf_keycache.c +++ b/mysys/mf_keycache.c @@ -3762,10 +3762,11 @@ static void free_block(SIMPLE_KEY_CACHE_CB *keycache, BLOCK_LINK *block) static int cmp_sec_link(const void *_a, const void *_b) { - BLOCK_LINK *const *a= _a; - BLOCK_LINK *const *b= _b; - return (((*a)->hash_link->diskpos < (*b)->hash_link->diskpos) ? -1 : - ((*a)->hash_link->diskpos > (*b)->hash_link->diskpos) ? 1 : 0); + const BLOCK_LINK *a= *(const BLOCK_LINK **)_a; + const BLOCK_LINK *b= *(const BLOCK_LINK **)_b; + + return (a->hash_link->diskpos < b->hash_link->diskpos) ? -1 : + (a->hash_link->diskpos > b->hash_link->diskpos) ? 1 : 0; } diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c index 1682bd42893..650732f9178 100644 --- a/storage/maria/ma_pagecache.c +++ b/storage/maria/ma_pagecache.c @@ -4726,10 +4726,10 @@ static my_bool free_block(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *block, static int cmp_sec_link(const void *a_, const void *b_) { - PAGECACHE_BLOCK_LINK *const *a= a_; - PAGECACHE_BLOCK_LINK *const *b= b_; - return (((*a)->hash_link->pageno < (*b)->hash_link->pageno) ? -1 : - ((*a)->hash_link->pageno > (*b)->hash_link->pageno) ? 1 : 0); + const PAGECACHE_BLOCK_LINK *a= *(const PAGECACHE_BLOCK_LINK **) a_; + const PAGECACHE_BLOCK_LINK *b= *(const PAGECACHE_BLOCK_LINK **) b_; + return ((a->hash_link->pageno < b->hash_link->pageno) ? -1 : + (a->hash_link->pageno > b->hash_link->pageno) ? 1 : 0); } From 43c5d1303f5c7c726db276815c459436110f342f Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 10 Feb 2025 14:36:56 +0200 Subject: [PATCH 26/47] MDEV-35958 Cost estimates for materialized derived tables are poor Backport of commit 74f70c394456767c1c1b0bee6bf34faff5b72dce to 10.11. The new logic is disabled by default, to enable, use optimizer_adjust_secondary_key_costs=fix_derived_table_read_cost. == Original commit comment == Fixed costs in JOIN_TAB::estimate_scan_time() and HEAP Estimate_scan_time() calculates the cost of scanning a derivied table. The old code did not take into account that the temporary table heap table may be converted to Aria. Things fixed: - Added checking if the temporary tables data will fit in the heap. If not, then calculate the cost based on the designated internal temporary table engine (Aria). - Removed MY_MAX(records, 1000) and instead trust the optimizer's estimate of records. This reduces the cost of temporary tables a bit for small tables, which caused a few changes in mtr results. - Fixed cost calculation for HEAP. - HEAP costs->row_next_find_cost was not set. This does not affect old costs calculation as this cost slot was not used anywhere. Now HEAP cost->row_next_find_cost is set, which allowed me to remove some duplicated computation in ha_heap::scan_time() --- mysql-test/main/mysqld--help.result | 3 +- mysql-test/main/secondary_key_costs.result | 76 +++++++++++++++++++ mysql-test/main/secondary_key_costs.test | 37 +++++++++ .../sys_vars/r/sysvars_server_embedded.result | 4 +- .../r/sysvars_server_notembedded.result | 4 +- sql/sql_priv.h | 1 + sql/sql_select.cc | 51 ++++++++++++- sql/sys_vars.cc | 7 +- 8 files changed, 173 insertions(+), 10 deletions(-) diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 6032d2f37b5..c8c850176a2 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -748,7 +748,8 @@ The following specify which files/extra groups are read (specified before remain keys. fix_reuse_range_for_ref = Do a better job at reusing range access estimates when estimating ref access. fix_card_multiplier = Fix the computation in - selectivity_for_indexes. selectivity_multiplier. This + selectivity_for_indexes. fix_derived_table_read_cost = + Fix the cost of reading materialized derived table. This variable will be deleted in MariaDB 11.0 as it is not needed with the new 11.0 optimizer. Use 'ALL' to set all combinations. diff --git a/mysql-test/main/secondary_key_costs.result b/mysql-test/main/secondary_key_costs.result index 09c6fa31cab..6e02bf475b6 100644 --- a/mysql-test/main/secondary_key_costs.result +++ b/mysql-test/main/secondary_key_costs.result @@ -177,4 +177,80 @@ disconnect user2; drop table t1,t2; set global userstat=@save_userstat; set global innodb_stats_persistent_sample_pages=@save_ispsp; +# +# MDEV-35958: Cost estimates for materialized derived tables are poor +# +set optimizer_trace=1; +create table t1 ( +a int +); +insert into t1 select seq from seq_1_to_10000; +explain +select * +from +t1 as t1_base, +(select a from t1 limit 10000) as TBL; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1_base ALL NULL NULL NULL NULL 10000 +1 PRIMARY ALL NULL NULL NULL NULL 10000 Using join buffer (flat, BNL join) +2 DERIVED t1 ALL NULL NULL NULL NULL 10000 +set @trace=(select trace from information_schema.optimizer_trace); +# BEFORE, without fix_derived_table_read_cost: derived2 has cost=rows=10000 +select json_detailed( +json_extract(json_extract(@trace, '$**.rows_estimation'), '$[1]') +) as Trace; +Trace +[ + { + "table": "t1_base", + "table_scan": + { + "rows": 10000, + "cost": 19.08984375 + } + }, + { + "table": "", + "table_scan": + { + "rows": 10000, + "cost": 10000 + } + } +] +set optimizer_adjust_secondary_key_costs='fix_derived_table_read_cost'; +explain +select * +from +t1 as t1_base, +(select a from t1 limit 10000) as TBL; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1_base ALL NULL NULL NULL NULL 10000 +1 PRIMARY ALL NULL NULL NULL NULL 10000 Using join buffer (flat, BNL join) +2 DERIVED t1 ALL NULL NULL NULL NULL 10000 +set @trace=(select trace from information_schema.optimizer_trace); +# AFTER, with fix_derived_table_read_cost: derived2 has more realistic cost +select json_detailed( +json_extract(json_extract(@trace, '$**.rows_estimation'), '$[1]') +) as Trace; +Trace +[ + { + "table": "t1_base", + "table_scan": + { + "rows": 10000, + "cost": 19.08984375 + } + }, + { + "table": "", + "table_scan": + { + "rows": 10000, + "cost": 501 + } + } +] +drop table t1; set @@optimizer_adjust_secondary_key_costs=default; diff --git a/mysql-test/main/secondary_key_costs.test b/mysql-test/main/secondary_key_costs.test index 32e30156a08..67443a6b2dc 100644 --- a/mysql-test/main/secondary_key_costs.test +++ b/mysql-test/main/secondary_key_costs.test @@ -109,4 +109,41 @@ drop table t1,t2; set global userstat=@save_userstat; set global innodb_stats_persistent_sample_pages=@save_ispsp; +--echo # +--echo # MDEV-35958: Cost estimates for materialized derived tables are poor +--echo # +set optimizer_trace=1; +create table t1 ( + a int +); +insert into t1 select seq from seq_1_to_10000; + +explain +select * +from + t1 as t1_base, + (select a from t1 limit 10000) as TBL; + +set @trace=(select trace from information_schema.optimizer_trace); +--echo # BEFORE, without fix_derived_table_read_cost: derived2 has cost=rows=10000 +select json_detailed( + json_extract(json_extract(@trace, '$**.rows_estimation'), '$[1]') + ) as Trace; + +set optimizer_adjust_secondary_key_costs='fix_derived_table_read_cost'; + +explain +select * +from + t1 as t1_base, + (select a from t1 limit 10000) as TBL; + +set @trace=(select trace from information_schema.optimizer_trace); +--echo # AFTER, with fix_derived_table_read_cost: derived2 has more realistic cost +select json_detailed( + json_extract(json_extract(@trace, '$**.rows_estimation'), '$[1]') + ) as Trace; + +drop table t1; + set @@optimizer_adjust_secondary_key_costs=default; diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index c231e5aba57..212ebb4c2ba 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -2325,11 +2325,11 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME OPTIMIZER_ADJUST_SECONDARY_KEY_COSTS VARIABLE_SCOPE SESSION VARIABLE_TYPE SET -VARIABLE_COMMENT A bit field with the following values: adjust_secondary_key_cost = Update secondary key costs for ranges to be at least 5x of clustered primary key costs. disable_max_seek = Disable 'max_seek optimization' for secondary keys and slight adjustment of filter cost. disable_forced_index_in_group_by = Disable automatic forced index in GROUP BY. fix_innodb_cardinality = Disable doubling of the Cardinality for InnoDB secondary keys. fix_reuse_range_for_ref = Do a better job at reusing range access estimates when estimating ref access. fix_card_multiplier = Fix the computation in selectivity_for_indexes. selectivity_multiplier. This variable will be deleted in MariaDB 11.0 as it is not needed with the new 11.0 optimizer. +VARIABLE_COMMENT A bit field with the following values: adjust_secondary_key_cost = Update secondary key costs for ranges to be at least 5x of clustered primary key costs. disable_max_seek = Disable 'max_seek optimization' for secondary keys and slight adjustment of filter cost. disable_forced_index_in_group_by = Disable automatic forced index in GROUP BY. fix_innodb_cardinality = Disable doubling of the Cardinality for InnoDB secondary keys. fix_reuse_range_for_ref = Do a better job at reusing range access estimates when estimating ref access. fix_card_multiplier = Fix the computation in selectivity_for_indexes. fix_derived_table_read_cost = Fix the cost of reading materialized derived table. This variable will be deleted in MariaDB 11.0 as it is not needed with the new 11.0 optimizer. NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST adjust_secondary_key_cost,disable_max_seek,disable_forced_index_in_group_by,fix_innodb_cardinality,fix_reuse_range_for_ref,fix_card_multiplier +ENUM_VALUE_LIST adjust_secondary_key_cost,disable_max_seek,disable_forced_index_in_group_by,fix_innodb_cardinality,fix_reuse_range_for_ref,fix_card_multiplier,fix_derived_table_read_cost READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME OPTIMIZER_EXTRA_PRUNING_DEPTH diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index f452e1c4c91..2cdba5a988e 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -2495,11 +2495,11 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME OPTIMIZER_ADJUST_SECONDARY_KEY_COSTS VARIABLE_SCOPE SESSION VARIABLE_TYPE SET -VARIABLE_COMMENT A bit field with the following values: adjust_secondary_key_cost = Update secondary key costs for ranges to be at least 5x of clustered primary key costs. disable_max_seek = Disable 'max_seek optimization' for secondary keys and slight adjustment of filter cost. disable_forced_index_in_group_by = Disable automatic forced index in GROUP BY. fix_innodb_cardinality = Disable doubling of the Cardinality for InnoDB secondary keys. fix_reuse_range_for_ref = Do a better job at reusing range access estimates when estimating ref access. fix_card_multiplier = Fix the computation in selectivity_for_indexes. selectivity_multiplier. This variable will be deleted in MariaDB 11.0 as it is not needed with the new 11.0 optimizer. +VARIABLE_COMMENT A bit field with the following values: adjust_secondary_key_cost = Update secondary key costs for ranges to be at least 5x of clustered primary key costs. disable_max_seek = Disable 'max_seek optimization' for secondary keys and slight adjustment of filter cost. disable_forced_index_in_group_by = Disable automatic forced index in GROUP BY. fix_innodb_cardinality = Disable doubling of the Cardinality for InnoDB secondary keys. fix_reuse_range_for_ref = Do a better job at reusing range access estimates when estimating ref access. fix_card_multiplier = Fix the computation in selectivity_for_indexes. fix_derived_table_read_cost = Fix the cost of reading materialized derived table. This variable will be deleted in MariaDB 11.0 as it is not needed with the new 11.0 optimizer. NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST adjust_secondary_key_cost,disable_max_seek,disable_forced_index_in_group_by,fix_innodb_cardinality,fix_reuse_range_for_ref,fix_card_multiplier +ENUM_VALUE_LIST adjust_secondary_key_cost,disable_max_seek,disable_forced_index_in_group_by,fix_innodb_cardinality,fix_reuse_range_for_ref,fix_card_multiplier,fix_derived_table_read_cost READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME OPTIMIZER_EXTRA_PRUNING_DEPTH diff --git a/sql/sql_priv.h b/sql/sql_priv.h index 2ff4867c4c0..779b0afbbd0 100644 --- a/sql/sql_priv.h +++ b/sql/sql_priv.h @@ -281,6 +281,7 @@ #define OPTIMIZER_FIX_INNODB_CARDINALITY (8) #define OPTIMIZER_ADJ_FIX_REUSE_RANGE_FOR_REF (16) #define OPTIMIZER_ADJ_FIX_CARD_MULT (32) +#define OPTIMIZER_ADJ_FIX_DERIVED_TABLE_READ_COST (64) #define OPTIMIZER_ADJ_DEFAULT (OPTIMIZER_ADJ_FIX_REUSE_RANGE_FOR_REF | \ OPTIMIZER_ADJ_FIX_CARD_MULT) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 572a59f1989..730d57ff877 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -6001,7 +6001,10 @@ make_join_statistics(JOIN *join, List &tables_list, s->table->opt_range_condition_rows=s->records; } else + { + /* Update s->records and s->read_time */ s->scan_time(); + } if (s->table->is_splittable()) s->add_keyuses_for_splitting(); @@ -15468,6 +15471,7 @@ void JOIN_TAB::cleanup() double JOIN_TAB::scan_time() { double res; + THD *thd= join->thd; if (table->is_created()) { if (table->is_filled_at_execution()) @@ -15488,10 +15492,53 @@ double JOIN_TAB::scan_time() } res= read_time; } + else if (!(thd->variables.optimizer_adjust_secondary_key_costs & + OPTIMIZER_ADJ_FIX_DERIVED_TABLE_READ_COST)) + { + /* + Old code, do not merge into 11.0+: + */ + found_records= records=table->stat_records(); + read_time= found_records ? (double)found_records: 10.0; + res= read_time; + } else { - found_records= records=table->stat_records(); - read_time= found_records ? (double)found_records: 10.0;// TODO:fix this stub + bool using_heap= 0; + TABLE_SHARE *share= table->s; + found_records= records= table->stat_records(); + + if (share->db_type() == heap_hton) + { + /* Check that the rows will fit into the heap table */ + ha_rows max_rows; + max_rows= (ha_rows) ((MY_MIN(thd->variables.tmp_memory_table_size, + thd->variables.max_heap_table_size)) / + MY_ALIGN(share->reclength, sizeof(char*))); + if (records <= max_rows) + { + /* The rows will fit into the heap table */ + using_heap= 1; + } + } + + /* + Code for the following is taken from the heap and aria storage engine. + In 11.# this is done without explict engine code + */ + if (using_heap) + read_time= (records / 20.0) + 1; + else + { + handler *file= table->file; + file->stats.data_file_length= share->reclength * records; + /* + Call the default scan_time() method as this is the cost for the + scan when heap is converted to Aria + */ + read_time= file->handler::scan_time(); + file->stats.data_file_length= 0; + } res= read_time; } return res; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index a10c1865042..a595d550e0a 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -2982,7 +2982,7 @@ static const char *adjust_secondary_key_cost[]= { "adjust_secondary_key_cost", "disable_max_seek", "disable_forced_index_in_group_by", "fix_innodb_cardinality", "fix_reuse_range_for_ref", - "fix_card_multiplier", 0 + "fix_card_multiplier", "fix_derived_table_read_cost", 0 }; @@ -2999,8 +2999,9 @@ static Sys_var_set Sys_optimizer_adjust_secondary_key_costs( "secondary keys. " "fix_reuse_range_for_ref = Do a better job at reusing range access estimates " "when estimating ref access. " - "fix_card_multiplier = Fix the computation in selectivity_for_indexes." - " selectivity_multiplier. " + "fix_card_multiplier = Fix the computation in selectivity_for_indexes. " + "fix_derived_table_read_cost = Fix the cost of reading materialized " + "derived table. " "This variable will be deleted in MariaDB 11.0 as it is not needed with the " "new 11.0 optimizer.", From ef966af801afc2a07222b5df65dddd52c77431dd Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Thu, 23 Jan 2025 20:20:00 +0200 Subject: [PATCH 27/47] MDEV-30877: Output cardinality for derived table ignores GROUP BY (Variant 3) (commit in 11.4) When a derived table has a GROUP BY clause: SELECT ... FROM (SELECT ... GROUP BY col1, col2) AS tbl The optimizer would use inner join's output cardinality as an estimate of derived table size, ignoring the fact that GROUP BY operation would produce much fewer groups. Add code to produce tighter bounds: - The GROUP BY list is split into per-table lists. If GROUP BY list has expressions that refer to multiple tables, we fall back to join output cardinality. - For each table, the first cardinality estimate is join_tab->read_records. - Then, we try to get a tighter bound by using index statistics. - If indexes do not cover all GROUP BY columns, we try to use per-column EITS statistics. --- libmysqld/CMakeLists.txt | 1 + mysql-test/main/derived.result | 2 +- mysql-test/main/derived_cond_pushdown.result | 6 +- mysql-test/main/derived_split_innodb.result | 4 +- mysql-test/main/explain.result | 10 +- mysql-test/main/func_gconcat.result | 6 +- mysql-test/main/group_by_cardinality.result | 213 +++++++++++ mysql-test/main/group_by_cardinality.test | 108 ++++++ mysql-test/main/opt_trace.result | 65 +++- mysql-test/main/selectivity.result | 8 +- mysql-test/main/selectivity_innodb.result | 8 +- mysql-test/main/table_elim.result | 8 +- sql/CMakeLists.txt | 1 + sql/opt_group_by_cardinality.cc | 376 +++++++++++++++++++ sql/sql_array.h | 13 + sql/sql_select.cc | 4 + sql/sql_select.h | 2 + 17 files changed, 794 insertions(+), 41 deletions(-) create mode 100644 mysql-test/main/group_by_cardinality.result create mode 100644 mysql-test/main/group_by_cardinality.test create mode 100644 sql/opt_group_by_cardinality.cc diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt index 12bcc7ce1aa..8b155b5c3d5 100644 --- a/libmysqld/CMakeLists.txt +++ b/libmysqld/CMakeLists.txt @@ -71,6 +71,7 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc ../sql/log_event.cc ../sql/log_event_server.cc ../sql/mf_iocache.cc ../sql/my_decimal.cc ../sql/net_serv.cc ../sql/opt_range.cc + ../sql/opt_group_by_cardinality.cc ../sql/opt_rewrite_date_cmp.cc ../sql/opt_rewrite_remove_casefold.cc ../sql/opt_sum.cc diff --git a/mysql-test/main/derived.result b/mysql-test/main/derived.result index 29870b01e62..47c0ce12e41 100644 --- a/mysql-test/main/derived.result +++ b/mysql-test/main/derived.result @@ -317,7 +317,7 @@ a 7.0000 b 3.5000 explain SELECT s.name, AVG(s.val) AS median FROM (SELECT x.name, x.val FROM t1 x, t1 y WHERE x.name=y.name GROUP BY x.name, x.val HAVING SUM(y.val <= x.val) >= COUNT(*)/2 AND SUM(y.val >= x.val) >= COUNT(*)/2) AS s GROUP BY s.name; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY ALL NULL NULL NULL NULL 289 Using temporary; Using filesort +1 PRIMARY ALL NULL NULL NULL NULL 17 Using temporary; Using filesort 2 DERIVED x ALL NULL NULL NULL NULL 17 Using temporary; Using filesort 2 DERIVED y ALL NULL NULL NULL NULL 17 Using where; Using join buffer (flat, BNL join) drop table t1; diff --git a/mysql-test/main/derived_cond_pushdown.result b/mysql-test/main/derived_cond_pushdown.result index 6877acba92d..c0354efe90c 100644 --- a/mysql-test/main/derived_cond_pushdown.result +++ b/mysql-test/main/derived_cond_pushdown.result @@ -20183,7 +20183,7 @@ where t2.b < 40 and t2.a=t3.a and t3.c=t.c; id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 90 60.00 Using where 1 PRIMARY t3 ref idx_a idx_a 5 test.t2.a 1 100.00 Using where -1 PRIMARY ref key0 key0 128 test.t3.c 10 100.00 +1 PRIMARY ref key0 key0 128 test.t3.c 5 100.00 2 DERIVED t4 ALL idx_c NULL NULL NULL 160 100.00 Using temporary; Using filesort Warnings: Note 1003 /* select#1 */ select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c`,`t`.`c` AS `t_c`,`t`.`max` AS `max`,`t`.`min` AS `min` from `test`.`t2` join `test`.`t3` join (/* select#2 */ select `test`.`t4`.`c` AS `c`,max(`test`.`t4`.`b`) AS `max`,min(`test`.`t4`.`b`) AS `min` from `test`.`t4` group by `test`.`t4`.`c`) `t` where `test`.`t3`.`a` = `test`.`t2`.`a` and `t`.`c` = `test`.`t3`.`c` and `test`.`t2`.`b` < 40 @@ -20233,7 +20233,7 @@ EXPLAIN "used_key_parts": ["c"], "ref": ["test.t3.c"], "loops": 80.99999987, - "rows": 10, + "rows": 5, "cost": "COST_REPLACED", "filtered": 100, "materialized": { @@ -22796,7 +22796,7 @@ from t1 limit 5; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 1000 2 DEPENDENT SUBQUERY t3 ALL NULL NULL NULL NULL 2 Using where -2 DEPENDENT SUBQUERY ref key0 key0 5 test.t1.a 100 +2 DEPENDENT SUBQUERY ref key0 key0 5 test.t1.a 1 3 DERIVED t2 ALL a NULL NULL NULL 1000 Using temporary; Using filesort select a, diff --git a/mysql-test/main/derived_split_innodb.result b/mysql-test/main/derived_split_innodb.result index 90b980bb4e0..327f3c247c1 100644 --- a/mysql-test/main/derived_split_innodb.result +++ b/mysql-test/main/derived_split_innodb.result @@ -283,7 +283,7 @@ on t3.a=t.a and t3.c=t.c where t3.b > 15; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t3 range idx_b idx_b 5 NULL 2 Using index condition; Using where -1 PRIMARY ref key0 key0 133 test.t3.a,test.t3.c 4 +1 PRIMARY ref key0 key0 133 test.t3.a,test.t3.c 2 2 DERIVED t4 ALL NULL NULL NULL NULL 40 Using filesort drop table t3, t4; # End of 10.3 tests @@ -809,7 +809,7 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 5 1 PRIMARY t2 ALL NULL NULL NULL NULL 50 Using where; Using join buffer (flat, BNL join) 1 PRIMARY t3 ALL NULL NULL NULL NULL 15 Using where; Using join buffer (incremental, BNL join) -1 PRIMARY ref key0 key0 5 test.t1.b 1000 Using where +1 PRIMARY ref key0 key0 5 test.t1.b 10 Using where 2 DERIVED t10 ALL grp_id NULL NULL NULL 10000 Using temporary; Using filesort 2 DERIVED t11 ALL NULL NULL NULL NULL 10 Using where; Using join buffer (flat, BNL join) select * diff --git a/mysql-test/main/explain.result b/mysql-test/main/explain.result index e4d972743d9..91974042fd6 100644 --- a/mysql-test/main/explain.result +++ b/mysql-test/main/explain.result @@ -154,7 +154,7 @@ INSERT INTO t2 VALUES (1),(2); EXPLAIN EXTENDED SELECT 1 FROM (SELECT COUNT(DISTINCT t1.a) FROM t1,t2 GROUP BY t1.a) AS s1; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL NULL NULL NULL NULL 4 100.00 +1 PRIMARY ALL NULL NULL NULL NULL 2 100.00 2 DERIVED t1 ALL NULL NULL NULL NULL 2 100.00 Using temporary; Using filesort 2 DERIVED t2 ALL NULL NULL NULL NULL 2 100.00 Using join buffer (flat, BNL join) Warnings: @@ -162,7 +162,7 @@ Note 1003 /* select#1 */ select 1 AS `1` from (/* select#2 */ select count(disti EXPLAIN EXTENDED SELECT 1 FROM (SELECT COUNT(DISTINCT t1.a) FROM t1,t2 GROUP BY t1.a) AS s1; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL NULL NULL NULL NULL 4 100.00 +1 PRIMARY ALL NULL NULL NULL NULL 2 100.00 2 DERIVED t1 ALL NULL NULL NULL NULL 2 100.00 Using temporary; Using filesort 2 DERIVED t2 ALL NULL NULL NULL NULL 2 100.00 Using join buffer (flat, BNL join) Warnings: @@ -172,7 +172,7 @@ prepare s1 from FROM (SELECT COUNT(DISTINCT t1.a) FROM t1,t2 GROUP BY t1.a) AS s1'; execute s1; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL NULL NULL NULL NULL 4 100.00 +1 PRIMARY ALL NULL NULL NULL NULL 2 100.00 2 DERIVED t1 ALL NULL NULL NULL NULL 2 100.00 Using temporary; Using filesort 2 DERIVED t2 ALL NULL NULL NULL NULL 2 100.00 Using join buffer (flat, BNL join) Warnings: @@ -182,14 +182,14 @@ prepare s1 from FROM (SELECT COUNT(DISTINCT t1.a) FROM t1,t2 GROUP BY t1.a) AS s1'; execute s1; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL NULL NULL NULL NULL 4 100.00 +1 PRIMARY ALL NULL NULL NULL NULL 2 100.00 2 DERIVED t1 ALL NULL NULL NULL NULL 2 100.00 Using temporary; Using filesort 2 DERIVED t2 ALL NULL NULL NULL NULL 2 100.00 Using join buffer (flat, BNL join) Warnings: Note 1003 /* select#1 */ select 1 AS `1` from (/* select#2 */ select count(distinct `test`.`t1`.`a`) AS `COUNT(DISTINCT t1.a)` from `test`.`t1` join `test`.`t2` group by `test`.`t1`.`a`) `s1` execute s1; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL NULL NULL NULL NULL 4 100.00 +1 PRIMARY ALL NULL NULL NULL NULL 2 100.00 2 DERIVED t1 ALL NULL NULL NULL NULL 2 100.00 Using temporary; Using filesort 2 DERIVED t2 ALL NULL NULL NULL NULL 2 100.00 Using join buffer (flat, BNL join) Warnings: diff --git a/mysql-test/main/func_gconcat.result b/mysql-test/main/func_gconcat.result index d725029a533..3694d6ebbbe 100644 --- a/mysql-test/main/func_gconcat.result +++ b/mysql-test/main/func_gconcat.result @@ -990,8 +990,8 @@ INSERT INTO t1 VALUES (),(); EXPLAIN EXTENDED SELECT 1 FROM (SELECT DISTINCT GROUP_CONCAT(td.f1) FROM t1,t1 AS td GROUP BY td.f1) AS d,t1; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY t1 ALL NULL NULL NULL NULL 2 100.00 -1 PRIMARY ALL NULL NULL NULL NULL 4 100.00 Using join buffer (flat, BNL join) +1 PRIMARY ALL NULL NULL NULL NULL 2 100.00 +1 PRIMARY t1 ALL NULL NULL NULL NULL 2 100.00 Using join buffer (flat, BNL join) 2 DERIVED t1 ALL NULL NULL NULL NULL 2 100.00 Using temporary; Using filesort 2 DERIVED td ALL NULL NULL NULL NULL 2 100.00 Using join buffer (flat, BNL join) Warnings: @@ -1012,7 +1012,7 @@ EXPLAIN EXTENDED SELECT 1 FROM (SELECT GROUP_CONCAT(t1.a ORDER BY t1.a ASC) FROM t1 t2, t1 GROUP BY t1.a) AS d; id select_type table type possible_keys key key_len ref rows filtered Extra -1 PRIMARY ALL NULL NULL NULL NULL 4 100.00 +1 PRIMARY ALL NULL NULL NULL NULL 2 100.00 2 DERIVED t2 ALL NULL NULL NULL NULL 2 100.00 Using temporary; Using filesort 2 DERIVED t1 ALL NULL NULL NULL NULL 2 100.00 Using join buffer (flat, BNL join) Warnings: diff --git a/mysql-test/main/group_by_cardinality.result b/mysql-test/main/group_by_cardinality.result new file mode 100644 index 00000000000..e68af93d47c --- /dev/null +++ b/mysql-test/main/group_by_cardinality.result @@ -0,0 +1,213 @@ +# +# MDEV-30877: Output cardinality for derived table ignores GROUP BY +# +create table t1 ( +groups_20 int, +groups_20_2 int, +b int, +index (groups_20) +); +insert into t1 select seq/1000, seq/1000, seq from seq_1_to_20000; +create table t2 (a int, b int, index(a)); +insert into t2 select seq, seq from seq_1_to_10; +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +set optimizer_trace=1; +# Case 1: one indexed column +explain +select * +from +t2, +(select count(*) cnt, groups_20 from t1 +group by groups_20) TBL +where +1; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 10 +1 PRIMARY ALL NULL NULL NULL NULL 20 Using join buffer (flat, BNL join) +2 DERIVED t1 index NULL groups_20 5 NULL 20000 Using index +select +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from +information_schema.optimizer_trace; +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +[ + { + "join_output_cardinality": 20000, + "estimation": + [ + { + "table": "t1", + "steps": + [ + { + "index_name": "groups_20", + "cardinality": 20.99999895 + } + ], + "cardinality": 20.99999895 + } + ], + "post_group_cardinality": 20.99999895 + } +] +# Case 2: one non- indexed column +explain +select * +from +t2, +(select count(*) cnt, groups_20_2 from t1 +group by groups_20_2) TBL +where +1; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 10 +1 PRIMARY ALL NULL NULL NULL NULL 20 Using join buffer (flat, BNL join) +2 DERIVED t1 ALL NULL NULL NULL NULL 20000 Using temporary; Using filesort +select +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from +information_schema.optimizer_trace; +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +[ + { + "join_output_cardinality": 20000, + "estimation": + [ + { + "table": "t1", + "steps": + [ + { + "column": "groups_20_2", + "cardinality": 20.99999895 + } + ], + "cardinality": 20.99999895 + } + ], + "post_group_cardinality": 20.99999895 + } +] +# Case 4: one indexed column, multiple tables +create table t3(c int); +insert into t3 select seq from seq_1_to_10; +explain +select * +from +t2, +(select count(*) cnt, groups_20 from t1,t3 +group by groups_20) TBL; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 10 +1 PRIMARY ALL NULL NULL NULL NULL 20 Using join buffer (flat, BNL join) +2 DERIVED t3 ALL NULL NULL NULL NULL 10 Using temporary; Using filesort +2 DERIVED t1 index NULL groups_20 5 NULL 20000 Using index; Using join buffer (flat, BNL join) +select +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from +information_schema.optimizer_trace; +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +[ + { + "join_output_cardinality": 200000, + "estimation": + [ + { + "table": "t1", + "steps": + [ + { + "index_name": "groups_20", + "cardinality": 20.99999895 + } + ], + "cardinality": 20.99999895 + } + ], + "post_group_cardinality": 20.99999895 + } +] +# Case 5: group by two tables +explain +select * +from +t2, +(select count(*) cnt, groups_20 from t1,t3 +group by groups_20, t3.c) TBL; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 10 +1 PRIMARY ALL NULL NULL NULL NULL 209 Using join buffer (flat, BNL join) +2 DERIVED t3 ALL NULL NULL NULL NULL 10 Using temporary; Using filesort +2 DERIVED t1 index NULL groups_20 5 NULL 20000 Using index; Using join buffer (flat, BNL join) +select +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from +information_schema.optimizer_trace; +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +[ + { + "join_output_cardinality": 200000, + "estimation": + [ + { + "table": "t1", + "steps": + [ + { + "index_name": "groups_20", + "cardinality": 20.99999895 + } + ], + "cardinality": 20.99999895 + }, + { + "table": "t3", + "steps": + [], + "cardinality": 10 + } + ], + "post_group_cardinality": 209.9999895 + } +] +# Now, without an index +explain +select * +from +t2, +(select count(*) cnt, groups_20 from t1 use index(),t3 group by groups_20) TBL; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 10 +1 PRIMARY ALL NULL NULL NULL NULL 20 Using join buffer (flat, BNL join) +2 DERIVED t3 ALL NULL NULL NULL NULL 10 Using temporary; Using filesort +2 DERIVED t1 ALL NULL NULL NULL NULL 20000 Using join buffer (flat, BNL join) +select +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from +information_schema.optimizer_trace; +json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +[ + { + "join_output_cardinality": 200000, + "estimation": + [ + { + "table": "t1", + "steps": + [ + { + "column": "groups_20", + "cardinality": 20.99999895 + } + ], + "cardinality": 20.99999895 + } + ], + "post_group_cardinality": 20.99999895 + } +] +set optimizer_trace=default; +drop table t1, t2, t3; diff --git a/mysql-test/main/group_by_cardinality.test b/mysql-test/main/group_by_cardinality.test new file mode 100644 index 00000000000..a2fecf40b9f --- /dev/null +++ b/mysql-test/main/group_by_cardinality.test @@ -0,0 +1,108 @@ +--echo # +--echo # MDEV-30877: Output cardinality for derived table ignores GROUP BY +--echo # + +--source include/have_sequence.inc +--source include/not_embedded.inc + +create table t1 ( + groups_20 int, + groups_20_2 int, + b int, + index (groups_20) +); + +insert into t1 select seq/1000, seq/1000, seq from seq_1_to_20000; + +create table t2 (a int, b int, index(a)); +insert into t2 select seq, seq from seq_1_to_10; + +analyze table t1 persistent for all; + +set optimizer_trace=1; +--echo # Case 1: one indexed column +explain +select * +from + t2, + (select count(*) cnt, groups_20 from t1 + group by groups_20) TBL +where + 1; + +--disable_view_protocol +select + json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from + information_schema.optimizer_trace; +--enable_view_protocol + + +--echo # Case 2: one non- indexed column +explain +select * +from + t2, + (select count(*) cnt, groups_20_2 from t1 + group by groups_20_2) TBL +where + 1; + +--disable_view_protocol +select + json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from + information_schema.optimizer_trace; +--enable_view_protocol + +--echo # Case 4: one indexed column, multiple tables +create table t3(c int); +insert into t3 select seq from seq_1_to_10; +explain +select * +from + t2, + (select count(*) cnt, groups_20 from t1,t3 + group by groups_20) TBL; + +--disable_view_protocol +select + json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from + information_schema.optimizer_trace; +--enable_view_protocol + + +--echo # Case 5: group by two tables +explain +select * +from + t2, + (select count(*) cnt, groups_20 from t1,t3 + group by groups_20, t3.c) TBL; + +--disable_view_protocol +select + json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from + information_schema.optimizer_trace; +--enable_view_protocol + +--echo # Now, without an index +explain +select * +from + t2, + (select count(*) cnt, groups_20 from t1 use index(),t3 group by groups_20) TBL; + +--disable_view_protocol +select + json_detailed(json_extract(trace, '$**.materialized_output_cardinality')) +from + information_schema.optimizer_trace; +--enable_view_protocol + + +set optimizer_trace=default; +drop table t1, t2, t3; + diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result index 4c47741ef03..1082cf1a492 100644 --- a/mysql-test/main/opt_trace.result +++ b/mysql-test/main/opt_trace.result @@ -514,6 +514,23 @@ select * from v2 { } ] }, + { + "materialized_output_cardinality": { + "join_output_cardinality": 1, + "estimation": [ + { + "table": "t1", + "steps": [ + { + "column": "b", + "cardinality": 2 + } + ], + "cardinality": 2 + } + ] + } + }, { "check_split_materialized": { "not_applicable": "no candidate field can be accessed through ref" @@ -803,7 +820,7 @@ explain select * from v2 { # Non-Mergeable view explain select * from v1 ; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY ALL NULL NULL NULL NULL 10 +1 PRIMARY ALL NULL NULL NULL NULL 4 2 DERIVED t1 ALL NULL NULL NULL NULL 10 Using temporary; Using filesort QUERY TRACE MISSING_BYTES_BEYOND_MAX_MEM_SIZE INSUFFICIENT_PRIVILEGES explain select * from v1 { @@ -908,6 +925,24 @@ explain select * from v1 { } ] }, + { + "materialized_output_cardinality": { + "join_output_cardinality": 10, + "estimation": [ + { + "table": "t1", + "steps": [ + { + "column": "b", + "cardinality": 4 + } + ], + "cardinality": 4 + } + ], + "post_group_cardinality": 4 + } + }, { "check_split_materialized": { "not_applicable": "group list has no candidates" @@ -953,9 +988,9 @@ explain select * from v1 { { "table": "", "table_scan": { - "rows": 10, - "read_cost": 0.010103506, - "read_and_compare_cost": 0.010446846 + "rows": 4, + "read_cost": 0.010041402, + "read_and_compare_cost": 0.010178738 } } ] @@ -974,19 +1009,19 @@ explain select * from v1 { "considered_access_paths": [ { "access_type": "scan", - "rows": 10, - "rows_after_filter": 10, - "rows_out": 10, - "cost": 0.010446846, + "rows": 4, + "rows_after_filter": 4, + "rows_out": 4, + "cost": 0.010178738, "index_only": false, "chosen": true } ], "chosen_access_method": { "type": "scan", - "rows_read": 10, - "rows_out": 10, - "cost": 0.010446846, + "rows_read": 4, + "rows_out": 4, + "cost": 0.010178738, "uses_join_buffering": false } } @@ -996,15 +1031,15 @@ explain select * from v1 { { "plan_prefix": "", "table": "", - "rows_for_plan": 10, - "cost_for_plan": 0.010446846 + "rows_for_plan": 4, + "cost_for_plan": 0.010178738 } ] }, { "best_join_order": [""], - "rows": 10, - "cost": 0.010446846 + "rows": 4, + "cost": 0.010178738 }, { "attaching_conditions_to_tables": { diff --git a/mysql-test/main/selectivity.result b/mysql-test/main/selectivity.result index f07720bb689..923967766e5 100644 --- a/mysql-test/main/selectivity.result +++ b/mysql-test/main/selectivity.result @@ -143,9 +143,9 @@ and total_revenue = (select max(total_revenue) from revenue0) order by s_suppkey; id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY supplier ALL PRIMARY NULL NULL NULL 10 100.00 Using filesort -1 PRIMARY ref key0 key0 5 dbt3_s001.supplier.s_suppkey 10 100.00 Using where +1 PRIMARY ref key0 key0 5 dbt3_s001.supplier.s_suppkey 1 100.00 Using where 3 DERIVED lineitem range i_l_shipdate,i_l_suppkey i_l_shipdate 4 NULL 269 100.00 Using where; Using temporary; Using filesort -2 SUBQUERY ALL NULL NULL NULL NULL 269 100.00 +2 SUBQUERY ALL NULL NULL NULL NULL 10 100.00 4 DERIVED lineitem range i_l_shipdate i_l_shipdate 4 NULL 269 100.00 Using where; Using temporary; Using filesort Warnings: Note 1003 /* select#1 */ select `dbt3_s001`.`supplier`.`s_suppkey` AS `s_suppkey`,`dbt3_s001`.`supplier`.`s_name` AS `s_name`,`dbt3_s001`.`supplier`.`s_address` AS `s_address`,`dbt3_s001`.`supplier`.`s_phone` AS `s_phone`,`revenue0`.`total_revenue` AS `total_revenue` from `dbt3_s001`.`supplier` join `dbt3_s001`.`revenue0` where `revenue0`.`supplier_no` = `dbt3_s001`.`supplier`.`s_suppkey` and `revenue0`.`total_revenue` = (/* select#2 */ select max(`revenue0`.`total_revenue`) from `dbt3_s001`.`revenue0`) order by `dbt3_s001`.`supplier`.`s_suppkey` @@ -164,9 +164,9 @@ and total_revenue = (select max(total_revenue) from revenue0) order by s_suppkey; id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY supplier ALL PRIMARY NULL NULL NULL 10 100.00 Using filesort -1 PRIMARY ref key0 key0 5 dbt3_s001.supplier.s_suppkey 10 100.00 Using where +1 PRIMARY ref key0 key0 5 dbt3_s001.supplier.s_suppkey 1 100.00 Using where 3 DERIVED lineitem range i_l_shipdate,i_l_suppkey i_l_shipdate 4 NULL 269 100.00 Using where; Using temporary; Using filesort -2 SUBQUERY ALL NULL NULL NULL NULL 269 100.00 +2 SUBQUERY ALL NULL NULL NULL NULL 10 100.00 4 DERIVED lineitem range i_l_shipdate i_l_shipdate 4 NULL 269 100.00 Using where; Using temporary; Using filesort Warnings: Note 1003 /* select#1 */ select `dbt3_s001`.`supplier`.`s_suppkey` AS `s_suppkey`,`dbt3_s001`.`supplier`.`s_name` AS `s_name`,`dbt3_s001`.`supplier`.`s_address` AS `s_address`,`dbt3_s001`.`supplier`.`s_phone` AS `s_phone`,`revenue0`.`total_revenue` AS `total_revenue` from `dbt3_s001`.`supplier` join `dbt3_s001`.`revenue0` where `revenue0`.`supplier_no` = `dbt3_s001`.`supplier`.`s_suppkey` and `revenue0`.`total_revenue` = (/* select#2 */ select max(`revenue0`.`total_revenue`) from `dbt3_s001`.`revenue0`) order by `dbt3_s001`.`supplier`.`s_suppkey` diff --git a/mysql-test/main/selectivity_innodb.result b/mysql-test/main/selectivity_innodb.result index 102242dca76..5c3c88c9dd5 100644 --- a/mysql-test/main/selectivity_innodb.result +++ b/mysql-test/main/selectivity_innodb.result @@ -148,9 +148,9 @@ and total_revenue = (select max(total_revenue) from revenue0) order by s_suppkey; id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY supplier index PRIMARY PRIMARY 4 NULL 10 100.00 -1 PRIMARY ref key0 key0 5 dbt3_s001.supplier.s_suppkey 10 100.00 Using where +1 PRIMARY ref key0 key0 5 dbt3_s001.supplier.s_suppkey 1 100.00 Using where 3 DERIVED lineitem range i_l_shipdate,i_l_suppkey i_l_shipdate 4 NULL 229 100.00 Using where; Using temporary; Using filesort -2 SUBQUERY ALL NULL NULL NULL NULL 229 100.00 +2 SUBQUERY ALL NULL NULL NULL NULL 10 100.00 4 DERIVED lineitem range i_l_shipdate i_l_shipdate 4 NULL 229 100.00 Using where; Using temporary; Using filesort Warnings: Note 1003 /* select#1 */ select `dbt3_s001`.`supplier`.`s_suppkey` AS `s_suppkey`,`dbt3_s001`.`supplier`.`s_name` AS `s_name`,`dbt3_s001`.`supplier`.`s_address` AS `s_address`,`dbt3_s001`.`supplier`.`s_phone` AS `s_phone`,`revenue0`.`total_revenue` AS `total_revenue` from `dbt3_s001`.`supplier` join `dbt3_s001`.`revenue0` where `revenue0`.`supplier_no` = `dbt3_s001`.`supplier`.`s_suppkey` and `revenue0`.`total_revenue` = (/* select#2 */ select max(`revenue0`.`total_revenue`) from `dbt3_s001`.`revenue0`) order by `dbt3_s001`.`supplier`.`s_suppkey` @@ -169,9 +169,9 @@ and total_revenue = (select max(total_revenue) from revenue0) order by s_suppkey; id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY supplier index PRIMARY PRIMARY 4 NULL 10 100.00 -1 PRIMARY ref key0 key0 5 dbt3_s001.supplier.s_suppkey 10 100.00 Using where +1 PRIMARY ref key0 key0 5 dbt3_s001.supplier.s_suppkey 1 100.00 Using where 3 DERIVED lineitem range i_l_shipdate,i_l_suppkey i_l_shipdate 4 NULL 229 100.00 Using where; Using temporary; Using filesort -2 SUBQUERY ALL NULL NULL NULL NULL 229 100.00 +2 SUBQUERY ALL NULL NULL NULL NULL 10 100.00 4 DERIVED lineitem range i_l_shipdate i_l_shipdate 4 NULL 229 100.00 Using where; Using temporary; Using filesort Warnings: Note 1003 /* select#1 */ select `dbt3_s001`.`supplier`.`s_suppkey` AS `s_suppkey`,`dbt3_s001`.`supplier`.`s_name` AS `s_name`,`dbt3_s001`.`supplier`.`s_address` AS `s_address`,`dbt3_s001`.`supplier`.`s_phone` AS `s_phone`,`revenue0`.`total_revenue` AS `total_revenue` from `dbt3_s001`.`supplier` join `dbt3_s001`.`revenue0` where `revenue0`.`supplier_no` = `dbt3_s001`.`supplier`.`s_suppkey` and `revenue0`.`total_revenue` = (/* select#2 */ select max(`revenue0`.`total_revenue`) from `dbt3_s001`.`revenue0`) order by `dbt3_s001`.`supplier`.`s_suppkey` diff --git a/mysql-test/main/table_elim.result b/mysql-test/main/table_elim.result index 9437675cacd..4533dcb046c 100644 --- a/mysql-test/main/table_elim.result +++ b/mysql-test/main/table_elim.result @@ -772,7 +772,7 @@ id select_type table type possible_keys key key_len ref rows Extra explain select t1.* from t1 left join v2b on t1.a=v2b.b; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 10 -1 PRIMARY ref key0 key0 8 test.t1.a 10 Using where +1 PRIMARY ref key0 key0 8 test.t1.a 1 Using where 2 DERIVED t11 ALL NULL NULL NULL NULL 1000 Using temporary; Using filesort # Check format JSON as well explain format=JSON select t1.* from t1 left join v2b on t1.a=v2b.b; @@ -803,7 +803,7 @@ EXPLAIN "used_key_parts": ["b"], "ref": ["test.t1.a"], "loops": 10, - "rows": 10, + "rows": 1, "cost": "COST_REPLACED", "filtered": 100, "attached_condition": "trigcond(t1.a = v2b.b and trigcond(t1.a is not null))", @@ -870,7 +870,7 @@ EXPLAIN explain select t1.* from t1 left join v2c on t1.a=v2c.b; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 10 -1 PRIMARY ref key0 key0 5 test.t1.a 10 Using where +1 PRIMARY ref key0 key0 5 test.t1.a 1 Using where 2 DERIVED t11 ALL NULL NULL NULL NULL 1000 Using temporary; Using filesort 2 DERIVED t12 eq_ref PRIMARY PRIMARY 4 test.t11.b 1 Using where # Check format JSON as well @@ -902,7 +902,7 @@ EXPLAIN "used_key_parts": ["b"], "ref": ["test.t1.a"], "loops": 10, - "rows": 10, + "rows": 1, "cost": "COST_REPLACED", "filtered": 100, "attached_condition": "trigcond(trigcond(t1.a is not null))", diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 0195555efaf..0cc4bbcd683 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -114,6 +114,7 @@ SET (SQL_SOURCE mysqld.cc net_serv.cc keycaches.cc ../sql-common/client_plugin.c opt_range.cc + opt_group_by_cardinality.cc opt_rewrite_date_cmp.cc opt_rewrite_remove_casefold.cc opt_sum.cc diff --git a/sql/opt_group_by_cardinality.cc b/sql/opt_group_by_cardinality.cc new file mode 100644 index 00000000000..8b58096cfd1 --- /dev/null +++ b/sql/opt_group_by_cardinality.cc @@ -0,0 +1,376 @@ +/* + Copyright (c) 2025, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +/** + @file + + Contains estimate_post_group_cardinality() which estimates cardinality + after GROUP BY operation is applied. +*/ + +#include "mariadb.h" +#include "sql_priv.h" +#include "sql_select.h" +#include "sql_statistics.h" +#include "opt_trace.h" + +static +double estimate_table_group_cardinality(JOIN *join, Item ***group_list, + Item* const *end); + +inline bool has_one_bit_set(table_map val) +{ + return val && !(val & (val-1)); +} + + +/* + @brief + Sort the Items that refer to one table (so have only one bit in + used_tables()). Used to get the items that refer to the same table + to be next to each other. +*/ + +int cmp_items_by_used_tables(const void *a_val, const void *b_val) +{ + table_map v1= (*((Item**)a_val))->used_tables(); + table_map v2= (*((Item**)b_val))->used_tables(); + return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); +} + + +/* + @brief + Given a SELECT with GROUP BY clause, estimate the cardinality of output + after the grouping operation is performed. + + @detail + Consider a query + + SELECT ... + FROM t1, t2, t3 ... + WHERE ... + GROUP BY + col1, col2, ... + + Join optimizer produces an estimate of number of record combinations we'll + get after all join operations are performed (denote this join_output_card). + This function produces a conservative (i.e. upper bound) estimate of how + many groups will be produced by the GROUP BY operation. + + It does it as follows: + * Split the GROUP BY clause into per-table lists. + (if there are GROUP BY items that refer to multiple tables, refuse + to work and return join_output_card). + * Compute n_groups estimate for each table and its GROUP BY sub-list. + * Compute a product of these estimates, n_groups_prod. + * Return MIN(join_ouput_card, n_groups_prod). + + @param + join_output_card Number of rows after join operation + + @return + Number of rows that will be left after grouping operation +*/ + +double estimate_post_group_cardinality(JOIN *join, double join_output_card) +{ + Dynamic_array group_cols(join->thd->mem_root); + ORDER *cur_group; + + Json_writer_object wrapper(join->thd); + Json_writer_object trace(join->thd, "materialized_output_cardinality"); + trace.add("join_output_cardinality", join_output_card); + + /* + Walk the GROUP BY list and put items into group_cols array. Array is + easier to work with: we will sort it and then produce estimates for + sub-arrays that refer to just one table. + Also check that each item depends on just one table (if not, bail out). + */ + for (cur_group= join->group_list; cur_group; cur_group= cur_group->next) + { + Item *item= *cur_group->item; + table_map map= item->used_tables(); + if ((map & PSEUDO_TABLE_BITS) || !has_one_bit_set(map)) + { + /* Can't estimate */ + return join_output_card; + } + group_cols.append(item); + } + DBUG_ASSERT(group_cols.size()); + + group_cols.sort(cmp_items_by_used_tables); + + double new_card= 1.0; + Item **pos= group_cols.front(); + Json_writer_array trace_steps(join->thd, "estimation"); + + while (pos != group_cols.end()) + { + new_card *= estimate_table_group_cardinality(join, &pos, group_cols.end()); + + if (new_card > join_output_card) + return join_output_card; + } + + trace_steps.end(); + trace.add("post_group_cardinality", new_card); + return new_card; +} + + +/* + @brief + Compute number of groups for a GROUP BY list that refers to a single table + + @detail + Consider a query: + + SELECT ... + FROM t1, t2, t3 ... + WHERE ... + GROUP BY + t1.col1, ... t1.colN -- expressions only refer to t1. + + The number of groups is estimated using the following: + + == 1. Use found_records == + There cannot be more rows than the number of records in t1 that match the + WHERE clause, that is, JOIN_TAB(t1)->found_records. + This estimate doesn't depend on the expressions in the GROUP BY list, so we + use it as a fall-back estimate. + + == 2. Use index statistics == + If t1 has an INDEX(col1, ... colN) then the number of different + combinations of {col1, ..., colN} can be obtained from index statistics. + + It is possible to cover the GROUP BY list with several indexes (without + overlaps) and use a product of n_distinct statistics. For example, for + + GROUP BY key1part1, key1part2, key2part1, key2part2, key2part3 + + the estimate would be: + + n_groups= n_distinct(key1, parts=2) * n_distinct(key2, parts=3) + + There can be multiple ways one can cover GROUP BY list with different + indexes. We try to use indexes that cover more GROUP BY columns, first. + This may cause us to fail later. For example, for + + GROUP BY a, b, c, d + + and indexes + INDEX idx1(a,b,c) + INDEX idx2(a,b) + INDEX idx3(c,d) + + We will use idx1 and then will be unable to get any estimate for column d. + We could have used idx2 and idx3, instead, and could have covered all + columns. We ignore such cases. + + Note that when using index statistics, we ignore the WHERE condition + selectivity. That's because we cannot tell how the WHERE affects index + stats. Does it + A. reduce the number of GROUP BY groups, or + B. make each GROUP BY group smaller ? + We conservatively assume that B holds. + + == 3 Use per-column EITS statistics == + If we fail to cover GROUP BY with indexes, we try to use column statistics + for the remaining columns. + + @param join the Join object we're computing for + @param group_list INOUT Array of Item* from GROUP BY clause, ordered + by table. This function should process the table + it is pointing to, and advance the pointer so it + points at 'end' or at the next table. + @param end IN End of the above array. + +*/ + +double estimate_table_group_cardinality(JOIN *join, Item ***group_list, + Item* const *end) +{ + TABLE *table= NULL; + key_map possible_keys; + Dynamic_array columns(join->thd->mem_root); + double card= 1.0; + double table_records_after_where= DBL_MAX; // Safety + + table_map table_bit= (**group_list)->used_tables(); + /* + join->map2table is not set yet, so find our table in JOIN_TABs. + */ + for (JOIN_TAB *tab= join->join_tab; + tab < join->join_tab + join->top_join_tab_count; + tab++) + { + if (tab->table->map == table_bit) + { + table= tab->table; + table_records_after_where= rows2double(tab->found_records); + break; + } + } + DBUG_ASSERT(table); + + Json_writer_object trace_obj(join->thd); + trace_obj.add_table_name(table); + Json_writer_array trace_steps(join->thd, "steps"); + + possible_keys.clear_all(); + bool found_complex_item= false; + + /* + Walk through the group list and collect references to fields. + If there are other kinds of items, return table's cardinality. + */ + Item **p; + for (p= *group_list; + p != end && (*p)->used_tables() == table_bit; + p++) + { + Item *real= (*p)->real_item(); + if (real->type() == Item::FIELD_ITEM) + { + Field *field= ((Item_field*)real)->field; + possible_keys.merge(field->part_of_key); + columns.append(field->field_index); + } + else + found_complex_item= true; + } + + /* Tell the caller where group_list ended */ + *group_list= p; + + if (found_complex_item) + goto whole_table; + + possible_keys.intersect(table->keys_in_use_for_query); + /* + Ok, group_list has only columns and we've got them in 'columns'. + */ + while (!possible_keys.is_clear_all()) + { + /* Find the index which has the longest prefix covered by columns. */ + uint longest_key= UINT_MAX; + int longest_len= 0; + key_map::Iterator key_it(possible_keys); + uint key; + while ((key= key_it++) != key_map::Iterator::BITMAP_END) + { + const KEY *keyinfo= table->key_info + key; + + /* Find the length of index prefix covered by GROUP BY columns */ + int part; + for (part= 0; part < (int)keyinfo->usable_key_parts; part++) + { + uint field_index= keyinfo->key_part[part].field->field_index; + if (columns.find_first(field_index) == columns.NOT_FOUND) + break; + } + + if (part > 0) // At least one column is covered + { + /* Make sure the index has statistics available */ + if (!keyinfo->actual_rec_per_key(part - 1)) + { + possible_keys.clear_bit(key); + continue; + } + if (part > longest_len) + { + longest_len= part; + longest_key= key; + } + } + else + { + /* + The index can't cover even one-column prefix. Remove it from + consideration. + */ + possible_keys.clear_bit(key); + } + } + + if (longest_key == UINT_MAX) + break; // No indexes are usable, stop. + + possible_keys.clear_bit(longest_key); + /* Multiply cardinality by index prefix's cardinality */ + const KEY *keyinfo= table->key_info + longest_key; + double index_card= (rows2double(table->stat_records()) / + keyinfo->actual_rec_per_key(longest_len-1)); + + /* Safety in case of inconsistent statistics: */ + set_if_bigger(index_card, 1.0); + + Json_writer_object trace_idx(join->thd); + trace_idx.add("index_name", keyinfo->name) + .add("cardinality", index_card); + card *= index_card; + if (card > table_records_after_where) + goto whole_table; + + /* Remove the columns we've handled from consideration */ + for (int part= 0; part < longest_len; part++) + { + uint field_index= keyinfo->key_part[part].field->field_index; + size_t idx= columns.find_first(field_index); + if (idx != columns.NOT_FOUND) + columns.del(idx); + else + DBUG_ASSERT(0); // Can't happen, we've found it above. + } + + if (!columns.size()) + break; // If we've covered all columns, stop. + } + + /* + If there are some columns left for which we couldn't get cardinality + from index statistics, try getting it from columns' histograms + */ + for (size_t i=0; i < columns.size(); i++) + { + double freq; + Field *field= table->field[columns.at(i)]; + if (!field->read_stats || + (freq= field->read_stats->get_avg_frequency()) == 0.0) + goto whole_table; + double column_card= rows2double(table->stat_records()) / freq; + Json_writer_object trace_col(join->thd); + trace_col.add("column", field->field_name) + .add("cardinality", column_card); + card *= column_card; + if (card > table_records_after_where) + goto whole_table; + } + +normal_exit: + trace_steps.end(); + trace_obj.add("cardinality", card); + return card; + +whole_table: + card= table_records_after_where; + goto normal_exit; +} + diff --git a/sql/sql_array.h b/sql/sql_array.h index d3b20af605e..812e8aae12b 100644 --- a/sql/sql_array.h +++ b/sql/sql_array.h @@ -182,6 +182,19 @@ public: return ((const Elem*)array.buffer) + array.elements - 1; } + static const size_t NOT_FOUND= (size_t)-1; + + /// @returns index of the first element equal to el, or NOT_FOUND + size_t find_first(const Elem &el) const + { + for (size_t i=0; i < size() ; i++) + { + if (el == at(i)) + return i; + } + return NOT_FOUND; + } + size_t size() const { return array.elements; } const Elem *end() const diff --git a/sql/sql_select.cc b/sql/sql_select.cc index d9b90c49196..a4c31658f0a 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -6357,6 +6357,10 @@ make_join_statistics(JOIN *join, List &tables_list, for (i= 0; i < join->table_count ; i++) if (double rr= join->best_positions[i].records_read) records= COST_MULT(records, rr); + + if (join->group_list) + records= estimate_post_group_cardinality(join, records); + rows= double_to_rows(records); set_if_smaller(rows, unit->lim.get_select_limit()); join->select_lex->increase_derived_records(rows); diff --git a/sql/sql_select.h b/sql/sql_select.h index 9f2cc464e2e..3bae2203b90 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -2712,6 +2712,8 @@ void propagate_new_equalities(THD *thd, Item *cond, #define PREV_BITS(type, N_BITS) ((type)my_set_bits(N_BITS)) +double estimate_post_group_cardinality(JOIN *join, double join_output_card); + bool dbug_user_var_equals_str(THD *thd, const char *name, const char *value); #endif /* SQL_SELECT_INCLUDED */ From 3009b5439dd2fc88b1e255b3ee57d32333829fb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 10 Feb 2025 09:19:23 +0200 Subject: [PATCH 28/47] MDEV-35941 : galera_bf_abort_lock_table fails with wait for metadata lock Problem was missing case from wsrep_handle_mdl_conflict. Test case was trying to confirm that LOCK TABLE thread is not BF-aborted. However as case was missing it was BF-aborted. Test case passed because BF-aborting takes time and used wait condition might see expected thread status before it was BF-aborted. Test naturally failed if BF-aborting was done early enough. Fix is to add missing case for SQLCOM_LOCK_TABLES to wsrep_handle_mdl_conflict. Note that using LOCK TABLE is still not recomended on cluster because it could cause cluster hang. Signed-off-by: Julius Goryavsky --- .../suite/galera/t/galera_bf_abort_lock_table.cnf | 5 ----- .../suite/galera/t/galera_bf_abort_lock_table.test | 5 ++--- sql/wsrep_mysqld.cc | 13 +++++++++---- 3 files changed, 11 insertions(+), 12 deletions(-) delete mode 100644 mysql-test/suite/galera/t/galera_bf_abort_lock_table.cnf diff --git a/mysql-test/suite/galera/t/galera_bf_abort_lock_table.cnf b/mysql-test/suite/galera/t/galera_bf_abort_lock_table.cnf deleted file mode 100644 index 033e6f8b99a..00000000000 --- a/mysql-test/suite/galera/t/galera_bf_abort_lock_table.cnf +++ /dev/null @@ -1,5 +0,0 @@ -!include ../galera_2nodes.cnf - -[mysqld.1] -wsrep-debug=1 -loose-galera-bf-abort-lock-table=1 diff --git a/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test b/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test index 06009712c7b..fe8aea9f248 100644 --- a/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test +++ b/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test @@ -1,6 +1,5 @@ --source include/galera_cluster.inc --source include/have_innodb.inc ---source include/force_restart.inc # # Test that a local LOCK TABLE will NOT be broken by an incoming remote transaction against that table @@ -20,13 +19,13 @@ INSERT INTO t1 VALUES (2); SET SESSION wsrep_sync_wait = 0; --let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'Waiting for table metadata lock' --let $wait_condition_on_error_output = SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST ---source include/wait_condition_with_debug_and_kill.inc +--source include/wait_condition_with_debug.inc UNLOCK TABLES; --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'Waiting for table metadata lock' --let $wait_condition_on_error_output = SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST ---source include/wait_condition_with_debug_and_kill.inc +--source include/wait_condition_with_debug.inc COMMIT; SELECT COUNT(*) = 1 FROM t1; diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index 4e240432939..de40c24bebe 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -3191,11 +3191,9 @@ void wsrep_to_isolation_end(THD *thd) @param requestor_ctx The MDL context of the requestor @param ticket MDL ticket for the requested lock + @param key The key of the object (data) being protected - @retval TRUE Lock request can be granted - @retval FALSE Lock request cannot be granted */ - void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx, const MDL_ticket *ticket, const MDL_key *key) @@ -3268,14 +3266,21 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx, (granted_thd->system_thread != NON_SYSTEM_THREAD && granted_thd->mdl_context.has_explicit_locks())) { - WSREP_DEBUG("BF thread waiting for FLUSH"); + WSREP_DEBUG("BF thread waiting for %s", + granted_thd->lex->sql_command == SQLCOM_FLUSH ? "FLUSH" : "BACKUP"); ticket->wsrep_report(wsrep_debug); + if (granted_thd->current_backup_stage != BACKUP_FINISHED && wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP)) { wsrep_abort_thd(request_thd, granted_thd, 1); } } + else if (granted_thd->lex->sql_command == SQLCOM_LOCK_TABLES) + { + WSREP_DEBUG("BF thread waiting for LOCK TABLES"); + ticket->wsrep_report(wsrep_debug); + } else if (request_thd->lex->sql_command == SQLCOM_DROP_TABLE) { WSREP_DEBUG("DROP caused BF abort, conf %s", From 44e1f7238aab19a4cb530d4e2ad84b394b633f75 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Wed, 12 Feb 2025 01:29:09 +0100 Subject: [PATCH 29/47] MDEV-35941 addendum: additional corrections for mtr tests --- mysql-test/suite/galera/r/mysql-wsrep#198.result | 3 +++ mysql-test/suite/galera/t/galera_bf_abort_lock_table.test | 4 ++-- mysql-test/suite/galera/t/mysql-wsrep#198.cnf | 2 -- mysql-test/suite/galera/t/mysql-wsrep#198.test | 8 ++++++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/galera/r/mysql-wsrep#198.result b/mysql-test/suite/galera/r/mysql-wsrep#198.result index 5b569ffae27..7759c4f1982 100644 --- a/mysql-test/suite/galera/r/mysql-wsrep#198.result +++ b/mysql-test/suite/galera/r/mysql-wsrep#198.result @@ -31,3 +31,6 @@ test.t1 repair note The storage engine for the table doesn't support repair test.t2 repair note The storage engine for the table doesn't support repair DROP TABLE t1; DROP TABLE t2; +connection node_1; +disconnect node_2a; +disconnect node_2b; diff --git a/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test b/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test index fe8aea9f248..71c3a7198f2 100644 --- a/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test +++ b/mysql-test/suite/galera/t/galera_bf_abort_lock_table.test @@ -17,13 +17,13 @@ INSERT INTO t1 VALUES (2); --connection node_2 SET SESSION wsrep_sync_wait = 0; ---let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'Waiting for table metadata lock' +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE LIKE 'Waiting for table metadata lock%' OR STATE LIKE 'Waiting to execute in isolation%'); --let $wait_condition_on_error_output = SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST --source include/wait_condition_with_debug.inc UNLOCK TABLES; ---let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'Waiting for table metadata lock' +--let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE LIKE 'Waiting for table metadata lock%' OR STATE LIKE 'Waiting to execute in isolation%'); --let $wait_condition_on_error_output = SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST --source include/wait_condition_with_debug.inc diff --git a/mysql-test/suite/galera/t/mysql-wsrep#198.cnf b/mysql-test/suite/galera/t/mysql-wsrep#198.cnf index bbeb0e31c31..4c62448fe3d 100644 --- a/mysql-test/suite/galera/t/mysql-wsrep#198.cnf +++ b/mysql-test/suite/galera/t/mysql-wsrep#198.cnf @@ -2,5 +2,3 @@ [mysqld] log-bin -wsrep-debug=1 -loose-mysql-wsrep198=1 diff --git a/mysql-test/suite/galera/t/mysql-wsrep#198.test b/mysql-test/suite/galera/t/mysql-wsrep#198.test index 2c89f859328..dceae6e17e9 100644 --- a/mysql-test/suite/galera/t/mysql-wsrep#198.test +++ b/mysql-test/suite/galera/t/mysql-wsrep#198.test @@ -1,6 +1,5 @@ --source include/galera_cluster.inc --source include/have_innodb.inc ---source include/force_restart.inc CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; CREATE TABLE t2 (id INT PRIMARY KEY) ENGINE=InnoDB; @@ -21,7 +20,7 @@ LOCK TABLE t2 WRITE; --connection node_2 SET SESSION wsrep_sync_wait = 0; ---let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE = 'Waiting for table metadata lock' +--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE STATE LIKE 'Waiting for table metadata lock%' OR STATE LIKE 'Waiting to execute in isolation%'; --let $wait_condition_on_error_output = SELECT * FROM INFORMATION_SCHEMA.PROCESSLIST --source include/wait_condition_with_debug_and_kill.inc @@ -39,3 +38,8 @@ UNLOCK TABLES; DROP TABLE t1; DROP TABLE t2; + +--connection node_1 + +--disconnect node_2a +--disconnect node_2b From c07e355c40379d37127082f5f5d419783ed778a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 12 Feb 2025 10:14:10 +0200 Subject: [PATCH 30/47] MDEV-36015: unrepresentable value in row_parse_int() row_parse_int(): Refactor the code and define the function static in one compilation unit. For any negative values, we must return 0. row_search_get_max_rec(), row_search_max_autoinc(): Moved to the same compilation unit with row_parse_int(). We also remove a work-around of an internal compiler error when targeting ARMv8 on GCC 4.8.5, a compiler that is no longer supported. Reviewed by: Debarun Banerjee --- .../suite/innodb/r/autoinc_persist.result | 37 +++++- .../suite/innodb/t/autoinc_persist.test | 21 +++- storage/innobase/include/row0row.h | 16 --- storage/innobase/include/row0row.inl | 49 -------- storage/innobase/include/row0sel.h | 5 +- storage/innobase/row/row0ins.cc | 109 ++++++++++++++++-- storage/innobase/row/row0sel.cc | 108 ----------------- 7 files changed, 151 insertions(+), 194 deletions(-) diff --git a/mysql-test/suite/innodb/r/autoinc_persist.result b/mysql-test/suite/innodb/r/autoinc_persist.result index 93594db3f05..59d708e5364 100644 --- a/mysql-test/suite/innodb/r/autoinc_persist.result +++ b/mysql-test/suite/innodb/r/autoinc_persist.result @@ -190,8 +190,7 @@ a 100000000000 100000000006 CREATE TABLE t11(a FLOAT AUTO_INCREMENT KEY) ENGINE = InnoDB; -INSERT INTO t11 VALUES(0), (0), (0), (0), (-1), (-10), (0), -(20), (30), (31); +INSERT INTO t11 VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); SELECT * FROM t11; a -10 @@ -204,9 +203,22 @@ a 20 30 31 +CREATE TABLE t11u(a FLOAT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t11u VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); +ERROR 22003: Out of range value for column 'a' at row 5 +INSERT INTO t11u VALUES(0), (0), (0), (0), (0), (20), (30), (31); +SELECT * FROM t11u; +a +11 +12 +13 +14 +15 +20 +30 +31 CREATE TABLE t12(a DOUBLE AUTO_INCREMENT KEY) ENGINE = InnoDB; -INSERT INTO t12 VALUES(0), (0), (0), (0), (-1), (-10), (0), -(20), (30), (31); +INSERT INTO t12 VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); SELECT * FROM t12; a -10 @@ -219,6 +231,20 @@ a 20 30 31 +CREATE TABLE t12u(a DOUBLE UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +INSERT INTO t12u VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); +ERROR 22003: Out of range value for column 'a' at row 5 +INSERT INTO t12u VALUES(0), (0), (0), (0), (0), (20), (30), (31); +SELECT * FROM t12u; +a +11 +12 +13 +14 +15 +20 +30 +31 # Scenario 1: Normal restart, to test if the counters are persisted # Scenario 2: Delete some values, to test the counters should not be the # one which is the largest in current table @@ -981,4 +1007,5 @@ a b 10 1 2 2 3 4 -DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t30, t32, t33; +DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t11u, t12u, +t30, t32, t33; diff --git a/mysql-test/suite/innodb/t/autoinc_persist.test b/mysql-test/suite/innodb/t/autoinc_persist.test index a2832bd5187..5364c933f57 100644 --- a/mysql-test/suite/innodb/t/autoinc_persist.test +++ b/mysql-test/suite/innodb/t/autoinc_persist.test @@ -82,15 +82,25 @@ INSERT INTO t10 VALUES(0), (0), (0), (0), (8), (10), (0), SELECT * FROM t10; CREATE TABLE t11(a FLOAT AUTO_INCREMENT KEY) ENGINE = InnoDB; -INSERT INTO t11 VALUES(0), (0), (0), (0), (-1), (-10), (0), -(20), (30), (31); +INSERT INTO t11 VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); SELECT * FROM t11; +CREATE TABLE t11u(a FLOAT UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +--error ER_WARN_DATA_OUT_OF_RANGE +INSERT INTO t11u VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); +INSERT INTO t11u VALUES(0), (0), (0), (0), (0), (20), (30), (31); +SELECT * FROM t11u; + CREATE TABLE t12(a DOUBLE AUTO_INCREMENT KEY) ENGINE = InnoDB; -INSERT INTO t12 VALUES(0), (0), (0), (0), (-1), (-10), (0), -(20), (30), (31); +INSERT INTO t12 VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); SELECT * FROM t12; +CREATE TABLE t12u(a DOUBLE UNSIGNED AUTO_INCREMENT KEY) ENGINE = InnoDB; +--error ER_WARN_DATA_OUT_OF_RANGE +INSERT INTO t12u VALUES(0), (0), (0), (0), (-1), (-10), (0), (20), (30), (31); +INSERT INTO t12u VALUES(0), (0), (0), (0), (0), (20), (30), (31); +SELECT * FROM t12u; + --echo # Scenario 1: Normal restart, to test if the counters are persisted --echo # Scenario 2: Delete some values, to test the counters should not be the --echo # one which is the largest in current table @@ -556,4 +566,5 @@ INSERT INTO t33 VALUES(3, NULL); SELECT MAX(b) AS `Expect 4` FROM t33; SELECT * FROM t33; -DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t30, t32, t33; +DROP TABLE t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t11u, t12u, +t30, t32, t33; diff --git a/storage/innobase/include/row0row.h b/storage/innobase/include/row0row.h index 7056c77f2e6..63427d597fc 100644 --- a/storage/innobase/include/row0row.h +++ b/storage/innobase/include/row0row.h @@ -328,22 +328,6 @@ row_get_clust_rec( mtr_t* mtr) /*!< in: mtr */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -/** Parse the integer data from specified data, which could be -DATA_INT, DATA_FLOAT or DATA_DOUBLE. If the value is less than 0 -and the type is not unsigned then we reset the value to 0 -@param[in] data data to read -@param[in] len length of data -@param[in] mtype mtype of data -@param[in] unsigned_type if the data is unsigned -@return the integer value from the data */ -inline -ib_uint64_t -row_parse_int( - const byte* data, - ulint len, - ulint mtype, - bool unsigned_type); - /** Result of row_search_index_entry */ enum row_search_result { ROW_FOUND = 0, /*!< the record was found */ diff --git a/storage/innobase/include/row0row.inl b/storage/innobase/include/row0row.inl index e89adb581f4..0ccf00b9b45 100644 --- a/storage/innobase/include/row0row.inl +++ b/storage/innobase/include/row0row.inl @@ -170,52 +170,3 @@ row_build_row_ref_fast( } } } - -/** Parse the integer data from specified data, which could be -DATA_INT, DATA_FLOAT or DATA_DOUBLE. If the value is less than 0 -and the type is not unsigned then we reset the value to 0 -@param[in] data data to read -@param[in] len length of data -@param[in] mtype mtype of data -@param[in] unsigned_type if the data is unsigned -@return the integer value from the data */ -ib_uint64_t -row_parse_int( - const byte* data, - ulint len, - ulint mtype, - bool unsigned_type) -{ - ib_uint64_t value = 0; - - switch (mtype) { - case DATA_INT: - - ut_a(len <= sizeof value); - value = mach_read_int_type(data, len, unsigned_type); - break; - - case DATA_FLOAT: - - ut_a(len == sizeof(float)); - value = static_cast(mach_float_read(data)); - break; - - case DATA_DOUBLE: - - ut_a(len == sizeof(double)); - value = static_cast(mach_double_read(data)); - break; - - default: - ut_error; - - } - - if (!unsigned_type && static_cast(value) < 0) { - value = 0; - } - - return(value); -} - diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index 54e4a1d283f..35e3cbe6631 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -182,9 +182,8 @@ dberr_t row_check_index(row_prebuilt_t *prebuilt, ulint *n_rows) @param[in] index index starting with an AUTO_INCREMENT column @return the largest AUTO_INCREMENT value @retval 0 if no records were found */ -ib_uint64_t -row_search_max_autoinc(dict_index_t* index) - MY_ATTRIBUTE((nonnull, warn_unused_result)); +uint64_t row_search_max_autoinc(dict_index_t *index) noexcept + MY_ATTRIBUTE((nonnull, warn_unused_result)); /** A structure for caching column values for prefetched rows */ struct sel_buf_t{ diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index a03415e218a..adc852725dd 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2560,12 +2560,44 @@ row_ins_index_entry_big_rec( return(error); } -#if defined __aarch64__&&defined __GNUC__&&__GNUC__==4&&!defined __clang__ -/* Avoid GCC 4.8.5 internal compiler error due to srw_mutex::wr_unlock(). -We would only need this for row_ins_clust_index_entry_low(), -but GCC 4.8.5 does not support pop_options. */ -# pragma GCC optimize ("O0") -#endif +/** Parse the integer data from specified data, which could be +DATA_INT, DATA_FLOAT or DATA_DOUBLE. If the value is less than 0 +and the type is not unsigned then we reset the value to 0 +@param data data to read +@param len length of data +@param mtype main type of the column +@param prtype precise type of the column +@return the integer value from the data +@retval 0 if the value is negative or the type or length invalid */ +static uint64_t row_parse_int(const byte *data, size_t len, + ulint mtype, ulint prtype) noexcept +{ + switch (mtype) { + case DATA_FLOAT: + if (len != sizeof(float)) + return 0; + { + float f= mach_float_read(data); + return f <= 0.0 ? 0 : uint64_t(f); + } + case DATA_DOUBLE: + if (len != sizeof(double)) + return 0; + { + double d= mach_double_read(data); + return d <= 0.0 ? 0 : uint64_t(d); + } + case DATA_INT: + if (len == 0 || len > 8) + return 0; + const ibool unsigned_type{prtype & DATA_UNSIGNED}; + uint64_t value= mach_read_int_type(data, len, unsigned_type); + return !unsigned_type && int64_t(value) < 0 ? 0 : value; + } + + ut_ad("invalid type" == 0); + return 0; +} /***************************************************************//** Tries to insert an entry into a clustered index, ignoring foreign key @@ -2652,8 +2684,7 @@ row_ins_clust_index_entry_low( dfield->data), dfield->len, dfield->type.mtype, - dfield->type.prtype - & DATA_UNSIGNED); + dfield->type.prtype); if (auto_inc && mode != BTR_MODIFY_TREE) { mode = btr_latch_mode( @@ -3810,3 +3841,65 @@ error_handling: return(thr); } + +/** Read the AUTOINC column from an index record +@param index index of the record +@param rec the record +@return value read from the first column +@retval 0 if the value would be NULL or negative */ +static uint64_t row_read_autoinc(const dict_index_t &index, const rec_t *rec) + noexcept +{ + const dict_field_t &field= index.fields[0]; + ut_ad(!DATA_BIG_COL(field.col)); + ut_ad(!(rec_get_info_bits(rec, index.table->not_redundant()) & + (REC_INFO_MIN_REC_FLAG | REC_INFO_DELETED_FLAG))); + mem_heap_t *heap= nullptr; + rec_offs offsets_[REC_OFFS_HEADER_SIZE + 2]; + rec_offs_init(offsets_); + rec_offs *offsets= rec_get_offsets(rec, &index, offsets_, + index.n_core_fields, 1, &heap); + ut_ad(!heap); + + size_t len; + ut_d(size_t first_offset=) rec_get_nth_field_offs(offsets, 0, &len); + ut_ad(!first_offset); + return row_parse_int(rec, len, field.col->mtype, field.col->prtype); +} + +/** Get the maximum and non-delete-marked record in an index. +@param index index B-tree +@param mtr mini-transaction (may be committed and restarted) +@return maximum record, page s-latched in mtr +@retval nullptr if there are no records, or if all of them are delete-marked */ +static +const rec_t *row_search_get_max_rec(dict_index_t *index, mtr_t *mtr) noexcept +{ + btr_pcur_t pcur; + /* Open at the high/right end (false), and init cursor */ + if (pcur.open_leaf(false, index, BTR_SEARCH_LEAF, mtr) != DB_SUCCESS) + return nullptr; + + do + { + const page_t *page= btr_pcur_get_page(&pcur); + const rec_t *rec= page_find_rec_max_not_deleted(page); + if (page_rec_is_user_rec_low(rec - page)) + return rec; + btr_pcur_move_before_first_on_page(&pcur); + } + while (btr_pcur_move_to_prev(&pcur, mtr)); + + return nullptr; +} + +uint64_t row_search_max_autoinc(dict_index_t *index) noexcept +{ + uint64_t value= 0; + mtr_t mtr; + mtr.start(); + if (const rec_t *rec= row_search_get_max_rec(index, &mtr)) + value= row_read_autoinc(*index, rec); + mtr.commit(); + return value; +} diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 38b79af8da4..5200c10fe18 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -6856,111 +6856,3 @@ next_rec: goto rec_loop; } - -/*******************************************************************//** -Read the AUTOINC column from the current row. If the value is less than -0 and the type is not unsigned then we reset the value to 0. -@return value read from the column */ -static -ib_uint64_t -row_search_autoinc_read_column( -/*===========================*/ - dict_index_t* index, /*!< in: index to read from */ - const rec_t* rec, /*!< in: current rec */ - ulint col_no, /*!< in: column number */ - ulint mtype, /*!< in: column main type */ - ibool unsigned_type) /*!< in: signed or unsigned flag */ -{ - ulint len; - const byte* data; - ib_uint64_t value; - mem_heap_t* heap = NULL; - rec_offs offsets_[REC_OFFS_NORMAL_SIZE]; - rec_offs* offsets = offsets_; - - rec_offs_init(offsets_); - ut_ad(page_rec_is_leaf(rec)); - - offsets = rec_get_offsets(rec, index, offsets, index->n_core_fields, - col_no + 1, &heap); - - if (rec_offs_nth_sql_null(offsets, col_no)) { - /* There is no non-NULL value in the auto-increment column. */ - value = 0; - goto func_exit; - } - - data = rec_get_nth_field(rec, offsets, col_no, &len); - - value = row_parse_int(data, len, mtype, unsigned_type); - -func_exit: - if (UNIV_LIKELY_NULL(heap)) { - mem_heap_free(heap); - } - - return(value); -} - -/** Get the maximum and non-delete-marked record in an index. -@param[in] index index tree -@param[in,out] mtr mini-transaction (may be committed and restarted) -@return maximum record, page s-latched in mtr -@retval NULL if there are no records, or if all of them are delete-marked */ -static -const rec_t* -row_search_get_max_rec( - dict_index_t* index, - mtr_t* mtr) -{ - btr_pcur_t pcur; - const rec_t* rec; - /* Open at the high/right end (false), and init cursor */ - if (pcur.open_leaf(false, index, BTR_SEARCH_LEAF, mtr) != DB_SUCCESS) { - return nullptr; - } - - do { - const page_t* page; - - page = btr_pcur_get_page(&pcur); - rec = page_find_rec_max_not_deleted(page); - - if (page_rec_is_user_rec(rec)) { - break; - } else { - rec = NULL; - } - btr_pcur_move_before_first_on_page(&pcur); - } while (btr_pcur_move_to_prev(&pcur, mtr)); - - ut_ad(!rec - || !(rec_get_info_bits(rec, dict_table_is_comp(index->table)) - & (REC_INFO_MIN_REC_FLAG | REC_INFO_DELETED_FLAG))); - return(rec); -} - -/** Read the max AUTOINC value from an index. -@param[in] index index starting with an AUTO_INCREMENT column -@return the largest AUTO_INCREMENT value -@retval 0 if no records were found */ -ib_uint64_t -row_search_max_autoinc(dict_index_t* index) -{ - const dict_field_t* dfield = dict_index_get_nth_field(index, 0); - - ib_uint64_t value = 0; - - mtr_t mtr; - mtr.start(); - - if (const rec_t* rec = row_search_get_max_rec(index, &mtr)) { - value = row_search_autoinc_read_column( - index, rec, 0, - dfield->col->mtype, - dfield->col->prtype & DATA_UNSIGNED); - } - - mtr.commit(); - return(value); -} From 7587b0ec84e01b4fc977d5c9bb3f5be8f12e3d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 12 Feb 2025 14:24:19 +0200 Subject: [PATCH 31/47] MDEV-36061 Incorrect error handling on DDL with FULLTEXT INDEX row_create_index_for_mysql(): Tolerate DB_LOCK_TABLE_FULL better. fts_create_one_common_table(), fts_create_one_index_table(): Do not corrupt the error state of a non-active transaction object. fts_config_set_value(): Only run another statement if there was no error yet. --- mysql-test/suite/innodb_fts/r/index_table.result | 3 +++ mysql-test/suite/innodb_fts/t/index_table.test | 6 ++++++ storage/innobase/fts/fts0config.cc | 2 +- storage/innobase/fts/fts0fts.cc | 13 +++++++++---- storage/innobase/row/row0mysql.cc | 4 +--- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/innodb_fts/r/index_table.result b/mysql-test/suite/innodb_fts/r/index_table.result index 570e367a7d4..909a889db42 100644 --- a/mysql-test/suite/innodb_fts/r/index_table.result +++ b/mysql-test/suite/innodb_fts/r/index_table.result @@ -5,6 +5,9 @@ id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, title VARCHAR(200), content TEXT ) ENGINE= InnoDB; +SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +CREATE FULLTEXT INDEX idx ON articles (title, content); +ERROR HY000: Got error 11 "Resource temporarily unavailable" from storage engine InnoDB CREATE FULLTEXT INDEX idx ON articles (title, content); INSERT INTO articles (title, content) VALUES ('MySQL Tutorial','DBMS stands for MySQL DataBase ...'), diff --git a/mysql-test/suite/innodb_fts/t/index_table.test b/mysql-test/suite/innodb_fts/t/index_table.test index 4b484877be1..89c09053230 100644 --- a/mysql-test/suite/innodb_fts/t/index_table.test +++ b/mysql-test/suite/innodb_fts/t/index_table.test @@ -3,6 +3,9 @@ -- source include/have_innodb.inc -- source include/have_debug.inc +--disable_query_log +call mtr.add_suppression("InnoDB: \\(Deadlock\\) writing `use_stopword'"); +--enable_query_log SET @optimize=@@GLOBAL.INNODB_OPTIMIZE_FULLTEXT_ONLY; SET GLOBAL INNODB_OPTIMIZE_FULLTEXT_ONLY=1; @@ -14,6 +17,9 @@ CREATE TABLE articles ( content TEXT ) ENGINE= InnoDB; +--error ER_GET_ERRNO +SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +CREATE FULLTEXT INDEX idx ON articles (title, content); CREATE FULLTEXT INDEX idx ON articles (title, content); INSERT INTO articles (title, content) VALUES diff --git a/storage/innobase/fts/fts0config.cc b/storage/innobase/fts/fts0config.cc index 4566224e171..524f648676e 100644 --- a/storage/innobase/fts/fts0config.cc +++ b/storage/innobase/fts/fts0config.cc @@ -231,7 +231,7 @@ fts_config_set_value( n_rows_updated = trx->undo_no - undo_no; /* Check if we need to do an insert. */ - if (n_rows_updated == 0) { + if (error == DB_SUCCESS && n_rows_updated == 0) { info = pars_info_create(); pars_info_bind_varchar_literal( diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 83ae2827230..a878905ace4 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -37,6 +37,7 @@ Full Text Search interface #include "fts0plugin.h" #include "dict0stats.h" #include "btr0pcur.h" +#include "log.h" static const ulint FTS_MAX_ID_LEN = 32; @@ -1870,8 +1871,10 @@ fts_create_one_common_table( } } - ib::warn() << "Failed to create FTS common table " << fts_table_name; - trx->error_state = error; + ut_ad(trx->state == TRX_STATE_NOT_STARTED + || trx->error_state == error); + sql_print_warning("InnoDB: Failed to create FTS common table %s: %s", + fts_table_name, ut_strerr(error)); return NULL; } @@ -2055,8 +2058,10 @@ fts_create_one_index_table( } } - ib::warn() << "Failed to create FTS index table " << table_name; - trx->error_state = error; + ut_ad(trx->state == TRX_STATE_NOT_STARTED + || trx->error_state == error); + sql_print_warning("InnoDB: Failed to create FTS index table %s: %s", + table_name, ut_strerr(error)); return NULL; } diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index c72e71bf047..1db5ec04ff1 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2181,11 +2181,9 @@ row_create_index_for_mysql( index = node->index; - ut_ad(!index == (err != DB_SUCCESS)); - que_graph_free((que_t*) que_node_get_parent(thr)); - if (index && (index->type & DICT_FTS)) { + if (err == DB_SUCCESS && (index->type & DICT_FTS)) { err = fts_create_index_tables(trx, index, table->id); } From f1d7e0c17e33f77278e6226dd94aeb30fc856bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 13 Feb 2025 12:18:03 +0200 Subject: [PATCH 32/47] MDEV-35436 dict_stats_fetch_from_ps() unnecessarily holds exclusive dict_sys.latch dict_stats_fetch_from_ps(): Acquire dict_sys.latch as few times as possible, and release dict_sys.latch after invoking pars_sql(), so that we will not be unnecessarily holding dict_sys.latch while possibly waiting for data to be read into the buffer pool. --- storage/innobase/dict/dict0stats.cc | 165 ++++++++++++++-------------- storage/innobase/pars/pars0pars.cc | 5 - 2 files changed, 85 insertions(+), 85 deletions(-) diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 0bd899ccd32..4d1e28b912a 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -3485,9 +3485,7 @@ dict_stats_fetch_from_ps( dict_table_t* table) /*!< in/out: table */ { index_fetch_t index_fetch_arg; - trx_t* trx; pars_info_t* pinfo; - dberr_t ret; char db_utf8[MAX_DB_UTF8_LEN]; char table_utf8[MAX_TABLE_UTF8_LEN]; @@ -3501,34 +3499,36 @@ dict_stats_fetch_from_ps( MDL_ticket *mdl_table = nullptr, *mdl_index = nullptr; dict_table_t* table_stats = dict_table_open_on_name( TABLE_STATS_NAME, false, DICT_ERR_IGNORE_NONE); - if (table_stats) { - dict_sys.freeze(SRW_LOCK_CALL); - table_stats = dict_acquire_mdl_shared(table_stats, thd, - &mdl_table); - dict_sys.unfreeze(); + if (!table_stats) { + return DB_STATS_DO_NOT_EXIST; } + dict_table_t* index_stats = dict_table_open_on_name( + INDEX_STATS_NAME, false, DICT_ERR_IGNORE_NONE); + if (!index_stats) { + dict_table_close(table_stats); + return DB_STATS_DO_NOT_EXIST; + } + + dict_sys.freeze(SRW_LOCK_CALL); + table_stats = dict_acquire_mdl_shared(table_stats, thd, + &mdl_table); if (!table_stats || strcmp(table_stats->name.m_name, TABLE_STATS_NAME)) { release_and_exit: if (table_stats) { - dict_table_close(table_stats, false, thd, mdl_table); + dict_table_close(table_stats, true, thd, mdl_table); } + if (index_stats) { + dict_table_close(index_stats, true, thd, mdl_index); + } + dict_sys.unfreeze(); return DB_STATS_DO_NOT_EXIST; } - dict_table_t* index_stats = dict_table_open_on_name( - INDEX_STATS_NAME, false, DICT_ERR_IGNORE_NONE); - if (index_stats) { - dict_sys.freeze(SRW_LOCK_CALL); - index_stats = dict_acquire_mdl_shared(index_stats, thd, - &mdl_index); - dict_sys.unfreeze(); - } - if (!index_stats) { - goto release_and_exit; - } - if (strcmp(index_stats->name.m_name, INDEX_STATS_NAME)) { - dict_table_close(index_stats, false, thd, mdl_index); + index_stats = dict_acquire_mdl_shared(index_stats, thd, + &mdl_index); + if (!index_stats + || strcmp(index_stats->name.m_name, INDEX_STATS_NAME)) { goto release_and_exit; } @@ -3536,10 +3536,6 @@ release_and_exit: DEBUG_SYNC(thd, "dict_stats_mdl_acquired"); #endif /* ENABLED_DEBUG_SYNC */ - trx = trx_create(); - - trx_start_internal_read_only(trx); - dict_fs2utf8(table->name.m_name, db_utf8, sizeof(db_utf8), table_utf8, sizeof(table_utf8)); @@ -3560,76 +3556,85 @@ release_and_exit: "fetch_index_stats_step", dict_stats_fetch_index_stats_step, &index_fetch_arg); - dict_sys.lock(SRW_LOCK_CALL); /* FIXME: remove this */ - ret = que_eval_sql(pinfo, - "PROCEDURE FETCH_STATS () IS\n" - "found INT;\n" - "DECLARE FUNCTION fetch_table_stats_step;\n" - "DECLARE FUNCTION fetch_index_stats_step;\n" - "DECLARE CURSOR table_stats_cur IS\n" - " SELECT\n" - /* if you change the selected fields, be - sure to adjust - dict_stats_fetch_table_stats_step() */ - " n_rows,\n" - " clustered_index_size,\n" - " sum_of_other_index_sizes\n" - " FROM \"" TABLE_STATS_NAME "\"\n" - " WHERE\n" - " database_name = :database_name AND\n" - " table_name = :table_name;\n" - "DECLARE CURSOR index_stats_cur IS\n" - " SELECT\n" - /* if you change the selected fields, be - sure to adjust - dict_stats_fetch_index_stats_step() */ - " index_name,\n" - " stat_name,\n" - " stat_value,\n" - " sample_size\n" - " FROM \"" INDEX_STATS_NAME "\"\n" - " WHERE\n" - " database_name = :database_name AND\n" - " table_name = :table_name;\n" + dict_sys.unfreeze(); + dict_sys.lock(SRW_LOCK_CALL); + que_t* graph = pars_sql( + pinfo, + "PROCEDURE FETCH_STATS () IS\n" + "found INT;\n" + "DECLARE FUNCTION fetch_table_stats_step;\n" + "DECLARE FUNCTION fetch_index_stats_step;\n" + "DECLARE CURSOR table_stats_cur IS\n" + " SELECT\n" + /* if you change the selected fields, be + sure to adjust + dict_stats_fetch_table_stats_step() */ + " n_rows,\n" + " clustered_index_size,\n" + " sum_of_other_index_sizes\n" + " FROM \"" TABLE_STATS_NAME "\"\n" + " WHERE\n" + " database_name = :database_name AND\n" + " table_name = :table_name;\n" + "DECLARE CURSOR index_stats_cur IS\n" + " SELECT\n" + /* if you change the selected fields, be + sure to adjust + dict_stats_fetch_index_stats_step() */ + " index_name,\n" + " stat_name,\n" + " stat_value,\n" + " sample_size\n" + " FROM \"" INDEX_STATS_NAME "\"\n" + " WHERE\n" + " database_name = :database_name AND\n" + " table_name = :table_name;\n" - "BEGIN\n" + "BEGIN\n" - "OPEN table_stats_cur;\n" - "FETCH table_stats_cur INTO\n" - " fetch_table_stats_step();\n" - "IF (SQL % NOTFOUND) THEN\n" - " CLOSE table_stats_cur;\n" - " RETURN;\n" - "END IF;\n" - "CLOSE table_stats_cur;\n" + "OPEN table_stats_cur;\n" + "FETCH table_stats_cur INTO\n" + " fetch_table_stats_step();\n" + "IF (SQL % NOTFOUND) THEN\n" + " CLOSE table_stats_cur;\n" + " RETURN;\n" + "END IF;\n" + "CLOSE table_stats_cur;\n" - "OPEN index_stats_cur;\n" - "found := 1;\n" - "WHILE found = 1 LOOP\n" - " FETCH index_stats_cur INTO\n" - " fetch_index_stats_step();\n" - " IF (SQL % NOTFOUND) THEN\n" - " found := 0;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE index_stats_cur;\n" + "OPEN index_stats_cur;\n" + "found := 1;\n" + "WHILE found = 1 LOOP\n" + " FETCH index_stats_cur INTO\n" + " fetch_index_stats_step();\n" + " IF (SQL % NOTFOUND) THEN\n" + " found := 0;\n" + " END IF;\n" + "END LOOP;\n" + "CLOSE index_stats_cur;\n" - "END;", trx); - /* pinfo is freed by que_eval_sql() */ + "END;"); dict_sys.unlock(); + trx_t* trx = trx_create(); + trx->graph = nullptr; + graph->trx = trx; + + trx_start_internal_read_only(trx); + que_run_threads(que_fork_start_command(graph)); + que_graph_free(graph); + dict_table_close(table_stats, false, thd, mdl_table); dict_table_close(index_stats, false, thd, mdl_index); trx_commit_for_mysql(trx); - + dberr_t ret = trx->error_state; trx->free(); if (!index_fetch_arg.stats_were_modified) { - return(DB_STATS_DO_NOT_EXIST); + return DB_STATS_DO_NOT_EXIST; } - return(ret); + return ret; } /*********************************************************************//** diff --git a/storage/innobase/pars/pars0pars.cc b/storage/innobase/pars/pars0pars.cc index 51bcc9540fc..fff60b0bd46 100644 --- a/storage/innobase/pars/pars0pars.cc +++ b/storage/innobase/pars/pars0pars.cc @@ -783,11 +783,6 @@ pars_retrieve_table_list_defs( { ulint count = 0; - if (sym_node == NULL) { - - return(count); - } - while (sym_node) { pars_retrieve_table_def(sym_node); From 7e001b2a3cc1efc5ac182a179f061faacdf09085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 17 Feb 2025 15:55:58 +0200 Subject: [PATCH 33/47] MDEV-36082 Race condition between log_t::resize_start() and log_t::resize_abort() log_t::writer_update(): Add the parameter bool resizing, to indicate whether log resizing is in progress. We must enable log_writer_resizing only if resize_lsn>1, to ensure that log_t::resize_abort() will not choose the wrong log_sys.log_writer. log_t::resize_initiator: The thread that successfully invoked resize_start(). log_t::resize_start(): Simplify some logic, and assign resize_initiator if we successfully started log resizing. log_t::resize_abort(): Abort log resizing if we are the resize_initiator. innodb_log_file_size_update(): Clean up some logic. Reviewed by: Debarun Banerjee --- storage/innobase/buf/buf0flu.cc | 3 +- storage/innobase/handler/ha_innodb.cc | 13 +++--- storage/innobase/include/log0log.h | 17 +++++-- storage/innobase/log/log0log.cc | 66 +++++++++++++++------------ 4 files changed, 56 insertions(+), 43 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index ab4938136b0..5ef6af65f6f 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1912,7 +1912,8 @@ inline void log_t::write_checkpoint(lsn_t end_lsn) noexcept resize_flush_buf= nullptr; resize_target= 0; resize_lsn.store(0, std::memory_order_relaxed); - writer_update(); + resize_initiator= nullptr; + writer_update(false); } log_resize_release(); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 19a27510762..7e9439a760c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -18605,7 +18605,7 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, " innodb_log_buffer_size=%u", MYF(0), log_sys.buf_size); else { - switch (log_sys.resize_start(*static_cast(save))) { + switch (log_sys.resize_start(*static_cast(save), thd)) { case log_t::RESIZE_NO_CHANGE: break; case log_t::RESIZE_IN_PROGRESS: @@ -18617,12 +18617,11 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, ib_senderrf(thd, IB_LOG_LEVEL_ERROR, ER_CANT_CREATE_HANDLER_FILE); break; case log_t::RESIZE_STARTED: - const lsn_t start{log_sys.resize_in_progress()}; for (timespec abstime;;) { if (thd_kill_level(thd)) { - log_sys.resize_abort(); + log_sys.resize_abort(thd); break; } @@ -18637,13 +18636,15 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, resizing= log_sys.resize_in_progress(); } mysql_mutex_unlock(&buf_pool.flush_list_mutex); - if (start > log_sys.get_lsn()) + if (!resizing || !log_sys.resize_running(thd)) + break; + if (resizing > log_sys.get_lsn()) { ut_ad(!log_sys.is_mmap()); /* The server is almost idle. Write dummy FILE_CHECKPOINT records to ensure that the log resizing will complete. */ log_sys.latch.wr_lock(SRW_LOCK_CALL); - while (start > log_sys.get_lsn()) + while (resizing > log_sys.get_lsn()) { mtr_t mtr; mtr.start(); @@ -18651,8 +18652,6 @@ static void innodb_log_file_size_update(THD *thd, st_mysql_sys_var*, } log_sys.latch.wr_unlock(); } - if (!resizing || resizing > start /* only wait for our resize */) - break; } } } diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 2ca34a93789..33df9d39ba6 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -303,7 +303,6 @@ public: bool log_maybe_unbuffered; # endif #endif - /** Fields involved in checkpoints @{ */ lsn_t log_capacity; /*!< capacity of the log; if the checkpoint age exceeds this, it is @@ -326,6 +325,8 @@ public: /* @} */ private: + /** the thread that initiated resize_lsn() */ + Atomic_relaxed resize_initiator; /** A lock when the spin-only lock_lsn() is not being used */ log_lsn_lock lsn_lock; public: @@ -367,11 +368,17 @@ public: /** Start resizing the log and release the exclusive latch. @param size requested new file_size + @param thd the current thread identifier @return whether the resizing was started successfully */ - resize_start_status resize_start(os_offset_t size) noexcept; + resize_start_status resize_start(os_offset_t size, void *thd) noexcept; - /** Abort any resize_start(). */ - void resize_abort() noexcept; + /** Abort a resize_start() that we started. + @param thd thread identifier that had been passed to resize_start() */ + void resize_abort(void *thd) noexcept; + + /** @return whether a particular resize_start() is in progress */ + bool resize_running(void *thd) const noexcept + { return thd == resize_initiator; } /** Replicate a write to the log. @param lsn start LSN @@ -481,7 +488,7 @@ public: private: /** Update writer and mtr_t::finisher */ - void writer_update() noexcept; + void writer_update(bool resizing) noexcept; /** Wait in append_prepare() for buffer to become available @tparam spin whether to use the spin-only lock_lsn() diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 0e9d4c9ad45..d35f66a78ec 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -353,7 +353,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) # endif buf= static_cast(ptr); max_buf_free= 1; - writer_update(); + writer_update(false); # ifdef HAVE_PMEM if (is_pmem) return true; @@ -395,7 +395,7 @@ bool log_t::attach(log_file_t file, os_offset_t size) TRASH_ALLOC(buf, buf_size); TRASH_ALLOC(flush_buf, buf_size); max_buf_free= buf_size / LOG_BUF_FLUSH_RATIO - LOG_BUF_FLUSH_MARGIN; - writer_update(); + writer_update(false); memset_aligned<512>(checkpoint_buf, 0, write_size); func_exit: @@ -570,31 +570,35 @@ void log_t::set_buffered(bool buffered) /** Start resizing the log and release the exclusive latch. @param size requested new file_size +@param thd the current thread identifier @return whether the resizing was started successfully */ -log_t::resize_start_status log_t::resize_start(os_offset_t size) noexcept +log_t::resize_start_status log_t::resize_start(os_offset_t size, void *thd) + noexcept { ut_ad(size >= 4U << 20); ut_ad(!(size & 4095)); ut_ad(!srv_read_only_mode); + ut_ad(thd); log_resize_acquire(); - resize_start_status status= RESIZE_NO_CHANGE; - lsn_t start_lsn{0}; -#ifdef HAVE_PMEM - bool is_pmem{false}; -#endif + resize_start_status status; - if (resize_in_progress()) + if (size == file_size) + status= RESIZE_NO_CHANGE; + else if (resize_in_progress()) status= RESIZE_IN_PROGRESS; - else if (size != file_size) + else { + lsn_t start_lsn; ut_ad(!resize_in_progress()); ut_ad(!resize_log.is_opened()); ut_ad(!resize_buf); ut_ad(!resize_flush_buf); + ut_ad(!resize_initiator); std::string path{get_log_file_path("ib_logfile101")}; bool success; + resize_initiator= thd; resize_lsn.store(1, std::memory_order_relaxed); resize_target= 0; resize_log.m_file= @@ -612,6 +616,7 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size) noexcept #ifdef HAVE_PMEM else if (is_mmap()) { + bool is_pmem{false}; ptr= ::log_mmap(resize_log.m_file, is_pmem, size); if (ptr == MAP_FAILED) @@ -661,34 +666,33 @@ log_t::resize_start_status log_t::resize_start(os_offset_t size) noexcept else if (!is_opened()) resize_log.close(); - writer_update(); + resize_lsn.store(start_lsn, std::memory_order_relaxed); + writer_update(true); + log_resize_release(); + + mysql_mutex_lock(&buf_pool.flush_list_mutex); + lsn_t target_lsn= buf_pool.get_oldest_modification(0); + mysql_mutex_unlock(&buf_pool.flush_list_mutex); + buf_flush_ahead(start_lsn < target_lsn ? target_lsn + 1 : start_lsn, + false); + return RESIZE_STARTED; } - status= success ? RESIZE_STARTED : RESIZE_FAILED; } - resize_lsn.store(start_lsn, std::memory_order_relaxed); + resize_initiator= nullptr; + resize_lsn.store(0, std::memory_order_relaxed); + status= RESIZE_FAILED; } log_resize_release(); - - if (start_lsn) - { - mysql_mutex_lock(&buf_pool.flush_list_mutex); - lsn_t target_lsn= buf_pool.get_oldest_modification(0); - if (start_lsn < target_lsn) - start_lsn= target_lsn + 1; - mysql_mutex_unlock(&buf_pool.flush_list_mutex); - buf_flush_ahead(start_lsn, false); - } - return status; } -/** Abort log resizing. */ -void log_t::resize_abort() noexcept +/** Abort a resize_start() that we started. */ +void log_t::resize_abort(void *thd) noexcept { log_resize_acquire(); - if (resize_in_progress() > 1) + if (resize_running(thd)) { #ifdef HAVE_PMEM const bool is_mmap{this->is_mmap()}; @@ -715,11 +719,12 @@ void log_t::resize_abort() noexcept resize_buf= nullptr; resize_target= 0; resize_lsn.store(0, std::memory_order_relaxed); + resize_initiator= nullptr; std::string path{get_log_file_path("ib_logfile101")}; IF_WIN(DeleteFile(path.c_str()), unlink(path.c_str())); + writer_update(false); } - writer_update(); log_resize_release(); } @@ -1190,10 +1195,11 @@ ATTRIBUTE_COLD static lsn_t log_writer_resizing() noexcept return log_sys.write_buf(); } -void log_t::writer_update() noexcept +void log_t::writer_update(bool resizing) noexcept { ut_ad(latch_have_wr()); - writer= resize_in_progress() ? log_writer_resizing : log_writer; + ut_ad(resizing == (resize_in_progress() > 1)); + writer= resizing ? log_writer_resizing : log_writer; mtr_t::finisher_update(); } From a20c8fabe703d951c26a35f649e1a0c303a27610 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Wed, 19 Feb 2025 10:43:36 +0000 Subject: [PATCH 34/47] Fix sporadic failure of rpl.rpl_parallel_innodb_lock_conflict Make sure the table mysql.gtid_slave_pos is altered to InnoDB before starting parallel replication. The parallel replication of the suppression insertion in the test case was trying to update the GTID position in parallel with the ALTER TABLE, which could occasionally deadlock on the MDL lock. Reviewed-by: Monty Signed-off-by: Kristian Nielsen --- .../rpl_parallel_innodb_lock_conflict.result | 7 +++---- .../rpl/r/rpl_parallel_innodb_lock_conflict.result | 7 +++---- .../rpl/t/rpl_parallel_innodb_lock_conflict.test | 12 +++++------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/mysql-test/suite/binlog_encryption/rpl_parallel_innodb_lock_conflict.result b/mysql-test/suite/binlog_encryption/rpl_parallel_innodb_lock_conflict.result index 1411db16af6..a3d87641622 100644 --- a/mysql-test/suite/binlog_encryption/rpl_parallel_innodb_lock_conflict.result +++ b/mysql-test/suite/binlog_encryption/rpl_parallel_innodb_lock_conflict.result @@ -1,16 +1,15 @@ ***MDEV-5914: Parallel replication deadlock due to InnoDB lock conflicts *** include/master-slave.inc [connection master] -connection server_2; -SET sql_log_bin=0; +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CALL mtr.add_suppression("InnoDB: Transaction was aborted due to "); CALL mtr.add_suppression("Commit failed due to failure of an earlier commit on which this one depends"); -SET sql_log_bin=1; +connection server_2; SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; include/stop_slave.inc SET GLOBAL slave_parallel_threads=10; CHANGE MASTER TO master_use_gtid=slave_pos; connection server_1; -ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t4 (a INT PRIMARY KEY, b INT, KEY b_idx(b)) ENGINE=InnoDB; INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); connect con1,127.0.0.1,root,,test,$SERVER_MYPORT_1,; diff --git a/mysql-test/suite/rpl/r/rpl_parallel_innodb_lock_conflict.result b/mysql-test/suite/rpl/r/rpl_parallel_innodb_lock_conflict.result index 1411db16af6..a3d87641622 100644 --- a/mysql-test/suite/rpl/r/rpl_parallel_innodb_lock_conflict.result +++ b/mysql-test/suite/rpl/r/rpl_parallel_innodb_lock_conflict.result @@ -1,16 +1,15 @@ ***MDEV-5914: Parallel replication deadlock due to InnoDB lock conflicts *** include/master-slave.inc [connection master] -connection server_2; -SET sql_log_bin=0; +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CALL mtr.add_suppression("InnoDB: Transaction was aborted due to "); CALL mtr.add_suppression("Commit failed due to failure of an earlier commit on which this one depends"); -SET sql_log_bin=1; +connection server_2; SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; include/stop_slave.inc SET GLOBAL slave_parallel_threads=10; CHANGE MASTER TO master_use_gtid=slave_pos; connection server_1; -ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t4 (a INT PRIMARY KEY, b INT, KEY b_idx(b)) ENGINE=InnoDB; INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); connect con1,127.0.0.1,root,,test,$SERVER_MYPORT_1,; diff --git a/mysql-test/suite/rpl/t/rpl_parallel_innodb_lock_conflict.test b/mysql-test/suite/rpl/t/rpl_parallel_innodb_lock_conflict.test index 532eb58571c..47fb8ff8c75 100644 --- a/mysql-test/suite/rpl/t/rpl_parallel_innodb_lock_conflict.test +++ b/mysql-test/suite/rpl/t/rpl_parallel_innodb_lock_conflict.test @@ -5,21 +5,19 @@ --source include/have_debug_sync.inc --source include/master-slave.inc ---disable_query_log -call mtr.add_suppression("InnoDB: Transaction was aborted due to "); ---enable_query_log +ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; +CALL mtr.add_suppression("InnoDB: Transaction was aborted due to "); +CALL mtr.add_suppression("Commit failed due to failure of an earlier commit on which this one depends"); +--save_master_pos --connection server_2 -SET sql_log_bin=0; -CALL mtr.add_suppression("Commit failed due to failure of an earlier commit on which this one depends"); -SET sql_log_bin=1; +--sync_with_master SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; --source include/stop_slave.inc SET GLOBAL slave_parallel_threads=10; CHANGE MASTER TO master_use_gtid=slave_pos; --connection server_1 -ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t4 (a INT PRIMARY KEY, b INT, KEY b_idx(b)) ENGINE=InnoDB; INSERT INTO t4 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6); --connect (con1,127.0.0.1,root,,test,$SERVER_MYPORT_1,) From bac2358c9dafd2df9d0e92293097615032ae958b Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 23 Feb 2025 16:59:04 +0200 Subject: [PATCH 35/47] Removed outdated code comment --- sql/sql_table.cc | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sql/sql_table.cc b/sql/sql_table.cc index c20fb8d9bc4..16bb9696ac7 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -1728,18 +1728,6 @@ err: if (non_temp_tables_count) query_cache_invalidate3(thd, tables, 0); - /* - We are always logging drop of temporary tables. - The reason is to handle the following case: - - Use statement based replication - - CREATE TEMPORARY TABLE foo (logged) - - set row based replication - - DROP TEMPORARY TABLE foo (needs to be logged) - This should be fixed so that we remember if creation of the - temporary table was logged and only log it if the creation was - logged. - */ - if (non_trans_tmp_table_deleted || trans_tmp_table_deleted || non_tmp_table_deleted) { From 49d976feaa9de7d7d1cdde1560d43e637f7b607b Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Wed, 15 Jan 2025 18:10:16 +1100 Subject: [PATCH 36/47] MDEV-29605 Reset queued ping info of all spider connections associated with a closed spider handler A spider_conn may outlive its associated ha_spider (in the field queued_ping_spider) used for connecting to and pinging the data node (a call to spider_db_ping(), guarded by the boolean field queued_ping). In a call to ha_spider::close() (which is often preceded with the deletion of the ha_spider itself), many cleanups happen, including freeing the associated spider_share, which is used by the spider_conn in spider_db_ping. Therefore it is necessary to reset both the queued_ping_spider and queued_ping fields, so that any further spider interaction with the data node will not trigger the call using the ha_spider including its freed spider_share. Also out of caution added an assert and internal error in case a connection has not been established (the db_conn field of type MYSQL * is NULL), and attempt to connect is skipped because both queued_connect and queued_ping are false. Note that this unlikely (if not impossible) scenario would not be a regression caused by this change, as it strictly falls under the scenario of this bug. --- storage/spider/ha_spider.cc | 23 +++++++++++++++++ .../spider/bugfix/r/mdev_29605.result | 19 ++++++++++++++ .../spider/bugfix/t/mdev_29605.test | 25 +++++++++++++++++++ storage/spider/spd_db_mysql.cc | 4 +++ 4 files changed, 71 insertions(+) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_29605.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_29605.test diff --git a/storage/spider/ha_spider.cc b/storage/spider/ha_spider.cc index b1df83de162..ef529470a60 100644 --- a/storage/spider/ha_spider.cc +++ b/storage/spider/ha_spider.cc @@ -468,6 +468,28 @@ static void spider_update_current_trx_ha_with_freed_share(SPIDER_SHARE *share) } } +/* + Given an ha_spider that is being closed, reset the queued ping info + of SPIDER_CONN of the current spider trx that has the given + ha_spider as the queued_ping_spider. +*/ +static void spider_reset_conn_queued_ping(ha_spider *spider) +{ + SPIDER_TRX *trx= spider_current_trx; + if (trx) + { + for (uint i= 0; i < trx->trx_conn_hash.records; i++) + { + SPIDER_CONN *conn= (SPIDER_CONN *) my_hash_element(&trx->trx_conn_hash, i); + if (conn->queued_ping_spider == spider) + { + conn->queued_ping= FALSE; + conn->queued_ping_spider= NULL; + } + } + } +} + int ha_spider::close() { int error_num= 0, roop_count; @@ -562,6 +584,7 @@ int ha_spider::close() result_list.tmp_sqls = NULL; } + spider_reset_conn_queued_ping(this); spider_update_current_trx_ha_with_freed_share(share); spider_free_share(share); is_clone = FALSE; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29605.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29605.result new file mode 100644 index 00000000000..15265d739dd --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29605.result @@ -0,0 +1,19 @@ +for master_1 +for child2 +for child3 +set spider_same_server_link= 1; +CREATE USER spider@localhost IDENTIFIED BY 'pwd'; +GRANT ALL ON test.* TO spider@localhost; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'spider', password 'pwd'); +SET autocommit=0; +set @old_init_connect=@@global.init_connect; +set global init_connect="dummy"; +CREATE TABLE t ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"' AS SELECT 1; +Got one of the listed errors +set global init_connect=@old_init_connect; +drop server srv; +drop user spider@localhost; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29605.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29605.test new file mode 100644 index 00000000000..bb751c73f9b --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29605.test @@ -0,0 +1,25 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +set spider_same_server_link= 1; +CREATE USER spider@localhost IDENTIFIED BY 'pwd'; +GRANT ALL ON test.* TO spider@localhost; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'spider', password 'pwd'); +SET autocommit=0; +set @old_init_connect=@@global.init_connect; +set global init_connect="dummy"; +--error ER_NET_ERROR_ON_WRITE,12701 +CREATE TABLE t ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"' AS SELECT 1; +set global init_connect=@old_init_connect; +drop server srv; +drop user spider@localhost; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index aa91a6963a4..63d29e69de4 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -2100,6 +2100,10 @@ int spider_db_mbase::exec_query( general_log_write(current_thd, COM_QUERY, tmp_query_str.ptr(), tmp_query_str.length()); } + /* There should be a live connection to the data node */ + DBUG_ASSERT(db_conn); + if (!db_conn) + DBUG_RETURN(HA_ERR_INTERNAL_ERROR); if (!spider_param_dry_access()) { error_num = mysql_real_query(db_conn, query, length); From 809a0cebdc4811bfdce20bc3173cc4bae4275418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Feb 2025 11:41:49 +0200 Subject: [PATCH 37/47] MDEV-36152 mariadb-backup --backup crash during innodb_undo_log_truncate=ON, innodb_encrypt_log=ON recv_sys_t::parse(): Allocate decrypt_buf also for storing==BACKUP but limit its size to correspond to 1 byte of record payload. Ensure that last_offset=0 for storing==BACKUP. When parsing INIT_PAGE or FREE_PAGE, avoid an unnecessary l.copy_if_needed() for storing!=YES. When parsing EXTENDED in storing==BACKUP, properly invoke l.copy_if_needed() on a large enough decrypt_buf. When parsing WRITE, MEMMOVE, MEMSET in storing==BACKUP, skip further validation (and potential overflow of the tiny decrypt_buf), like we used to do before commit 46aaf328ce424aededdb61e59a48db05630563d5 (MDEV-35830). Reviewed by: Debarun Banerjee --- storage/innobase/log/log0recv.cc | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 4b794bc7fee..3c7b63bdfed 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2507,9 +2507,11 @@ restart: ut_ad(log_sys.is_latest()); alignas(8) byte iv[MY_AES_BLOCK_SIZE]; - byte *decrypt_buf= storing != BACKUP - ? static_cast(alloca(srv_page_size)) : nullptr; - + byte *decrypt_buf= + static_cast(alloca(storing == BACKUP + ? 1/*type,length*/ + 5/*space_id*/ + + 5/*page_no*/ + 1/*rlen*/ + : srv_page_size)); const lsn_t start_lsn{lsn}; /* Check that the entire mini-transaction is included within the buffer */ @@ -2599,7 +2601,10 @@ restart: ut_d(std::set modified); #endif - uint32_t space_id= 0, page_no= 0, last_offset= 0; + uint32_t space_id= 0, page_no= 0; + /* The end offset the last write (always 0 in storing==BACKUP). + The value 1 means that no "same page" record is allowed. */ + uint last_offset= 0; bool got_page_op= false; for (l= begin;; l+= rlen) @@ -2712,8 +2717,7 @@ restart: { mach_write_to_4(iv + 8, space_id); mach_write_to_4(iv + 12, page_no); - byte eb[1/*type,length*/ + 5/*space_id*/ + 5/*page_no*/ + 1/*rlen*/]; - if (*l.copy_if_needed(iv, eb, recs, 1) == TRIM_PAGES) + if (*l.copy_if_needed(iv, decrypt_buf, recs, 1) == TRIM_PAGES) undo_space_trunc(space_id); } continue; @@ -2762,10 +2766,10 @@ restart: case FREE_PAGE: ut_ad(freed.emplace(id).second); /* the next record must not be same_page */ - last_offset= 1; + if (storing != BACKUP) last_offset= 1; goto free_or_init_page; case INIT_PAGE: - last_offset= FIL_PAGE_TYPE; + if (storing != BACKUP) last_offset= FIL_PAGE_TYPE; free_or_init_page: if (UNIV_UNLIKELY(rlen != 0)) goto record_corrupted; @@ -2797,7 +2801,8 @@ restart: erase(r); continue; } - cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen); + if (storing == YES) + cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen); break; case EXTENDED: if (storing == NO) @@ -2811,7 +2816,8 @@ restart: continue; if (UNIV_UNLIKELY(!rlen)) goto record_corrupted; - cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen); + if (storing == YES || rlen == 1) + cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen); if (rlen == 1 && *cl == TRIM_PAGES) { if (!srv_is_undo_tablespace(space_id) || @@ -2825,7 +2831,7 @@ restart: truncated_undo_spaces[space_id - srv_undo_space_id_start]= { start_lsn, page_no }; /* the next record must not be same_page */ - last_offset= 1; + if (storing != BACKUP) last_offset= 1; if (undo_space_trunc) undo_space_trunc(space_id); continue; @@ -2833,7 +2839,7 @@ restart: /* This record applies to an undo log or index page, and it may be followed by subsequent WRITE or similar records for the same page in the same mini-transaction. */ - last_offset= FIL_PAGE_TYPE; + if (storing != BACKUP) last_offset= FIL_PAGE_TYPE; break; case OPTION: /* OPTION records can be safely ignored in recovery */ @@ -2850,6 +2856,8 @@ restart: case WRITE: case MEMMOVE: case MEMSET: + if (storing == BACKUP) + continue; if (storing == NO && UNIV_LIKELY(page_no != 0)) /* fil_space_set_recv_size_and_flags() is mandatory for storing==NO. It is only applicable to page_no == 0. Other than that, we can just From aea440d3e7fd0dd74e884da79e9537c8b6deea1b Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 9 Feb 2025 14:48:38 +0200 Subject: [PATCH 38/47] Fixed mysqld_list_processes to remove a possibility to access null pointers --- sql/sql_show.cc | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 18d9e48537a..c2186a667a2 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2943,25 +2943,27 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose) while (thread_info *thd_info= arg.thread_infos.get()) { + const char *str; + ulonglong start_time; + CSET_STRING query; + protocol->prepare_for_resend(); protocol->store(thd_info->thread_id); protocol->store(thd_info->user, strlen(thd_info->user), system_charset_info); protocol->store(thd_info->host, strlen(thd_info->host), system_charset_info); protocol->store_string_or_null(thd_info->db, system_charset_info); - if (thd_info->proc_info) - protocol->store(thd_info->proc_info, strlen(thd_info->proc_info), - system_charset_info); + if ((str= thd_info->proc_info)) + protocol->store(str, strlen(str), system_charset_info); else protocol->store(&command_name[thd_info->command], system_charset_info); - if (thd_info->start_time && now > thd_info->start_time) - protocol->store_long((now - thd_info->start_time) / HRTIME_RESOLUTION); + if ((start_time= thd_info->start_time) && now > start_time) + protocol->store_long((now - start_time) / HRTIME_RESOLUTION); else protocol->store_null(); protocol->store_string_or_null(thd_info->state_info, system_charset_info); - if (thd_info->query_string.length()) - protocol->store(thd_info->query_string.str(), - thd_info->query_string.length(), - thd_info->query_string.charset()); + query= thd_info->query_string; + if (query.length() && query.str()) + protocol->store(query.str(), query.length(), query.charset()); else protocol->store_null(); if (!(thd->variables.old_behavior & OLD_MODE_NO_PROGRESS_INFO)) From 7b59a4dbc2fafdbeed7acb182ea9930b19d4883e Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 25 Feb 2025 15:50:23 +0200 Subject: [PATCH 39/47] Allow 'mariadb' as a connection wrapper name for FederatedX. One can now use 'mariadb' and/or 'mysql' asr wrapper name to connect to MariaDB or MySQL. --- mysql-test/suite/federated/federatedx.result | 2 +- mysql-test/suite/federated/federatedx.test | 2 +- mysql-test/suite/federated/federatedx_create_handlers.result | 4 ++-- mysql-test/suite/federated/federatedx_create_handlers.test | 4 ++-- storage/federatedx/federatedx_io.cc | 1 + 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/federated/federatedx.result b/mysql-test/suite/federated/federatedx.result index 7f815c1a61c..14d3db9dfcd 100644 --- a/mysql-test/suite/federated/federatedx.result +++ b/mysql-test/suite/federated/federatedx.result @@ -79,7 +79,7 @@ CREATE TABLE federated.t1 ( `name` varchar(32) NOT NULL default '' ) ENGINE="FEDERATED" DEFAULT CHARSET=latin1 -CONNECTION='mysql://root@127.0.0.1:SLAVE_PORT/federated/t1'; +CONNECTION='mariadb://root@127.0.0.1:SLAVE_PORT/federated/t1'; INSERT INTO federated.t1 (id, name) VALUES (1, 'foo'); INSERT INTO federated.t1 (id, name) VALUES (2, 'fee'); INSERT INTO federated.t1 (id, `group`) VALUES (3, 42); diff --git a/mysql-test/suite/federated/federatedx.test b/mysql-test/suite/federated/federatedx.test index 7e5a335b786..bc75edc5a40 100644 --- a/mysql-test/suite/federated/federatedx.test +++ b/mysql-test/suite/federated/federatedx.test @@ -92,7 +92,7 @@ eval CREATE TABLE federated.t1 ( `name` varchar(32) NOT NULL default '' ) ENGINE="FEDERATED" DEFAULT CHARSET=latin1 - CONNECTION='mysql://root@127.0.0.1:$SLAVE_MYPORT/federated/t1'; + CONNECTION='mariadb://root@127.0.0.1:$SLAVE_MYPORT/federated/t1'; INSERT INTO federated.t1 (id, name) VALUES (1, 'foo'); INSERT INTO federated.t1 (id, name) VALUES (2, 'fee'); diff --git a/mysql-test/suite/federated/federatedx_create_handlers.result b/mysql-test/suite/federated/federatedx_create_handlers.result index 7288c520cbc..d402bd5ccb1 100644 --- a/mysql-test/suite/federated/federatedx_create_handlers.result +++ b/mysql-test/suite/federated/federatedx_create_handlers.result @@ -479,12 +479,12 @@ CREATE TABLE federated.t3 (a INT); INSERT INTO federated.t3 VALUES (1),(2),(3); CREATE TABLE federated.t4 (a INT); connection master; -CREATE SERVER fedlink FOREIGN DATA WRAPPER mysql +CREATE SERVER fedlink FOREIGN DATA WRAPPER mariadb OPTIONS (USER 'root', HOST '127.0.0.1', DATABASE 'federated', PORT SLAVE_PORT); CREATE TABLE federated.t3 (a INT) ENGINE=FEDERATED -CONNECTION='mysql://root@127.0.0.1:$SLAVE_MYPORT/federated/t3' +CONNECTION='mariadb://root@127.0.0.1:$SLAVE_MYPORT/federated/t3' PARTITION BY list (a) (PARTITION p1 VALUES IN (1) CONNECTION='fedlink/t3', PARTITION p2 VALUES IN (2) CONNECTION='fedlink/t4'); diff --git a/mysql-test/suite/federated/federatedx_create_handlers.test b/mysql-test/suite/federated/federatedx_create_handlers.test index c09ba28a7ea..75eb32e7fbe 100644 --- a/mysql-test/suite/federated/federatedx_create_handlers.test +++ b/mysql-test/suite/federated/federatedx_create_handlers.test @@ -307,13 +307,13 @@ CREATE TABLE federated.t4 (a INT); connection master; --replace_result $SLAVE_MYPORT SLAVE_PORT -eval CREATE SERVER fedlink FOREIGN DATA WRAPPER mysql +eval CREATE SERVER fedlink FOREIGN DATA WRAPPER mariadb OPTIONS (USER 'root', HOST '127.0.0.1', DATABASE 'federated', PORT $SLAVE_MYPORT); CREATE TABLE federated.t3 (a INT) ENGINE=FEDERATED - CONNECTION='mysql://root@127.0.0.1:$SLAVE_MYPORT/federated/t3' + CONNECTION='mariadb://root@127.0.0.1:$SLAVE_MYPORT/federated/t3' PARTITION BY list (a) (PARTITION p1 VALUES IN (1) CONNECTION='fedlink/t3', PARTITION p2 VALUES IN (2) CONNECTION='fedlink/t4'); diff --git a/storage/federatedx/federatedx_io.cc b/storage/federatedx/federatedx_io.cc index 5baec617cda..b3c88001b27 100644 --- a/storage/federatedx/federatedx_io.cc +++ b/storage/federatedx/federatedx_io.cc @@ -51,6 +51,7 @@ struct io_schemes_st static const io_schemes_st federated_io_schemes[] = { { "mysql", &instantiate_io_mysql }, + { "mariadb", &instantiate_io_mysql }, { "null", instantiate_io_null } /* must be last element */ }; From cf01bfe811dcdad121ed04922adf231e2c85520d Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 25 Feb 2025 15:43:20 +0200 Subject: [PATCH 40/47] Extended mariadb-test-run to define MARIADB_TOPDIR and MARIADB_DATADIR This is for preparing MariaDB for catalogs. mtr tests should in the future use MARIADB_TOPDIR or MARIADB_DATADIR instead of '@@datadir'. This is especially true for replication tests that access binlog files. MARIADB_TOPDIR is the top directory where binary log and engine log files resides. MARIADB_DATADIR is the directory where the database/schema directories resides. MARIADB_TOPDIR is not depending on catalogs. When catalogs is used MARIADB_DATADIR will point to the directory of the current catalog. If catalogs is not used then MARIAD_DATADIR=MARIADB_TOPDIR. --- mysql-test/mariadb-test-run.pl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index f72ff055a33..57fb5580139 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -268,6 +268,9 @@ our $opt_force= 0; our $opt_skip_not_found= 0; our $opt_mem= $ENV{'MTR_MEM'}; our $opt_clean_vardir= $ENV{'MTR_CLEAN_VARDIR'}; +our $opt_catalogs= 0; +our $opt_catalog_name=""; +our $catalog_name="def"; our $opt_gcov; our $opt_gprof; @@ -3945,6 +3948,23 @@ sub run_testcase ($$) { } } + # Set up things for catalogs + # The values of MARIADB_TOPDIR and MARIAD_DATADIR should + # be taken from the values used by the default (first) + # connection that is used by mariadb-test. + my ($mysqld, @servers); + @servers= all_servers(); + $mysqld= $servers[0]; + $ENV{'MARIADB_TOPDIR'}= $mysqld->value('datadir'); + if (!$opt_catalogs) + { + $ENV{'MARIADB_DATADIR'}= $mysqld->value('datadir'); + } + else + { + $ENV{'MARIADB_DATADIR'}= $mysqld->value('datadir') . "/" . $catalog_name; + } + # Write start of testcase to log mark_log($path_current_testlog, $tinfo); From 2c0ba2680b4203d04b9cf27199c75da85ae10e4b Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 25 Feb 2025 15:45:44 +0200 Subject: [PATCH 41/47] load_db_opt was always doing a file access if db.opt file did not exist Added caching of database directories that did not have a db.opt file. This was common for older MariaDB installaiton or if a user created a database with 'mkdir'. Other things: - Give a note "no db.opt file" if one uses SHOW CREATE DATABASE one a database without a db.opt file. --- mysql-test/main/comment_database.result | 13 ++++++ mysql-test/main/comment_database.test | 8 ++++ mysql-test/main/ctype_utf8_def_upgrade.result | 2 + mysql-test/main/mysql_upgrade-34014.result | 2 + sql/sql_db.cc | 46 +++++++++++++------ sql/sql_db.h | 4 +- sql/sql_show.cc | 9 +++- 7 files changed, 67 insertions(+), 17 deletions(-) diff --git a/mysql-test/main/comment_database.result b/mysql-test/main/comment_database.result index 8e3cf1c904e..d67f0c054ca 100644 --- a/mysql-test/main/comment_database.result +++ b/mysql-test/main/comment_database.result @@ -76,3 +76,16 @@ WHERE schema_name='comment'; CATALOG_NAME SCHEMA_NAME DEFAULT_CHARACTER_SET_NAME DEFAULT_COLLATION_NAME SQL_PATH SCHEMA_COMMENT def comment latin2 latin2_general_ci NULL comment DROP DATABASE comment; +CREATE DATABASE db1; +# restart +SHOW CREATE DATABASE db1; +Database Create Database +db1 CREATE DATABASE `db1` /*!40100 DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci */ +Warnings: +Note 1105 Database 'db1' does not have a db.opt file. You can create one with ALTER DATABASE if needed +SHOW CREATE DATABASE db1; +Database Create Database +db1 CREATE DATABASE `db1` /*!40100 DEFAULT CHARACTER SET latin1 COLLATE latin1_swedish_ci */ +Warnings: +Note 1105 Database 'db1' does not have a db.opt file. You can create one with ALTER DATABASE if needed +DROP DATABASE db1; diff --git a/mysql-test/main/comment_database.test b/mysql-test/main/comment_database.test index bf89a57c7b2..a6f43a91e5d 100644 --- a/mysql-test/main/comment_database.test +++ b/mysql-test/main/comment_database.test @@ -63,3 +63,11 @@ SELECT * FROM information_schema.schemata WHERE schema_name='comment'; DROP DATABASE comment; --enable_service_connection + +CREATE DATABASE db1; +--remove_file $MARIADB_DATADIR/db1/db.opt +--source include/restart_mysqld.inc +# We need to call this two times to ensure all code paths are used +SHOW CREATE DATABASE db1; +SHOW CREATE DATABASE db1; +DROP DATABASE db1; diff --git a/mysql-test/main/ctype_utf8_def_upgrade.result b/mysql-test/main/ctype_utf8_def_upgrade.result index d30f8670536..fbe43fb208d 100644 --- a/mysql-test/main/ctype_utf8_def_upgrade.result +++ b/mysql-test/main/ctype_utf8_def_upgrade.result @@ -53,6 +53,8 @@ SET @@character_set_database=DEFAULT; SHOW CREATE DATABASE db1; Database Create Database db1 CREATE DATABASE `db1` /*!40100 DEFAULT CHARACTER SET utf8mb3 COLLATE utf8mb3_general_ci */ +Warnings: +Note 1105 Database 'db1' does not have a db.opt file. You can create one with ALTER DATABASE if needed USE db1; SELECT @@character_set_database, 'taken from defaults' AS comment; @@character_set_database comment diff --git a/mysql-test/main/mysql_upgrade-34014.result b/mysql-test/main/mysql_upgrade-34014.result index d53da1c85df..78dd75da8e8 100644 --- a/mysql-test/main/mysql_upgrade-34014.result +++ b/mysql-test/main/mysql_upgrade-34014.result @@ -12,6 +12,8 @@ FLUSH TABLES; SHOW CREATE DATABASE sys; Database Create Database sys CREATE DATABASE `sys` /*!40100 DEFAULT CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci */ +Warnings: +Note 1105 Database 'sys' does not have a db.opt file. You can create one with ALTER DATABASE if needed Phase 1/8: Checking and upgrading mysql database Processing databases mysql diff --git a/sql/sql_db.cc b/sql/sql_db.cc index 44426184497..6b7949ccc41 100644 --- a/sql/sql_db.cc +++ b/sql/sql_db.cc @@ -536,36 +536,53 @@ static bool write_db_opt(THD *thd, const char *path, DESCRIPTION + create->default_table_charset is guaranteed to be alway set + Required by some callers + RETURN VALUES 0 File found - 1 No database file or could not open it - + -1 No database file (file was not found or 'empty' file was cached) + 1 Could not open it */ -bool load_db_opt(THD *thd, const char *path, Schema_specification_st *create) +int load_db_opt(THD *thd, const char *path, Schema_specification_st *create) { File file; char buf[256+DATABASE_COMMENT_MAXLEN]; DBUG_ENTER("load_db_opt"); - bool error=1; + int error= 0; size_t nbytes; myf utf8_flag= thd->get_utf8_flag(); bzero((char*) create,sizeof(*create)); - create->default_table_charset= thd->variables.collation_server; /* Check if options for this database are already in the hash */ if (!get_dbopt(thd, path, create)) - DBUG_RETURN(0); + { + if (!create->default_table_charset) + error= -1; // db.opt did not exists + goto err1; + } /* Otherwise, load options from the .opt file */ if ((file= mysql_file_open(key_file_dbopt, path, O_RDONLY | O_SHARE, MYF(0))) < 0) + { + /* + Create an empty entry, to avoid doing an extra file open for every create + table. + */ + put_dbopt(path, create); + error= -1; goto err1; + } IO_CACHE cache; if (init_io_cache(&cache, file, IO_SIZE, READ_CACHE, 0, 0, MYF(0))) - goto err2; + { + error= 1; + goto err2; // Not cached + } while ((int) (nbytes= my_b_gets(&cache, (char*) buf, sizeof(buf))) > 0) { @@ -586,7 +603,7 @@ bool load_db_opt(THD *thd, const char *path, Schema_specification_st *create) default-collation commands. */ if (!(create->default_table_charset= - get_charset_by_csname(pos+1, MY_CS_PRIMARY, MYF(utf8_flag))) && + get_charset_by_csname(pos+1, MY_CS_PRIMARY, MYF(utf8_flag))) && !(create->default_table_charset= get_charset_by_name(pos+1, MYF(utf8_flag)))) { @@ -621,10 +638,11 @@ bool load_db_opt(THD *thd, const char *path, Schema_specification_st *create) err2: mysql_file_close(file, MYF(0)); err1: + if (!create->default_table_charset) // In case of error + create->default_table_charset= thd->variables.collation_server; DBUG_RETURN(error); } - /* Retrieve database options by name. Load database options file or fetch from cache. @@ -651,11 +669,12 @@ err1: db_create_info right after that. RETURN VALUES (read NOTE!) - FALSE Success - TRUE Failed to retrieve options + 0 File found + -1 No database file (file was not found or 'empty' file was cached) + 1 Could not open it */ -bool load_db_opt_by_name(THD *thd, const char *db_name, +int load_db_opt_by_name(THD *thd, const char *db_name, Schema_specification_st *db_create_info) { char db_opt_path[FN_REFLEN + 1]; @@ -1951,8 +1970,7 @@ bool mysql_upgrade_db(THD *thd, const LEX_CSTRING *old_db) build_table_filename(path, sizeof(path)-1, old_db->str, "", MY_DB_OPT_FILE, 0); - if ((load_db_opt(thd, path, &create_info))) - create_info.default_table_charset= thd->variables.collation_server; + load_db_opt(thd, path, &create_info); length= build_table_filename(path, sizeof(path)-1, old_db->str, "", "", 0); if (length && path[length-1] == FN_LIBCHAR) diff --git a/sql/sql_db.h b/sql/sql_db.h index 3c037d668e0..03517dc684e 100644 --- a/sql/sql_db.h +++ b/sql/sql_db.h @@ -37,8 +37,8 @@ bool mysql_opt_change_db(THD *thd, bool my_dboptions_cache_init(void); void my_dboptions_cache_free(void); bool check_db_dir_existence(const char *db_name); -bool load_db_opt(THD *thd, const char *path, Schema_specification_st *create); -bool load_db_opt_by_name(THD *thd, const char *db_name, +int load_db_opt(THD *thd, const char *path, Schema_specification_st *create); +int load_db_opt_by_name(THD *thd, const char *db_name, Schema_specification_st *db_create_info); CHARSET_INFO *get_default_db_collation(THD *thd, const char *db_name); bool my_dbopt_init(void); diff --git a/sql/sql_show.cc b/sql/sql_show.cc index c2186a667a2..d70e20fe0b3 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -1435,7 +1435,14 @@ bool mysqld_show_create_db(THD *thd, LEX_CSTRING *dbname, DBUG_RETURN(TRUE); } - load_db_opt_by_name(thd, dbname->str, &create); + if (load_db_opt_by_name(thd, dbname->str, &create) < 0) + { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, + ER_UNKNOWN_ERROR, + "Database '%.192s' does not have a db.opt file. " + "You can create one with ALTER DATABASE if needed", + dbname->str); + } } mysqld_show_create_db_get_fields(thd, &field_list); From 937ae4137ee80f591d63d3d0e6c5c1f409b2b47d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Feb 2025 18:49:15 +1100 Subject: [PATCH 42/47] MDEV-36155: MSAN use-of-uninitialized-value innodb.log_file_size_online Writing the redo log resized will write uninitialized data. There is a MEM_MAKE_DEFINED construct in the code to bless this however it was correct on the initial length, but not the changed length. The MEM_MAKE_DEFINED is moved earlier in the code where the length contains the correct value. --- storage/innobase/log/log0log.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index d35f66a78ec..77225b1c809 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1044,12 +1044,13 @@ lsn_t log_t::write_buf() noexcept the current LSN are generated. */ MEM_MAKE_DEFINED(buf + length, (write_size_1 + 1) - new_buf_free); buf[length]= 0; /* allow recovery to catch EOF faster */ + if (UNIV_LIKELY_NULL(re_write_buf)) + MEM_MAKE_DEFINED(re_write_buf + length, (write_size_1 + 1) - + new_buf_free); length&= ~write_size_1; memcpy_aligned<16>(flush_buf, buf + length, (new_buf_free + 15) & ~15); if (UNIV_LIKELY_NULL(re_write_buf)) { - MEM_MAKE_DEFINED(re_write_buf + length, (write_size_1 + 1) - - new_buf_free); memcpy_aligned<16>(resize_flush_buf, re_write_buf + length, (new_buf_free + 15) & ~15); re_write_buf[length + new_buf_free]= 0; From 733852d4c3a0040e1db2453727f01f8367789ef0 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Wed, 26 Feb 2025 17:57:43 +0700 Subject: [PATCH 43/47] BKA join cache buffer is employed despite join_cache_level=3 (flat BNLH) In the `check_join_cache_usage()` function there is a branching issue where an accidental fall-through to BKA/BKAH buffers may occur, even when the join_cache_level setting does not permit their use. This patch corrects the condition to ensure that BKA/BKAH join caching is only enabled when explicitly allowed by join_cache_level Reviewer: Sergei Petrunia --- mysql-test/main/join_cache.result | 26 ++++++++++++++++++++++++++ mysql-test/main/join_cache.test | 27 +++++++++++++++++++++++++++ sql/sql_select.cc | 2 +- 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/join_cache.result b/mysql-test/main/join_cache.result index c93446e6103..4a9436cfa70 100644 --- a/mysql-test/main/join_cache.result +++ b/mysql-test/main/join_cache.result @@ -6443,3 +6443,29 @@ DROP TABLE t1, t2; # # End of 10.5 tests # +# +# MDEV-36165: BKA join cache buffer is employed despite join_cache_level=3 (flat BNLH) +# +CREATE TABLE t1(a INT); +INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +CREATE TABLE t2(a INT, b INT); +INSERT INTO t2 SELECT a, a from t1; +CREATE TABLE t3(a INT, b INT, c INT, key (a,b)); +INSERT INTO t3 select a, a, a FROM t1; +SET optimizer_switch = 'join_cache_hashed=off,join_cache_bka=on,mrr=on'; +SET join_cache_level = 3; +EXPLAIN SELECT * FROM t2, t3 WHERE t2.a=t3.a AND (t3.b+1 <= t2.b+1); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 10 Using where +1 SIMPLE t3 ref a a 5 test.t2.a 1 Using index condition +SET join_cache_level = 4; +EXPLAIN SELECT * FROM t2, t3 WHERE t2.a=t3.a AND (t3.b+1 <= t2.b+1); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 ALL NULL NULL NULL NULL 10 Using where +1 SIMPLE t3 ref a a 5 test.t2.a 1 Using index condition +SET join_cache_level = default; +SET optimizer_switch = default; +DROP TABLE t1, t2, t3; +# +# End of 10.11 tests +# diff --git a/mysql-test/main/join_cache.test b/mysql-test/main/join_cache.test index 3d3efa7655f..b707cd25166 100644 --- a/mysql-test/main/join_cache.test +++ b/mysql-test/main/join_cache.test @@ -4321,3 +4321,30 @@ DROP TABLE t1, t2; --echo # --echo # End of 10.5 tests --echo # + +--echo # +--echo # MDEV-36165: BKA join cache buffer is employed despite join_cache_level=3 (flat BNLH) +--echo # +--source include/have_sequence.inc +CREATE TABLE t1(a INT); +INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +CREATE TABLE t2(a INT, b INT); +INSERT INTO t2 SELECT a, a from t1; +CREATE TABLE t3(a INT, b INT, c INT, key (a,b)); +INSERT INTO t3 select a, a, a FROM t1; + +SET optimizer_switch = 'join_cache_hashed=off,join_cache_bka=on,mrr=on'; + +SET join_cache_level = 3; +EXPLAIN SELECT * FROM t2, t3 WHERE t2.a=t3.a AND (t3.b+1 <= t2.b+1); + +SET join_cache_level = 4; +EXPLAIN SELECT * FROM t2, t3 WHERE t2.a=t3.a AND (t3.b+1 <= t2.b+1); + +SET join_cache_level = default; +SET optimizer_switch = default; +DROP TABLE t1, t2, t3; + +--echo # +--echo # End of 10.11 tests +--echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 730d57ff877..13802a91591 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -14769,7 +14769,7 @@ uint check_join_cache_usage(JOIN_TAB *tab, } goto no_join_cache; } - if (cache_level > 4 && no_bka_cache) + if (cache_level < 5 || no_bka_cache) goto no_join_cache; if ((flags & HA_MRR_NO_ASSOCIATION) && From 1965b2be160038d726e6cdbf039becfb5963d3af Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 15 Jul 2024 16:00:32 +0300 Subject: [PATCH 44/47] MDEV-34620: Lots of index_merge created and discarded for many-way OR If a query has many OR-ed constructs which can use multiple indexes (key1=1 AND key2=10) OR (key1=2 AND key2=20) OR (key1=3 AND key2=30) OR ... The range optimizer would construct and then discard a lot of potential index_merge plans. This process 1. is CPU-intensive 2. can hit the @@optimizer_max_sel_args limitation after which all potential range or index_merge plans are discarded. The fix is to apply a heuristic: if there is an OR clause with more than MAX_OR_ELEMENTS_FOR_INDEX_MERGE=100 branches (hard-coded constant), disallow construction of index_merge plans for the OR branches. --- mysql-test/main/range_notembedded.result | 67 ++++++++++++++++++++++++ mysql-test/main/range_notembedded.test | 48 +++++++++++++++++ sql/opt_range.cc | 11 ++++ sql/opt_range.h | 29 ++++++++++ 4 files changed, 155 insertions(+) diff --git a/mysql-test/main/range_notembedded.result b/mysql-test/main/range_notembedded.result index e1bcc7463d5..1babba7548e 100644 --- a/mysql-test/main/range_notembedded.result +++ b/mysql-test/main/range_notembedded.result @@ -247,3 +247,70 @@ SELECT id FROM t1 WHERE id IS NULL OR id NOT BETWEEN 1 AND 4; id 5 DROP TABLE t1; +# +# MDEV-34620: Many index_merge variants made and discarded for a big OR +# +CREATE TABLE t1 ( +a1 int NOT NULL, +a2 int NOT NULL, +filler char(100), +KEY key1 (a1,a2), +KEY key2 (a2,a1) +); +insert into t1 (a1,a2) values (1,1),(2,2),(3,3); +set @query= concat( +"explain select * from t1 where\n", +(select +group_concat(concat("a1=", seq, " and a2=", seq, " ") separator "\nor " ) +from seq_1_to_30) +); +set optimizer_trace=1; +prepare s from @query; +execute s; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL key1,key2 NULL NULL NULL 3 Using where +set @trace=json_extract((select trace from information_schema.optimizer_trace), '$**.range_analysis'); +# Observe that "key1" is a a part of several index_merge_union: +select json_pretty(json_search(@trace, 'all', 'key1')); +json_pretty(json_search(@trace, 'all', 'key1')) +[ + "$[0].potential_range_indexes[0].index", + "$[0].analyzing_range_alternatives.range_scan_alternatives[0].index", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[0].indexes_to_merge[0].range_scan_alternatives[0].index", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[0].indexes_to_merge[0].index_to_merge", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[0].indexes_to_merge[1].range_scan_alternatives[0].index", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[0].indexes_to_merge[1].index_to_merge", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[1].indexes_to_merge[0].range_scan_alternatives[0].index", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[1].indexes_to_merge[0].index_to_merge", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[1].indexes_to_merge[1].range_scan_alternatives[0].index", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[1].indexes_to_merge[1].index_to_merge", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[1].indexes_to_merge[2].range_scan_alternatives[0].index", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[1].indexes_to_merge[2].index_to_merge", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[2].indexes_to_merge[0].range_scan_alternatives[0].index", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[2].indexes_to_merge[0].index_to_merge", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[2].indexes_to_merge[1].range_scan_alternatives[0].index", + "$[0].analyzing_range_alternatives.analyzing_index_merge_union[2].indexes_to_merge[1].index_to_merge" +] +# +# Now, same as above but for a long IN-list +# +set @query= concat( +"explain select * from t1 where\n", +(select +group_concat(concat("a1=", seq, " and a2=", seq, " ") separator "\nor " ) +from seq_1_to_120) +); +set optimizer_trace=1; +prepare s from @query; +execute s; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL key1,key2 NULL NULL NULL 3 Using where +set @trace=json_extract((select trace from information_schema.optimizer_trace), '$**.range_analysis'); +# Observe that there are NO index_merge_union candidates. Only one potential range scan: +select json_pretty(json_search(@trace, 'all', 'key1')); +json_pretty(json_search(@trace, 'all', 'key1')) +[ + "$[0].potential_range_indexes[0].index", + "$[0].analyzing_range_alternatives.range_scan_alternatives[0].index" +] +drop table t1; diff --git a/mysql-test/main/range_notembedded.test b/mysql-test/main/range_notembedded.test index df9fe0fbd40..2088be9bf20 100644 --- a/mysql-test/main/range_notembedded.test +++ b/mysql-test/main/range_notembedded.test @@ -162,3 +162,51 @@ INSERT INTO t1 VALUES (1),(5); SELECT id FROM t1 WHERE id IS NULL OR id NOT BETWEEN 1 AND 4; DROP TABLE t1; +--echo # +--echo # MDEV-34620: Many index_merge variants made and discarded for a big OR +--echo # + +CREATE TABLE t1 ( + a1 int NOT NULL, + a2 int NOT NULL, + filler char(100), + KEY key1 (a1,a2), + KEY key2 (a2,a1) +); +insert into t1 (a1,a2) values (1,1),(2,2),(3,3); + + +set @query= concat( + "explain select * from t1 where\n", + (select + group_concat(concat("a1=", seq, " and a2=", seq, " ") separator "\nor " ) + from seq_1_to_30) + ); + +set optimizer_trace=1; +prepare s from @query; +execute s; +set @trace=json_extract((select trace from information_schema.optimizer_trace), '$**.range_analysis'); + +--echo # Observe that "key1" is a a part of several index_merge_union: +select json_pretty(json_search(@trace, 'all', 'key1')); + +--echo # +--echo # Now, same as above but for a long IN-list +--echo # +set @query= concat( + "explain select * from t1 where\n", + (select + group_concat(concat("a1=", seq, " and a2=", seq, " ") separator "\nor " ) + from seq_1_to_120) + ); + +set optimizer_trace=1; +prepare s from @query; +execute s; +set @trace=json_extract((select trace from information_schema.optimizer_trace), '$**.range_analysis'); + +--echo # Observe that there are NO index_merge_union candidates. Only one potential range scan: +select json_pretty(json_search(@trace, 'all', 'key1')); +drop table t1; + diff --git a/sql/opt_range.cc b/sql/opt_range.cc index c2e9909106a..d649926abf7 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -8513,6 +8513,11 @@ SEL_TREE *Item_cond::get_mm_tree(RANGE_OPT_PARAM *param, Item **cond_ptr) SEL_TREE *tree= li.ref()[0]->get_mm_tree(param, li.ref()); if (param->statement_should_be_aborted()) DBUG_RETURN(NULL); + bool orig_disable_index_merge= param->disable_index_merge_plans; + + if (list.elements > MAX_OR_ELEMENTS_FOR_INDEX_MERGE) + param->disable_index_merge_plans= true; + if (tree) { if (tree->type == SEL_TREE::IMPOSSIBLE && @@ -8529,7 +8534,10 @@ SEL_TREE *Item_cond::get_mm_tree(RANGE_OPT_PARAM *param, Item **cond_ptr) { SEL_TREE *new_tree= li.ref()[0]->get_mm_tree(param, li.ref()); if (new_tree == NULL || param->statement_should_be_aborted()) + { + param->disable_index_merge_plans= orig_disable_index_merge; DBUG_RETURN(NULL); + } tree= tree_or(param, tree, new_tree); if (tree == NULL || tree->type == SEL_TREE::ALWAYS) { @@ -8561,6 +8569,7 @@ SEL_TREE *Item_cond::get_mm_tree(RANGE_OPT_PARAM *param, Item **cond_ptr) if (replace_cond) *cond_ptr= replacement_item; } + param->disable_index_merge_plans= orig_disable_index_merge; DBUG_RETURN(tree); } @@ -9952,6 +9961,8 @@ tree_or(RANGE_OPT_PARAM *param,SEL_TREE *tree1,SEL_TREE *tree2) { bool must_be_ored= sel_trees_must_be_ored(param, tree1, tree2, ored_keys); no_imerge_from_ranges= must_be_ored; + if (param->disable_index_merge_plans) + no_imerge_from_ranges= true; if (no_imerge_from_ranges && no_merges1 && no_merges2) { diff --git a/sql/opt_range.h b/sql/opt_range.h index f434fbf7444..b4db3850de4 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -39,6 +39,32 @@ class JOIN; class Item_sum; +/* + When processing an OR clause with more than MAX_OR_ELEMENTS_FOR_INDEX_MERGE + disjuncts (i.e. OR-parts), do not construct index_merge plans from it. + + Some users have OR clauses with extremely large number of disjuncts, like: + + (key1=1 AND key2=10) OR + (key1=2 AND key2=20) OR + (key1=3 AND key2=30) OR + ... + + When processing this, the optimizer would try to build a lot of potential + index_merge plans. Hypothetically this could be useful as the cheapest plan + could be to pick a specific index for each disjunct and build: + + index_merge(key1 IN (1,3,8,15...), key2 IN (20, 40, 50 ...)) + + In practice this causes combinatorial amount of time to be spent in the range + analyzer, and most variants will be discarded when the range optimizer tries + to avoid this combinatorial explosion (which may or may not work depending on + the form of the WHERE clause). + In practice, very long ORs are served well enough by just considering range + accesses on individual indexes. +*/ +const int MAX_OR_ELEMENTS_FOR_INDEX_MERGE=100; + struct KEY_PART { uint16 key,part; /* See KEY_PART_INFO for meaning of the next two: */ @@ -889,6 +915,9 @@ public: */ bool remove_false_where_parts; + /* If TRUE, do not construct index_merge plans */ + bool disable_index_merge_plans; + /* Which functions should give SQL notes for unusable keys. */ From 1ed09cfdcbb66807d2e3804884545b405f0f7ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Feb 2025 08:55:16 +0200 Subject: [PATCH 45/47] MDEV-35000 preparation: Clean up dict_table_t::stat innodb_stats_transient_sample_pages, innodb_stats_persistent_sample_pages: Change the type to UNSIGNED, because the number of pages in a table is limited to 32 bits by the InnoDB file format. btr_get_size_and_reserved(), fseg_get_n_frag_pages(), fseg_n_reserved_pages_low(), fseg_n_reserved_pages(): Return uint32_t. The file format limits page numbers to 32 bits. dict_table_t::stat: An Atomic_relaxed that combines a number of metadata fields. innodb_copy_stat_flags(): Copy the statistics flags from TABLE_SHARE or HA_CREATE_INFO. dict_table_t::stats_initialized(), dict_table_t::stats_is_persistent(): Accessors to dict_table_t::stat. Reviewed by: Thirunarayanan Balathandayuthapani --- .../suite/sys_vars/r/sysvars_innodb.result | 8 +- storage/innobase/dict/dict0defrag_bg.cc | 24 ++-- storage/innobase/dict/dict0dict.cc | 5 +- storage/innobase/dict/dict0stats.cc | 88 ++++++------ storage/innobase/dict/dict0stats_bg.cc | 14 +- storage/innobase/fsp/fsp0fsp.cc | 33 ++--- storage/innobase/handler/ha_innodb.cc | 123 ++++++++--------- storage/innobase/handler/handler0alter.cc | 5 +- storage/innobase/handler/i_s.cc | 4 +- storage/innobase/ibuf/ibuf0ibuf.cc | 29 ++-- storage/innobase/include/dict0dict.h | 2 +- storage/innobase/include/dict0mem.h | 94 +++++++------ storage/innobase/include/dict0stats.h | 38 ------ storage/innobase/include/dict0stats.inl | 125 +----------------- storage/innobase/include/fsp0fsp.h | 6 +- storage/innobase/include/ibuf0ibuf.h | 10 +- storage/innobase/include/srv0srv.h | 4 +- storage/innobase/row/row0mysql.cc | 2 +- storage/innobase/row/row0purge.cc | 2 +- storage/innobase/row/row0uins.cc | 2 +- storage/innobase/row/row0umod.cc | 2 +- storage/innobase/srv/srv0srv.cc | 4 +- 22 files changed, 226 insertions(+), 398 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 2d8e1573db8..a82eab33313 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -1571,10 +1571,10 @@ VARIABLE_NAME INNODB_STATS_PERSISTENT_SAMPLE_PAGES SESSION_VALUE NULL DEFAULT_VALUE 20 VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT The number of leaf index pages to sample when calculating persistent statistics (by ANALYZE, default 20) NUMERIC_MIN_VALUE 1 -NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_MAX_VALUE 4294967295 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO @@ -1595,10 +1595,10 @@ VARIABLE_NAME INNODB_STATS_TRANSIENT_SAMPLE_PAGES SESSION_VALUE NULL DEFAULT_VALUE 8 VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT The number of leaf index pages to sample when calculating transient statistics (if persistent statistics are not used, default 8) NUMERIC_MIN_VALUE 1 -NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_MAX_VALUE 4294967295 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/storage/innobase/dict/dict0defrag_bg.cc b/storage/innobase/dict/dict0defrag_bg.cc index bec6da8e6af..81b8dbf077d 100644 --- a/storage/innobase/dict/dict0defrag_bg.cc +++ b/storage/innobase/dict/dict0defrag_bg.cc @@ -284,18 +284,18 @@ release_and_exit: /**************************************************************//** Gets the number of reserved and used pages in a B-tree. -@return number of pages reserved, or ULINT_UNDEFINED if the index -is unavailable */ +@return number of pages reserved +@retval 0 if the index is unavailable */ static -ulint +uint32_t btr_get_size_and_reserved( dict_index_t* index, /*!< in: index */ ulint flag, /*!< in: BTR_N_LEAF_PAGES or BTR_TOTAL_SIZE */ - ulint* used, /*!< out: number of pages used (<= reserved) */ + uint32_t* used, /*!< out: number of pages used (<= reserved) */ mtr_t* mtr) /*!< in/out: mini-transaction where index is s-latched */ { - ulint dummy; + uint32_t dummy; ut_ad(mtr->memo_contains(index->lock, MTR_MEMO_SX_LOCK)); ut_a(flag == BTR_N_LEAF_PAGES || flag == BTR_TOTAL_SIZE); @@ -304,19 +304,19 @@ btr_get_size_and_reserved( || dict_index_is_online_ddl(index) || !index->is_committed() || !index->table->space) { - return(ULINT_UNDEFINED); + return 0; } dberr_t err; buf_block_t* root = btr_root_block_get(index, RW_SX_LATCH, mtr, &err); *used = 0; if (!root) { - return ULINT_UNDEFINED; + return 0; } mtr->x_lock_space(index->table->space); - ulint n = fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_LEAF + auto n = fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_LEAF + root->page.frame, used, mtr); if (flag == BTR_TOTAL_SIZE) { n += fseg_n_reserved_pages(*root, @@ -343,14 +343,14 @@ dict_stats_save_defrag_stats( const time_t now= time(nullptr); mtr_t mtr; - ulint n_leaf_pages; + uint32_t n_leaf_pages; mtr.start(); mtr_sx_lock_index(index, &mtr); - ulint n_leaf_reserved= btr_get_size_and_reserved(index, BTR_N_LEAF_PAGES, - &n_leaf_pages, &mtr); + uint32_t n_leaf_reserved= btr_get_size_and_reserved(index, BTR_N_LEAF_PAGES, + &n_leaf_pages, &mtr); mtr.commit(); - if (n_leaf_reserved == ULINT_UNDEFINED) + if (!n_leaf_reserved) return DB_SUCCESS; THD *thd= current_thd; diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 2e6a6d896e1..38e95282f26 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -198,8 +198,7 @@ dict_tables_have_same_db( /** Decrement the count of open handles */ void dict_table_close(dict_table_t *table) { - if (table->get_ref_count() == 1 && - dict_stats_is_persistent_enabled(table) && + if (table->get_ref_count() == 1 && table->stats_is_persistent() && strchr(table->name.m_name, '/')) { /* It looks like we are closing the last handle. The user could @@ -238,7 +237,7 @@ dict_table_close( dict_table_close(table); else { - if (table->release() && dict_stats_is_persistent_enabled(table) && + if (table->release() && table->stats_is_persistent() && strchr(table->name.m_name, '/')) { /* Force persistent stats re-read upon next open of the table so diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 34c2306feb5..b9bb3645dd5 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -599,7 +599,7 @@ void dict_stats_empty_table( table->stat_clustered_index_size = 1; /* 1 page for each index, not counting the clustered */ table->stat_sum_of_other_index_sizes - = UT_LIST_GET_LEN(table->indexes) - 1; + = uint32_t(UT_LIST_GET_LEN(table->indexes) - 1); table->stat_modified_counter = 0; dict_index_t* index; @@ -617,7 +617,7 @@ void dict_stats_empty_table( dict_stats_empty_index(index, empty_defrag_stats); } - table->stat_initialized = TRUE; + table->stat = table->stat | dict_table_t::STATS_INITIALIZED; table->stats_mutex_unlock(); } @@ -658,16 +658,10 @@ dict_stats_assert_initialized( /*==========================*/ const dict_table_t* table) /*!< in: table */ { - ut_a(table->stat_initialized); - MEM_CHECK_DEFINED(&table->stats_last_recalc, sizeof table->stats_last_recalc); - MEM_CHECK_DEFINED(&table->stat_persistent, - sizeof table->stat_persistent); - - MEM_CHECK_DEFINED(&table->stats_auto_recalc, - sizeof table->stats_auto_recalc); + MEM_CHECK_DEFINED(&table->stat, sizeof table->stat); MEM_CHECK_DEFINED(&table->stats_sample_pages, sizeof table->stats_sample_pages); @@ -844,8 +838,8 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, ulint n_cols; ib_uint64_t* n_diff; ib_uint64_t* n_not_null; - ibool stats_null_not_equal; - uintmax_t n_sample_pages=1; /* number of pages to sample */ + bool stats_null_not_equal; + uint32_t n_sample_pages=1; /* number of pages to sample */ ulint not_empty_flag = 0; ulint total_external_size = 0; uintmax_t add_on; @@ -883,11 +877,11 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, case SRV_STATS_NULLS_UNEQUAL: /* for both SRV_STATS_NULLS_IGNORED and SRV_STATS_NULLS_UNEQUAL case, we will treat NULLs as unequal value */ - stats_null_not_equal = TRUE; + stats_null_not_equal = true; break; case SRV_STATS_NULLS_EQUAL: - stats_null_not_equal = FALSE; + stats_null_not_equal = false; break; default: @@ -938,19 +932,21 @@ btr_estimate_number_of_different_key_vals(dict_index_t* index, so taking all case2 paths is I, our expression is: n_pages = S < I? min(I,L) : I - */ - if (index->stat_index_size > 1) { - n_sample_pages = (srv_stats_transient_sample_pages < index->stat_index_size) - ? ut_min(index->stat_index_size, - static_cast( - log2(double(index->stat_index_size)) - * double(srv_stats_transient_sample_pages))) - : index->stat_index_size; + */ + if (uint32_t I = index->stat_index_size) { + const uint32_t S{srv_stats_transient_sample_pages}; + n_sample_pages = S < I + ? std::min(I, + uint32_t(log2(double(I)) + * double(S))) + : I; } } /* Sanity check */ - ut_ad(n_sample_pages > 0 && n_sample_pages <= (index->stat_index_size <= 1 ? 1 : index->stat_index_size)); + ut_ad(n_sample_pages); + ut_ad(n_sample_pages <= (index->stat_index_size <= 1 + ? 1 : index->stat_index_size)); /* We sample some pages in the index to get an estimate */ btr_cur_t cursor; @@ -1169,7 +1165,7 @@ invalid: mtr.x_lock_space(index->table->space); - ulint dummy, size; + uint32_t dummy, size; index->stat_index_size = fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_LEAF @@ -1226,7 +1222,7 @@ dict_stats_update_transient( ut_ad(!table->stats_mutex_is_owner()); dict_index_t* index; - ulint sum_of_index_sizes = 0; + uint32_t sum_of_index_sizes = 0; dberr_t err = DB_SUCCESS; /* Find out the sizes of the indexes and how many different values @@ -1285,7 +1281,7 @@ empty_table: table->stat_modified_counter = 0; - table->stat_initialized = TRUE; + table->stat = table->stat | dict_table_t::STATS_INITIALIZED; table->stats_mutex_unlock(); @@ -2225,8 +2221,8 @@ dict_stats_analyze_index_for_n_prefix( struct index_stats_t { std::vector stats; - ulint index_size; - ulint n_leaf_pages; + uint32_t index_size; + uint32_t n_leaf_pages; index_stats_t(ulint n_uniq) : index_size(1), n_leaf_pages(1) { @@ -2365,7 +2361,7 @@ empty_index: uint16_t root_level = btr_page_get_level(root->page.frame); mtr.x_lock_space(index->table->space); - ulint dummy, size; + uint32_t dummy, size; result.index_size = fseg_n_reserved_pages(*root, PAGE_HEADER + PAGE_BTR_SEG_LEAF + root->page.frame, &size, &mtr) @@ -2742,7 +2738,7 @@ dict_stats_update_persistent( table->stat_modified_counter = 0; - table->stat_initialized = TRUE; + table->stat = table->stat | dict_table_t::STATS_INITIALIZED; dict_stats_assert_initialized(table); @@ -3164,8 +3160,7 @@ dict_stats_fetch_table_stats_step( ut_a(len == 8); table->stat_clustered_index_size - = std::max( - (ulint) mach_read_from_8(data), 1); + = std::max(mach_read_from_4(data + 4), 1U); break; } @@ -3174,18 +3169,9 @@ dict_stats_fetch_table_stats_step( ut_a(dtype_get_mtype(type) == DATA_INT); ut_a(len == 8); - ulint stat_other_idx_size - = (ulint) mach_read_from_8(data); - if (!stat_other_idx_size - && UT_LIST_GET_LEN(table->indexes) > 1) { - stat_other_idx_size - = UT_LIST_GET_LEN(table->indexes) - 1; - } - table->stat_sum_of_other_index_sizes - = std::max( - (ulint) mach_read_from_8(data), - UT_LIST_GET_LEN(table->indexes) - 1); - + table->stat_sum_of_other_index_sizes = std::max( + mach_read_from_4(data + 4), + uint32_t(UT_LIST_GET_LEN(table->indexes) - 1)); break; } default: @@ -3370,14 +3356,12 @@ dict_stats_fetch_index_stats_step( if (stat_name_len == 4 /* strlen("size") */ && strncasecmp("size", stat_name, stat_name_len) == 0) { - index->stat_index_size - = std::max((ulint) stat_value, 1); + index->stat_index_size = std::max(uint32_t(stat_value), 1U); arg->stats_were_modified = true; } else if (stat_name_len == 12 /* strlen("n_leaf_pages") */ && strncasecmp("n_leaf_pages", stat_name, stat_name_len) == 0) { - index->stat_n_leaf_pages - = std::max((ulint) stat_value, 1); + index->stat_n_leaf_pages = std::max(uint32_t(stat_value), 1U); arg->stats_were_modified = true; } else if (stat_name_len == 12 /* strlen("n_page_split") */ && strncasecmp("n_page_split", stat_name, stat_name_len) @@ -3648,7 +3632,9 @@ dict_stats_update_for_index( { DBUG_ENTER("dict_stats_update_for_index"); - if (dict_stats_is_persistent_enabled(index->table)) { + ut_ad(index->table->stat_initialized()); + + if (index->table->stats_is_persistent()) { if (dict_stats_persistent_storage_check(false)) { index_stats_t stats = dict_stats_analyze_index(index); @@ -3789,7 +3775,7 @@ dict_stats_update( /* If table is using persistent stats, then save the stats on disk */ - if (dict_stats_is_persistent_enabled(table)) { + if (table->stats_is_persistent()) { if (dict_stats_persistent_storage_check(false)) { @@ -3806,7 +3792,7 @@ dict_stats_update( /* fetch requested, either fetch from persistent statistics storage or use the old method */ - if (table->stat_initialized) { + if (table->stat_initialized()) { return(DB_SUCCESS); } @@ -3850,7 +3836,7 @@ dict_stats_update( goto transient; } #endif - if (dict_stats_auto_recalc_is_enabled(table)) { + if (table->stats_is_auto_recalc()) { return(dict_stats_update( table, DICT_STATS_RECALC_PERSISTENT)); diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index b0c34dc6d30..e44e97e2b8e 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -135,7 +135,9 @@ schedule new estimates for table and index statistics to be calculated. void dict_stats_update_if_needed_func(dict_table_t *table) #endif { - if (UNIV_UNLIKELY(!table->stat_initialized)) { + uint32_t stat{table->stat}; + + if (UNIV_UNLIKELY(!table->stat_initialized(stat))) { /* The table may have been evicted from dict_sys and reloaded internally by InnoDB for FOREIGN KEY processing, but not reloaded by the SQL layer. @@ -154,13 +156,9 @@ void dict_stats_update_if_needed_func(dict_table_t *table) ulonglong counter = table->stat_modified_counter++; ulonglong n_rows = dict_table_get_n_rows(table); - if (dict_stats_is_persistent_enabled(table)) { - if (table->name.is_temporary()) { - return; - } - if (counter > n_rows / 10 /* 10% */ - && dict_stats_auto_recalc_is_enabled(table)) { - + if (table->stats_is_persistent(stat)) { + if (table->stats_is_auto_recalc(stat) + && counter > n_rows / 10 && !table->name.is_temporary()) { #ifdef WITH_WSREP /* Do not add table to background statistic calculation if this thread is not a diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 677e9b68662..0fb03a0cd41 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1644,12 +1644,11 @@ fseg_find_last_used_frag_page_slot( /** Calculate reserved fragment page slots. @param inode file segment index @return number of fragment pages */ -static ulint fseg_get_n_frag_pages(const fseg_inode_t *inode) +static uint32_t fseg_get_n_frag_pages(const fseg_inode_t *inode) noexcept { - ulint i; - ulint count = 0; + uint32_t count = 0; - for (i = 0; i < FSEG_FRAG_ARR_N_SLOTS; i++) { + for (ulint i = 0; i < FSEG_FRAG_ARR_N_SLOTS; i++) { if (FIL_NULL != fseg_get_nth_frag_page_no(inode, i)) { count++; } @@ -1794,21 +1793,24 @@ Calculates the number of pages reserved by a segment, and how many pages are currently used. @return number of reserved pages */ static -ulint +uint32_t fseg_n_reserved_pages_low( /*======================*/ const fseg_inode_t* inode, /*!< in: segment inode */ - ulint* used) /*!< out: number of pages used (not + uint32_t* used) /*!< out: number of pages used (not more than reserved) */ + noexcept { + const uint32_t extent_size = FSP_EXTENT_SIZE; + *used = mach_read_from_4(inode + FSEG_NOT_FULL_N_USED) - + FSP_EXTENT_SIZE * flst_get_len(inode + FSEG_FULL) + + extent_size * flst_get_len(inode + FSEG_FULL) + fseg_get_n_frag_pages(inode); return fseg_get_n_frag_pages(inode) - + FSP_EXTENT_SIZE * flst_get_len(inode + FSEG_FREE) - + FSP_EXTENT_SIZE * flst_get_len(inode + FSEG_NOT_FULL) - + FSP_EXTENT_SIZE * flst_get_len(inode + FSEG_FULL); + + extent_size * flst_get_len(inode + FSEG_FREE) + + extent_size * flst_get_len(inode + FSEG_NOT_FULL) + + extent_size * flst_get_len(inode + FSEG_FULL); } /** Calculate the number of pages reserved by a segment, @@ -1818,9 +1820,9 @@ and how many pages are currently used. @param[out] used number of pages that are used (not more than reserved) @param[in,out] mtr mini-transaction @return number of reserved pages */ -ulint fseg_n_reserved_pages(const buf_block_t &block, - const fseg_header_t *header, ulint *used, - mtr_t *mtr) +uint32_t fseg_n_reserved_pages(const buf_block_t &block, + const fseg_header_t *header, uint32_t *used, + mtr_t *mtr) noexcept { ut_ad(page_align(header) == block.page.frame); buf_block_t *iblock; @@ -1845,7 +1847,7 @@ static dberr_t fseg_fill_free_list(const fseg_inode_t *inode, buf_block_t *iblock, fil_space_t *space, uint32_t hint, mtr_t *mtr) { - ulint used; + uint32_t used; ut_ad(!((page_offset(inode) - FSEG_ARR_OFFSET) % FSEG_INODE_SIZE)); ut_d(space->modify_check(*mtr)); @@ -1996,8 +1998,7 @@ fseg_alloc_free_page_low( dberr_t* err) { ib_id_t seg_id; - ulint used; - ulint reserved; + uint32_t used, reserved; xdes_t* descr; /*!< extent of the hinted page */ uint32_t ret_page; /*!< the allocated page offset, FIL_NULL if could not be allocated */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 7e9439a760c..faec1410236 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1622,7 +1622,7 @@ inline void ha_innobase::reload_statistics() if (table->is_readable()) dict_stats_init(table); else - table->stat_initialized= 1; + table->stat.fetch_or(dict_table_t::STATS_INITIALIZED); } } @@ -2978,6 +2978,44 @@ static int innobase_rollback_by_xid(handlerton*, XID *xid) noexcept return XAER_NOTA; } +/** Initialize the InnoDB persistent statistics attributes. +@param table InnoDB table +@param table_options MariaDB table options +@param sar the value of STATS_AUTO_RECALC +@param initialized whether the InnoDB statistics were already initialized +@return whether table->stats_sample_pages needs to be initialized */ +static bool innodb_copy_stat_flags(dict_table_t *table, + ulong table_options, + enum_stats_auto_recalc sar, + bool initialized) noexcept +{ + if (table->is_temporary() || table->no_rollback()) + { + table->stat= dict_table_t::STATS_INITIALIZED | + dict_table_t::STATS_PERSISTENT_OFF | dict_table_t::STATS_AUTO_RECALC_OFF; + table->stats_sample_pages= 1; + return false; + } + + static_assert(HA_OPTION_STATS_PERSISTENT == + dict_table_t::STATS_PERSISTENT_ON << 11, ""); + static_assert(HA_OPTION_NO_STATS_PERSISTENT == + dict_table_t::STATS_PERSISTENT_OFF << 11, ""); + uint32_t stat= + (table_options & + (HA_OPTION_STATS_PERSISTENT | HA_OPTION_NO_STATS_PERSISTENT)) >> 11; + static_assert(uint32_t{HA_STATS_AUTO_RECALC_ON} << 3 == + dict_table_t::STATS_AUTO_RECALC_ON, ""); + static_assert(uint32_t{HA_STATS_AUTO_RECALC_OFF} << 3 == + dict_table_t::STATS_AUTO_RECALC_OFF, ""); + static_assert(true == dict_table_t::STATS_INITIALIZED, ""); + stat|= (sar & (HA_STATS_AUTO_RECALC_ON | HA_STATS_AUTO_RECALC_OFF)) << 3 | + initialized; + + table->stat= stat; + return true; +} + /*********************************************************************//** Copy table flags from MySQL's HA_CREATE_INFO into an InnoDB table object. Those flags are stored in .frm file and end up in the MySQL table object, @@ -2990,29 +3028,9 @@ innobase_copy_frm_flags_from_create_info( dict_table_t* innodb_table, /*!< in/out: InnoDB table */ const HA_CREATE_INFO* create_info) /*!< in: create info */ { - ibool ps_on; - ibool ps_off; - - if (innodb_table->is_temporary() - || innodb_table->no_rollback()) { - /* Temp tables do not use persistent stats. */ - ps_on = FALSE; - ps_off = TRUE; - } else { - ps_on = create_info->table_options - & HA_OPTION_STATS_PERSISTENT; - ps_off = create_info->table_options - & HA_OPTION_NO_STATS_PERSISTENT; - } - - dict_stats_set_persistent(innodb_table, ps_on, ps_off); - - dict_stats_auto_recalc_set( - innodb_table, - create_info->stats_auto_recalc == HA_STATS_AUTO_RECALC_ON, - create_info->stats_auto_recalc == HA_STATS_AUTO_RECALC_OFF); - - innodb_table->stats_sample_pages = create_info->stats_sample_pages; + if (innodb_copy_stat_flags(innodb_table, create_info->table_options, + create_info->stats_auto_recalc, false)) + innodb_table->stats_sample_pages= create_info->stats_sample_pages; } /*********************************************************************//** @@ -3026,28 +3044,10 @@ innobase_copy_frm_flags_from_table_share( dict_table_t* innodb_table, /*!< in/out: InnoDB table */ const TABLE_SHARE* table_share) /*!< in: table share */ { - ibool ps_on; - ibool ps_off; - - if (innodb_table->is_temporary()) { - /* Temp tables do not use persistent stats */ - ps_on = FALSE; - ps_off = TRUE; - } else { - ps_on = table_share->db_create_options - & HA_OPTION_STATS_PERSISTENT; - ps_off = table_share->db_create_options - & HA_OPTION_NO_STATS_PERSISTENT; - } - - dict_stats_set_persistent(innodb_table, ps_on, ps_off); - - dict_stats_auto_recalc_set( - innodb_table, - table_share->stats_auto_recalc == HA_STATS_AUTO_RECALC_ON, - table_share->stats_auto_recalc == HA_STATS_AUTO_RECALC_OFF); - - innodb_table->stats_sample_pages = table_share->stats_sample_pages; + if (innodb_copy_stat_flags(innodb_table, table_share->db_create_options, + table_share->stats_auto_recalc, + innodb_table->stat_initialized())) + innodb_table->stats_sample_pages= table_share->stats_sample_pages; } /*********************************************************************//** @@ -13387,6 +13387,8 @@ ha_innobase::discard_or_import_tablespace( DBUG_RETURN(HA_ERR_TABLE_NEEDS_UPGRADE); } + ut_ad(m_prebuilt->table->stat_initialized()); + if (m_prebuilt->table->space == fil_system.sys_space) { ib_senderrf( m_prebuilt->trx->mysql_thd, IB_LOG_LEVEL_ERROR, @@ -13460,7 +13462,7 @@ ha_innobase::discard_or_import_tablespace( err, m_prebuilt->table->flags, NULL)); } - if (dict_stats_is_persistent_enabled(m_prebuilt->table)) { + if (m_prebuilt->table->stats_is_persistent()) { dberr_t ret; /* Adjust the persistent statistics. */ @@ -13645,7 +13647,7 @@ int ha_innobase::delete_table(const char *name) /* This looks like the rollback of ALTER TABLE...ADD PARTITION that was caused by MDL timeout. We could have written undo log for inserting the data into the new partitions. */ - if (table->stat_persistent != DICT_STATS_PERSISTENT_OFF) + if (!(table->stat & dict_table_t::STATS_PERSISTENT_OFF)) { /* We do not really know if we are holding MDL_EXCLUSIVE. Even though this code is handling the case that we are not holding @@ -13660,7 +13662,7 @@ int ha_innobase::delete_table(const char *name) DEBUG_SYNC(thd, "before_delete_table_stats"); - if (err == DB_SUCCESS && dict_stats_is_persistent_enabled(table) && + if (err == DB_SUCCESS && table->stats_is_persistent() && !table->is_stats_table()) { table_stats= dict_table_open_on_name(TABLE_STATS_NAME, false, @@ -14033,7 +14035,7 @@ int ha_innobase::truncate() std::this_thread::sleep_for(std::chrono::milliseconds(50)); } - if (error == DB_SUCCESS && dict_stats_is_persistent_enabled(ib_table) && + if (error == DB_SUCCESS && ib_table->stats_is_persistent() && !ib_table->is_stats_table()) { table_stats= dict_table_open_on_name(TABLE_STATS_NAME, false, @@ -14529,7 +14531,7 @@ ha_innobase::scan_time() ulint stat_clustered_index_size; - ut_a(m_prebuilt->table->stat_initialized); + ut_ad(m_prebuilt->table->stat_initialized()); stat_clustered_index_size = m_prebuilt->table->stat_clustered_index_size; @@ -14656,7 +14658,7 @@ innodb_rec_per_key( rec_per_key_t rec_per_key; ib_uint64_t n_diff; - ut_a(index->table->stat_initialized); + ut_ad(index->table->stat_initialized()); ut_ad(i < dict_index_get_n_unique(index)); ut_ad(!dict_index_is_spatial(index)); @@ -14806,7 +14808,7 @@ ha_innobase::info_low( m_prebuilt->trx->op_info = "updating table statistics"; - if (dict_stats_is_persistent_enabled(ib_table)) { + if (ib_table->stats_is_persistent()) { if (is_analyze) { if (!srv_read_only_mode) { dict_stats_recalc_pool_del( @@ -14844,7 +14846,7 @@ ha_innobase::info_low( ulint stat_clustered_index_size; ulint stat_sum_of_other_index_sizes; - ut_a(ib_table->stat_initialized); + ut_ad(ib_table->stat_initialized()); #if !defined NO_ELISION && !defined SUX_LOCK_GENERIC if (xbegin()) { @@ -14998,7 +15000,7 @@ ha_innobase::info_low( auto _ = make_scope_exit([ib_table]() { ib_table->stats_shared_unlock(); }); - ut_a(ib_table->stat_initialized); + ut_ad(ib_table->stat_initialized()); for (uint i = 0; i < table->s->keys; i++) { ulong j; @@ -15925,8 +15927,7 @@ ha_innobase::extra( /* During copy alter operation, InnoDB updates the stats only for non-persistent tables. */ - if (!dict_stats_is_persistent_enabled( - m_prebuilt->table)) { + if (!m_prebuilt->table->stats_is_persistent()) { dict_stats_update_if_needed( m_prebuilt->table, *trx); } @@ -19156,12 +19157,12 @@ static MYSQL_SYSVAR_BOOL(stats_on_metadata, innobase_stats_on_metadata, " SHOW TABLE STATUS for tables that use transient statistics (off by default)", NULL, NULL, FALSE); -static MYSQL_SYSVAR_ULONGLONG(stats_transient_sample_pages, +static MYSQL_SYSVAR_UINT(stats_transient_sample_pages, srv_stats_transient_sample_pages, PLUGIN_VAR_RQCMDARG, "The number of leaf index pages to sample when calculating transient" " statistics (if persistent statistics are not used, default 8)", - NULL, NULL, 8, 1, ~0ULL, 0); + NULL, NULL, 8, 1, ~0U, 0); static MYSQL_SYSVAR_BOOL(stats_persistent, srv_stats_persistent, PLUGIN_VAR_OPCMDARG, @@ -19177,12 +19178,12 @@ static MYSQL_SYSVAR_BOOL(stats_auto_recalc, srv_stats_auto_recalc, " new statistics)", NULL, NULL, TRUE); -static MYSQL_SYSVAR_ULONGLONG(stats_persistent_sample_pages, +static MYSQL_SYSVAR_UINT(stats_persistent_sample_pages, srv_stats_persistent_sample_pages, PLUGIN_VAR_RQCMDARG, "The number of leaf index pages to sample when calculating persistent" " statistics (by ANALYZE, default 20)", - NULL, NULL, 20, 1, ~0ULL, 0); + NULL, NULL, 20, 1, ~0U, 0); static MYSQL_SYSVAR_ULONGLONG(stats_modified_counter, srv_stats_modified_counter, PLUGIN_VAR_RQCMDARG, diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 798c90133d3..b808c74b262 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -11180,7 +11180,7 @@ alter_stats_norebuild( DBUG_ENTER("alter_stats_norebuild"); DBUG_ASSERT(!ctx->need_rebuild()); - if (!dict_stats_is_persistent_enabled(ctx->new_table)) { + if (!ctx->new_table->stats_is_persistent()) { DBUG_VOID_RETURN; } @@ -11214,8 +11214,7 @@ alter_stats_rebuild( { DBUG_ENTER("alter_stats_rebuild"); - if (!table->space - || !dict_stats_is_persistent_enabled(table)) { + if (!table->space || !table->stats_is_persistent()) { DBUG_VOID_RETURN; } diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 0f00a0d39f9..6f0987f153e 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -4777,9 +4777,9 @@ i_s_dict_fill_sys_tablestats(THD* thd, dict_table_t *table, OK(field_store_string(fields[SYS_TABLESTATS_NAME], table->name.m_name)); - OK(fields[SYS_TABLESTATS_INIT]->store(table->stat_initialized, true)); + OK(fields[SYS_TABLESTATS_INIT]->store(table->stat_initialized(), true)); - if (table->stat_initialized) + if (table->stat_initialized()) { OK(fields[SYS_TABLESTATS_NROW]->store(table->stat_n_rows, true)); diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index d90527c0107..b594b86d1ea 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -375,7 +375,7 @@ ibuf_size_update( ibuf.free_list_len = flst_get_len(root + PAGE_HEADER + PAGE_BTR_IBUF_FREE_LIST); - ibuf.height = 1 + btr_page_get_level(root); + ibuf.height = uint8_t(1 + btr_page_get_level(root)); /* the '1 +' is the ibuf header page */ ibuf.size = ibuf.seg_size - (1 + ibuf.free_list_len); @@ -443,18 +443,11 @@ err_exit: goto err_exit; } - /* At startup we intialize ibuf to have a maximum of - CHANGE_BUFFER_DEFAULT_SIZE in terms of percentage of the - buffer pool size. Once ibuf struct is initialized this - value is updated with the user supplied size by calling - ibuf_max_size_update(). */ - ibuf.max_size = ((buf_pool_get_curr_size() >> srv_page_size_shift) - * CHANGE_BUFFER_DEFAULT_SIZE) / 100; - mysql_mutex_init(ibuf_mutex_key, &ibuf_mutex, nullptr); mysql_mutex_init(ibuf_pessimistic_insert_mutex_key, &ibuf_pessimistic_insert_mutex, nullptr); + ibuf_max_size_update(CHANGE_BUFFER_DEFAULT_SIZE); mysql_mutex_lock(&ibuf_mutex); ibuf_size_update(root); mysql_mutex_unlock(&ibuf_mutex); @@ -506,10 +499,12 @@ ibuf_max_size_update( percentage of the buffer pool size */ { if (UNIV_UNLIKELY(!ibuf.index)) return; - ulint new_size = ((buf_pool_get_curr_size() >> srv_page_size_shift) - * new_val) / 100; + ulint new_size = std::min( + (buf_pool_get_curr_size() >> srv_page_size_shift) * new_val + / 100, uint32_t(~0U)); + mysql_mutex_lock(&ibuf_mutex); - ibuf.max_size = new_size; + ibuf.max_size = uint32_t(new_size); mysql_mutex_unlock(&ibuf_mutex); } @@ -4483,17 +4478,17 @@ ibuf_print( return; } - const ulint size= ibuf.size; - const ulint free_list_len= ibuf.free_list_len; - const ulint seg_size= ibuf.seg_size; + const uint32_t size= ibuf.size; + const uint32_t free_list_len= ibuf.free_list_len; + const uint32_t seg_size= ibuf.seg_size; mysql_mutex_unlock(&ibuf_mutex); fprintf(file, "-------------\n" "INSERT BUFFER\n" "-------------\n" - "size " ULINTPF ", free list len " ULINTPF "," - " seg size " ULINTPF ", " ULINTPF " merges\n", + "size %" PRIu32 ", free list len %" PRIu32 "," + " seg size %" PRIu32 ", " ULINTPF " merges\n", size, free_list_len, seg_size, ulint{ibuf.n_merges}); ibuf_print_ops("merged operations:\n", ibuf.n_merged_ops, file); ibuf_print_ops("discarded operations:\n", ibuf.n_discarded_ops, file); diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 97568c61997..842b9fec9cd 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -674,7 +674,7 @@ TPOOL_SUPPRESS_TSAN @return estimated number of rows */ inline uint64_t dict_table_get_n_rows(const dict_table_t *table) { - ut_ad(table->stat_initialized); + ut_ad(table->stat_initialized()); return table->stat_n_rows; } diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index ffc26510679..bec735537fc 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1106,10 +1106,10 @@ struct dict_index_t { is indexed from 0 to n_uniq-1); This is used when innodb_stats_method is "nulls_ignored". */ - ulint stat_index_size; + uint32_t stat_index_size; /*!< approximate index size in database pages */ - ulint stat_n_leaf_pages; + uint32_t stat_n_leaf_pages; /*!< approximate number of leaf pages in the index tree */ bool stats_error_printed; @@ -2358,63 +2358,32 @@ public: /** Statistics for query optimization. Mostly protected by dict_sys.latch and stats_mutex_lock(). @{ */ - /** TRUE if statistics have been calculated the first time after - database startup or table creation. */ - unsigned stat_initialized:1; - /** Timestamp of last recalc of the stats. */ time_t stats_last_recalc; - /** The two bits below are set in the 'stat_persistent' member. They - have the following meaning: - 1. _ON=0, _OFF=0, no explicit persistent stats setting for this table, - the value of the global srv_stats_persistent is used to determine - whether the table has persistent stats enabled or not - 2. _ON=0, _OFF=1, persistent stats are explicitly disabled for this - table, regardless of the value of the global srv_stats_persistent - 3. _ON=1, _OFF=0, persistent stats are explicitly enabled for this - table, regardless of the value of the global srv_stats_persistent - 4. _ON=1, _OFF=1, not allowed, we assert if this ever happens. */ - #define DICT_STATS_PERSISTENT_ON (1 << 1) - #define DICT_STATS_PERSISTENT_OFF (1 << 2) + static constexpr uint32_t STATS_INITIALIZED= 1U; + static constexpr uint32_t STATS_PERSISTENT_ON= 1U << 1; + static constexpr uint32_t STATS_PERSISTENT_OFF= 1U << 2; + static constexpr uint32_t STATS_AUTO_RECALC_ON= 1U << 3; + static constexpr uint32_t STATS_AUTO_RECALC_OFF= 1U << 4; - /** Indicates whether the table uses persistent stats or not. See - DICT_STATS_PERSISTENT_ON and DICT_STATS_PERSISTENT_OFF. */ - ib_uint32_t stat_persistent; + /** flags for index cardinality statistics */ + Atomic_relaxed stat; + /** Approximate clustered index size in database pages. */ + uint32_t stat_clustered_index_size; + /** Approximate size of other indexes in database pages. */ + uint32_t stat_sum_of_other_index_sizes; - /** The two bits below are set in the 'stats_auto_recalc' member. They - have the following meaning: - 1. _ON=0, _OFF=0, no explicit auto recalc setting for this table, the - value of the global srv_stats_persistent_auto_recalc is used to - determine whether the table has auto recalc enabled or not - 2. _ON=0, _OFF=1, auto recalc is explicitly disabled for this table, - regardless of the value of the global srv_stats_persistent_auto_recalc - 3. _ON=1, _OFF=0, auto recalc is explicitly enabled for this table, - regardless of the value of the global srv_stats_persistent_auto_recalc - 4. _ON=1, _OFF=1, not allowed, we assert if this ever happens. */ - #define DICT_STATS_AUTO_RECALC_ON (1 << 1) - #define DICT_STATS_AUTO_RECALC_OFF (1 << 2) - /** Indicates whether the table uses automatic recalc for persistent - stats or not. See DICT_STATS_AUTO_RECALC_ON and - DICT_STATS_AUTO_RECALC_OFF. */ - ib_uint32_t stats_auto_recalc; - - /** The number of pages to sample for this table during persistent - stats estimation. If this is 0, then the value of the global - srv_stats_persistent_sample_pages will be used instead. */ - ulint stats_sample_pages; + /** The number of pages to sample for this table during persistent + stats estimation. If this is 0, then the value of the global + srv_stats_persistent_sample_pages will be used instead. */ + uint32_t stats_sample_pages; /** Approximate number of rows in the table. We periodically calculate new estimates. */ ib_uint64_t stat_n_rows; - /** Approximate clustered index size in database pages. */ - ulint stat_clustered_index_size; - - /** Approximate size of other indexes in database pages. */ - ulint stat_sum_of_other_index_sizes; - /** How many rows are modified since last stats recalc. When a row is inserted, updated, or deleted, we add 1 to this number; we calculate new estimates for the table and the indexes if the table has changed @@ -2551,6 +2520,35 @@ public: /** @return the index for that starts with a specific column */ dict_index_t *get_index(const dict_col_t &col) const; + /** @return whether the statistics are initialized */ + static bool stat_initialized(uint32_t stat) noexcept + { return stat & STATS_INITIALIZED; } + + /** @return whether STATS_PERSISTENT is enabled */ + static bool stats_is_persistent(uint32_t stat) noexcept + { + ut_ad(~(stat & (STATS_PERSISTENT_ON | STATS_PERSISTENT_OFF))); + if (stat & STATS_PERSISTENT_ON) return true; + return !(stat & STATS_PERSISTENT_OFF) && srv_stats_persistent; + } + /** @return whether STATS_AUTO_RECALC is enabled */ + static bool stats_is_auto_recalc(uint32_t stat) noexcept + { + ut_ad(stat_initialized(stat)); + ut_ad(~(stat & (STATS_AUTO_RECALC_ON | STATS_AUTO_RECALC_OFF))); + if (stat & STATS_AUTO_RECALC_ON) return true; + return !(stat & STATS_AUTO_RECALC_OFF) && srv_stats_auto_recalc; + } + + /** @return whether the statistics are initialized */ + bool stat_initialized() const noexcept { return stat_initialized(stat); } + /** @return whether STATS_PERSISTENT is enabled */ + bool stats_is_persistent() const noexcept + { return stats_is_persistent(stat); } + /** @return whether STATS_AUTO_RECALC is enabled */ + bool stats_is_auto_recalc() const noexcept + { return stats_is_auto_recalc(stat); } + /** Create metadata. @param name table name @param space tablespace diff --git a/storage/innobase/include/dict0stats.h b/storage/innobase/include/dict0stats.h index 3b006daf53c..e1d9893dd0a 100644 --- a/storage/innobase/include/dict0stats.h +++ b/storage/innobase/include/dict0stats.h @@ -52,44 +52,6 @@ enum dict_stats_upd_option_t { otherwise do nothing */ }; -/*********************************************************************//** -Set the persistent statistics flag for a given table. This is set only -in the in-memory table object and is not saved on disk. It will be read -from the .frm file upon first open from MySQL after a server restart. */ -UNIV_INLINE -void -dict_stats_set_persistent( -/*======================*/ - dict_table_t* table, /*!< in/out: table */ - ibool ps_on, /*!< in: persistent stats explicitly enabled */ - ibool ps_off) /*!< in: persistent stats explicitly disabled */ - MY_ATTRIBUTE((nonnull)); - -/** @return whether persistent statistics is enabled for a given table */ -UNIV_INLINE -bool -dict_stats_is_persistent_enabled(const dict_table_t* table) - MY_ATTRIBUTE((nonnull, warn_unused_result)); - -/*********************************************************************//** -Set the auto recalc flag for a given table (only honored for a persistent -stats enabled table). The flag is set only in the in-memory table object -and is not saved in InnoDB files. It will be read from the .frm file upon -first open from MySQL after a server restart. */ -UNIV_INLINE -void -dict_stats_auto_recalc_set( -/*=======================*/ - dict_table_t* table, /*!< in/out: table */ - ibool auto_recalc_on, /*!< in: explicitly enabled */ - ibool auto_recalc_off); /*!< in: explicitly disabled */ - -/** @return whether auto recalc is enabled for a given table*/ -UNIV_INLINE -bool -dict_stats_auto_recalc_is_enabled(const dict_table_t* table) - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /*********************************************************************//** Initialize table's stats for the first time when opening a table. */ UNIV_INLINE diff --git a/storage/innobase/include/dict0stats.inl b/storage/innobase/include/dict0stats.inl index dd516275156..fa5b4cccf19 100644 --- a/storage/innobase/include/dict0stats.inl +++ b/storage/innobase/include/dict0stats.inl @@ -25,120 +25,6 @@ Created Jan 23, 2012 Vasil Dimov *******************************************************/ #include "dict0dict.h" -#include "srv0srv.h" - -/*********************************************************************//** -Set the persistent statistics flag for a given table. This is set only -in the in-memory table object and is not saved on disk. It will be read -from the .frm file upon first open from MySQL after a server restart. */ -UNIV_INLINE -void -dict_stats_set_persistent( -/*======================*/ - dict_table_t* table, /*!< in/out: table */ - ibool ps_on, /*!< in: persistent stats explicitly enabled */ - ibool ps_off) /*!< in: persistent stats explicitly disabled */ -{ - /* Not allowed to have both flags set, but a CREATE or ALTER - statement that contains "STATS_PERSISTENT=0 STATS_PERSISTENT=1" would - end up having both set. In this case we clear the OFF flag. */ - if (ps_on && ps_off) { - ps_off = FALSE; - } - - ib_uint32_t stat_persistent = 0; - - if (ps_on) { - stat_persistent |= DICT_STATS_PERSISTENT_ON; - } - - if (ps_off) { - stat_persistent |= DICT_STATS_PERSISTENT_OFF; - } - - /* we rely on this assignment to be atomic */ - table->stat_persistent = stat_persistent; -} - -/** @return whether persistent statistics is enabled for a given table */ -UNIV_INLINE -bool -dict_stats_is_persistent_enabled(const dict_table_t* table) -{ - /* Because of the nature of this check (non-locking) it is possible - that a table becomes: - * PS-disabled immediately after this function has returned TRUE or - * PS-enabled immediately after this function has returned FALSE. - This means that it is possible that we do: - + dict_stats_update(DICT_STATS_RECALC_PERSISTENT) on a table that has - just been PS-disabled or - + dict_stats_update(DICT_STATS_RECALC_TRANSIENT) on a table that has - just been PS-enabled. - This is acceptable. Avoiding this would mean that we would have to - hold dict_sys.latch or stats_mutex_lock() like for accessing the - other ::stat_ members which would be too big performance penalty, - especially when this function is called from - dict_stats_update_if_needed(). */ - - /* we rely on this read to be atomic */ - ib_uint32_t stat_persistent = table->stat_persistent; - - if (stat_persistent & DICT_STATS_PERSISTENT_ON) { - ut_ad(!(stat_persistent & DICT_STATS_PERSISTENT_OFF)); - return(true); - } else if (stat_persistent & DICT_STATS_PERSISTENT_OFF) { - return(false); - } else { - return(srv_stats_persistent); - } -} - -/*********************************************************************//** -Set the auto recalc flag for a given table (only honored for a persistent -stats enabled table). The flag is set only in the in-memory table object -and is not saved in InnoDB files. It will be read from the .frm file upon -first open from MySQL after a server restart. */ -UNIV_INLINE -void -dict_stats_auto_recalc_set( -/*=======================*/ - dict_table_t* table, /*!< in/out: table */ - ibool auto_recalc_on, /*!< in: explicitly enabled */ - ibool auto_recalc_off) /*!< in: explicitly disabled */ -{ - ut_ad(!auto_recalc_on || !auto_recalc_off); - - ib_uint32_t stats_auto_recalc = 0; - - if (auto_recalc_on) { - stats_auto_recalc |= DICT_STATS_AUTO_RECALC_ON; - } - - if (auto_recalc_off) { - stats_auto_recalc |= DICT_STATS_AUTO_RECALC_OFF; - } - - /* we rely on this assignment to be atomic */ - table->stats_auto_recalc = stats_auto_recalc; -} - -/** @return whether auto recalc is enabled for a given table*/ -UNIV_INLINE -bool -dict_stats_auto_recalc_is_enabled(const dict_table_t* table) -{ - /* we rely on this read to be atomic */ - ib_uint32_t stats_auto_recalc = table->stats_auto_recalc; - - if (stats_auto_recalc & DICT_STATS_AUTO_RECALC_ON) { - ut_ad(!(stats_auto_recalc & DICT_STATS_AUTO_RECALC_OFF)); - return(true); - } else if (stats_auto_recalc & DICT_STATS_AUTO_RECALC_OFF) { - return(false); - } else { - return(srv_stats_auto_recalc); - } -} /*********************************************************************//** Initialize table's stats for the first time when opening a table. */ @@ -150,13 +36,15 @@ dict_stats_init( { ut_ad(!table->stats_mutex_is_owner()); - if (table->stat_initialized) { + uint32_t stat = table->stat; + + if (stat & dict_table_t::STATS_INITIALIZED) { return; } dict_stats_upd_option_t opt; - if (dict_stats_is_persistent_enabled(table)) { + if (table->stats_is_persistent(stat)) { opt = DICT_STATS_FETCH_ONLY_IF_NOT_IN_MEMORY; } else { opt = DICT_STATS_RECALC_TRANSIENT; @@ -178,7 +66,7 @@ dict_stats_deinit( ut_ad(table->get_ref_count() == 0); #ifdef HAVE_valgrind - if (!table->stat_initialized) { + if (!table->stat_initialized()) { return; } @@ -215,5 +103,6 @@ dict_stats_deinit( sizeof(index->stat_n_leaf_pages)); } #endif /* HAVE_valgrind */ - table->stat_initialized = FALSE; + + table->stat = table->stat & ~dict_table_t::STATS_INITIALIZED; } diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 4b8dd20a224..fe8d5980ac4 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -355,9 +355,9 @@ and how many pages are currently used. @param[out] used number of pages that are used (not more than reserved) @param[in,out] mtr mini-transaction @return number of reserved pages */ -ulint fseg_n_reserved_pages(const buf_block_t &block, - const fseg_header_t *header, ulint *used, - mtr_t *mtr) +uint32_t fseg_n_reserved_pages(const buf_block_t &block, + const fseg_header_t *header, uint32_t *used, + mtr_t *mtr) noexcept MY_ATTRIBUTE((nonnull)); /**********************************************************************//** Allocates a single free page from a segment. This function implements diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index c246b2ef513..b00532f9a02 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -62,11 +62,11 @@ extern ulong innodb_change_buffering; /** Insert buffer struct */ struct ibuf_t{ - Atomic_relaxed size; /*!< current size of the ibuf index + Atomic_relaxed size; /*!< current size of the ibuf index tree, in pages */ - Atomic_relaxed max_size; /*!< recommended maximum size of the + Atomic_relaxed max_size;/*!< recommended maximum size of the ibuf index tree, in pages */ - ulint seg_size; /*!< allocated pages of the file + uint32_t seg_size; /*!< allocated pages of the file segment containing ibuf header and tree */ bool empty; /*!< Protected by the page @@ -75,8 +75,8 @@ struct ibuf_t{ (FSP_IBUF_TREE_ROOT_PAGE_NO). true if and only if the insert buffer tree is empty. */ - ulint free_list_len; /*!< length of the free list */ - ulint height; /*!< tree height */ + uint8_t height; /*!< tree height */ + uint32_t free_list_len; /*!< length of the free list */ dict_index_t* index; /*!< insert buffer index */ /** number of pages merged */ diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index bca858ca409..374b15f0d66 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -294,9 +294,9 @@ extern uint srv_fast_shutdown; extern ibool srv_innodb_status; -extern unsigned long long srv_stats_transient_sample_pages; +extern uint32_t srv_stats_transient_sample_pages; extern my_bool srv_stats_persistent; -extern unsigned long long srv_stats_persistent_sample_pages; +extern uint32_t srv_stats_persistent_sample_pages; extern my_bool srv_stats_auto_recalc; extern my_bool srv_stats_include_delete_marked; extern unsigned long long srv_stats_modified_counter; diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 4c8468dbc41..8b20d70fb15 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1599,7 +1599,7 @@ row_update_for_mysql(row_prebuilt_t* prebuilt) ut_a(prebuilt->magic_n == ROW_PREBUILT_ALLOCATED); ut_a(prebuilt->magic_n2 == ROW_PREBUILT_ALLOCATED); ut_a(prebuilt->template_type == ROW_MYSQL_WHOLE_ROW); - ut_ad(table->stat_initialized); + ut_ad(table->stat_initialized()); if (!table->is_readable()) { return row_mysql_get_table_error(trx, table); diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 442dd4f6e1d..547936e34c1 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -1564,7 +1564,7 @@ row_purge_record_func( case TRX_UNDO_DEL_MARK_REC: purged = row_purge_del_mark(node); if (purged) { - if (node->table->stat_initialized + if (node->table->stat_initialized() && srv_stats_include_delete_marked) { dict_stats_update_if_needed( node->table, *thr->graph->trx); diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc index 23255cc98f9..b02cbec56ea 100644 --- a/storage/innobase/row/row0uins.cc +++ b/storage/innobase/row/row0uins.cc @@ -614,7 +614,7 @@ row_undo_ins( err = row_undo_ins_remove_clust_rec(node); } - if (err == DB_SUCCESS && node->table->stat_initialized) { + if (err == DB_SUCCESS && node->table->stat_initialized()) { /* Not protected by dict_sys.latch or table->stats_mutex_lock() for performance reasons, we would rather get garbage diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index d8a3b29b630..a1bc1df3dd2 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -1388,7 +1388,7 @@ rollback_clust: bool update_statistics = !(node->cmpl_info & UPD_NODE_NO_ORD_CHANGE); - if (err == DB_SUCCESS && node->table->stat_initialized) { + if (err == DB_SUCCESS && node->table->stat_initialized()) { switch (node->rec_type) { case TRX_UNDO_UPD_EXIST_REC: break; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index b3bdfef9c66..f73a95d5226 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -291,13 +291,13 @@ this many index pages, there are 2 ways to calculate statistics: in the innodb database. * quick transient stats, that are used if persistent stats for the given table/index are not found in the innodb database */ -unsigned long long srv_stats_transient_sample_pages; +uint32_t srv_stats_transient_sample_pages; /** innodb_stats_persistent */ my_bool srv_stats_persistent; /** innodb_stats_include_delete_marked */ my_bool srv_stats_include_delete_marked; /** innodb_stats_persistent_sample_pages */ -unsigned long long srv_stats_persistent_sample_pages; +uint32_t srv_stats_persistent_sample_pages; /** innodb_stats_auto_recalc */ my_bool srv_stats_auto_recalc; From 6e6a1b316ca8df5116613fbe4ca2dc37b3c73bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Feb 2025 09:00:16 +0200 Subject: [PATCH 46/47] MDEV-35000: dict_table_close() breaks STATS_AUTO_RECALC stats_deinit(): Replaces dict_stats_deinit(). Deinitialize the statistics for persistent tables, so that they will be reloaded or recalculated on a subsequent ha_innobase::open(). ha_innobase::rename_table(): Invoke stats_deinit() so that the subsequent ha_innobase::open() will reload the InnoDB persistent statistics. That is, it will remain possible to have the InnoDB persistent statistics reloaded by executing the following: RENAME TABLE t TO tmp, tmp TO t; dict_table_close(table): Replaced with table->release(). There will no longer be any logic that would attempt to ensure that the InnoDB persistent statistics will be reloaded after FLUSH TABLES has been executed. This also fixes the problem that dict_table_t::stat_modified_counter would be frequently reset to 0, whenever ha_innobase::open() is invoked after the table reference count had dropped to 0. dict_table_close(table, thd, mdl): Remove the parameter "dict_locked". Do not try to invalidate the statistics. ha_innobase::statistics_init(): Replaces dict_stats_init(table). Reviewed by: Thirunarayanan Balathandayuthapani --- mysql-test/main/type_num_innodb.result | 128 ++++- .../suite/gcol/r/innodb_virtual_stats.result | 52 ++ ...db_stats_auto_recalc_on_nonexistent.result | 6 +- .../suite/innodb/r/innodb_stats_fetch.result | 2 +- ...nodb_stats_auto_recalc_on_nonexistent.test | 14 +- .../suite/innodb/t/innodb_stats_fetch.test | 2 +- storage/innobase/CMakeLists.txt | 1 - storage/innobase/dict/dict0defrag_bg.cc | 18 +- storage/innobase/dict/dict0dict.cc | 72 +-- storage/innobase/dict/dict0stats.cc | 452 +++++------------- storage/innobase/dict/dict0stats_bg.cc | 8 +- storage/innobase/fts/fts0opt.cc | 2 +- storage/innobase/handler/ha_innodb.cc | 301 ++++++++---- storage/innobase/handler/ha_innodb.h | 3 + storage/innobase/handler/handler0alter.cc | 40 +- storage/innobase/handler/i_s.cc | 14 +- storage/innobase/include/dict0dict.h | 28 +- storage/innobase/include/dict0dict.inl | 4 +- storage/innobase/include/dict0mem.h | 5 +- storage/innobase/include/dict0stats.h | 103 ++-- storage/innobase/include/dict0stats.inl | 108 ----- storage/innobase/row/row0ins.cc | 2 +- storage/innobase/row/row0mysql.cc | 2 +- storage/innobase/row/row0uins.cc | 8 +- storage/innobase/row/row0umod.cc | 5 +- storage/innobase/row/row0upd.cc | 4 +- storage/innobase/trx/trx0purge.cc | 4 +- 27 files changed, 644 insertions(+), 744 deletions(-) delete mode 100644 storage/innobase/include/dict0stats.inl diff --git a/mysql-test/main/type_num_innodb.result b/mysql-test/main/type_num_innodb.result index 76ab147ea7e..72e7752b3d1 100644 --- a/mysql-test/main/type_num_innodb.result +++ b/mysql-test/main/type_num_innodb.result @@ -46,23 +46,70 @@ ALTER TABLE t1 MODIFY a DECIMAL(10,0); SELECT * FROM t1,t2 WHERE a=d; a b c pk d e Warnings: -Warning 1292 Truncated incorrect DECIMAL value: 'd' -Warning 1292 Truncated incorrect DECIMAL value: 'd' -Warning 1292 Truncated incorrect DECIMAL value: 'f' -Warning 1292 Truncated incorrect DECIMAL value: 'f' -Warning 1292 Truncated incorrect DECIMAL value: 'g' -Warning 1292 Truncated incorrect DECIMAL value: 'k' -Warning 1292 Truncated incorrect DECIMAL value: 'm' -Warning 1292 Truncated incorrect DECIMAL value: 'm' -Warning 1292 Truncated incorrect DECIMAL value: 'm' -Warning 1292 Truncated incorrect DECIMAL value: 'o' -Warning 1292 Truncated incorrect DECIMAL value: 'q' -Warning 1292 Truncated incorrect DECIMAL value: 'r' -Warning 1292 Truncated incorrect DECIMAL value: 'u' -Warning 1292 Truncated incorrect DECIMAL value: 'w' -Warning 1292 Truncated incorrect DECIMAL value: 'x' -Warning 1292 Truncated incorrect DECIMAL value: 'x' -Warning 1292 Truncated incorrect DECIMAL value: 'y' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'g' +Warning 1292 Truncated incorrect DOUBLE value: 'k' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'o' +Warning 1292 Truncated incorrect DOUBLE value: 'q' +Warning 1292 Truncated incorrect DOUBLE value: 'r' +Warning 1292 Truncated incorrect DOUBLE value: 'u' +Warning 1292 Truncated incorrect DOUBLE value: 'w' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'y' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'g' +Warning 1292 Truncated incorrect DOUBLE value: 'k' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'o' +Warning 1292 Truncated incorrect DOUBLE value: 'q' +Warning 1292 Truncated incorrect DOUBLE value: 'r' +Warning 1292 Truncated incorrect DOUBLE value: 'u' +Warning 1292 Truncated incorrect DOUBLE value: 'w' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'y' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'g' +Warning 1292 Truncated incorrect DOUBLE value: 'k' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'o' +Warning 1292 Truncated incorrect DOUBLE value: 'q' +Warning 1292 Truncated incorrect DOUBLE value: 'r' +Warning 1292 Truncated incorrect DOUBLE value: 'u' +Warning 1292 Truncated incorrect DOUBLE value: 'w' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'y' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'g' +Warning 1292 Truncated incorrect DOUBLE value: 'k' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'o' +Warning 1292 Truncated incorrect DOUBLE value: 'q' +Warning 1292 Truncated incorrect DOUBLE value: 'r' +Warning 1292 Truncated incorrect DOUBLE value: 'u' ALTER TABLE t1 MODIFY a DOUBLE; SELECT * FROM t1,t2 WHERE a=d; a b c pk d e @@ -84,6 +131,53 @@ Warning 1292 Truncated incorrect DOUBLE value: 'w' Warning 1292 Truncated incorrect DOUBLE value: 'x' Warning 1292 Truncated incorrect DOUBLE value: 'x' Warning 1292 Truncated incorrect DOUBLE value: 'y' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'g' +Warning 1292 Truncated incorrect DOUBLE value: 'k' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'o' +Warning 1292 Truncated incorrect DOUBLE value: 'q' +Warning 1292 Truncated incorrect DOUBLE value: 'r' +Warning 1292 Truncated incorrect DOUBLE value: 'u' +Warning 1292 Truncated incorrect DOUBLE value: 'w' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'y' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'g' +Warning 1292 Truncated incorrect DOUBLE value: 'k' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'o' +Warning 1292 Truncated incorrect DOUBLE value: 'q' +Warning 1292 Truncated incorrect DOUBLE value: 'r' +Warning 1292 Truncated incorrect DOUBLE value: 'u' +Warning 1292 Truncated incorrect DOUBLE value: 'w' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'x' +Warning 1292 Truncated incorrect DOUBLE value: 'y' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'd' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'f' +Warning 1292 Truncated incorrect DOUBLE value: 'g' +Warning 1292 Truncated incorrect DOUBLE value: 'k' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'm' +Warning 1292 Truncated incorrect DOUBLE value: 'o' +Warning 1292 Truncated incorrect DOUBLE value: 'q' +Warning 1292 Truncated incorrect DOUBLE value: 'r' +Warning 1292 Truncated incorrect DOUBLE value: 'u' DROP TABLE t1,t2; # # End of 10.2 tests diff --git a/mysql-test/suite/gcol/r/innodb_virtual_stats.result b/mysql-test/suite/gcol/r/innodb_virtual_stats.result index 74a0480883d..c0f595263df 100644 --- a/mysql-test/suite/gcol/r/innodb_virtual_stats.result +++ b/mysql-test/suite/gcol/r/innodb_virtual_stats.result @@ -38,6 +38,10 @@ idxa n_diff_pfx01 a idxa n_diff_pfx02 a,DB_ROW_ID idxa n_leaf_pages Number of leaf pages in the index idxa size Number of pages in the index +idxb n_diff_pfx01 b +idxb n_diff_pfx02 b,DB_ROW_ID +idxb n_leaf_pages Number of leaf pages in the index +idxb size Number of pages in the index vidxcd n_diff_pfx01 c vidxcd n_diff_pfx02 c,d vidxcd n_diff_pfx03 c,d,DB_ROW_ID @@ -54,6 +58,14 @@ index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index GEN_CLUST_INDEX size Number of pages in the index +idxb n_diff_pfx01 b +idxb n_diff_pfx02 b,DB_ROW_ID +idxb n_leaf_pages Number of leaf pages in the index +idxb size Number of pages in the index +vidxcd n_diff_pfx01 d +vidxcd n_diff_pfx02 d,DB_ROW_ID +vidxcd n_leaf_pages Number of leaf pages in the index +vidxcd size Number of pages in the index ALTER TABLE t ADD INDEX vidxe (e), ALGORITHM=INPLACE; select count(*) from t; count(*) @@ -65,6 +77,18 @@ index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index GEN_CLUST_INDEX size Number of pages in the index +idxb n_diff_pfx01 b +idxb n_diff_pfx02 b,DB_ROW_ID +idxb n_leaf_pages Number of leaf pages in the index +idxb size Number of pages in the index +vidxcd n_diff_pfx01 d +vidxcd n_diff_pfx02 d,DB_ROW_ID +vidxcd n_leaf_pages Number of leaf pages in the index +vidxcd size Number of pages in the index +vidxe n_diff_pfx01 e +vidxe n_diff_pfx02 e,DB_ROW_ID +vidxe n_leaf_pages Number of leaf pages in the index +vidxe size Number of pages in the index ALTER TABLE t ADD COLUMN f INT GENERATED ALWAYS AS(a + a), ADD INDEX vidxf (f), ALGORITHM=INPLACE; select count(*) from t; count(*) @@ -76,6 +100,22 @@ index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index GEN_CLUST_INDEX size Number of pages in the index +idxb n_diff_pfx01 b +idxb n_diff_pfx02 b,DB_ROW_ID +idxb n_leaf_pages Number of leaf pages in the index +idxb size Number of pages in the index +vidxcd n_diff_pfx01 d +vidxcd n_diff_pfx02 d,DB_ROW_ID +vidxcd n_leaf_pages Number of leaf pages in the index +vidxcd size Number of pages in the index +vidxe n_diff_pfx01 e +vidxe n_diff_pfx02 e,DB_ROW_ID +vidxe n_leaf_pages Number of leaf pages in the index +vidxe size Number of pages in the index +vidxf n_diff_pfx01 f +vidxf n_diff_pfx02 f,DB_ROW_ID +vidxf n_leaf_pages Number of leaf pages in the index +vidxf size Number of pages in the index ALTER TABLE t DROP INDEX vidxcd; SELECT index_name, stat_name, stat_description FROM mysql.innodb_index_stats @@ -84,4 +124,16 @@ index_name stat_name stat_description GEN_CLUST_INDEX n_diff_pfx01 DB_ROW_ID GEN_CLUST_INDEX n_leaf_pages Number of leaf pages in the index GEN_CLUST_INDEX size Number of pages in the index +idxb n_diff_pfx01 b +idxb n_diff_pfx02 b,DB_ROW_ID +idxb n_leaf_pages Number of leaf pages in the index +idxb size Number of pages in the index +vidxe n_diff_pfx01 e +vidxe n_diff_pfx02 e,DB_ROW_ID +vidxe n_leaf_pages Number of leaf pages in the index +vidxe size Number of pages in the index +vidxf n_diff_pfx01 f +vidxf n_diff_pfx02 f,DB_ROW_ID +vidxf n_leaf_pages Number of leaf pages in the index +vidxf size Number of pages in the index DROP TABLE t; diff --git a/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_on_nonexistent.result b/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_on_nonexistent.result index 6b2397432df..9a216374125 100644 --- a/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_on_nonexistent.result +++ b/mysql-test/suite/innodb/r/innodb_stats_auto_recalc_on_nonexistent.result @@ -5,13 +5,13 @@ COUNT(*) 1 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; COUNT(*) 3 SELECT * FROM t; -FLUSH TABLE t; DELETE FROM mysql.innodb_index_stats WHERE table_name = 't'; DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 0 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; COUNT(*) 0 +RENAME TABLE t TO tmp, tmp TO t; SELECT * FROM t; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 1 @@ -25,13 +25,13 @@ COUNT(*) 1 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; COUNT(*) 3 SELECT * FROM t; -FLUSH TABLE t; DELETE FROM mysql.innodb_index_stats WHERE table_name = 't'; DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 0 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; COUNT(*) 0 +RENAME TABLE t TO tmp, tmp TO t; SELECT * FROM t; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 1 @@ -45,13 +45,13 @@ COUNT(*) 1 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; COUNT(*) 3 SELECT * FROM t; -FLUSH TABLE t; DELETE FROM mysql.innodb_index_stats WHERE table_name = 't'; DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 0 SELECT COUNT(*) FROM mysql.innodb_index_stats WHERE table_name = 't'; COUNT(*) 0 +RENAME TABLE t TO tmp, tmp TO t; SELECT * FROM t; SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE table_name = 't'; COUNT(*) 0 diff --git a/mysql-test/suite/innodb/r/innodb_stats_fetch.result b/mysql-test/suite/innodb/r/innodb_stats_fetch.result index d2fce6301d3..772cc5d146a 100644 --- a/mysql-test/suite/innodb/r/innodb_stats_fetch.result +++ b/mysql-test/suite/innodb/r/innodb_stats_fetch.result @@ -125,7 +125,7 @@ WHERE table_name = 'test_ps_fetch' AND index_name = 'idx' AND stat_name = 'n_diff_pfx02'; -FLUSH TABLE test_ps_fetch; +RENAME TABLE test_ps_fetch TO tmp, tmp TO test_ps_fetch; SELECT seq_in_index, column_name, cardinality FROM information_schema.statistics WHERE table_name = 'test_ps_fetch' ORDER BY index_name, seq_in_index; diff --git a/mysql-test/suite/innodb/t/innodb_stats_auto_recalc_on_nonexistent.test b/mysql-test/suite/innodb/t/innodb_stats_auto_recalc_on_nonexistent.test index 4cd910071e8..b9ef74ea30d 100644 --- a/mysql-test/suite/innodb/t/innodb_stats_auto_recalc_on_nonexistent.test +++ b/mysql-test/suite/innodb/t/innodb_stats_auto_recalc_on_nonexistent.test @@ -17,9 +17,7 @@ CREATE TABLE t (a INT, PRIMARY KEY (a)) ENGINE=INNODB; -- eval $check_stats1 -- eval $check_stats2 -# open and close the table SELECT * FROM t; -FLUSH TABLE t; DELETE FROM mysql.innodb_index_stats WHERE table_name = 't'; DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; @@ -27,7 +25,8 @@ DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; -- eval $check_stats1 -- eval $check_stats2 -# open the table, causing stats recalc/save +# rename and open the table, causing stats recalc/save +RENAME TABLE t TO tmp, tmp TO t; SELECT * FROM t; -- eval $check_stats1 @@ -43,9 +42,7 @@ CREATE TABLE t (a INT, PRIMARY KEY (a)) ENGINE=INNODB STATS_AUTO_RECALC=1; -- eval $check_stats1 -- eval $check_stats2 -# open and close the table SELECT * FROM t; -FLUSH TABLE t; DELETE FROM mysql.innodb_index_stats WHERE table_name = 't'; DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; @@ -53,7 +50,7 @@ DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; -- eval $check_stats1 -- eval $check_stats2 -# open the table, causing stats recalc/save +RENAME TABLE t TO tmp, tmp TO t; SELECT * FROM t; -- eval $check_stats1 @@ -69,9 +66,7 @@ CREATE TABLE t (a INT, PRIMARY KEY (a)) ENGINE=INNODB STATS_AUTO_RECALC=0; -- eval $check_stats1 -- eval $check_stats2 -# open and close the table SELECT * FROM t; -FLUSH TABLE t; DELETE FROM mysql.innodb_index_stats WHERE table_name = 't'; DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; @@ -79,7 +74,8 @@ DELETE FROM mysql.innodb_table_stats WHERE table_name = 't'; -- eval $check_stats1 -- eval $check_stats2 -# open the table, stats should not be present, since autorecalc is disabled +# rename the table, stats should not be present, since autorecalc is disabled +RENAME TABLE t TO tmp, tmp TO t; SELECT * FROM t; -- eval $check_stats1 diff --git a/mysql-test/suite/innodb/t/innodb_stats_fetch.test b/mysql-test/suite/innodb/t/innodb_stats_fetch.test index 031144d82bc..844e17a521b 100644 --- a/mysql-test/suite/innodb/t/innodb_stats_fetch.test +++ b/mysql-test/suite/innodb/t/innodb_stats_fetch.test @@ -69,7 +69,7 @@ table_name = 'test_ps_fetch' AND index_name = 'idx' AND stat_name = 'n_diff_pfx02'; -FLUSH TABLE test_ps_fetch; +RENAME TABLE test_ps_fetch TO tmp, tmp TO test_ps_fetch; SELECT seq_in_index, column_name, cardinality FROM information_schema.statistics WHERE table_name = 'test_ps_fetch' diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 7dde90286f5..64540c2256d 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -226,7 +226,6 @@ SET(INNOBASE_SOURCES include/dict0pagecompress.h include/dict0pagecompress.inl include/dict0stats.h - include/dict0stats.inl include/dict0stats_bg.h include/dict0types.h include/dyn0buf.h diff --git a/storage/innobase/dict/dict0defrag_bg.cc b/storage/innobase/dict/dict0defrag_bg.cc index 81b8dbf077d..32470a4e2e0 100644 --- a/storage/innobase/dict/dict0defrag_bg.cc +++ b/storage/innobase/dict/dict0defrag_bg.cc @@ -196,7 +196,7 @@ static void dict_stats_process_entry_from_defrag_pool(THD *thd) ? dict_table_find_index_on_id(table, index_id) : nullptr) if (index->is_btree()) dict_stats_save_defrag_stats(index); - dict_table_close(table, false, thd, mdl); + dict_table_close(table, thd, mdl); } } @@ -230,7 +230,7 @@ dberr_t dict_stats_save_defrag_summary(dict_index_t *index, THD *thd) { release_and_exit: if (table_stats) - dict_table_close(table_stats, false, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); return DB_STATS_DO_NOT_EXIST; } @@ -246,7 +246,7 @@ release_and_exit: goto release_and_exit; if (strcmp(index_stats->name.m_name, INDEX_STATS_NAME)) { - dict_table_close(index_stats, false, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); goto release_and_exit; } @@ -272,9 +272,9 @@ release_and_exit: trx->rollback(); if (table_stats) - dict_table_close(table_stats, true, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); if (index_stats) - dict_table_close(index_stats, true, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); row_mysql_unlock_data_dictionary(trx); trx->free(); @@ -367,7 +367,7 @@ dict_stats_save_defrag_stats( { release_and_exit: if (table_stats) - dict_table_close(table_stats, false, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); return DB_STATS_DO_NOT_EXIST; } @@ -384,7 +384,7 @@ release_and_exit: if (strcmp(index_stats->name.m_name, INDEX_STATS_NAME)) { - dict_table_close(index_stats, false, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); goto release_and_exit; } @@ -424,9 +424,9 @@ release_and_exit: trx->rollback(); if (table_stats) - dict_table_close(table_stats, true, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); if (index_stats) - dict_table_close(index_stats, true, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); row_mysql_unlock_data_dictionary(trx); trx->free(); diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 38e95282f26..1540d0f812a 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -195,70 +195,6 @@ dict_tables_have_same_db( return(FALSE); } -/** Decrement the count of open handles */ -void dict_table_close(dict_table_t *table) -{ - if (table->get_ref_count() == 1 && table->stats_is_persistent() && - strchr(table->name.m_name, '/')) - { - /* It looks like we are closing the last handle. The user could - have executed FLUSH TABLES in order to have the statistics reloaded - from the InnoDB persistent statistics tables. We must acquire - exclusive dict_sys.latch to prevent a race condition with another - thread concurrently acquiring a handle on the table. */ - dict_sys.lock(SRW_LOCK_CALL); - if (table->release()) - { - table->stats_mutex_lock(); - if (table->get_ref_count() == 0) - dict_stats_deinit(table); - table->stats_mutex_unlock(); - } - dict_sys.unlock(); - } - else - table->release(); -} - -/** Decrements the count of open handles of a table. -@param[in,out] table table -@param[in] dict_locked whether dict_sys.latch is being held -@param[in] thd thread to release MDL -@param[in] mdl metadata lock or NULL if the thread - is a foreground one. */ -void -dict_table_close( - dict_table_t* table, - bool dict_locked, - THD* thd, - MDL_ticket* mdl) -{ - if (!dict_locked) - dict_table_close(table); - else - { - if (table->release() && table->stats_is_persistent() && - strchr(table->name.m_name, '/')) - { - /* Force persistent stats re-read upon next open of the table so - that FLUSH TABLE can be used to forcibly fetch stats from disk if - they have been manually modified. */ - table->stats_mutex_lock(); - if (table->get_ref_count() == 0) - dict_stats_deinit(table); - table->stats_mutex_unlock(); - } - - ut_ad(dict_lru_validate()); - ut_ad(dict_sys.find(table)); - } - - if (!thd || !mdl); - else if (MDL_context *mdl_context= static_cast - (thd_mdl_context(thd))) - mdl_context->release_lock(mdl); -} - /** Check if the table has a given (non_virtual) column. @param[in] table table object @param[in] col_name column name @@ -585,6 +521,14 @@ dict_index_get_nth_field_pos( return(ULINT_UNDEFINED); } +void mdl_release(THD *thd, MDL_ticket *mdl) noexcept +{ + if (!thd || !mdl); + else if (MDL_context *mdl_context= static_cast + (thd_mdl_context(thd))) + mdl_context->release_lock(mdl); +} + /** Parse the table file name into table name and database name. @tparam dict_frozen whether the caller holds dict_sys.latch @param[in,out] db_name database name buffer diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index b9bb3645dd5..9cdc62bc49d 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -359,7 +359,7 @@ dict_table_schema_check( if (!table) { if (opt_bootstrap) - return DB_TABLE_NOT_FOUND; + return DB_STATS_DO_NOT_EXIST; if (req_schema == &table_stats_schema) { if (innodb_table_stats_not_found_reported) { return DB_STATS_DO_NOT_EXIST; @@ -377,10 +377,10 @@ dict_table_schema_check( snprintf(errstr, errstr_sz, "Table %s not found.", req_schema->table_name_sql); - return DB_TABLE_NOT_FOUND; + return DB_STATS_DO_NOT_EXIST; } - if (!table->is_readable() && !table->space) { + if (!table->is_readable() || !table->space) { /* missing tablespace */ snprintf(errstr, errstr_sz, "Tablespace for table %s is missing.", @@ -491,11 +491,8 @@ dict_table_schema_check( return DB_SUCCESS; } -/*********************************************************************//** -Checks whether the persistent statistics storage exists and that all -tables have the proper structure. -@return true if exists and all tables are ok */ -static bool dict_stats_persistent_storage_check(bool dict_already_locked) +dict_stats_schema_check +dict_stats_persistent_storage_check(bool dict_already_locked) noexcept { char errstr[512]; dberr_t ret; @@ -521,14 +518,14 @@ static bool dict_stats_persistent_storage_check(bool dict_already_locked) switch (ret) { case DB_SUCCESS: - return true; + return SCHEMA_OK; + case DB_STATS_DO_NOT_EXIST: + return SCHEMA_NOT_EXIST; default: if (!opt_bootstrap) { - ib::error() << errstr; + sql_print_error("InnoDB: %s", errstr); } - /* fall through */ - case DB_STATS_DO_NOT_EXIST: - return false; + return SCHEMA_INVALID; } } @@ -544,13 +541,16 @@ dberr_t dict_stats_exec_sql(pars_info_t *pinfo, const char* sql, trx_t *trx) { ut_ad(dict_sys.locked()); - if (!dict_stats_persistent_storage_check(true)) - { - pars_info_free(pinfo); - return DB_STATS_DO_NOT_EXIST; + switch (dict_stats_persistent_storage_check(true)) { + case SCHEMA_OK: + return que_eval_sql(pinfo, sql, trx); + case SCHEMA_INVALID: + case SCHEMA_NOT_EXIST: + break; } - return que_eval_sql(pinfo, sql, trx); + pars_info_free(pinfo); + return DB_STATS_DO_NOT_EXIST; } @@ -1205,19 +1205,7 @@ invalid: return err; } -/*********************************************************************//** -Calculates new estimates for table and index statistics. This function -is relatively quick and is used to calculate transient statistics that -are not saved on disk. -This was the only way to calculate statistics before the -Persistent Statistics feature was introduced. -@return error code -@retval DB_SUCCESS_LOCKED REC if the table under bulk insert operation */ -static -dberr_t -dict_stats_update_transient( -/*========================*/ - dict_table_t* table) /*!< in/out: table */ +dberr_t dict_stats_update_transient(dict_table_t *table) noexcept { ut_ad(!table->stats_mutex_is_owner()); @@ -1230,17 +1218,16 @@ dict_stats_update_transient( index = dict_table_get_first_index(table); - if (!table->space) { - /* Nothing to do. */ -empty_table: + if (!index || !table->space) { dict_stats_empty_table(table, true); - return err; - } else if (index == NULL) { - /* Table definition is corrupt */ + return DB_SUCCESS; + } - ib::warn() << "Table " << table->name - << " has no indexes. Cannot calculate statistics."; - goto empty_table; + if (trx_id_t bulk_trx_id = table->bulk_trx_id) { + if (trx_sys.find(nullptr, bulk_trx_id, false)) { + dict_stats_empty_table(table, false); + return DB_SUCCESS_LOCKED_REC; + } } for (; index != NULL; index = dict_table_get_next_index(index)) { @@ -2631,17 +2618,7 @@ found_level: DBUG_RETURN(result); } -/*********************************************************************//** -Calculates new estimates for table and index statistics. This function -is relatively slow and is used to calculate persistent statistics that -will be saved on disk. -@return DB_SUCCESS or error code -@retval DB_SUCCESS_LOCKED_REC if the table under bulk insert operation */ -static -dberr_t -dict_stats_update_persistent( -/*=========================*/ - dict_table_t* table) /*!< in/out: table */ +dberr_t dict_stats_update_persistent(dict_table_t *table) noexcept { dict_index_t* index; @@ -2649,6 +2626,13 @@ dict_stats_update_persistent( DEBUG_SYNC_C("dict_stats_update_persistent"); + if (trx_id_t bulk_trx_id = table->bulk_trx_id) { + if (trx_sys.find(nullptr, bulk_trx_id, false)) { + dict_stats_empty_table(table, false); + return DB_SUCCESS_LOCKED_REC; + } + } + /* analyze the clustered index first */ index = dict_table_get_first_index(table); @@ -2747,6 +2731,18 @@ dict_stats_update_persistent( return(DB_SUCCESS); } +dberr_t dict_stats_update_persistent_try(dict_table_t *table) +{ + if (table->stats_is_persistent() && + dict_stats_persistent_storage_check(false) == SCHEMA_OK) + { + if (dberr_t err= dict_stats_update_persistent(table)) + return err; + return dict_stats_save(table); + } + return DB_SUCCESS; +} + #include "mysql_com.h" /** Save an individual index's statistic into the persistent statistics storage. @@ -2825,14 +2821,14 @@ dict_stats_save_index_stat( "END;", trx); if (UNIV_UNLIKELY(ret != DB_SUCCESS)) { - if (innodb_index_stats_not_found == false && - index->stats_error_printed == false) { + if (innodb_index_stats_not_found == false + && !index->table->stats_error_printed) { + index->table->stats_error_printed = true; ib::error() << "Cannot save index statistics for table " << index->table->name << ", index " << index->name << ", stat name \"" << stat_name << "\": " << ret; - index->stats_error_printed = true; } } @@ -2874,17 +2870,11 @@ dict_stats_report_error(dict_table_t* table, bool defragment) return err; } -/** Save the table's statistics into the persistent statistics storage. -@param[in] table table whose stats to save -@param[in] only_for_index if this is non-NULL, then stats for indexes -that are not equal to it will not be saved, if NULL, then all indexes' stats -are saved +/** Save the persistent statistics of a table or an index. +@param table table whose stats to save +@param only_for_index the index ID to save statistics for (0=all) @return DB_SUCCESS or error code */ -static -dberr_t -dict_stats_save( - dict_table_t* table, - const index_id_t* only_for_index) +dberr_t dict_stats_save(dict_table_t* table, index_id_t index_id) { pars_info_t* pinfo; char db_utf8[MAX_DB_UTF8_LEN]; @@ -2921,7 +2911,7 @@ dict_stats_save( || strcmp(table_stats->name.m_name, TABLE_STATS_NAME)) { release_and_exit: if (table_stats) { - dict_table_close(table_stats, false, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); } return DB_STATS_DO_NOT_EXIST; } @@ -2938,7 +2928,7 @@ release_and_exit: goto release_and_exit; } if (strcmp(index_stats->name.m_name, INDEX_STATS_NAME)) { - dict_table_close(index_stats, false, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); goto release_and_exit; } @@ -2998,8 +2988,14 @@ release_and_exit: "END;", trx); if (UNIV_UNLIKELY(ret != DB_SUCCESS)) { - ib::error() << "Cannot save table statistics for table " - << table->name << ": " << ret; + sql_print_error("InnoDB: Cannot save table statistics for" +#ifdef EMBEDDED_LIBRARY + " table %.*s.%s: %s", +#else + " table %`.*s.%`s: %s", +#endif + int(table->name.dblen()), table->name.m_name, + table->name.basename(), ut_strerr(ret)); rollback_and_exit: trx->rollback(); free_and_exit: @@ -3007,8 +3003,8 @@ free_and_exit: dict_sys.unlock(); unlocked_free_and_exit: trx->free(); - dict_table_close(table_stats, false, thd, mdl_table); - dict_table_close(index_stats, false, thd, mdl_index); + dict_table_close(table_stats, thd, mdl_table); + dict_table_close(index_stats, thd, mdl_index); return ret; } @@ -3042,7 +3038,7 @@ unlocked_free_and_exit: index = it->second; - if (only_for_index != NULL && index->id != *only_for_index) { + if (index_id != 0 && index->id != index_id) { continue; } @@ -3112,6 +3108,14 @@ unlocked_free_and_exit: goto free_and_exit; } +void dict_stats_empty_table_and_save(dict_table_t *table) +{ + dict_stats_empty_table(table, true); + if (table->stats_is_persistent() && + dict_stats_persistent_storage_check(false) == SCHEMA_OK) + dict_stats_save(table); +} + /*********************************************************************//** Called for the row that is selected by SELECT ... FROM mysql.innodb_table_stats WHERE table='...' @@ -3461,14 +3465,8 @@ dict_stats_fetch_index_stats_step( return(TRUE); } -/*********************************************************************//** -Read table's statistics from the persistent statistics storage. -@return DB_SUCCESS or error code */ -static -dberr_t -dict_stats_fetch_from_ps( -/*=====================*/ - dict_table_t* table) /*!< in/out: table */ +/** Read the stored persistent statistics of a table. */ +dberr_t dict_stats_fetch_from_ps(dict_table_t *table) { index_fetch_t index_fetch_arg; pars_info_t* pinfo; @@ -3491,7 +3489,7 @@ dict_stats_fetch_from_ps( dict_table_t* index_stats = dict_table_open_on_name( INDEX_STATS_NAME, false, DICT_ERR_IGNORE_NONE); if (!index_stats) { - dict_table_close(table_stats); + table_stats->release(); return DB_STATS_DO_NOT_EXIST; } @@ -3501,13 +3499,13 @@ dict_stats_fetch_from_ps( if (!table_stats || strcmp(table_stats->name.m_name, TABLE_STATS_NAME)) { release_and_exit: + dict_sys.unfreeze(); if (table_stats) { - dict_table_close(table_stats, true, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); } if (index_stats) { - dict_table_close(index_stats, true, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); } - dict_sys.unfreeze(); return DB_STATS_DO_NOT_EXIST; } @@ -3609,8 +3607,8 @@ release_and_exit: que_run_threads(que_fork_start_command(graph)); que_graph_free(graph); - dict_table_close(table_stats, false, thd, mdl_table); - dict_table_close(index_stats, false, thd, mdl_index); + dict_table_close(table_stats, thd, mdl_table); + dict_table_close(index_stats, thd, mdl_index); trx_commit_for_mysql(trx); dberr_t ret = trx->error_state; @@ -3630,252 +3628,46 @@ dict_stats_update_for_index( /*========================*/ dict_index_t* index) /*!< in/out: index */ { - DBUG_ENTER("dict_stats_update_for_index"); + dict_table_t *const table= index->table; + ut_ad(table->stat_initialized()); - ut_ad(index->table->stat_initialized()); - - if (index->table->stats_is_persistent()) { - - if (dict_stats_persistent_storage_check(false)) { - index_stats_t stats = dict_stats_analyze_index(index); - index->table->stats_mutex_lock(); - index->stat_index_size = stats.index_size; - index->stat_n_leaf_pages = stats.n_leaf_pages; - for (size_t i = 0; i < stats.stats.size(); ++i) { - index->stat_n_diff_key_vals[i] - = stats.stats[i].n_diff_key_vals; - index->stat_n_sample_sizes[i] - = stats.stats[i].n_sample_sizes; - index->stat_n_non_null_key_vals[i] - = stats.stats[i].n_non_null_key_vals; - } - index->table->stat_sum_of_other_index_sizes - += index->stat_index_size; - index->table->stats_mutex_unlock(); - - dict_stats_save(index->table, &index->id); - DBUG_VOID_RETURN; - } - /* else */ - - if (innodb_index_stats_not_found == false && - index->stats_error_printed == false) { - /* Fall back to transient stats since the persistent - storage is not present or is corrupted */ - - ib::info() << "Recalculation of persistent statistics" - " requested for table " << index->table->name - << " index " << index->name - << " but the required" - " persistent statistics storage is not present or is" - " corrupted. Using transient stats instead."; - index->stats_error_printed = false; - } - } - - dict_stats_update_transient_for_index(index); - - DBUG_VOID_RETURN; -} - -/*********************************************************************//** -Calculates new estimates for table and index statistics. The statistics -are used in query optimization. -@return DB_SUCCESS or error code -@retval DB_SUCCESS_LOCKED_REC if the table under bulk insert operation */ -dberr_t -dict_stats_update( -/*==============*/ - dict_table_t* table, /*!< in/out: table */ - dict_stats_upd_option_t stats_upd_option) - /*!< in: whether to (re) calc - the stats or to fetch them from - the persistent statistics - storage */ -{ - ut_ad(!table->stats_mutex_is_owner()); - - if (!table->is_readable()) { - return (dict_stats_report_error(table)); - } else if (srv_force_recovery >= SRV_FORCE_NO_UNDO_LOG_SCAN) { - /* If we have set a high innodb_force_recovery level, do - not calculate statistics, as a badly corrupted index can - cause a crash in it. */ - dict_stats_empty_table(table, false); - return(DB_SUCCESS); - } - - if (trx_id_t bulk_trx_id = table->bulk_trx_id) { - if (trx_sys.find(nullptr, bulk_trx_id, false)) { - dict_stats_empty_table(table, false); - return DB_SUCCESS_LOCKED_REC; - } - } - - switch (stats_upd_option) { - case DICT_STATS_RECALC_PERSISTENT: - - if (srv_read_only_mode) { - goto transient; - } - - /* Persistent recalculation requested, called from - 1) ANALYZE TABLE, or - 2) the auto recalculation background thread, or - 3) open table if stats do not exist on disk and auto recalc - is enabled */ - - /* InnoDB internal tables (e.g. SYS_TABLES) cannot have - persistent stats enabled */ - ut_a(strchr(table->name.m_name, '/') != NULL); - - /* check if the persistent statistics storage exists - before calling the potentially slow function - dict_stats_update_persistent(); that is a - prerequisite for dict_stats_save() succeeding */ - if (dict_stats_persistent_storage_check(false)) { - - dberr_t err; - - err = dict_stats_update_persistent(table); - - if (err != DB_SUCCESS) { - return(err); - } - - err = dict_stats_save(table, NULL); - - return(err); - } - - /* Fall back to transient stats since the persistent - storage is not present or is corrupted */ - - if (innodb_table_stats_not_found == false && - table->stats_error_printed == false) { - ib::warn() << "Recalculation of persistent statistics" - " requested for table " - << table->name - << " but the required persistent" - " statistics storage is not present or is corrupted." - " Using transient stats instead."; - table->stats_error_printed = true; - } - - goto transient; - - case DICT_STATS_RECALC_TRANSIENT: - - goto transient; - - case DICT_STATS_EMPTY_TABLE: - - dict_stats_empty_table(table, true); - - /* If table is using persistent stats, - then save the stats on disk */ - - if (table->stats_is_persistent()) { - - if (dict_stats_persistent_storage_check(false)) { - - return(dict_stats_save(table, NULL)); - } - - return(DB_STATS_DO_NOT_EXIST); - } - - return(DB_SUCCESS); - - case DICT_STATS_FETCH_ONLY_IF_NOT_IN_MEMORY: - - /* fetch requested, either fetch from persistent statistics - storage or use the old method */ - - if (table->stat_initialized()) { - return(DB_SUCCESS); - } - - /* InnoDB internal tables (e.g. SYS_TABLES) cannot have - persistent stats enabled */ - ut_a(strchr(table->name.m_name, '/') != NULL); - - if (!dict_stats_persistent_storage_check(false)) { - /* persistent statistics storage does not exist - or is corrupted, calculate the transient stats */ - - if (innodb_table_stats_not_found == false && - table->stats_error_printed == false && - !opt_bootstrap) { - ib::error() << "Fetch of persistent statistics" - " requested for table " - << table->name - << " but the required system tables " - << TABLE_STATS_NAME_PRINT - << " and " << INDEX_STATS_NAME_PRINT - << " are not present or have unexpected" - " structure. Using transient stats instead."; - table->stats_error_printed = true; - } - - goto transient; - } - - dberr_t err = dict_stats_fetch_from_ps(table); - - switch (err) { - case DB_SUCCESS: - return(DB_SUCCESS); - case DB_STATS_DO_NOT_EXIST: - - if (srv_read_only_mode) { - goto transient; - } -#ifdef WITH_WSREP - if (wsrep_thd_skip_locking(current_thd)) { - goto transient; - } + if (table->stats_is_persistent()) + switch (dict_stats_persistent_storage_check(false)) { + case SCHEMA_NOT_EXIST: + break; + case SCHEMA_INVALID: + if (table->stats_error_printed) + break; + table->stats_error_printed= true; + sql_print_information("InnoDB: Recalculation of persistent statistics" +#ifdef EMBEDDED_LIBRARY + " requested for table %.*s.%s index %s but" +#else + " requested for table %`.*s.%`s index %`s but" #endif - if (table->stats_is_auto_recalc()) { - return(dict_stats_update( - table, - DICT_STATS_RECALC_PERSISTENT)); - } + " the required persistent statistics storage" + " is corrupted. Using transient stats instead.", + int(table->name.dblen()), table->name.m_name, + table->name.basename(), index->name()); + break; + case SCHEMA_OK: + index_stats_t stats{dict_stats_analyze_index(index)}; + table->stats_mutex_lock(); + index->stat_index_size = stats.index_size; + index->stat_n_leaf_pages = stats.n_leaf_pages; + for (size_t i = 0; i < stats.stats.size(); ++i) + { + index->stat_n_diff_key_vals[i]= stats.stats[i].n_diff_key_vals; + index->stat_n_sample_sizes[i]= stats.stats[i].n_sample_sizes; + index->stat_n_non_null_key_vals[i]= stats.stats[i].n_non_null_key_vals; + } + table->stat_sum_of_other_index_sizes+= index->stat_index_size; + table->stats_mutex_unlock(); + dict_stats_save(table, index->id); + return; + } - ib::info() << "Trying to use table " << table->name - << " which has persistent statistics enabled," - " but auto recalculation turned off and the" - " statistics do not exist in " - TABLE_STATS_NAME_PRINT - " and " INDEX_STATS_NAME_PRINT - ". Please either run \"ANALYZE TABLE " - << table->name << ";\" manually or enable the" - " auto recalculation with \"ALTER TABLE " - << table->name << " STATS_AUTO_RECALC=1;\"." - " InnoDB will now use transient statistics for " - << table->name << "."; - - goto transient; - default: - - if (innodb_table_stats_not_found == false && - table->stats_error_printed == false) { - ib::error() << "Error fetching persistent statistics" - " for table " - << table->name - << " from " TABLE_STATS_NAME_PRINT " and " - INDEX_STATS_NAME_PRINT ": " << err - << ". Using transient stats method instead."; - } - - goto transient; - } - /* no "default:" in order to produce a compilation warning - about unhandled enumeration value */ - } - -transient: - return dict_stats_update_transient(table); + dict_stats_update_transient_for_index(index); } /** Execute DELETE FROM mysql.innodb_table_stats @@ -4025,7 +3817,7 @@ dberr_t dict_stats_rename_index(const char *db, const char *table, const char *old_name, const char *new_name, trx_t *trx) { - if (!dict_stats_persistent_storage_check(true)) + if (dict_stats_persistent_storage_check(true) != SCHEMA_OK) return DB_STATS_DO_NOT_EXIST; pars_info_t *pinfo= pars_info_create(); @@ -4161,7 +3953,7 @@ test_dict_stats_save() index2_stat_n_sample_sizes[2] = TEST_IDX2_N_DIFF3_SAMPLE_SIZE; index2_stat_n_sample_sizes[3] = TEST_IDX2_N_DIFF4_SAMPLE_SIZE; - ret = dict_stats_save(&table, NULL); + ret = dict_stats_save(&table); ut_a(ret == DB_SUCCESS); diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index e44e97e2b8e..c5dca24fcfc 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -201,7 +201,7 @@ void dict_stats_update_if_needed_func(dict_table_t *table) if (counter > threshold) { /* this will reset table->stat_modified_counter to 0 */ - dict_stats_update(table, DICT_STATS_RECALC_TRANSIENT); + dict_stats_update_transient(table); } } @@ -329,7 +329,7 @@ invalid_table_id: if (!mdl || !table->is_accessible()) { - dict_table_close(table, false, thd, mdl); + dict_table_close(table, thd, mdl); goto invalid_table_id; } @@ -343,10 +343,10 @@ invalid_table_id: difftime(time(nullptr), table->stats_last_recalc) >= MIN_RECALC_INTERVAL; const dberr_t err= update_now - ? dict_stats_update(table, DICT_STATS_RECALC_PERSISTENT) + ? dict_stats_update_persistent_try(table) : DB_SUCCESS_LOCKED_REC; - dict_table_close(table, false, thd, mdl); + dict_table_close(table, thd, mdl); mysql_mutex_lock(&recalc_pool_mutex); auto i= std::find_if(recalc_pool.begin(), recalc_pool.end(), diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 30889e5903c..38b45a94142 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -2809,7 +2809,7 @@ static void fts_optimize_sync_table(dict_table_t *table, std::this_thread::sleep_for(std::chrono::seconds(6));); if (mdl_ticket) - dict_table_close(sync_table, false, fts_opt_thd, mdl_ticket); + dict_table_close(sync_table, fts_opt_thd, mdl_ticket); } /**********************************************************************//** diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index faec1410236..427be74e9c3 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1464,9 +1464,9 @@ static void innodb_drop_database(handlerton*, char *path) trx->commit(); if (table_stats) - dict_table_close(table_stats, true, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); if (index_stats) - dict_table_close(index_stats, true, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); row_mysql_unlock_data_dictionary(trx); trx->free(); @@ -1620,7 +1620,7 @@ inline void ha_innobase::reload_statistics() if (dict_table_t *table= m_prebuilt ? m_prebuilt->table : nullptr) { if (table->is_readable()) - dict_stats_init(table); + statistics_init(table, true); else table->stat.fetch_or(dict_table_t::STATS_INITIALIZED); } @@ -1932,7 +1932,7 @@ static int innodb_check_version(handlerton *hton, const char *path, { const trx_id_t trx_id= table->def_trx_id; DBUG_ASSERT(trx_id <= create_id); - dict_table_close(table); + table->release(); DBUG_PRINT("info", ("create_id: %llu trx_id: %" PRIu64, create_id, trx_id)); DBUG_RETURN(create_id != trx_id); } @@ -3010,7 +3010,7 @@ static bool innodb_copy_stat_flags(dict_table_t *table, dict_table_t::STATS_AUTO_RECALC_OFF, ""); static_assert(true == dict_table_t::STATS_INITIALIZED, ""); stat|= (sar & (HA_STATS_AUTO_RECALC_ON | HA_STATS_AUTO_RECALC_OFF)) << 3 | - initialized; + uint32_t(initialized); table->stat= stat; return true; @@ -3288,7 +3288,7 @@ static bool innobase_query_caching_table_check( bool allow = innobase_query_caching_table_check_low(table, trx); - dict_table_close(table); + table->release(); if (allow) { /* If the isolation level is high, assign a read view for the @@ -5837,6 +5837,70 @@ static void initialize_auto_increment(dict_table_t *table, const Field& field, table->autoinc_mutex.wr_unlock(); } +dberr_t ha_innobase::statistics_init(dict_table_t *table, bool recalc) +{ + ut_ad(table->is_readable()); + ut_ad(!table->stats_mutex_is_owner()); + + uint32_t stat= table->stat; + dberr_t err= DB_SUCCESS; + + if (!recalc && dict_table_t::stat_initialized(stat)); + else if (srv_force_recovery >= SRV_FORCE_NO_UNDO_LOG_SCAN) + dict_stats_empty_table(table, false); + else + { + if (dict_table_t::stats_is_persistent(stat) && !srv_read_only_mode +#ifdef WITH_WSREP + && !wsrep_thd_skip_locking(m_user_thd) +#endif + ) + { + switch (dict_stats_persistent_storage_check(false)) { + case SCHEMA_OK: + if (recalc) + { + recalc: + err= dict_stats_update_persistent(table); + if (err == DB_SUCCESS) + err= dict_stats_save(table); + } + else + { + err= dict_stats_fetch_from_ps(table); + if (err == DB_STATS_DO_NOT_EXIST && table->stats_is_auto_recalc()) + goto recalc; + } + if (err == DB_SUCCESS) + return err; + if (!recalc) + break; + /* fall through */ + case SCHEMA_INVALID: + if (table->stats_error_printed) + break; + table->stats_error_printed = true; + if (opt_bootstrap) + break; + sql_print_warning("InnoDB: %s of persistent statistics requested" + " for table %`.*s.%`s" + " but the required persistent statistics storage" + " is corrupted.", + recalc ? "Recalculation" : "Fetch", + int(table->name.dblen()), table->name.m_name, + table->name.basename()); + /* fall through */ + case SCHEMA_NOT_EXIST: + err= DB_STATS_DO_NOT_EXIST; + } + } + + dict_stats_update_transient(table); + } + + return err; +} + /** Open an InnoDB table @param[in] name table name @return error code @@ -13338,7 +13402,7 @@ ha_innobase::create(const char *name, TABLE *form, HA_CREATE_INFO *create_info, if (!error) { - dict_stats_update(info.table(), DICT_STATS_EMPTY_TABLE); + dict_stats_empty_table_and_save(info.table()); if (!info.table()->is_temporary()) log_write_up_to(trx->commit_lsn, true); info.table()->release(); @@ -13462,23 +13526,17 @@ ha_innobase::discard_or_import_tablespace( err, m_prebuilt->table->flags, NULL)); } - if (m_prebuilt->table->stats_is_persistent()) { - dberr_t ret; + dict_table_t* t = m_prebuilt->table; - /* Adjust the persistent statistics. */ - ret = dict_stats_update(m_prebuilt->table, - DICT_STATS_RECALC_PERSISTENT); - - if (ret != DB_SUCCESS) { - push_warning_printf( - ha_thd(), - Sql_condition::WARN_LEVEL_WARN, - ER_ALTER_INFO, - "Error updating stats for table '%s'" - " after table rebuild: %s", - m_prebuilt->table->name.m_name, - ut_strerr(ret)); - } + if (dberr_t ret = dict_stats_update_persistent_try(t)) { + push_warning_printf( + ha_thd(), + Sql_condition::WARN_LEVEL_WARN, + ER_ALTER_INFO, + "Error updating stats after" + " ALTER TABLE %`.*s.%`s IMPORT TABLESPACE: %s", + int(t->name.dblen()), t->name.m_name, + t->name.basename(), ut_strerr(ret)); } DBUG_RETURN(0); @@ -13700,8 +13758,8 @@ int ha_innobase::delete_table(const char *name) ut_ad(err == DB_LOCK_WAIT); ut_ad(trx->error_state == DB_SUCCESS); err= DB_SUCCESS; - dict_table_close(table_stats, false, thd, mdl_table); - dict_table_close(index_stats, false, thd, mdl_index); + dict_table_close(table_stats, thd, mdl_table); + dict_table_close(index_stats, thd, mdl_index); table_stats= nullptr; index_stats= nullptr; } @@ -13775,9 +13833,9 @@ err_exit: purge_sys.resume_FTS(); #endif if (table_stats) - dict_table_close(table_stats, true, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); if (index_stats) - dict_table_close(index_stats, true, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); row_mysql_unlock_data_dictionary(trx); if (trx != parent_trx) trx->free(); @@ -13807,9 +13865,9 @@ err_exit: std::vector deleted; trx->commit(deleted); if (table_stats) - dict_table_close(table_stats, true, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); if (index_stats) - dict_table_close(index_stats, true, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); row_mysql_unlock_data_dictionary(trx); for (pfs_os_file_t d : deleted) os_file_close(d); @@ -14024,15 +14082,18 @@ int ha_innobase::truncate() error= fts_lock_tables(trx, *ib_table); } - /* Wait for purge threads to stop using the table. */ - for (uint n = 15; ib_table->get_ref_count() > 1; ) + if (error == DB_SUCCESS) { - if (!--n) + /* Wait for purge threads to stop using the table. */ + for (uint n = 15; ib_table->get_ref_count() > 1; ) { - error= DB_LOCK_WAIT_TIMEOUT; - break; + if (!--n) + { + error= DB_LOCK_WAIT_TIMEOUT; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(50)); } - std::this_thread::sleep_for(std::chrono::milliseconds(50)); } if (error == DB_SUCCESS && ib_table->stats_is_persistent() && @@ -14125,7 +14186,7 @@ int ha_innobase::truncate() if (!err) { - dict_stats_update(m_prebuilt->table, DICT_STATS_EMPTY_TABLE); + dict_stats_empty_table_and_save(m_prebuilt->table); log_write_up_to(trx->commit_lsn, true); row_prebuilt_t *prebuilt= m_prebuilt; uchar *upd_buf= m_upd_buf; @@ -14157,13 +14218,49 @@ int ha_innobase::truncate() mem_heap_free(heap); if (table_stats) - dict_table_close(table_stats, false, m_user_thd, mdl_table); + dict_table_close(table_stats, m_user_thd, mdl_table); if (index_stats) - dict_table_close(index_stats, false, m_user_thd, mdl_index); + dict_table_close(index_stats, m_user_thd, mdl_index); DBUG_RETURN(err); } +/** Deinitialize InnoDB persistent statistics, forcing them +to be reloaded on subsequent ha_innobase::open(). +@param t table for which the cached STATS_PERSISTENT are to be evicted */ +static void stats_deinit(dict_table_t *t) noexcept +{ + ut_ad(dict_sys.frozen()); + ut_ad(t->get_ref_count() == 0); + + if (t->is_temporary() || t->no_rollback()) + return; + + t->stats_mutex_lock(); + t->stat= t->stat & ~dict_table_t::STATS_INITIALIZED; + MEM_UNDEFINED(&t->stat_n_rows, sizeof t->stat_n_rows); + MEM_UNDEFINED(&t->stat_clustered_index_size, + sizeof t->stat_clustered_index_size); + MEM_UNDEFINED(&t->stat_sum_of_other_index_sizes, + sizeof t->stat_sum_of_other_index_sizes); + MEM_UNDEFINED(&t->stat_modified_counter, sizeof t->stat_modified_counter); +#ifdef HAVE_valgrind + for (dict_index_t *i= dict_table_get_first_index(t); i; + i= dict_table_get_next_index(i)) + { + MEM_UNDEFINED(i->stat_n_diff_key_vals, + i->n_uniq * sizeof *i->stat_n_diff_key_vals); + MEM_UNDEFINED(i->stat_n_sample_sizes, + i->n_uniq * sizeof *i->stat_n_sample_sizes); + MEM_UNDEFINED(i->stat_n_non_null_key_vals, + i->n_uniq * sizeof *i->stat_n_non_null_key_vals); + MEM_UNDEFINED(&i->stat_index_size, sizeof i->stat_index_size); + MEM_UNDEFINED(&i->stat_n_leaf_pages, sizeof i->stat_n_leaf_pages); + } +#endif /* HAVE_valgrind */ + t->stats_mutex_unlock(); +} + /*********************************************************************//** Renames an InnoDB table. @return 0 or error code */ @@ -14197,15 +14294,20 @@ ha_innobase::rename_table( dberr_t error = DB_SUCCESS; const bool from_temp = dict_table_t::is_temporary_name(norm_from); + dict_table_t* t; + if (from_temp) { /* There is no need to lock any FOREIGN KEY child tables. */ - } else if (dict_table_t *table = dict_table_open_on_name( - norm_from, false, DICT_ERR_IGNORE_FK_NOKEY)) { - error = lock_table_children(table, trx); - if (error == DB_SUCCESS) { - error = lock_table_for_trx(table, trx, LOCK_X); + t = nullptr; + } else { + t = dict_table_open_on_name( + norm_from, false, DICT_ERR_IGNORE_FK_NOKEY); + if (t) { + error = lock_table_children(t, trx); + if (error == DB_SUCCESS) { + error = lock_table_for_trx(t, trx, LOCK_X); + } } - table->release(); } if (strcmp(norm_from, TABLE_STATS_NAME) @@ -14246,10 +14348,8 @@ ha_innobase::rename_table( we cannot lock the tables, when the table is being renamed from from a temporary name. */ - dict_table_close(table_stats, false, thd, - mdl_table); - dict_table_close(index_stats, false, thd, - mdl_index); + dict_table_close(table_stats, thd, mdl_table); + dict_table_close(index_stats, thd, mdl_index); table_stats = nullptr; index_stats = nullptr; } @@ -14291,16 +14391,27 @@ ha_innobase::rename_table( if (error == DB_SUCCESS) { trx->flush_log_later = true; + if (t) { + ut_ad(dict_sys.frozen()); + if (UNIV_LIKELY(t->release())) { + stats_deinit(t); + } else { + ut_ad("unexpected references" == 0); + } + } innobase_commit_low(trx); } else { + if (t) { + t->release(); + } trx->rollback(); } if (table_stats) { - dict_table_close(table_stats, true, thd, mdl_table); + dict_table_close(table_stats, thd, mdl_table); } if (index_stats) { - dict_table_close(index_stats, true, thd, mdl_index); + dict_table_close(index_stats, thd, mdl_index); } row_mysql_unlock_data_dictionary(trx); if (error == DB_SUCCESS) { @@ -14314,10 +14425,10 @@ ha_innobase::rename_table( during DDL operations, because the duplicate key would exist in metadata tables, not in the user table. */ my_error(ER_TABLE_EXISTS_ERROR, MYF(0), to); - error = DB_ERROR; + DBUG_RETURN(HA_ERR_GENERIC); } else if (error == DB_LOCK_WAIT_TIMEOUT) { my_error(ER_LOCK_WAIT_TIMEOUT, MYF(0)); - error = DB_LOCK_WAIT; + DBUG_RETURN(HA_ERR_GENERIC); } DBUG_RETURN(convert_error_code_to_mysql(error, 0, NULL)); @@ -14796,51 +14907,69 @@ ha_innobase::info_low( ib_table = m_prebuilt->table; DBUG_ASSERT(ib_table->get_ref_count() > 0); - if (!ib_table->is_readable()) { + if (!ib_table->is_readable() + || srv_force_recovery >= SRV_FORCE_NO_UNDO_LOG_SCAN) { dict_stats_empty_table(ib_table, true); - } + } else if (flag & HA_STATUS_TIME) { + stats.update_time = ib_table->update_time; + if (!is_analyze && !innobase_stats_on_metadata) { + goto stats_fetch; + } - if (flag & HA_STATUS_TIME) { - if (is_analyze || innobase_stats_on_metadata) { + dberr_t ret; + m_prebuilt->trx->op_info = "updating table statistics"; - dict_stats_upd_option_t opt; - dberr_t ret; - - m_prebuilt->trx->op_info = "updating table statistics"; - - if (ib_table->stats_is_persistent()) { - if (is_analyze) { - if (!srv_read_only_mode) { - dict_stats_recalc_pool_del( - ib_table->id, false); - } - opt = DICT_STATS_RECALC_PERSISTENT; - } else { - /* This is e.g. 'SHOW INDEXES', fetch - the persistent stats from disk. */ - opt = DICT_STATS_FETCH_ONLY_IF_NOT_IN_MEMORY; - } + if (ib_table->stats_is_persistent() + && !srv_read_only_mode + && dict_stats_persistent_storage_check(false) + == SCHEMA_OK) { + if (is_analyze) { + dict_stats_recalc_pool_del(ib_table->id, + false); +recalc: + ret = statistics_init(ib_table, is_analyze); } else { - opt = DICT_STATS_RECALC_TRANSIENT; + /* This is e.g. 'SHOW INDEXES' */ + ret = statistics_init(ib_table, is_analyze); + switch (ret) { + case DB_SUCCESS: + break; + default: + goto error; + case DB_STATS_DO_NOT_EXIST: + if (!ib_table + ->stats_is_auto_recalc()) { + break; + } + + if (opt_bootstrap) { + break; + } +#ifdef WITH_WSREP + if (wsrep_thd_skip_locking( + m_user_thd)) { + break; + } +#endif + is_analyze = true; + goto recalc; + } } - - ret = dict_stats_update(ib_table, opt); - + } else { + ret = dict_stats_update_transient(ib_table); if (ret != DB_SUCCESS) { +error: m_prebuilt->trx->op_info = ""; DBUG_RETURN(HA_ERR_GENERIC); } - - m_prebuilt->trx->op_info = - "returning various info to MariaDB"; } - - stats.update_time = (ulong) ib_table->update_time; + m_prebuilt->trx->op_info = "returning various info to MariaDB"; + } else { +stats_fetch: + statistics_init(ib_table, false); } - dict_stats_init(ib_table); - if (flag & HA_STATUS_VARIABLE) { ulint stat_clustered_index_size; @@ -15696,7 +15825,7 @@ get_foreign_key_info( << foreign->foreign_table_name; } } else { - dict_table_close(ref_table, true); + ref_table->release(); } } @@ -17581,7 +17710,7 @@ static int innodb_ft_aux_table_validate(THD *thd, st_mysql_sys_var*, table_name, false, DICT_ERR_IGNORE_NONE)) { const table_id_t id = dict_table_has_fts_index(table) ? table->id : 0; - dict_table_close(table); + table->release(); if (id) { innodb_ft_aux_table_id = id; if (table_name == buf) { diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 534936f8fae..9cc79e59a50 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -101,6 +101,9 @@ public: int open(const char *name, int mode, uint test_if_locked) override; + /** Fetch or recalculate InnoDB table statistics */ + dberr_t statistics_init(dict_table_t *table, bool recalc); + handler* clone(const char *name, MEM_ROOT *mem_root) override; int close(void) override; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index b808c74b262..e44cd9f476f 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -7542,10 +7542,11 @@ error_handled: } } - /* n_ref_count must be 1, because background threads cannot + /* n_ref_count must be 1 (+ InnoDB_share), + because background threads cannot be executing on this very table as we are holding MDL_EXCLUSIVE. */ - ut_ad(ctx->online || user_table->get_ref_count() == 1); + ut_ad(ctx->online || ((user_table->get_ref_count() - 1) <= 1)); if (new_clustered) { online_retry_drop_indexes_low(user_table, ctx->trx); @@ -11180,7 +11181,10 @@ alter_stats_norebuild( DBUG_ENTER("alter_stats_norebuild"); DBUG_ASSERT(!ctx->need_rebuild()); - if (!ctx->new_table->stats_is_persistent()) { + auto stat = ctx->new_table->stat; + + if (!dict_table_t::stat_initialized(stat) + || !dict_table_t::stats_is_persistent(stat)) { DBUG_VOID_RETURN; } @@ -11189,7 +11193,6 @@ alter_stats_norebuild( DBUG_ASSERT(index->table == ctx->new_table); if (!(index->type & DICT_FTS)) { - dict_stats_init(ctx->new_table); dict_stats_update_for_index(index); } } @@ -11214,11 +11217,15 @@ alter_stats_rebuild( { DBUG_ENTER("alter_stats_rebuild"); - if (!table->space || !table->stats_is_persistent()) { + if (!table->space || !table->stats_is_persistent() + || dict_stats_persistent_storage_check(false) != SCHEMA_OK) { DBUG_VOID_RETURN; } - dberr_t ret = dict_stats_update(table, DICT_STATS_RECALC_PERSISTENT); + dberr_t ret = dict_stats_update_persistent(table); + if (ret == DB_SUCCESS) { + ret = dict_stats_save(table); + } if (ret != DB_SUCCESS) { push_warning_printf( @@ -11331,6 +11338,13 @@ ha_innobase::commit_inplace_alter_table( /* A rollback is being requested. So far we may at most have created stubs for ADD INDEX or a copy of the table for rebuild. */ +#if 0 /* FIXME: is there a better way for innodb.innodb-index-online? */ + lock_shared_ha_data(); + auto share = static_cast(get_ha_share_ptr()); + set_ha_share_ptr(nullptr); + unlock_shared_ha_data(); + delete share; +#endif DBUG_RETURN(rollback_inplace_alter_table( ha_alter_info, table, m_prebuilt)); } @@ -11600,12 +11614,10 @@ err_index: } if (error != DB_SUCCESS) { if (table_stats) { - dict_table_close(table_stats, false, m_user_thd, - mdl_table); + dict_table_close(table_stats, m_user_thd, mdl_table); } if (index_stats) { - dict_table_close(index_stats, false, m_user_thd, - mdl_index); + dict_table_close(index_stats, m_user_thd, mdl_index); } my_error_innodb(error, table_share->table_name.str, 0); if (fts_exist) { @@ -11641,11 +11653,11 @@ fail: trx->rollback(); ut_ad(!trx->fts_trx); if (table_stats) { - dict_table_close(table_stats, true, m_user_thd, + dict_table_close(table_stats, m_user_thd, mdl_table); } if (index_stats) { - dict_table_close(index_stats, true, m_user_thd, + dict_table_close(index_stats, m_user_thd, mdl_index); } row_mysql_unlock_data_dictionary(trx); @@ -11699,10 +11711,10 @@ fail: } if (table_stats) { - dict_table_close(table_stats, true, m_user_thd, mdl_table); + dict_table_close(table_stats, m_user_thd, mdl_table); } if (index_stats) { - dict_table_close(index_stats, true, m_user_thd, mdl_index); + dict_table_close(index_stats, m_user_thd, mdl_index); } /* Commit or roll back the changes to the data dictionary. */ diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 6f0987f153e..0554a229e5e 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -2230,7 +2230,7 @@ i_s_fts_deleted_generic_fill( DBUG_RETURN(0); } else if (!dict_table_has_fts_index(user_table) || !user_table->is_readable()) { - dict_table_close(user_table, false, thd, mdl_ticket); + dict_table_close(user_table, thd, mdl_ticket); DBUG_RETURN(0); } @@ -2245,7 +2245,7 @@ i_s_fts_deleted_generic_fill( fts_table_fetch_doc_ids(trx, &fts_table, deleted); - dict_table_close(user_table, false, thd, mdl_ticket); + dict_table_close(user_table, thd, mdl_ticket); trx->free(); @@ -2578,7 +2578,7 @@ i_s_fts_index_cache_fill( } if (!user_table->fts || !user_table->fts->cache) { - dict_table_close(user_table, false, thd, mdl_ticket); + dict_table_close(user_table, thd, mdl_ticket); DBUG_RETURN(0); } @@ -2603,7 +2603,7 @@ i_s_fts_index_cache_fill( } mysql_mutex_unlock(&cache->lock); - dict_table_close(user_table, false, thd, mdl_ticket); + dict_table_close(user_table, thd, mdl_ticket); DBUG_RETURN(ret); } @@ -3020,7 +3020,7 @@ i_s_fts_index_table_fill( } } - dict_table_close(user_table, false, thd, mdl_ticket); + dict_table_close(user_table, thd, mdl_ticket); ut_free(conv_str.f_str); @@ -3145,7 +3145,7 @@ i_s_fts_config_fill( } if (!dict_table_has_fts_index(user_table)) { - dict_table_close(user_table, false, thd, mdl_ticket); + dict_table_close(user_table, thd, mdl_ticket); DBUG_RETURN(0); } @@ -3202,7 +3202,7 @@ i_s_fts_config_fill( fts_sql_commit(trx); - dict_table_close(user_table, false, thd, mdl_ticket); + dict_table_close(user_table, thd, mdl_ticket); trx->free(); diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 842b9fec9cd..64f8d0a95da 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -146,21 +146,21 @@ dict_table_open_on_id(table_id_t table_id, bool dict_locked, MDL_ticket **mdl= nullptr) MY_ATTRIBUTE((warn_unused_result)); -/** Decrement the count of open handles */ -void dict_table_close(dict_table_t *table); +/** Release a metadata lock. +@param thd connection that holds mdl +@param mdl metadata lock, or nullptr */ +void mdl_release(THD *thd, MDL_ticket *mdl) noexcept; -/** Decrements the count of open handles of a table. -@param[in,out] table table -@param[in] dict_locked whether dict_sys.latch is being held -@param[in] thd thread to release MDL -@param[in] mdl metadata lock or NULL if the thread is a - foreground one. */ -void -dict_table_close( - dict_table_t* table, - bool dict_locked, - THD* thd = NULL, - MDL_ticket* mdl = NULL); +/** Release a table reference and a metadata lock. +@param table referenced table +@param thd connection that holds mdl +@param mdl metadata lock, or nullptr */ +inline void dict_table_close(dict_table_t* table, THD *thd, MDL_ticket *mdl) + noexcept +{ + table->release(); + mdl_release(thd, mdl); +} /*********************************************************************//** Gets the minimum number of bytes per character. diff --git a/storage/innobase/include/dict0dict.inl b/storage/innobase/include/dict0dict.inl index 4c0ff1497d8..ee440101ede 100644 --- a/storage/innobase/include/dict0dict.inl +++ b/storage/innobase/include/dict0dict.inl @@ -1076,8 +1076,8 @@ dict_table_is_file_per_table( /** Acquire the table handle. */ inline void dict_table_t::acquire() { - ut_ad(dict_sys.frozen()); - n_ref_count++; + ut_d(const auto old=) n_ref_count++; + ut_ad(old || dict_sys.frozen()); } /** Release the table handle. diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index bec735537fc..dfeefb2cb33 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1112,9 +1112,6 @@ struct dict_index_t { uint32_t stat_n_leaf_pages; /*!< approximate number of leaf pages in the index tree */ - bool stats_error_printed; - /*!< has persistent statistics error printed - for this index ? */ /* @} */ /** Statistics for defragmentation, these numbers are estimations and could be very inaccurate at certain times, e.g. right after restart, @@ -2393,7 +2390,7 @@ public: ib_uint64_t stat_modified_counter; bool stats_error_printed; - /*!< Has persistent stats error beein + /*!< Has persistent stats error been already printed for this table ? */ /* @} */ diff --git a/storage/innobase/include/dict0stats.h b/storage/innobase/include/dict0stats.h index e1d9893dd0a..91ec4953599 100644 --- a/storage/innobase/include/dict0stats.h +++ b/storage/innobase/include/dict0stats.h @@ -30,46 +30,6 @@ Created Jan 06, 2010 Vasil Dimov #include "dict0types.h" #include "trx0types.h" -enum dict_stats_upd_option_t { - DICT_STATS_RECALC_PERSISTENT,/* (re) calculate the - statistics using a precise and slow - algo and save them to the persistent - storage, if the persistent storage is - not present then emit a warning and - fall back to transient stats */ - DICT_STATS_RECALC_TRANSIENT,/* (re) calculate the statistics - using an imprecise quick algo - without saving the results - persistently */ - DICT_STATS_EMPTY_TABLE, /* Write all zeros (or 1 where it makes sense) - into a table and its indexes' statistics - members. The resulting stats correspond to an - empty table. If the table is using persistent - statistics, then they are saved on disk. */ - DICT_STATS_FETCH_ONLY_IF_NOT_IN_MEMORY /* fetch the stats - from the persistent storage if the in-memory - structures have not been initialized yet, - otherwise do nothing */ -}; - -/*********************************************************************//** -Initialize table's stats for the first time when opening a table. */ -UNIV_INLINE -void -dict_stats_init( -/*============*/ - dict_table_t* table); /*!< in/out: table */ - -/*********************************************************************//** -Deinitialize table's stats after the last close of the table. This is -used to detect "FLUSH TABLE" and refresh the stats upon next open. */ -UNIV_INLINE -void -dict_stats_deinit( -/*==============*/ - dict_table_t* table) /*!< in/out: table */ - MY_ATTRIBUTE((nonnull)); - #ifdef WITH_WSREP /** Update the table modification counter and if necessary, schedule new estimates for table and index statistics to be calculated. @@ -86,19 +46,6 @@ void dict_stats_update_if_needed_func(dict_table_t *table) # define dict_stats_update_if_needed(t,trx) dict_stats_update_if_needed_func(t) #endif -/*********************************************************************//** -Calculates new estimates for table and index statistics. The statistics -are used in query optimization. -@return DB_* error code or DB_SUCCESS */ -dberr_t -dict_stats_update( -/*==============*/ - dict_table_t* table, /*!< in/out: table */ - dict_stats_upd_option_t stats_upd_option); - /*!< in: whether to (re) calc - the stats or to fetch them from - the persistent storage */ - /** Execute DELETE FROM mysql.innodb_table_stats @param database_name database name @param table_name table name @@ -135,6 +82,50 @@ dict_stats_update_for_index( dict_index_t* index) /*!< in/out: index */ MY_ATTRIBUTE((nonnull)); +enum dict_stats_schema_check { + /** The InnoDB persistent statistics tables do not exist. */ + SCHEMA_NOT_EXIST= -1, + /** The schema of the InnoDB persistent statistics tables is valid. */ + SCHEMA_OK= 0, + /** The schema is invalid. */ + SCHEMA_INVALID +}; + +/** @return whether the persistent statistics storage is usable */ +dict_stats_schema_check +dict_stats_persistent_storage_check(bool dict_already_locked= false) noexcept; + +/** Save the persistent statistics of a table or an index. +@param table table whose stats to save +@param only_for_index the index ID to save statistics for (0=all) +@return DB_SUCCESS or error code */ +dberr_t dict_stats_save(dict_table_t* table, index_id_t index_id= 0); + +/** Read the stored persistent statistics of a table. */ +dberr_t dict_stats_fetch_from_ps(dict_table_t *table); + +/** +Calculate new estimates for table and index statistics. This function +is relatively quick and is used to calculate non-persistent statistics. +@param table table for which the non-persistent statistics are being updated +@return error code +@retval DB_SUCCESS_LOCKED REC if the table under bulk insert operation */ +dberr_t dict_stats_update_transient(dict_table_t *table) noexcept; + +/** +Calculate new estimates for table and index statistics. This function +is slower than dict_stats_update_transient(). +@param table table for which the persistent statistics are being updated +@return DB_SUCCESS or error code +@retval DB_SUCCESS_LOCKED_REC if the table under bulk insert operation */ +dberr_t dict_stats_update_persistent(dict_table_t *table) noexcept; + +/** +Try to calculate and save new estimates for persistent statistics. +If persistent statistics are not enabled for the table or not available, +this does nothing. */ +dberr_t dict_stats_update_persistent_try(dict_table_t *table); + /** Rename a table in InnoDB persistent stats storage. @param old_name old table name @param new_name new table name @@ -191,8 +182,6 @@ dberr_t dict_stats_report_error(dict_table_t* table, bool defragment = false) MY_ATTRIBUTE((nonnull, warn_unused_result)); -#include "dict0stats.inl" - #ifdef UNIV_ENABLE_UNIT_TEST_DICT_STATS void test_dict_stats_all(); #endif /* UNIV_ENABLE_UNIT_TEST_DICT_STATS */ @@ -206,4 +195,8 @@ void dict_stats_empty_table( dict_table_t* table, bool empty_defrag_stats); + +/** Clear the statistics for a table and save them if +persistent statistics are enabled. */ +void dict_stats_empty_table_and_save(dict_table_t *table); #endif /* dict0stats_h */ diff --git a/storage/innobase/include/dict0stats.inl b/storage/innobase/include/dict0stats.inl deleted file mode 100644 index fa5b4cccf19..00000000000 --- a/storage/innobase/include/dict0stats.inl +++ /dev/null @@ -1,108 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. -Copyright (c) 2017, 2021, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file include/dict0stats.ic -Code used for calculating and manipulating table statistics. - -Created Jan 23, 2012 Vasil Dimov -*******************************************************/ - -#include "dict0dict.h" - -/*********************************************************************//** -Initialize table's stats for the first time when opening a table. */ -UNIV_INLINE -void -dict_stats_init( -/*============*/ - dict_table_t* table) /*!< in/out: table */ -{ - ut_ad(!table->stats_mutex_is_owner()); - - uint32_t stat = table->stat; - - if (stat & dict_table_t::STATS_INITIALIZED) { - return; - } - - dict_stats_upd_option_t opt; - - if (table->stats_is_persistent(stat)) { - opt = DICT_STATS_FETCH_ONLY_IF_NOT_IN_MEMORY; - } else { - opt = DICT_STATS_RECALC_TRANSIENT; - } - - dict_stats_update(table, opt); -} - -/*********************************************************************//** -Deinitialize table's stats after the last close of the table. This is -used to detect "FLUSH TABLE" and refresh the stats upon next open. */ -UNIV_INLINE -void -dict_stats_deinit( -/*==============*/ - dict_table_t* table) /*!< in/out: table */ -{ - ut_ad(table->stats_mutex_is_owner()); - ut_ad(table->get_ref_count() == 0); - -#ifdef HAVE_valgrind - if (!table->stat_initialized()) { - return; - } - - MEM_UNDEFINED(&table->stat_n_rows, sizeof table->stat_n_rows); - MEM_UNDEFINED(&table->stat_clustered_index_size, - sizeof table->stat_clustered_index_size); - MEM_UNDEFINED(&table->stat_sum_of_other_index_sizes, - sizeof table->stat_sum_of_other_index_sizes); - MEM_UNDEFINED(&table->stat_modified_counter, - sizeof table->stat_modified_counter); - - dict_index_t* index; - - for (index = dict_table_get_first_index(table); - index != NULL; - index = dict_table_get_next_index(index)) { - MEM_UNDEFINED( - index->stat_n_diff_key_vals, - index->n_uniq - * sizeof index->stat_n_diff_key_vals[0]); - MEM_UNDEFINED( - index->stat_n_sample_sizes, - index->n_uniq - * sizeof index->stat_n_sample_sizes[0]); - MEM_UNDEFINED( - index->stat_n_non_null_key_vals, - index->n_uniq - * sizeof index->stat_n_non_null_key_vals[0]); - MEM_UNDEFINED( - &index->stat_index_size, - sizeof(index->stat_index_size)); - MEM_UNDEFINED( - &index->stat_n_leaf_pages, - sizeof(index->stat_n_leaf_pages)); - } -#endif /* HAVE_valgrind */ - - table->stat = table->stat & ~dict_table_t::STATS_INITIALIZED; -} diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index e6e36a6d8d2..e364c67589d 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1955,7 +1955,7 @@ row_ins_check_foreign_constraints( TRUE, foreign, table, ref_tuple, thr); if (ref_table) { - dict_table_close(ref_table); + ref_table->release(); } } } diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 8b20d70fb15..30b4f5975a1 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -981,7 +981,7 @@ void row_prebuilt_free(row_prebuilt_t *prebuilt) rtr_clean_rtr_info(prebuilt->rtr_info, true); } if (prebuilt->table) { - dict_table_close(prebuilt->table); + prebuilt->table->release(); } mem_heap_free(prebuilt->heap); diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc index b02cbec56ea..ba47481d852 100644 --- a/storage/innobase/row/row0uins.cc +++ b/storage/innobase/row/row0uins.cc @@ -244,8 +244,7 @@ func_exit: btr_pcur_commit_specify_mtr(&node->pcur, &mtr); if (UNIV_LIKELY_NULL(table)) { - dict_table_close(table, dict_locked, - node->trx->mysql_thd, mdl_ticket); + dict_table_close(table, node->trx->mysql_thd, mdl_ticket); } return(err); @@ -452,7 +451,7 @@ close_table: would probably be better to just drop all temporary tables (and temporary undo log records) of the current connection, instead of doing this rollback. */ - dict_table_close(node->table, dict_locked); + node->table->release(); node->table = NULL; return false; } else { @@ -644,8 +643,7 @@ row_undo_ins( break; } - dict_table_close(node->table, dict_locked); - + node->table->release(); node->table = NULL; return(err); diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index a1bc1df3dd2..fa228cba93a 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -1259,7 +1259,7 @@ close_table: would probably be better to just drop all temporary tables (and temporary undo log records) of the current connection, instead of doing this rollback. */ - dict_table_close(node->table, dict_locked); + node->table->release(); node->table = NULL; return false; } @@ -1418,8 +1418,7 @@ rollback_clust: } } - dict_table_close(node->table, dict_locked); - + node->table->release(); node->table = NULL; return(err); diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 03118fb25f2..2c40fdf32d4 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -253,7 +253,7 @@ row_upd_check_references_constraints( FALSE, foreign, table, entry, thr); if (ref_table) { - dict_table_close(ref_table); + ref_table->release(); } if (err != DB_SUCCESS) { @@ -338,7 +338,7 @@ wsrep_row_upd_check_foreign_constraints( TRUE, foreign, table, entry, thr); if (opened) { - dict_table_close(opened); + opened->release(); } if (err != DB_SUCCESS) { diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 8b0d8e2f138..6b96d388fea 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1060,7 +1060,7 @@ static void trx_purge_close_tables(purge_node_t *node, THD *thd) else if (t.second.first == reinterpret_cast(-1)); else { - dict_table_close(t.second.first, false, thd, t.second.second); + dict_table_close(t.second.first, thd, t.second.second); t.second.first= reinterpret_cast(-1); } } @@ -1198,7 +1198,7 @@ dict_table_t *purge_sys_t::close_and_reopen(table_id_t id, THD *thd, if (t.second.first == reinterpret_cast(-1)) { if (table) - dict_table_close(table, false, thd, *mdl); + dict_table_close(table, thd, *mdl); goto retry; } } From 2183f302c8f0965c7a70b70ed7baf356f48c64c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 5 Mar 2025 11:44:38 +0200 Subject: [PATCH 47/47] MDEV-33489 atomic.alter_table is too slow with SSL The test is killing and restarting the server very many times. This may lead to timeouts on architectures or builds that lack an SIMD based encryption implementation, such as IBM System Z (s390x) or cmake -DWITH_MSAN=ON builds. --- mysql-test/suite/atomic/alter_table.opt | 1 + mysql-test/suite/atomic/alter_table.test | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/suite/atomic/alter_table.opt b/mysql-test/suite/atomic/alter_table.opt index 61f3ce08d27..df19ac60a2a 100644 --- a/mysql-test/suite/atomic/alter_table.opt +++ b/mysql-test/suite/atomic/alter_table.opt @@ -1 +1,2 @@ --innodb-max-dirty-pages-pct=0 +--skip-ssl diff --git a/mysql-test/suite/atomic/alter_table.test b/mysql-test/suite/atomic/alter_table.test index aa265c92b7e..cd23fae9fd9 100644 --- a/mysql-test/suite/atomic/alter_table.test +++ b/mysql-test/suite/atomic/alter_table.test @@ -5,7 +5,6 @@ if (!$BIG_TEST) { --source include/not_valgrind.inc - --source include/not_msan.inc } #