From 899cedb33c8e92a0067f2d74015300454c6235ae Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Wed, 26 Oct 2022 19:52:17 -0400 Subject: [PATCH 01/22] Fix building my_gethwaddr() on OpenBSD --- mysys/my_gethwaddr.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mysys/my_gethwaddr.c b/mysys/my_gethwaddr.c index 70e1d549e15..5bb0ed75fb3 100644 --- a/mysys/my_gethwaddr.c +++ b/mysys/my_gethwaddr.c @@ -33,8 +33,14 @@ static my_bool memcpy_and_test(uchar *to, uchar *from, uint len) return res; } -#if defined(__APPLE__) || defined(__FreeBSD__) +#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) +#ifdef __OpenBSD__ +#include +#include +#include +#else #include +#endif #include #include #include From ce443c855478e931b77058f3a7108c07500220d7 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 26 Oct 2022 14:48:03 +0400 Subject: [PATCH 02/22] MDEV-29495 Generalize can_convert_xxx() hook engine API to cover any arbitrary data type --- sql/field.h | 22 ------------------- sql/ha_partition.cc | 31 ++++----------------------- sql/ha_partition.h | 12 ++--------- sql/handler.h | 19 ++-------------- sql/sql_table.cc | 2 +- storage/innobase/handler/ha_innodb.cc | 22 ++++++++++++++++++- storage/innobase/handler/ha_innodb.h | 22 +++++++++++-------- 7 files changed, 43 insertions(+), 87 deletions(-) diff --git a/sql/field.h b/sql/field.h index 9d40caf0932..40ff99a5b2e 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1552,12 +1552,6 @@ public: Used by the ALTER TABLE */ virtual bool is_equal(const Column_definition &new_field) const= 0; - // Used as double dispatch pattern: calls virtual method of handler - virtual bool - can_be_converted_by_engine(const Column_definition &new_type) const - { - return false; - } /* convert decimal to longlong with overflow check */ longlong convert_decimal2longlong(const my_decimal *val, bool unsigned_flag, int *err); @@ -3621,10 +3615,6 @@ public: void sql_type(String &str) const; void sql_rpl_type(String*) const; bool is_equal(const Column_definition &new_field) const; - bool can_be_converted_by_engine(const Column_definition &new_type) const - { - return table->file->can_convert_string(this, new_type); - } virtual uchar *pack(uchar *to, const uchar *from, uint max_length); virtual const uchar *unpack(uchar* to, const uchar *from, @@ -3751,10 +3741,6 @@ public: uchar *new_ptr, uint32 length, uchar *new_null_ptr, uint new_null_bit); bool is_equal(const Column_definition &new_field) const; - bool can_be_converted_by_engine(const Column_definition &new_type) const - { - return table->file->can_convert_varstring(this, new_type); - } void hash(ulong *nr, ulong *nr2); uint length_size() const { return length_bytes; } void print_key_value(String *out, uint32 length); @@ -4128,10 +4114,6 @@ public: uint32 char_length() const; uint32 character_octet_length() const; bool is_equal(const Column_definition &new_field) const; - bool can_be_converted_by_engine(const Column_definition &new_type) const - { - return table->file->can_convert_blob(this, new_type); - } void print_key_value(String *out, uint32 length); friend void TABLE::remember_blob_values(String *blob_storage); @@ -4247,10 +4229,6 @@ public: !table->copy_blobs; } bool is_equal(const Column_definition &new_field) const; - bool can_be_converted_by_engine(const Column_definition &new_type) const - { - return table->file->can_convert_geom(this, new_type); - } int store(const char *to, size_t length, CHARSET_INFO *charset); int store(double nr); diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 18a8cd5e027..06ae329ee3a 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2005, 2019, Oracle and/or its affiliates. - Copyright (c) 2009, 2021, MariaDB + Copyright (c) 2009, 2022, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -12021,35 +12021,12 @@ void ha_partition::clear_top_table_fields() } bool -ha_partition::can_convert_string(const Field_string* field, - const Column_definition& new_type) const +ha_partition::can_convert_nocopy(const Field &field, + const Column_definition &new_type) const { for (uint index= 0; index < m_tot_parts; index++) { - if (!m_file[index]->can_convert_string(field, new_type)) - return false; - } - return true; -} - -bool -ha_partition::can_convert_varstring(const Field_varstring* field, - const Column_definition& new_type) const{ - for (uint index= 0; index < m_tot_parts; index++) - { - if (!m_file[index]->can_convert_varstring(field, new_type)) - return false; - } - return true; -} - -bool -ha_partition::can_convert_blob(const Field_blob* field, - const Column_definition& new_type) const -{ - for (uint index= 0; index < m_tot_parts; index++) - { - if (!m_file[index]->can_convert_blob(field, new_type)) + if (!m_file[index]->can_convert_nocopy(field, new_type)) return false; } return true; diff --git a/sql/ha_partition.h b/sql/ha_partition.h index 507db9efddf..2670784c759 100644 --- a/sql/ha_partition.h +++ b/sql/ha_partition.h @@ -1624,16 +1624,8 @@ public: friend int cmp_key_rowid_part_id(void *ptr, uchar *ref1, uchar *ref2); friend int cmp_key_part_id(void *key_p, uchar *ref1, uchar *ref2); - bool can_convert_string( - const Field_string* field, - const Column_definition& new_field) const override; - bool can_convert_varstring( - const Field_varstring* field, - const Column_definition& new_field) const override; - - bool can_convert_blob( - const Field_blob* field, - const Column_definition& new_field) const override; + bool can_convert_nocopy(const Field &field, + const Column_definition &new_field) const override; }; #endif /* HA_PARTITION_INCLUDED */ diff --git a/sql/handler.h b/sql/handler.h index aa68c30480e..541c408178b 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -4851,23 +4851,8 @@ public: These functions check for such possibility. Implementation could be based on Field_xxx::is_equal() */ - virtual bool can_convert_string(const Field_string *field, - const Column_definition &new_type) const - { - return false; - } - virtual bool can_convert_varstring(const Field_varstring *field, - const Column_definition &new_type) const - { - return false; - } - virtual bool can_convert_blob(const Field_blob *field, - const Column_definition &new_type) const - { - return false; - } - virtual bool can_convert_geom(const Field_geom *field, - const Column_definition &new_type) const + virtual bool can_convert_nocopy(const Field &, + const Column_definition &) const { return false; } diff --git a/sql/sql_table.cc b/sql/sql_table.cc index efa5c06dd2a..63289e3ba59 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -6948,7 +6948,7 @@ static bool fill_alter_inplace_info(THD *thd, TABLE *table, bool varchar, bool is_equal= field->is_equal(*new_field); if (!is_equal) { - if (field->can_be_converted_by_engine(*new_field)) + if (field->table->file->can_convert_nocopy(*field, *new_field)) { /* New column type differs from the old one, but storage engine can diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3fa2a0c4d30..7acbe79d732 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -21159,6 +21159,26 @@ bool ha_innobase::can_convert_blob(const Field_blob *field, return true; } + +bool ha_innobase::can_convert_nocopy(const Field &field, + const Column_definition &new_type) const +{ + if (const Field_string *tf= dynamic_cast(&field)) + return can_convert_string(tf, new_type); + + if (const Field_varstring *tf= dynamic_cast(&field)) + return can_convert_varstring(tf, new_type); + + if (dynamic_cast(&field)) + return false; + + if (const Field_blob *tf= dynamic_cast(&field)) + return can_convert_blob(tf, new_type); + + return false; +} + + Compare_keys ha_innobase::compare_key_parts( const Field &old_field, const Column_definition &new_field, const KEY_PART_INFO &old_part, const KEY_PART_INFO &new_part) const @@ -21169,7 +21189,7 @@ Compare_keys ha_innobase::compare_key_parts( if (!is_equal) { - if (!old_field.can_be_converted_by_engine(new_field)) + if (!old_field.table->file->can_convert_nocopy(old_field, new_field)) return Compare_keys::NotEqual; if (!Charset(old_cs).eq_collation_specific_names(new_cs)) diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 97f62f4b398..b5f9438921c 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -419,15 +419,9 @@ public: @retval false if pushed (always) */ bool rowid_filter_push(Rowid_filter *rowid_filter) override; - bool - can_convert_string(const Field_string* field, - const Column_definition& new_field) const override; - bool can_convert_varstring( - const Field_varstring* field, - const Column_definition& new_field) const override; - bool - can_convert_blob(const Field_blob* field, - const Column_definition& new_field) const override; + bool can_convert_nocopy(const Field &field, + const Column_definition& new_field) const + override; /** @return whether innodb_strict_mode is active */ static bool is_innodb_strict_mode(THD* thd); @@ -442,6 +436,16 @@ public: const KEY_PART_INFO& new_part) const override; protected: + bool + can_convert_string(const Field_string* field, + const Column_definition& new_field) const; + bool can_convert_varstring( + const Field_varstring* field, + const Column_definition& new_field) const; + bool + can_convert_blob(const Field_blob* field, + const Column_definition& new_field) const; + dberr_t innobase_get_autoinc(ulonglong* value); dberr_t innobase_lock_autoinc(); ulonglong innobase_peek_autoinc(); From 4ebc8d8c27446f884feb3d4771b0685e333f2892 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 28 Oct 2022 19:59:35 +1100 Subject: [PATCH 03/22] MDEV-29847: Wrong warning on rlimit capping of max_open_files (#2315) Per the code my_set_max_open_files 3 lines earlier, we attempt to set the nofile (number of open files), rlimit to max_open_files. We should use this in the warning because wanted_files may not be the number. --- sql/mysqld.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5d58d42faf9..84c3b9fb0a6 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4580,8 +4580,8 @@ static int init_common_variables() files= my_set_max_open_files(max_open_files); SYSVAR_AUTOSIZE_IF_CHANGED(open_files_limit, files, ulong); - if (files < wanted_files && global_system_variables.log_warnings) - sql_print_warning("Could not increase number of max_open_files to more than %u (request: %u)", files, wanted_files); + if (files < max_open_files && global_system_variables.log_warnings) + sql_print_warning("Could not increase number of max_open_files to more than %u (request: %u)", files, max_open_files); /* If we required too much tc_instances than we reduce */ SYSVAR_AUTOSIZE_IF_CHANGED(tc_instances, From 7d96cb4703693cbf7a23308cfffee955022c86a9 Mon Sep 17 00:00:00 2001 From: Brad Smith Date: Sun, 30 Oct 2022 17:25:32 -0400 Subject: [PATCH 04/22] Fix warning with signal typedef for *BSD /usr/ports/pobj/mariadb-10.9.3/mariadb-10.9.3/mysys/my_lock.c:183:7: warning: incompatible function pointer types assigning to 'sig_return' (aka 'void (*)(void)') from 'void (*)(int)' [-Wincompatible-function-pointer-types] ALARM_INIT; ^~~~~~~~~~ /usr/ports/pobj/mariadb-10.9.3/mariadb-10.9.3/include/my_alarm.h:43:16: note: expanded from macro 'ALARM_INIT' alarm_signal=signal(SIGALRM,my_set_alarm_variable); ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ /usr/ports/pobj/mariadb-10.9.3/mariadb-10.9.3/mysys/my_lock.c:189:7: warning: incompatible function pointer types passing 'sig_return' (aka 'void (*)(void)') to parameter of type 'void (*)(int)' [-Wincompatible-function-pointer-types] ALARM_END; ^~~~~~~~~ /usr/ports/pobj/mariadb-10.9.3/mariadb-10.9.3/include/my_alarm.h:44:41: note: expanded from macro 'ALARM_END' ^~~~~~~~~~~~ /usr/include/sys/signal.h:199:27: note: passing argument to parameter here void (*signal(int, void (*)(int)))(int); ^ 2 warnings generated. The prototype is the same for all of the *BSD's. void (*signal(int sigcatch, void (*func)(int sigraised)))(int); --- include/my_alarm.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/my_alarm.h b/include/my_alarm.h index bc0004476ca..287d226fbac 100644 --- a/include/my_alarm.h +++ b/include/my_alarm.h @@ -31,7 +31,9 @@ extern ulong my_time_to_wait_for_lock; #include #ifdef HAVE_SIGHANDLER_T #define sig_return sighandler_t -#elif defined(SOLARIS) || defined(__sun) || defined(__APPLE__) +#elif defined(SOLARIS) || defined(__sun) || defined(__APPLE__) || \ + defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ + defined(__DragonFly__) typedef void (*sig_return)(int); /* Returns type from signal */ #else typedef void (*sig_return)(void); /* Returns type from signal */ From 0d927a57d2bfb32384dd024b9b4d1009fa22555a Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Wed, 19 Oct 2022 13:26:19 +0400 Subject: [PATCH 05/22] MDEV-29624 MDEV-29655 Fix ASAN errors on pushdown of derived table Deallocation of TABLE_LIST::dt_handler and TABLE_LIST::pushdown_derived was performed in multiple places if code. This not only made the code more difficult to maintain but also led to memory leaks and ASAN heap-use-after-free errors. This commit puts deallocation of TABLE_LIST::dt_handler and TABLE_LIST::pushdown_derived to the single point - JOIN::cleanup() --- .../federatedx_create_handlers.result | 45 ++++++++++++++ .../federated/federatedx_create_handlers.test | 59 ++++++++++++++++++- sql/derived_handler.cc | 5 -- sql/sql_derived.cc | 5 -- sql/sql_select.cc | 34 +++++++++-- sql/sql_select.h | 3 +- 6 files changed, 132 insertions(+), 19 deletions(-) diff --git a/mysql-test/suite/federated/federatedx_create_handlers.result b/mysql-test/suite/federated/federatedx_create_handlers.result index 29ce2c4348b..612b89fa04e 100644 --- a/mysql-test/suite/federated/federatedx_create_handlers.result +++ b/mysql-test/suite/federated/federatedx_create_handlers.result @@ -469,6 +469,51 @@ a 1 2 3 +# +# MDEV-29655: ASAN heap-use-after-free in +# Pushdown_derived::Pushdown_derived +# +connection slave; +DROP TABLE IF EXISTS federated.t1; +CREATE TABLE federated.t1 ( +id int(20) NOT NULL, +name varchar(16) NOT NULL default '' +) +DEFAULT CHARSET=latin1; +INSERT INTO federated.t1 VALUES +(3,'xxx'), (7,'yyy'), (4,'xxx'), (1,'zzz'), (5,'yyy'); +connection master; +DROP TABLE IF EXISTS federated.t1; +CREATE TABLE federated.t1 ( +id int(20) NOT NULL, +name varchar(16) NOT NULL default '' +) +ENGINE="FEDERATED" DEFAULT CHARSET=latin1 +CONNECTION='mysql://root@127.0.0.1:SLAVE_PORT/federated/t1'; +use federated; +SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM t1 where id=3) dt3 +WHERE id=2) dt2) dt; +id name +connection slave; +CREATE TABLE federated.t10 (a INT,b INT); +CREATE TABLE federated.t11 (a INT, b INT); +INSERT INTO federated.t10 VALUES (1,1),(2,2); +INSERT INTO federated.t11 VALUES (1,1),(2,2); +connection master; +CREATE TABLE federated.t10 +ENGINE="FEDERATED" DEFAULT CHARSET=latin1 +CONNECTION='mysql://root@127.0.0.1:SLAVE_PORT/federated/t10'; +CREATE TABLE federated.t11 +ENGINE="FEDERATED" DEFAULT CHARSET=latin1 +CONNECTION='mysql://root@127.0.0.1:SLAVE_PORT/federated/t11'; +use federated; +SELECT * FROM t10 LEFT JOIN +(t11, (SELECT * FROM (SELECT * FROM (SELECT * FROM t1 where id=3) dt3 +WHERE id=2) dt2) dt +) ON t10.a=t11.a; +a b a b id name +1 1 NULL NULL NULL NULL +2 2 NULL NULL NULL NULL set global federated_pushdown=0; connection master; DROP TABLE IF EXISTS federated.t1; diff --git a/mysql-test/suite/federated/federatedx_create_handlers.test b/mysql-test/suite/federated/federatedx_create_handlers.test index 2d6c2bc4197..f827c141f3d 100644 --- a/mysql-test/suite/federated/federatedx_create_handlers.test +++ b/mysql-test/suite/federated/federatedx_create_handlers.test @@ -267,7 +267,6 @@ INSERT INTO federated.t2 SELECT * FROM (SELECT * FROM federated.t1 LIMIT 70000) dt; SELECT COUNT(DISTINCT a) FROM federated.t2; - --echo # --echo # MDEV-29640 FederatedX does not properly handle pushdown --echo # in case of difference in local and remote table names @@ -314,6 +313,64 @@ CREATE TABLE federated.t3 (a INT) EXPLAIN SELECT * FROM federated.t3; SELECT * FROM federated.t3; +--echo # +--echo # MDEV-29655: ASAN heap-use-after-free in +--echo # Pushdown_derived::Pushdown_derived +--echo # + +connection slave; +DROP TABLE IF EXISTS federated.t1; + +CREATE TABLE federated.t1 ( + id int(20) NOT NULL, + name varchar(16) NOT NULL default '' +) +DEFAULT CHARSET=latin1; + +INSERT INTO federated.t1 VALUES + (3,'xxx'), (7,'yyy'), (4,'xxx'), (1,'zzz'), (5,'yyy'); + +connection master; +DROP TABLE IF EXISTS federated.t1; + +--replace_result $SLAVE_MYPORT SLAVE_PORT +eval +CREATE TABLE federated.t1 ( + id int(20) NOT NULL, + name varchar(16) NOT NULL default '' +) +ENGINE="FEDERATED" DEFAULT CHARSET=latin1 +CONNECTION='mysql://root@127.0.0.1:$SLAVE_MYPORT/federated/t1'; + +use federated; +SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM t1 where id=3) dt3 + WHERE id=2) dt2) dt; + +connection slave; +CREATE TABLE federated.t10 (a INT,b INT); +CREATE TABLE federated.t11 (a INT, b INT); +INSERT INTO federated.t10 VALUES (1,1),(2,2); +INSERT INTO federated.t11 VALUES (1,1),(2,2); + +connection master; +--replace_result $SLAVE_MYPORT SLAVE_PORT +eval +CREATE TABLE federated.t10 +ENGINE="FEDERATED" DEFAULT CHARSET=latin1 +CONNECTION='mysql://root@127.0.0.1:$SLAVE_MYPORT/federated/t10'; + +--replace_result $SLAVE_MYPORT SLAVE_PORT +eval +CREATE TABLE federated.t11 +ENGINE="FEDERATED" DEFAULT CHARSET=latin1 +CONNECTION='mysql://root@127.0.0.1:$SLAVE_MYPORT/federated/t11'; + +use federated; +SELECT * FROM t10 LEFT JOIN + (t11, (SELECT * FROM (SELECT * FROM (SELECT * FROM t1 where id=3) dt3 + WHERE id=2) dt2) dt + ) ON t10.a=t11.a; + set global federated_pushdown=0; source include/federated_cleanup.inc; diff --git a/sql/derived_handler.cc b/sql/derived_handler.cc index f48b95cbf76..70cc04bf9c7 100644 --- a/sql/derived_handler.cc +++ b/sql/derived_handler.cc @@ -44,11 +44,6 @@ Pushdown_derived::Pushdown_derived(TABLE_LIST *tbl, derived_handler *h) } -Pushdown_derived::~Pushdown_derived() -{ - delete handler; -} - int Pushdown_derived::execute() { diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc index a4e0fd6b683..4dcc8a61985 100644 --- a/sql/sql_derived.cc +++ b/sql/sql_derived.cc @@ -1000,11 +1000,7 @@ bool mysql_derived_optimize(THD *thd, LEX *lex, TABLE_LIST *derived) /* Create an object for execution of the query specifying the table */ if (!(derived->pushdown_derived= new (thd->mem_root) Pushdown_derived(derived, derived->dt_handler))) - { - delete derived->dt_handler; - derived->dt_handler= NULL; DBUG_RETURN(TRUE); - } } lex->current_select= first_select; @@ -1229,7 +1225,6 @@ bool mysql_derived_fill(THD *thd, LEX *lex, TABLE_LIST *derived) /* Execute the query that specifies the derived table by a foreign engine */ res= derived->pushdown_derived->execute(); unit->executed= true; - delete derived->pushdown_derived; DBUG_RETURN(res); } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 5ec88e5259c..c7a0bfe93ac 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -68,6 +68,7 @@ #include "select_handler.h" #include "my_json_writer.h" #include "opt_trace.h" +#include "derived_handler.h" /* A key part number that means we're using a fulltext scan. @@ -14086,6 +14087,7 @@ void JOIN::cleanup(bool full) } } } + free_pushdown_handlers(*join_list); } /* Restore ref array to original state */ if (current_ref_ptrs != items0) @@ -14096,6 +14098,32 @@ void JOIN::cleanup(bool full) DBUG_VOID_RETURN; } +/** + Clean up all derived pushdown handlers in this join. + + @detail + Note that dt_handler is picked at the prepare stage (as opposed + to optimization stage where one could expect this). + Because of that, we have to do cleanups in this function that is called + from JOIN::cleanup() and not in JOIN_TAB::cleanup. + */ +void JOIN::free_pushdown_handlers(List& join_list) +{ + List_iterator li(join_list); + TABLE_LIST *table_ref; + while ((table_ref= li++)) + { + if (table_ref->nested_join) + free_pushdown_handlers(table_ref->nested_join->join_list); + if (table_ref->pushdown_derived) + { + delete table_ref->pushdown_derived; + table_ref->pushdown_derived= NULL; + } + delete table_ref->dt_handler; + table_ref->dt_handler= NULL; + } +} /** Remove the following expressions from ORDER BY and GROUP BY: @@ -27400,12 +27428,6 @@ bool mysql_explain_union(THD *thd, SELECT_LEX_UNIT *unit, select_result *result) result, unit, first); } - if (unit->derived && unit->derived->pushdown_derived) - { - delete unit->derived->pushdown_derived; - unit->derived->pushdown_derived= NULL; - } - DBUG_RETURN(res || thd->is_error()); } diff --git a/sql/sql_select.h b/sql/sql_select.h index ecfa4c6fd11..b3945c499f8 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -1840,6 +1840,7 @@ private: bool add_having_as_table_cond(JOIN_TAB *tab); bool make_aggr_tables_info(); bool add_fields_for_current_rowid(JOIN_TAB *cur, List *fields); + void free_pushdown_handlers(List& join_list); }; enum enum_with_bush_roots { WITH_BUSH_ROOTS, WITHOUT_BUSH_ROOTS}; @@ -2507,8 +2508,6 @@ public: Pushdown_derived(TABLE_LIST *tbl, derived_handler *h); - ~Pushdown_derived(); - int execute(); }; From e7be2d3142699aa73e8f50e01657a3ec2570c7ff Mon Sep 17 00:00:00 2001 From: Ian Gilfillan Date: Wed, 2 Nov 2022 16:24:36 +0200 Subject: [PATCH 06/22] Fix duplicate entry in mysqld_safe man page --- man/mysqld_safe.1 | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/man/mysqld_safe.1 b/man/mysqld_safe.1 index dbb50e11ac4..2b902ac123e 100644 --- a/man/mysqld_safe.1 +++ b/man/mysqld_safe.1 @@ -340,7 +340,9 @@ program to set the server\'s scheduling priority to the given value\&. .\} .\" mysqld_safe: no-auto-restart option .\" no-auto-restart option: mysqld_safe -\fB\-\-no\-auto\-restart\fR +\fB\-\-no\-auto\-restart\fR, +\fB\-\-nowatch\fR, +\fB\-\-no\-watch\fR .sp Exit after starting mysqld\&. .RE @@ -368,21 +370,6 @@ Do not read any option files\&. This must be the first option on the command lin .sp -1 .IP \(bu 2.3 .\} -.\" mysqld_safe: no-watch option -.\" no-watch option: mysqld_safe -\fB\-\-no\-auto\-restart\fR -.sp -Exit after starting mysqld\&. -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -.sp -1 -.IP \(bu 2.3 -.\} .\" mysqld_safe: numa-interleave option .\" numa-interleave option: mysqld_safe \fB\-\-numa\-interleave\fR From 92be8d20480ee0e2fb8ec72c005648f2573558ce Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 5 Nov 2022 18:36:43 +0100 Subject: [PATCH 07/22] MDEV-29951 server hang in crash handler When trying to output stacktrace, and addr2line is not installed, the child process forked by start_addr2line_fork() will fail to do exec(), and finish with exit(1). There is a problem with exit() though - it runs exit handlers, and for the forked copy of crashing process, it is a bad idea. In 10.5+ code for example, exit handlers include tpool::task_group static destructors, and it will hang infinitely waiting for completion of the outstanding tasks. The fix is to use _exit() instead, which skips the execution of exit handlers --- mysys/my_addr_resolve.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysys/my_addr_resolve.c b/mysys/my_addr_resolve.c index f0c0d214171..ac1bdae187c 100644 --- a/mysys/my_addr_resolve.c +++ b/mysys/my_addr_resolve.c @@ -202,7 +202,7 @@ int start_addr2line_fork(const char *binary_path) close(out[0]); close(out[1]); execlp("addr2line", "addr2line", "-C", "-f", "-e", binary_path, NULL); - exit(1); + _exit(1); } close(in[0]); From 10132ad26119205e1b53e5d3389bedb71d444201 Mon Sep 17 00:00:00 2001 From: Sachin Date: Mon, 14 Sep 2020 15:27:02 +0100 Subject: [PATCH 08/22] MDEV-23264 Unique blobs allow duplicate values upon UPDATE Problem:- We are able to insert duplicate value in table because cmp_binary_offset is not able to differentiate between NULL and empty string. So check_duplicate_long_entry_key is never called and we don't check for duplicate. Solution Added a if condition with is_null() on field which can differentiate between NULL and empty string. --- mysql-test/main/long_unique_bugs.result | 9 +++++++++ mysql-test/main/long_unique_bugs.test | 11 +++++++++++ sql/handler.cc | 9 +++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/long_unique_bugs.result b/mysql-test/main/long_unique_bugs.result index 1d896059289..f359913b5f4 100644 --- a/mysql-test/main/long_unique_bugs.result +++ b/mysql-test/main/long_unique_bugs.result @@ -314,6 +314,15 @@ ERROR 23000: Duplicate entry '1' for key 'v2' update t1,t2 set v1 = v2 , v5 = 0; ERROR 23000: Duplicate entry '-128' for key 'v1' drop table t1, t2; +CREATE TABLE t1 (f TEXT UNIQUE); +INSERT INTO t1 VALUES (NULL),(NULL); +UPDATE t1 SET f = ''; +ERROR 23000: Duplicate entry '' for key 'f' +SELECT * FROM t1; +f + +NULL +DROP TABLE t1; # # MDEV-21540 Initialization of already inited long unique index on reorganize partition # diff --git a/mysql-test/main/long_unique_bugs.test b/mysql-test/main/long_unique_bugs.test index f62ba2ac61a..9bef617ca90 100644 --- a/mysql-test/main/long_unique_bugs.test +++ b/mysql-test/main/long_unique_bugs.test @@ -396,6 +396,17 @@ update t1 set v2 = 1, v3 = -128; update t1,t2 set v1 = v2 , v5 = 0; drop table t1, t2; +# +# MDEV-23264 Unique blobs allow duplicate values upon UPDATE +# + +CREATE TABLE t1 (f TEXT UNIQUE); +INSERT INTO t1 VALUES (NULL),(NULL); +--error ER_DUP_ENTRY +UPDATE t1 SET f = ''; +SELECT * FROM t1; +DROP TABLE t1; + --echo # --echo # MDEV-21540 Initialization of already inited long unique index on reorganize partition --echo # diff --git a/sql/handler.cc b/sql/handler.cc index bf819733b81..5e9c9dc3a71 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -6750,8 +6750,13 @@ static int check_duplicate_long_entries_update(TABLE *table, handler *h, uchar * for (uint j= 0; j < key_parts; j++, keypart++) { field= keypart->field; - /* Compare fields if they are different then check for duplicates*/ - if(field->cmp_binary_offset(reclength)) + /* + Compare fields if they are different then check for duplicates + cmp_binary_offset cannot differentiate between null and empty string + So also check for that too + */ + if((field->is_null(0) != field->is_null(reclength)) || + field->cmp_binary_offset(reclength)) { if((error= check_duplicate_long_entry_key(table, table->update_handler, new_rec, i))) From f7e6198c0221112fdeb42668f89d29c828636156 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 7 Nov 2022 17:36:08 +0530 Subject: [PATCH 09/22] MDEV-27121 mariabackup incompatible with disabled dedicated undo log tablespaces - mariabackup fails to assign srv_undo_space_id_start when the dedicated undo tablespaces are disabled --- extra/mariabackup/xtrabackup.cc | 4 ---- mysql-test/suite/mariabackup/full_backup.opt | 1 + .../suite/mariabackup/full_backup.result | 14 +++++++++++ mysql-test/suite/mariabackup/full_backup.test | 24 +++++++++++++++++++ 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 mysql-test/suite/mariabackup/full_backup.opt diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 2b276856303..f7910fc9c23 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -3578,10 +3578,6 @@ static dberr_t xb_assign_undo_space_start() ulint space; int n_retries = 5; - if (srv_undo_tablespaces == 0) { - return error; - } - file = os_file_create(0, srv_sys_space.first_datafile()->filepath(), OS_FILE_OPEN, OS_FILE_NORMAL, OS_DATA_FILE, true, &ret); diff --git a/mysql-test/suite/mariabackup/full_backup.opt b/mysql-test/suite/mariabackup/full_backup.opt new file mode 100644 index 00000000000..7928cd812d0 --- /dev/null +++ b/mysql-test/suite/mariabackup/full_backup.opt @@ -0,0 +1 @@ +--innodb_undo_tablespaces=2 diff --git a/mysql-test/suite/mariabackup/full_backup.result b/mysql-test/suite/mariabackup/full_backup.result index 954151bbad7..10f4dc44ed4 100644 --- a/mysql-test/suite/mariabackup/full_backup.result +++ b/mysql-test/suite/mariabackup/full_backup.result @@ -12,3 +12,17 @@ SELECT * FROM t; i 1 DROP TABLE t; +# +# MDEV-27121 mariabackup incompatible with disabled dedicated +# undo log tablespaces +# +call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); +# xtrabackup backup +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +# Display undo log files from target directory +undo001 +undo002 diff --git a/mysql-test/suite/mariabackup/full_backup.test b/mysql-test/suite/mariabackup/full_backup.test index 66bed34cf3d..a0243527438 100644 --- a/mysql-test/suite/mariabackup/full_backup.test +++ b/mysql-test/suite/mariabackup/full_backup.test @@ -29,3 +29,27 @@ SELECT * FROM t; DROP TABLE t; rmdir $targetdir; +--echo # +--echo # MDEV-27121 mariabackup incompatible with disabled dedicated +--echo # undo log tablespaces +--echo # +call mtr.add_suppression("InnoDB: innodb_undo_tablespaces=0 disables dedicated undo log tablespaces"); + +let $restart_parameters=--innodb_undo_tablespaces=0; +--source include/restart_mysqld.inc + +echo # xtrabackup backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log + +--echo # Display undo log files from target directory +list_files $targetdir undo*; + +rmdir $targetdir; From baa6b052a2ad012aaedb1db0e1e549de3c9e0c29 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Mon, 7 Nov 2022 08:08:55 -0500 Subject: [PATCH 10/22] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 92848892dc8..876e7c96e80 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=3 -MYSQL_VERSION_PATCH=37 +MYSQL_VERSION_PATCH=38 SERVER_MATURITY=stable From 625fff0d8f97e813676d9cfd8f4b4b326762037a Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Mon, 7 Nov 2022 08:35:15 -0500 Subject: [PATCH 11/22] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ec02fb54be8..156ceae1801 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=4 -MYSQL_VERSION_PATCH=27 +MYSQL_VERSION_PATCH=28 SERVER_MATURITY=stable From eabb3b35d5ab76ceacd9006bb388e310f7016315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 08:53:49 +0200 Subject: [PATCH 12/22] MDEV-27121 fixup: mariabackup.mdev-14447 fault injection --- storage/innobase/buf/buf0buf.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 01111157920..fe2c0c6b880 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1141,7 +1141,7 @@ buf_page_is_corrupted( DBUG_EXECUTE_IF( "page_intermittent_checksum_mismatch", { static int page_counter; - if (page_counter++ == 2) return true; + if (page_counter++ == 3) return true; }); if ((checksum_field1 != crc32 From 456d4a508c510f4cb3798145944a47c8f3954465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 08:54:07 +0200 Subject: [PATCH 13/22] Remove an unused file The file plugin_exports became unused in commit fec844aca88e1c6b9c36bb0b811e92d9d023ffb9 --- storage/innobase/plugin_exports | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 storage/innobase/plugin_exports diff --git a/storage/innobase/plugin_exports b/storage/innobase/plugin_exports deleted file mode 100644 index 235ae3d5e72..00000000000 --- a/storage/innobase/plugin_exports +++ /dev/null @@ -1,14 +0,0 @@ -{ - global: - _maria_plugin_interface_version_; - _maria_sizeof_struct_st_plugin_; - _maria_plugin_declarations_; - my_snprintf_service; - thd_alloc_service; - thd_autoinc_service; - thd_error_context_service; - thd_kill_statement_service; - thd_wait_service; - local: - *; -}; From 314ed9f5eccc527654b0a63064561749a7b842f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 09:00:58 +0200 Subject: [PATCH 14/22] Work around MDEV-24813 in some tests Not creating explicit record locks will speed up the test. Also, disable the use of InnoDB persistent statistics in the test of MDEV-27270 to avoid intermittent failures in 10.6 or later (after commit 9608773f75e2ca21491ef6825c3616cdc96d1ca5) due to the nondeterministic scheduling of STATS_AUTO_PERSISTENT. --- mysql-test/main/order_by_innodb.result | 30 ++++++++++------------ mysql-test/main/order_by_innodb.test | 35 +++++++++++--------------- mysql-test/main/range_innodb.result | 15 ++++------- mysql-test/main/range_innodb.test | 22 +++++----------- 4 files changed, 38 insertions(+), 64 deletions(-) diff --git a/mysql-test/main/order_by_innodb.result b/mysql-test/main/order_by_innodb.result index 28922ef65f2..17d39eb12e6 100644 --- a/mysql-test/main/order_by_innodb.result +++ b/mysql-test/main/order_by_innodb.result @@ -1,8 +1,8 @@ -drop table if exists t0,t1,t2,t3; # # MDEV-6434: Wrong result (extra rows) with ORDER BY, multiple-column index, InnoDB # -CREATE TABLE t1 (a INT, b INT, c INT, d TEXT, KEY idx(a,b,c)) ENGINE=InnoDB; +CREATE TABLE t1 (a INT, b INT, c INT, d TEXT, KEY idx(a,b,c)) ENGINE=InnoDB +STATS_PERSISTENT=0; INSERT INTO t1 (a,c) VALUES (8, 9),(8, 10),(13, 15),(16, 17),(16, 18),(16, 19),(20, 21), (20, 22),(20, 24),(20, 25),(20, 26),(20, 27),(20, 28); @@ -14,8 +14,6 @@ DROP TABLE t1; # # MDEV-9457: Poor query plan chosen for ORDER BY query by a recent 10.1 # -create table t0 (a int); -insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t1 ( pk int primary key, key1 int, @@ -23,15 +21,9 @@ key2 int, col1 char(255), key(key1), key(key2) -) engine=innodb; -set @a=-1; +) engine=innodb stats_persistent=0; insert into t1 -select -@a:=@a+1, -@a, -@a, -repeat('abcd', 63) -from t0 A, t0 B, t0 C, t0 D; +select seq,seq,seq,repeat('abcd', 63) from seq_0_to_9999; # The following must NOT use 'index' on PK. # It should use index_merge(key1,key2) + filesort explain @@ -47,7 +39,7 @@ from t1 where key1<3 or key2<3; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index_merge key1,key2 key1,key2 5,5 NULL # Using sort_union(key1,key2); Using where -drop table t0, t1; +drop table t1; # # MDEV-18094: Query with order by limit picking index scan over filesort # @@ -78,9 +70,12 @@ drop table t1,t0; # MDEV-14071: wrong results with orderby_uses_equalities=on # (duplicate of MDEV-13994) # -CREATE TABLE t1 (i int, j int, z int,PRIMARY KEY (i,j), KEY (z)) ENGINE=InnoDB; -CREATE TABLE t2 (i int, j int, PRIMARY KEY (i,j)) ENGINE=InnoDB; -CREATE TABLE t3 (j int, n varchar(5), PRIMARY KEY (j)) ENGINE=InnoDB; +CREATE TABLE t1 (i int, j int, z int,PRIMARY KEY (i,j), KEY (z)) ENGINE=InnoDB +STATS_PERSISTENT=0; +CREATE TABLE t2 (i int, j int, PRIMARY KEY (i,j)) ENGINE=InnoDB +STATS_PERSISTENT=0; +CREATE TABLE t3 (j int, n varchar(5), PRIMARY KEY (j)) ENGINE=InnoDB +STATS_PERSISTENT=0; INSERT INTO t1 VALUES (127,0,1),(188,0,1),(206,0,1),(218,0,1),(292,0,1),(338,0,1),(375,0,1), (381,0,1),(409,0,1),(466,0,1),(469,0,1),(498,0,1),(656,0,1); @@ -150,7 +145,8 @@ DROP TABLE t1,t2,t3; # # MDEV-25858: Query results are incorrect when indexes are added # -CREATE TABLE t1 (id int NOT NULL PRIMARY KEY) engine=innodb; +CREATE TABLE t1 (id int NOT NULL PRIMARY KEY) engine=innodb +STATS_PERSISTENT=0; insert into t1 values (1),(2),(3); CREATE TABLE t2 ( id int NOT NULL PRIMARY KEY, diff --git a/mysql-test/main/order_by_innodb.test b/mysql-test/main/order_by_innodb.test index af12644c073..29b796f67bc 100644 --- a/mysql-test/main/order_by_innodb.test +++ b/mysql-test/main/order_by_innodb.test @@ -2,16 +2,14 @@ # ORDER BY handling (e.g. filesort) tests that require innodb # -- source include/have_innodb.inc - ---disable_warnings -drop table if exists t0,t1,t2,t3; ---enable_warnings +-- source include/have_sequence.inc --echo # --echo # MDEV-6434: Wrong result (extra rows) with ORDER BY, multiple-column index, InnoDB --echo # -CREATE TABLE t1 (a INT, b INT, c INT, d TEXT, KEY idx(a,b,c)) ENGINE=InnoDB; +CREATE TABLE t1 (a INT, b INT, c INT, d TEXT, KEY idx(a,b,c)) ENGINE=InnoDB +STATS_PERSISTENT=0; INSERT INTO t1 (a,c) VALUES (8, 9),(8, 10),(13, 15),(16, 17),(16, 18),(16, 19),(20, 21), @@ -24,9 +22,6 @@ DROP TABLE t1; --echo # --echo # MDEV-9457: Poor query plan chosen for ORDER BY query by a recent 10.1 --echo # -create table t0 (a int); -insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); - create table t1 ( pk int primary key, key1 int, @@ -34,16 +29,10 @@ create table t1 ( col1 char(255), key(key1), key(key2) -) engine=innodb; +) engine=innodb stats_persistent=0; -set @a=-1; insert into t1 -select - @a:=@a+1, - @a, - @a, - repeat('abcd', 63) -from t0 A, t0 B, t0 C, t0 D; +select seq,seq,seq,repeat('abcd', 63) from seq_0_to_9999; --echo # The following must NOT use 'index' on PK. --echo # It should use index_merge(key1,key2) + filesort @@ -60,7 +49,7 @@ select * from t1 where key1<3 or key2<3; -drop table t0, t1; +drop table t1; --echo # --echo # MDEV-18094: Query with order by limit picking index scan over filesort @@ -93,9 +82,12 @@ drop table t1,t0; --echo # (duplicate of MDEV-13994) --echo # -CREATE TABLE t1 (i int, j int, z int,PRIMARY KEY (i,j), KEY (z)) ENGINE=InnoDB; -CREATE TABLE t2 (i int, j int, PRIMARY KEY (i,j)) ENGINE=InnoDB; -CREATE TABLE t3 (j int, n varchar(5), PRIMARY KEY (j)) ENGINE=InnoDB; +CREATE TABLE t1 (i int, j int, z int,PRIMARY KEY (i,j), KEY (z)) ENGINE=InnoDB +STATS_PERSISTENT=0; +CREATE TABLE t2 (i int, j int, PRIMARY KEY (i,j)) ENGINE=InnoDB +STATS_PERSISTENT=0; +CREATE TABLE t3 (j int, n varchar(5), PRIMARY KEY (j)) ENGINE=InnoDB +STATS_PERSISTENT=0; INSERT INTO t1 VALUES (127,0,1),(188,0,1),(206,0,1),(218,0,1),(292,0,1),(338,0,1),(375,0,1), @@ -139,7 +131,8 @@ DROP TABLE t1,t2,t3; --echo # MDEV-25858: Query results are incorrect when indexes are added --echo # -CREATE TABLE t1 (id int NOT NULL PRIMARY KEY) engine=innodb; +CREATE TABLE t1 (id int NOT NULL PRIMARY KEY) engine=innodb +STATS_PERSISTENT=0; insert into t1 values (1),(2),(3); CREATE TABLE t2 ( diff --git a/mysql-test/main/range_innodb.result b/mysql-test/main/range_innodb.result index df111bc05be..cbe763bcf6e 100644 --- a/mysql-test/main/range_innodb.result +++ b/mysql-test/main/range_innodb.result @@ -1,15 +1,11 @@ # # Range optimizer (and related) tests that need InnoDB. # -drop table if exists t0, t1, t2; # # MDEV-6735: Range checked for each record used with key # create table t0(a int); insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); -create table t1(a int); -insert into t1 select A.a + B.a* 10 + C.a * 100 + D.a * 1000 -from t0 A, t0 B, t0 C, t0 D; create table t2 ( a int, b int, @@ -22,12 +18,12 @@ key(b) ) engine=innodb; insert into t2 select -a,a, +seq,seq, repeat('0123456789', 10), repeat('0123456789', 10), repeat('0123456789', 10), repeat('0123456789', 10) -from t1; +from seq_0_to_9999; analyze table t2; Table Op Msg_type Msg_text test.t2 analyze status OK @@ -36,7 +32,7 @@ explain select * from t0 left join t2 on t2.a Date: Tue, 8 Nov 2022 10:30:03 +0200 Subject: [PATCH 15/22] MDEV-22512: Disable frequently failing tests InnoDB crash recovery can run out of memory before commit 50324ce62448f284522ee1e3be24d8f5ba3bf904 in MariaDB Server 10.5. Let us disable some frequently failing recovery tests in earlier versions. --- mysql-test/suite/innodb_gis/disabled.def | 1 + mysql-test/suite/innodb_zip/disabled.def | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/innodb_gis/disabled.def b/mysql-test/suite/innodb_gis/disabled.def index 2d4d3686dd1..ba492f4f033 100644 --- a/mysql-test/suite/innodb_gis/disabled.def +++ b/mysql-test/suite/innodb_gis/disabled.def @@ -13,3 +13,4 @@ rtree_concurrent_srch : MDEV-15284 COUNT(*) mismatch rtree_recovery : MDEV-15284 COUNT(*) mismatch rtree_compress2 : MDEV-16269 CHECK TABLE reports wrong count +types : MDEV-22512 recovery runs out of memory before 10.5 diff --git a/mysql-test/suite/innodb_zip/disabled.def b/mysql-test/suite/innodb_zip/disabled.def index 8b137891791..fcf7333f843 100644 --- a/mysql-test/suite/innodb_zip/disabled.def +++ b/mysql-test/suite/innodb_zip/disabled.def @@ -1 +1 @@ - +recover : MDEV-22512 recovery runs out of memory before 10.5 From 9ac8be4e2980aa995117147e39ae5b7ad79fc980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 10:39:29 +0200 Subject: [PATCH 16/22] Include some advice in the crash-upgrade message --- storage/innobase/log/log0recv.cc | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index d8b7bd2ce85..e7effd9224f 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -58,6 +58,7 @@ Created 9/20/1997 Heikki Tuuri #include "trx0roll.h" #include "row0merge.h" #include "fil0pagecompress.h" +#include "log.h" /** Log records are stored in the hash table in chunks at most of this size; this must be less than srv_page_size as it is stored in the buffer pool */ @@ -1137,7 +1138,7 @@ static dberr_t recv_log_format_0_recover(lsn_t lsn, bool crypt) byte* buf = log_sys.buf; static const char* NO_UPGRADE_RECOVERY_MSG = - "Upgrade after a crash is not supported." + "InnoDB: Upgrade after a crash is not supported." " This redo log was created before MariaDB 10.2.2"; fil_io(IORequestLogRead, true, @@ -1150,21 +1151,24 @@ static dberr_t recv_log_format_0_recover(lsn_t lsn, bool crypt) if (log_block_calc_checksum_format_0(buf) != log_block_get_checksum(buf) && !log_crypt_101_read_block(buf)) { - ib::error() << NO_UPGRADE_RECOVERY_MSG - << ", and it appears corrupted."; - return(DB_CORRUPTION); + sql_print_error("%s, and it appears corrupted.", + NO_UPGRADE_RECOVERY_MSG); + return DB_CORRUPTION; } if (log_block_get_data_len(buf) == (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) { } else if (crypt) { - ib::error() << "Cannot decrypt log for upgrading." - " The encrypted log was created" - " before MariaDB 10.2.2."; + sql_print_error("InnoDB: Cannot decrypt log for upgrading." + " The encrypted log was created" + " before MariaDB 10.2.2."); return DB_ERROR; } else { - ib::error() << NO_UPGRADE_RECOVERY_MSG << "."; - return(DB_ERROR); + sql_print_error("%s. You must start up and shut down" + " MariaDB 10.1 or MySQL 5.6 or earlier" + " on the data directory.", + NO_UPGRADE_RECOVERY_MSG); + return DB_ERROR; } /* Mark the redo log for upgrading. */ From 1e8189fceed795a16d31f484f844b7a4a048b920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 10:55:22 +0200 Subject: [PATCH 17/22] MDEV-13564 follow-up: Correct a bogus comment This fixes up commit e3c39c0be82e69acb813e7c5e0f2aea7d5546e31 --- storage/innobase/ibuf/ibuf0ibuf.cc | 2 +- storage/innobase/include/ibuf0ibuf.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index dbb19bec7a0..38f6a50f7bb 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -4661,7 +4661,7 @@ reset_bit: } /** Delete all change buffer entries for a tablespace, -in DISCARD TABLESPACE, IMPORT TABLESPACE, or crash recovery. +in DISCARD TABLESPACE, IMPORT TABLESPACE, or read-ahead. @param[in] space missing or to-be-discarded tablespace */ void ibuf_delete_for_discarded_space(ulint space) { diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index 73f7054c9fb..79f9c7b1c07 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2019, MariaDB Corporation. +Copyright (c) 2016, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -338,7 +338,7 @@ void ibuf_merge_or_delete_for_page(buf_block_t *block, const page_id_t page_id, ulint zip_size); /** Delete all change buffer entries for a tablespace, -in DISCARD TABLESPACE, IMPORT TABLESPACE, or crash recovery. +in DISCARD TABLESPACE, IMPORT TABLESPACE, or read-ahead. @param[in] space missing or to-be-discarded tablespace */ void ibuf_delete_for_discarded_space(ulint space); From 49a0ad695b83f98993e068d89c02bdcc15724a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 11:24:49 +0200 Subject: [PATCH 18/22] MDEV-23371: Crash in _db_doprnt_ via que_thr_step() Something appears to be broken in the DBUG subsystem. Let us remove frequent calls to it from the InnoDB internal SQL interpreter that is used in the purge of transaction history. The DBUG_PRINT in que_eval_sql() can remain for now, because those operations are much less frequent. --- storage/innobase/que/que0que.cc | 76 +-------------------------------- 1 file changed, 2 insertions(+), 74 deletions(-) diff --git a/storage/innobase/que/que0que.cc b/storage/innobase/que/que0que.cc index 67da45a64e4..91e44adbdee 100644 --- a/storage/innobase/que/que0que.cc +++ b/storage/innobase/que/que0que.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2021, MariaDB Corporation. +Copyright (c) 2017, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -402,17 +402,10 @@ que_graph_free_recursive( ind_node_t* cre_ind; purge_node_t* purge; - DBUG_ENTER("que_graph_free_recursive"); - if (node == NULL) { - - DBUG_VOID_RETURN; + return; } - DBUG_PRINT("que_graph_free_recursive", - ("node: %p, type: " ULINTPF, node, - que_node_get_type(node))); - switch (que_node_get_type(node)) { case QUE_NODE_FORK: @@ -558,8 +551,6 @@ que_graph_free_recursive( default: ut_error; } - - DBUG_VOID_RETURN; } /**********************************************************************//** @@ -892,65 +883,6 @@ que_node_get_containing_loop_node( return(node); } -#ifndef DBUG_OFF -/** Gets information of an SQL query graph node. -@return type description */ -static MY_ATTRIBUTE((warn_unused_result, nonnull)) -const char* -que_node_type_string( -/*=================*/ - const que_node_t* node) /*!< in: query graph node */ -{ - switch (que_node_get_type(node)) { - case QUE_NODE_SELECT: - return("SELECT"); - case QUE_NODE_INSERT: - return("INSERT"); - case QUE_NODE_UPDATE: - return("UPDATE"); - case QUE_NODE_WHILE: - return("WHILE"); - case QUE_NODE_ASSIGNMENT: - return("ASSIGNMENT"); - case QUE_NODE_IF: - return("IF"); - case QUE_NODE_FETCH: - return("FETCH"); - case QUE_NODE_OPEN: - return("OPEN"); - case QUE_NODE_PROC: - return("STORED PROCEDURE"); - case QUE_NODE_FUNC: - return("FUNCTION"); - case QUE_NODE_LOCK: - return("LOCK"); - case QUE_NODE_THR: - return("QUERY THREAD"); - case QUE_NODE_COMMIT: - return("COMMIT"); - case QUE_NODE_UNDO: - return("UNDO ROW"); - case QUE_NODE_PURGE: - return("PURGE ROW"); - case QUE_NODE_ROLLBACK: - return("ROLLBACK"); - case QUE_NODE_CREATE_TABLE: - return("CREATE TABLE"); - case QUE_NODE_CREATE_INDEX: - return("CREATE INDEX"); - case QUE_NODE_FOR: - return("FOR LOOP"); - case QUE_NODE_RETURN: - return("RETURN"); - case QUE_NODE_EXIT: - return("EXIT"); - default: - ut_ad(0); - return("UNKNOWN NODE TYPE"); - } -} -#endif /* !DBUG_OFF */ - /**********************************************************************//** Performs an execution step on a query thread. @return query thread to run next: it may differ from the input @@ -978,10 +910,6 @@ que_thr_step( old_thr = thr; - DBUG_PRINT("ib_que", ("Execute %u (%s) at %p", - unsigned(type), que_node_type_string(node), - (const void*) node)); - if (type & QUE_NODE_CONTROL_STAT) { if ((thr->prev_node != que_node_get_parent(node)) && que_node_get_next(thr->prev_node)) { From b737d09dbc6ab588f1b1a61bb98e33ed8000acd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 11:37:43 +0200 Subject: [PATCH 19/22] MDEV-29905 Change buffer operations fail to check for log file overflow Every operation that is going to write redo log is supposed to invoke log_free_check() before acquiring any latches. If there is a risk of log buffer overrun, a log checkpoint would be triggered by that call. ibuf_merge_space(), ibuf_merge_in_background(), ibuf_delete_for_discarded_space(): Invoke log_free_check() when the current thread is not holding any page latches. Unfortunately, in lower-level code called from ibuf_insert() or ibuf_merge_or_delete_for_page(), some page latches may be held and a call to log_free_check() could hang. ibuf_set_bitmap_for_bulk_load(): Use the caller's mini-transaction. The caller should have invoked log_free_check() while not holding any page latches. --- storage/innobase/btr/btr0bulk.cc | 5 +-- storage/innobase/ibuf/ibuf0ibuf.cc | 46 +++++++++------------------- storage/innobase/include/ibuf0ibuf.h | 23 ++++++-------- 3 files changed, 28 insertions(+), 46 deletions(-) diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc index 0b53438feb7..cf14ffc5376 100644 --- a/storage/innobase/btr/btr0bulk.cc +++ b/storage/innobase/btr/btr0bulk.cc @@ -398,8 +398,9 @@ PageBulk::finish() void PageBulk::commit(bool success) { finish(); - if (success && !dict_index_is_clust(m_index) && page_is_leaf(m_page)) - ibuf_set_bitmap_for_bulk_load(m_block, innobase_fill_factor == 100); + if (success && !m_index->is_clust() && page_is_leaf(m_page)) + ibuf_set_bitmap_for_bulk_load(m_block, &m_mtr, + innobase_fill_factor == 100); m_mtr.commit(); } diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 65abacf3fd5..ce6705ea369 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -1067,12 +1067,12 @@ bitmap page) bitmap page if the page is not one of the fixed address ibuf pages, or NULL, in which case a new transaction is created. @return TRUE if level 2 or level 3 page */ -ibool +bool ibuf_page_low( const page_id_t page_id, const page_size_t& page_size, #ifdef UNIV_DEBUG - ibool x_latch, + bool x_latch, #endif /* UNIV_DEBUG */ const char* file, unsigned line, @@ -1954,10 +1954,7 @@ ibuf_data_too_much_free(void) Allocates a new page from the ibuf file segment and adds it to the free list. @return TRUE on success, FALSE if no space left */ -static -ibool -ibuf_add_free_page(void) -/*====================*/ +static bool ibuf_add_free_page() { mtr_t mtr; page_t* header_page; @@ -1966,7 +1963,7 @@ ibuf_add_free_page(void) page_t* root; page_t* bitmap_page; - mtr_start(&mtr); + mtr.start(); /* Acquire the fsp latch before the ibuf header, obeying the latching order */ mtr_x_lock_space(fil_system.sys_space, &mtr); @@ -1987,9 +1984,8 @@ ibuf_add_free_page(void) &mtr); if (block == NULL) { - mtr_commit(&mtr); - - return(FALSE); + mtr.commit(); + return false; } ut_ad(rw_lock_get_x_lock_count(&block->lock) == 1); @@ -2023,8 +2019,7 @@ ibuf_add_free_page(void) IBUF_BITMAP_IBUF, TRUE, &mtr); ibuf_mtr_commit(&mtr); - - return(TRUE); + return true; } /*********************************************************************//** @@ -2520,6 +2515,7 @@ ibuf_merge_space( ut_ad(space < SRV_LOG_SPACE_FIRST_ID); + log_free_check(); ibuf_mtr_start(&mtr); /* Position the cursor on the first matching record. */ @@ -2675,6 +2671,8 @@ ibuf_merge_in_background( #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ while (sum_pages < n_pages) { + log_free_check(); + ulint n_bytes; n_bytes = ibuf_merge(&n_pag2, false); @@ -4729,6 +4727,7 @@ ibuf_delete_for_discarded_space( memset(dops, 0, sizeof(dops)); loop: + log_free_check(); ibuf_mtr_start(&mtr); /* Position pcur in the insert buffer at the first entry for the @@ -4886,9 +4885,6 @@ dberr_t ibuf_check_bitmap_on_import(const trx_t* trx, fil_space_t* space) } mtr_start(&mtr); - - mtr_set_log_mode(&mtr, MTR_LOG_NO_REDO); - ibuf_enter(&mtr); bitmap_page = ibuf_bitmap_get_map_page( @@ -4978,36 +4974,24 @@ dberr_t ibuf_check_bitmap_on_import(const trx_t* trx, fil_space_t* space) return(DB_SUCCESS); } -/** Updates free bits and buffered bits for bulk loaded page. -@param[in] block index page -@param[in] reset flag if reset free val */ -void -ibuf_set_bitmap_for_bulk_load( - buf_block_t* block, - bool reset) +void ibuf_set_bitmap_for_bulk_load(buf_block_t *block, mtr_t *mtr, bool reset) { page_t* bitmap_page; - mtr_t mtr; ulint free_val; ut_a(page_is_leaf(buf_block_get_frame(block))); free_val = ibuf_index_page_calc_free(block); - mtr_start(&mtr); - mtr.set_named_space_id(block->page.id.space()); - bitmap_page = ibuf_bitmap_get_map_page(block->page.id, - block->page.size, &mtr); + block->page.size, mtr); free_val = reset ? 0 : ibuf_index_page_calc_free(block); ibuf_bitmap_page_set_bits( bitmap_page, block->page.id, block->page.size, - IBUF_BITMAP_FREE, free_val, &mtr); + IBUF_BITMAP_FREE, free_val, mtr); ibuf_bitmap_page_set_bits( bitmap_page, block->page.id, block->page.size, - IBUF_BITMAP_BUFFERED, FALSE, &mtr); - - mtr_commit(&mtr); + IBUF_BITMAP_BUFFERED, FALSE, mtr); } diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index f9d9ab27549..94fd56c62d6 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2019, MariaDB Corporation. +Copyright (c) 2016, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -261,12 +261,12 @@ bitmap page) bitmap page if the page is not one of the fixed address ibuf pages, or NULL, in which case a new transaction is created. @return TRUE if level 2 or level 3 page */ -ibool +bool ibuf_page_low( const page_id_t page_id, const page_size_t& page_size, #ifdef UNIV_DEBUG - ibool x_latch, + bool x_latch, #endif /* UNIV_DEBUG */ const char* file, unsigned line, @@ -274,7 +274,6 @@ ibuf_page_low( MY_ATTRIBUTE((warn_unused_result)); #ifdef UNIV_DEBUG - /** Checks if a page is a level 2 or 3 page in the ibuf hierarchy of pages. Must not be called when recv_no_ibuf_operations==true. @param[in] page_id tablespace/page identifier @@ -284,7 +283,7 @@ Must not be called when recv_no_ibuf_operations==true. # define ibuf_page(page_id, page_size, mtr) \ ibuf_page_low(page_id, page_size, TRUE, __FILE__, __LINE__, mtr) -#else /* UVIV_DEBUG */ +#else /* UNIV_DEBUG */ /** Checks if a page is a level 2 or 3 page in the ibuf hierarchy of pages. Must not be called when recv_no_ibuf_operations==true. @@ -295,7 +294,7 @@ Must not be called when recv_no_ibuf_operations==true. # define ibuf_page(page_id, page_size, mtr) \ ibuf_page_low(page_id, page_size, __FILE__, __LINE__, mtr) -#endif /* UVIV_DEBUG */ +#endif /* UNIV_DEBUG */ /***********************************************************************//** Frees excess pages from the ibuf free list. This function is called when an OS thread calls fsp services to allocate a new file segment, or a new page to a @@ -418,13 +417,11 @@ ibuf_close(void); dberr_t ibuf_check_bitmap_on_import(const trx_t* trx, fil_space_t* space) MY_ATTRIBUTE((nonnull, warn_unused_result)); -/** Updates free bits and buffered bits for bulk loaded page. -@param[in] block index page -@param]in] reset flag if reset free val */ -void -ibuf_set_bitmap_for_bulk_load( - buf_block_t* block, - bool reset); +/** Update free bits and buffered bits for bulk loaded page. +@param block secondary index leaf page +@param mtr mini-transaction +@param reset whether the page is full */ +void ibuf_set_bitmap_for_bulk_load(buf_block_t *block, mtr_t *mtr, bool reset); #define IBUF_HEADER_PAGE_NO FSP_IBUF_HEADER_PAGE_NO #define IBUF_TREE_ROOT_PAGE_NO FSP_IBUF_TREE_ROOT_PAGE_NO From 2ef2e2322a03b6decf4ad00721a27e6dc308847f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 15:26:34 +0200 Subject: [PATCH 20/22] MDEV-29856 heap-use-after-poison in row_merge_spatial_rows() w/ column prefix spatial_index_info: Replaces index_tuple_info_t. Always take a memory heap as a parameter to the member functions. Remove pointer indirection for m_dtuple_vec. spatial_index_info::add(): Duplicate any PRIMARY KEY fields that would point to within ext->buf because that buffer will be allocated in a shorter-lifetime memory heap. --- .../innodb_gis/r/alter_spatial_index.result | 10 ++ .../innodb_gis/t/alter_spatial_index.test | 12 ++ storage/innobase/row/row0merge.cc | 168 +++++++----------- 3 files changed, 90 insertions(+), 100 deletions(-) diff --git a/mysql-test/suite/innodb_gis/r/alter_spatial_index.result b/mysql-test/suite/innodb_gis/r/alter_spatial_index.result index e6ac128bf9f..26c35d0c49f 100644 --- a/mysql-test/suite/innodb_gis/r/alter_spatial_index.result +++ b/mysql-test/suite/innodb_gis/r/alter_spatial_index.result @@ -794,4 +794,14 @@ ENGINE=InnoDB; INSERT INTO t VALUES (REPEAT('MariaDB Corporation Ab ',351),POINT(0,0)); ALTER TABLE t FORCE; DROP TABLE t; +# +# MDEV-29856 heap-use-after-poison in row_merge_spatial_rows() +# with PRIMARY KEY on column prefix +# +CREATE TABLE t (id INT, f TEXT, s POINT NOT NULL, +PRIMARY KEY(id,f(1)), SPATIAL(s)) ENGINE=InnoDB; +INSERT INTO t VALUES +(1,REPEAT('x',8192),@p:=ST_GeomFromText('POINT(0 0)')),(2,'',@p); +ALTER TABLE t FORCE; +DROP TABLE t; # End of 10.3 tests diff --git a/mysql-test/suite/innodb_gis/t/alter_spatial_index.test b/mysql-test/suite/innodb_gis/t/alter_spatial_index.test index 4cfa1daf657..5c110e13fb6 100644 --- a/mysql-test/suite/innodb_gis/t/alter_spatial_index.test +++ b/mysql-test/suite/innodb_gis/t/alter_spatial_index.test @@ -793,4 +793,16 @@ ALTER TABLE t FORCE; # Cleanup DROP TABLE t; +--echo # +--echo # MDEV-29856 heap-use-after-poison in row_merge_spatial_rows() +--echo # with PRIMARY KEY on column prefix +--echo # + +CREATE TABLE t (id INT, f TEXT, s POINT NOT NULL, + PRIMARY KEY(id,f(1)), SPATIAL(s)) ENGINE=InnoDB; +INSERT INTO t VALUES +(1,REPEAT('x',8192),@p:=ST_GeomFromText('POINT(0 0)')),(2,'',@p); +ALTER TABLE t FORCE; +DROP TABLE t; + --echo # End of 10.3 tests diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 3d729b54f7c..7bd15177dad 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -60,63 +60,48 @@ Completed by Sunny Bains and Marko Makela /* Whether to disable file system cache */ char srv_disable_sort_file_cache; -/** Class that caches index row tuples made from a single cluster +/** Class that caches spatial index row tuples made from a single cluster index page scan, and then insert into corresponding index tree */ -class index_tuple_info_t { +class spatial_index_info { public: - /** constructor - @param[in] heap memory heap - @param[in] index index to be created */ - index_tuple_info_t( - mem_heap_t* heap, - dict_index_t* index) UNIV_NOTHROW - { - m_heap = heap; - m_index = index; - m_dtuple_vec = UT_NEW_NOKEY(idx_tuple_vec()); - } + /** constructor + @param index spatial index to be created */ + spatial_index_info(dict_index_t *index) : index(index) + { + ut_ad(index->is_spatial()); + } - /** destructor */ - ~index_tuple_info_t() - { - UT_DELETE(m_dtuple_vec); - } - - /** Get the index object - @return the index object */ - dict_index_t* get_index() UNIV_NOTHROW - { - return(m_index); - } - - /** Caches an index row into index tuple vector - @param[in] row table row - @param[in] ext externally stored column - prefixes, or NULL */ - void add( - const dtuple_t* row, - const row_ext_t* ext) UNIV_NOTHROW - { - dtuple_t* dtuple; - - dtuple = row_build_index_entry(row, ext, m_index, m_heap); - - ut_ad(dtuple); - - m_dtuple_vec->push_back(dtuple); - } + /** Caches an index row into index tuple vector + @param[in] row table row + @param[in] ext externally stored column prefixes, or NULL */ + void add(const dtuple_t *row, const row_ext_t *ext, mem_heap_t *heap) + { + dtuple_t *dtuple= row_build_index_entry(row, ext, index, heap); + ut_ad(dtuple); + ut_ad(dtuple->n_fields == index->n_fields); + if (ext) + { + /* Replace any references to ext, because ext will be allocated + from row_heap. */ + for (ulint i= 1; i < dtuple->n_fields; i++) + { + dfield_t &dfield= dtuple->fields[i]; + if (dfield.data >= ext->buf && + dfield.data <= &ext->buf[ext->n_ext * ext->max_len]) + dfield_dup(&dfield, heap); + } + } + m_dtuple_vec.push_back(dtuple); + } /** Insert spatial index rows cached in vector into spatial index @param[in] trx_id transaction id - @param[in,out] row_heap memory heap @param[in] pcur cluster index scanning cursor + @param[in,out] heap temporary memory heap @param[in,out] scan_mtr mini-transaction for pcur @return DB_SUCCESS if successful, else error number */ - inline dberr_t insert( - trx_id_t trx_id, - mem_heap_t* row_heap, - btr_pcur_t* pcur, - mtr_t* scan_mtr) + inline dberr_t insert(trx_id_t trx_id, btr_pcur_t* pcur, + mem_heap_t* heap, mtr_t* scan_mtr) { big_rec_t* big_rec; rec_t* rec; @@ -130,14 +115,12 @@ public: | BTR_NO_LOCKING_FLAG | BTR_KEEP_SYS_FLAG | BTR_CREATE_FLAG; - ut_ad(dict_index_is_spatial(m_index)); - DBUG_EXECUTE_IF("row_merge_instrument_log_check_flush", log_sys.check_flush_or_checkpoint = true; ); - for (idx_tuple_vec::iterator it = m_dtuple_vec->begin(); - it != m_dtuple_vec->end(); + for (idx_tuple_vec::iterator it = m_dtuple_vec.begin(); + it != m_dtuple_vec.end(); ++it) { dtuple = *it; ut_ad(dtuple); @@ -153,14 +136,14 @@ public: } mtr.start(); - m_index->set_modified(mtr); + index->set_modified(mtr); - ins_cur.index = m_index; - rtr_init_rtr_info(&rtr_info, false, &ins_cur, m_index, + ins_cur.index = index; + rtr_init_rtr_info(&rtr_info, false, &ins_cur, index, false); rtr_info_update_btr(&ins_cur, &rtr_info); - btr_cur_search_to_nth_level(m_index, 0, dtuple, + btr_cur_search_to_nth_level(index, 0, dtuple, PAGE_CUR_RTREE_INSERT, BTR_MODIFY_LEAF, &ins_cur, __FILE__, __LINE__, @@ -169,44 +152,44 @@ public: /* It need to update MBR in parent entry, so change search mode to BTR_MODIFY_TREE */ if (rtr_info.mbr_adj) { - mtr_commit(&mtr); + mtr.commit(); rtr_clean_rtr_info(&rtr_info, true); rtr_init_rtr_info(&rtr_info, false, &ins_cur, - m_index, false); + index, false); rtr_info_update_btr(&ins_cur, &rtr_info); - mtr_start(&mtr); - m_index->set_modified(mtr); + mtr.start(); + index->set_modified(mtr); btr_cur_search_to_nth_level( - m_index, 0, dtuple, + index, 0, dtuple, PAGE_CUR_RTREE_INSERT, BTR_MODIFY_TREE, &ins_cur, __FILE__, __LINE__, &mtr); } error = btr_cur_optimistic_insert( - flag, &ins_cur, &ins_offsets, &row_heap, + flag, &ins_cur, &ins_offsets, &heap, dtuple, &rec, &big_rec, 0, NULL, &mtr); if (error == DB_FAIL) { ut_ad(!big_rec); mtr.commit(); mtr.start(); - m_index->set_modified(mtr); + index->set_modified(mtr); rtr_clean_rtr_info(&rtr_info, true); rtr_init_rtr_info(&rtr_info, false, - &ins_cur, m_index, false); + &ins_cur, index, false); rtr_info_update_btr(&ins_cur, &rtr_info); btr_cur_search_to_nth_level( - m_index, 0, dtuple, + index, 0, dtuple, PAGE_CUR_RTREE_INSERT, BTR_MODIFY_TREE, &ins_cur, __FILE__, __LINE__, &mtr); error = btr_cur_pessimistic_insert( flag, &ins_cur, &ins_offsets, - &row_heap, dtuple, &rec, + &heap, dtuple, &rec, &big_rec, 0, NULL, &mtr); } @@ -229,30 +212,26 @@ public: } } - mtr_commit(&mtr); + mtr.commit(); rtr_clean_rtr_info(&rtr_info, true); } - m_dtuple_vec->clear(); + m_dtuple_vec.clear(); return(error); } private: - /** Cache index rows made from a cluster index scan. Usually - for rows on single cluster index page */ - typedef std::vector > - idx_tuple_vec; + /** Cache index rows made from a cluster index scan. Usually + for rows on single cluster index page */ + typedef std::vector > idx_tuple_vec; - /** vector used to cache index rows made from cluster index scan */ - idx_tuple_vec* m_dtuple_vec; - - /** the index being built */ - dict_index_t* m_index; - - /** memory heap for creating index tuples */ - mem_heap_t* m_heap; + /** vector used to cache index rows made from cluster index scan */ + idx_tuple_vec m_dtuple_vec; +public: + /** the index being built */ + dict_index_t*const index; }; /* Maximum pending doc memory limit in bytes for a fts tokenization thread */ @@ -1566,7 +1545,6 @@ row_mtuple_cmp( @param[in] trx_id transaction id @param[in] sp_tuples cached spatial rows @param[in] num_spatial number of spatial indexes -@param[in,out] row_heap heap for insert @param[in,out] sp_heap heap for tuples @param[in,out] pcur cluster index cursor @param[in,out] mtr mini transaction @@ -1575,9 +1553,8 @@ static dberr_t row_merge_spatial_rows( trx_id_t trx_id, - index_tuple_info_t** sp_tuples, + spatial_index_info** sp_tuples, ulint num_spatial, - mem_heap_t* row_heap, mem_heap_t* sp_heap, btr_pcur_t* pcur, mtr_t* mtr) @@ -1591,7 +1568,7 @@ row_merge_spatial_rows( ut_ad(sp_heap != NULL); for (ulint j = 0; j < num_spatial; j++) { - err = sp_tuples[j]->insert(trx_id, row_heap, pcur, mtr); + err = sp_tuples[j]->insert(trx_id, pcur, sp_heap, mtr); if (err != DB_SUCCESS) { return(err); @@ -1714,8 +1691,7 @@ row_merge_read_clustered_index( os_event_t fts_parallel_sort_event = NULL; ibool fts_pll_sort = FALSE; int64_t sig_count = 0; - index_tuple_info_t** sp_tuples = NULL; - mem_heap_t* sp_heap = NULL; + spatial_index_info** sp_tuples = NULL; ulint num_spatial = 0; BtrBulk* clust_btr_bulk = NULL; bool clust_temp_file = false; @@ -1805,9 +1781,7 @@ row_merge_read_clustered_index( if (num_spatial > 0) { ulint count = 0; - sp_heap = mem_heap_create(512); - - sp_tuples = static_cast( + sp_tuples = static_cast( ut_malloc_nokey(num_spatial * sizeof(*sp_tuples))); @@ -1815,9 +1789,7 @@ row_merge_read_clustered_index( if (dict_index_is_spatial(index[i])) { sp_tuples[count] = UT_NEW_NOKEY( - index_tuple_info_t( - sp_heap, - index[i])); + spatial_index_info(index[i])); count++; } } @@ -1948,7 +1920,7 @@ row_merge_read_clustered_index( /* Insert the cached spatial index rows. */ err = row_merge_spatial_rows( trx->id, sp_tuples, num_spatial, - row_heap, sp_heap, &pcur, &mtr); + row_heap, &pcur, &mtr); if (err != DB_SUCCESS) { goto func_exit; @@ -2329,7 +2301,7 @@ write_buffers: continue; } - ut_ad(sp_tuples[s_idx_cnt]->get_index() + ut_ad(sp_tuples[s_idx_cnt]->index == buf->index); /* If the geometry field is invalid, report @@ -2339,7 +2311,7 @@ write_buffers: break; } - sp_tuples[s_idx_cnt]->add(row, ext); + sp_tuples[s_idx_cnt]->add(row, ext, buf->heap); s_idx_cnt++; continue; @@ -2469,7 +2441,7 @@ write_buffers: err = row_merge_spatial_rows( trx->id, sp_tuples, num_spatial, - row_heap, sp_heap, + row_heap, &pcur, &mtr); if (err != DB_SUCCESS) { @@ -2847,10 +2819,6 @@ wait_again: UT_DELETE(sp_tuples[i]); } ut_free(sp_tuples); - - if (sp_heap) { - mem_heap_free(sp_heap); - } } /* Update the next Doc ID we used. Table should be locked, so From 12f20c154d24b2d29b29bdd1fbbd03ee3b6f5697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 16:04:16 +0200 Subject: [PATCH 21/22] Work around MDEV-24813 in main.rowid_filter_innodb_debug Let us avoid excessive allocation of explicit record locks. --- mysql-test/include/rowid_filter_debug_kill.inc | 18 ++++++------------ .../main/rowid_filter_innodb_debug.result | 15 +++++---------- mysql-test/main/rowid_filter_innodb_debug.test | 1 - .../main/rowid_filter_myisam_debug.result | 14 +++++--------- 4 files changed, 16 insertions(+), 32 deletions(-) diff --git a/mysql-test/include/rowid_filter_debug_kill.inc b/mysql-test/include/rowid_filter_debug_kill.inc index c701d206297..513efed8a4c 100644 --- a/mysql-test/include/rowid_filter_debug_kill.inc +++ b/mysql-test/include/rowid_filter_debug_kill.inc @@ -1,17 +1,14 @@ --source include/have_debug.inc --source include/have_debug_sync.inc +--source include/have_sequence.inc --source include/count_sessions.inc --echo # --echo # MDEV-22761 KILL QUERY during rowid_filter, crashes --echo # -create table t0(a int); -insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); - -# 100 rows create table t2(a int); -insert into t2 select A.a + B.a* 10 from t0 A, t0 B; +insert into t2 select * from seq_0_to_99; # 10K rows CREATE TABLE t3 ( @@ -26,11 +23,10 @@ where table_schema=database() and table_name='t3'; insert into t3 select - A.a, - B.a, + A.seq, + B.seq, 'filler-data-filler-data' -from - t2 A, t2 B; +from seq_0_to_99 A, seq_0_to_99 B; analyze table t2,t3; @@ -48,7 +44,6 @@ where t3.key1=t2.a and t3.key2 in (2,3); connect (con1, localhost, root,,); -connection con1; set debug_sync='now WAIT_FOR at_rowid_filter_check'; evalp kill query $target_id; set debug_sync='now SIGNAL go'; @@ -60,6 +55,5 @@ disconnect con1; reap; set debug_sync='RESET'; -drop table t0,t2,t3; +drop table t2,t3; --source include/wait_until_count_sessions.inc - diff --git a/mysql-test/main/rowid_filter_innodb_debug.result b/mysql-test/main/rowid_filter_innodb_debug.result index f989e00919b..7741a949e7d 100644 --- a/mysql-test/main/rowid_filter_innodb_debug.result +++ b/mysql-test/main/rowid_filter_innodb_debug.result @@ -2,10 +2,8 @@ set default_storage_engine=innodb; # # MDEV-22761 KILL QUERY during rowid_filter, crashes # -create table t0(a int); -insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t2(a int); -insert into t2 select A.a + B.a* 10 from t0 A, t0 B; +insert into t2 select * from seq_0_to_99; CREATE TABLE t3 ( key1 int , key2 int, @@ -19,11 +17,10 @@ engine InnoDB insert into t3 select -A.a, -B.a, +A.seq, +B.seq, 'filler-data-filler-data' -from -t2 A, t2 B; +from seq_0_to_99 A, seq_0_to_99 B; analyze table t2,t3; Table Op Msg_type Msg_text test.t2 analyze status Engine-independent statistics collected @@ -42,7 +39,6 @@ select * from t2, t3 where t3.key1=t2.a and t3.key2 in (2,3); connect con1, localhost, root,,; -connection con1; set debug_sync='now WAIT_FOR at_rowid_filter_check'; kill query $target_id; set debug_sync='now SIGNAL go'; @@ -50,7 +46,7 @@ connection default; disconnect con1; ERROR 70100: Query execution was interrupted set debug_sync='RESET'; -drop table t0,t2,t3; +drop table t2,t3; set default_storage_engine=default; set @save_optimizer_switch= @@optimizer_switch; set @save_use_stat_tables= @@use_stat_tables; @@ -77,7 +73,6 @@ test.t1 analyze status OK set debug_sync='handler_rowid_filter_check SIGNAL killme WAIT_FOR go'; SELECT * FROM t1 WHERE a > 0 AND b=0; connect con1, localhost, root,,; -connection con1; set debug_sync='now WAIT_FOR killme'; kill query @id; set debug_sync='now SIGNAL go'; diff --git a/mysql-test/main/rowid_filter_innodb_debug.test b/mysql-test/main/rowid_filter_innodb_debug.test index 74deaa8ccc9..6353ffa335e 100644 --- a/mysql-test/main/rowid_filter_innodb_debug.test +++ b/mysql-test/main/rowid_filter_innodb_debug.test @@ -39,7 +39,6 @@ set debug_sync='handler_rowid_filter_check SIGNAL killme WAIT_FOR go'; send SELECT * FROM t1 WHERE a > 0 AND b=0; connect (con1, localhost, root,,); -connection con1; let $ignore= `SELECT @id := $ID`; set debug_sync='now WAIT_FOR killme'; kill query @id; diff --git a/mysql-test/main/rowid_filter_myisam_debug.result b/mysql-test/main/rowid_filter_myisam_debug.result index 32a989f50da..75a8fad6947 100644 --- a/mysql-test/main/rowid_filter_myisam_debug.result +++ b/mysql-test/main/rowid_filter_myisam_debug.result @@ -1,10 +1,8 @@ # # MDEV-22761 KILL QUERY during rowid_filter, crashes # -create table t0(a int); -insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t2(a int); -insert into t2 select A.a + B.a* 10 from t0 A, t0 B; +insert into t2 select * from seq_0_to_99; CREATE TABLE t3 ( key1 int , key2 int, @@ -18,11 +16,10 @@ engine MyISAM insert into t3 select -A.a, -B.a, +A.seq, +B.seq, 'filler-data-filler-data' -from -t2 A, t2 B; +from seq_0_to_99 A, seq_0_to_99 B; analyze table t2,t3; Table Op Msg_type Msg_text test.t2 analyze status Engine-independent statistics collected @@ -41,7 +38,6 @@ select * from t2, t3 where t3.key1=t2.a and t3.key2 in (2,3); connect con1, localhost, root,,; -connection con1; set debug_sync='now WAIT_FOR at_rowid_filter_check'; kill query $target_id; set debug_sync='now SIGNAL go'; @@ -49,4 +45,4 @@ connection default; disconnect con1; ERROR 70100: Query execution was interrupted set debug_sync='RESET'; -drop table t0,t2,t3; +drop table t2,t3; From 8fb176c3c1c71516f0562399d055e92686519588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 8 Nov 2022 16:59:36 +0200 Subject: [PATCH 22/22] MDEV-27121 fixup: mariabackup.mdev-14447,full_crc32 --- mysql-test/suite/mariabackup/mdev-14447.combinations | 5 +++++ storage/innobase/buf/buf0buf.cc | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/mariabackup/mdev-14447.combinations diff --git a/mysql-test/suite/mariabackup/mdev-14447.combinations b/mysql-test/suite/mariabackup/mdev-14447.combinations new file mode 100644 index 00000000000..79e5f7836ed --- /dev/null +++ b/mysql-test/suite/mariabackup/mdev-14447.combinations @@ -0,0 +1,5 @@ +[crc32] +--innodb-checksum-algorithm=crc32 + +[full_crc32] +--innodb-checksum-algorithm=full_crc32 diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index d5415d6b249..59d35ddcd5a 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -998,7 +998,7 @@ buf_page_is_corrupted( DBUG_EXECUTE_IF( "page_intermittent_checksum_mismatch", { static int page_counter; - if (page_counter++ == 2) { + if (page_counter++ == 3) { crc32++; } });