From ae142c21a5f7d17cb98f18434b397cb0190725ef Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Thu, 23 Feb 2017 14:24:34 +0200 Subject: [PATCH 01/28] MDEV-12106 Valgrind tests fail all over in buildbot on 10.0 (with blocks are still reachable). There was a similar suppression already, but it had an extra line comparing to failures which we are getting, so it wasn't applied. Added another variant of the suppression. --- mysql-test/valgrind.supp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 18c9559ea78..cbc9493e385 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -1089,6 +1089,15 @@ fun:SSL_library_init } +{ + OpenSSL still reachable. + Memcheck:Leak + fun:*alloc + fun:CRYPTO_malloc + fun:sk_new + fun:SSL_COMP_get_compression_methods + fun:SSL_library_init +} { OpenSSL still reachable. From fdeeab01c0b48b5f56e543b98decb76cb58bedbe Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Sun, 26 Feb 2017 23:01:23 +0400 Subject: [PATCH 02/28] MDEV-6390 CONVERT TO CHARACTER SET utf8 doesn't change DEFAULT CHARSET. ALTER_CONVERT flag removed and replaced for ALTER_OPTIONS for the CONVERT TO CHARACTER SET command. --- mysql-test/r/alter_table.result | 21 ++++++++++++++++++++ mysql-test/t/alter_table.test | 11 +++++++++++ sql/sql_alter.h | 35 +++++++++++++++------------------ sql/sql_yacc.yy | 2 +- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index 72e81895816..cac4c477b5a 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -2091,3 +2091,24 @@ Warnings: Note 1061 Duplicate key name 'id1' DROP TABLE t2; DROP TABLE t1; +# +# MDEV-6390 CONVERT TO CHARACTER SET utf8 doesn't change DEFAULT CHARSET. +# +CREATE TABLE t1 (id int(11) NOT NULL, a int(11) NOT NULL, b int(11)) +ENGINE=InnoDB DEFAULT CHARSET=latin1; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL, + `a` int(11) NOT NULL, + `b` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 +ALTER TABLE t1 CONVERT TO CHARACTER SET utf8; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL, + `a` int(11) NOT NULL, + `b` int(11) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8 +DROP TABLE t1; diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index 6fe0f4d30ee..3670e871bb0 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1756,3 +1756,14 @@ REFERENCES t1 (id); DROP TABLE t2; DROP TABLE t1; + +--echo # +--echo # MDEV-6390 CONVERT TO CHARACTER SET utf8 doesn't change DEFAULT CHARSET. +--echo # + +CREATE TABLE t1 (id int(11) NOT NULL, a int(11) NOT NULL, b int(11)) + ENGINE=InnoDB DEFAULT CHARSET=latin1; +SHOW CREATE TABLE t1; +ALTER TABLE t1 CONVERT TO CHARACTER SET utf8; +SHOW CREATE TABLE t1; +DROP TABLE t1; diff --git a/sql/sql_alter.h b/sql/sql_alter.h index 526442e83e2..7114694124b 100644 --- a/sql/sql_alter.h +++ b/sql/sql_alter.h @@ -70,59 +70,56 @@ public: // Set for DISABLE KEYS | ENABLE KEYS static const uint ALTER_KEYS_ONOFF = 1L << 9; - // Set for CONVERT TO CHARACTER SET - static const uint ALTER_CONVERT = 1L << 10; - // Set for FORCE // Set for ENGINE(same engine) // Set by mysql_recreate_table() - static const uint ALTER_RECREATE = 1L << 11; + static const uint ALTER_RECREATE = 1L << 10; // Set for ADD PARTITION - static const uint ALTER_ADD_PARTITION = 1L << 12; + static const uint ALTER_ADD_PARTITION = 1L << 11; // Set for DROP PARTITION - static const uint ALTER_DROP_PARTITION = 1L << 13; + static const uint ALTER_DROP_PARTITION = 1L << 12; // Set for COALESCE PARTITION - static const uint ALTER_COALESCE_PARTITION = 1L << 14; + static const uint ALTER_COALESCE_PARTITION = 1L << 13; // Set for REORGANIZE PARTITION ... INTO - static const uint ALTER_REORGANIZE_PARTITION = 1L << 15; + static const uint ALTER_REORGANIZE_PARTITION = 1L << 14; // Set for partition_options - static const uint ALTER_PARTITION = 1L << 16; + static const uint ALTER_PARTITION = 1L << 15; // Set for LOAD INDEX INTO CACHE ... PARTITION // Set for CACHE INDEX ... PARTITION - static const uint ALTER_ADMIN_PARTITION = 1L << 17; + static const uint ALTER_ADMIN_PARTITION = 1L << 16; // Set for REORGANIZE PARTITION - static const uint ALTER_TABLE_REORG = 1L << 18; + static const uint ALTER_TABLE_REORG = 1L << 17; // Set for REBUILD PARTITION - static const uint ALTER_REBUILD_PARTITION = 1L << 19; + static const uint ALTER_REBUILD_PARTITION = 1L << 18; // Set for partitioning operations specifying ALL keyword - static const uint ALTER_ALL_PARTITION = 1L << 20; + static const uint ALTER_ALL_PARTITION = 1L << 19; // Set for REMOVE PARTITIONING - static const uint ALTER_REMOVE_PARTITIONING = 1L << 21; + static const uint ALTER_REMOVE_PARTITIONING = 1L << 20; // Set for ADD FOREIGN KEY - static const uint ADD_FOREIGN_KEY = 1L << 22; + static const uint ADD_FOREIGN_KEY = 1L << 21; // Set for DROP FOREIGN KEY - static const uint DROP_FOREIGN_KEY = 1L << 23; + static const uint DROP_FOREIGN_KEY = 1L << 22; // Set for EXCHANGE PARITION - static const uint ALTER_EXCHANGE_PARTITION = 1L << 24; + static const uint ALTER_EXCHANGE_PARTITION = 1L << 23; // Set by Sql_cmd_alter_table_truncate_partition::execute() - static const uint ALTER_TRUNCATE_PARTITION = 1L << 25; + static const uint ALTER_TRUNCATE_PARTITION = 1L << 24; // Set for ADD [COLUMN] FIRST | AFTER - static const uint ALTER_COLUMN_ORDER = 1L << 26; + static const uint ALTER_COLUMN_ORDER = 1L << 25; enum enum_enable_or_disable { LEAVE_AS_IS, ENABLE, DISABLE }; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index aca579f0a98..9885cd729b4 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -7643,7 +7643,7 @@ alter_list_item: LEX *lex= Lex; if (lex->create_info.add_alter_list_item_convert_to_charset($5)) MYSQL_YYABORT; - lex->alter_info.flags|= Alter_info::ALTER_CONVERT; + lex->alter_info.flags|= Alter_info::ALTER_OPTIONS; } | create_table_options_space_separated { From e5b877ce2790bb88fca05ea1c5a455add50a5da9 Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Thu, 23 Feb 2017 21:50:55 +0100 Subject: [PATCH 03/28] MDEV-11935: Queries in stored procedures with and EXISTS(SELECT * FROM VIEW) crashes and closes hte conneciton. Use correct start point even for taken out from subselect items in process of exists2in conversion. --- mysql-test/r/sp.result | 38 ++++++++++++++++++++++++++++++++++++++ mysql-test/t/sp.test | 40 ++++++++++++++++++++++++++++++++++++++++ sql/sql_base.cc | 2 +- 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/sp.result b/mysql-test/r/sp.result index c059428be16..2cb1b701e2d 100644 --- a/mysql-test/r/sp.result +++ b/mysql-test/r/sp.result @@ -7952,3 +7952,41 @@ set global table_open_cache= @tmp_toc; set global table_definition_cache= @tmp_tdc; drop procedure p1; drop table t1,t2,t3,t4,t5,t6; +# +# MDEV-11935: Queries in stored procedures with and +# EXISTS(SELECT * FROM VIEW) crashes and closes hte conneciton. +# +CREATE TABLE ANY_TABLE ( +ENTITY_UID BIGINT NOT NULL +); +CREATE TABLE SECURITY_PATH( +origid BIGINT UNSIGNED NOT NULL, +destid BIGINT UNSIGNED NOT NULL, +KEY (destid) +); +CREATE VIEW ENTITY_ACCESS ( +ENTITY_UID, +OWNER_UID +) AS +SELECT SP1.origid, +SP2.destid +FROM SECURITY_PATH SP1 +JOIN SECURITY_PATH SP2 ON SP1.destid = SP2.origid +; +CREATE PROCEDURE SP_EXAMPLE_SELECT () +BEGIN +SELECT * +FROM ANY_TABLE AT1 +WHERE EXISTS ( SELECT * +FROM ENTITY_ACCESS EA +WHERE AT1.ENTITY_UID = EA.ENTITY_UID +AND EA.OWNER_UID IS NULL ); +END +// +CALL SP_EXAMPLE_SELECT (); +ENTITY_UID +CALL SP_EXAMPLE_SELECT (); +ENTITY_UID +drop procedure SP_EXAMPLE_SELECT; +drop view ENTITY_ACCESS; +drop table ANY_TABLE, SECURITY_PATH; diff --git a/mysql-test/t/sp.test b/mysql-test/t/sp.test index 10c95f9a919..c5d37d1017d 100644 --- a/mysql-test/t/sp.test +++ b/mysql-test/t/sp.test @@ -9422,3 +9422,43 @@ drop procedure p1; drop table t1,t2,t3,t4,t5,t6; +--echo # +--echo # MDEV-11935: Queries in stored procedures with and +--echo # EXISTS(SELECT * FROM VIEW) crashes and closes hte conneciton. +--echo # + +CREATE TABLE ANY_TABLE ( + ENTITY_UID BIGINT NOT NULL +); +CREATE TABLE SECURITY_PATH( +origid BIGINT UNSIGNED NOT NULL, +destid BIGINT UNSIGNED NOT NULL, +KEY (destid) +); +CREATE VIEW ENTITY_ACCESS ( +ENTITY_UID, +OWNER_UID +) AS +SELECT SP1.origid, + SP2.destid +FROM SECURITY_PATH SP1 +JOIN SECURITY_PATH SP2 ON SP1.destid = SP2.origid +; +--delimiter // +CREATE PROCEDURE SP_EXAMPLE_SELECT () +BEGIN + SELECT * + FROM ANY_TABLE AT1 + WHERE EXISTS ( SELECT * + FROM ENTITY_ACCESS EA + WHERE AT1.ENTITY_UID = EA.ENTITY_UID + AND EA.OWNER_UID IS NULL ); +END +// +--delimiter ; +CALL SP_EXAMPLE_SELECT (); +CALL SP_EXAMPLE_SELECT (); + +drop procedure SP_EXAMPLE_SELECT; +drop view ENTITY_ACCESS; +drop table ANY_TABLE, SECURITY_PATH; diff --git a/sql/sql_base.cc b/sql/sql_base.cc index f2c61cdca89..73f71a07f15 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -6628,7 +6628,7 @@ find_field_in_tables(THD *thd, Item_ident *item, if (!table_ref->belong_to_view && !table_ref->belong_to_derived) { - SELECT_LEX *current_sel= thd->lex->current_select; + SELECT_LEX *current_sel= item->context->select_lex; SELECT_LEX *last_select= table_ref->select_lex; bool all_merged= TRUE; for (SELECT_LEX *sl= current_sel; sl && sl!=last_select; From b54566d73bb96ef9eb7d734742d3743fb3369a40 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 28 Feb 2017 10:08:12 +1100 Subject: [PATCH 04/28] MDEV-11619: mtr --mem {no argument of a directory} (#320) This removes functionality of where ./mtr --mem /tmp/dir could be a directory. Now MTR_MEM=/tmp/dir ./mtr is needed. The case where MTR_MEM=/tmp/dir ./mtr --mem has the equivalent effect. Signed-off-by: Daniel Black --mem works better as a pure flag, because it can be followed by command-line arguments (test names). If the option is allowed to have a value, the test name which directly follows it will be treated as the option value instead. It is possible to implement workarounds to avoid this, but they would not be completely reliable, and there is no practical purpose of such extension of functionality to justify them. --- mysql-test/mysql-test-run.pl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 568b9292344..69ab23c6f40 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1194,7 +1194,7 @@ sub command_line_setup { # Directories 'tmpdir=s' => \$opt_tmpdir, 'vardir=s' => \$opt_vardir, - 'mem:s' => \$opt_mem, + 'mem' => \$opt_mem, 'clean-vardir' => \$opt_clean_vardir, 'client-bindir=s' => \$path_client_bindir, 'client-libdir=s' => \$path_client_libdir, @@ -1457,9 +1457,6 @@ sub command_line_setup { # Use $ENV{'MTR_MEM'} as first location to look (if defined) unshift(@tmpfs_locations, $ENV{'MTR_MEM'}) if defined $ENV{'MTR_MEM'}; - # however if the opt_mem was given a value, use this first - unshift(@tmpfs_locations, $opt_mem) if $opt_mem ne ''; - foreach my $fs (@tmpfs_locations) { if ( -d $fs && ! -l $fs ) From fc673a2c121f524a7ed4101b8b9a0cedf8d6fd58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 28 Feb 2017 09:54:12 +0200 Subject: [PATCH 05/28] MDEV-12127 InnoDB: Assertion failure loop_count < 5 in file log0log.cc As suggested in MySQL Bug#58536, increase the limit in this debug assertion in order to avoid false positives on heavily loaded systems. --- storage/innobase/log/log0log.cc | 12 +----------- storage/xtradb/log/log0log.cc | 12 +----------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 89c4ed2d7bf..57516d7c8f4 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1460,17 +1460,7 @@ log_write_up_to( } loop: -#ifdef UNIV_DEBUG - loop_count++; - - ut_ad(loop_count < 5); - -# if 0 - if (loop_count > 2) { - fprintf(stderr, "Log loop count %lu\n", loop_count); - } -# endif -#endif + ut_ad(++loop_count < 100); mutex_enter(&(log_sys->mutex)); ut_ad(!recv_no_log_write); diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index c39f108b1de..b39a8ed1829 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1570,17 +1570,7 @@ log_write_up_to( } loop: -#ifdef UNIV_DEBUG - loop_count++; - - ut_ad(loop_count < 5); - -# if 0 - if (loop_count > 2) { - fprintf(stderr, "Log loop count %lu\n", loop_count); - } -# endif -#endif + ut_ad(++loop_count < 100); mutex_enter(&(log_sys->mutex)); ut_ad(!recv_no_log_write); From d4baeca441b248de024719f556b681ce868501ae Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 28 Feb 2017 12:57:33 +0000 Subject: [PATCH 06/28] Windows : Fix server compile errors when compile with /Zc:strictStrings option --- client/mysqlbinlog.cc | 2 +- include/my_global.h | 7 +++---- sql/mysql_install_db.cc | 4 ++-- sql/mysqld.cc | 2 +- sql/mysqld.h | 3 ++- sql/nt_servc.cc | 2 +- sql/nt_servc.h | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 4be31421f9b..29355d013c7 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -106,7 +106,7 @@ static const char* sock= 0; static char *opt_plugindir= 0, *opt_default_auth= 0; #ifdef HAVE_SMEM -static char *shared_memory_base_name= 0; +static const char *shared_memory_base_name= 0; #endif static char* user = 0; static char* pass = 0; diff --git a/include/my_global.h b/include/my_global.h index 007c9400ec0..a61fd765995 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -1078,10 +1078,9 @@ typedef ulong myf; /* Type of MyFlags in my_funcs */ static inline char *dlerror(void) { static char win_errormsg[2048]; - if(FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, - 0, GetLastError(), 0, win_errormsg, 2048, NULL)) - return win_errormsg; - return ""; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, + 0, GetLastError(), 0, win_errormsg, 2048, NULL); + return win_errormsg; } #define HAVE_DLOPEN 1 #define HAVE_DLERROR 1 diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index c23a20ebac9..de799874a8f 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -386,8 +386,8 @@ static int register_service() CloseServiceHandle(sc_manager); die("CreateService failed (%u)", GetLastError()); } - - SERVICE_DESCRIPTION sd= { "MariaDB database server" }; + char description[] = "MariaDB database server"; + SERVICE_DESCRIPTION sd= { description }; ChangeServiceConfig2(sc_service, SERVICE_CONFIG_DESCRIPTION, &sd); CloseServiceHandle(sc_service); CloseServiceHandle(sc_manager); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 6e40168990f..a6b6cdfae9b 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1414,7 +1414,7 @@ ulong query_cache_min_res_unit= QUERY_CACHE_MIN_RESULT_DATA_SIZE; Query_cache query_cache; #endif #ifdef HAVE_SMEM -char *shared_memory_base_name= default_shared_memory_base_name; +const char *shared_memory_base_name= default_shared_memory_base_name; my_bool opt_enable_shared_memory; HANDLE smem_event_connect_request= 0; #endif diff --git a/sql/mysqld.h b/sql/mysqld.h index 732f00c887a..e7394230fe4 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -118,7 +118,8 @@ extern my_bool sp_automatic_privileges, opt_noacl; extern ulong use_stat_tables; extern my_bool opt_old_style_user_limits, trust_function_creators; extern uint opt_crash_binlog_innodb; -extern char *shared_memory_base_name, *mysqld_unix_port; +extern const char *shared_memory_base_name; +extern char *mysqld_unix_port; extern my_bool opt_enable_shared_memory; extern ulong opt_replicate_events_marked_for_skip; extern char *default_tz_name; diff --git a/sql/nt_servc.cc b/sql/nt_servc.cc index d6a8eac7ed5..e05e43a0a59 100644 --- a/sql/nt_servc.cc +++ b/sql/nt_servc.cc @@ -508,7 +508,7 @@ BOOL NTService::IsService(LPCSTR ServiceName) } /* ------------------------------------------------------------------------ -------------------------------------------------------------------------- */ -BOOL NTService::got_service_option(char **argv, char *service_option) +BOOL NTService::got_service_option(char **argv, const char *service_option) { char *option; for (option= argv[1]; *option; option++) diff --git a/sql/nt_servc.h b/sql/nt_servc.h index 949499d8d7f..6781fe0ddfa 100644 --- a/sql/nt_servc.h +++ b/sql/nt_servc.h @@ -61,7 +61,7 @@ class NTService BOOL SeekStatus(LPCSTR szInternName, int OperationType); BOOL Remove(LPCSTR szInternName); BOOL IsService(LPCSTR ServiceName); - BOOL got_service_option(char **argv, char *service_option); + BOOL got_service_option(char **argv, const char *service_option); BOOL is_super_user(); /* From ce903428a8517f5207b1e236c9187fa299bc0213 Mon Sep 17 00:00:00 2001 From: iangilfillan Date: Thu, 23 Feb 2017 11:27:52 +0200 Subject: [PATCH 07/28] Update MariaDB Foundation sponsors --- CREDITS | 22 ++++++++++++---------- mysql-test/r/contributors.result | 14 ++++++++------ sql/contributors.h | 14 ++++++++------ 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/CREDITS b/CREDITS index 35ab4d48a8f..d352232ad2e 100644 --- a/CREDITS +++ b/CREDITS @@ -3,16 +3,18 @@ organization registered in the USA. The current main sponsors of the MariaDB Foundation are: -Booking.com http://www.booking.com (2013 - 2016) -Development Bank of Singapore http://dbs.com (2016) -MariaDB Corporation https://www.mariadb.com (2013 - 2016) -Visma http://visma.com (2015 - 2016) -Acronis http://acronis.com (2016) -Nexedi https://www.nexedi.com (2016) -Automattic https://automattic.com (2014 - 2016) -Tencent Game DBA http://tencentdba.com/about (2016) -Verkkokauppa.com https://www.verkkokauppa.com (2015 - 2016) -Virtuozzo https://virtuozzo.com (2016) +Alibaba Cloud https://intl.aliyun.com (2017) +Booking.com https://www.booking.com (2013 - 2017) +Development Bank of Singapore https://dbs.com (2016 - 2017) +MariaDB Corporation https://www.mariadb.com (2013 - 2017) +Visma https://visma.com (2015 - 2017) +Acronis http://acronis.com (2016 - 2017) +Nexedi https://www.nexedi.com (2016 - 2017) +Automattic https://automattic.com (2014 - 2017) +Tencent Game DBA http://tencentdba.com/about (2016 - 2017) +Tencent TDSQL http://tdsql.org/ (2016 - 2017) +Verkkokauppa.com https://www.verkkokauppa.com (2015 - 2017) +Virtuozzo https://virtuozzo.com (2016 - 2017) For a full list of sponsors, see https://mariadb.org/about/supporters/ diff --git a/mysql-test/r/contributors.result b/mysql-test/r/contributors.result index f3f5e227d3a..4a26d0f19dd 100644 --- a/mysql-test/r/contributors.result +++ b/mysql-test/r/contributors.result @@ -1,15 +1,17 @@ SHOW CONTRIBUTORS; Name Location Comment -Booking.com http://www.booking.com Founding member, Platinum Sponsor of the MariaDB Foundation +Booking.com https://www.booking.com Founding member, Platinum Sponsor of the MariaDB Foundation +Alibaba Cloud https://intl.aliyun.com Platinum Sponsor of the MariaDB Foundation MariaDB Corporation https://mariadb.com Founding member, Gold Sponsor of the MariaDB Foundation -Visma http://visma.com Gold Sponsor of the MariaDB Foundation -DBS http://dbs.com Gold Sponsor of the MariaDB Foundation +Visma https://visma.com Gold Sponsor of the MariaDB Foundation +DBS https://dbs.com Gold Sponsor of the MariaDB Foundation Nexedi https://www.nexedi.com Silver Sponsor of the MariaDB Foundation Acronis http://www.acronis.com Silver Sponsor of the MariaDB Foundation Auttomattic https://automattic.com Bronze Sponsor of the MariaDB Foundation -Verkkokauppa.com https://virtuozzo.com Bronze Sponsor of the MariaDB Foundation -Virtuozzo https://virtuozzo.com/ Bronze Sponsor of the MariaDB Foundation -Tencent Game DBA http://tencentdba.com/about/ Bronze Sponsor of the MariaDB Foundation +Verkkokauppa.com https://www.verkkokauppa.com Bronze Sponsor of the MariaDB Foundation +Virtuozzo https://virtuozzo.com Bronze Sponsor of the MariaDB Foundation +Tencent Game DBA http://tencentdba.com/about Bronze Sponsor of the MariaDB Foundation +Tencent TDSQL http://tdsql.org Bronze Sponsor of the MariaDB Foundation Google USA Sponsoring encryption, parallel replication and GTID Facebook USA Sponsoring non-blocking API, LIMIT ROWS EXAMINED etc Ronald Bradford Brisbane, Australia EFF contribution for UC2006 Auction diff --git a/sql/contributors.h b/sql/contributors.h index 0359ec54022..3a771e2b493 100644 --- a/sql/contributors.h +++ b/sql/contributors.h @@ -37,16 +37,18 @@ struct show_table_contributors_st { struct show_table_contributors_st show_table_contributors[]= { /* MariaDB foundation sponsors, in contribution, size , time order */ - {"Booking.com", "http://www.booking.com", "Founding member, Platinum Sponsor of the MariaDB Foundation"}, + {"Booking.com", "https://www.booking.com", "Founding member, Platinum Sponsor of the MariaDB Foundation"}, + {"Alibaba Cloud", "https://intl.aliyun.com", "Platinum Sponsor of the MariaDB Foundation"}, {"MariaDB Corporation", "https://mariadb.com", "Founding member, Gold Sponsor of the MariaDB Foundation"}, - {"Visma", "http://visma.com", "Gold Sponsor of the MariaDB Foundation"}, - {"DBS", "http://dbs.com", "Gold Sponsor of the MariaDB Foundation"}, + {"Visma", "https://visma.com", "Gold Sponsor of the MariaDB Foundation"}, + {"DBS", "https://dbs.com", "Gold Sponsor of the MariaDB Foundation"}, {"Nexedi", "https://www.nexedi.com", "Silver Sponsor of the MariaDB Foundation"}, {"Acronis", "http://www.acronis.com", "Silver Sponsor of the MariaDB Foundation"}, {"Auttomattic", "https://automattic.com", "Bronze Sponsor of the MariaDB Foundation"}, - {"Verkkokauppa.com", "https://virtuozzo.com", "Bronze Sponsor of the MariaDB Foundation"}, - {"Virtuozzo", "https://virtuozzo.com/", "Bronze Sponsor of the MariaDB Foundation"}, - {"Tencent Game DBA", "http://tencentdba.com/about/", "Bronze Sponsor of the MariaDB Foundation"}, + {"Verkkokauppa.com", "https://www.verkkokauppa.com", "Bronze Sponsor of the MariaDB Foundation"}, + {"Virtuozzo", "https://virtuozzo.com", "Bronze Sponsor of the MariaDB Foundation"}, + {"Tencent Game DBA", "http://tencentdba.com/about", "Bronze Sponsor of the MariaDB Foundation"}, + {"Tencent TDSQL", "http://tdsql.org", "Bronze Sponsor of the MariaDB Foundation"}, /* Sponsors of important features */ {"Google", "USA", "Sponsoring encryption, parallel replication and GTID"}, From d5c54f3990b49d7c7d6a410016e85e8e58803895 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 29 Jan 2017 18:18:19 +0200 Subject: [PATCH 08/28] Fixed compiler warnings - Removed not used variables - Added __attribute__() - Added static to some local functions (gcc 5.4 gives a warning for external functions without an external definition) --- extra/yassl/src/ssl.cpp | 1 - storage/connect/ha_connect.cc | 6 +----- storage/connect/ioapi.c | 21 +++++++++++---------- storage/connect/zip.c | 14 +++++++------- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/extra/yassl/src/ssl.cpp b/extra/yassl/src/ssl.cpp index 7069140dcda..982d462e3b8 100644 --- a/extra/yassl/src/ssl.cpp +++ b/extra/yassl/src/ssl.cpp @@ -774,7 +774,6 @@ int SSL_CTX_load_verify_locations(SSL_CTX* ctx, const char* file, const char* path) { int ret = SSL_FAILURE; - const int HALF_PATH = 128; if (file) ret = read_file(ctx, file, SSL_FILETYPE_PEM, CA); diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 33f02d8338c..e66f03e8174 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -5173,7 +5173,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd, TABLE_SHARE *table_s, HA_CREATE_INFO *create_info) { - char v=0, spc= ',', qch= 0; + char v=0; const char *fncn= "?"; const char *user, *fn, *db, *host, *pwd, *sep, *tbl, *src; const char *col, *ocl, *rnk, *pic, *fcl, *skc, *zfn; @@ -5225,10 +5225,6 @@ static int connect_assisted_discovery(handlerton *, THD* thd, fncn= topt->catfunc; fnc= GetFuncID(fncn); sep= topt->separator; - spc= (!sep) ? ',' : *sep; - qch= topt->qchar ? *topt->qchar : (signed)topt->quoted >= 0 ? '"' : 0; - mul = (int)topt->multiple; - tbl= topt->tablist; col= topt->colist; if (topt->oplist) { diff --git a/storage/connect/ioapi.c b/storage/connect/ioapi.c index 7f5c191b2af..a49da91f7f0 100644 --- a/storage/connect/ioapi.c +++ b/storage/connect/ioapi.c @@ -27,6 +27,7 @@ #include "ioapi.h" +#include "my_attribute.h" voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode) { @@ -92,7 +93,7 @@ static long ZCALLBACK fseek64_file_func OF((voidpf opaque, voidpf stream, ZPO static int ZCALLBACK fclose_file_func OF((voidpf opaque, voidpf stream)); static int ZCALLBACK ferror_file_func OF((voidpf opaque, voidpf stream)); -static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, int mode) +static voidpf ZCALLBACK fopen_file_func (voidpf opaque __attribute__((unused)), const char* filename, int mode) { FILE* file = NULL; const char* mode_fopen = NULL; @@ -110,7 +111,7 @@ static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, in return file; } -static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, int mode) +static voidpf ZCALLBACK fopen64_file_func (voidpf opaque __attribute__((unused)), const void* filename, int mode) { FILE* file = NULL; const char* mode_fopen = NULL; @@ -129,21 +130,21 @@ static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, } -static uLong ZCALLBACK fread_file_func (voidpf opaque, voidpf stream, void* buf, uLong size) +static uLong ZCALLBACK fread_file_func (voidpf opaque __attribute__((unused)), voidpf stream, void* buf, uLong size) { uLong ret; ret = (uLong)fread(buf, 1, (size_t)size, (FILE *)stream); return ret; } -static uLong ZCALLBACK fwrite_file_func (voidpf opaque, voidpf stream, const void* buf, uLong size) +static uLong ZCALLBACK fwrite_file_func (voidpf opaque __attribute__((unused)), voidpf stream, const void* buf, uLong size) { uLong ret; ret = (uLong)fwrite(buf, 1, (size_t)size, (FILE *)stream); return ret; } -static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream) +static long ZCALLBACK ftell_file_func (voidpf opaque __attribute__((unused)), voidpf stream) { long ret; ret = ftell((FILE *)stream); @@ -151,14 +152,14 @@ static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream) } -static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque, voidpf stream) +static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque __attribute__((unused)), voidpf stream) { ZPOS64_T ret; ret = FTELLO_FUNC((FILE *)stream); return ret; } -static long ZCALLBACK fseek_file_func (voidpf opaque, voidpf stream, uLong offset, int origin) +static long ZCALLBACK fseek_file_func (voidpf opaque __attribute__((unused)), voidpf stream, uLong offset, int origin) { int fseek_origin=0; long ret; @@ -181,7 +182,7 @@ static long ZCALLBACK fseek_file_func (voidpf opaque, voidpf stream, uLong offs return ret; } -static long ZCALLBACK fseek64_file_func (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) +static long ZCALLBACK fseek64_file_func (voidpf opaque __attribute__((unused)), voidpf stream, ZPOS64_T offset, int origin) { int fseek_origin=0; long ret; @@ -207,14 +208,14 @@ static long ZCALLBACK fseek64_file_func (voidpf opaque, voidpf stream, ZPOS64_T } -static int ZCALLBACK fclose_file_func (voidpf opaque, voidpf stream) +static int ZCALLBACK fclose_file_func (voidpf opaque __attribute__((unused)), voidpf stream) { int ret; ret = fclose((FILE *)stream); return ret; } -static int ZCALLBACK ferror_file_func (voidpf opaque, voidpf stream) +static int ZCALLBACK ferror_file_func (voidpf opaque __attribute__((unused)), voidpf stream) { int ret; ret = ferror((FILE *)stream); diff --git a/storage/connect/zip.c b/storage/connect/zip.c index ea54853e858..4bbe31ab7dd 100644 --- a/storage/connect/zip.c +++ b/storage/connect/zip.c @@ -637,7 +637,7 @@ local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib return relativeOffset; } -int LoadCentralDirectoryRecord(zip64_internal* pziinit) +static int LoadCentralDirectoryRecord(zip64_internal* pziinit) { int err=ZIP_OK; ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ @@ -846,7 +846,7 @@ int LoadCentralDirectoryRecord(zip64_internal* pziinit) /************************************************************/ -extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) +static zipFile zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) { zip64_internal ziinit; zip64_internal* zi; @@ -955,7 +955,7 @@ extern zipFile ZEXPORT zipOpen64 (const void* pathname, int append) return zipOpen3(pathname,append,NULL,NULL); } -int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) +static int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) { /* write the local header */ int err; @@ -1752,7 +1752,7 @@ extern int ZEXPORT zipCloseFileInZip (zipFile file) return zipCloseFileInZipRaw (file,0,0); } -int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) +static int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) { int err = ZIP_OK; ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writting_offset; @@ -1774,7 +1774,7 @@ int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eo return err; } -int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +static int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) { int err = ZIP_OK; @@ -1813,7 +1813,7 @@ int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centra } return err; } -int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +static int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) { int err = ZIP_OK; @@ -1861,7 +1861,7 @@ int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, return err; } -int Write_GlobalComment(zip64_internal* zi, const char* global_comment) +static int Write_GlobalComment(zip64_internal* zi, const char* global_comment) { int err = ZIP_OK; uInt size_global_comment = 0; From e65f667bb60244610512efd7491fc77eccceb9db Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 29 Jan 2017 22:10:56 +0200 Subject: [PATCH 09/28] MDEV-9573 'Stop slave' hangs on replication slave The reason for this is that stop slave takes LOCK_active_mi over the whole operation while some slave operations will also need LOCK_active_mi which causes deadlocks. Fixed by introducing object counting for Master_info and not taking LOCK_active_mi over stop slave or even stop_all_slaves() Another benefit of this approach is that it allows: - Multiple threads can run SHOW SLAVE STATUS at the same time - START/STOP/RESET/SLAVE STATUS on a slave will not block other slaves - Simpler interface for handling get_master_info() - Added some missing unlock of 'log_lock' in error condtions - Moved rpl_parallel_inactivate_pool(&global_rpl_thread_pool) to end of stop_slave() to not have to use LOCK_active_mi inside terminate_slave_threads() - Changed argument for remove_master_info() to Master_info, as we always have this available - Fixed core dump when doing FLUSH TABLES WITH READ LOCK and parallel replication. Problem was that waiting for pause_for_ftwrl was not done when deleting rpt->current_owner after a force_abort. --- mysys/mf_iocache.c | 5 +- sql/item_func.cc | 8 +- sql/mysqld.cc | 66 +++----- sql/rpl_mi.cc | 375 ++++++++++++++++++++++++++++++++------------ sql/rpl_mi.h | 16 +- sql/rpl_parallel.cc | 62 +++++++- sql/rpl_parallel.h | 1 + sql/slave.cc | 79 +++++----- sql/slave.h | 1 + sql/sql_parse.cc | 66 +++++--- sql/sql_reload.cc | 56 +++---- sql/sql_repl.cc | 19 +-- sql/sys_vars.cc | 130 +++++---------- 13 files changed, 539 insertions(+), 345 deletions(-) diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c index f891a22b17d..8687c2e0c48 100644 --- a/mysys/mf_iocache.c +++ b/mysys/mf_iocache.c @@ -1815,6 +1815,7 @@ int my_b_flush_io_cache(IO_CACHE *info, It's currently safe to call this if one has called init_io_cache() on the 'info' object, even if init_io_cache() failed. This function is also safe to call twice with the same handle. + Note that info->file is not reset as the caller may still use ut for my_close() RETURN 0 ok @@ -1850,10 +1851,12 @@ int end_io_cache(IO_CACHE *info) if (info->type == SEQ_READ_APPEND) { /* Destroy allocated mutex */ - info->type= TYPE_NOT_SET; mysql_mutex_destroy(&info->append_buffer_lock); } info->share= 0; + info->type= TYPE_NOT_SET; /* Ensure that flush_io_cache() does nothing */ + info->write_end= 0; /* Ensure that my_b_write() fails */ + info->write_function= 0; /* my_b_write will crash if used */ DBUG_RETURN(error); } /* end_io_cache */ diff --git a/sql/item_func.cc b/sql/item_func.cc index 3149ebb855d..2c43181e8bb 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -3958,12 +3958,7 @@ longlong Item_master_pos_wait::val_int() else connection_name= thd->variables.default_master_connection; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) // master_info_index is set to NULL on shutdown. - mi= master_info_index->get_master_info(&connection_name, - Sql_condition::WARN_LEVEL_WARN); - mysql_mutex_unlock(&LOCK_active_mi); - if (!mi) + if (!(mi= get_master_info(&connection_name, Sql_condition::WARN_LEVEL_WARN))) goto err; if ((event_count = mi->rli.wait_for_pos(thd, log_name, pos, timeout)) == -2) @@ -3971,6 +3966,7 @@ longlong Item_master_pos_wait::val_int() null_value = 1; event_count=0; } + mi->release(); #endif return event_count; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index a6b6cdfae9b..ca45e9cdcdb 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -702,12 +702,15 @@ mysql_mutex_t LOCK_delayed_insert, LOCK_delayed_status, LOCK_delayed_create, LOCK_crypt, LOCK_global_system_variables, - LOCK_user_conn, LOCK_slave_list, LOCK_active_mi, + LOCK_user_conn, LOCK_slave_list, LOCK_connection_count, LOCK_error_messages, LOCK_slave_init; mysql_mutex_t LOCK_stats, LOCK_global_user_client_stats, LOCK_global_table_stats, LOCK_global_index_stats; +/* This protects against changes in master_info_index */ +mysql_mutex_t LOCK_active_mi; + /** The below lock protects access to two global server variables: max_prepared_stmt_count and prepared_stmt_count. These variables @@ -1651,7 +1654,7 @@ static void close_connections(void) mysql_mutex_unlock(&LOCK_thread_count); // For unlink from list Events::deinit(); - end_slave(); + slave_prepare_for_shutdown(); /* Give threads time to die. @@ -1700,6 +1703,7 @@ static void close_connections(void) DBUG_PRINT("quit",("Unlocking LOCK_thread_count")); mysql_mutex_unlock(&LOCK_thread_count); } + end_slave(); /* All threads has now been aborted */ DBUG_PRINT("quit",("Waiting for threads to die (count=%u)",thread_count)); mysql_mutex_lock(&LOCK_thread_count); @@ -7214,17 +7218,14 @@ static int show_slave_running(THD *thd, SHOW_VAR *var, char *buff) var->type= SHOW_MY_BOOL; var->value= buff; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if ((mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_NOTE))) { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_NOTE); - if (mi) - tmp= (my_bool) (mi->slave_running == MYSQL_SLAVE_RUN_READING && - mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN); + tmp= (my_bool) (mi->slave_running == MYSQL_SLAVE_RUN_READING && + mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); if (mi) *((my_bool *)buff)= tmp; else @@ -7256,38 +7257,26 @@ static int show_slaves_running(THD *thd, SHOW_VAR *var, char *buff) { var->type= SHOW_LONGLONG; var->value= buff; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) - *((longlong *)buff)= master_info_index->any_slave_sql_running(); - else - *((longlong *)buff)= 0; + *((longlong *)buff)= any_slave_sql_running(); - mysql_mutex_unlock(&LOCK_active_mi); return 0; } static int show_slave_received_heartbeats(THD *thd, SHOW_VAR *var, char *buff) { - Master_info *mi= NULL; - longlong tmp; - LINT_INIT(tmp); + Master_info *mi; var->type= SHOW_LONGLONG; var->value= buff; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if ((mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_NOTE))) { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_NOTE); - if (mi) - tmp= mi->received_heartbeats; + *((longlong *)buff)= mi->received_heartbeats; + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); - if (mi) - *((longlong *)buff)= tmp; else var->type= SHOW_UNDEF; return 0; @@ -7297,23 +7286,16 @@ static int show_slave_received_heartbeats(THD *thd, SHOW_VAR *var, char *buff) static int show_heartbeat_period(THD *thd, SHOW_VAR *var, char *buff) { Master_info *mi= NULL; - float tmp; - LINT_INIT(tmp); var->type= SHOW_CHAR; var->value= buff; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if ((mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_NOTE))) { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_NOTE); - if (mi) - tmp= mi->heartbeat_period; + sprintf(buff, "%.3f", mi->heartbeat_period); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); - if (mi) - sprintf(buff, "%.3f", tmp); else var->type= SHOW_UNDEF; return 0; diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index 249bf7608e5..7aa84e72bef 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -40,7 +40,9 @@ Master_info::Master_info(LEX_STRING *connection_name_arg, sync_counter(0), heartbeat_period(0), received_heartbeats(0), master_id(0), prev_master_id(0), using_gtid(USE_GTID_NO), events_queued_since_last_gtid(0), - gtid_reconnect_event_skip_count(0), gtid_event_seen(false) + gtid_reconnect_event_skip_count(0), gtid_event_seen(false), + in_start_all_slaves(0), in_stop_all_slaves(0), + users(0), killed(0) { host[0] = 0; user[0] = 0; password[0] = 0; ssl_ca[0]= 0; ssl_capath[0]= 0; ssl_cert[0]= 0; @@ -87,8 +89,27 @@ Master_info::Master_info(LEX_STRING *connection_name_arg, mysql_cond_init(key_master_info_sleep_cond, &sleep_cond, NULL); } + +/** + Wait until no one is using Master_info +*/ + +void Master_info::wait_until_free() +{ + mysql_mutex_lock(&sleep_lock); + killed= 1; + while (users) + mysql_cond_wait(&sleep_cond, &sleep_lock); + mysql_mutex_unlock(&sleep_lock); +} + +/** + Delete master_info +*/ + Master_info::~Master_info() { + wait_until_free(); rpl_filters.delete_element(connection_name.str, connection_name.length, (void (*)(const char*, uchar*)) free_rpl_filter); my_free(connection_name.str); @@ -709,8 +730,13 @@ uchar *get_key_master_info(Master_info *mi, size_t *length, void free_key_master_info(Master_info *mi) { DBUG_ENTER("free_key_master_info"); + /* Ensure that we are not in reset_slave while this is done */ + lock_slave_threads(mi); terminate_slave_threads(mi,SLAVE_FORCE_ALL); + /* We use 2 here instead of 1 just to make it easier when debugging */ + mi->killed= 2; end_master_info(mi); + unlock_slave_threads(mi); delete mi; DBUG_VOID_RETURN; } @@ -867,9 +893,27 @@ Master_info_index::Master_info_index() index_file.file= -1; } + +/** + Free all connection threads + + This is done during early stages of shutdown + to give connection threads and slave threads time + to die before ~Master_info_index is called +*/ + +void Master_info_index::free_connections() +{ + my_hash_reset(&master_info_hash); +} + + +/** + Free all connection threads and free structures +*/ + Master_info_index::~Master_info_index() { - /* This will close connection for all objects in the cache */ my_hash_free(&master_info_hash); end_io_cache(&index_file); if (index_file.file >= 0) @@ -1001,7 +1045,6 @@ bool Master_info_index::init_all_master_info() if (master_info_index->add_master_info(mi, FALSE)) DBUG_RETURN(1); succ_num++; - unlock_slave_threads(mi); if (!opt_skip_slave_start) { @@ -1022,6 +1065,7 @@ bool Master_info_index::init_all_master_info() (int) connection_name.length, connection_name.str); } + unlock_slave_threads(mi); } } @@ -1072,6 +1116,71 @@ bool Master_info_index::write_master_name_to_index_file(LEX_STRING *name, } +/** + Get Master_info for a connection and lock the object from deletion + + @param + connection_name Connection name + warning WARN_LEVEL_NOTE -> Don't print anything + WARN_LEVEL_WARN -> Issue warning if not exists + WARN_LEVEL_ERROR-> Issue error if not exists +*/ + +Master_info *get_master_info(LEX_STRING *connection_name, + Sql_condition::enum_warning_level warning) +{ + Master_info *mi; + DBUG_ENTER("get_master_info"); + + /* Protect against inserts into hash */ + mysql_mutex_lock(&LOCK_active_mi); + /* + The following can only be true during shutdown when slave has been killed + but some other threads are still trying to access slave statistics. + */ + if (unlikely(!master_info_index)) + { + if (warning != Sql_condition::WARN_LEVEL_NOTE) + my_error(WARN_NO_MASTER_INFO, + MYF(warning == Sql_condition::WARN_LEVEL_WARN ? + ME_JUST_WARNING : 0), + (int) connection_name->length, connection_name->str); + mysql_mutex_unlock(&LOCK_active_mi); + DBUG_RETURN(0); + } + if ((mi= master_info_index->get_master_info(connection_name, warning))) + { + /* + We have to use sleep_lock here. If we would use LOCK_active_mi + then we would take locks in wrong order in Master_info::release() + */ + mysql_mutex_lock(&mi->sleep_lock); + mi->users++; + DBUG_PRINT("info",("users: %d", mi->users)); + mysql_mutex_unlock(&mi->sleep_lock); + } + mysql_mutex_unlock(&LOCK_active_mi); + DBUG_RETURN(mi); +} + + +/** + Release master info. + Signals ~Master_info that it's now safe to delete it +*/ + +void Master_info::release() +{ + mysql_mutex_lock(&sleep_lock); + if (!--users && killed) + { + /* Signal ~Master_info that it's ok to now free it */ + mysql_cond_signal(&sleep_cond); + } + mysql_mutex_unlock(&sleep_lock); +} + + /** Get Master_info for a connection @@ -1094,8 +1203,6 @@ Master_info_index::get_master_info(LEX_STRING *connection_name, ("connection_name: '%.*s'", (int) connection_name->length, connection_name->str)); - mysql_mutex_assert_owner(&LOCK_active_mi); - /* Make name lower case for comparison */ res= strmake(buff, connection_name->str, connection_name->length); my_casedn_str(system_charset_info, buff); @@ -1187,105 +1294,123 @@ bool Master_info_index::add_master_info(Master_info *mi, bool write_to_file) atomic */ -bool Master_info_index::remove_master_info(LEX_STRING *name) +bool Master_info_index::remove_master_info(Master_info *mi) { - Master_info* mi; DBUG_ENTER("remove_master_info"); + mysql_mutex_assert_owner(&LOCK_active_mi); - if ((mi= get_master_info(name, Sql_condition::WARN_LEVEL_WARN))) + // Delete Master_info and rewrite others to file + if (!my_hash_delete(&master_info_hash, (uchar*) mi)) { - // Delete Master_info and rewrite others to file - if (!my_hash_delete(&master_info_hash, (uchar*) mi)) + File index_file_nr; + + // Close IO_CACHE and FILE handler fisrt + end_io_cache(&index_file); + my_close(index_file.file, MYF(MY_WME)); + + // Reopen File and truncate it + if ((index_file_nr= my_open(index_file_name, + O_RDWR | O_CREAT | O_TRUNC | O_BINARY , + MYF(MY_WME))) < 0 || + init_io_cache(&index_file, index_file_nr, + IO_SIZE, WRITE_CACHE, + my_seek(index_file_nr,0L,MY_SEEK_END,MYF(0)), + 0, MYF(MY_WME | MY_WAIT_IF_FULL))) { - File index_file_nr; + int error= my_errno; + if (index_file_nr >= 0) + my_close(index_file_nr,MYF(0)); - // Close IO_CACHE and FILE handler fisrt - end_io_cache(&index_file); - my_close(index_file.file, MYF(MY_WME)); - - // Reopen File and truncate it - if ((index_file_nr= my_open(index_file_name, - O_RDWR | O_CREAT | O_TRUNC | O_BINARY , - MYF(MY_WME))) < 0 || - init_io_cache(&index_file, index_file_nr, - IO_SIZE, WRITE_CACHE, - my_seek(index_file_nr,0L,MY_SEEK_END,MYF(0)), - 0, MYF(MY_WME | MY_WAIT_IF_FULL))) - { - int error= my_errno; - if (index_file_nr >= 0) - my_close(index_file_nr,MYF(0)); - - sql_print_error("Create of Master Info Index file '%s' failed with " - "error: %M", - index_file_name, error); - DBUG_RETURN(TRUE); - } - - // Rewrite Master_info.index - for (uint i= 0; i< master_info_hash.records; ++i) - { - Master_info *tmp_mi; - tmp_mi= (Master_info *) my_hash_element(&master_info_hash, i); - write_master_name_to_index_file(&tmp_mi->connection_name, 0); - } - my_sync(index_file_nr, MYF(MY_WME)); + sql_print_error("Create of Master Info Index file '%s' failed with " + "error: %M", + index_file_name, error); + DBUG_RETURN(TRUE); } + + // Rewrite Master_info.index + for (uint i= 0; i< master_info_hash.records; ++i) + { + Master_info *tmp_mi; + tmp_mi= (Master_info *) my_hash_element(&master_info_hash, i); + write_master_name_to_index_file(&tmp_mi->connection_name, 0); + } + if (my_sync(index_file_nr, MYF(MY_WME))) + DBUG_RETURN(TRUE); } DBUG_RETURN(FALSE); } /** - Master_info_index::give_error_if_slave_running() + give_error_if_slave_running() + + @param + already_locked 0 if we need to lock, 1 if we have LOCK_active_mi_locked @return TRUE If some slave is running. An error is printed FALSE No slave is running */ -bool Master_info_index::give_error_if_slave_running() +bool give_error_if_slave_running(bool already_locked) { + bool ret= 0; DBUG_ENTER("give_error_if_slave_running"); - mysql_mutex_assert_owner(&LOCK_active_mi); - for (uint i= 0; i< master_info_hash.records; ++i) + if (!already_locked) + mysql_mutex_lock(&LOCK_active_mi); + if (!master_info_index) { - Master_info *mi; - mi= (Master_info *) my_hash_element(&master_info_hash, i); - if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) + my_error(ER_SERVER_SHUTDOWN, MYF(0)); + ret= 1; + } + else + { + HASH *hash= &master_info_index->master_info_hash; + for (uint i= 0; i< hash->records; ++i) { - my_error(ER_SLAVE_MUST_STOP, MYF(0), (int) mi->connection_name.length, - mi->connection_name.str); - DBUG_RETURN(TRUE); + Master_info *mi; + mi= (Master_info *) my_hash_element(hash, i); + if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) + { + my_error(ER_SLAVE_MUST_STOP, MYF(0), (int) mi->connection_name.length, + mi->connection_name.str); + ret= 1; + break; + } } } - DBUG_RETURN(FALSE); + if (!already_locked) + mysql_mutex_unlock(&LOCK_active_mi); + DBUG_RETURN(ret); } /** - Master_info_index::any_slave_sql_running() - - The LOCK_active_mi must be held while calling this function. + any_slave_sql_running() @return 0 No Slave SQL thread is running # Number of slave SQL thread running */ -uint Master_info_index::any_slave_sql_running() +uint any_slave_sql_running() { uint count= 0; DBUG_ENTER("any_slave_sql_running"); - mysql_mutex_assert_owner(&LOCK_active_mi); - for (uint i= 0; i< master_info_hash.records; ++i) + mysql_mutex_lock(&LOCK_active_mi); + if (likely(master_info_index)) // Not shutdown { - Master_info *mi= (Master_info *)my_hash_element(&master_info_hash, i); - if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) - count++; + HASH *hash= &master_info_index->master_info_hash; + for (uint i= 0; i< hash->records; ++i) + { + Master_info *mi= (Master_info *)my_hash_element(hash, i); + if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) + count++; + } } + mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(count); } @@ -1298,15 +1423,25 @@ uint Master_info_index::any_slave_sql_running() @return TRUE Error FALSE Everything ok. + + This code is written so that we don't keep LOCK_active_mi active + while we are starting a slave. */ bool Master_info_index::start_all_slaves(THD *thd) { bool result= FALSE; - DBUG_ENTER("warn_if_slave_running"); + DBUG_ENTER("start_all_slaves"); mysql_mutex_assert_owner(&LOCK_active_mi); - for (uint i= 0; i< master_info_hash.records; ++i) + for (uint i= 0; i< master_info_hash.records; i++) + { + Master_info *mi; + mi= (Master_info *) my_hash_element(&master_info_hash, i); + mi->in_start_all_slaves= 0; + } + + for (uint i= 0; i< master_info_hash.records; ) { int error; Master_info *mi; @@ -1316,25 +1451,40 @@ bool Master_info_index::start_all_slaves(THD *thd) Try to start all slaves that are configured (host is defined) and are not already running */ - if ((mi->slave_running == MYSQL_SLAVE_NOT_RUN || - !mi->rli.slave_running) && *mi->host) + if (!((mi->slave_running == MYSQL_SLAVE_NOT_RUN || + !mi->rli.slave_running) && *mi->host) || + mi->in_start_all_slaves) { - if ((error= start_slave(thd, mi, 1))) - { - my_error(ER_CANT_START_STOP_SLAVE, MYF(0), - "START", - (int) mi->connection_name.length, - mi->connection_name.str); - result= 1; - if (error < 0) // fatal error - break; - } - else - push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, - ER_SLAVE_STARTED, ER(ER_SLAVE_STARTED), - (int) mi->connection_name.length, - mi->connection_name.str); + i++; + continue; } + mi->in_start_all_slaves= 1; + + mysql_mutex_lock(&mi->sleep_lock); + mi->users++; // Mark used + mysql_mutex_unlock(&mi->sleep_lock); + mysql_mutex_unlock(&LOCK_active_mi); + error= start_slave(thd, mi, 1); + mi->release(); + mysql_mutex_lock(&LOCK_active_mi); + if (error) + { + my_error(ER_CANT_START_STOP_SLAVE, MYF(0), + "START", + (int) mi->connection_name.length, + mi->connection_name.str); + result= 1; + if (error < 0) // fatal error + break; + } + else + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, + ER_SLAVE_STARTED, ER(ER_SLAVE_STARTED), + (int) mi->connection_name.length, + mi->connection_name.str); + /* Restart from first element as master_info_hash may have changed */ + i= 0; + continue; } DBUG_RETURN(result); } @@ -1348,38 +1498,63 @@ bool Master_info_index::start_all_slaves(THD *thd) @return TRUE Error FALSE Everything ok. + + This code is written so that we don't keep LOCK_active_mi active + while we are stopping a slave. */ bool Master_info_index::stop_all_slaves(THD *thd) { bool result= FALSE; - DBUG_ENTER("warn_if_slave_running"); + DBUG_ENTER("stop_all_slaves"); mysql_mutex_assert_owner(&LOCK_active_mi); - for (uint i= 0; i< master_info_hash.records; ++i) + for (uint i= 0; i< master_info_hash.records; i++) + { + Master_info *mi; + mi= (Master_info *) my_hash_element(&master_info_hash, i); + mi->in_stop_all_slaves= 0; + } + + for (uint i= 0; i< master_info_hash.records ;) { int error; Master_info *mi; mi= (Master_info *) my_hash_element(&master_info_hash, i); - if ((mi->slave_running != MYSQL_SLAVE_NOT_RUN || - mi->rli.slave_running)) + if (!(mi->slave_running != MYSQL_SLAVE_NOT_RUN || + mi->rli.slave_running) || + mi->in_stop_all_slaves) { - if ((error= stop_slave(thd, mi, 1))) - { - my_error(ER_CANT_START_STOP_SLAVE, MYF(0), - "STOP", - (int) mi->connection_name.length, - mi->connection_name.str); - result= 1; - if (error < 0) // Fatal error - break; - } - else - push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, - ER_SLAVE_STOPPED, ER(ER_SLAVE_STOPPED), - (int) mi->connection_name.length, - mi->connection_name.str); + i++; + continue; } + mi->in_stop_all_slaves= 1; // Protection for loops + + mysql_mutex_lock(&mi->sleep_lock); + mi->users++; // Mark used + mysql_mutex_unlock(&mi->sleep_lock); + mysql_mutex_unlock(&LOCK_active_mi); + error= stop_slave(thd, mi, 1); + mi->release(); + mysql_mutex_lock(&LOCK_active_mi); + if (error) + { + my_error(ER_CANT_START_STOP_SLAVE, MYF(0), + "STOP", + (int) mi->connection_name.length, + mi->connection_name.str); + result= 1; + if (error < 0) // Fatal error + break; + } + else + push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, + ER_SLAVE_STOPPED, ER(ER_SLAVE_STOPPED), + (int) mi->connection_name.length, + mi->connection_name.str); + /* Restart from first element as master_info_hash may have changed */ + i= 0; + continue; } DBUG_RETURN(result); } diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index e58df0150f5..32591c56caa 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -79,6 +79,8 @@ class Master_info : public Slave_reporting_capability { return opt_slave_parallel_threads > 0; } + void release(); + void wait_until_free(); /* the variables below are needed because we can change masters on the fly */ char master_log_name[FN_REFLEN+6]; /* Room for multi-*/ @@ -182,7 +184,11 @@ class Master_info : public Slave_reporting_capability uint64 gtid_reconnect_event_skip_count; /* gtid_event_seen is false until we receive first GTID event from master. */ bool gtid_event_seen; + bool in_start_all_slaves, in_stop_all_slaves; + uint users; /* Active user for object */ + uint killed; }; + int init_master_info(Master_info* mi, const char* master_info_fname, const char* slave_info_fname, bool abort_if_no_master_info_file, @@ -218,13 +224,12 @@ public: bool check_duplicate_master_info(LEX_STRING *connection_name, const char *host, uint port); bool add_master_info(Master_info *mi, bool write_to_file); - bool remove_master_info(LEX_STRING *connection_name); + bool remove_master_info(Master_info *mi); Master_info *get_master_info(LEX_STRING *connection_name, Sql_condition::enum_warning_level warning); - bool give_error_if_slave_running(); - uint any_slave_sql_running(); bool start_all_slaves(THD *thd); bool stop_all_slaves(THD *thd); + void free_connections(); }; @@ -237,6 +242,8 @@ public: }; +Master_info *get_master_info(LEX_STRING *connection_name, + Sql_condition::enum_warning_level warning); bool check_master_connection_name(LEX_STRING *name); void create_logfile_name_with_suffix(char *res_file_name, size_t length, const char *info_file, @@ -246,7 +253,8 @@ void create_logfile_name_with_suffix(char *res_file_name, size_t length, uchar *get_key_master_info(Master_info *mi, size_t *length, my_bool not_used __attribute__((unused))); void free_key_master_info(Master_info *mi); - +uint any_slave_sql_running(); +bool give_error_if_slave_running(bool already_lock); #endif /* HAVE_REPLICATION */ #endif /* RPL_MI_H */ diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 23f61a82a90..ff2c7d68467 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -1312,6 +1312,29 @@ handle_rpl_parallel_thread(void *arg) } if (!in_event_group) { + /* If we are in a FLUSH TABLES FOR READ LOCK, wait for it */ + while (rpt->current_entry && rpt->pause_for_ftwrl) + { + /* + We are currently in the delicate process of pausing parallel + replication while FLUSH TABLES WITH READ LOCK is starting. We must + not de-allocate the thread (setting rpt->current_owner= NULL) until + rpl_unpause_after_ftwrl() has woken us up. + */ + rpl_parallel_entry *e= rpt->current_entry; + /* + Ensure that we will unblock rpl_pause_for_ftrwl() + e->pause_sub_id may be LONGLONG_MAX if rpt->current_entry has changed + */ + DBUG_ASSERT(e->pause_sub_id == (uint64)ULONGLONG_MAX || + e->last_committed_sub_id >= e->pause_sub_id); + mysql_mutex_lock(&e->LOCK_parallel_entry); + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); + if (rpt->pause_for_ftwrl) + mysql_cond_wait(&e->COND_parallel_entry, &e->LOCK_parallel_entry); + mysql_mutex_unlock(&e->LOCK_parallel_entry); + mysql_mutex_lock(&rpt->LOCK_rpl_thread); + } rpt->current_owner= NULL; /* Tell wait_for_done() that we are done, if it is waiting. */ if (likely(rpt->current_entry) && @@ -1369,6 +1392,28 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, if ((res= pool_mark_busy(pool, current_thd))) return res; + /* Protect against parallel pool resizes */ + if (pool->count == new_count) + { + pool_mark_not_busy(pool); + return 0; + } + + /* + If we are about to delete pool, do an extra check that there are no new + slave threads running since we marked pool busy + */ + if (!new_count) + { + if (any_slave_sql_running()) + { + DBUG_PRINT("warning", + ("SQL threads running while trying to reset parallel pool")); + pool_mark_not_busy(pool); + return 0; // Ok to not resize pool + } + } + /* Allocate the new list of threads up-front. That way, if we fail half-way, we only need to free whatever we managed @@ -1382,7 +1427,7 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, { my_error(ER_OUTOFMEMORY, MYF(0), (int(new_count*sizeof(*new_list) + new_count*sizeof(*rpt_array)))); - goto err;; + goto err; } for (i= 0; i < new_count; ++i) @@ -1503,6 +1548,20 @@ err: return 1; } +/* + Deactivate the parallel replication thread pool, if there are now no more + SQL threads running. +*/ + +int rpl_parallel_resize_pool_if_no_slaves(void) +{ + /* master_info_index is set to NULL on shutdown */ + if (opt_slave_parallel_threads > 0 && !any_slave_sql_running() && + master_info_index) + return rpl_parallel_inactivate_pool(&global_rpl_thread_pool); + return 0; +} + int rpl_parallel_activate_pool(rpl_parallel_thread_pool *pool) @@ -1814,6 +1873,7 @@ rpl_parallel_thread_pool::get_thread(rpl_parallel_thread **owner, { rpl_parallel_thread *rpt; + DBUG_ASSERT(count > 0); mysql_mutex_lock(&LOCK_rpl_thread_pool); while (unlikely(busy) || !(rpt= free_list)) mysql_cond_wait(&COND_rpl_thread_pool, &LOCK_rpl_thread_pool); diff --git a/sql/rpl_parallel.h b/sql/rpl_parallel.h index a02c1af3b3e..0d7cd4f2e9b 100644 --- a/sql/rpl_parallel.h +++ b/sql/rpl_parallel.h @@ -342,6 +342,7 @@ struct rpl_parallel { extern struct rpl_parallel_thread_pool global_rpl_thread_pool; +extern int rpl_parallel_resize_pool_if_no_slaves(void); extern int rpl_parallel_activate_pool(rpl_parallel_thread_pool *pool); extern int rpl_parallel_inactivate_pool(rpl_parallel_thread_pool *pool); extern bool process_gtid_for_restart_pos(Relay_log_info *rli, rpl_gtid *gtid); diff --git a/sql/slave.cc b/sql/slave.cc index c4817ef4794..a2b8439b4b4 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -614,6 +614,7 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) if (!mi->inited) DBUG_RETURN(0); /* successfully do nothing */ int error,force_all = (thread_mask & SLAVE_FORCE_ALL); + int retval= 0; mysql_mutex_t *sql_lock = &mi->rli.run_lock, *io_lock = &mi->run_lock; mysql_mutex_t *log_lock= mi->rli.relay_log.get_log_lock(); @@ -633,24 +634,19 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) skip_lock)) && !force_all) DBUG_RETURN(error); + retval= error; mysql_mutex_lock(log_lock); DBUG_PRINT("info",("Flushing relay-log info file.")); if (current_thd) THD_STAGE_INFO(current_thd, stage_flushing_relay_log_info_file); - if (flush_relay_log_info(&mi->rli)) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); - - if (my_sync(mi->rli.info_fd, MYF(MY_WME))) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); + if (flush_relay_log_info(&mi->rli) || + my_sync(mi->rli.info_fd, MYF(MY_WME))) + retval= ER_ERROR_DURING_FLUSH_LOGS; mysql_mutex_unlock(log_lock); } - if (opt_slave_parallel_threads > 0 && - master_info_index &&// master_info_index is set to NULL on server shutdown - !master_info_index->any_slave_sql_running()) - rpl_parallel_inactivate_pool(&global_rpl_thread_pool); if (thread_mask & (SLAVE_IO|SLAVE_FORCE_ALL)) { DBUG_PRINT("info",("Terminating IO thread")); @@ -661,25 +657,26 @@ int terminate_slave_threads(Master_info* mi,int thread_mask,bool skip_lock) skip_lock)) && !force_all) DBUG_RETURN(error); + if (!retval) + retval= error; mysql_mutex_lock(log_lock); DBUG_PRINT("info",("Flushing relay log and master info file.")); if (current_thd) THD_STAGE_INFO(current_thd, stage_flushing_relay_log_and_master_info_repository); - if (flush_master_info(mi, TRUE, FALSE)) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); - + if (likely(mi->fd >= 0)) + { + if (flush_master_info(mi, TRUE, FALSE) || my_sync(mi->fd, MYF(MY_WME))) + retval= ER_ERROR_DURING_FLUSH_LOGS; + } if (mi->rli.relay_log.is_open() && my_sync(mi->rli.relay_log.get_log_file()->file, MYF(MY_WME))) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); - - if (my_sync(mi->fd, MYF(MY_WME))) - DBUG_RETURN(ER_ERROR_DURING_FLUSH_LOGS); + retval= ER_ERROR_DURING_FLUSH_LOGS; mysql_mutex_unlock(log_lock); } - DBUG_RETURN(0); + DBUG_RETURN(retval); } @@ -956,10 +953,7 @@ int start_slave_threads(bool need_slave_mutex, bool wait_for_start, mi); if (!error && (thread_mask & SLAVE_SQL)) { - if (opt_slave_parallel_threads > 0) - error= rpl_parallel_activate_pool(&global_rpl_thread_pool); - if (!error) - error= start_slave_thread( + error= start_slave_thread( #ifdef HAVE_PSI_INTERFACE key_thread_slave_sql, #endif @@ -975,10 +969,18 @@ int start_slave_threads(bool need_slave_mutex, bool wait_for_start, /* - Release slave threads at time of executing shutdown. + Kill slaves preparing for shutdown +*/ - SYNOPSIS - end_slave() +void slave_prepare_for_shutdown() +{ + mysql_mutex_lock(&LOCK_active_mi); + master_info_index->free_connections(); + mysql_mutex_unlock(&LOCK_active_mi); +} + +/* + Release slave threads at time of executing shutdown. */ void end_slave() @@ -996,7 +998,10 @@ void end_slave() startup parameter to the server was wrong. */ mysql_mutex_lock(&LOCK_active_mi); - /* This will call terminate_slave_threads() on all connections */ + /* + master_info_index should not have any threads anymore as they where + killed as part of slave_prepare_for_shutdown() + */ delete master_info_index; master_info_index= 0; active_mi= 0; @@ -2657,7 +2662,9 @@ static bool send_show_master_info_data(THD *thd, Master_info *mi, bool full, mysql_mutex_lock(&mi->data_lock); mysql_mutex_lock(&mi->rli.data_lock); + /* err_lock is to protect mi->last_error() */ mysql_mutex_lock(&mi->err_lock); + /* err_lock is to protect mi->rli.last_error() */ mysql_mutex_lock(&mi->rli.err_lock); protocol->store(mi->host, &my_charset_bin); protocol->store(mi->user, &my_charset_bin); @@ -4493,6 +4500,16 @@ pthread_handler_t handle_slave_sql(void *arg) rli->slave_running= MYSQL_SLAVE_RUN_NOT_CONNECT; pthread_detach_this_thread(); + + if (opt_slave_parallel_threads > 0 && + rpl_parallel_activate_pool(&global_rpl_thread_pool)) + { + mysql_cond_broadcast(&rli->start_cond); + rli->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, + "Failed during parallel slave pool activation"); + goto err_during_init; + } + if (init_slave_thread(thd, mi, SLAVE_THD_SQL)) { /* @@ -4862,17 +4879,7 @@ err_during_init: DBUG_EXECUTE_IF("simulate_slave_delay_at_terminate_bug38694", sleep(5);); mysql_mutex_unlock(&rli->run_lock); // tell the world we are done - /* - Deactivate the parallel replication thread pool, if there are now no more - SQL threads running. Do this here, when we have released all locks, but - while our THD (and current_thd) is still valid. - */ - mysql_mutex_lock(&LOCK_active_mi); - if (opt_slave_parallel_threads > 0 && - master_info_index &&// master_info_index is set to NULL on server shutdown - !master_info_index->any_slave_sql_running()) - rpl_parallel_inactivate_pool(&global_rpl_thread_pool); - mysql_mutex_unlock(&LOCK_active_mi); + rpl_parallel_resize_pool_if_no_slaves(); mysql_mutex_lock(&LOCK_thread_count); delete thd; diff --git a/sql/slave.h b/sql/slave.h index c6b78b96aca..cfff79da0e4 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -213,6 +213,7 @@ bool rpl_master_erroneous_autoinc(THD* thd); const char *print_slave_db_safe(const char *db); void skip_load_data_infile(NET* net); +void slave_prepare_for_shutdown(); void end_slave(); /* release slave threads */ void close_active_mi(); /* clean up slave threads data */ void clear_until_condition(Relay_log_info* rli); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 03bad46ae0e..97d3ce24c25 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2207,7 +2207,7 @@ err: int mysql_execute_command(THD *thd) { - int res= FALSE; + int res= 0; int up_result= 0; LEX *lex= thd->lex; /* first SELECT_LEX (have special meaning for many of non-SELECTcommands) */ @@ -2702,10 +2702,17 @@ case SQLCOM_PREPARE: if (check_global_access(thd, SUPER_ACL)) goto error; + /* + In this code it's ok to use LOCK_active_mi as we are adding new things + into master_info_index + */ mysql_mutex_lock(&LOCK_active_mi); - if (!master_info_index) + { + mysql_mutex_unlock(&LOCK_active_mi); + my_error(ER_SERVER_SHUTDOWN, MYF(0)); goto error; + } mi= master_info_index->get_master_info(&lex_mi->connection_name, Sql_condition::WARN_LEVEL_NOTE); @@ -2734,7 +2741,7 @@ case SQLCOM_PREPARE: If new master was not added, we still need to free mi. */ if (master_info_added) - master_info_index->remove_master_info(&lex_mi->connection_name); + master_info_index->remove_master_info(mi); else delete mi; } @@ -2752,22 +2759,24 @@ case SQLCOM_PREPARE: /* Accept one of two privileges */ if (check_global_access(thd, SUPER_ACL | REPL_CLIENT_ACL)) goto error; - mysql_mutex_lock(&LOCK_active_mi); if (lex->verbose) + { + mysql_mutex_lock(&LOCK_active_mi); res= show_all_master_info(thd); + mysql_mutex_unlock(&LOCK_active_mi); + } else { LEX_MASTER_INFO *lex_mi= &thd->lex->mi; Master_info *mi; - mi= master_info_index->get_master_info(&lex_mi->connection_name, - Sql_condition::WARN_LEVEL_ERROR); - if (mi != NULL) + if ((mi= get_master_info(&lex_mi->connection_name, + Sql_condition::WARN_LEVEL_ERROR))) { res= show_master_info(thd, mi, 0); + mi->release(); } } - mysql_mutex_unlock(&LOCK_active_mi); break; } case SQLCOM_SHOW_MASTER_STAT: @@ -3091,22 +3100,23 @@ end_with_restore_list: load_error= rpl_load_gtid_slave_state(thd); - mysql_mutex_lock(&LOCK_active_mi); - - if ((mi= (master_info_index-> - get_master_info(&lex_mi->connection_name, - Sql_condition::WARN_LEVEL_ERROR)))) + /* + We don't need to ensure that only one user is using master_info + as start_slave is protected against simultaneous usage + */ + if ((mi= get_master_info(&lex_mi->connection_name, + Sql_condition::WARN_LEVEL_ERROR))) { if (load_error) { /* - We cannot start a slave using GTID if we cannot load the GTID position - from the mysql.gtid_slave_pos table. But we can allow non-GTID - replication (useful eg. during upgrade). + We cannot start a slave using GTID if we cannot load the + GTID position from the mysql.gtid_slave_pos table. But we + can allow non-GTID replication (useful eg. during upgrade). */ if (mi->using_gtid != Master_info::USE_GTID_NO) { - mysql_mutex_unlock(&LOCK_active_mi); + mi->release(); break; } else @@ -3114,8 +3124,8 @@ end_with_restore_list: } if (!start_slave(thd, mi, 1 /* net report*/)) my_ok(thd); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); break; } case SQLCOM_SLAVE_STOP: @@ -3145,13 +3155,17 @@ end_with_restore_list: } lex_mi= &thd->lex->mi; - mysql_mutex_lock(&LOCK_active_mi); - if ((mi= (master_info_index-> - get_master_info(&lex_mi->connection_name, - Sql_condition::WARN_LEVEL_ERROR)))) - if (!stop_slave(thd, mi, 1/* net report*/)) + if ((mi= get_master_info(&lex_mi->connection_name, + Sql_condition::WARN_LEVEL_ERROR))) + { + if (stop_slave(thd, mi, 1/* net report*/)) + res= 1; + mi->release(); + if (rpl_parallel_resize_pool_if_no_slaves()) + res= 1; + if (!res) my_ok(thd); - mysql_mutex_unlock(&LOCK_active_mi); + } break; } case SQLCOM_SLAVE_ALL_START: @@ -4317,11 +4331,13 @@ end_with_restore_list: reload_acl_and_cache binlog interactions failed */ res= 1; - } + } if (!res) my_ok(thd); } + else + res= 1; // reload_acl_and_cache failed #ifdef HAVE_REPLICATION if (lex->type & REFRESH_READ_LOCK) rpl_unpause_after_ftwrl(thd); diff --git a/sql/sql_reload.cc b/sql/sql_reload.cc index bcf2585ca2a..f9c5757f721 100644 --- a/sql/sql_reload.cc +++ b/sql/sql_reload.cc @@ -174,24 +174,20 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options, slave is not likely to have the same connection names. */ tmp_write_to_binlog= 0; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if (!(mi= (get_master_info(&connection_name, + Sql_condition::WARN_LEVEL_ERROR)))) { - if (!(mi= (master_info_index-> - get_master_info(&connection_name, - Sql_condition::WARN_LEVEL_ERROR)))) - { - result= 1; - } - else - { - mysql_mutex_lock(&mi->data_lock); - if (rotate_relay_log(mi)) - *write_to_binlog= -1; - mysql_mutex_unlock(&mi->data_lock); - } + result= 1; + } + else + { + mysql_mutex_lock(&mi->data_lock); + if (rotate_relay_log(mi)) + *write_to_binlog= -1; + mysql_mutex_unlock(&mi->data_lock); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); #endif } #ifdef HAVE_QUERY_CACHE @@ -349,27 +345,33 @@ bool reload_acl_and_cache(THD *thd, unsigned long long options, LEX_MASTER_INFO* lex_mi= &thd->lex->mi; Master_info *mi; tmp_write_to_binlog= 0; - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index) + + if (!(mi= get_master_info(&lex_mi->connection_name, + Sql_condition::WARN_LEVEL_ERROR))) { - if (!(mi= (master_info_index-> - get_master_info(&lex_mi->connection_name, - Sql_condition::WARN_LEVEL_ERROR)))) - { - result= 1; - } - else if (reset_slave(thd, mi)) + result= 1; + } + else + { + /* The following will fail if slave is running */ + if (reset_slave(thd, mi)) { + mi->release(); /* NOTE: my_error() has been already called by reset_slave(). */ result= 1; } else if (mi->connection_name.length && thd->lex->reset_slave_info.all) { /* If not default connection and 'all' is used */ - master_info_index->remove_master_info(&mi->connection_name); + mi->release(); + mysql_mutex_lock(&LOCK_active_mi); + if (master_info_index->remove_master_info(mi)) + result= 1; + mysql_mutex_unlock(&LOCK_active_mi); } + else + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); } #endif if (options & REFRESH_USER_RESOURCES) diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 4505ad69880..29980dbbbf1 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -3265,8 +3265,8 @@ bool change_master(THD* thd, Master_info* mi, bool *master_info_added) LEX_MASTER_INFO* lex_mi= &thd->lex->mi; DBUG_ENTER("change_master"); - mysql_mutex_assert_owner(&LOCK_active_mi); DBUG_ASSERT(master_info_index); + mysql_mutex_assert_owner(&LOCK_active_mi); *master_info_added= false; /* @@ -3646,7 +3646,6 @@ bool mysql_show_binlog_events(THD* thd) int old_max_allowed_packet= thd->variables.max_allowed_packet; Master_info *mi= 0; LOG_INFO linfo; - DBUG_ENTER("mysql_show_binlog_events"); Log_event::init_show_field_list(&field_list); @@ -3674,13 +3673,9 @@ bool mysql_show_binlog_events(THD* thd) } else /* showing relay log contents */ { - mysql_mutex_lock(&LOCK_active_mi); - if (!master_info_index || - !(mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_ERROR))) + if (!(mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_ERROR))) { - mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(TRUE); } binary_log= &(mi->rli.relay_log); @@ -3700,7 +3695,7 @@ bool mysql_show_binlog_events(THD* thd) if (mi) { /* We can unlock the mutex as we have a lock on the file */ - mysql_mutex_unlock(&LOCK_active_mi); + mi->release(); mi= 0; } @@ -3722,6 +3717,7 @@ bool mysql_show_binlog_events(THD* thd) goto err; } + /* These locks is here to enable syncronization with log_in_use() */ mysql_mutex_lock(&LOCK_thread_count); thd->current_linfo = &linfo; mysql_mutex_unlock(&LOCK_thread_count); @@ -3799,7 +3795,7 @@ bool mysql_show_binlog_events(THD* thd) mysql_mutex_unlock(log_lock); } else if (mi) - mysql_mutex_unlock(&LOCK_active_mi); + mi->release(); // Check that linfo is still on the function scope. DEBUG_SYNC(thd, "after_show_binlog_events"); @@ -3820,8 +3816,9 @@ err: else my_eof(thd); + /* These locks is here to enable syncronization with log_in_use() */ mysql_mutex_lock(&LOCK_thread_count); - thd->current_linfo = 0; + thd->current_linfo= 0; mysql_mutex_unlock(&LOCK_thread_count); thd->variables.max_allowed_packet= old_max_allowed_packet; DBUG_RETURN(ret); diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 4adc19231bc..5ff79a2f235 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1527,7 +1527,6 @@ bool Sys_var_gtid_slave_pos::do_check(THD *thd, set_var *var) { String str, *res; - bool running; DBUG_ASSERT(var->type == OPT_GLOBAL); @@ -1538,11 +1537,7 @@ Sys_var_gtid_slave_pos::do_check(THD *thd, set_var *var) return true; } - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); - if (running) + if (give_error_if_slave_running(0)) return true; if (!(res= var->value->val_str(&str))) return true; @@ -1580,7 +1575,7 @@ Sys_var_gtid_slave_pos::global_update(THD *thd, set_var *var) mysql_mutex_unlock(&LOCK_global_system_variables); mysql_mutex_lock(&LOCK_active_mi); - if (!master_info_index || master_info_index->give_error_if_slave_running()) + if (give_error_if_slave_running(1)) err= true; else err= rpl_gtid_pos_update(thd, var->save_result.string_value.str, @@ -1766,16 +1761,7 @@ Sys_var_last_gtid::session_value_ptr(THD *thd, LEX_STRING *base) static bool check_slave_parallel_threads(sys_var *self, THD *thd, set_var *var) { - bool running; - - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); - if (running) - return true; - - return false; + return give_error_if_slave_running(0); } static bool @@ -1784,10 +1770,7 @@ fix_slave_parallel_threads(sys_var *self, THD *thd, enum_var_type type) bool err; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - err= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); + err= give_error_if_slave_running(0); mysql_mutex_lock(&LOCK_global_system_variables); return err; @@ -1810,16 +1793,7 @@ static Sys_var_ulong Sys_slave_parallel_threads( static bool check_slave_domain_parallel_threads(sys_var *self, THD *thd, set_var *var) { - bool running; - - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); - if (running) - return true; - - return false; + return give_error_if_slave_running(0); } static bool @@ -1828,13 +1802,10 @@ fix_slave_domain_parallel_threads(sys_var *self, THD *thd, enum_var_type type) bool running; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); + running= give_error_if_slave_running(0); mysql_mutex_lock(&LOCK_global_system_variables); - return running ? true : false; + return running; } @@ -1865,16 +1836,7 @@ static Sys_var_ulong Sys_slave_parallel_max_queued( static bool check_gtid_ignore_duplicates(sys_var *self, THD *thd, set_var *var) { - bool running; - - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); - if (running) - return true; - - return false; + return give_error_if_slave_running(0); } static bool @@ -1883,13 +1845,10 @@ fix_gtid_ignore_duplicates(sys_var *self, THD *thd, enum_var_type type) bool running; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - running= (!master_info_index || - master_info_index->give_error_if_slave_running()); - mysql_mutex_unlock(&LOCK_active_mi); + running= give_error_if_slave_running(0); mysql_mutex_lock(&LOCK_global_system_variables); - return running ? true : false; + return running; } @@ -2837,10 +2796,8 @@ Sys_var_replicate_events_marked_for_skip::global_update(THD *thd, set_var *var) DBUG_ENTER("Sys_var_replicate_events_marked_for_skip::global_update"); mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - if (master_info_index && !master_info_index->give_error_if_slave_running()) + if (!give_error_if_slave_running(0)) result= Sys_var_enum::global_update(thd, var); - mysql_mutex_unlock(&LOCK_active_mi); mysql_mutex_lock(&LOCK_global_system_variables); DBUG_RETURN(result); } @@ -4112,19 +4069,16 @@ bool Sys_var_rpl_filter::global_update(THD *thd, set_var *var) Master_info *mi; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); if (!var->base.length) // no base name { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_ERROR); + mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_ERROR); } else // has base name { - mi= master_info_index-> - get_master_info(&var->base, - Sql_condition::WARN_LEVEL_WARN); + mi= get_master_info(&var->base, + Sql_condition::WARN_LEVEL_WARN); } if (mi) @@ -4132,17 +4086,17 @@ bool Sys_var_rpl_filter::global_update(THD *thd, set_var *var) if (mi->rli.slave_running) { my_error(ER_SLAVE_MUST_STOP, MYF(0), - mi->connection_name.length, - mi->connection_name.str); + mi->connection_name.length, + mi->connection_name.str); result= true; } else { result= set_filter_value(var->save_result.string_value.str, mi); } + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); mysql_mutex_lock(&LOCK_global_system_variables); return result; } @@ -4150,8 +4104,10 @@ bool Sys_var_rpl_filter::global_update(THD *thd, set_var *var) bool Sys_var_rpl_filter::set_filter_value(const char *value, Master_info *mi) { bool status= true; - Rpl_filter* rpl_filter= mi ? mi->rpl_filter : global_rpl_filter; + Rpl_filter* rpl_filter= mi->rpl_filter; + /* Proctect against other threads */ + mysql_mutex_lock(&LOCK_active_mi); switch (opt_id) { case OPT_REPLICATE_DO_DB: status= rpl_filter->set_do_db(value); @@ -4172,7 +4128,7 @@ bool Sys_var_rpl_filter::set_filter_value(const char *value, Master_info *mi) status= rpl_filter->set_wild_ignore_table(value); break; } - + mysql_mutex_unlock(&LOCK_active_mi); return status; } @@ -4185,29 +4141,24 @@ uchar *Sys_var_rpl_filter::global_value_ptr(THD *thd, LEX_STRING *base) Rpl_filter *rpl_filter; mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); if (!base->length) // no base name { - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_ERROR); + mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_ERROR); } else // has base name - { - mi= master_info_index-> - get_master_info(base, - Sql_condition::WARN_LEVEL_WARN); - } - mysql_mutex_lock(&LOCK_global_system_variables); + mi= get_master_info(base, Sql_condition::WARN_LEVEL_WARN); if (!mi) { - mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_global_system_variables); return 0; } + rpl_filter= mi->rpl_filter; tmp.length(0); + mysql_mutex_lock(&LOCK_active_mi); switch (opt_id) { case OPT_REPLICATE_DO_DB: rpl_filter->get_do_db(&tmp); @@ -4228,9 +4179,12 @@ uchar *Sys_var_rpl_filter::global_value_ptr(THD *thd, LEX_STRING *base) rpl_filter->get_wild_ignore_table(&tmp); break; } + mysql_mutex_unlock(&LOCK_active_mi); + mysql_mutex_lock(&LOCK_global_system_variables); + + mi->release(); ret= (uchar *) thd->strmake(tmp.ptr(), tmp.length()); - mysql_mutex_unlock(&LOCK_active_mi); return ret; } @@ -4301,17 +4255,12 @@ get_master_info_ulonglong_value(THD *thd, ptrdiff_t offset) Master_info *mi; ulonglong res= 0; // Default value mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_WARN); - if (mi) + if ((mi= get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_WARN))) { - mysql_mutex_lock(&mi->rli.data_lock); res= *((ulonglong*) (((uchar*) mi) + master_info_offset)); - mysql_mutex_unlock(&mi->rli.data_lock); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); mysql_mutex_lock(&LOCK_global_system_variables); return res; } @@ -4326,19 +4275,16 @@ bool update_multi_source_variable(sys_var *self_var, THD *thd, if (type == OPT_GLOBAL) mysql_mutex_unlock(&LOCK_global_system_variables); - mysql_mutex_lock(&LOCK_active_mi); - mi= master_info_index-> - get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_ERROR); - if (mi) + if ((mi= (get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_ERROR)))) { mysql_mutex_lock(&mi->rli.run_lock); mysql_mutex_lock(&mi->rli.data_lock); result= self->update_variable(thd, mi); mysql_mutex_unlock(&mi->rli.data_lock); mysql_mutex_unlock(&mi->rli.run_lock); + mi->release(); } - mysql_mutex_unlock(&LOCK_active_mi); if (type == OPT_GLOBAL) mysql_mutex_lock(&LOCK_global_system_variables); return result; From c5e25c8b40e8c75d690dbc0e441abbefdd3cdcf4 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 29 Jan 2017 23:44:24 +0200 Subject: [PATCH 10/28] Added a separate lock for start/stop/reset slave. This solves some possible dead locks when one calls stop slave while slave is starting. --- mysys/hash.c | 12 +++++++-- sql/mysqld.cc | 3 ++- sql/mysqld.h | 2 +- sql/rpl_mi.cc | 61 ++++++++++++++++++++++++++++++++------------- sql/rpl_mi.h | 4 ++- sql/rpl_parallel.cc | 28 +++++++++++++++------ sql/slave.cc | 22 ++++++---------- sql/slave.h | 2 -- sql/sql_repl.cc | 57 ++++++++++++++++++++++++++++++++---------- 9 files changed, 132 insertions(+), 59 deletions(-) diff --git a/mysys/hash.c b/mysys/hash.c index 344b698a433..c2f3555396e 100644 --- a/mysys/hash.c +++ b/mysys/hash.c @@ -115,14 +115,19 @@ my_hash_init2(HASH *hash, uint growth_size, CHARSET_INFO *charset, static inline void my_hash_free_elements(HASH *hash) { + uint records= hash->records; + /* + Set records to 0 early to guard against anyone looking at the structure + during the free process + */ + hash->records= 0; if (hash->free) { HASH_LINK *data=dynamic_element(&hash->array,0,HASH_LINK*); - HASH_LINK *end= data + hash->records; + HASH_LINK *end= data + records; while (data < end) (*hash->free)((data++)->data); } - hash->records=0; } @@ -519,6 +524,9 @@ my_bool my_hash_insert(HASH *info, const uchar *record) The record with the same record ptr is removed. If there is a free-function it's called if record was found. + hash->free() is guarantee to be called only after the row has been + deleted from the hash and the hash can be reused by other threads. + @return @retval 0 ok @retval 1 Record not found diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ca45e9cdcdb..3e0cdf2d10e 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -863,7 +863,7 @@ PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_LOCK_system_variables_hash, key_LOCK_thd_data, key_LOCK_user_conn, key_LOCK_uuid_short_generator, key_LOG_LOCK_log, key_master_info_data_lock, key_master_info_run_lock, - key_master_info_sleep_lock, + key_master_info_sleep_lock, key_master_info_start_stop_lock, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_rpl_group_info_sleep_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, @@ -933,6 +933,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_LOCK_uuid_short_generator, "LOCK_uuid_short_generator", PSI_FLAG_GLOBAL}, { &key_LOG_LOCK_log, "LOG::LOCK_log", 0}, { &key_master_info_data_lock, "Master_info::data_lock", 0}, + { &key_master_info_start_stop_lock, "Master_info::start_stop_lock", 0}, { &key_master_info_run_lock, "Master_info::run_lock", 0}, { &key_master_info_sleep_lock, "Master_info::sleep_lock", 0}, { &key_mutex_slave_reporting_capability_err_lock, "Slave_reporting_capability::err_lock", 0}, diff --git a/sql/mysqld.h b/sql/mysqld.h index e7394230fe4..4538b7c4c70 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -266,7 +266,7 @@ extern PSI_mutex_key key_BINLOG_LOCK_index, key_BINLOG_LOCK_xid_list, key_LOCK_thd_data, key_LOCK_user_conn, key_LOG_LOCK_log, key_master_info_data_lock, key_master_info_run_lock, - key_master_info_sleep_lock, + key_master_info_sleep_lock, key_master_info_start_stop_lock, key_mutex_slave_reporting_capability_err_lock, key_relay_log_info_data_lock, key_relay_log_info_log_space_lock, key_relay_log_info_run_lock, key_rpl_group_info_sleep_lock, diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index 7aa84e72bef..8e9fcb85082 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -80,6 +80,8 @@ Master_info::Master_info(LEX_STRING *connection_name_arg, bzero((char*) &file, sizeof(file)); mysql_mutex_init(key_master_info_run_lock, &run_lock, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_master_info_data_lock, &data_lock, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_master_info_start_stop_lock, &start_stop_lock, + MY_MUTEX_INIT_SLOW); mysql_mutex_setflags(&run_lock, MYF_NO_DEADLOCK_DETECTION); mysql_mutex_setflags(&data_lock, MYF_NO_DEADLOCK_DETECTION); mysql_mutex_init(key_master_info_sleep_lock, &sleep_lock, MY_MUTEX_INIT_FAST); @@ -117,6 +119,7 @@ Master_info::~Master_info() mysql_mutex_destroy(&run_lock); mysql_mutex_destroy(&data_lock); mysql_mutex_destroy(&sleep_lock); + mysql_mutex_destroy(&start_stop_lock); mysql_cond_destroy(&data_cond); mysql_cond_destroy(&start_cond); mysql_cond_destroy(&stop_cond); @@ -727,17 +730,28 @@ uchar *get_key_master_info(Master_info *mi, size_t *length, return (uchar*) mi->cmp_connection_name.str; } +/* + Delete a master info + + Called from my_hash_delete(&master_info_hash) + Stops associated slave threads and frees master_info +*/ + void free_key_master_info(Master_info *mi) { DBUG_ENTER("free_key_master_info"); + mysql_mutex_unlock(&LOCK_active_mi); + /* Ensure that we are not in reset_slave while this is done */ - lock_slave_threads(mi); + mi->lock_slave_threads(); terminate_slave_threads(mi,SLAVE_FORCE_ALL); /* We use 2 here instead of 1 just to make it easier when debugging */ mi->killed= 2; end_master_info(mi); - unlock_slave_threads(mi); + mi->unlock_slave_threads(); delete mi; + + mysql_mutex_lock(&LOCK_active_mi); DBUG_VOID_RETURN; } @@ -904,6 +918,7 @@ Master_info_index::Master_info_index() void Master_info_index::free_connections() { + mysql_mutex_assert_owner(&LOCK_active_mi); my_hash_reset(&master_info_hash); } @@ -936,7 +951,6 @@ bool Master_info_index::init_all_master_info() File index_file_nr; DBUG_ENTER("init_all_master_info"); - mysql_mutex_assert_owner(&LOCK_active_mi); DBUG_ASSERT(master_info_index); if ((index_file_nr= my_open(index_file_name, @@ -984,7 +998,6 @@ bool Master_info_index::init_all_master_info() DBUG_RETURN(1); } - lock_slave_threads(mi); init_thread_mask(&thread_mask,mi,0 /*not inverse*/); create_logfile_name_with_suffix(buf_master_info_file, @@ -999,6 +1012,7 @@ bool Master_info_index::init_all_master_info() sql_print_information("Reading Master_info: '%s' Relay_info:'%s'", buf_master_info_file, buf_relay_log_info_file); + mi->lock_slave_threads(); if (init_master_info(mi, buf_master_info_file, buf_relay_log_info_file, 0, thread_mask)) { @@ -1012,14 +1026,14 @@ bool Master_info_index::init_all_master_info() if (master_info_index->add_master_info(mi, FALSE)) DBUG_RETURN(1); succ_num++; - unlock_slave_threads(mi); + mi->unlock_slave_threads(); } else { /* Master_info already in HASH */ sql_print_error(ER(ER_CONNECTION_ALREADY_EXISTS), (int) connection_name.length, connection_name.str); - unlock_slave_threads(mi); + mi->unlock_slave_threads(); delete mi; } continue; @@ -1036,7 +1050,7 @@ bool Master_info_index::init_all_master_info() /* Master_info was already registered */ sql_print_error(ER(ER_CONNECTION_ALREADY_EXISTS), (int) connection_name.length, connection_name.str); - unlock_slave_threads(mi); + mi->unlock_slave_threads(); delete mi; continue; } @@ -1065,7 +1079,7 @@ bool Master_info_index::init_all_master_info() (int) connection_name.length, connection_name.str); } - unlock_slave_threads(mi); + mi->unlock_slave_threads(); } } @@ -1268,7 +1282,12 @@ bool Master_info_index::check_duplicate_master_info(LEX_STRING *name_arg, /* Add a Master_info class to Hash Table */ bool Master_info_index::add_master_info(Master_info *mi, bool write_to_file) { - if (!my_hash_insert(&master_info_hash, (uchar*) mi)) + /* + We have to protect against shutdown to ensure we are not calling + my_hash_insert() while my_hash_free() is in progress + */ + if (unlikely(shutdown_in_progress) || + !my_hash_insert(&master_info_hash, (uchar*) mi)) { if (global_system_variables.log_warnings > 1) sql_print_information("Added new Master_info '%.*s' to hash table", @@ -1392,23 +1411,31 @@ bool give_error_if_slave_running(bool already_locked) @return 0 No Slave SQL thread is running # Number of slave SQL thread running + + Note that during shutdown we return 1. This is needed to ensure we + don't try to resize thread pool during shutdown as during shutdown + master_info_hash may be freeing the hash and during that time + hash entries can't be accessed. */ uint any_slave_sql_running() { uint count= 0; + HASH *hash; DBUG_ENTER("any_slave_sql_running"); mysql_mutex_lock(&LOCK_active_mi); - if (likely(master_info_index)) // Not shutdown + if (unlikely(shutdown_in_progress || !master_info_index)) { - HASH *hash= &master_info_index->master_info_hash; - for (uint i= 0; i< hash->records; ++i) - { - Master_info *mi= (Master_info *)my_hash_element(hash, i); - if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) - count++; - } + mysql_mutex_unlock(&LOCK_active_mi); + return 1; + } + hash= &master_info_index->master_info_hash; + for (uint i= 0; i< hash->records; ++i) + { + Master_info *mi= (Master_info *)my_hash_element(hash, i); + if (mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN) + count++; } mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(count); diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index 32591c56caa..5c42378ca2c 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -81,6 +81,8 @@ class Master_info : public Slave_reporting_capability } void release(); void wait_until_free(); + void lock_slave_threads(); + void unlock_slave_threads(); /* the variables below are needed because we can change masters on the fly */ char master_log_name[FN_REFLEN+6]; /* Room for multi-*/ @@ -99,7 +101,7 @@ class Master_info : public Slave_reporting_capability File fd; // we keep the file open, so we need to remember the file pointer IO_CACHE file; - mysql_mutex_t data_lock, run_lock, sleep_lock; + mysql_mutex_t data_lock, run_lock, sleep_lock, start_stop_lock; mysql_cond_t data_cond, start_cond, stop_cond, sleep_cond; THD *io_thd; MYSQL* mysql; diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index ff2c7d68467..474c6f063c7 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -1378,10 +1378,24 @@ dealloc_gco(group_commit_orderer *gco) my_free(gco); } +/** + Change thread count for global parallel worker threads + + @param pool parallel thread pool + @param new_count Number of threads to be in pool. 0 in shutdown + @param force Force thread count to new_count even if slave + threads are running + + By default we don't resize pool of there are running threads. + However during shutdown we will always do it. + This is needed as any_slave_sql_running() returns 1 during shutdown + as we don't want to access master_info while + Master_info_index::free_connections are running. +*/ static int rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, - uint32 new_count) + uint32 new_count, bool force) { uint32 i; rpl_parallel_thread **new_list= NULL; @@ -1403,7 +1417,7 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, If we are about to delete pool, do an extra check that there are no new slave threads running since we marked pool busy */ - if (!new_count) + if (!new_count && !force) { if (any_slave_sql_running()) { @@ -1556,8 +1570,7 @@ err: int rpl_parallel_resize_pool_if_no_slaves(void) { /* master_info_index is set to NULL on shutdown */ - if (opt_slave_parallel_threads > 0 && !any_slave_sql_running() && - master_info_index) + if (opt_slave_parallel_threads > 0 && !any_slave_sql_running()) return rpl_parallel_inactivate_pool(&global_rpl_thread_pool); return 0; } @@ -1567,7 +1580,8 @@ int rpl_parallel_activate_pool(rpl_parallel_thread_pool *pool) { if (!pool->count) - return rpl_parallel_change_thread_count(pool, opt_slave_parallel_threads); + return rpl_parallel_change_thread_count(pool, opt_slave_parallel_threads, + 0); return 0; } @@ -1575,7 +1589,7 @@ rpl_parallel_activate_pool(rpl_parallel_thread_pool *pool) int rpl_parallel_inactivate_pool(rpl_parallel_thread_pool *pool) { - return rpl_parallel_change_thread_count(pool, 0); + return rpl_parallel_change_thread_count(pool, 0, 0); } @@ -1854,7 +1868,7 @@ rpl_parallel_thread_pool::destroy() { if (!inited) return; - rpl_parallel_change_thread_count(this, 0); + rpl_parallel_change_thread_count(this, 0, 1); mysql_mutex_destroy(&LOCK_rpl_thread_pool); mysql_cond_destroy(&COND_rpl_thread_pool); inited= false; diff --git a/sql/slave.cc b/sql/slave.cc index a2b8439b4b4..1bb2b534693 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -228,16 +228,14 @@ void init_thread_mask(int* mask,Master_info* mi,bool inverse) /* - lock_slave_threads() + lock_slave_threads() against other threads doing STOP, START or RESET SLAVE + */ -void lock_slave_threads(Master_info* mi) +void Master_info::lock_slave_threads() { DBUG_ENTER("lock_slave_threads"); - - //TODO: see if we can do this without dual mutex - mysql_mutex_lock(&mi->run_lock); - mysql_mutex_lock(&mi->rli.run_lock); + mysql_mutex_lock(&start_stop_lock); DBUG_VOID_RETURN; } @@ -246,13 +244,10 @@ void lock_slave_threads(Master_info* mi) unlock_slave_threads() */ -void unlock_slave_threads(Master_info* mi) +void Master_info::unlock_slave_threads() { DBUG_ENTER("unlock_slave_threads"); - - //TODO: see if we can do this without dual mutex - mysql_mutex_unlock(&mi->rli.run_lock); - mysql_mutex_unlock(&mi->run_lock); + mysql_mutex_unlock(&start_stop_lock); DBUG_VOID_RETURN; } @@ -374,7 +369,6 @@ int init_slave() accepted. However bootstrap may conflict with us if it does START SLAVE. So it's safer to take the lock. */ - mysql_mutex_lock(&LOCK_active_mi); if (pthread_key_create(&RPL_MASTER_INFO, NULL)) goto err; @@ -383,7 +377,6 @@ int init_slave() if (!master_info_index || master_info_index->init_all_master_info()) { sql_print_error("Failed to initialize multi master structures"); - mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(1); } if (!(active_mi= new Master_info(&default_master_connection_name, @@ -441,7 +434,6 @@ int init_slave() } end: - mysql_mutex_unlock(&LOCK_active_mi); DBUG_RETURN(error); err: @@ -6159,7 +6151,7 @@ static int connect_to_master(THD* thd, MYSQL* mysql, Master_info* mi, suppress_warnings= 0; mi->report(ERROR_LEVEL, last_errno, NULL, "error %s to master '%s@%s:%d'" - " - retry-time: %d retries: %lu message: %s", + " - retry-time: %d maximum-retries: %lu message: %s", (reconnect ? "reconnecting" : "connecting"), mi->user, mi->host, mi->port, mi->connect_retry, master_retry_count, diff --git a/sql/slave.h b/sql/slave.h index cfff79da0e4..abcb4df984b 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -219,8 +219,6 @@ void close_active_mi(); /* clean up slave threads data */ void clear_until_condition(Relay_log_info* rli); void clear_slave_error(Relay_log_info* rli); void end_relay_log_info(Relay_log_info* rli); -void lock_slave_threads(Master_info* mi); -void unlock_slave_threads(Master_info* mi); void init_thread_mask(int* mask,Master_info* mi,bool inverse); Format_description_log_event * read_relay_log_description_event(IO_CACHE *cur_log, ulonglong start_pos, diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 29980dbbbf1..7d4fb110229 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -2833,7 +2833,16 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) relay_log_info_file, 0, &mi->cmp_connection_name); - lock_slave_threads(mi); // this allows us to cleanly read slave_running + mi->lock_slave_threads(); + if (mi->killed) + { + /* connection was deleted while we waited for lock_slave_threads */ + mi->unlock_slave_threads(); + my_error(WARN_NO_MASTER_INFO, mi->connection_name.length, + mi->connection_name.str); + DBUG_RETURN(-1); + } + // Get a mask of _stopped_ threads init_thread_mask(&thread_mask,mi,1 /* inverse */); @@ -2968,7 +2977,7 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) ER(ER_UNTIL_COND_IGNORED)); if (!slave_errno) - slave_errno = start_slave_threads(0 /*no mutex */, + slave_errno = start_slave_threads(1, 1 /* wait for start */, mi, master_info_file_tmp, @@ -2984,7 +2993,7 @@ int start_slave(THD* thd , Master_info* mi, bool net_report) } err: - unlock_slave_threads(mi); + mi->unlock_slave_threads(); if (slave_errno) { @@ -3024,8 +3033,12 @@ int stop_slave(THD* thd, Master_info* mi, bool net_report ) DBUG_RETURN(-1); THD_STAGE_INFO(thd, stage_killing_slave); int thread_mask; - lock_slave_threads(mi); - // Get a mask of _running_ threads + mi->lock_slave_threads(); + /* + Get a mask of _running_ threads. + We don't have to test for mi->killed as the thread_mask will take care + of checking if threads exists + */ init_thread_mask(&thread_mask,mi,0 /* not inverse*/); /* Below we will stop all running threads. @@ -3038,8 +3051,7 @@ int stop_slave(THD* thd, Master_info* mi, bool net_report ) if (thread_mask) { - slave_errno= terminate_slave_threads(mi,thread_mask, - 1 /*skip lock */); + slave_errno= terminate_slave_threads(mi,thread_mask, 0 /* get lock */); } else { @@ -3048,7 +3060,8 @@ int stop_slave(THD* thd, Master_info* mi, bool net_report ) push_warning(thd, Sql_condition::WARN_LEVEL_NOTE, ER_SLAVE_WAS_NOT_RUNNING, ER(ER_SLAVE_WAS_NOT_RUNNING)); } - unlock_slave_threads(mi); + + mi->unlock_slave_threads(); if (slave_errno) { @@ -3083,11 +3096,20 @@ int reset_slave(THD *thd, Master_info* mi) char relay_log_info_file_tmp[FN_REFLEN]; DBUG_ENTER("reset_slave"); - lock_slave_threads(mi); + mi->lock_slave_threads(); + if (mi->killed) + { + /* connection was deleted while we waited for lock_slave_threads */ + mi->unlock_slave_threads(); + my_error(WARN_NO_MASTER_INFO, mi->connection_name.length, + mi->connection_name.str); + DBUG_RETURN(-1); + } + init_thread_mask(&thread_mask,mi,0 /* not inverse */); if (thread_mask) // We refuse if any slave thread is running { - unlock_slave_threads(mi); + mi->unlock_slave_threads(); my_error(ER_SLAVE_MUST_STOP, MYF(0), (int) mi->connection_name.length, mi->connection_name.str); DBUG_RETURN(ER_SLAVE_MUST_STOP); @@ -3152,7 +3174,7 @@ int reset_slave(THD *thd, Master_info* mi) RUN_HOOK(binlog_relay_io, after_reset_slave, (thd, mi)); err: - unlock_slave_threads(mi); + mi->unlock_slave_threads(); if (error) my_error(sql_errno, MYF(0), errmsg); DBUG_RETURN(error); @@ -3286,7 +3308,16 @@ bool change_master(THD* thd, Master_info* mi, bool *master_info_added) lex_mi->port)) DBUG_RETURN(TRUE); - lock_slave_threads(mi); + mi->lock_slave_threads(); + if (mi->killed) + { + /* connection was deleted while we waited for lock_slave_threads */ + mi->unlock_slave_threads(); + my_error(WARN_NO_MASTER_INFO, mi->connection_name.length, + mi->connection_name.str); + DBUG_RETURN(TRUE); + } + init_thread_mask(&thread_mask,mi,0 /*not inverse*/); if (thread_mask) // We refuse if any slave thread is running { @@ -3593,7 +3624,7 @@ bool change_master(THD* thd, Master_info* mi, bool *master_info_added) mysql_mutex_unlock(&mi->rli.data_lock); err: - unlock_slave_threads(mi); + mi->unlock_slave_threads(); if (ret == FALSE) my_ok(thd); DBUG_RETURN(ret); From a2de378c00c17b358fa784dcca6ab4ac11b56821 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 30 Jan 2017 16:13:49 +0200 Subject: [PATCH 11/28] Add protection for reinitialization of mutex in parallel replaction Added mutex_lock/mutex_unlock of mutex that is to be destroyed in wait_for_commit::reinit() in a similar fashion that we do in ~wait_for_commit --- sql/sql_class.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 21c8d558437..6177210e576 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -6516,7 +6516,13 @@ wait_for_commit::reinit() So in this case, do a re-init of the mutex. In release builds, we want to avoid the overhead of a re-init though. + + To ensure that no one is locking the mutex, we take a lock of it first. + For full explanation, see wait_for_commit::~wait_for_commit() */ + mysql_mutex_lock(&LOCK_wait_commit); + mysql_mutex_unlock(&LOCK_wait_commit); + mysql_mutex_destroy(&LOCK_wait_commit); mysql_mutex_init(key_LOCK_wait_commit, &LOCK_wait_commit, MY_MUTEX_INIT_FAST); #endif From 4bad74e13955ee21015c6b8a9d4bd07624924efa Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 5 Feb 2017 02:23:49 +0200 Subject: [PATCH 12/28] Added error checking for all calls to flush_relay_log_info() and stmt_done() --- sql/log.cc | 11 +++++++---- sql/log_event.cc | 17 ++++++++++------- sql/log_event_old.cc | 4 ++-- sql/rpl_rli.cc | 10 ++++++---- sql/rpl_rli.h | 2 +- sql/slave.cc | 14 ++++++++++++-- sql/sql_repl.cc | 3 ++- 7 files changed, 40 insertions(+), 21 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index cd215822d44..bdd60b4d420 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -4129,7 +4129,7 @@ err: int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) { - int error; + int error, errcode; char *to_purge_if_included= NULL; inuse_relaylog *ir; ulonglong log_space_reclaimed= 0; @@ -4200,7 +4200,8 @@ int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) } /* Store where we are in the new file for the execution thread */ - flush_relay_log_info(rli); + if (flush_relay_log_info(rli)) + error= LOG_INFO_IO; DBUG_EXECUTE_IF("crash_before_purge_logs", DBUG_SUICIDE();); @@ -4216,11 +4217,13 @@ int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) * Need to update the log pos because purge logs has been called * after fetching initially the log pos at the begining of the method. */ - if((error=find_log_pos(&rli->linfo, rli->event_relay_log_name, 0))) + if ((errcode= find_log_pos(&rli->linfo, rli->event_relay_log_name, 0))) { char buff[22]; + if (!error) + error= errcode; sql_print_error("next log error: %d offset: %s log: %s included: %d", - error, + errcode, llstr(rli->linfo.index_file_offset,buff), rli->group_relay_log_name, included); diff --git a/sql/log_event.cc b/sql/log_event.cc index 14f6bb20b47..43e8df7b801 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -6244,9 +6244,11 @@ bool Rotate_log_event::write(IO_CACHE* file) @retval 0 ok + 1 error */ int Rotate_log_event::do_update_pos(rpl_group_info *rgi) { + int error= 0; Relay_log_info *rli= rgi->rli; DBUG_ENTER("Rotate_log_event::do_update_pos"); #ifndef DBUG_OFF @@ -6298,7 +6300,7 @@ int Rotate_log_event::do_update_pos(rpl_group_info *rgi) (ulong) rli->group_master_log_pos)); mysql_mutex_unlock(&rli->data_lock); rpl_global_gtid_slave_state->record_and_update_gtid(thd, rgi); - flush_relay_log_info(rli); + error= flush_relay_log_info(rli); /* Reset thd->variables.option_bits and sql_mode etc, because this could @@ -6316,8 +6318,7 @@ int Rotate_log_event::do_update_pos(rpl_group_info *rgi) else rgi->inc_event_relay_log_pos(); - - DBUG_RETURN(0); + DBUG_RETURN(error); } @@ -8174,6 +8175,7 @@ void Stop_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) int Stop_log_event::do_update_pos(rpl_group_info *rgi) { + int error= 0; Relay_log_info *rli= rgi->rli; DBUG_ENTER("Stop_log_event::do_update_pos"); /* @@ -8189,9 +8191,10 @@ int Stop_log_event::do_update_pos(rpl_group_info *rgi) { rpl_global_gtid_slave_state->record_and_update_gtid(thd, rgi); rli->inc_group_relay_log_pos(0, rgi); - flush_relay_log_info(rli); + if (flush_relay_log_info(rli)) + error= 1; } - DBUG_RETURN(0); + DBUG_RETURN(error); } #endif /* !MYSQL_CLIENT */ @@ -10178,8 +10181,8 @@ int Rows_log_event::do_update_pos(rpl_group_info *rgi) { Relay_log_info *rli= rgi->rli; - DBUG_ENTER("Rows_log_event::do_update_pos"); int error= 0; + DBUG_ENTER("Rows_log_event::do_update_pos"); DBUG_PRINT("info", ("flags: %s", get_flags(STMT_END_F) ? "STMT_END_F " : "")); @@ -10191,7 +10194,7 @@ Rows_log_event::do_update_pos(rpl_group_info *rgi) Step the group log position if we are not in a transaction, otherwise increase the event log position. */ - rli->stmt_done(log_pos, thd, rgi); + error= rli->stmt_done(log_pos, thd, rgi); /* Clear any errors in thd->net.last_err*. It is not known if this is needed or not. It is believed that any errors that may exist in diff --git a/sql/log_event_old.cc b/sql/log_event_old.cc index 22ef970dab9..8bdb81f5283 100644 --- a/sql/log_event_old.cc +++ b/sql/log_event_old.cc @@ -1843,8 +1843,8 @@ int Old_rows_log_event::do_update_pos(rpl_group_info *rgi) { Relay_log_info *rli= rgi->rli; - DBUG_ENTER("Old_rows_log_event::do_update_pos"); int error= 0; + DBUG_ENTER("Old_rows_log_event::do_update_pos"); DBUG_PRINT("info", ("flags: %s", get_flags(STMT_END_F) ? "STMT_END_F " : "")); @@ -1856,7 +1856,7 @@ Old_rows_log_event::do_update_pos(rpl_group_info *rgi) Step the group log position if we are not in a transaction, otherwise increase the event log position. */ - rli->stmt_done(log_pos, thd, rgi); + error= rli->stmt_done(log_pos, thd, rgi); /* Clear any errors in thd->net.last_err*. It is not known if this is needed or not. It is believed that any errors that may exist in diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index c570105cdf6..7eacf539d19 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -432,7 +432,7 @@ Failed to open the existing relay log info file '%s' (errno %d)", } rli->inited= 1; mysql_mutex_unlock(&rli->data_lock); - DBUG_RETURN(error); + DBUG_RETURN(0); err: sql_print_error("%s", msg); @@ -1304,9 +1304,10 @@ bool Relay_log_info::is_until_satisfied(THD *thd, Log_event *ev) } -void Relay_log_info::stmt_done(my_off_t event_master_log_pos, THD *thd, +bool Relay_log_info::stmt_done(my_off_t event_master_log_pos, THD *thd, rpl_group_info *rgi) { + int error= 0; DBUG_ENTER("Relay_log_info::stmt_done"); DBUG_ASSERT(rgi->rli == this); @@ -1358,10 +1359,11 @@ void Relay_log_info::stmt_done(my_off_t event_master_log_pos, THD *thd, } DBUG_EXECUTE_IF("inject_crash_before_flush_rli", DBUG_SUICIDE();); if (mi->using_gtid == Master_info::USE_GTID_NO) - flush_relay_log_info(this); + if (flush_relay_log_info(this)) + error= 1; DBUG_EXECUTE_IF("inject_crash_after_flush_rli", DBUG_SUICIDE();); } - DBUG_VOID_RETURN; + DBUG_RETURN(error); } diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index efcec83b880..7fc41786957 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -416,7 +416,7 @@ public: relay log info and used to produce information for SHOW SLAVE STATUS. */ - void stmt_done(my_off_t event_log_pos, THD *thd, rpl_group_info *rgi); + bool stmt_done(my_off_t event_log_pos, THD *thd, rpl_group_info *rgi); int alloc_inuse_relaylog(const char *name); void free_inuse_relaylog(inuse_relaylog *ir); void reset_inuse_relaylog(); diff --git a/sql/slave.cc b/sql/slave.cc index 1bb2b534693..1634a56a4b6 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -4788,8 +4788,15 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, if (rli->mi->using_gtid != Master_info::USE_GTID_NO) { ulong domain_count; + my_bool save_log_all_errors= thd->log_all_errors; + /* + We don't need to check return value for flush_relay_log_info() + as any errors should be logged to stderr + */ + thd->log_all_errors= 1; flush_relay_log_info(rli); + thd->log_all_errors= save_log_all_errors; if (mi->using_parallel()) { /* @@ -6715,9 +6722,12 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) } rli->event_relay_log_pos = BIN_LOG_HEADER_SIZE; strmake_buf(rli->event_relay_log_name,rli->linfo.log_file_name); - flush_relay_log_info(rli); + if (flush_relay_log_info(rli)) + { + errmsg= "error flushing relay log"; + goto err; + } } - /* Now we want to open this next log. To know if it's a hot log (the one being written by the I/O thread now) or a cold log, we can use diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 7d4fb110229..560b7ede183 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -3619,7 +3619,8 @@ bool change_master(THD* thd, Master_info* mi, bool *master_info_added) in-memory value at restart (thus causing errors, as the old relay log does not exist anymore). */ - flush_relay_log_info(&mi->rli); + if (flush_relay_log_info(&mi->rli)) + ret= 1; mysql_cond_broadcast(&mi->data_cond); mysql_mutex_unlock(&mi->rli.data_lock); From d7a9aed43f6a77b2a4fcbc45b08d16b40be921ba Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Wed, 8 Feb 2017 02:14:54 +0200 Subject: [PATCH 13/28] Fixed test failing as myisam table was deleted before oqgraph table. --- storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.result | 2 +- storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.test | 2 +- storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.result | 2 +- storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.test | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.result b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.result index 42c86405503..6b1d0a1854d 100644 --- a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.result +++ b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.result @@ -33,5 +33,5 @@ FROM `version_history` AS `v` INNER JOIN `db_history` AS `db` ON `db`.`nodeID` = WHERE `latch` = 'breadth_first' AND `origid` = '1' ORDER BY `weight` DESC LIMIT 1; version nodeID 0.0.3 3 -DROP TABLE db_history; DROP TABLE version_history; +DROP TABLE db_history; diff --git a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.test b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.test index 3a7a0e5af08..b8f0ae556c8 100644 --- a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.test +++ b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6282.test @@ -40,8 +40,8 @@ --disconnect con2 --connect (con3,localhost,root,,test) -DROP TABLE db_history; DROP TABLE version_history; +DROP TABLE db_history; --disconnect con3 diff --git a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.result b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.result index 8e680e15206..68002ce98a2 100644 --- a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.result +++ b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.result @@ -8,5 +8,5 @@ latch origid destid weight seq linkid breadth_first 1 6 NULL 0 1 breadth_first 1 6 1 1 2 breadth_first 1 6 1 2 6 -DROP TABLE IF EXISTS oq_backing; DROP TABLE IF EXISTS oq_graph; +DROP TABLE IF EXISTS oq_backing; diff --git a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.test b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.test index 72de926da2e..fefd03a7ed9 100644 --- a/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.test +++ b/storage/oqgraph/mysql-test/oqgraph/regression_mdev6345.test @@ -14,6 +14,6 @@ CREATE TABLE oq_graph (latch VARCHAR(32) NULL, origid BIGINT UNSIGNED NULL, dest SELECT * FROM oq_graph WHERE latch='breadth_first' AND origid=1 AND destid=6; -DROP TABLE IF EXISTS oq_backing; DROP TABLE IF EXISTS oq_graph; +DROP TABLE IF EXISTS oq_backing; From b624b41abb20b853cb033aa9b40bd0786e9d17fa Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 10 Feb 2017 20:25:01 +0200 Subject: [PATCH 14/28] Don't allow one to kill START SLAVE while the slaves IO_THREAD or SQL_THREAD is starting. This is needed as if we kill the START SLAVE thread too early during shutdown then the IO_THREAD or SQL_THREAD will not have time to properly initlize it's replication or THD structures and clean_up() will try to delete master_info structures that are still in use. --- sql/slave.cc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sql/slave.cc b/sql/slave.cc index 1634a56a4b6..1187031711d 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -831,6 +831,15 @@ int start_slave_thread( mysql_mutex_unlock(start_lock); DBUG_RETURN(ER_SLAVE_THREAD); } + + /* + In the following loop we can't check for thd->killed as we have to + wait until THD structures for the slave thread are created + before we can return. + This should be ok as there is no major work done in the slave + threads before they signal that we can stop waiting. + */ + if (start_cond && cond_lock) // caller has cond_lock { THD* thd = current_thd; @@ -848,16 +857,9 @@ int start_slave_thread( registered, we could otherwise go waiting though thd->killed is set. */ - if (!thd->killed) - mysql_cond_wait(start_cond, cond_lock); + mysql_cond_wait(start_cond, cond_lock); thd->EXIT_COND(& saved_stage); mysql_mutex_lock(cond_lock); // re-acquire it as exit_cond() released - if (thd->killed) - { - if (start_lock) - mysql_mutex_unlock(start_lock); - DBUG_RETURN(thd->killed_errno()); - } } } if (start_lock) From f3c65ce95189f4636ef853b819fb02c4e068bc0f Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 10 Feb 2017 20:30:37 +0200 Subject: [PATCH 15/28] Add protection to not access is_open() without LOCK_log mutex Protection added to reopen_file() and new_file_impl(). Without this we could get an assert in fn_format() as name == 0, because the file was closed and name reset, atthe same time new_file_impl() was called. --- sql/log.cc | 42 ++++++++++++++++++++---------------------- sql/mysqld.cc | 14 +++++++++++--- sql/rpl_rli.cc | 5 +++++ sql/slave.cc | 4 ++++ 4 files changed, 40 insertions(+), 25 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index bdd60b4d420..e99eefe4970 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -2702,16 +2702,16 @@ int MYSQL_LOG::generate_new_name(char *new_name, const char *log_name) void MYSQL_QUERY_LOG::reopen_file() { char *save_name; - DBUG_ENTER("MYSQL_LOG::reopen_file"); + + mysql_mutex_lock(&LOCK_log); if (!is_open()) { DBUG_PRINT("info",("log is closed")); + mysql_mutex_unlock(&LOCK_log); DBUG_VOID_RETURN; } - mysql_mutex_lock(&LOCK_log); - save_name= name; name= 0; // Don't free name close(LOG_CLOSE_TO_BE_OPENED); @@ -2870,13 +2870,6 @@ bool MYSQL_QUERY_LOG::write(THD *thd, time_t current_time, DBUG_ENTER("MYSQL_QUERY_LOG::write"); mysql_mutex_lock(&LOCK_log); - - if (!is_open()) - { - mysql_mutex_unlock(&LOCK_log); - DBUG_RETURN(0); - } - if (is_open()) { // Safety agains reopen int tmp_errno= 0; @@ -3100,7 +3093,9 @@ void MYSQL_BIN_LOG::cleanup() } inited= 0; + mysql_mutex_lock(&LOCK_log); close(LOG_CLOSE_INDEX|LOG_CLOSE_STOP_EVENT); + mysql_mutex_unlock(&LOCK_log); delete description_event_for_queue; delete description_event_for_exec; @@ -3257,10 +3252,11 @@ bool MYSQL_BIN_LOG::open(const char *log_name, { File file= -1; xid_count_per_binlog *new_xid_list_entry= NULL, *b; - DBUG_ENTER("MYSQL_BIN_LOG::open"); DBUG_PRINT("enter",("log_type: %d",(int) log_type_arg)); + mysql_mutex_assert_owner(&LOCK_log); + if (!is_relay_log) { if (!binlog_state_recover_done) @@ -4842,20 +4838,20 @@ int MYSQL_BIN_LOG::new_file_impl(bool need_lock) bool delay_close= false; File old_file; LINT_INIT(old_file); - DBUG_ENTER("MYSQL_BIN_LOG::new_file_impl"); - if (!is_open()) - { - DBUG_PRINT("info",("log is closed")); - DBUG_RETURN(error); - } if (need_lock) mysql_mutex_lock(&LOCK_log); - mysql_mutex_lock(&LOCK_index); - mysql_mutex_assert_owner(&LOCK_log); - mysql_mutex_assert_owner(&LOCK_index); + + if (!is_open()) + { + DBUG_PRINT("info",("log is closed")); + mysql_mutex_unlock(&LOCK_log); + DBUG_RETURN(error); + } + + mysql_mutex_lock(&LOCK_index); /* Reuse old name if not binlog and not update log */ new_name_ptr= name; @@ -4989,9 +4985,9 @@ end: new_name_ptr, errno); } + mysql_mutex_unlock(&LOCK_index); if (need_lock) mysql_mutex_unlock(&LOCK_log); - mysql_mutex_unlock(&LOCK_index); DBUG_RETURN(error); } @@ -7775,9 +7771,11 @@ int MYSQL_BIN_LOG::wait_for_update_bin_log(THD* thd, void MYSQL_BIN_LOG::close(uint exiting) { // One can't set log_type here! bool failed_to_save_state= false; - DBUG_ENTER("MYSQL_BIN_LOG::close"); DBUG_PRINT("enter",("exiting: %d", (int) exiting)); + + mysql_mutex_assert_owner(&LOCK_log); + if (log_state == LOG_OPENED) { #ifdef HAVE_REPLICATION diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 3e0cdf2d10e..5692c88d6c8 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4928,9 +4928,17 @@ static int init_server_components() unireg_abort(1); } - if (opt_bin_log && mysql_bin_log.open(opt_bin_logname, LOG_BIN, 0, - WRITE_CACHE, max_binlog_size, 0, TRUE)) - unireg_abort(1); + if (opt_bin_log) + { + int error; + mysql_mutex_t *log_lock= mysql_bin_log.get_log_lock(); + mysql_mutex_lock(log_lock); + error= mysql_bin_log.open(opt_bin_logname, LOG_BIN, 0, + WRITE_CACHE, max_binlog_size, 0, TRUE); + mysql_mutex_unlock(log_lock); + if (error) + unireg_abort(1); + } #ifdef HAVE_REPLICATION if (opt_bin_log && expire_logs_days) diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 7eacf539d19..928fbd3d7c1 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -208,6 +208,7 @@ a file name for --relay-log-index option", opt_relaylog_index_name); Master_info* mi= rli->mi; char buf_relay_logname[FN_REFLEN], buf_relaylog_index_name_buff[FN_REFLEN]; char *buf_relaylog_index_name= opt_relaylog_index_name; + mysql_mutex_t *log_lock; create_logfile_name_with_suffix(buf_relay_logname, sizeof(buf_relay_logname), @@ -227,14 +228,18 @@ a file name for --relay-log-index option", opt_relaylog_index_name); note, that if open() fails, we'll still have index file open but a destructor will take care of that */ + log_lock= rli->relay_log.get_log_lock(); + mysql_mutex_lock(log_lock); if (rli->relay_log.open_index_file(buf_relaylog_index_name, ln, TRUE) || rli->relay_log.open(ln, LOG_BIN, 0, SEQ_READ_APPEND, mi->rli.max_relay_log_size, 1, TRUE)) { + mysql_mutex_unlock(log_lock); mysql_mutex_unlock(&rli->data_lock); sql_print_error("Failed when trying to open logs for '%s' in init_relay_log_info(). Error: %M", ln, my_errno); DBUG_RETURN(1); } + mysql_mutex_unlock(log_lock); } /* if file does not exist */ diff --git a/sql/slave.cc b/sql/slave.cc index 1187031711d..36d28382c56 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -5990,6 +5990,7 @@ err: void end_relay_log_info(Relay_log_info* rli) { + mysql_mutex_t *log_lock; DBUG_ENTER("end_relay_log_info"); if (!rli->inited) @@ -6007,8 +6008,11 @@ void end_relay_log_info(Relay_log_info* rli) rli->cur_log_fd = -1; } rli->inited = 0; + log_lock= rli->relay_log.get_log_lock(); + mysql_mutex_lock(log_lock); rli->relay_log.close(LOG_CLOSE_INDEX | LOG_CLOSE_STOP_EVENT); rli->relay_log.harvest_bytes_written(&rli->log_space_total); + mysql_mutex_unlock(log_lock); /* Delete the slave's temporary tables from memory. In the future there will be other actions than this, to ensure persistance From 84ed5e1d5fe956af818c2f781e9987181d33fc06 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 10 Feb 2017 22:35:04 +0200 Subject: [PATCH 16/28] Fixed hang doing FLUSH TABLES WITH READ LOCK and parallel replication The problem was that waiting for pause_for_ftwrl was done before event_group was completed. This caused rpl_pause_for_ftwrl() to wait forever during FLUSH TABLES WITH READ LOCK. Now we only wait for FLUSH TABLES WITH READ LOCK when we are changing to a new event group. --- sql/rpl_parallel.cc | 42 +++++++++++++----------------------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc index 474c6f063c7..44f60e4482b 100644 --- a/sql/rpl_parallel.cc +++ b/sql/rpl_parallel.cc @@ -1253,39 +1253,16 @@ handle_rpl_parallel_thread(void *arg) */ rpt->batch_free(); - for (;;) + if ((events= rpt->event_queue) != NULL) { - if ((events= rpt->event_queue) != NULL) - { - /* - Take next group of events from the replication pool. - This is faster than having to wakeup the pool manager thread to give - us a new event. - */ - rpt->dequeue1(events); - mysql_mutex_unlock(&rpt->LOCK_rpl_thread); - goto more_events; - } - if (!rpt->pause_for_ftwrl || - (in_event_group && !group_rgi->parallel_entry->force_abort)) - break; /* - We are currently in the delicate process of pausing parallel - replication while FLUSH TABLES WITH READ LOCK is starting. We must - not de-allocate the thread (setting rpt->current_owner= NULL) until - rpl_unpause_after_ftwrl() has woken us up. + Take next group of events from the replication pool. + This is faster than having to wakeup the pool manager thread to give + us a new event. */ - mysql_mutex_lock(&rpt->current_entry->LOCK_parallel_entry); + rpt->dequeue1(events); mysql_mutex_unlock(&rpt->LOCK_rpl_thread); - if (rpt->pause_for_ftwrl) - mysql_cond_wait(&rpt->current_entry->COND_parallel_entry, - &rpt->current_entry->LOCK_parallel_entry); - mysql_mutex_unlock(&rpt->current_entry->LOCK_parallel_entry); - mysql_mutex_lock(&rpt->LOCK_rpl_thread); - /* - Now loop to check again for more events available, since we released - and re-aquired the LOCK_rpl_thread mutex. - */ + goto more_events; } rpt->inuse_relaylog_refcount_update(); @@ -1335,11 +1312,13 @@ handle_rpl_parallel_thread(void *arg) mysql_mutex_unlock(&e->LOCK_parallel_entry); mysql_mutex_lock(&rpt->LOCK_rpl_thread); } + rpt->current_owner= NULL; /* Tell wait_for_done() that we are done, if it is waiting. */ if (likely(rpt->current_entry) && unlikely(rpt->current_entry->force_abort)) mysql_cond_broadcast(&rpt->COND_rpl_thread_stop); + rpt->current_entry= NULL; if (!rpt->stop) rpt->pool->release_thread(rpt); @@ -2116,6 +2095,11 @@ rpl_parallel::find(uint32 domain_id) return e; } +/** + Wait until all sql worker threads has stopped processing + + This is called when sql thread has been killed/stopped +*/ void rpl_parallel::wait_for_done(THD *thd, Relay_log_info *rli) From 6a12c05347beb85a95318568e5f78221bde73b38 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 10 Feb 2017 22:39:22 +0200 Subject: [PATCH 17/28] Fixed wrong arguments to sql_print_error() --- sql/rpl_mi.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index 8e9fcb85082..59ada30a46d 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -1032,6 +1032,7 @@ bool Master_info_index::init_all_master_info() { /* Master_info already in HASH */ sql_print_error(ER(ER_CONNECTION_ALREADY_EXISTS), + (int) connection_name.length, connection_name.str, (int) connection_name.length, connection_name.str); mi->unlock_slave_threads(); delete mi; @@ -1049,6 +1050,7 @@ bool Master_info_index::init_all_master_info() { /* Master_info was already registered */ sql_print_error(ER(ER_CONNECTION_ALREADY_EXISTS), + (int) connection_name.length, connection_name.str, (int) connection_name.length, connection_name.str); mi->unlock_slave_threads(); delete mi; From 370cf70136778b4b89bc3d3d2dfc8a35c7f98478 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 22 Feb 2017 19:50:27 +0100 Subject: [PATCH 18/28] MDEV-11757 KEY_BLOCK_SIZE strangeness when UNCOMPRESSing COMPRESSed InnoDB tables in ALTER TABLE ... DROP KEY, ADD KEY, don't forget to compare old and new keys' block sizes. If they differ - the key definition has changed. --- .../r/alter_key_block_size-11757.result | 47 +++++++++++++ .../innodb/t/alter_key_block_size-11757.test | 23 +++++++ sql/handler.cc | 2 +- sql/sql_table.cc | 3 + storage/myisam/ha_myisam.cc | 68 +++++++++++++++++++ storage/myisam/ha_myisam.h | 2 + 6 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/innodb/r/alter_key_block_size-11757.result create mode 100644 mysql-test/suite/innodb/t/alter_key_block_size-11757.test diff --git a/mysql-test/suite/innodb/r/alter_key_block_size-11757.result b/mysql-test/suite/innodb/r/alter_key_block_size-11757.result new file mode 100644 index 00000000000..400a3d0df3c --- /dev/null +++ b/mysql-test/suite/innodb/r/alter_key_block_size-11757.result @@ -0,0 +1,47 @@ +set global innodb_file_format=barracuda; +create table t1 ( +id1 bigint(20) not null, +id2 bigint(20) not null, +primary key (id1), +unique key id2 (id2) +) engine=innodb row_format=compressed key_block_size=8; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id1` bigint(20) NOT NULL, + `id2` bigint(20) NOT NULL, + PRIMARY KEY (`id1`), + UNIQUE KEY `id2` (`id2`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8 +alter table t1 row_format=dynamic; +Warnings: +Warning 1478 InnoDB: ignoring KEY_BLOCK_SIZE=8 unless ROW_FORMAT=COMPRESSED. +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id1` bigint(20) NOT NULL, + `id2` bigint(20) NOT NULL, + PRIMARY KEY (`id1`), + UNIQUE KEY `id2` (`id2`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC KEY_BLOCK_SIZE=8 +alter table t1 key_block_size=0; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id1` bigint(20) NOT NULL, + `id2` bigint(20) NOT NULL, + PRIMARY KEY (`id1`) KEY_BLOCK_SIZE=8, + UNIQUE KEY `id2` (`id2`) KEY_BLOCK_SIZE=8 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC +alter table t1 drop primary key, add primary key (id1), +drop key id2, add unique (id2); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id1` bigint(20) NOT NULL, + `id2` bigint(20) NOT NULL, + PRIMARY KEY (`id1`), + UNIQUE KEY `id2` (`id2`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC +drop table t1; +set global innodb_file_format=default; diff --git a/mysql-test/suite/innodb/t/alter_key_block_size-11757.test b/mysql-test/suite/innodb/t/alter_key_block_size-11757.test new file mode 100644 index 00000000000..50a909870ba --- /dev/null +++ b/mysql-test/suite/innodb/t/alter_key_block_size-11757.test @@ -0,0 +1,23 @@ +# +# MDEV-11757 KEY_BLOCK_SIZE strangeness when UNCOMPRESSing COMPRESSed InnoDB tables +# +source include/have_innodb.inc; +set global innodb_file_format=barracuda; + +create table t1 ( + id1 bigint(20) not null, + id2 bigint(20) not null, + primary key (id1), + unique key id2 (id2) +) engine=innodb row_format=compressed key_block_size=8; +show create table t1; +alter table t1 row_format=dynamic; +show create table t1; +alter table t1 key_block_size=0; +show create table t1; +alter table t1 drop primary key, add primary key (id1), + drop key id2, add unique (id2); +show create table t1; +drop table t1; + +set global innodb_file_format=default; diff --git a/sql/handler.cc b/sql/handler.cc index b261983ea0d..249d68fccb1 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4209,7 +4209,7 @@ enum_alter_inplace_result handler::check_if_supported_inplace_alter(TABLE *altered_table, Alter_inplace_info *ha_alter_info) { - DBUG_ENTER("check_if_supported_alter"); + DBUG_ENTER("handler::check_if_supported_inplace_alter"); HA_CREATE_INFO *create_info= ha_alter_info->create_info; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 5fde13c1f88..ef1b348d2ce 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -6429,6 +6429,9 @@ static bool fill_alter_inplace_info(THD *thd, new_key->user_defined_key_parts)) goto index_changed; + if (table_key->block_size != new_key->block_size) + goto index_changed; + if (engine_options_differ(table_key->option_struct, new_key->option_struct, table->file->ht->index_options)) goto index_changed; diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index dac01b8af89..730a63067ac 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -2179,6 +2179,74 @@ uint ha_myisam::checksum() const } +enum_alter_inplace_result +ha_myisam::check_if_supported_inplace_alter(TABLE *new_table, + Alter_inplace_info *alter_info) +{ + DBUG_ENTER("ha_myisam::check_if_supported_inplace_alter"); + + const uint readd_index= Alter_inplace_info::ADD_INDEX | + Alter_inplace_info::DROP_INDEX; + const uint readd_unique= Alter_inplace_info::ADD_UNIQUE_INDEX | + Alter_inplace_info::DROP_UNIQUE_INDEX; + const uint readd_pk= Alter_inplace_info::ADD_PK_INDEX | + Alter_inplace_info::DROP_PK_INDEX; + + const uint op= alter_info->handler_flags; + + /* + ha_myisam::open() updates table->key_info->block_size to be the actual + MYI index block size, overwriting user-specified value (if any). + So, the server can not reliably detect whether ALTER TABLE changes + key_block_size or not, it might think the block size was changed, + when it wasn't, and in this case the server will recreate (drop+add) + the index unnecessary. Fix it. + */ + + if (table->s->keys == new_table->s->keys && + ((op & readd_pk) == readd_pk || + (op & readd_unique) == readd_unique || + (op & readd_index) == readd_index)) + { + for (uint i=0; i < table->s->keys; i++) + { + KEY *old_key= table->key_info + i; + KEY *new_key= new_table->key_info + i; + + if (old_key->block_size == new_key->block_size) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); // must differ somewhere else + + if (new_key->block_size && new_key->block_size != old_key->block_size) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); // really changed + + /* any difference besides the block_size, and we give up */ + if (old_key->key_length != new_key->key_length || + old_key->flags != new_key->flags || + old_key->user_defined_key_parts != new_key->user_defined_key_parts || + old_key->algorithm != new_key->algorithm || + strcmp(old_key->name, new_key->name)) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); + + for (uint j= 0; j < old_key->user_defined_key_parts; j++) + { + KEY_PART_INFO *old_kp= old_key->key_part + j; + KEY_PART_INFO *new_kp= new_key->key_part + j; + if (old_kp->offset != new_kp->offset || + old_kp->null_offset != new_kp->null_offset || + old_kp->length != new_kp->length || + old_kp->fieldnr != new_kp->fieldnr || + old_kp->key_part_flag != new_kp->key_part_flag || + old_kp->type != new_kp->type || + old_kp->null_bit != new_kp->null_bit) + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); + } + } + alter_info->handler_flags &= ~(readd_pk | readd_unique | readd_index); + } + DBUG_RETURN(handler::check_if_supported_inplace_alter(new_table, alter_info)); +} + + bool ha_myisam::check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes) { diff --git a/storage/myisam/ha_myisam.h b/storage/myisam/ha_myisam.h index 63fb0ea5a2a..b132c955795 100644 --- a/storage/myisam/ha_myisam.h +++ b/storage/myisam/ha_myisam.h @@ -138,6 +138,8 @@ class ha_myisam: public handler int optimize(THD* thd, HA_CHECK_OPT* check_opt); int assign_to_keycache(THD* thd, HA_CHECK_OPT* check_opt); int preload_keys(THD* thd, HA_CHECK_OPT* check_opt); + enum_alter_inplace_result check_if_supported_inplace_alter(TABLE *new_table, + Alter_inplace_info *alter_info); bool check_if_incompatible_data(HA_CREATE_INFO *info, uint table_changes); #ifdef HAVE_QUERY_CACHE my_bool register_query_cache_table(THD *thd, char *table_key, From cc413ce9a368b930aba5e63c0ab013f7b3ab3c04 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 23 Feb 2017 20:45:07 +0100 Subject: [PATCH 19/28] MDEV-11753 Link failure on missing -L${LIBLZ4_LIBRARY_DIR} On FreeBSD liblz4 is installed in /usr/local/lib. Groonga uses pkg_check_modules to check for liblz4 (that is, pkg-config), and then it used to set for libgroonga.a link_directories({$LIBLZ4_LIBRARY_DIRS}) target_link_libraries(... ${LIBLZ4_LIBRARIES}) Now groonga is a static library, linked into ha_mroonga.so. CMake won't link dynamic liblz4.so into libgroonga.a, instead it'll pass it as a dependency and will link it into ha_mroonga.so. Fine so far. But it will not pass link_directories from the static library as a dependency, so ha_mroonga.so won't find liblz4.so As suggested on cmake mailing list (e.g. here: http://public.kitware.com/pipermail/cmake/2011-November/047468.html) we switch to use the full path to liblz4.so, instead of the -l/-L pair. --- storage/mroonga/vendor/groonga/lib/CMakeLists.txt | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt index 8959f883ca3..ef3e13e236d 100644 --- a/storage/mroonga/vendor/groonga/lib/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/lib/CMakeLists.txt @@ -22,8 +22,14 @@ include_directories( ${ONIGMO_INCLUDE_DIRS} ${MRUBY_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS}) -link_directories( - ${LIBLZ4_LIBRARY_DIRS}) +if (LIBLZ4_LIBRARY_DIRS) + find_library(LZ4_LIBS + NAMES ${LIBLZ4_LIBRARIES} + PATHS ${LIBLZ4_LIBRARY_DIRS} + NO_DEFAULT_PATH) +else() + set(LZ4_LIBS ${LIBLZ4_LIBRARIES}) +endif() read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/sources.am LIBGROONGA_SOURCES) read_file_list(${CMAKE_CURRENT_SOURCE_DIR}/dat/sources.am LIBGRNDAT_SOURCES) @@ -60,7 +66,7 @@ set(GRN_ALL_LIBRARIES ${RT_LIBS} ${PTHREAD_LIBS} ${Z_LIBS} - ${LIBLZ4_LIBRARIES} + ${LZ4_LIBS} ${DL_LIBS} ${M_LIBS} ${WS2_32_LIBS} From e9ad4bdb4252e0a4a3799364e9a27c3865a19df1 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Tue, 28 Feb 2017 15:23:44 +0400 Subject: [PATCH 20/28] MDEV-11221 - main.events_restart failed in bb Applied lost in a merge revision 7f38a07: MDEV-10043 - main.events_restart fails sporadically in buildbot (crashes upon shutdown) There was race condition between shutdown thread and event worker threads. Shutdown thread waits for thread_count to become 0 in close_connections(). It may happen so that event worker thread was started but didn't increment thread_count by this time. In this case shutdown thread may miss wait for this working thread and continue deinitialization. Worker thread in turn may continue execution and crash on deinitialized data. Fixed by incrementing thread_count before thread is actually created like it is done for connection threads. Also let event scheduler not to inc/dec running threads counter for symmetry with other "service" threads. --- sql/event_scheduler.cc | 40 ++++++++++++++-------------------------- sql/mysqld.cc | 17 ----------------- sql/mysqld.h | 1 - 3 files changed, 14 insertions(+), 44 deletions(-) diff --git a/sql/event_scheduler.cc b/sql/event_scheduler.cc index 5c4926c830c..2423fac4ca6 100644 --- a/sql/event_scheduler.cc +++ b/sql/event_scheduler.cc @@ -133,12 +133,6 @@ post_init_event_thread(THD *thd) thd->cleanup(); return TRUE; } - - thread_safe_increment32(&thread_count, &thread_count_lock); - mysql_mutex_lock(&LOCK_thread_count); - threads.append(thd); - mysql_mutex_unlock(&LOCK_thread_count); - inc_thread_running(); return FALSE; } @@ -157,7 +151,13 @@ deinit_event_thread(THD *thd) thd->proc_info= "Clearing"; DBUG_PRINT("exit", ("Event thread finishing")); - delete_running_thd(thd); + mysql_mutex_lock(&LOCK_thread_count); + thd->unlink(); + mysql_mutex_unlock(&LOCK_thread_count); + + delete thd; + thread_safe_decrement32(&thread_count, &thread_count_lock); + signal_thd_deleted(); } @@ -191,8 +191,10 @@ pre_init_event_thread(THD* thd) thd->net.read_timeout= slave_net_timeout; thd->variables.option_bits|= OPTION_AUTO_IS_NULL; thd->client_capabilities|= CLIENT_MULTI_RESULTS; + thread_safe_increment32(&thread_count, &thread_count_lock); mysql_mutex_lock(&LOCK_thread_count); thd->thread_id= thd->variables.pseudo_thread_id= thread_id++; + threads.append(thd); mysql_mutex_unlock(&LOCK_thread_count); /* @@ -240,13 +242,8 @@ event_scheduler_thread(void *arg) my_free(arg); if (!res) scheduler->run(thd); - else - { - thd->proc_info= "Clearing"; - net_end(&thd->net); - delete thd; - } + deinit_event_thread(thd); DBUG_LEAVE; // Against gcc warnings my_thread_end(); return 0; @@ -310,6 +307,7 @@ Event_worker_thread::run(THD *thd, Event_queue_element_for_exec *event) DBUG_ENTER("Event_worker_thread::run"); DBUG_PRINT("info", ("Time is %ld, THD: 0x%lx", (long) my_time(0), (long) thd)); + inc_thread_running(); if (res) goto end; @@ -338,6 +336,7 @@ end: event->name.str)); delete event; + dec_thread_running(); deinit_event_thread(thd); DBUG_VOID_RETURN; @@ -442,13 +441,9 @@ Event_scheduler::start(int *err_no) " Can not create thread for event scheduler (errno=%d)", *err_no); - new_thd->proc_info= "Clearing"; - DBUG_ASSERT(new_thd->net.buff != 0); - net_end(&new_thd->net); - state= INITIALIZED; scheduler_thd= NULL; - delete new_thd; + deinit_event_thread(new_thd); delete scheduler_param_value; ret= true; @@ -515,7 +510,6 @@ Event_scheduler::run(THD *thd) } LOCK_DATA(); - deinit_event_thread(thd); scheduler_thd= NULL; state= INITIALIZED; DBUG_PRINT("info", ("Broadcasting COND_state back to the stoppers")); @@ -575,10 +569,7 @@ Event_scheduler::execute_top(Event_queue_element_for_exec *event_name) sql_print_error("Event_scheduler::execute_top: Can not create event worker" " thread (errno=%d). Stopping event scheduler", res); - new_thd->proc_info= "Clearing"; - DBUG_ASSERT(new_thd->net.buff != 0); - net_end(&new_thd->net); - + deinit_event_thread(new_thd); goto error; } @@ -590,9 +581,6 @@ Event_scheduler::execute_top(Event_queue_element_for_exec *event_name) error: DBUG_PRINT("error", ("Event_scheduler::execute_top() res: %d", res)); - if (new_thd) - delete new_thd; - delete event_name; DBUG_RETURN(TRUE); } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5692c88d6c8..63bc83af5d5 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2647,23 +2647,6 @@ void dec_connection_count(THD *thd) } -/* - Delete THD and decrement thread counters, including thread_running -*/ - -void delete_running_thd(THD *thd) -{ - mysql_mutex_lock(&LOCK_thread_count); - thd->unlink(); - mysql_mutex_unlock(&LOCK_thread_count); - - delete thd; - dec_thread_running(); - thread_safe_decrement32(&thread_count, &thread_count_lock); - signal_thd_deleted(); -} - - /* Send a signal to unblock close_conneciton() if there is no more threads running with a THD attached diff --git a/sql/mysqld.h b/sql/mysqld.h index 4538b7c4c70..28ac871d858 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -59,7 +59,6 @@ void kill_mysql(void); void close_connection(THD *thd, uint sql_errno= 0); void handle_connection_in_main_thread(THD *thd); void create_thread_to_handle_connection(THD *thd); -void delete_running_thd(THD *thd); void signal_thd_deleted(); void unlink_thd(THD *thd); bool one_thread_per_connection_end(THD *thd, bool put_in_cache); From c1c5b7a8d26b446db18c5d16aca523252e97d078 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Wed, 1 Mar 2017 11:41:48 +0400 Subject: [PATCH 21/28] Fixed missing DBUG_RETURN --- sql/rpl_mi.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc index 59ada30a46d..fd501e62898 100644 --- a/sql/rpl_mi.cc +++ b/sql/rpl_mi.cc @@ -1430,7 +1430,7 @@ uint any_slave_sql_running() if (unlikely(shutdown_in_progress || !master_info_index)) { mysql_mutex_unlock(&LOCK_active_mi); - return 1; + DBUG_RETURN(1); } hash= &master_info_index->master_info_hash; for (uint i= 0; i< hash->records; ++i) From 71f53bf72d918d99910f4a3e8cadbd5ed98a3021 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Thu, 2 Mar 2017 12:35:31 +0400 Subject: [PATCH 22/28] MDEV-11221 - main.events_restart failed in bb This is an addition to original fix. Buildbot revealed another sporadic failure in perfschema.threads_mysql test. Tests relies on data stored in performance_schema.threads, while performing waits on information_schema.processlist. These tables are not updated synchronously. Fixed by performing waits on performance_schema.threads instead. --- .../include/pfs_no_running_event_scheduler.inc | 10 ++++++++++ .../include/pfs_running_event_scheduler.inc | 10 ++++++++++ .../suite/perfschema/t/threads_mysql.test | 18 +++--------------- 3 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 mysql-test/suite/perfschema/include/pfs_no_running_event_scheduler.inc create mode 100644 mysql-test/suite/perfschema/include/pfs_running_event_scheduler.inc diff --git a/mysql-test/suite/perfschema/include/pfs_no_running_event_scheduler.inc b/mysql-test/suite/perfschema/include/pfs_no_running_event_scheduler.inc new file mode 100644 index 00000000000..eff3d7df854 --- /dev/null +++ b/mysql-test/suite/perfschema/include/pfs_no_running_event_scheduler.inc @@ -0,0 +1,10 @@ +# threads are removed from: +# - information_schema.processlist +# - performance_schema.threads +# at different times, so we may have to wait a little more +# for the event_scheduler to shutdown +# +let $wait_condition= + SELECT COUNT(*) = 0 FROM performance_schema.threads + WHERE name like 'thread/sql/event%'; +--source include/wait_condition.inc diff --git a/mysql-test/suite/perfschema/include/pfs_running_event_scheduler.inc b/mysql-test/suite/perfschema/include/pfs_running_event_scheduler.inc new file mode 100644 index 00000000000..219a41051fb --- /dev/null +++ b/mysql-test/suite/perfschema/include/pfs_running_event_scheduler.inc @@ -0,0 +1,10 @@ +# threads are removed from: +# - information_schema.processlist +# - performance_schema.threads +# at different times, so we may have to wait a little more +# for the event_scheduler to shutdown +# +let $wait_condition= + SELECT COUNT(*) = 1 FROM performance_schema.threads + WHERE name like 'thread/sql/event%'; +--source include/wait_condition.inc diff --git a/mysql-test/suite/perfschema/t/threads_mysql.test b/mysql-test/suite/perfschema/t/threads_mysql.test index 8576c8767d6..c33f421863e 100644 --- a/mysql-test/suite/perfschema/t/threads_mysql.test +++ b/mysql-test/suite/perfschema/t/threads_mysql.test @@ -9,22 +9,10 @@ # Ensure that the event scheduler (started via threads_mysql-master.opt) # is really running. ---source include/running_event_scheduler.inc +--source include/pfs_running_event_scheduler.inc SET GLOBAL event_scheduler = OFF; ---source include/no_running_event_scheduler.inc - -# threads are removed from: -# - information_schema.processlist -# - performance_schema.threads -# at different times, so we may have to wait a little more -# for the event_scheduler to shutdown -# -let $wait_timeout= 1; -let $wait_condition= - SELECT COUNT(*) = 0 FROM performance_schema.threads - WHERE name like 'thread/sql/event%'; ---source include/wait_condition.inc +--source include/pfs_no_running_event_scheduler.inc --vertical_results @@ -59,7 +47,7 @@ WHERE name LIKE 'thread/sql%'; SET GLOBAL event_scheduler = ON; ---source include/running_event_scheduler.inc +--source include/pfs_running_event_scheduler.inc # Show entries belonging to the just started event scheduler SELECT name, type, processlist_user, processlist_host, processlist_db, From 659047b8207a1212adeaa80c23b22da929f22b1b Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 23 Jan 2017 08:34:59 +1100 Subject: [PATCH 23/28] MDEV-11386: Advance Toochain library cache workaround (temporary) Due to the way Advance Toolchain before 10.0-3 and 8.0-8 is packaged, shared libraries installed after the library cache advance-toolchain-atX.Y-runtime package aren't updated in /opt/atX.Y/etc/ld.so.cache. This results in mysqld, configured with RUNPATH set to /opt/atX.Y/lib64, resulting in the Advance Toolchain loader being used and if libraries such as jemalloc, libssl or any other library that mysqld uses is installed after Advance Toolchain, these libraries aren't in the cache and therefore won't be found in the RPM postinstall when mysqld is executed by mysql_install_db. Signed-off-by: Daniel Black --- support-files/rpm/server-postin.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/support-files/rpm/server-postin.sh b/support-files/rpm/server-postin.sh index 377a752824d..5e7da5ab828 100644 --- a/support-files/rpm/server-postin.sh +++ b/support-files/rpm/server-postin.sh @@ -31,6 +31,11 @@ if [ $1 = 1 ] ; then # The user may already exist, make sure it has the proper group nevertheless (BUG#12823) usermod --gid %{mysqld_group} %{mysqld_user} 2> /dev/null || true + # Temporary Workaround for MDEV-11386 - will be corrected in Advance Toolchain 10.0-3 and 8.0-8 + for ldconfig in /opt/at*/sbin/ldconfig; do + test -x $ldconfig && $ldconfig + done + # Change permissions so that the user that will run the MySQL daemon # owns all database files. chown -R %{mysqld_user}:%{mysqld_group} $datadir From 33c1f20d8e549c4c48fda4c8bb95e024d96f7629 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 28 Feb 2017 20:21:19 +1100 Subject: [PATCH 24/28] MDEV-11610: Add --local to mysqladmin --- client/mysqladmin.cc | 20 +++++++++++++++++--- man/mysqladmin.1 | 12 ++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/client/mysqladmin.cc b/client/mysqladmin.cc index 7f32658f1b2..a41da5dc444 100644 --- a/client/mysqladmin.cc +++ b/client/mysqladmin.cc @@ -38,8 +38,8 @@ char ex_var_names[MAX_MYSQL_VAR][FN_REFLEN]; ulonglong last_values[MAX_MYSQL_VAR]; static int interval=0; static my_bool option_force=0,interrupted=0,new_line=0, - opt_compress=0, opt_relative=0, opt_verbose=0, opt_vertical=0, - tty_password= 0, opt_nobeep; + opt_compress= 0, opt_local= 0, opt_relative= 0, opt_verbose= 0, + opt_vertical= 0, tty_password= 0, opt_nobeep; static my_bool debug_info_flag= 0, debug_check_flag= 0; static uint tcp_port = 0, option_wait = 0, option_silent=0, nr_iterations; static uint opt_count_iterations= 0, my_end_arg; @@ -160,6 +160,9 @@ static struct my_option my_long_options[] = NO_ARG, 0, 0, 0, 0, 0, 0}, {"host", 'h', "Connect to host.", &host, &host, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"local", 'l', "Local command, don't write to binlog.", + &opt_local, &opt_local, 0, GET_BOOL, NO_ARG, 0, 0, 0, + 0, 0, 0}, {"no-beep", 'b', "Turn off beep on error.", &opt_nobeep, &opt_nobeep, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"password", 'p', @@ -617,6 +620,18 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv) */ struct my_rnd_struct rand_st; + char buff[FN_REFLEN + 20]; + + if (opt_local) + { + sprintf(buff, "set local sql_log_bin=0"); + if (mysql_query(mysql, buff)) + { + my_printf_error(0, "SET LOCAL SQL_LOG_BIN=0 failed; error: '%-.200s'", + error_flags, mysql_error(mysql)); + return -1; + } + } for (; argc > 0 ; argv++,argc--) { @@ -624,7 +639,6 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv) switch ((command= find_type(argv[0],&command_typelib,FIND_TYPE_BASIC))) { case ADMIN_CREATE: { - char buff[FN_REFLEN+20]; if (argc < 2) { my_printf_error(0, "Too few arguments to create", error_flags); diff --git a/man/mysqladmin.1 b/man/mysqladmin.1 index e2b1753ff9f..65f348cab0c 100644 --- a/man/mysqladmin.1 +++ b/man/mysqladmin.1 @@ -816,6 +816,18 @@ Connect to the MariaDB server on the given host\&. .sp -1 .IP \(bu 2.3 .\} +.\" mysqladmin: local option +.\" local option: mysqladmin +\fB\-\-local\fR, +\fB\-l\fR +.sp +Suppress the SQL command(s) from being written to the binary log by enabled sql_log_bin=0 for the session\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} .\" mysqladmin: no-beep option .\" no-beep option: mysqladmin \fB\-\-no\-beep\fR, From 0af8b565f282f5e765fb279e28740385f3dbdaf1 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 28 Feb 2017 21:39:34 +1100 Subject: [PATCH 25/28] MDEV-11610: mysqladmin flush-X-log options Add the following options to mysqladmin + flush-binary-log Flush binary log + flush-engine-log Flush engine log(s) + flush-error-log Flush error log + flush-general-log Flush general log + flush-relay-log Flush relay log + flush-user-resources Flush user resources --- client/mysqladmin.cc | 78 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 4 deletions(-) diff --git a/client/mysqladmin.cc b/client/mysqladmin.cc index a41da5dc444..78a73c01218 100644 --- a/client/mysqladmin.cc +++ b/client/mysqladmin.cc @@ -102,9 +102,12 @@ enum commands { ADMIN_PING, ADMIN_EXTENDED_STATUS, ADMIN_FLUSH_STATUS, ADMIN_FLUSH_PRIVILEGES, ADMIN_START_SLAVE, ADMIN_STOP_SLAVE, ADMIN_START_ALL_SLAVES, ADMIN_STOP_ALL_SLAVES, - ADMIN_FLUSH_THREADS, ADMIN_OLD_PASSWORD, ADMIN_FLUSH_SLOW_LOG, + ADMIN_FLUSH_THREADS, ADMIN_OLD_PASSWORD, ADMIN_FLUSH_BINARY_LOG, + ADMIN_FLUSH_ENGINE_LOG, ADMIN_FLUSH_ERROR_LOG, ADMIN_FLUSH_GENERAL_LOG, + ADMIN_FLUSH_RELAY_LOG, ADMIN_FLUSH_SLOW_LOG, ADMIN_FLUSH_TABLE_STATISTICS, ADMIN_FLUSH_INDEX_STATISTICS, ADMIN_FLUSH_USER_STATISTICS, ADMIN_FLUSH_CLIENT_STATISTICS, + ADMIN_FLUSH_USER_RESOURCES, ADMIN_FLUSH_ALL_STATUS, ADMIN_FLUSH_ALL_STATISTICS }; static const char *command_names[]= { @@ -116,9 +119,10 @@ static const char *command_names[]= { "ping", "extended-status", "flush-status", "flush-privileges", "start-slave", "stop-slave", "start-all-slaves", "stop-all-slaves", - "flush-threads", "old-password", "flush-slow-log", + "flush-threads", "old-password", "flush-binary-log", "flush-engine-log", + "flush-error-log", "flush-general-log", "flush-relay-log", "flush-slow-log", "flush-table-statistics", "flush-index-statistics", - "flush-user-statistics", "flush-client-statistics", + "flush-user-statistics", "flush-client-statistics", "flush-user-resources", "flush-all-status", "flush-all-statistics", NullS }; @@ -916,6 +920,56 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv) } break; } + case ADMIN_FLUSH_BINARY_LOG: + { + if (mysql_query(mysql, "flush binary logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } + case ADMIN_FLUSH_ENGINE_LOG: + { + if (mysql_query(mysql,"flush engine logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } + case ADMIN_FLUSH_ERROR_LOG: + { + if (mysql_query(mysql, "flush error logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } + case ADMIN_FLUSH_GENERAL_LOG: + { + if (mysql_query(mysql, "flush general logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } + case ADMIN_FLUSH_RELAY_LOG: + { + if (mysql_query(mysql, "flush relay logs")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } case ADMIN_FLUSH_SLOW_LOG: { if (mysql_query(mysql,"flush slow logs")) @@ -986,6 +1040,16 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv) } break; } + case ADMIN_FLUSH_USER_RESOURCES: + { + if (mysql_query(mysql, "flush user_resources")) + { + my_printf_error(0, "flush failed; error: '%s'", error_flags, + mysql_error(mysql)); + return -1; + } + break; + } case ADMIN_FLUSH_CLIENT_STATISTICS: { if (mysql_query(mysql,"flush client_statistics")) @@ -1309,12 +1373,18 @@ static void usage(void) flush-index-statistics Flush index statistics\n\ flush-logs Flush all logs\n\ flush-privileges Reload grant tables (same as reload)\n\ + flush-binary-log Flush binary log\n\ + flush-engine-log Flush engine log(s)\n\ + flush-error-log Flush error log\n\ + flush-general-log Flush general log\n\ + flush-relay-log Flush relay log\n\ flush-slow-log Flush slow query log\n\ - flush-status Clear status variables\n\ + flush-status Clear status variables\n\ flush-table-statistics Clear table statistics\n\ flush-tables Flush all tables\n\ flush-threads Flush the thread cache\n\ flush-user-statistics Flush user statistics\n\ + flush-user-resources Flush user resources\n\ kill id,id,... Kill mysql threads"); #if MYSQL_VERSION_ID >= 32200 puts("\ From 156cf86defdc59353f37f6bd5b685d91004f38aa Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 28 Feb 2017 21:47:44 +1100 Subject: [PATCH 26/28] MDEV-11610: Alter Debian log rotate to not rotate binary/relay logs Rotating binary/relay logs can cause interuption to the processing on the server. Binary and relay logs have their own mechanism already for not getting out of control (expire_logs_days). By no longer rotating binary and relay logs log rotation is limited to the following logs: * error log * general log * slow query log Writing these to the binary log would cause any logrotation on the slave to occur twice, once due to this and another due to the log- rotate script on the slave. Now --local is passed to mysqladmin to prevent this duplication. --- debian/mariadb-server-10.0.mysql-server.logrotate | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/mariadb-server-10.0.mysql-server.logrotate b/debian/mariadb-server-10.0.mysql-server.logrotate index a19e9ec46a2..dbe2a7e7dfe 100644 --- a/debian/mariadb-server-10.0.mysql-server.logrotate +++ b/debian/mariadb-server-10.0.mysql-server.logrotate @@ -14,7 +14,8 @@ if [ -f `my_print_defaults --mysqld | grep -oP "pid-file=\K[^$]+"` ]; then # If this fails, check debian.conf! - mysqladmin --defaults-file=/etc/mysql/debian.cnf flush-logs + mysqladmin --defaults-file=/etc/mysql/debian.cnf --local flush-error-log \ + flush-engine-log flush-general-log flush-slow-log fi endscript } From d04d835f64b39edf0310e0471a7ed261b5cb2a20 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 28 Feb 2017 22:26:53 +1100 Subject: [PATCH 27/28] MDEV-11610: support-files/mysql-log-rotate.sh not binlog either For the same reason as removing binlog rotation on Debian. --- support-files/mysql-log-rotate.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/support-files/mysql-log-rotate.sh b/support-files/mysql-log-rotate.sh index 0ee78b0f7ca..5d1b30b208e 100644 --- a/support-files/mysql-log-rotate.sh +++ b/support-files/mysql-log-rotate.sh @@ -30,7 +30,8 @@ if test -x @bindir@/mysqladmin && \ @bindir@/mysqladmin ping &>/dev/null then - @bindir@/mysqladmin flush-logs + @bindir@/mysqladmin --local flush-error-log \ + flush-engine-log flush-general-log flush-slow-log fi endscript } From 29c776cfd1e560846e394f39d79ae43ff7d70c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 3 Mar 2017 12:03:33 +0200 Subject: [PATCH 28/28] MDEV-11520: Retry posix_fallocate() after EINTR. The function posix_fallocate() as well as the Linux system call fallocate() can return EINTR when the operation was interrupted by a signal. In that case, keep retrying the operation, except if InnoDB shutdown has been initiated. --- storage/innobase/fil/fil0fil.cc | 7 ++++++- storage/innobase/os/os0file.cc | 7 ++++++- storage/xtradb/fil/fil0fil.cc | 7 ++++++- storage/xtradb/os/os0file.cc | 7 ++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index b0d489cf701..72ee8d74a82 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -5023,7 +5023,12 @@ retry: = size_after_extend - start_page_no; const os_offset_t len = os_offset_t(n_pages) * page_size; - int err = posix_fallocate(node->handle, start_offset, len); + int err; + do { + err = posix_fallocate(node->handle, start_offset, len); + } while (err == EINTR + && srv_shutdown_state == SRV_SHUTDOWN_NONE); + success = !err; if (!success) { ib_logf(IB_LOG_LEVEL_ERROR, "extending file %s" diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 9fdd52bf736..eb8b3a53d32 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2130,7 +2130,12 @@ os_file_set_size( #ifdef HAVE_POSIX_FALLOCATE if (srv_use_posix_fallocate) { - int err = posix_fallocate(file, 0, size); + int err; + do { + err = posix_fallocate(file, 0, size); + } while (err == EINTR + && srv_shutdown_state == SRV_SHUTDOWN_NONE); + if (err) { ib_logf(IB_LOG_LEVEL_ERROR, "preallocating " INT64PF " bytes for" diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 3b0b1da57e9..1a866a693ca 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -5063,7 +5063,12 @@ retry: = size_after_extend - start_page_no; const os_offset_t len = os_offset_t(n_pages) * page_size; - int err = posix_fallocate(node->handle, start_offset, len); + int err; + do { + err = posix_fallocate(node->handle, start_offset, len); + } while (err == EINTR + && srv_shutdown_state == SRV_SHUTDOWN_NONE); + success = !err; if (!success) { ib_logf(IB_LOG_LEVEL_ERROR, "extending file %s" diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 5e7a0251a00..bed201991e9 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2348,7 +2348,12 @@ os_file_set_size( #ifdef HAVE_POSIX_FALLOCATE if (srv_use_posix_fallocate) { - int err = posix_fallocate(file, 0, size); + int err; + do { + err = posix_fallocate(file, 0, size); + } while (err == EINTR + && srv_shutdown_state == SRV_SHUTDOWN_NONE); + if (err) { ib_logf(IB_LOG_LEVEL_ERROR, "preallocating " INT64PF " bytes for"