From 653aadcd59f4ac94750d9ebf398d6e44029ebe3f Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Tue, 15 Sep 2015 16:27:04 -0400 Subject: [PATCH 1/6] MDEV-8804: bootstrap command missing in debian init script Add 'bootstrap' to debian init script. --- debian/mariadb-galera-server-5.5.mysql.init | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/debian/mariadb-galera-server-5.5.mysql.init b/debian/mariadb-galera-server-5.5.mysql.init index 07edda2f079..d604075b9e4 100644 --- a/debian/mariadb-galera-server-5.5.mysql.init +++ b/debian/mariadb-galera-server-5.5.mysql.init @@ -179,8 +179,15 @@ case "${1:-''}" in fi ;; + 'bootstrap') + # Bootstrap the cluster, start the first node + # that initiates the cluster + log_daemon_msg "Bootstrapping the cluster" "mysqld" + $SELF start "${@:2}" --wsrep-new-cluster + ;; + *) - echo "Usage: $SELF start|stop|restart|reload|force-reload|status" + echo "Usage: $SELF start|stop|restart|reload|force-reload|status|bootstrap" exit 1 ;; esac From fd1b2e486a9a81ffb5416e7a0a0d85d15598c77c Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Tue, 15 Sep 2015 17:07:41 -0400 Subject: [PATCH 2/6] MDEV-8803: Debian jessie 8.2 + MariaDB 10.1.7 + GaleraCluster * Merge fix for issue #127 from mysql-wsrep (commit : a0ba0d7) * Also, cherry-picked a similar fix for OpenSUSE (commit : 7790cb1) --- debian/mariadb-galera-server-5.5.mysql.init | 3 +++ support-files/mysql.server.sh | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/debian/mariadb-galera-server-5.5.mysql.init b/debian/mariadb-galera-server-5.5.mysql.init index d604075b9e4..874d099a0c6 100644 --- a/debian/mariadb-galera-server-5.5.mysql.init +++ b/debian/mariadb-galera-server-5.5.mysql.init @@ -17,6 +17,9 @@ set -e set -u ${DEBIAN_SCRIPT_DEBUG:+ set -v -x} +# Prevent Debian's init scripts from calling systemctl +_SYSTEMCTL_SKIP_REDIRECT=true + test -x /usr/sbin/mysqld || exit 0 . /lib/lsb/init-functions diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh index 30d1ecd5f4c..fb9e01ded43 100644 --- a/support-files/mysql.server.sh +++ b/support-files/mysql.server.sh @@ -24,6 +24,14 @@ # Short-Description: start and stop MySQL # Description: MySQL is a very fast and reliable SQL database engine. ### END INIT INFO + +# Prevent OpenSUSE's init scripts from calling systemd, so that +# both 'bootstrap' and 'start' are handled entirely within this +# script +SYSTEMD_NO_WRAP=1 + +# Prevent Debian's init scripts from calling systemctl +_SYSTEMCTL_SKIP_REDIRECT=true # If you install MySQL on some other places than @prefix@, then you # have to do one of the following things for this script to work: From db2e21bf3ea32115e25bc6d9f67249042f16cadb Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Wed, 16 Sep 2015 23:20:57 -0400 Subject: [PATCH 3/6] MDEV-8208: Sporadic SEGFAULT on startup Problem: When mysqld starts as a galera node, it creates 2 system threads (applier & rollbacker) using start_wsrep_THD(). These threads are created before plugin initialization (plugin_init()) for SST methods like rsync and xtrabackup. The threads' initialization itself can proceed in parallel to mysqld's main thread of execution. As a result, the thread initialization code (start_wsrep_THD()) can end up accessing some un/partially initialized structures (like maria_hton, in this particular case) resulting in segfault. Solution: Fixed by calling THD::init_for_queries() (which accesses maria_hton) only after the plugins have been initialized. --- sql/mysqld.cc | 66 ++++++++++++++++++++++++++++++++------------- sql/wsrep_mysqld.cc | 8 ++++++ sql/wsrep_mysqld.h | 1 + 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index a59780f8f4f..415f6c3d617 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4507,11 +4507,6 @@ will be ignored as the --log-bin option is not defined."); // to know mysqld_port now - lp:1071882 #endif /* !EMBEDDED_LIBRARY */ - /* - Plugin initialization (plugin_init()) hasn't happened yet, set - maria_hton to 0. - */ - maria_hton= 0; wsrep_init_startup(true); } } @@ -4593,6 +4588,29 @@ a file name for --log-bin-index option", opt_binlog_index_name); } plugins_are_initialized= TRUE; /* Don't separate from init function */ +#ifdef WITH_WSREP + /* Wait for wsrep threads to get created. */ + if (wsrep_creating_startup_threads == 1) { + mysql_mutex_lock(&LOCK_thread_count); + while (wsrep_running_threads < 2) + { + mysql_cond_wait(&COND_thread_count, &LOCK_thread_count); + } + + /* Now is the time to initialize threads for queries. */ + THD *tmp; + I_List_iterator it(threads); + while ((tmp= it++)) + { + if (tmp->wsrep_applier == true) + { + tmp->init_for_queries(); + } + } + mysql_mutex_unlock(&LOCK_thread_count); + } +#endif + have_csv= plugin_status(STRING_WITH_LEN("csv"), MYSQL_STORAGE_ENGINE_PLUGIN); have_ndbcluster= plugin_status(STRING_WITH_LEN("ndbcluster"), @@ -4825,16 +4843,11 @@ pthread_handler_t start_wsrep_THD(void *arg) THD *thd; wsrep_thd_processor_fun processor= (wsrep_thd_processor_fun)arg; - if (my_thread_init()) + if (my_thread_init() || (!(thd= new THD(true)))) { - WSREP_ERROR("Could not initialize thread"); - return(NULL); + goto error; } - if (!(thd= new THD(true))) - { - return(NULL); - } mysql_mutex_lock(&LOCK_thread_count); thd->thread_id=thread_id++; @@ -4859,13 +4872,13 @@ pthread_handler_t start_wsrep_THD(void *arg) mysql_thread_set_psi_id(thd->thread_id); thd->thr_create_utime= microsecond_interval_timer(); + if (MYSQL_CALLBACK_ELSE(thread_scheduler, init_new_connection_thread, (), 0)) { close_connection(thd, ER_OUT_OF_RESOURCES); statistic_increment(aborted_connects,&LOCK_status); MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0)); - - return(NULL); + goto error; } // @@ -4888,8 +4901,7 @@ pthread_handler_t start_wsrep_THD(void *arg) statistic_increment(aborted_connects,&LOCK_status); MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0)); delete thd; - - return(NULL); + goto error; } thd->system_thread= SYSTEM_THREAD_SLAVE_SQL; @@ -4899,12 +4911,21 @@ pthread_handler_t start_wsrep_THD(void *arg) //thd->version= refresh_version; thd->proc_info= 0; thd->command= COM_SLEEP; - thd->set_time(); - thd->init_for_queries(); + + if (plugins_are_initialized) + { + thd->init_for_queries(); + } mysql_mutex_lock(&LOCK_thread_count); wsrep_running_threads++; mysql_cond_broadcast(&COND_thread_count); + + if (wsrep_running_threads > 2) + { + wsrep_creating_startup_threads= 0; + } + mysql_mutex_unlock(&LOCK_thread_count); processor(thd); @@ -4942,6 +4963,15 @@ pthread_handler_t start_wsrep_THD(void *arg) mysql_mutex_unlock(&LOCK_thread_count); } return(NULL); + +error: + WSREP_ERROR("Failed to create/initialize system thread"); + + /* Abort if its the first applier/rollbacker thread. */ + if (wsrep_creating_startup_threads < 2) + unireg_abort(1); + else + return NULL; } /**/ diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index a154809d132..c570c7b831d 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -65,6 +65,13 @@ my_bool wsrep_restart_slave_activated = 0; // node has dropped, and slave // restart will be needed my_bool wsrep_slave_UK_checks = 0; // slave thread does UK checks my_bool wsrep_slave_FK_checks = 0; // slave thread does FK checks + +/* + Set during the creation of first wsrep applier and rollback threads. + Since these threads are critical, abort if the thread creation fails. +*/ +my_bool wsrep_creating_startup_threads = 0; + /* * End configuration options */ @@ -700,6 +707,7 @@ void wsrep_init_startup (bool first) if (!wsrep_start_replication()) unireg_abort(1); + wsrep_creating_startup_threads= 1; wsrep_create_rollbacker(); wsrep_create_appliers(1); diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index dd47d4dc1a4..e6fe0c04fac 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -99,6 +99,7 @@ extern my_bool wsrep_restart_slave; extern my_bool wsrep_restart_slave_activated; extern my_bool wsrep_slave_FK_checks; extern my_bool wsrep_slave_UK_checks; +extern my_bool wsrep_creating_startup_threads; enum enum_wsrep_OSU_method { WSREP_OSU_TOI, WSREP_OSU_RSU }; enum enum_wsrep_sync_wait { From 9d5767cf43e1243e039b20c1853f5b7812587f2d Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 21 Sep 2015 20:47:05 -0400 Subject: [PATCH 4/6] Post-merge fix. --- sql/wsrep_sst.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc index a73c9706c7d..8008a977a84 100644 --- a/sql/wsrep_sst.cc +++ b/sql/wsrep_sst.cc @@ -458,7 +458,7 @@ static ssize_t sst_prepare_other (const char* method, mysql_real_data_home, wsrep_defaults_file, (int)getpid()); - if (ret < 0 || ret >= (int)sizeof(cmd_str)) + if (ret < 0 || ret >= cmd_len) { WSREP_ERROR("sst_prepare_other(): snprintf() failed: %d", ret); return (ret < 0 ? ret : -EMSGSIZE); @@ -1025,7 +1025,7 @@ static int sst_donate_other (const char* method, uuid, (long long) seqno, bypass ? " "WSREP_SST_OPT_BYPASS : ""); - if (ret < 0 || ret >= (int) sizeof(cmd_str)) + if (ret < 0 || ret >= cmd_len) { WSREP_ERROR("sst_donate_other(): snprintf() failed: %d", ret); return (ret < 0 ? ret : -EMSGSIZE); From d54bc3c0d1ed50b3c5aa3a22b2cb653851315e8a Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Mon, 21 Sep 2015 20:49:31 -0400 Subject: [PATCH 5/6] Cleanup: remove dead code which could also lead to race. --- .../suite/galera/r/galera_read_only.result | 18 +++++++++ .../suite/galera/t/galera_read_only.test | 39 +++++++++++++++++++ sql/sql_parse.cc | 20 +--------- 3 files changed, 59 insertions(+), 18 deletions(-) create mode 100644 mysql-test/suite/galera/r/galera_read_only.result create mode 100644 mysql-test/suite/galera/t/galera_read_only.test diff --git a/mysql-test/suite/galera/r/galera_read_only.result b/mysql-test/suite/galera/r/galera_read_only.result new file mode 100644 index 00000000000..82736c5f4ba --- /dev/null +++ b/mysql-test/suite/galera/r/galera_read_only.result @@ -0,0 +1,18 @@ +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +SET GLOBAL read_only=TRUE; +INSERT INTO t1 VALUES (1); +SELECT COUNT(*) = 1 FROM t1; +COUNT(*) = 1 +1 +CREATE USER foo@localhost; +# Open connection to node 2 using 'foo' user. + +# Connect with foo_node_2 +INSERT INTO t1 VALUES (2); +ERROR HY000: The MariaDB server is running with the --read-only option so it cannot execute this statement +SELECT COUNT(*) = 1 FROM t1; +COUNT(*) = 1 +1 +SET GLOBAL read_only=FALSE; +DROP TABLE t1; +DROP USER foo@localhost; diff --git a/mysql-test/suite/galera/t/galera_read_only.test b/mysql-test/suite/galera/t/galera_read_only.test new file mode 100644 index 00000000000..c0fa4af07e0 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_read_only.test @@ -0,0 +1,39 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +# +# Ensure that the read_only option does not apply to Galera appliers and that replication +# continues, the way MySQL replication would. +# + +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; + +--connection node_2 +SET GLOBAL read_only=TRUE; + +--connection node_1 +INSERT INTO t1 VALUES (1); + +--connection node_2 +SELECT COUNT(*) = 1 FROM t1; + +CREATE USER foo@localhost; + +--echo # Open connection to node 2 using 'foo' user. +--let $port_2= \$NODE_MYPORT_2 +--connect(foo_node_2,127.0.0.1,foo,,test,$port_2,) + +--echo +--echo # Connect with foo_node_2 +--connection foo_node_2 +--error ER_OPTION_PREVENTS_STATEMENT +INSERT INTO t1 VALUES (2); + +--connection node_2 +SELECT COUNT(*) = 1 FROM t1; + +# Cleanup +SET GLOBAL read_only=FALSE; +DROP TABLE t1; +DROP USER foo@localhost; + diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 3629f3dd198..566a50340aa 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -982,22 +982,6 @@ static my_bool deny_updates_if_read_only_option(THD *thd, } #ifdef WITH_WSREP -static my_bool wsrep_read_only_option(THD *thd, TABLE_LIST *all_tables) -{ - int opt_readonly_saved = opt_readonly; - ulong flag_saved = (ulong)(thd->security_ctx->master_access & SUPER_ACL); - - opt_readonly = 0; - thd->security_ctx->master_access &= ~SUPER_ACL; - - my_bool ret = !deny_updates_if_read_only_option(thd, all_tables); - - opt_readonly = opt_readonly_saved; - thd->security_ctx->master_access |= flag_saved; - - return ret; -} - static void wsrep_copy_query(THD *thd) { thd->wsrep_retry_command = thd->command; @@ -1011,6 +995,7 @@ static void wsrep_copy_query(THD *thd) thd->wsrep_retry_query[thd->wsrep_retry_query_len] = '\0'; } #endif /* WITH_WSREP */ + /** Perform one connection-level (COM_XXXX) command. @@ -6355,8 +6340,7 @@ static void wsrep_mysql_parse(THD *thd, char *rawbuf, uint length, bool is_autocommit= !thd->in_multi_stmt_transaction_mode() && thd->wsrep_conflict_state == NO_CONFLICT && - !thd->wsrep_applier && - wsrep_read_only_option(thd, thd->lex->query_tables); + !thd->wsrep_applier; do { From 13615c5e18eed62fa2dee80402dfebe3e74053c4 Mon Sep 17 00:00:00 2001 From: Nirbhay Choubey Date: Fri, 25 Sep 2015 13:56:02 -0400 Subject: [PATCH 6/6] MDEV-8208: Sporadic SEGFAULT on startup Addendum: * Before calling THD::init_for_queries(), flip the current_thd to wsrep thread so that memory gets allocated for the right THD. * Use wsrep_creating_startup_threads instead of plugins_are_initialized as the condition for the execution of THD::init_for_queries() within start_wsrep_THD(), as use of latter could still leave some room for race. --- mysys/my_alloc.c | 5 ++++- sql/mysqld.cc | 12 +++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/mysys/my_alloc.c b/mysys/my_alloc.c index 1054db6cee4..9e44707b1a2 100644 --- a/mysys/my_alloc.c +++ b/mysys/my_alloc.c @@ -93,6 +93,7 @@ void init_alloc_root(MEM_ROOT *mem_root, size_t block_size, void reset_root_defaults(MEM_ROOT *mem_root, size_t block_size, size_t pre_alloc_size __attribute__((unused))) { + DBUG_ENTER("reset_root_defaults"); DBUG_ASSERT(alloc_root_inited(mem_root)); mem_root->block_size= block_size - ALLOC_ROOT_MIN_BLOCK_SIZE; @@ -114,7 +115,7 @@ void reset_root_defaults(MEM_ROOT *mem_root, size_t block_size, { /* We found a suitable block, no need to do anything else */ mem_root->pre_alloc= mem; - return; + DBUG_VOID_RETURN; } if (mem->left + ALIGN_SIZE(sizeof(USED_MEM)) == mem->size) { @@ -142,6 +143,8 @@ void reset_root_defaults(MEM_ROOT *mem_root, size_t block_size, else #endif mem_root->pre_alloc= 0; + + DBUG_VOID_RETURN; } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 415f6c3d617..b42d586d748 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4604,7 +4604,17 @@ a file name for --log-bin-index option", opt_binlog_index_name); { if (tmp->wsrep_applier == true) { + /* + Set THR_THD to temporally point to this THD to register all the + variables that allocates memory for this THD. + */ + THD *current_thd_saved= current_thd; + my_pthread_setspecific_ptr(THR_THD, tmp); + tmp->init_for_queries(); + + /* Restore current_thd. */ + my_pthread_setspecific_ptr(THR_THD, current_thd_saved); } } mysql_mutex_unlock(&LOCK_thread_count); @@ -4912,7 +4922,7 @@ pthread_handler_t start_wsrep_THD(void *arg) thd->proc_info= 0; thd->command= COM_SLEEP; - if (plugins_are_initialized) + if (wsrep_creating_startup_threads == 0) { thd->init_for_queries(); }