MDEV-7108: Make long semaphore wait timeout configurable

Merge Facebook commit cd063ab930
authored by Peng Tian from https://github.com/facebook/mysql-5.6

Introduced a new configuration variable innodb_fatal_semaphore_wait_threshold,
it makes the fatal semaphore timeout configurable. Modified original commit
so that no MariaDB server files are changed, instead introduced a new
InnoDB/XtraDB configuration variable.

Its default/min/max vlaues are 600/1/2^32-1 in seconds (it was hardcoded
as 600, now its default value is 600, so the default behavior of this diff
should be no change).

mysql-test/suite/sys_vars/r/innodb_fatal_semaphore_wait_threshold.result

@@ -0,0 +1,25 @@
+# Establish connection con1 (user=root)
+# Establish connection con2 (user=root)
+drop table if exists t1;
+# Switch to connection con1
+create table t1 (id integer, x integer) engine = InnoDB;
+insert into t1 values(0, 0);
+set @@debug_dbug="d,fatal-semaphore-timeout";
+set autocommit=0;
+# Sending query on con1,
+# the session will hold lock table mutex and sleep
+SELECT * from t1 where id = 0 FOR UPDATE;
+# Switch to connection con2
+set autocommit=0;
+# Sending query on con2,
+# the session will be blocked on the lock table mutex and
+# thus be put into sync arry
+SELECT * from t1 where id = 0 FOR UPDATE;
+# Switched to the default connection
+# Waitting for mysqld to crash
+# Mysqld crash was detected
+# Waitting for reconnect after mysqld restarts
+# Reconnected after mysqld was successfully restarted
+# Cleaning up before exit
+drop table if exists t1;
+# Clean exit

mysql-test/suite/sys_vars/r/innodb_fatal_semaphore_wait_threshold_basic.result

@@ -0,0 +1,21 @@
+select @@global.innodb_fatal_semaphore_wait_threshold;
+@@global.innodb_fatal_semaphore_wait_threshold
+600
+select @@session.innodb_fatal_semaphore_wait_threshold;
+ERROR HY000: Variable 'innodb_fatal_semaphore_wait_threshold' is a GLOBAL variable
+show global variables like 'innodb_fatal_semaphore_wait_threshold';
+Variable_name	Value
+innodb_fatal_semaphore_wait_threshold	600
+show session variables like 'innodb_fatal_semaphore_wait_threshold';
+Variable_name	Value
+innodb_fatal_semaphore_wait_threshold	600
+select * from information_schema.global_variables where variable_name='innodb_fatal_semaphore_wait_threshold';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_FATAL_SEMAPHORE_WAIT_THRESHOLD	600
+select * from information_schema.session_variables where variable_name='innodb_fatal_semaphore_wait_threshold';
+VARIABLE_NAME	VARIABLE_VALUE
+INNODB_FATAL_SEMAPHORE_WAIT_THRESHOLD	600
+set global innodb_fatal_semaphore_wait_threshold=1;
+ERROR HY000: Variable 'innodb_fatal_semaphore_wait_threshold' is a read only variable
+set session innodb_fatal_semaphore_wait_threshold=1;
+ERROR HY000: Variable 'innodb_fatal_semaphore_wait_threshold' is a read only variable

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -663,6 +663,20 @@ NUMERIC_BLOCK_SIZE	0
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	OPTIONAL
+VARIABLE_NAME	INNODB_FATAL_SEMAPHORE_WAIT_THRESHOLD
+SESSION_VALUE	NULL
+GLOBAL_VALUE	600
+GLOBAL_VALUE_ORIGIN	COMPILE-TIME
+DEFAULT_VALUE	600
+VARIABLE_SCOPE	GLOBAL
+VARIABLE_TYPE	BIGINT UNSIGNED
+VARIABLE_COMMENT	Maximum number of seconds that semaphore times out in InnoDB.
+NUMERIC_MIN_VALUE	1
+NUMERIC_MAX_VALUE	4294967295
+NUMERIC_BLOCK_SIZE	0
+ENUM_VALUE_LIST	NULL
+READ_ONLY	YES
+COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	INNODB_FILE_FORMAT
 SESSION_VALUE	NULL
 GLOBAL_VALUE	Antelope

mysql-test/suite/sys_vars/t/innodb_fatal_semaphore_wait_threshold-master.opt

@@ -0,0 +1 @@
+--innodb-fatal-semaphore-wait-threshold=1

mysql-test/suite/sys_vars/t/innodb_fatal_semaphore_wait_threshold.test

@@ -0,0 +1,120 @@
+# Only test in debug mode since DBUG_EXECUTE_IF is used
+--source include/have_debug.inc
+
+# Can't test this with embedded server
+--source include/not_embedded.inc
+
+# Don't test this with valgrind
+--source include/not_valgrind.inc
+
+# Only test for innodb
+--source include/have_innodb.inc
+
+--echo # Establish connection con1 (user=root)
+connect (con1,localhost,root,,);
+--echo # Establish connection con2 (user=root)
+connect (con2,localhost,root,,);
+
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+--echo # Switch to connection con1
+connection con1;
+eval create table t1 (id integer, x integer) engine = InnoDB;
+insert into t1 values(0, 0);
+
+# Enable the debug injection.
+set @@debug_dbug="d,fatal-semaphore-timeout";
+set autocommit=0;
+
+# The following query will hang for an hour since the debug injection
+# code will sleep an hour after holding the lock table mutex
+--echo # Sending query on con1,
+--echo # the session will hold lock table mutex and sleep
+--send
+SELECT * from t1 where id = 0 FOR UPDATE;
+
+# To make sure con1 holding the lock table mutex and sleeping
+--sleep 2
+
+--echo # Switch to connection con2
+connection con2;
+set autocommit=0;
+
+# The following query will be blocked on the lock table mutex held by
+# con1 so it will be put into sync array.
+--echo # Sending query on con2,
+--echo # the session will be blocked on the lock table mutex and
+--echo # thus be put into sync arry
+--send
+SELECT * from t1 where id = 0 FOR UPDATE;
+
+# Waitting for mysqld to abort due to fatal semaphore timeout.
+# Please note that, in the master.opt file, the fatal timeout
+# was set to 1 second, but in mysqld debug mode, this timeout
+# value will be timed 10 because UNIV_DEBUG_VALGRIND is set
+# (see sync_array_print_long_waits_low() in storage/innobase/sync/sync0arr.cc)
+# so the actual timeout will be 1 * 10 = 10 seconds. Besides,
+# mysqld will abort after detecting this fatal timeout 10 times in
+# a loop with interval of 1 second (see srv_error_monitor_thread
+# thread in torage/innobase/srv/srv0srv.cc), so mysqld will abort
+# in 1 * 10 + 1 * 10 = 20 seconds after con2 being blocked on
+# the lock table mutex.
+#
+# P.S. the default fatal sempahore timeout is 600 seconds,
+# so mysqld will abort after 600 * 10 + 1 * 10 = 6010 seconds
+# in debug mode and 600 + 1 * 10 = 610 seconds in release mode.
+
+--echo # Switched to the default connection
+connection default;
+
+--disable_result_log
+--disable_query_log
+
+# Since this test generates lot of errors in log, suppress checking errors
+call mtr.add_suppression(".*");
+
+# The crash is expected
+exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect;
+
+--echo # Waitting for mysqld to crash
+
+# It will take 20 seconds to detect the long semaphore and mysqld to abort.
+# This test will be treated as pass as long as mysqld crash/restart is dectected
+# in 60 seconds.
+let $counter= 60;
+let $mysql_errno= 0;
+while (!$mysql_errno)
+{
+  --error 0,1040,1053,2002,2003,2006,2013
+  show status;
+
+  dec $counter;
+  if (!$counter)
+  {
+    # This will fail this test.
+    --die Server failed to dissapear
+  }
+  --sleep 1
+}
+
+--echo # Mysqld crash was detected
+--echo # Waitting for reconnect after mysqld restarts
+
+enable_reconnect;
+connection default;
+
+--exec echo "restart:--log-error=$error_log" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+
+# Call script that will poll the server waiting for it to be back online again
+source include/wait_until_connected_again.inc;
+
+--echo # Reconnected after mysqld was successfully restarted
+
+--echo # Cleaning up before exit
+--disable_warnings
+drop table if exists t1;
+--enable_warnings
+
+--echo # Clean exit

mysql-test/suite/sys_vars/t/innodb_fatal_semaphore_wait_threshold_basic.test

@@ -0,0 +1,21 @@
+--source include/have_innodb.inc
+# bool readonly
+
+#
+# show values;
+#
+select @@global.innodb_fatal_semaphore_wait_threshold;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+select @@session.innodb_fatal_semaphore_wait_threshold;
+show global variables like 'innodb_fatal_semaphore_wait_threshold';
+show session variables like 'innodb_fatal_semaphore_wait_threshold';
+select * from information_schema.global_variables where variable_name='innodb_fatal_semaphore_wait_threshold';
+select * from information_schema.session_variables where variable_name='innodb_fatal_semaphore_wait_threshold';
+
+#
+# show that it's read-only
+#
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+set global innodb_fatal_semaphore_wait_threshold=1;
+--error ER_INCORRECT_GLOBAL_LOCAL_VAR
+set session innodb_fatal_semaphore_wait_threshold=1;