MDEV-35710 support for threadpool

When client connections use threadpool, i.e. configuration has:
thread_handling = pool-of-threads it turned out that during wsrep
replication shutdown, not all client connections could be closed.
Reason was that some client threads has stmt_da in state DA_EOF,
and this state was earlier used to detect if client connection
was issuing SHUTDOWN command.

To fix this, the connection executing SHUTDOWN is now detected by
looking at the actual command being executed:
thd->get_command() == COM_SHUTDOWN
During replication shutdown, all other connections but the
SHUTDOWN executor, are terminated.

This commit has new mtr test galera.galera_threadpool, which
opens a number of threadpool client connections, and then
restarts the node to verify that connections in threadpool are
terminated during shutdown.

Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>

mysql-test/suite/galera/r/galera_threadpool.result

@@ -0,0 +1,36 @@
+connection node_2;
+connection node_1;
+connection node_1;
+connection node_2;
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connect node_2c, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connect node_2d, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connect node_2e, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_2a;
+CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY, f2 char) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (0,'a');
+INSERT INTO t1 VALUES (1,'a');
+connection node_2b;
+SELECT * FROM t1;
+f1	f2
+0	a
+1	a
+connection node_2c;
+INSERT INTO t1 VALUES (2,'c');
+connection node_2d;
+BEGIN;
+SELECT * FROM t1;
+f1	f2
+0	a
+1	a
+2	c
+connection node_2e;
+BEGIN;
+UPDATE t1 SET f2='e' WHERE f1=0;
+connection node_2;
+connection node_1;
+connection node_2;
+connection node_1;
+connection node_2;
+DROP TABLE t1;

mysql-test/suite/galera/t/galera_threadpool.cnf

@@ -0,0 +1,17 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-node-name="node1"
+log_bin=binlog
+log_slave_updates=ON
+wsrep_sst_method=rsync
+thread_handling = pool-of-threads
+
+[mysqld.2]
+wsrep-node-name="node2"
+log_bin=binlog
+log_slave_updates=ON
+wsrep_sst_method=rsync
+thread_handling = pool-of-threads
+
+

mysql-test/suite/galera/t/galera_threadpool.test

@@ -0,0 +1,67 @@
+#
+# Tests for threadpool support
+#
+--source include/galera_cluster.inc
+
+--let $node_1 = node_1
+--let $node_2 = node_2
+
+--source ../galera/include/auto_increment_offset_save.inc
+
+#
+# start connections in node 2, and execute some SQL statements
+# leave also open transactions in the node
+#
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connect node_2b, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connect node_2c, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connect node_2d, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connect node_2e, 127.0.0.1, root, , test, $NODE_MYPORT_2
+
+--connection node_2a
+CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY, f2 char) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (0,'a');
+INSERT INTO t1 VALUES (1,'a');
+
+--connection node_2b
+SELECT * FROM t1;
+
+--connection node_2c
+INSERT INTO t1 VALUES (2,'c');
+
+--connection node_2d
+BEGIN;
+SELECT * FROM t1;
+
+--connection node_2e
+BEGIN;
+UPDATE t1 SET f2='e' WHERE f1=0;
+
+#
+# Shut down node 2, all open connections should be closed
+#
+--connection node_2
+--source include/shutdown_mysqld.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+# And restart the node
+--connection node_2
+let $restart_noprint=2;
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--source include/start_mysqld.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+--connection node_2
+
+DROP TABLE t1;
+
+#
+# Restore auto increment variables.
+#
+--source ../galera/include/auto_increment_offset_restore.inc

sql/wsrep_mysqld.cc

@@ -2925,8 +2925,6 @@ static inline bool is_committing_connection(THD *thd)
 
 static my_bool have_client_connections(THD *thd, void*)
 {
-  DBUG_PRINT("quit",("Informing thread %lld that it's time to die",
-                     (longlong) thd->thread_id));
   if (is_client_connection(thd))
   {
     if (thd->killed == KILL_CONNECTION ||
@@ -2980,13 +2978,18 @@ static my_bool kill_all_threads(THD *thd, THD *caller_thd)
   /* We skip slave threads & scheduler on this first loop through. */
   if (is_client_connection(thd) && thd != caller_thd)
   {
-    if (thd->get_stmt_da()->is_eof())
+    /* the connection executing SHUTDOWN, should do clean exit,
+       not aborting here */
+    if (thd->get_command() == COM_SHUTDOWN)
     {
+      WSREP_DEBUG("leaving SHUTDOWN executing connection alive, thread: %lld",
+                 (longlong) thd->thread_id);
       return 0;
     }
-
+    /* replaying connection is killed by signal */
     if (is_replaying_connection(thd))
     {
+      WSREP_DEBUG("closing connection is replaying %lld", (longlong) thd->thread_id);
       thd->set_killed(KILL_CONNECTION_HARD);
       return 0;
     }
@@ -2995,7 +2998,7 @@ static my_bool kill_all_threads(THD *thd, THD *caller_thd)
     {
       /* replicated transactions must be skipped */
       WSREP_DEBUG("closing connection %lld", (longlong) thd->thread_id);
-      /* instead of wsrep_close_thread() we do now  soft kill by THD::awake */
+      /* instead of wsrep_close_thread() we do now hard kill by THD::awake */
       thd->awake(KILL_CONNECTION_HARD);
       return 0;
     }
@@ -3036,8 +3039,10 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD* except_caller_thd)
   */
   server_threads.iterate(kill_remaining_threads, except_caller_thd);
 
-  DBUG_PRINT("quit", ("Waiting for threads to die (count=%u)", THD_count::value()));
+  DBUG_PRINT("quit", ("Waiting for threads to die (count=%u)",
-  WSREP_DEBUG("waiting for client connections to close: %u", THD_count::value());
+                      THD_count::value()));
+  WSREP_DEBUG("waiting for client connections to close: %u",
+              THD_count::value());
 
   while (wait_to_end && server_threads.iterate(have_client_connections))
   {