From 9d97f92febc89941784d17d59c60275e21140ce0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Mon, 13 Sep 2021 08:19:29 +0300
Subject: [PATCH 01/31] Revert "MDEV-23328 Server hang due to Galera lock
 conflict resolution" and Revert "MDEV-24873 galera.galera_as_slave_ctas MTR
 failed:..."

This reverts commit 29bbcac0ee841faaa68eeb09c86ff825eabbe6b6 and
later commit 5ecaf52d42a1e464c71515f35be97855072bcafe.
---
 sql/wsrep_mysqld.cc                   |   2 +
 storage/innobase/handler/ha_innodb.cc | 184 ++++++++++----------------
 2 files changed, 74 insertions(+), 112 deletions(-)

diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index f22d8bf0f5a..e60100e2e90 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -2762,7 +2762,9 @@ extern "C" void wsrep_thd_awake(THD *thd, my_bool signal)
 {
   if (signal)
   {
+    mysql_mutex_lock(&thd->LOCK_thd_data);
     thd->awake(KILL_QUERY);
+    mysql_mutex_unlock(&thd->LOCK_thd_data);
   }
   else
   {
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index c17bb219a72..d4142481a62 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -60,7 +60,6 @@ this program; if not, write to the Free Software Foundation, Inc.,
 
 #include <my_service_manager.h>
 #include <key.h>
-#include <sql_manager.h>
 
 /* Include necessary InnoDB headers */
 #include "btr0btr.h"
@@ -19536,68 +19535,61 @@ wsrep_abort_slave_trx(
 		(long long)bf_seqno, (long long)victim_seqno);
 	abort();
 }
-
-struct bg_wsrep_kill_trx_arg {
-	my_thread_id	thd_id;
-	trx_id_t	trx_id;
-	int64_t		bf_seqno;
-	ibool		signal;
-};
-
-static void bg_wsrep_kill_trx(
-	void *void_arg)
+/*******************************************************************//**
+This function is used to kill one transaction in BF. */
+UNIV_INTERN
+void
+wsrep_innobase_kill_one_trx(
+/*========================*/
+	MYSQL_THD const bf_thd,
+	const trx_t * const bf_trx,
+	trx_t *victim_trx,
+	ibool signal)
 {
-	bg_wsrep_kill_trx_arg *arg = (bg_wsrep_kill_trx_arg*)void_arg;
-	THD *thd 		   = find_thread_by_id(arg->thd_id, false);
-	trx_t *victim_trx 	   = NULL;
-	bool awake 		   = false;
-	DBUG_ENTER("bg_wsrep_kill_trx");
+        ut_ad(bf_thd);
+        ut_ad(victim_trx);
+        ut_ad(lock_mutex_own());
+        ut_ad(trx_mutex_own(victim_trx));
 
-	if (thd) {
-		victim_trx= thd_to_trx(thd);
-		/* Victim trx might not exist e.g. on MDL-conflict. */
-		if (victim_trx) {
-			lock_mutex_enter();
-			trx_mutex_enter(victim_trx);
-			if (victim_trx->id != arg->trx_id ||
-			    victim_trx->state == TRX_STATE_COMMITTED_IN_MEMORY)
-			{
-				/* Victim was meanwhile rolled back or
-				committed */
-				trx_mutex_exit(victim_trx);
-				lock_mutex_exit();
-				wsrep_thd_UNLOCK(thd);
-				victim_trx= NULL;
-			}
-		} else {
-			/* find_thread_by_id locked
-			THD::LOCK_thd_data */
-			wsrep_thd_UNLOCK(thd);
-		}
+	DBUG_ENTER("wsrep_innobase_kill_one_trx");
+	THD *thd          = (THD *) victim_trx->mysql_thd;
+	int64_t bf_seqno  = wsrep_thd_trx_seqno(bf_thd);
+
+	if (!thd) {
+		DBUG_PRINT("wsrep", ("no thd for conflicting lock"));
+		WSREP_WARN("no THD for trx: " TRX_ID_FMT, victim_trx->id);
+		DBUG_VOID_RETURN;
 	}
 
-	if (!victim_trx) {
-		/* Victim trx might not exist (MDL-conflict) or victim
-		was meanwhile rolled back or committed because of
-		a KILL statement or a disconnect. */
-		goto ret;
-	}
+	WSREP_LOG_CONFLICT(bf_thd, thd, TRUE);
 
 	WSREP_DEBUG("BF kill (" ULINTPF ", seqno: " INT64PF
 		    "), victim: (%lu) trx: " TRX_ID_FMT,
-		    arg->signal, arg->bf_seqno,
+		    signal, bf_seqno,
 		    thd_get_thread_id(thd),
 		    victim_trx->id);
 
 	WSREP_DEBUG("Aborting query: %s conf %d trx: %" PRId64,
-		    (wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void",
+		    (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void",
 		    wsrep_thd_conflict_state(thd, FALSE),
 		    wsrep_thd_ws_handle(thd)->trx_id);
 
+	wsrep_thd_LOCK(thd);
+        DBUG_EXECUTE_IF("sync.wsrep_after_BF_victim_lock",
+                 {
+                   const char act[]=
+                     "now "
+                     "wait_for signal.wsrep_after_BF_victim_lock";
+                   DBUG_ASSERT(!debug_sync_set_action(bf_thd,
+                                                      STRING_WITH_LEN(act)));
+                 };);
+
+
 	if (wsrep_thd_query_state(thd) == QUERY_EXITING) {
 		WSREP_DEBUG("kill trx EXITING for " TRX_ID_FMT,
 			    victim_trx->id);
-		goto ret_unlock;
+		wsrep_thd_UNLOCK(thd);
+		DBUG_VOID_RETURN;
 	}
 
 	if (wsrep_thd_exec_mode(thd) != LOCAL_STATE) {
@@ -19613,13 +19605,18 @@ static void bg_wsrep_kill_trx(
         case MUST_ABORT:
 		WSREP_DEBUG("victim " TRX_ID_FMT " in MUST ABORT state",
 			    victim_trx->id);
-		goto ret_awake;
+		wsrep_thd_UNLOCK(thd);
+		wsrep_thd_awake(thd, signal);
+		DBUG_VOID_RETURN;
+		break;
 	case ABORTED:
 	case ABORTING: // fall through
 	default:
 		WSREP_DEBUG("victim " TRX_ID_FMT " in state %d",
 			    victim_trx->id, wsrep_thd_get_conflict_state(thd));
-		goto ret_unlock;
+		wsrep_thd_UNLOCK(thd);
+		DBUG_VOID_RETURN;
+		break;
 	}
 
 	switch (wsrep_thd_query_state(thd)) {
@@ -19632,12 +19629,12 @@ static void bg_wsrep_kill_trx(
 			    victim_trx->id);
 
 		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
-			wsrep_abort_slave_trx(arg->bf_seqno,
+			wsrep_abort_slave_trx(bf_seqno,
 					      wsrep_thd_trx_seqno(thd));
 		} else {
 			wsrep_t *wsrep= get_wsrep();
 			rcode = wsrep->abort_pre_commit(
-				wsrep, arg->bf_seqno,
+				wsrep, bf_seqno,
 				(wsrep_trx_id_t)wsrep_thd_ws_handle(thd)->trx_id
 			);
 
@@ -19646,7 +19643,10 @@ static void bg_wsrep_kill_trx(
 				WSREP_DEBUG("cancel commit warning: "
 					    TRX_ID_FMT,
 					    victim_trx->id);
-				goto ret_awake;
+				wsrep_thd_UNLOCK(thd);
+				wsrep_thd_awake(thd, signal);
+				DBUG_VOID_RETURN;
+				break;
 			case WSREP_OK:
 				break;
 			default:
@@ -19659,9 +19659,12 @@ static void bg_wsrep_kill_trx(
 				 * kill the lock holder first.
 				 */
 				abort();
+				break;
 			}
 		}
-		goto ret_awake;
+		wsrep_thd_UNLOCK(thd);
+		wsrep_thd_awake(thd, signal);
+		break;
 	case QUERY_EXEC:
 		/* it is possible that victim trx is itself waiting for some
 		 * other lock. We need to cancel this waiting
@@ -19682,20 +19685,26 @@ static void bg_wsrep_kill_trx(
 				lock_cancel_waiting_and_release(wait_lock);
 			}
 
+			wsrep_thd_UNLOCK(thd);
+			wsrep_thd_awake(thd, signal);
 		} else {
 			/* abort currently executing query */
 			DBUG_PRINT("wsrep",("sending KILL_QUERY to: %lu",
                                             thd_get_thread_id(thd)));
 			WSREP_DEBUG("kill query for: %ld",
 				thd_get_thread_id(thd));
+			/* Note that innobase_kill_query will take lock_mutex
+			and trx_mutex */
+			wsrep_thd_UNLOCK(thd);
+			wsrep_thd_awake(thd, signal);
 
 			/* for BF thd, we need to prevent him from committing */
 			if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
-				wsrep_abort_slave_trx(arg->bf_seqno,
+				wsrep_abort_slave_trx(bf_seqno,
 						    wsrep_thd_trx_seqno(thd));
 			}
 		}
-		goto ret_awake;
+		break;
 	case QUERY_IDLE:
 	{
 		WSREP_DEBUG("kill IDLE for " TRX_ID_FMT, victim_trx->id);
@@ -19703,9 +19712,10 @@ static void bg_wsrep_kill_trx(
 		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
 			WSREP_DEBUG("kill BF IDLE, seqno: %lld",
 				    (long long)wsrep_thd_trx_seqno(thd));
-			wsrep_abort_slave_trx(arg->bf_seqno,
+			wsrep_thd_UNLOCK(thd);
+			wsrep_abort_slave_trx(bf_seqno,
 					      wsrep_thd_trx_seqno(thd));
-			goto ret_unlock;
+			DBUG_VOID_RETURN;
 		}
                 /* This will lock thd from proceeding after net_read() */
 		wsrep_thd_set_conflict_state(thd, ABORTING);
@@ -19726,67 +19736,17 @@ static void bg_wsrep_kill_trx(
 		DBUG_PRINT("wsrep",("signalling wsrep rollbacker"));
 		WSREP_DEBUG("signaling aborter");
 		wsrep_unlock_rollback();
-		goto ret_unlock;
+		wsrep_thd_UNLOCK(thd);
+
+		break;
 	}
 	default:
 		WSREP_WARN("bad wsrep query state: %d",
 			  wsrep_thd_query_state(thd));
-		goto ret_unlock;
+		wsrep_thd_UNLOCK(thd);
+		break;
 	}
 
-ret_awake:
-	awake= true;
-
-ret_unlock:
-	trx_mutex_exit(victim_trx);
-	lock_mutex_exit();
-	if (awake)
-		wsrep_thd_awake(thd, arg->signal);
-	wsrep_thd_UNLOCK(thd);
-
-ret:
-	free(arg);
-	DBUG_VOID_RETURN;
-
-}
-
-/*******************************************************************//**
-This function is used to kill one transaction in BF. */
-UNIV_INTERN
-void
-wsrep_innobase_kill_one_trx(
-/*========================*/
-	MYSQL_THD const bf_thd,
-	const trx_t * const bf_trx,
-	trx_t *victim_trx,
-	ibool signal)
-{
-	ut_ad(bf_thd);
-	ut_ad(victim_trx);
-	ut_ad(lock_mutex_own());
-	ut_ad(trx_mutex_own(victim_trx));
-
-	bg_wsrep_kill_trx_arg *arg = (bg_wsrep_kill_trx_arg*)malloc(sizeof(*arg));
-	arg->thd_id	= thd_get_thread_id(victim_trx->mysql_thd);
-	arg->trx_id	= victim_trx->id;
-	arg->bf_seqno	= wsrep_thd_trx_seqno((THD*)bf_thd);
-	arg->signal	= signal;
-
-	DBUG_ENTER("wsrep_innobase_kill_one_trx");
-
-	WSREP_LOG_CONFLICT(bf_thd, victim_trx->mysql_thd, TRUE);
-
-	DBUG_EXECUTE_IF("sync.wsrep_after_BF_victim_lock",
-		{
-		  const char act[]=
-		    "now "
-		    "wait_for signal.wsrep_after_BF_victim_lock";
-		  DBUG_ASSERT(!debug_sync_set_action(bf_thd,
-						     STRING_WITH_LEN(act)));
-		};);
-
-
-	mysql_manager_submit(bg_wsrep_kill_trx, arg);
 	DBUG_VOID_RETURN;
 }
 
@@ -19821,8 +19781,8 @@ wsrep_abort_transaction(
 		WSREP_DEBUG("victim does not have transaction");
 		wsrep_thd_LOCK(victim_thd);
 		wsrep_thd_set_conflict_state(victim_thd, MUST_ABORT);
-		wsrep_thd_awake(victim_thd, signal);
 		wsrep_thd_UNLOCK(victim_thd);
+		wsrep_thd_awake(victim_thd, signal);
 	}
 
 	DBUG_VOID_RETURN;

From 88a4be75a5f3b8d59ac8f6347ff2c197813c05dc Mon Sep 17 00:00:00 2001
From: sjaakola <seppo.jaakola@iki.fi>
Date: Wed, 15 Sep 2021 09:16:44 +0300
Subject: [PATCH 02/31] MDEV-25114 Crash: WSREP: invalid state ROLLED_BACK
 (FATAL)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch is the plan D variant for fixing potetial mutex locking
order exercised by BF aborting and KILL command execution.

In this approach, KILL command is replicated as TOI operation.
This guarantees total isolation for the KILL command execution
in the first node: there is no concurrent replication applying
and no concurrent DDL executing. Therefore there is no risk of
BF aborting to happen in parallel with KILL command execution
either. Potential mutex deadlocks between the different mutex
access paths with KILL command execution and BF aborting cannot
therefore happen.

TOI replication is used, in this approach,  purely as means
to provide isolated KILL command execution in the first node.
KILL command should not (and must not) be applied in secondary
nodes. In this patch, we make this sure by skipping KILL
execution in secondary nodes, in applying phase, where we
bail out if applier thread is trying to execute KILL command.
This is effective, but skipping the applying of KILL command
could happen much earlier as well.

This patch also fixes mutex locking order and unprotected
THD member accesses on bf aborting case. We try to hold
THD::LOCK_thd_data during bf aborting. Only case where it
is not possible is at wsrep_abort_transaction before
call wsrep_innobase_kill_one_trx where we take InnoDB
mutexes first and then THD::LOCK_thd_data.

This will also fix possible race condition during
close_connection and while wsrep is disconnecting
connections.

Added wsrep_bf_kill_debug test case

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>
---
 .../suite/galera/r/galera_UK_conflict.result  |   7 +
 .../galera/r/galera_bf_kill_debug.result      | 163 ++++++++++
 .../galera/r/galera_toi_ddl_fk_insert.result  |  16 -
 .../suite/galera/t/galera_UK_conflict.test    |   4 +
 .../suite/galera/t/galera_bf_kill_debug.cnf   |   9 +
 .../suite/galera/t/galera_bf_kill_debug.test  | 283 ++++++++++++++++++
 .../galera/t/galera_toi_ddl_fk_insert.test    |  14 +-
 mysql-test/suite/wsrep/t/variables.test       |   6 +-
 sql/mysqld.cc                                 |   9 +-
 sql/mysqld.h                                  |   2 +-
 sql/sql_class.cc                              |  24 +-
 sql/sql_class.h                               |   9 +-
 sql/sql_parse.cc                              |  36 ++-
 sql/wsrep_mysqld.cc                           |  94 ++++--
 sql/wsrep_thd.cc                              |   8 +-
 sql/wsrep_thd.h                               |   1 -
 storage/innobase/handler/ha_innodb.cc         | 186 +++++++-----
 storage/innobase/include/ha_prototypes.h      |   3 +-
 18 files changed, 724 insertions(+), 150 deletions(-)
 create mode 100644 mysql-test/suite/galera/r/galera_bf_kill_debug.result
 create mode 100644 mysql-test/suite/galera/t/galera_bf_kill_debug.cnf
 create mode 100644 mysql-test/suite/galera/t/galera_bf_kill_debug.test

diff --git a/mysql-test/suite/galera/r/galera_UK_conflict.result b/mysql-test/suite/galera/r/galera_UK_conflict.result
index 76649f1b268..2795a86d6a6 100644
--- a/mysql-test/suite/galera/r/galera_UK_conflict.result
+++ b/mysql-test/suite/galera/r/galera_UK_conflict.result
@@ -68,6 +68,9 @@ f1	f2	f3
 10	10	0
 INSERT INTO t1 VALUES (7,7,7);
 INSERT INTO t1 VALUES (8,8,8);
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+7
 SELECT * FROM t1;
 f1	f2	f3
 1	1	0
@@ -78,6 +81,9 @@ f1	f2	f3
 8	8	8
 10	10	0
 connection node_1;
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+7
 SELECT * FROM t1;
 f1	f2	f3
 1	1	0
@@ -85,5 +91,6 @@ f1	f2	f3
 4	4	2
 5	5	2
 7	7	7
+8	8	8
 10	10	0
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/r/galera_bf_kill_debug.result b/mysql-test/suite/galera/r/galera_bf_kill_debug.result
new file mode 100644
index 00000000000..c20bb80131a
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_bf_kill_debug.result
@@ -0,0 +1,163 @@
+#
+# Case 1: We execute bf kill to wsrep_innobase_kill_one_trx
+# function just before wsrep_thd_LOCK(thd) call. Then we
+# try to kill victim transaction by KILL QUERY
+#
+CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
+INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+begin;
+update t1 set b = b * 10 where id between 2 and 4;
+connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET DEBUG_SYNC='wsrep_before_BF_victim_lock SIGNAL bf_kill WAIT_FOR bf_continue';
+ALTER TABLE t1 ADD UNIQUE KEY b1(b);;
+connection node_1;
+SET DEBUG_SYNC='now WAIT_FOR bf_kill';
+connection node_1b;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `id` int(11) NOT NULL,
+  `b` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `b1` (`b`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+id	b
+1	1
+2	2
+3	3
+4	4
+5	5
+connection node_1;
+SET DEBUG_SYNC= 'RESET';
+DROP TABLE t1;
+disconnect node_1a;
+disconnect node_1b;
+disconnect node_1c;
+#
+# Case 2: We execute bf kill to wsrep_innobase_kill_one_trx
+# function just after wsrep_thd_LOCK(thd) call. Then we
+# try to kill victim transaction by KILL QUERY
+#
+CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
+INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+begin;
+update t1 set b = b * 10 where id between 2 and 4;
+connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET DEBUG_SYNC='wsrep_after_BF_victim_lock SIGNAL bf_kill WAIT_FOR bf_continue';
+ALTER TABLE t1 ADD UNIQUE KEY b1(b);;
+connection node_1;
+SET DEBUG_SYNC='now WAIT_FOR bf_kill';
+connection node_1b;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `id` int(11) NOT NULL,
+  `b` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `b1` (`b`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+id	b
+1	1
+2	2
+3	3
+4	4
+5	5
+connection node_1;
+SET DEBUG_SYNC= 'RESET';
+DROP TABLE t1;
+disconnect node_1a;
+disconnect node_1b;
+disconnect node_1c;
+#
+# Case 3: Create victim transaction and try to send user KILL
+# from several threads
+#
+CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
+INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+begin;
+update t1 set b = b * 10 where id between 2 and 4;
+connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1d, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connection node_1b;
+connection node_1c;
+connection node_1d;
+connection node_1;
+disconnect node_1a;
+disconnect node_1b;
+disconnect node_1c;
+disconnect node_1d;
+DROP TABLE t1;
+#
+# Case 4: MDL-conflict, we execute ALTER until we hit gap in
+# wsrep_abort_transaction, while we are there we try to
+# manually KILL conflicting transaction (UPDATE) and
+# send conflicting transaction from other node to be executed
+# in this node by applier. As ALTER and KILL are TOI they
+# are not executed concurrently. Similarly UPDATE from other
+# node will wait for certification.
+#
+CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
+INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+begin;
+update t1 set b = b * 10 where id between 2 and 4;
+connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET DEBUG_SYNC='wsrep_abort_victim_unlocked SIGNAL bf_kill_unlocked WAIT_FOR bf_continue';
+ALTER TABLE t1 ADD UNIQUE KEY b1(b);;
+connection node_1;
+SET DEBUG_SYNC='now WAIT_FOR bf_kill_unlocked';
+connection node_1b;
+connection node_2;
+update t1 set b = b + 1000 where id between 2 and 4;;
+connection node_1;
+SET DEBUG_SYNC='now SIGNAL bf_continue';
+connection node_1c;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `id` int(11) NOT NULL,
+  `b` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `b1` (`b`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+SELECT * FROM t1;
+id	b
+1	1
+5	5
+2	1002
+3	1003
+4	1004
+connection node_1b;
+connection node_1;
+SET DEBUG_SYNC= 'RESET';
+SELECT * FROM t1;
+id	b
+1	1
+5	5
+2	1002
+3	1003
+4	1004
+connection node_2;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `id` int(11) NOT NULL,
+  `b` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `b1` (`b`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+SELECT * FROM t1;
+id	b
+1	1
+5	5
+2	1002
+3	1003
+4	1004
+DROP TABLE t1;
+disconnect node_1a;
+disconnect node_1c;
diff --git a/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result b/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result
index 94752ed7c76..a972394f87c 100644
--- a/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result
+++ b/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result
@@ -21,22 +21,6 @@ connection node_1a;
 connection node_1b;
 connection node_2;
 connection node_2a;
-connection node_1;
-SET SESSION wsrep_sync_wait=15;
-SELECT COUNT(*) FROM parent;
-COUNT(*)
-20001
-SELECT COUNT(*) FROM child;
-COUNT(*)
-10000
-connection node_2;
-SET SESSION wsrep_sync_wait=15;
-SELECT COUNT(*) FROM parent;
-COUNT(*)
-20001
-SELECT COUNT(*) FROM child;
-COUNT(*)
-10000
 DROP TABLE child;
 DROP TABLE parent;
 DROP TABLE ten;
diff --git a/mysql-test/suite/galera/t/galera_UK_conflict.test b/mysql-test/suite/galera/t/galera_UK_conflict.test
index 57bafbf8ae0..83d0e47dc3d 100644
--- a/mysql-test/suite/galera/t/galera_UK_conflict.test
+++ b/mysql-test/suite/galera/t/galera_UK_conflict.test
@@ -140,9 +140,13 @@ SELECT * FROM t1;
 # original state in node 1
 INSERT INTO t1 VALUES (7,7,7);
 INSERT INTO t1 VALUES (8,8,8);
+SELECT COUNT(*) FROM t1;
 SELECT * FROM t1;
 
 --connection node_1
+--let $wait_condition = SELECT COUNT(*) = 7 FROM t1
+--source include/wait_condition.inc
+SELECT COUNT(*) FROM t1;
 SELECT * FROM t1;
 
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/t/galera_bf_kill_debug.cnf b/mysql-test/suite/galera/t/galera_bf_kill_debug.cnf
new file mode 100644
index 00000000000..77bb6af9f35
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_bf_kill_debug.cnf
@@ -0,0 +1,9 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep_log_conflicts=ON
+wsrep_debug=1
+
+[mysqld.2]
+wsrep_log_conflicts=ON
+wsrep_debug=1
diff --git a/mysql-test/suite/galera/t/galera_bf_kill_debug.test b/mysql-test/suite/galera/t/galera_bf_kill_debug.test
new file mode 100644
index 00000000000..f83d4a28ce9
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_bf_kill_debug.test
@@ -0,0 +1,283 @@
+--source include/galera_cluster.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+
+--echo #
+--echo # Case 1: We execute bf kill to wsrep_innobase_kill_one_trx
+--echo # function just before wsrep_thd_LOCK(thd) call. Then we
+--echo # try to kill victim transaction by KILL QUERY
+--echo #
+
+CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
+INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+
+#
+# This will be victim transaction for both bf kill and
+# user KILL
+#
+--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+begin;
+update t1 set b = b * 10 where id between 2 and 4;
+
+#
+# Take thread id for above query
+#
+--connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
+
+#
+# Set DEBUG_SYNC and send conflicting DDL that will be TOI (bf) and
+# cause bf_kill
+#
+--connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
+SET DEBUG_SYNC='wsrep_before_BF_victim_lock SIGNAL bf_kill WAIT_FOR bf_continue';
+--send ALTER TABLE t1 ADD UNIQUE KEY b1(b);
+
+#
+# Wait until we have reached the sync point
+#
+--connection node_1
+SET DEBUG_SYNC='now WAIT_FOR bf_kill';
+
+#
+# Try to kill update query
+#
+--connection node_1b
+--disable_query_log
+--send_eval KILL QUERY $k_thread;
+
+
+#
+# Let bf_kill continue
+#
+--connection node_1
+SET DEBUG_SYNC='now SIGNAL bf_continue';
+--connection node_1c
+--reap
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+
+--connection node_1b
+--reap
+--enable_query_log
+
+--connection node_1
+SET DEBUG_SYNC= 'RESET';
+DROP TABLE t1;
+
+--disconnect node_1a
+--disconnect node_1b
+--disconnect node_1c
+
+--echo #
+--echo # Case 2: We execute bf kill to wsrep_innobase_kill_one_trx
+--echo # function just after wsrep_thd_LOCK(thd) call. Then we
+--echo # try to kill victim transaction by KILL QUERY
+--echo #
+
+CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
+INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+
+#
+# This will be victim transaction for both bf kill and
+# user KILL
+#
+--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+begin;
+update t1 set b = b * 10 where id between 2 and 4;
+
+#
+# Take thread id for above query
+#
+--connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
+
+#
+# Set DEBUG_SYNC and send conflicting DDL that will be TOI (bf) and
+# cause bf_kill
+#
+--connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
+SET DEBUG_SYNC='wsrep_after_BF_victim_lock SIGNAL bf_kill WAIT_FOR bf_continue';
+--send ALTER TABLE t1 ADD UNIQUE KEY b1(b);
+
+#
+# Wait until we have reached the sync point
+#
+--connection node_1
+SET DEBUG_SYNC='now WAIT_FOR bf_kill';
+
+#
+# Try to kill update query
+#
+--connection node_1b
+--disable_query_log
+--send_eval KILL QUERY $k_thread;
+
+#
+# Let bf_kill continue
+#
+--connection node_1
+SET DEBUG_SYNC='now SIGNAL bf_continue';
+--connection node_1c
+--reap
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+
+--connection node_1b
+--reap
+--enable_query_log
+
+--connection node_1
+SET DEBUG_SYNC= 'RESET';
+DROP TABLE t1;
+
+--disconnect node_1a
+--disconnect node_1b
+--disconnect node_1c
+
+--echo #
+--echo # Case 3: Create victim transaction and try to send user KILL
+--echo # from several threads
+--echo #
+
+CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
+INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+
+#
+# This will be victim transaction for user KILL
+#
+--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+begin;
+update t1 set b = b * 10 where id between 2 and 4;
+
+#
+# Take thread id for above query
+#
+--connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--connect node_1d, 127.0.0.1, root, , test, $NODE_MYPORT_1
+
+--connection node_1b
+--let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
+
+#
+# Try to kill update query from several connections concurrently
+#
+--disable_query_log
+--send_eval KILL QUERY $k_thread;
+
+--connection node_1c
+--disable_query_log
+--send_eval KILL QUERY $k_thread;
+
+--connection node_1d
+--disable_query_log
+--send_eval KILL QUERY $k_thread;
+
+#
+# We do not know execution order so any of these could fail as KILL
+# has been already done
+#
+--connection node_1b
+--enable_query_log
+--error 0,ER_KILL_DENIED_ERROR
+--reap
+--connection node_1c
+--enable_query_log
+--error 0,ER_KILL_DENIED_ERROR
+--reap
+--connection node_1d
+--enable_query_log
+--error 0,ER_KILL_DENIED_ERROR
+--reap
+
+--connection node_1
+--disconnect node_1a
+--disconnect node_1b
+--disconnect node_1c
+--disconnect node_1d
+DROP TABLE t1;
+
+--echo #
+--echo # Case 4: MDL-conflict, we execute ALTER until we hit gap in
+--echo # wsrep_abort_transaction, while we are there we try to
+--echo # manually KILL conflicting transaction (UPDATE) and
+--echo # send conflicting transaction from other node to be executed
+--echo # in this node by applier. As ALTER and KILL are TOI they
+--echo # are not executed concurrently. Similarly UPDATE from other
+--echo # node will wait for certification.
+--echo #
+
+CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
+INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
+
+#
+# This will be victim transaction for both bf kill and
+# user KILL, and should not have any effect on result
+#
+--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+begin;
+update t1 set b = b * 10 where id between 2 and 4;
+
+#
+# Take thread id for above query
+#
+--connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
+
+#
+# Set DEBUG_SYNC and send conflicting DDL that will be TOI (bf) and
+# cause bf_kill but let's execute it only to gap in wsrep_abort_transaction
+#
+--connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
+SET DEBUG_SYNC='wsrep_abort_victim_unlocked SIGNAL bf_kill_unlocked WAIT_FOR bf_continue';
+--send ALTER TABLE t1 ADD UNIQUE KEY b1(b);
+
+#
+# Wait until we have reached the sync point
+#
+--connection node_1
+SET DEBUG_SYNC='now WAIT_FOR bf_kill_unlocked';
+
+#
+# Try to kill update query
+#
+--connection node_1b
+--disable_query_log
+--send_eval KILL QUERY $k_thread;
+
+#
+# Send conflicting update from other node, this should be applied on both nodes
+# but should not kill ALTER
+#
+--enable_query_log
+--connection node_2
+--send update t1 set b = b + 1000 where id between 2 and 4;
+
+#
+# Let bf_kill continue
+#
+--connection node_1
+SET DEBUG_SYNC='now SIGNAL bf_continue';
+--connection node_1c
+--reap
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+
+--connection node_1b
+--reap
+--enable_query_log
+
+--connection node_1
+SET DEBUG_SYNC= 'RESET';
+SELECT * FROM t1;
+
+--connection node_2
+--reap
+SHOW CREATE TABLE t1;
+SELECT * FROM t1;
+DROP TABLE t1;
+
+--disconnect node_1a
+--disconnect node_1c
+
diff --git a/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test b/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test
index fadc94d78ff..3b4b427f551 100644
--- a/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test
+++ b/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test
@@ -54,15 +54,11 @@ INSERT INTO parent VALUES (1, 0);
 --connection node_2a
 --reap
 
---connection node_1
-SET SESSION wsrep_sync_wait=15;
-SELECT COUNT(*) FROM parent;
-SELECT COUNT(*) FROM child;
-
---connection node_2
-SET SESSION wsrep_sync_wait=15;
-SELECT COUNT(*) FROM parent;
-SELECT COUNT(*) FROM child;
+#
+# ALTER TABLE could bf kill one or more of INSERTs to parent, so
+# the actual number of rows in PARENT depends on whether
+# the INSERT is committed before ALTER TABLE is executed
+#
 
 DROP TABLE child;
 DROP TABLE parent;
diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test
index 0cf13650ce0..875315c0e7c 100644
--- a/mysql-test/suite/wsrep/t/variables.test
+++ b/mysql-test/suite/wsrep/t/variables.test
@@ -66,7 +66,7 @@ call mtr.add_suppression("WSREP: Failed to get provider options");
 
 #evalp SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
 
---replace_regex /.*libgalera_smm.*/libgalera_smm.so/
+--replace_regex /.*libgalera.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_slave_threads;
 SELECT @@global.wsrep_cluster_address;
@@ -77,7 +77,7 @@ SHOW STATUS LIKE 'wsrep_thread_count';
 
 #evalp SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
 
---replace_regex /.*libgalera_smm.*/libgalera_smm.so/
+--replace_regex /.*libgalera.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_cluster_address;
 SELECT @@global.wsrep_on;
@@ -101,7 +101,7 @@ SELECT VARIABLE_VALUE AS EXPECT_1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VA
 SELECT VARIABLE_VALUE AS EXPECT_1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
 SELECT VARIABLE_VALUE AS EXPECT_2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
 
---replace_regex /.*libgalera_smm.*/libgalera_smm.so/
+--replace_regex /.*libgalera.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_cluster_address;
 SELECT @@global.wsrep_on;
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 9afc701c6ba..269ad67165a 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -2391,7 +2391,7 @@ static void clean_up_mutexes()
 static void set_ports()
 {
 }
-void close_connection(THD *thd, uint sql_errno)
+void close_connection(THD *thd, uint sql_errno, my_bool locked)
 {
 }
 #else
@@ -2867,7 +2867,7 @@ static void network_init(void)
     For the connection that is doing shutdown, this is called twice
 */
 
-void close_connection(THD *thd, uint sql_errno)
+void close_connection(THD *thd, uint sql_errno, my_bool locked)
 {
   DBUG_ENTER("close_connection");
 
@@ -2877,7 +2877,10 @@ void close_connection(THD *thd, uint sql_errno)
   thd->print_aborted_warning(3, sql_errno ? ER_DEFAULT(sql_errno)
                                           : "CLOSE_CONNECTION");
 
-  thd->disconnect();
+  if (locked)
+    thd->disconnect_mutexed();
+  else
+    thd->disconnect();
 
   MYSQL_CONNECTION_DONE((int) sql_errno, thd->thread_id);
 
diff --git a/sql/mysqld.h b/sql/mysqld.h
index 44b0491f138..1084173ae5f 100644
--- a/sql/mysqld.h
+++ b/sql/mysqld.h
@@ -83,7 +83,7 @@ enum enum_slave_parallel_mode {
 
 /* Function prototypes */
 void kill_mysql(THD *thd= 0);
-void close_connection(THD *thd, uint sql_errno= 0);
+void close_connection(THD *thd, uint sql_errno= 0, my_bool locked=false);
 void handle_connection_in_main_thread(CONNECT *thd);
 void create_thread_to_handle_connection(CONNECT *connect);
 void signal_thd_deleted();
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index 5ada018e540..05ec9a1c369 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -1804,11 +1804,11 @@ void THD::awake(killed_state state_to_set)
           the Vio might be disassociated concurrently.
 */
 
-void THD::disconnect()
+void THD::disconnect_mutexed()
 {
   Vio *vio= NULL;
 
-  mysql_mutex_lock(&LOCK_thd_data);
+  mysql_mutex_assert_owner(&LOCK_thd_data);
 
   set_killed(KILL_CONNECTION);
 
@@ -1826,8 +1826,6 @@ void THD::disconnect()
   if (net.vio != vio)
     vio_close(net.vio);
   net.thd= 0;                                   // Don't collect statistics
-
-  mysql_mutex_unlock(&LOCK_thd_data);
 }
 
 
@@ -1879,16 +1877,18 @@ bool THD::notify_shared_lock(MDL_context_owner *ctx_in_use,
           thread can see those instances (e.g. see partitioning code).
         */
         if (!thd_table->needs_reopen())
-        {
           signalled|= mysql_lock_abort_for_thread(this, thd_table);
-          if (WSREP(this) && wsrep_thd_is_BF(this, FALSE))
-          {
-            WSREP_DEBUG("remove_table_from_cache: %llu",
-                        (unsigned long long) this->real_id);
-            wsrep_abort_thd((void *)this, (void *)in_use, FALSE);
-          }
-        }
       }
+#ifdef WITH_WSREP
+      if (WSREP(this) && wsrep_thd_is_BF(this, false))
+      {
+        WSREP_DEBUG("notify_shared_lock: BF thread %llu query %s"
+                    " victim %llu query %s",
+                    this->real_id, wsrep_thd_query(this),
+                    in_use->real_id, wsrep_thd_query(in_use));
+        wsrep_abort_thd((void *)this, (void *)in_use, false);
+      }
+#endif /* WITH_WSREP */
     }
     mysql_mutex_unlock(&in_use->LOCK_thd_data);
   }
diff --git a/sql/sql_class.h b/sql/sql_class.h
index 5f871f9caf6..7dbc81be770 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -3231,8 +3231,13 @@ public:
   void awake(killed_state state_to_set);
  
   /** Disconnect the associated communication endpoint. */
-  void disconnect();
-
+  inline void disconnect()
+  {
+    mysql_mutex_lock(&LOCK_thd_data);
+    disconnect_mutexed();
+    mysql_mutex_unlock(&LOCK_thd_data);
+  }
+  void disconnect_mutexed();
 
   /*
     Allows this thread to serve as a target for others to schedule Async 
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index 3e1f248b082..2bec9c6b6cd 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2017, Oracle and/or its affiliates.
-   Copyright (c) 2008, 2020, MariaDB
+   Copyright (c) 2008, 2021, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -9069,6 +9069,18 @@ static
 void sql_kill(THD *thd, longlong id, killed_state state, killed_type type)
 {
   uint error;
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    WSREP_DEBUG("sql_kill called");
+    if (thd->wsrep_applier)
+    {
+      WSREP_DEBUG("KILL in applying, bailing out here");
+      return;
+    }
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
+  }
+#endif /* WITH_WSREP */
   if (!(error= kill_one_thread(thd, id, state, type)))
   {
     if (!thd->killed)
@@ -9078,6 +9090,11 @@ void sql_kill(THD *thd, longlong id, killed_state state, killed_type type)
   }
   else
     my_error(error, MYF(0), id);
+#ifdef WITH_WSREP
+  return;
+ wsrep_error_label:
+  my_error(ER_CANNOT_USER, MYF(0), wsrep_thd_query(thd));
+#endif /* WITH_WSREP */
 }
 
 
@@ -9086,6 +9103,18 @@ void sql_kill_user(THD *thd, LEX_USER *user, killed_state state)
 {
   uint error;
   ha_rows rows;
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    WSREP_DEBUG("sql_kill_user called");
+    if (thd->wsrep_applier)
+    {
+      WSREP_DEBUG("KILL in applying, bailing out here");
+      return;
+    }
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
+  }
+#endif /* WITH_WSREP */
   if (!(error= kill_threads_for_user(thd, user, state, &rows)))
     my_ok(thd, rows);
   else
@@ -9096,6 +9125,11 @@ void sql_kill_user(THD *thd, LEX_USER *user, killed_state state)
     */
     my_error(error, MYF(0), user->host.str, user->user.str);
   }
+#ifdef WITH_WSREP
+  return;
+ wsrep_error_label:
+  my_error(ER_CANNOT_USER, MYF(0), user->user.str);
+#endif /* WITH_WSREP */
 }
 
 
diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index e60100e2e90..00a6dfe2f8a 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -1,4 +1,4 @@
-/* Copyright 2008-2015 Codership Oy <http://www.codership.com>
+/* Copyright 2008-2021 Codership Oy <http://www.codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -835,13 +835,25 @@ void wsrep_thr_init()
   DBUG_VOID_RETURN;
 }
 
+/* This is wrapper for wsrep_break_lock in thr_lock.c */
+static int wsrep_thr_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
+{
+  THD* victim_thd= (THD *) victim_thd_ptr;
+  /* We need to lock THD::LOCK_thd_data to protect victim
+  from concurrent usage or disconnect or delete. */
+  mysql_mutex_lock(&victim_thd->LOCK_thd_data);
+  int res= wsrep_abort_thd(bf_thd_ptr, victim_thd_ptr, signal);
+  mysql_mutex_unlock(&victim_thd->LOCK_thd_data);
+  return res;
+}
+
 void wsrep_init_startup (bool first)
 {
   if (wsrep_init()) unireg_abort(1);
 
   wsrep_thr_lock_init(
      (wsrep_thd_is_brute_force_fun)wsrep_thd_is_BF,
-     (wsrep_abort_thd_fun)wsrep_abort_thd,
+     (wsrep_abort_thd_fun)wsrep_thr_abort_thd,
      wsrep_debug, wsrep_convert_LOCK_to_trx,
      (wsrep_on_fun)wsrep_on);
 
@@ -1694,6 +1706,11 @@ static int wsrep_TOI_begin(THD *thd, char *db_, char *table_,
   case SQLCOM_DROP_TABLE:
     buf_err= wsrep_drop_table_query(thd, &buf, &buf_len);
     break;
+  case SQLCOM_KILL:
+    WSREP_DEBUG("KILL as TOI: %s", thd->query());
+    buf_err= wsrep_to_buf_helper(thd, thd->query(), thd->query_length(),
+                                 &buf, &buf_len);
+    break;
   case SQLCOM_CREATE_ROLE:
     if (sp_process_definer(thd))
     {
@@ -2058,8 +2075,11 @@ bool wsrep_grant_mdl_exception(MDL_context *requestor_ctx,
         ticket->wsrep_report(true);
       }
 
-      mysql_mutex_unlock(&granted_thd->LOCK_thd_data);
+      /* This will call wsrep_abort_transaction so we should hold
+      THD::LOCK_thd_data to protect victim from concurrent usage
+      or disconnect or delete. */
       wsrep_abort_thd((void *) request_thd, (void *) granted_thd, 1);
+      mysql_mutex_unlock(&granted_thd->LOCK_thd_data);
       ret= false;
     }
   }
@@ -2241,6 +2261,7 @@ error:
 static bool abort_replicated(THD *thd)
 {
   bool ret_code= false;
+  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
   if (thd->wsrep_query_state== QUERY_COMMITTING)
   {
     WSREP_DEBUG("aborting replicated trx: %llu", (ulonglong)(thd->real_id));
@@ -2255,6 +2276,7 @@ static bool abort_replicated(THD *thd)
 /**/
 static inline bool is_client_connection(THD *thd)
 {
+  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
   return (thd->wsrep_client_thread && thd->variables.wsrep_on);
 }
 
@@ -2263,9 +2285,8 @@ static inline bool is_replaying_connection(THD *thd)
 {
   bool ret;
 
-  mysql_mutex_lock(&thd->LOCK_thd_data);
+  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
   ret=  (thd->wsrep_conflict_state == REPLAYING) ? true : false;
-  mysql_mutex_unlock(&thd->LOCK_thd_data);
 
   return ret;
 }
@@ -2275,9 +2296,8 @@ static inline bool is_committing_connection(THD *thd)
 {
   bool ret;
 
-  mysql_mutex_lock(&thd->LOCK_thd_data);
+  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
   ret=  (thd->wsrep_query_state == QUERY_COMMITTING) ? true : false;
-  mysql_mutex_unlock(&thd->LOCK_thd_data);
 
   return ret;
 }
@@ -2290,13 +2310,17 @@ static bool have_client_connections()
   I_List_iterator<THD> it(threads);
   while ((tmp=it++))
   {
+    /* Protect thread from concurrent usage or disconnect or delete. */
+    mysql_mutex_lock(&tmp->LOCK_thd_data);
     DBUG_PRINT("quit",("Informing thread %lld that it's time to die",
                        (longlong) tmp->thread_id));
     if (is_client_connection(tmp) && tmp->killed == KILL_CONNECTION)
     {
       (void)abort_replicated(tmp);
+      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       return true;
     }
+    mysql_mutex_unlock(&tmp->LOCK_thd_data);
   }
   return false;
 }
@@ -2328,14 +2352,21 @@ static my_bool have_committing_connections()
   I_List_iterator<THD> it(threads);
   while ((tmp=it++))
   {
+    /* Protect from concurrent usage or disconnect or delete */
+    mysql_mutex_lock(&tmp->LOCK_thd_data);
     if (!is_client_connection(tmp))
+    {
+      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
+    }
 
     if (is_committing_connection(tmp))
     {
       mysql_mutex_unlock(&LOCK_thread_count);
+      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       return TRUE;
     }
+    mysql_mutex_unlock(&tmp->LOCK_thd_data);
   }
   mysql_mutex_unlock(&LOCK_thread_count);
   return FALSE;
@@ -2378,33 +2409,44 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD *except_caller_thd)
   {
     DBUG_PRINT("quit",("Informing thread %lld that it's time to die",
                        (longlong) tmp->thread_id));
+    /* Protect from concurrent usage or disconnect or delete */
+    mysql_mutex_lock(&tmp->LOCK_thd_data);
     /* We skip slave threads & scheduler on this first loop through. */
     if (!is_client_connection(tmp))
+    {
+      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
+    }
 
     if (tmp == except_caller_thd)
     {
       DBUG_ASSERT(is_client_connection(tmp));
+      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
     }
 
     if (is_replaying_connection(tmp))
     {
       tmp->set_killed(KILL_CONNECTION);
+      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
     }
 
-    /* replicated transactions must be skipped */
+    /* replicated transactions must be skipped and aborted
+    with wsrep_abort_thd. */
     if (abort_replicated(tmp))
+    {
+      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
+    }
 
     WSREP_DEBUG("closing connection %lld", (longlong) tmp->thread_id);
 
     /*
-      instead of wsrep_close_thread() we do now  soft kill by THD::awake
-     */
-    mysql_mutex_lock(&tmp->LOCK_thd_data);
-
+      instead of wsrep_close_thread() we do now  soft kill by
+      THD::awake(). Here also victim needs to be protected from
+      concurrent usage or disconnect or delete.
+    */
     tmp->awake(KILL_CONNECTION);
 
     mysql_mutex_unlock(&tmp->LOCK_thd_data);
@@ -2423,16 +2465,19 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD *except_caller_thd)
   I_List_iterator<THD> it2(threads);
   while ((tmp=it2++))
   {
-#ifndef __bsdi__				// Bug in BSDI kernel
-    if (is_client_connection(tmp) &&
-        !abort_replicated(tmp)    &&
-        !is_replaying_connection(tmp) &&
-        tmp != except_caller_thd)
+    /* Protect from concurrent usage or disconnect or delete */
+    mysql_mutex_lock(&tmp->LOCK_thd_data);
+    if (is_client_connection(tmp))
     {
-      WSREP_INFO("killing local connection: %lld", (longlong) tmp->thread_id);
-      close_connection(tmp,0);
+      if (!abort_replicated(tmp) &&
+          !is_replaying_connection(tmp) &&
+          tmp != except_caller_thd)
+      {
+        WSREP_INFO("killing local connection: %lld", (longlong) tmp->thread_id);
+        close_connection(tmp,0, true);
+      }
     }
-#endif
+    mysql_mutex_unlock(&tmp->LOCK_thd_data);
   }
 
   DBUG_PRINT("quit",("Waiting for threads to die (count=%u)",thread_count));
@@ -2621,7 +2666,9 @@ extern "C" void wsrep_thd_set_query_state(
 
 void wsrep_thd_set_conflict_state(THD *thd, enum wsrep_conflict_state state)
 {
-  if (WSREP(thd)) thd->wsrep_conflict_state= state;
+  DBUG_ASSERT(thd);
+  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
+  thd->wsrep_conflict_state= state;
 }
 
 
@@ -2762,9 +2809,10 @@ extern "C" void wsrep_thd_awake(THD *thd, my_bool signal)
 {
   if (signal)
   {
-    mysql_mutex_lock(&thd->LOCK_thd_data);
+    /* Here we should hold THD::LOCK_thd_data to
+    protect from concurrent usage. */
+    mysql_mutex_assert_owner(&thd->LOCK_thd_data);
     thd->awake(KILL_QUERY);
-    mysql_mutex_unlock(&thd->LOCK_thd_data);
   }
   else
   {
diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc
index ef8c0e132f7..d849cd2cd78 100644
--- a/sql/wsrep_thd.cc
+++ b/sql/wsrep_thd.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2013 Codership Oy <info@codership.com>
+/* Copyright (C) 2013-2021 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -804,10 +804,12 @@ my_bool wsrep_thd_is_local(void *thd_ptr, my_bool sync)
 
 int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
 {
-  THD *victim_thd = (THD *) victim_thd_ptr;
-  THD *bf_thd     = (THD *) bf_thd_ptr;
+  THD *victim_thd= (THD *) victim_thd_ptr;
+  THD *bf_thd= (THD *) bf_thd_ptr;
   DBUG_ENTER("wsrep_abort_thd");
 
+  mysql_mutex_assert_owner(&victim_thd->LOCK_thd_data);
+
   if ( (WSREP(bf_thd) ||
          ( (WSREP_ON || bf_thd->variables.wsrep_OSU_method == WSREP_OSU_RSU) &&
            bf_thd->wsrep_exec_mode == TOTAL_ORDER) )                         &&
diff --git a/sql/wsrep_thd.h b/sql/wsrep_thd.h
index 46bc08a466a..1284cad1acf 100644
--- a/sql/wsrep_thd.h
+++ b/sql/wsrep_thd.h
@@ -32,7 +32,6 @@ void wsrep_create_rollbacker();
 
 int  wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr,
                                 my_bool signal);
-
 /*
   PA = Parallel Applying (on the slave side)
 */
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index d4142481a62..efcdd201fa6 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -5233,17 +5233,21 @@ UNIV_INTERN void lock_cancel_waiting_and_release(lock_t* lock);
 @sa THD::awake() @sa ha_kill_query() */
 static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels)
 {
-	DBUG_ENTER("innobase_kill_query");
+  DBUG_ENTER("innobase_kill_query");
 #ifdef WITH_WSREP
-	if (wsrep_thd_get_conflict_state(thd) != NO_CONFLICT) {
-		/* if victim has been signaled by BF thread and/or aborting
-		   is already progressing, following query aborting is not necessary
-		   any more.
-		   Also, BF thread should own trx mutex for the victim, which would
-		   conflict with trx_mutex_enter() below
-		*/
-		DBUG_VOID_RETURN;
-	}
+  if (wsrep_thd_get_conflict_state(thd) != NO_CONFLICT)
+  {
+    /* if victim has been signaled by BF thread and/or aborting
+    is already progressing, following query aborting is not necessary
+    any more. E.g. wsrep_innobase_kill_one_trx().
+    Also, BF thread should own trx mutex for the victim, which would
+    conflict with trx_mutex_enter() below
+    */
+    WSREP_DEBUG("Victim thread %ld bail out conflict_state %s query %s",
+		thd_get_thread_id(thd),
+		wsrep_thd_conflict_state_str(thd), wsrep_thd_query(thd));
+    DBUG_VOID_RETURN;
+  }
 #endif /* WITH_WSREP */
 
   if (trx_t* trx= thd_to_trx(thd))
@@ -19520,9 +19524,9 @@ static struct st_mysql_storage_engine innobase_storage_engine=
 { MYSQL_HANDLERTON_INTERFACE_VERSION };
 
 #ifdef WITH_WSREP
+static
 void
 wsrep_abort_slave_trx(
-/*==================*/
 	wsrep_seqno_t bf_seqno,
 	wsrep_seqno_t victim_seqno)
 {
@@ -19532,19 +19536,17 @@ wsrep_abort_slave_trx(
 		"2) a bug in the code.\n\t"
 		"3) a database corruption.\n Node consistency compromized, "
 		"need to abort. Restart the node to resync with cluster.",
-		(long long)bf_seqno, (long long)victim_seqno);
+		bf_seqno, victim_seqno);
 	abort();
 }
 /*******************************************************************//**
 This function is used to kill one transaction in BF. */
-UNIV_INTERN
 void
 wsrep_innobase_kill_one_trx(
-/*========================*/
 	MYSQL_THD const bf_thd,
 	const trx_t * const bf_trx,
 	trx_t *victim_trx,
-	ibool signal)
+	my_bool signal)
 {
         ut_ad(bf_thd);
         ut_ad(victim_trx);
@@ -19552,38 +19554,41 @@ wsrep_innobase_kill_one_trx(
         ut_ad(trx_mutex_own(victim_trx));
 
 	DBUG_ENTER("wsrep_innobase_kill_one_trx");
-	THD *thd          = (THD *) victim_trx->mysql_thd;
-	int64_t bf_seqno  = wsrep_thd_trx_seqno(bf_thd);
+	THD *thd= (THD *) victim_trx->mysql_thd;
+	int64_t bf_seqno= wsrep_thd_trx_seqno(bf_thd);
 
 	if (!thd) {
-		DBUG_PRINT("wsrep", ("no thd for conflicting lock"));
 		WSREP_WARN("no THD for trx: " TRX_ID_FMT, victim_trx->id);
 		DBUG_VOID_RETURN;
 	}
 
-	WSREP_LOG_CONFLICT(bf_thd, thd, TRUE);
-
-	WSREP_DEBUG("BF kill (" ULINTPF ", seqno: " INT64PF
-		    "), victim: (%lu) trx: " TRX_ID_FMT,
-		    signal, bf_seqno,
-		    thd_get_thread_id(thd),
-		    victim_trx->id);
-
-	WSREP_DEBUG("Aborting query: %s conf %d trx: %" PRId64,
-		    (thd && wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void",
-		    wsrep_thd_conflict_state(thd, FALSE),
-		    wsrep_thd_ws_handle(thd)->trx_id);
-
+	/* Here we need to lock THD::LOCK_thd_data to protect from
+	concurrent usage or disconnect or delete. */
+	DEBUG_SYNC(bf_thd, "wsrep_before_BF_victim_lock");
 	wsrep_thd_LOCK(thd);
-        DBUG_EXECUTE_IF("sync.wsrep_after_BF_victim_lock",
-                 {
-                   const char act[]=
-                     "now "
-                     "wait_for signal.wsrep_after_BF_victim_lock";
-                   DBUG_ASSERT(!debug_sync_set_action(bf_thd,
-                                                      STRING_WITH_LEN(act)));
-                 };);
+	DEBUG_SYNC(bf_thd, "wsrep_after_BF_victim_lock");
 
+	WSREP_DEBUG("Aborter %s trx_id: " TRX_ID_FMT " thread: %ld "
+		    "seqno: %lld query_state: %s conflict_state: %s query: %s",
+		    wsrep_thd_is_BF(bf_thd, false) ? "BF" : "normal",
+		    bf_trx ? bf_trx->id : TRX_ID_MAX,
+		    thd_get_thread_id(bf_thd),
+		    bf_seqno,
+		    wsrep_thd_query_state_str(bf_thd),
+		    wsrep_thd_conflict_state_str(bf_thd),
+		    wsrep_thd_query(bf_thd));
+
+	WSREP_DEBUG("Victim %s trx_id: " TRX_ID_FMT " thread: %ld "
+		    "seqno: %lld query_state: %s  conflict_state: %s query: %s",
+		    wsrep_thd_is_BF(thd, false) ? "BF" : "normal",
+		    victim_trx->id,
+		    thd_get_thread_id(thd),
+		    wsrep_thd_trx_seqno(thd),
+		    wsrep_thd_query_state_str(thd),
+		    wsrep_thd_conflict_state_str(thd),
+		    wsrep_thd_query(thd));
+
+	WSREP_LOG_CONFLICT(bf_thd, thd, TRUE);
 
 	if (wsrep_thd_query_state(thd) == QUERY_EXITING) {
 		WSREP_DEBUG("kill trx EXITING for " TRX_ID_FMT,
@@ -19593,27 +19598,32 @@ wsrep_innobase_kill_one_trx(
 	}
 
 	if (wsrep_thd_exec_mode(thd) != LOCAL_STATE) {
-		WSREP_DEBUG("withdraw for BF trx: " TRX_ID_FMT ", state: %d",
+		WSREP_DEBUG("withdraw for BF trx: " TRX_ID_FMT
+			    ", state: %s exec %s",
 			    victim_trx->id,
-		wsrep_thd_get_conflict_state(thd));
+			    wsrep_thd_conflict_state_str(thd),
+			    wsrep_thd_exec_mode_str(thd));
 	}
 
 	switch (wsrep_thd_get_conflict_state(thd)) {
 	case NO_CONFLICT:
+		/* This will cause any call to innobase_kill_query()
+		for this thd to bail out. */
 		wsrep_thd_set_conflict_state(thd, MUST_ABORT);
 		break;
         case MUST_ABORT:
 		WSREP_DEBUG("victim " TRX_ID_FMT " in MUST ABORT state",
 			    victim_trx->id);
-		wsrep_thd_UNLOCK(thd);
 		wsrep_thd_awake(thd, signal);
+		wsrep_thd_UNLOCK(thd);
 		DBUG_VOID_RETURN;
 		break;
 	case ABORTED:
 	case ABORTING: // fall through
 	default:
-		WSREP_DEBUG("victim " TRX_ID_FMT " in state %d",
-			    victim_trx->id, wsrep_thd_get_conflict_state(thd));
+		WSREP_DEBUG("victim " TRX_ID_FMT " in state %s",
+			    victim_trx->id,
+			    wsrep_thd_conflict_state_str(thd));
 		wsrep_thd_UNLOCK(thd);
 		DBUG_VOID_RETURN;
 		break;
@@ -19643,8 +19653,8 @@ wsrep_innobase_kill_one_trx(
 				WSREP_DEBUG("cancel commit warning: "
 					    TRX_ID_FMT,
 					    victim_trx->id);
-				wsrep_thd_UNLOCK(thd);
 				wsrep_thd_awake(thd, signal);
+				wsrep_thd_UNLOCK(thd);
 				DBUG_VOID_RETURN;
 				break;
 			case WSREP_OK:
@@ -19662,8 +19672,8 @@ wsrep_innobase_kill_one_trx(
 				break;
 			}
 		}
-		wsrep_thd_UNLOCK(thd);
 		wsrep_thd_awake(thd, signal);
+		wsrep_thd_UNLOCK(thd);
 		break;
 	case QUERY_EXEC:
 		/* it is possible that victim trx is itself waiting for some
@@ -19685,23 +19695,19 @@ wsrep_innobase_kill_one_trx(
 				lock_cancel_waiting_and_release(wait_lock);
 			}
 
-			wsrep_thd_UNLOCK(thd);
 			wsrep_thd_awake(thd, signal);
+			wsrep_thd_UNLOCK(thd);
 		} else {
 			/* abort currently executing query */
-			DBUG_PRINT("wsrep",("sending KILL_QUERY to: %lu",
-                                            thd_get_thread_id(thd)));
 			WSREP_DEBUG("kill query for: %ld",
 				thd_get_thread_id(thd));
-			/* Note that innobase_kill_query will take lock_mutex
-			and trx_mutex */
-			wsrep_thd_UNLOCK(thd);
 			wsrep_thd_awake(thd, signal);
+			wsrep_thd_UNLOCK(thd);
 
 			/* for BF thd, we need to prevent him from committing */
 			if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
 				wsrep_abort_slave_trx(bf_seqno,
-						    wsrep_thd_trx_seqno(thd));
+						      wsrep_thd_trx_seqno(thd));
 			}
 		}
 		break;
@@ -19711,29 +19717,27 @@ wsrep_innobase_kill_one_trx(
 
 		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
 			WSREP_DEBUG("kill BF IDLE, seqno: %lld",
-				    (long long)wsrep_thd_trx_seqno(thd));
+				    wsrep_thd_trx_seqno(thd));
 			wsrep_thd_UNLOCK(thd);
 			wsrep_abort_slave_trx(bf_seqno,
 					      wsrep_thd_trx_seqno(thd));
 			DBUG_VOID_RETURN;
 		}
-                /* This will lock thd from proceeding after net_read() */
+		/* This will lock thd from proceeding after net_read()
+		and innobase_kill_query to bail out for this thd. */
 		wsrep_thd_set_conflict_state(thd, ABORTING);
 
 		wsrep_lock_rollback();
 
 		if (wsrep_aborting_thd_contains(thd)) {
 			WSREP_WARN("duplicate thd aborter %lu",
-			           (ulong) thd_get_thread_id(thd));
+			           thd_get_thread_id(thd));
 		} else {
 			wsrep_aborting_thd_enqueue(thd);
-			DBUG_PRINT("wsrep",("enqueuing trx abort for %lu",
-			                    thd_get_thread_id(thd)));
 			WSREP_DEBUG("enqueuing trx abort for (%lu)",
-			            thd_get_thread_id(thd));
+				    thd_get_thread_id(thd));
 		}
 
-		DBUG_PRINT("wsrep",("signalling wsrep rollbacker"));
 		WSREP_DEBUG("signaling aborter");
 		wsrep_unlock_rollback();
 		wsrep_thd_UNLOCK(thd);
@@ -19741,10 +19745,7 @@ wsrep_innobase_kill_one_trx(
 		break;
 	}
 	default:
-		WSREP_WARN("bad wsrep query state: %d",
-			  wsrep_thd_query_state(thd));
-		wsrep_thd_UNLOCK(thd);
-		break;
+		ut_error;
 	}
 
 	DBUG_VOID_RETURN;
@@ -19753,7 +19754,6 @@ wsrep_innobase_kill_one_trx(
 static
 void
 wsrep_abort_transaction(
-/*====================*/
 	handlerton* hton,
 	THD *bf_thd,
 	THD *victim_thd,
@@ -19761,27 +19761,65 @@ wsrep_abort_transaction(
 {
 	DBUG_ENTER("wsrep_abort_transaction");
 
-	trx_t* victim_trx	= thd_to_trx(victim_thd);
-	trx_t* bf_trx		= (bf_thd) ? thd_to_trx(bf_thd) : NULL;
+	ut_ad(bf_thd);
+	ut_ad(victim_thd);
+	trx_t* victim_trx= thd_to_trx(victim_thd);
+	trx_t* bf_trx= thd_to_trx(bf_thd);
 
-	WSREP_DEBUG("abort transaction: BF: %s victim: %s victim conf: %d",
-			wsrep_thd_query(bf_thd),
-			wsrep_thd_query(victim_thd),
-			wsrep_thd_conflict_state(victim_thd, FALSE));
+	/* Here we should hold THD::LOCK_thd_data to protect
+	victim from concurrent usage or disconnect or delete. */
+	WSREP_DEBUG("wsrep_abort_transaction: BF:"
+		    " thread %ld query_state %s conflict_state %s"
+		    " exec %s query %s trx " TRX_ID_FMT,
+		    thd_get_thread_id(bf_thd),
+		    wsrep_thd_query_state_str(bf_thd),
+		    wsrep_thd_conflict_state_str(bf_thd),
+		    wsrep_thd_exec_mode_str(bf_thd),
+		    wsrep_thd_query(bf_thd),
+		    bf_trx ? bf_trx->id : 0);
+
+	WSREP_DEBUG("wsrep_abort_transaction: victim:"
+		    " thread %ld query_state %s conflict_state %s"
+		    " exec %s query %s trx " TRX_ID_FMT,
+		    thd_get_thread_id(victim_thd),
+		    wsrep_thd_query_state_str(victim_thd),
+		    wsrep_thd_conflict_state_str(victim_thd),
+		    wsrep_thd_exec_mode_str(victim_thd),
+		    wsrep_thd_query(victim_thd),
+		    victim_trx ? victim_trx->id : 0);
 
 	if (victim_trx) {
+		WSREP_DEBUG("wsrep_abort_transaction: Victim thread %ld "
+			    "transaction " TRX_ID_FMT " trx_state %d",
+			    thd_get_thread_id(victim_thd),
+			    victim_trx->id,
+			    victim_trx->state);
+		/* This is necessary as correct mutexing order is
+		lock_sys -> trx -> THD::LOCK_thd_data and below
+		function assumes we have lock_sys and trx locked
+		and takes THD::LOCK_thd_data for THD state check. */
+		wsrep_thd_UNLOCK(victim_thd);
+		DEBUG_SYNC(bf_thd, "wsrep_abort_victim_unlocked");
+		DBUG_EXECUTE_IF("wsrep_abort_replicated_sleep",
+				WSREP_DEBUG("wsrep_abort_transaction: sleeping "
+					    "for thread %ld ",
+					    thd_get_thread_id(victim_thd));
+				my_sleep(100000););
 		lock_mutex_enter();
 		trx_mutex_enter(victim_trx);
 		wsrep_innobase_kill_one_trx(bf_thd, bf_trx, victim_trx, signal);
 		lock_mutex_exit();
 		trx_mutex_exit(victim_trx);
 		wsrep_srv_conc_cancel_wait(victim_trx);
+		wsrep_thd_LOCK(victim_thd);
 		DBUG_VOID_RETURN;
 	} else {
-		WSREP_DEBUG("victim does not have transaction");
-		wsrep_thd_LOCK(victim_thd);
+		WSREP_DEBUG("wsrep_abort_transaction: Victim thread %ld "
+			    "no transaction",
+			    thd_get_thread_id(victim_thd));
+		/* This will cause any call to innobase_kill_query()
+		for this thd to bail out. */
 		wsrep_thd_set_conflict_state(victim_thd, MUST_ABORT);
-		wsrep_thd_UNLOCK(victim_thd);
 		wsrep_thd_awake(victim_thd, signal);
 	}
 
diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h
index 3eab2135969..427e57f09d2 100644
--- a/storage/innobase/include/ha_prototypes.h
+++ b/storage/innobase/include/ha_prototypes.h
@@ -233,12 +233,11 @@ innobase_casedn_str(
 	char*	a);	/*!< in/out: string to put in lower case */
 
 #ifdef WITH_WSREP
-UNIV_INTERN
 void
 wsrep_innobase_kill_one_trx(MYSQL_THD const thd_ptr,
                             const trx_t * const bf_trx,
                             trx_t *victim_trx,
-                            ibool signal);
+                            my_bool signal);
 int wsrep_innobase_mysql_sort(int mysql_type, uint charset_number,
                              unsigned char* str, unsigned int str_length,
                              unsigned int buf_length);

From 47ba5523046094db33e68c92a182491a629bbd56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Fri, 24 Sep 2021 07:33:27 +0300
Subject: [PATCH 03/31] Revert "MDEV-24978 : SIGABRT in __libc_message"

This reverts commit 30dea4599e44e3008fb9bc5fe79ab5747841f21f.
---
 .../suite/galera/r/galera_fulltext.result     | 28 --------
 .../suite/galera/t/galera_fulltext.test       | 23 -------
 storage/innobase/handler/ha_innodb.cc         | 64 ++++++-------------
 3 files changed, 21 insertions(+), 94 deletions(-)

diff --git a/mysql-test/suite/galera/r/galera_fulltext.result b/mysql-test/suite/galera/r/galera_fulltext.result
index bb482b7f4f7..f52f5c996a3 100644
--- a/mysql-test/suite/galera/r/galera_fulltext.result
+++ b/mysql-test/suite/galera/r/galera_fulltext.result
@@ -36,34 +36,6 @@ DROP TABLE t1;
 DROP TABLE ten;
 connection node_1;
 SET @value=REPEAT (1,5001);
-CREATE TABLE t (a VARCHAR(5000),FULLTEXT (a)) engine=innodb;
-INSERT IGNORE INTO t VALUES(@value);
-Warnings:
-Warning	1265	Data truncated for column 'a' at row 1
-SELECT COUNT(*) FROM t;
-COUNT(*)
-1
-connection node_2;
-SELECT COUNT(*) FROM t;
-COUNT(*)
-1
-connection node_1;
-DROP TABLE t;
-CREATE TABLE t (a VARCHAR(5000)) engine=innodb;
-INSERT IGNORE INTO t VALUES(@value);
-Warnings:
-Warning	1265	Data truncated for column 'a' at row 1
-SELECT COUNT(*) FROM t;
-COUNT(*)
-1
-connection node_2;
-SELECT COUNT(*) FROM t;
-COUNT(*)
-1
-connection node_1;
-DROP TABLE t;
-connection node_1;
-SET @value=REPEAT (1,5001);
 CREATE TABLE t (a VARCHAR(5000),FULLTEXT (a)) engine=innodb DEFAULT CHARSET=utf8;
 INSERT IGNORE INTO t VALUES(@value);
 Warnings:
diff --git a/mysql-test/suite/galera/t/galera_fulltext.test b/mysql-test/suite/galera/t/galera_fulltext.test
index 25f4f83b7b7..76c29da4123 100644
--- a/mysql-test/suite/galera/t/galera_fulltext.test
+++ b/mysql-test/suite/galera/t/galera_fulltext.test
@@ -60,29 +60,6 @@ SELECT COUNT(f1) = 1000 FROM t1 WHERE MATCH(f1) AGAINST ('abcdefjhk');
 
 DROP TABLE t1;
 DROP TABLE ten;
-#
-# MDEV-24978 : SIGABRT in __libc_message
-#
---connection node_1
-SET @value=REPEAT (1,5001);
-CREATE TABLE t (a VARCHAR(5000),FULLTEXT (a)) engine=innodb;
-INSERT IGNORE INTO t VALUES(@value);
-SELECT COUNT(*) FROM t;
-
---connection node_2
-SELECT COUNT(*) FROM t;
-
---connection node_1
-DROP TABLE t;
-CREATE TABLE t (a VARCHAR(5000)) engine=innodb;
-INSERT IGNORE INTO t VALUES(@value);
-SELECT COUNT(*) FROM t;
-
---connection node_2
-SELECT COUNT(*) FROM t;
-
---connection node_1
-DROP TABLE t;
 
 #
 # Case 2: UTF-8
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index efcdd201fa6..921b8282e45 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -6727,8 +6727,8 @@ wsrep_innobase_mysql_sort(
 	case MYSQL_TYPE_LONG_BLOB:
 	case MYSQL_TYPE_VARCHAR:
 	{
-		uchar *tmp_str;
-		uint tmp_length;
+		uchar tmp_str[REC_VERSION_56_MAX_INDEX_COL_LEN] = {'\0'};
+		uint tmp_length = REC_VERSION_56_MAX_INDEX_COL_LEN;
 
 		/* Use the charset number to pick the right charset struct for
 		the comparison. Since the MySQL function get_charset may be
@@ -6751,12 +6751,7 @@ wsrep_innobase_mysql_sort(
 			}
 		}
 
-		// Note that strnxfrm may change length of string
-		tmp_length= charset->coll->strnxfrmlen(charset, str_length);
-		tmp_length= tmp_length * charset->mbmaxlen;
-		tmp_length= ut_max(str_length, tmp_length) + charset->mbmaxlen;
-		tmp_str= static_cast<uchar *>(ut_malloc_nokey(tmp_length));
-		ut_ad(str_length <= tmp_length);
+		ut_a(str_length <= tmp_length);
 		memcpy(tmp_str, str, str_length);
 
 		tmp_length = charset->coll->strnxfrm(charset, str, str_length,
@@ -6780,7 +6775,6 @@ wsrep_innobase_mysql_sort(
 			ret_length = tmp_length;
 		}
 
-		ut_free(tmp_str);
 		break;
 	}
 	case MYSQL_TYPE_DECIMAL :
@@ -7132,7 +7126,7 @@ wsrep_store_key_val_for_row(
 	THD* 		thd,
 	TABLE*		table,
 	uint		keynr,	/*!< in: key number */
-	uchar*		buff,	/*!< in/out: buffer for the key value (in MySQL
+	char*		buff,	/*!< in/out: buffer for the key value (in MySQL
 				format) */
 	uint		buff_len,/*!< in: buffer length */
 	const uchar*	record,
@@ -7141,7 +7135,7 @@ wsrep_store_key_val_for_row(
 	KEY*		key_info	= table->key_info + keynr;
 	KEY_PART_INFO*	key_part	= key_info->key_part;
 	KEY_PART_INFO*	end		= key_part + key_info->user_defined_key_parts;
-	uchar*		buff_start	= buff;
+	char*		buff_start	= buff;
 	enum_field_types mysql_type;
 	Field*		field;
 	uint buff_space = buff_len;
@@ -7153,8 +7147,7 @@ wsrep_store_key_val_for_row(
 
 	for (; key_part != end; key_part++) {
 
-		uchar *sorted=NULL;
-		uint  max_len=0;
+		uchar sorted[REC_VERSION_56_MAX_INDEX_COL_LEN] = {'\0'};
 		ibool part_is_null = FALSE;
 
 		if (key_part->null_bit) {
@@ -7233,14 +7226,10 @@ wsrep_store_key_val_for_row(
 				true_len = key_len;
 			}
 
-			max_len= true_len;
-			sorted= static_cast<uchar *>(ut_malloc_nokey(max_len+1));
 			memcpy(sorted, data, true_len);
 			true_len = wsrep_innobase_mysql_sort(
 				mysql_type, cs->number, sorted, true_len,
-				max_len);
-			ut_ad(true_len <= max_len);
-
+				REC_VERSION_56_MAX_INDEX_COL_LEN);
 			if (wsrep_protocol_version > 1) {
 				/* Note that we always reserve the maximum possible
 				length of the true VARCHAR in the key value, though
@@ -7325,13 +7314,11 @@ wsrep_store_key_val_for_row(
 				true_len = key_len;
 			}
 
-			max_len= true_len;
-			sorted= static_cast<uchar *>(ut_malloc_nokey(max_len+1));
 			memcpy(sorted, blob_data, true_len);
 			true_len = wsrep_innobase_mysql_sort(
 				mysql_type, cs->number, sorted, true_len,
-				max_len);
-			ut_ad(true_len <= max_len);
+				REC_VERSION_56_MAX_INDEX_COL_LEN);
+
 
 			/* Note that we always reserve the maximum possible
 			length of the BLOB prefix in the key value. */
@@ -7407,14 +7394,10 @@ wsrep_store_key_val_for_row(
 								cs->mbmaxlen),
 							&error);
 				}
-
-				max_len= true_len;
-				sorted= static_cast<uchar *>(ut_malloc_nokey(max_len+1));
 				memcpy(sorted, src_start, true_len);
 				true_len = wsrep_innobase_mysql_sort(
 					mysql_type, cs->number, sorted, true_len,
-					max_len);
-				ut_ad(true_len <= max_len);
+					REC_VERSION_56_MAX_INDEX_COL_LEN);
 
 				if (true_len > buff_space) {
 					fprintf (stderr,
@@ -7429,11 +7412,6 @@ wsrep_store_key_val_for_row(
 			buff       += true_len;
 			buff_space -= true_len;
 		}
-
-		if (sorted) {
-			ut_free(sorted);
-			sorted= NULL;
-		}
 	}
 
 	ut_a(buff <= buff_start + buff_len);
@@ -10510,7 +10488,7 @@ wsrep_append_key(
 	trx_t 		*trx,
 	TABLE_SHARE 	*table_share,
 	TABLE 		*table,
-	const uchar*	key,
+	const char*	key,
 	uint16_t        key_len,
 	wsrep_key_type	key_type	/*!< in: access type of this key
 					(shared, exclusive, semi...) */
@@ -10614,8 +10592,8 @@ ha_innobase::wsrep_append_keys(
 
 	if (wsrep_protocol_version == 0) {
 		uint	len;
-		uchar 	keyval[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
-		uchar 	*key 		= &keyval[0];
+		char 	keyval[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+		char 	*key 		= &keyval[0];
 		ibool    is_null;
 
 		len = wsrep_store_key_val_for_row(
@@ -10647,18 +10625,18 @@ ha_innobase::wsrep_append_keys(
 
 		for (i=0; i<table->s->keys; ++i) {
 			uint  len;
-			uchar  keyval0[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
-			uchar  keyval1[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
-			uchar* key0 		= &keyval0[1];
-			uchar* key1 		= &keyval1[1];
+			char  keyval0[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+			char  keyval1[WSREP_MAX_SUPPORTED_KEY_LENGTH+1] = {'\0'};
+			char* key0 		= &keyval0[1];
+			char* key1 		= &keyval1[1];
 			KEY*  key_info	= table->key_info + i;
 			ibool is_null;
 
 			dict_index_t* idx  = innobase_get_index(i);
 			dict_table_t* tab  = (idx) ? idx->table : NULL;
 
-			keyval0[0] = (uchar)i;
-			keyval1[0] = (uchar)i;
+			keyval0[0] = (char)i;
+			keyval1[0] = (char)i;
 
 			if (!tab) {
 				WSREP_WARN("MariaDB-InnoDB key mismatch %s %s",
@@ -10716,7 +10694,7 @@ ha_innobase::wsrep_append_keys(
 
 		wsrep_calc_row_hash(digest, record0, table, m_prebuilt, thd);
 		if ((rcode = wsrep_append_key(thd, trx, table_share, table,
-					      (const uchar*) digest, 16,
+					      (const char*) digest, 16,
 					      key_type))) {
 			DBUG_RETURN(rcode);
 		}
@@ -10726,7 +10704,7 @@ ha_innobase::wsrep_append_keys(
 				digest, record1, table, m_prebuilt, thd);
 			if ((rcode = wsrep_append_key(thd, trx, table_share,
 						      table,
-						      (const uchar*) digest,
+						      (const char*) digest,
 						      16, key_type))) {
 				DBUG_RETURN(rcode);
 			}

From 467011bcac3b3f42ae6f21dde8d88e78708b21d1 Mon Sep 17 00:00:00 2001
From: Alexey Bychko <alexey.bychko@mariadb.com>
Date: Wed, 22 Sep 2021 18:08:50 +0700
Subject: [PATCH 04/31] MDEV-26612 Two different ways to start MariaDB service
 can cause data corruption

RedHat systems have both files for lsb and init functions.
Old code was written as if/else, so second file (RedHat-specific) was not processed.
So, systemd redirect didn't work, because its logic is described in
RedHat-specific functions file
---
 support-files/mysql.server.sh | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/support-files/mysql.server.sh b/support-files/mysql.server.sh
index 6eb2d0bc257..7f034601539 100644
--- a/support-files/mysql.server.sh
+++ b/support-files/mysql.server.sh
@@ -91,16 +91,15 @@ datadir_set=
 
 #
 # Use LSB init script functions for printing messages, if possible
-#
+# Include non-LSB RedHat init functions to make systemctl redirect work
+init_functions="/etc/init.d/functions"
 lsb_functions="/lib/lsb/init-functions"
-if test -f $lsb_functions ; then
+if test -f $lsb_functions; then
   . $lsb_functions
-else
-  # Include non-LSB RedHat init functions to make systemctl redirect work
-  init_functions="/etc/init.d/functions"
-  if test -f $init_functions; then
-    . $init_functions
-  fi
+fi
+
+if test -f $init_functions; then
+  . $init_functions
   log_success_msg()
   {
     echo " SUCCESS! $@"

From 77b11965220e249b8fe1dc178e0aff4a8a58db2a Mon Sep 17 00:00:00 2001
From: Julius Goryavsky <julius.goryavsky@mariadb.com>
Date: Thu, 23 Sep 2021 16:14:54 +0200
Subject: [PATCH 05/31] MDEV-26360: Using hostnames breaks certificate
 validation

Fixed flaws with overly strict or, conversely,
overly soft verification of certificates in some
scenarios:

1. Removed the check that the 'commonname' (CN) in the
   certificate matches the 'localhost' value on the side
   of the joiner node, which was performed earlier, even
   if the address was received by the script only as an
   argument (out of the exchange via the Galera protocol) -
   since for the joining node this argument always contains
   its own local address, not the address of the remote host,
   so it is always treated as 'localhost', which is not
   necessarily true (outside of mtr testing);
2. Removed checking the domain name or IP-address of the
   peer node in the encrypt=2 mode;
3. Fixed checking of compliance of certificates when
   rsync SST is used;
4. Added the ability to specify CA not only as a file,
   but also as a path to the directory where the certificates
   are stored. To do this, the user just needs to specify the
   path to this directory as the value ssl-ca or tca parameter,
   ending with the '/' character.
---
 scripts/wsrep_sst_common.sh        | 16 ++++++--
 scripts/wsrep_sst_mariabackup.sh   | 55 +++++++++++++++++++-------
 scripts/wsrep_sst_rsync.sh         | 63 +++++++++++++++++++-----------
 scripts/wsrep_sst_xtrabackup-v2.sh | 61 +++++++++++++++++++++--------
 4 files changed, 136 insertions(+), 59 deletions(-)

diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh
index 67244a7c622..a1293fcb749 100644
--- a/scripts/wsrep_sst_common.sh
+++ b/scripts/wsrep_sst_common.sh
@@ -1065,8 +1065,9 @@ check_for_dhparams()
 #
 verify_ca_matches_cert()
 {
-    local ca_path="$1"
-    local cert_path="$2"
+    local ca="$1"
+    local cert="$2"
+    local path=${3:-0}
 
     # If the openssl utility is not installed, then
     # we will not do this certificate check:
@@ -1075,8 +1076,15 @@ verify_ca_matches_cert()
         return
     fi
 
-    if ! "$OPENSSL_BINARY" verify -verbose -CAfile "$ca_path" "$cert_path" >/dev/null 2>&1
-    then
+    local not_match=0
+
+    if [ $path -eq 0 ]; then
+        "$OPENSSL_BINARY" verify -verbose -CAfile "$ca" "$cert" >/dev/null 2>&1 || not_match=1
+    else
+        "$OPENSSL_BINARY" verify -verbose -CApath "$ca" "$cert" >/dev/null 2>&1 || not_match=1
+    fi
+
+    if [ $not_match -eq 1 ]; then
         wsrep_log_error "******** FATAL ERROR ********************************************"
         wsrep_log_error "* The certifcate and CA (certificate authority) do not match.   *"
         wsrep_log_error "* It does not appear that the certificate was issued by the CA. *"
diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
index 54632e5f79b..3fe3bf5c206 100644
--- a/scripts/wsrep_sst_mariabackup.sh
+++ b/scripts/wsrep_sst_mariabackup.sh
@@ -34,6 +34,7 @@ ssyslog=""
 ssystag=""
 BACKUP_PID=""
 tcert=""
+tpath=0
 tpem=""
 tkey=""
 tmode="DISABLED"
@@ -85,7 +86,6 @@ readonly SECRET_TAG="secret"
 
 # Required for backup locks
 # For backup locks it is 1 sent by joiner
-# 5.6.21 PXC and later can't donate to an older joiner
 sst_ver=1
 
 if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then
@@ -339,64 +339,83 @@ get_transfer()
             fi
         fi
 
+        CN_option=",commonname=''"
+
         if [ $encrypt -eq 2 ]; then
             wsrep_log_info "Using openssl based encryption with socat: with crt and pem"
             if [ -z "$tpem" -o -z "$tcert" ]; then
-                wsrep_log_error "Both PEM and CRT files required"
+                wsrep_log_error \
+                    "Both PEM file and CRT file (or path) are required"
                 exit 22
             fi
             if [ ! -r "$tpem" -o ! -r "$tcert" ]; then
-                wsrep_log_error "Both PEM and CRT files must be readable"
+                wsrep_log_error \
+                    "Both PEM file and CRT file (or path) must be readable"
                 exit 22
             fi
-            verify_ca_matches_cert "$tcert" "$tpem"
-            tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt"
+            verify_ca_matches_cert "$tcert" "$tpem" $tpath
+            if [ $tpath -eq 0 ]; then
+                tcmd="$tcmd,cert='$tpem',cafile='$tcert'"
+            else
+                tcmd="$tcmd,cert='$tpem',capath='$tcert'"
+            fi
             stagemsg="$stagemsg-OpenSSL-Encrypted-2"
-            wsrep_log_info "$action with cert=$tpem, cafile=$tcert"
+            wsrep_log_info "$action with cert=$tpem, ca=$tcert"
         elif [ $encrypt -eq 3 -o $encrypt -eq 4 ]; then
             wsrep_log_info "Using openssl based encryption with socat: with key and crt"
             if [ -z "$tpem" -o -z "$tkey" ]; then
-                wsrep_log_error "Both certificate and key files required"
+                wsrep_log_error "Both certificate file (or path) " \
+                                "and key file are required"
                 exit 22
             fi
             if [ ! -r "$tpem" -o ! -r "$tkey" ]; then
-                wsrep_log_error "Both certificate and key files must be readable"
+                wsrep_log_error "Both certificate file (or path) " \
+                                "and key file must be readable"
                 exit 22
             fi
             verify_cert_matches_key "$tpem" "$tkey"
             stagemsg="$stagemsg-OpenSSL-Encrypted-3"
             if [ -z "$tcert" ]; then
                 if [ $encrypt -eq 4 ]; then
-                    wsrep_log_error "Peer certificate required if encrypt=4"
+                    wsrep_log_error \
+                        "Peer certificate file (or path) required if encrypt=4"
                     exit 22
                 fi
                 # no verification
-                tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0$sockopt"
+                CN_option=""
+                tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0"
                 wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0"
             else
                 # CA verification
                 if [ ! -r "$tcert" ]; then
-                    wsrep_log_error "Certificate file must be readable"
+                    wsrep_log_error "Certificate file or path must be readable"
                     exit 22
                 fi
-                verify_ca_matches_cert "$tcert" "$tpem"
+                verify_ca_matches_cert "$tcert" "$tpem" $tpath
                 if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
                     CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'"
-                elif [ $encrypt -eq 4 ]; then
+                elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' -o $encrypt -eq 4 ]
+                then
                     CN_option=",commonname=''"
                 elif is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then
                     CN_option=',commonname=localhost'
                 else
                     CN_option=",commonname='$WSREP_SST_OPT_HOST_UNESCAPED'"
                 fi
-                tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'$CN_option$sockopt"
-                wsrep_log_info "$action with cert=$tpem, key=$tkey, cafile=$tcert"
+                if [ $tpath -eq 0 ]; then
+                    tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'"
+                else
+                    tcmd="$tcmd,cert='$tpem',key='$tkey',capath='$tcert'"
+                fi
+                wsrep_log_info "$action with cert=$tpem, key=$tkey, ca=$tcert"
             fi
         else
             wsrep_log_info "Unknown encryption mode: encrypt=$encrypt"
             exit 22
         fi
 
+        tcmd="$tcmd$CN_option$sockopt"
+
         if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
             tcmd="$tcmd stdio"
         fi
@@ -473,6 +492,12 @@ check_server_ssl_config()
                            "of the tca, tcert and/or tkey in the [sst] section"
         fi
     fi
+    if [ -n "$tcert" ]; then
+       tcert=$(trim_string "$tcert")
+       if [ "${tcert%/}" != "$tcert" ]; then
+           tpath=1
+       fi
+    fi
 }
 
 read_cnf()
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
index e16ed75cb16..ad9688011e1 100644
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -236,11 +236,18 @@ check_server_ssl_config()
 SSLMODE=$(parse_cnf "$SST_SECTIONS" 'ssl-mode' | tr [:lower:] [:upper:])
 
 # no old-style SSL config in [sst], check for new one:
-if [ -z "$SSTKEY" -a -z "$SSTCERT" -a -z "$SSTCA" ]
-then
+if [ -z "$SSTKEY" -a -z "$SSTCERT" -a -z "$SSTCA" ]; then
     check_server_ssl_config
 fi
 
+SSTPATH=0
+if [ -n "$SSTCA" ]; then
+   SSTCA=$(trim_string "$SSTCA")
+   if [ "${SSTCA%/}" != "$SSTCA" ]; then
+       SSTPATH=1
+   fi
+fi
+
 if [ -z "$SSLMODE" ]; then
     # Implicit verification if CA is set and the SSL mode
     # is not specified by user:
@@ -254,9 +261,19 @@ if [ -z "$SSLMODE" ]; then
     fi
 fi
 
-if [ -n "$SSTCA" ]
-then
-    CAFILE_OPT="CAfile = $SSTCA"
+if [ -n "$SSTCERT" -a -n "$SSTKEY" ]; then
+    verify_cert_matches_key "$SSTCERT" "$SSTKEY"
+fi
+
+if [ -n "$SSTCA" ]; then
+    if [ $SSTPATH -eq 0 ]; then
+        CAFILE_OPT="CAfile = $SSTCA"
+    else
+        CAFILE_OPT="CApath = $SSTCA"
+    fi
+    if [ -n "$SSTCERT" ]; then
+        verify_ca_matches_cert "$SSTCA" "$SSTCERT" $SSTPATH
+    fi
 else
     CAFILE_OPT=""
 fi
@@ -272,38 +289,38 @@ then
         ;;
     'VERIFY_CA')
         VERIFY_OPT='verifyChain = yes'
-        if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
-            CHECK_OPT="checkHost = $WSREP_SST_OPT_REMOTE_USER"
-        else
-            # check if the address is an ip-address (v4 or v6):
-            if echo "$WSREP_SST_OPT_HOST_UNESCAPED" | \
-               grep -q -E '^([0-9]+(\.[0-9]+){3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$'
-            then
-                CHECK_OPT="checkIP = $WSREP_SST_OPT_HOST_UNESCAPED"
-            else
-                CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST"
-            fi
-            if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then
-                CHECK_OPT_LOCAL="checkHost = localhost"
-            fi
-        fi
         ;;
     *)
         wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'"
         exit 22 # EINVAL
         ;;
     esac
-    if [ -z "$CAFILE_OPT" ]; then
-        wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file"
+    if [ -z "$SSTCA" ]; then
+        wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file or path"
         exit 22 # EINVAL
     fi
+    if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
+        CHECK_OPT="checkHost = $WSREP_SST_OPT_REMOTE_USER"
+    elif [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
+        # check if the address is an ip-address (v4 or v6):
+        if echo "$WSREP_SST_OPT_HOST_UNESCAPED" | \
+           grep -q -E '^([0-9]+(\.[0-9]+){3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$'
+        then
+            CHECK_OPT="checkIP = $WSREP_SST_OPT_HOST_UNESCAPED"
+        else
+            CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST"
+        fi
+        if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then
+            CHECK_OPT_LOCAL="checkHost = localhost"
+        fi
+    fi
 fi
 
 STUNNEL=""
 if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then
     STUNNEL_BIN="$(command -v stunnel)"
     if [ -n "$STUNNEL_BIN" ]; then
-        wsrep_log_info "Using stunnel for SSL encryption: CAfile: '$SSTCA', ssl-mode='$SSLMODE'"
+        wsrep_log_info "Using stunnel for SSL encryption: CA: '$SSTCA', ssl-mode='$SSLMODE'"
         STUNNEL="$STUNNEL_BIN $STUNNEL_CONF"
     fi
 fi
diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh
index 999c6c649ba..7cc05185ce3 100644
--- a/scripts/wsrep_sst_xtrabackup-v2.sh
+++ b/scripts/wsrep_sst_xtrabackup-v2.sh
@@ -34,6 +34,7 @@ ssyslog=""
 ssystag=""
 BACKUP_PID=""
 tcert=""
+tpath=0
 tpem=""
 tkey=""
 tmode="DISABLED"
@@ -201,9 +202,10 @@ get_keys()
         if [ -z "$ekey" ]; then
             ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key-file='$ekeyfile'"
         else
-            wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key"
-            wsrep_log_warning "to be set via the command-line and is considered insecure."
-            wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead."
+            wsrep_log_warning \
+                "Using the 'encrypt-key' option causes the encryption key " \
+                "to be set via the command-line and is considered insecure. " \
+                "It is recommended to use the 'encrypt-key-file' option instead."
             ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key='$ekey'"
         fi
         if [ -n "$encrypt_threads" ]; then
@@ -341,64 +343,83 @@ get_transfer()
             fi
         fi
 
+        CN_option=",commonname=''"
+
         if [ $encrypt -eq 2 ]; then
             wsrep_log_info "Using openssl based encryption with socat: with crt and pem"
             if [ -z "$tpem" -o -z "$tcert" ]; then
-                wsrep_log_error "Both PEM and CRT files required"
+                wsrep_log_error \
+                    "Both PEM file and CRT file (or path) are required"
                 exit 22
             fi
             if [ ! -r "$tpem" -o ! -r "$tcert" ]; then
-                wsrep_log_error "Both PEM and CRT files must be readable"
+                wsrep_log_error \
+                    "Both PEM file and CRT file (or path) must be readable"
                 exit 22
             fi
-            verify_ca_matches_cert "$tcert" "$tpem"
-            tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt"
+            verify_ca_matches_cert "$tcert" "$tpem" $tpath
+            if [ $tpath -eq 0 ]; then
+                tcmd="$tcmd,cert='$tpem',cafile='$tcert'"
+            else
+                tcmd="$tcmd,cert='$tpem',capath='$tcert'"
+            fi
             stagemsg="$stagemsg-OpenSSL-Encrypted-2"
-            wsrep_log_info "$action with cert=$tpem, cafile=$tcert"
+            wsrep_log_info "$action with cert=$tpem, ca=$tcert"
         elif [ $encrypt -eq 3 -o $encrypt -eq 4 ]; then
             wsrep_log_info "Using openssl based encryption with socat: with key and crt"
             if [ -z "$tpem" -o -z "$tkey" ]; then
-                wsrep_log_error "Both certificate and key files required"
+                wsrep_log_error "Both certificate file (or path) " \
+                                "and key file are required"
                 exit 22
             fi
             if [ ! -r "$tpem" -o ! -r "$tkey" ]; then
-                wsrep_log_error "Both certificate and key files must be readable"
+                wsrep_log_error "Both certificate file (or path) " \
+                                "and key file must be readable"
                 exit 22
             fi
             verify_cert_matches_key "$tpem" "$tkey"
             stagemsg="$stagemsg-OpenSSL-Encrypted-3"
             if [ -z "$tcert" ]; then
                 if [ $encrypt -eq 4 ]; then
-                    wsrep_log_error "Peer certificate required if encrypt=4"
+                    wsrep_log_error \
+                        "Peer certificate file (or path) required if encrypt=4"
                     exit 22
                 fi
                 # no verification
-                tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0$sockopt"
+                CN_option=""
+                tcmd="$tcmd,cert='$tpem',key='$tkey',verify=0"
                 wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0"
             else
                 # CA verification
                 if [ ! -r "$tcert" ]; then
-                    wsrep_log_error "Certificate file must be readable"
+                    wsrep_log_error "Certificate file or path must be readable"
                     exit 22
                 fi
-                verify_ca_matches_cert "$tcert" "$tpem"
+                verify_ca_matches_cert "$tcert" "$tpem" $tpath
                 if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
                     CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'"
-                elif [ $encrypt -eq 4 ]; then
+                elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' -o $encrypt -eq 4 ]
+                then
                     CN_option=",commonname=''"
                 elif is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then
                     CN_option=',commonname=localhost'
                 else
                     CN_option=",commonname='$WSREP_SST_OPT_HOST_UNESCAPED'"
                 fi
-                tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'$CN_option$sockopt"
-                wsrep_log_info "$action with cert=$tpem, key=$tkey, cafile=$tcert"
+                if [ $tpath -eq 0 ]; then
+                    tcmd="$tcmd,cert='$tpem',key='$tkey',cafile='$tcert'"
+                else
+                    tcmd="$tcmd,cert='$tpem',key='$tkey',capath='$tcert'"
+                fi
+                wsrep_log_info "$action with cert=$tpem, key=$tkey, ca=$tcert"
             fi
         else
             wsrep_log_info "Unknown encryption mode: encrypt=$encrypt"
             exit 22
         fi
 
+        tcmd="$tcmd$CN_option$sockopt"
+
         if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
             tcmd="$tcmd stdio"
         fi
@@ -475,6 +496,12 @@ check_server_ssl_config()
                            "of the tca, tcert and/or tkey in the [sst] section"
         fi
     fi
+    if [ -n "$tcert" ]; then
+       tcert=$(trim_string "$tcert")
+       if [ "${tcert%/}" != "$tcert" ]; then
+           tpath=1
+       fi
+    fi
 }
 
 read_cnf()

From ca7046dc19d8bbb5916c89d9e98e9d4632294fa8 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Fri, 24 Sep 2021 01:33:05 +0200
Subject: [PATCH 06/31] MDEV-11499 mysqltest, Windows : improve diagnostics if
 server fails to shutdown

Create minidump when server fails to shutdown. If process is being
debugged, cause a debug break.

Moves some code which is part of safe_kill into mysys, as both safe_kill,
and mysqltest produce minidumps on different timeouts.

Small cleanup in wait_until_dead() - replace inefficient loop with a single
wait.
---
 client/mysqltest.cc                           |  54 ++++++--
 include/my_minidump.h                         |  25 ++++
 mysql-test/lib/My/SafeProcess/CMakeLists.txt  |   3 +-
 .../lib/My/SafeProcess/safe_kill_win.cc       | 109 +----------------
 mysys/CMakeLists.txt                          |   4 +-
 mysys/my_minidump.cc                          | 115 ++++++++++++++++++
 6 files changed, 193 insertions(+), 117 deletions(-)
 create mode 100644 include/my_minidump.h
 create mode 100644 mysys/my_minidump.cc

diff --git a/client/mysqltest.cc b/client/mysqltest.cc
index ad1eb29ab70..82bd1abf1a3 100644
--- a/client/mysqltest.cc
+++ b/client/mysqltest.cc
@@ -5026,13 +5026,34 @@ int query_get_string(MYSQL* mysql, const char* query,
 }
 
 
+#ifdef _WIN32
+#define SIGKILL 9
+#include <my_minidump.h>
 static int my_kill(int pid, int sig)
 {
-  DBUG_PRINT("info", ("Killing server, pid: %d", pid));
-#ifdef _WIN32
-#define SIGKILL 9 /* ignored anyway, see below */
   HANDLE proc;
-  if ((proc= OpenProcess(SYNCHRONIZE|PROCESS_TERMINATE, FALSE, pid)) == NULL)
+  if (sig == SIGABRT)
+  {
+    /*
+     Create a minidump. If process is being debugged, debug break
+     Otherwise, terminate.
+    */
+    verbose_msg("Aborting %d",pid);
+    my_create_minidump(pid,TRUE);
+    proc= OpenProcess(PROCESS_ALL_ACCESS, FALSE, pid);
+    if(!proc)
+      return -1;
+    BOOL debugger_present;
+    if (CheckRemoteDebuggerPresent(proc,&debugger_present) && debugger_present)
+    {
+      if (DebugBreakProcess(proc))
+      {
+        CloseHandle(proc);
+        return 0;
+      }
+    }
+  }
+  else if ((proc= OpenProcess(SYNCHRONIZE|PROCESS_TERMINATE, FALSE, pid)) == NULL)
     return -1;
   if (sig == 0)
   {
@@ -5043,12 +5064,30 @@ static int my_kill(int pid, int sig)
   (void)TerminateProcess(proc, 201);
   CloseHandle(proc);
   return 1;
-#else
-  return kill(pid, sig);
-#endif
 }
 
 
+/* Wait until process is gone, with timeout */
+static int wait_until_dead(int pid, int timeout)
+{
+  HANDLE proc= OpenProcess(SYNCHRONIZE, FALSE, pid);
+  if (!proc)
+    return 0; /* already dead */
+  DBUG_ASSERT(timeout >= 0);
+  DBUG_ASSERT(timeout <= UINT_MAX/1000);
+  DWORD wait_result= WaitForSingleObject(proc, (DWORD)timeout*1000);
+  CloseHandle(proc);
+  return (int)wait_result;
+}
+
+#else /* !_WIN32 */
+
+
+static int my_kill(int pid, int sig)
+{
+  DBUG_PRINT("info", ("Killing server, pid: %d", pid));
+  return kill(pid, sig);
+}
 
 /*
   Shutdown the server of current connection and
@@ -5083,6 +5122,7 @@ static int wait_until_dead(int pid, int timeout)
   }
   DBUG_RETURN(1);                               // Did not die
 }
+#endif /* _WIN32 */
 
 
 void do_shutdown_server(struct st_command *command)
diff --git a/include/my_minidump.h b/include/my_minidump.h
new file mode 100644
index 00000000000..0c0e843be8c
--- /dev/null
+++ b/include/my_minidump.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2021, MariaDB Corporation
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA */
+
+#include <windows.h>
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+BOOL my_create_minidump(DWORD pid, BOOL verbose);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/mysql-test/lib/My/SafeProcess/CMakeLists.txt b/mysql-test/lib/My/SafeProcess/CMakeLists.txt
index 33ff99da89b..4946af7d899 100644
--- a/mysql-test/lib/My/SafeProcess/CMakeLists.txt
+++ b/mysql-test/lib/My/SafeProcess/CMakeLists.txt
@@ -17,7 +17,8 @@
 IF (WIN32)
   ADD_EXECUTABLE(my_safe_process safe_process_win.cc)
   ADD_EXECUTABLE(my_safe_kill safe_kill_win.cc)
-  TARGET_LINK_LIBRARIES(my_safe_kill dbghelp psapi)
+  TARGET_INCLUDE_DIRECTORIES(my_safe_kill PRIVATE ${CMAKE_SOURCE_DIR}/include)
+  TARGET_LINK_LIBRARIES(my_safe_kill mysys psapi)
 ELSE()
   ADD_EXECUTABLE(my_safe_process safe_process.cc)
 ENDIF()
diff --git a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc
index 4a9d5f2b8cc..375ed80b292 100644
--- a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc
+++ b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc
@@ -26,19 +26,7 @@
 #include <signal.h>
 #include <stdlib.h>
 #include <psapi.h>
-
-#ifdef _MSC_VER
-/* Silence warning in OS header dbghelp.h */
-#pragma warning(push)
-#pragma warning(disable : 4091)
-#endif
-
-#include <dbghelp.h>
-
-#ifdef _MSC_VER
-/* Silence warning in OS header dbghelp.h */
-#pragma warning(pop)
-#endif
+#include <my_minidump.h>
 
 #include <tlhelp32.h>
 #include <vector>
@@ -64,106 +52,13 @@ static std::vector<DWORD> find_children(DWORD pid)
   return children;
 }
 
-void dump_single_process(DWORD pid)
-{
-  HANDLE file = 0;
-  HANDLE process= 0;
-  DWORD size= MAX_PATH;
-  char path[MAX_PATH];
-  char working_dir[MAX_PATH];
-  char tmpname[MAX_PATH];
-  char *filename= 0;
-
-  process= OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid);
-  if (!process)
-  {
-    fprintf(stderr, "safe_kill : cannot open process pid=%lu to create dump, last error %lu\n",
-      pid, GetLastError());
-    goto exit;
-  }
-
-  if (QueryFullProcessImageName(process, 0, path, &size) == 0)
-  {
-    fprintf(stderr, "safe_kill : cannot read process path for pid %lu, last error %lu\n",
-      pid, GetLastError());
-    goto exit;
-  }
-
-  filename= strrchr(path, '\\');
-  if (filename)
-  {
-    filename++;
-    // We are not interested in dump of some proceses (my_safe_process.exe,cmd.exe)
-    // since they are only used to start up other programs.
-    // We're interested however in their children;
-    const char *exclude_programs[] = {"my_safe_process.exe","cmd.exe", 0};
-    for(size_t i=0; exclude_programs[i]; i++)
-      if (_stricmp(filename, exclude_programs[i]) == 0)
-        goto exit;
-  }
-  else
-    filename= path;
-
-  // Add .dmp extension
-  char *p;
-  if ((p= strrchr(filename, '.')) == 0)
-    p= filename + strlen(filename);
-
-  strncpy(p, ".dmp", path + MAX_PATH - p);
-
-  // Íf file with this name exist, generate unique name with .dmp extension
-  if (GetFileAttributes(filename) != INVALID_FILE_ATTRIBUTES)
-  {
-    if (!GetTempFileName(".", filename, 0, tmpname))
-    {
-      fprintf(stderr, "GetTempFileName failed, last error %lu", GetLastError());
-      goto exit;
-    }
-    strncat_s(tmpname, ".dmp", sizeof(tmpname));
-    filename= tmpname;
-  }
-
-  
-  if (!GetCurrentDirectory(MAX_PATH, working_dir))
-  {
-    fprintf(stderr, "GetCurrentDirectory failed, last error %lu", GetLastError());
-    goto exit;
-  }
-
-  file= CreateFile(filename, GENERIC_READ | GENERIC_WRITE,
-    0, 0, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
-
-  if (file == INVALID_HANDLE_VALUE)
-  {
-    fprintf(stderr, "safe_kill : CreateFile() failed for file %s, working dir %s, last error = %lu\n",
-      filename, working_dir, GetLastError());
-    goto exit;
-  }
-
-  if (!MiniDumpWriteDump(process, pid, file, MiniDumpNormal, 0, 0, 0))
-  {
-    fprintf(stderr, "Failed to write minidump to %s, working dir %s, last error %lu\n",
-      filename, working_dir, GetLastError());
-    goto exit;
-  }
-
-  fprintf(stderr, "Minidump written to %s, directory %s\n", filename, working_dir);
-
-exit:
-  if (process != 0 && process != INVALID_HANDLE_VALUE)
-    CloseHandle(process);
-
-  if (file != 0 && file != INVALID_HANDLE_VALUE)
-    CloseHandle(file);
-}
-
 
 static int create_dump(DWORD pid, int recursion_depth= 5)
 {
   if (recursion_depth < 0)
     return 0;
 
-  dump_single_process(pid);
+  my_create_minidump(pid, TRUE);
   std::vector<DWORD> children= find_children(pid);
   for(size_t i=0; i < children.size(); i++)
     create_dump(children[i], recursion_depth -1);
diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt
index 3505f417762..f97e3b4d390 100644
--- a/mysys/CMakeLists.txt
+++ b/mysys/CMakeLists.txt
@@ -48,7 +48,7 @@ SET(MYSYS_SOURCES  array.c charset-def.c charset.c checksum.c my_default.c
                                 file_logger.c my_dlerror.c)
 
 IF (WIN32)
- SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_winthread.c my_wincond.c my_winerr.c my_winfile.c my_windac.c my_conio.c)
+ SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_winthread.c my_wincond.c my_winerr.c my_winfile.c my_windac.c my_conio.c my_minidump.cc)
 ENDIF()
 
 IF(UNIX)
@@ -83,7 +83,7 @@ IF(HAVE_BFD_H)
 ENDIF(HAVE_BFD_H)
 
 IF (WIN32)
-  TARGET_LINK_LIBRARIES(mysys IPHLPAPI)  
+  TARGET_LINK_LIBRARIES(mysys iphlpapi dbghelp)
 ENDIF(WIN32)
 
 # Need explicit pthread for gcc -fsanitize=address
diff --git a/mysys/my_minidump.cc b/mysys/my_minidump.cc
new file mode 100644
index 00000000000..d81aab2f764
--- /dev/null
+++ b/mysys/my_minidump.cc
@@ -0,0 +1,115 @@
+/* Copyright (c) 2021, MariaDB Corporation
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335  USA */
+
+#include <windows.h>
+#include <dbghelp.h>
+#include <stdio.h>
+#include <my_minidump.h>
+
+#define VERBOSE(fmt,...) \
+  if (verbose)  { fprintf(stderr, "my_create_minidump : " fmt,__VA_ARGS__); }
+
+extern "C" BOOL my_create_minidump(DWORD pid, BOOL verbose)
+{
+  HANDLE file = 0;
+  HANDLE process= 0;
+  DWORD size= MAX_PATH;
+  char path[MAX_PATH];
+  char working_dir[MAX_PATH];
+  char tmpname[MAX_PATH];
+  char *filename= 0;
+  bool ret= FALSE;
+  process= OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid);
+  if (!process)
+  {
+    VERBOSE("cannot open process pid=%lu to create dump, last error %lu\n",
+      pid, GetLastError());
+    goto exit;
+  }
+
+  if (QueryFullProcessImageName(process, 0, path, &size) == 0)
+  {
+    VERBOSE("cannot read process path for pid %lu, last error %lu\n",
+      pid, GetLastError());
+    goto exit;
+  }
+
+  filename= strrchr(path, '\\');
+  if (filename)
+  {
+    filename++;
+    // We are not interested in dump of some proceses (my_safe_process.exe,cmd.exe)
+    // since they are only used to start up other programs.
+    // We're interested however in their children;
+    const char *exclude_programs[] = {"my_safe_process.exe","cmd.exe", 0};
+    for(size_t i=0; exclude_programs[i]; i++)
+      if (_stricmp(filename, exclude_programs[i]) == 0)
+        goto exit;
+  }
+  else
+    filename= path;
+
+  // Add .dmp extension
+  char *p;
+  if ((p= strrchr(filename, '.')) == 0)
+    p= filename + strlen(filename);
+
+  strncpy(p, ".dmp", path + MAX_PATH - p);
+
+  // �f file with this name exist, generate unique name with .dmp extension
+  if (GetFileAttributes(filename) != INVALID_FILE_ATTRIBUTES)
+  {
+    if (!GetTempFileName(".", filename, 0, tmpname))
+    {
+      fprintf(stderr, "GetTempFileName failed, last error %lu", GetLastError());
+      goto exit;
+    }
+    strncat_s(tmpname, ".dmp", sizeof(tmpname));
+    filename= tmpname;
+  }
+
+  if (!GetCurrentDirectory(MAX_PATH, working_dir))
+  {
+    VERBOSE("GetCurrentDirectory failed, last error %lu", GetLastError());
+    goto exit;
+  }
+
+  file= CreateFile(filename, GENERIC_READ | GENERIC_WRITE,
+    0, 0, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
+
+  if (file == INVALID_HANDLE_VALUE)
+  {
+    VERBOSE("CreateFile() failed for file %s, working dir %s, last error = %lu\n",
+      filename, working_dir, GetLastError());
+    goto exit;
+  }
+
+  if (!MiniDumpWriteDump(process, pid, file, MiniDumpNormal, 0, 0, 0))
+  {
+    VERBOSE("Failed to write minidump to %s, working dir %s, last error %lu\n",
+      filename, working_dir, GetLastError());
+    goto exit;
+  }
+
+  VERBOSE("Minidump written to %s, directory %s\n", filename, working_dir);
+  ret= TRUE;
+exit:
+  if (process != 0 && process != INVALID_HANDLE_VALUE)
+    CloseHandle(process);
+
+  if (file != 0 && file != INVALID_HANDLE_VALUE)
+    CloseHandle(file);
+  return ret;
+}

From 8221708e389728aef799046eef3c49b1eec2e400 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Fri, 24 Sep 2021 10:07:01 +0200
Subject: [PATCH 07/31] Windows, mysqltest : cleanup, remove dead code
 USE_CYGWIN

---
 client/mysqltest.cc | 54 ---------------------------------------------
 1 file changed, 54 deletions(-)

diff --git a/client/mysqltest.cc b/client/mysqltest.cc
index 82bd1abf1a3..736e708e0f4 100644
--- a/client/mysqltest.cc
+++ b/client/mysqltest.cc
@@ -71,10 +71,6 @@ static my_bool non_blocking_api_enabled= 0;
 #include "../tests/nonblock-wrappers.h"
 #endif
 
-/* Use cygwin for --exec and --system before 5.0 */
-#if MYSQL_VERSION_ID < 50000
-#define USE_CYGWIN
-#endif
 
 #define MAX_VAR_NAME_LENGTH    256
 #define MAX_COLUMNS            256
@@ -618,11 +614,6 @@ void fix_win_paths(char *val, size_t len);
 const char *get_errname_from_code (uint error_code);
 int multi_reg_replace(struct st_replace_regex* r,char* val);
 
-#ifdef _WIN32
-void free_tmp_sh_file();
-void free_win_path_patterns();
-#endif
-
 
 /* For replace_column */
 static char *replace_column[MAX_COLUMNS];
@@ -1471,10 +1462,6 @@ void free_used_memory()
   free_root(&require_file_root, MYF(0));
   free_re();
   my_free(read_command_buf);
-#ifdef _WIN32
-  free_tmp_sh_file();
-  free_win_path_patterns();
-#endif
   DBUG_VOID_RETURN;
 }
 
@@ -3204,33 +3191,6 @@ void do_source(struct st_command *command)
 }
 
 
-#if defined _WIN32
-
-#ifdef USE_CYGWIN
-/* Variables used for temporary sh files used for emulating Unix on Windows */
-char tmp_sh_name[64], tmp_sh_cmd[70];
-#endif
-
-void init_tmp_sh_file()
-{
-#ifdef USE_CYGWIN
-  /* Format a name for the tmp sh file that is unique for this process */
-  my_snprintf(tmp_sh_name, sizeof(tmp_sh_name), "tmp_%d.sh", getpid());
-  /* Format the command to execute in order to run the script */
-  my_snprintf(tmp_sh_cmd, sizeof(tmp_sh_cmd), "sh %s", tmp_sh_name);
-#endif
-}
-
-
-void free_tmp_sh_file()
-{
-#ifdef USE_CYGWIN
-  my_delete(tmp_sh_name, MYF(0));
-#endif
-}
-#endif
-
-
 static void init_builtin_echo(void)
 {
 #ifdef _WIN32
@@ -3346,14 +3306,12 @@ void do_exec(struct st_command *command)
   }
 
 #ifdef _WIN32
-#ifndef USE_CYGWIN
   /* Replace /dev/null with NUL */
   while(replace(&ds_cmd, "/dev/null", 9, "NUL", 3) == 0)
     ;
   /* Replace "closed stdout" with non existing output fd */
   while(replace(&ds_cmd, ">&-", 3, ">&4", 3) == 0)
     ;
-#endif
 #endif
 
   if (disable_result_log)
@@ -3512,13 +3470,7 @@ int do_modify_var(struct st_command *command,
 
 int my_system(DYNAMIC_STRING* ds_cmd)
 {
-#if defined _WIN32 && defined USE_CYGWIN
-  /* Dump the command into a sh script file and execute with system */
-  str_to_file(tmp_sh_name, ds_cmd->str, ds_cmd->length);
-  return system(tmp_sh_cmd);
-#else
   return system(ds_cmd->str);
-#endif
 }
 
 
@@ -3552,12 +3504,10 @@ void do_system(struct st_command *command)
   do_eval(&ds_cmd, command->first_argument, command->end, !is_windows);
 
 #ifdef _WIN32
-#ifndef USE_CYGWIN
    /* Replace /dev/null with NUL */
    while(replace(&ds_cmd, "/dev/null", 9, "NUL", 3) == 0)
      ;
 #endif
-#endif
 
 
   DBUG_PRINT("info", ("running system command '%s' as '%s'",
@@ -9221,12 +9171,8 @@ int main(int argc, char **argv)
 
   init_builtin_echo();
 #ifdef _WIN32
-#ifndef USE_CYGWIN
   is_windows= 1;
 #endif
-  init_tmp_sh_file();
-  init_win_path_patterns();
-#endif
 
   read_command_buf= (char*)my_malloc(read_command_buflen= 65536, MYF(MY_FAE));
 

From cfe1a258e884a4e21df9c752c2c3351da96bfb1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 24 Sep 2021 15:33:53 +0300
Subject: [PATCH 08/31] Update libmariadb

---
 libmariadb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libmariadb b/libmariadb
index 42cb1e442c4..b99172386a7 160000
--- a/libmariadb
+++ b/libmariadb
@@ -1 +1 @@
-Subproject commit 42cb1e442c43902e2866bea38d15f2ed1f5d38b3
+Subproject commit b99172386a740ef0c8136e9a6cd7d9ad9a77b31f

From 76972163711f965402d51055f081ab51ae4a3bb7 Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Tue, 21 Sep 2021 13:29:27 +0530
Subject: [PATCH 09/31] MDEV-26631 InnoDB fails to fetch page from doublewrite
 buffer

Problem:
========
InnoDB fails to fetch the page0 from dblwr if page0 is
corrupted.In that case, InnoDB defers the tablespace
and doesn't find the INIT_PAGE redo log record for page0
and it leads to failure.

Solution:
=========
 InnoDB should recover page0 from dblwr if space_id can
be found for deferred tablespace.
---
 mysql-test/suite/innodb/r/doublewrite.result |  1 +
 mysql-test/suite/innodb/t/doublewrite.test   |  2 ++
 storage/innobase/fsp/fsp0file.cc             | 32 +++++++++++++++-----
 3 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result
index 234d58012d3..ba1965ed4cd 100644
--- a/mysql-test/suite/innodb/r/doublewrite.result
+++ b/mysql-test/suite/innodb/r/doublewrite.result
@@ -65,6 +65,7 @@ where name = 'test/t1';
 # Ensure that dirty pages of table t1 is flushed.
 flush tables t1 for export;
 unlock tables;
+set global innodb_log_checkpoint_now=1;
 begin;
 insert into t1 values (6, repeat('%', 12));
 # Make the first page dirty for table t1
diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test
index cedd2c9942b..d8dac955348 100644
--- a/mysql-test/suite/innodb/t/doublewrite.test
+++ b/mysql-test/suite/innodb/t/doublewrite.test
@@ -159,6 +159,8 @@ where name = 'test/t1';
 flush tables t1 for export;
 unlock tables;
 
+set global innodb_log_checkpoint_now=1;
+
 begin;
 insert into t1 values (6, repeat('%', 12));
 
diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc
index aebe6c9ce23..9a552e6dc5c 100644
--- a/storage/innobase/fsp/fsp0file.cc
+++ b/storage/innobase/fsp/fsp0file.cc
@@ -400,10 +400,19 @@ Datafile::validate_for_recovery()
 	err = validate_first_page(0);
 
 	switch (err) {
-	case DB_SUCCESS:
 	case DB_TABLESPACE_EXISTS:
 		break;
-
+	case DB_SUCCESS:
+		if (!m_defer || !m_space_id) {
+			break;
+		}
+		/* InnoDB should check whether the deferred
+		tablespace page0 can be recovered from
+		double write buffer. InnoDB should try
+	        to recover only if m_space_id exists because
+		dblwr pages can be searched via {space_id, 0}.
+		m_space_id is set in read_first_page(). */
+		/* fall through */
 	default:
 		/* Re-open the file in read-write mode  Attempt to restore
 		page 0 from doublewrite and read the space ID from a survey
@@ -414,23 +423,30 @@ Datafile::validate_for_recovery()
 			return(err);
 		}
 
-		err = find_space_id();
-		if (err != DB_SUCCESS || m_space_id == 0) {
-			ib::error() << "Datafile '" << m_filepath << "' is"
-				" corrupted. Cannot determine the space ID from"
-				" the first 64 pages.";
-			return(err);
+		if (!m_defer) {
+			err = find_space_id();
+			if (err != DB_SUCCESS || m_space_id == 0) {
+				ib::error() << "Datafile '" << m_filepath
+					<< "' is corrupted. Cannot determine "
+					"the space ID from the first 64 pages.";
+				return(err);
+			}
 		}
+
 		if (m_space_id == ULINT_UNDEFINED) {
 			return DB_SUCCESS; /* empty file */
 		}
 
 		if (restore_from_doublewrite()) {
+			if (m_defer) {
+				return err;
+			}
 			return(DB_CORRUPTION);
 		}
 
 		/* Free the previously read first page and then re-validate. */
 		free_first_page();
+		m_defer = false;
 		err = validate_first_page(0);
 	}
 

From a5df5aec06d429cfaf0c7a8994e67eaeeebf080b Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Fri, 24 Sep 2021 15:17:52 +0200
Subject: [PATCH 10/31] Fixup "Windows, mysqltest : cleanup, remove dead code
 USE_CYGWIN"

last commit  8221708e389728aef799046eef3c49b1eec2e400 removed too much,
mtr is failing
---
 client/mysqltest.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/client/mysqltest.cc b/client/mysqltest.cc
index 736e708e0f4..bca35b74a64 100644
--- a/client/mysqltest.cc
+++ b/client/mysqltest.cc
@@ -614,6 +614,10 @@ void fix_win_paths(char *val, size_t len);
 const char *get_errname_from_code (uint error_code);
 int multi_reg_replace(struct st_replace_regex* r,char* val);
 
+#ifdef _WIN32
+void free_win_path_patterns();
+#endif
+
 
 /* For replace_column */
 static char *replace_column[MAX_COLUMNS];
@@ -1462,6 +1466,9 @@ void free_used_memory()
   free_root(&require_file_root, MYF(0));
   free_re();
   my_free(read_command_buf);
+#ifdef _WIN32
+  free_win_path_patterns();
+#endif
   DBUG_VOID_RETURN;
 }
 
@@ -9172,6 +9179,7 @@ int main(int argc, char **argv)
   init_builtin_echo();
 #ifdef _WIN32
   is_windows= 1;
+  init_win_path_patterns();
 #endif
 
   read_command_buf= (char*)my_malloc(read_command_buflen= 65536, MYF(MY_FAE));

From f59f5c4a10151b18d1407065ca48746384b6a25a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 24 Sep 2021 16:21:20 +0300
Subject: [PATCH 11/31] Revert MDEV-25114

Revert 88a4be75a5f3b8d59ac8f6347ff2c197813c05dc and
9d97f92febc89941784d17d59c60275e21140ce0, which had been
prematurely pushed by accident.
---
 .../suite/galera/r/galera_UK_conflict.result  |   7 -
 .../galera/r/galera_bf_kill_debug.result      | 163 ---------
 .../galera/r/galera_toi_ddl_fk_insert.result  |  16 +
 .../suite/galera/t/galera_UK_conflict.test    |   4 -
 .../suite/galera/t/galera_bf_kill_debug.cnf   |   9 -
 .../suite/galera/t/galera_bf_kill_debug.test  | 283 ----------------
 .../galera/t/galera_toi_ddl_fk_insert.test    |  14 +-
 mysql-test/suite/wsrep/t/variables.test       |   6 +-
 sql/mysqld.cc                                 |   9 +-
 sql/mysqld.h                                  |   2 +-
 sql/sql_class.cc                              |  24 +-
 sql/sql_class.h                               |   9 +-
 sql/sql_parse.cc                              |  36 +-
 sql/wsrep_mysqld.cc                           |  92 ++----
 sql/wsrep_thd.cc                              |   8 +-
 sql/wsrep_thd.h                               |   1 +
 storage/innobase/handler/ha_innodb.cc         | 308 +++++++++---------
 storage/innobase/include/ha_prototypes.h      |   3 +-
 18 files changed, 229 insertions(+), 765 deletions(-)
 delete mode 100644 mysql-test/suite/galera/r/galera_bf_kill_debug.result
 delete mode 100644 mysql-test/suite/galera/t/galera_bf_kill_debug.cnf
 delete mode 100644 mysql-test/suite/galera/t/galera_bf_kill_debug.test

diff --git a/mysql-test/suite/galera/r/galera_UK_conflict.result b/mysql-test/suite/galera/r/galera_UK_conflict.result
index 2795a86d6a6..76649f1b268 100644
--- a/mysql-test/suite/galera/r/galera_UK_conflict.result
+++ b/mysql-test/suite/galera/r/galera_UK_conflict.result
@@ -68,9 +68,6 @@ f1	f2	f3
 10	10	0
 INSERT INTO t1 VALUES (7,7,7);
 INSERT INTO t1 VALUES (8,8,8);
-SELECT COUNT(*) FROM t1;
-COUNT(*)
-7
 SELECT * FROM t1;
 f1	f2	f3
 1	1	0
@@ -81,9 +78,6 @@ f1	f2	f3
 8	8	8
 10	10	0
 connection node_1;
-SELECT COUNT(*) FROM t1;
-COUNT(*)
-7
 SELECT * FROM t1;
 f1	f2	f3
 1	1	0
@@ -91,6 +85,5 @@ f1	f2	f3
 4	4	2
 5	5	2
 7	7	7
-8	8	8
 10	10	0
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/r/galera_bf_kill_debug.result b/mysql-test/suite/galera/r/galera_bf_kill_debug.result
deleted file mode 100644
index c20bb80131a..00000000000
--- a/mysql-test/suite/galera/r/galera_bf_kill_debug.result
+++ /dev/null
@@ -1,163 +0,0 @@
-#
-# Case 1: We execute bf kill to wsrep_innobase_kill_one_trx
-# function just before wsrep_thd_LOCK(thd) call. Then we
-# try to kill victim transaction by KILL QUERY
-#
-CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
-INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
-connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-begin;
-update t1 set b = b * 10 where id between 2 and 4;
-connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-SET DEBUG_SYNC='wsrep_before_BF_victim_lock SIGNAL bf_kill WAIT_FOR bf_continue';
-ALTER TABLE t1 ADD UNIQUE KEY b1(b);;
-connection node_1;
-SET DEBUG_SYNC='now WAIT_FOR bf_kill';
-connection node_1b;
-Table	Create Table
-t1	CREATE TABLE `t1` (
-  `id` int(11) NOT NULL,
-  `b` int(11) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `b1` (`b`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
-id	b
-1	1
-2	2
-3	3
-4	4
-5	5
-connection node_1;
-SET DEBUG_SYNC= 'RESET';
-DROP TABLE t1;
-disconnect node_1a;
-disconnect node_1b;
-disconnect node_1c;
-#
-# Case 2: We execute bf kill to wsrep_innobase_kill_one_trx
-# function just after wsrep_thd_LOCK(thd) call. Then we
-# try to kill victim transaction by KILL QUERY
-#
-CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
-INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
-connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-begin;
-update t1 set b = b * 10 where id between 2 and 4;
-connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-SET DEBUG_SYNC='wsrep_after_BF_victim_lock SIGNAL bf_kill WAIT_FOR bf_continue';
-ALTER TABLE t1 ADD UNIQUE KEY b1(b);;
-connection node_1;
-SET DEBUG_SYNC='now WAIT_FOR bf_kill';
-connection node_1b;
-Table	Create Table
-t1	CREATE TABLE `t1` (
-  `id` int(11) NOT NULL,
-  `b` int(11) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `b1` (`b`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
-id	b
-1	1
-2	2
-3	3
-4	4
-5	5
-connection node_1;
-SET DEBUG_SYNC= 'RESET';
-DROP TABLE t1;
-disconnect node_1a;
-disconnect node_1b;
-disconnect node_1c;
-#
-# Case 3: Create victim transaction and try to send user KILL
-# from several threads
-#
-CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
-INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
-connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-begin;
-update t1 set b = b * 10 where id between 2 and 4;
-connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-connect node_1d, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-connection node_1b;
-connection node_1c;
-connection node_1d;
-connection node_1;
-disconnect node_1a;
-disconnect node_1b;
-disconnect node_1c;
-disconnect node_1d;
-DROP TABLE t1;
-#
-# Case 4: MDL-conflict, we execute ALTER until we hit gap in
-# wsrep_abort_transaction, while we are there we try to
-# manually KILL conflicting transaction (UPDATE) and
-# send conflicting transaction from other node to be executed
-# in this node by applier. As ALTER and KILL are TOI they
-# are not executed concurrently. Similarly UPDATE from other
-# node will wait for certification.
-#
-CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
-INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
-connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-begin;
-update t1 set b = b * 10 where id between 2 and 4;
-connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-SET DEBUG_SYNC='wsrep_abort_victim_unlocked SIGNAL bf_kill_unlocked WAIT_FOR bf_continue';
-ALTER TABLE t1 ADD UNIQUE KEY b1(b);;
-connection node_1;
-SET DEBUG_SYNC='now WAIT_FOR bf_kill_unlocked';
-connection node_1b;
-connection node_2;
-update t1 set b = b + 1000 where id between 2 and 4;;
-connection node_1;
-SET DEBUG_SYNC='now SIGNAL bf_continue';
-connection node_1c;
-SHOW CREATE TABLE t1;
-Table	Create Table
-t1	CREATE TABLE `t1` (
-  `id` int(11) NOT NULL,
-  `b` int(11) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `b1` (`b`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
-SELECT * FROM t1;
-id	b
-1	1
-5	5
-2	1002
-3	1003
-4	1004
-connection node_1b;
-connection node_1;
-SET DEBUG_SYNC= 'RESET';
-SELECT * FROM t1;
-id	b
-1	1
-5	5
-2	1002
-3	1003
-4	1004
-connection node_2;
-SHOW CREATE TABLE t1;
-Table	Create Table
-t1	CREATE TABLE `t1` (
-  `id` int(11) NOT NULL,
-  `b` int(11) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `b1` (`b`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
-SELECT * FROM t1;
-id	b
-1	1
-5	5
-2	1002
-3	1003
-4	1004
-DROP TABLE t1;
-disconnect node_1a;
-disconnect node_1c;
diff --git a/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result b/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result
index a972394f87c..94752ed7c76 100644
--- a/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result
+++ b/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result
@@ -21,6 +21,22 @@ connection node_1a;
 connection node_1b;
 connection node_2;
 connection node_2a;
+connection node_1;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) FROM parent;
+COUNT(*)
+20001
+SELECT COUNT(*) FROM child;
+COUNT(*)
+10000
+connection node_2;
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) FROM parent;
+COUNT(*)
+20001
+SELECT COUNT(*) FROM child;
+COUNT(*)
+10000
 DROP TABLE child;
 DROP TABLE parent;
 DROP TABLE ten;
diff --git a/mysql-test/suite/galera/t/galera_UK_conflict.test b/mysql-test/suite/galera/t/galera_UK_conflict.test
index 83d0e47dc3d..57bafbf8ae0 100644
--- a/mysql-test/suite/galera/t/galera_UK_conflict.test
+++ b/mysql-test/suite/galera/t/galera_UK_conflict.test
@@ -140,13 +140,9 @@ SELECT * FROM t1;
 # original state in node 1
 INSERT INTO t1 VALUES (7,7,7);
 INSERT INTO t1 VALUES (8,8,8);
-SELECT COUNT(*) FROM t1;
 SELECT * FROM t1;
 
 --connection node_1
---let $wait_condition = SELECT COUNT(*) = 7 FROM t1
---source include/wait_condition.inc
-SELECT COUNT(*) FROM t1;
 SELECT * FROM t1;
 
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/t/galera_bf_kill_debug.cnf b/mysql-test/suite/galera/t/galera_bf_kill_debug.cnf
deleted file mode 100644
index 77bb6af9f35..00000000000
--- a/mysql-test/suite/galera/t/galera_bf_kill_debug.cnf
+++ /dev/null
@@ -1,9 +0,0 @@
-!include ../galera_2nodes.cnf
-
-[mysqld.1]
-wsrep_log_conflicts=ON
-wsrep_debug=1
-
-[mysqld.2]
-wsrep_log_conflicts=ON
-wsrep_debug=1
diff --git a/mysql-test/suite/galera/t/galera_bf_kill_debug.test b/mysql-test/suite/galera/t/galera_bf_kill_debug.test
deleted file mode 100644
index f83d4a28ce9..00000000000
--- a/mysql-test/suite/galera/t/galera_bf_kill_debug.test
+++ /dev/null
@@ -1,283 +0,0 @@
---source include/galera_cluster.inc
---source include/have_debug.inc
---source include/have_debug_sync.inc
-
---echo #
---echo # Case 1: We execute bf kill to wsrep_innobase_kill_one_trx
---echo # function just before wsrep_thd_LOCK(thd) call. Then we
---echo # try to kill victim transaction by KILL QUERY
---echo #
-
-CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
-INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
-
-#
-# This will be victim transaction for both bf kill and
-# user KILL
-#
---connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
-begin;
-update t1 set b = b * 10 where id between 2 and 4;
-
-#
-# Take thread id for above query
-#
---connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
---let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
-
-#
-# Set DEBUG_SYNC and send conflicting DDL that will be TOI (bf) and
-# cause bf_kill
-#
---connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
-SET DEBUG_SYNC='wsrep_before_BF_victim_lock SIGNAL bf_kill WAIT_FOR bf_continue';
---send ALTER TABLE t1 ADD UNIQUE KEY b1(b);
-
-#
-# Wait until we have reached the sync point
-#
---connection node_1
-SET DEBUG_SYNC='now WAIT_FOR bf_kill';
-
-#
-# Try to kill update query
-#
---connection node_1b
---disable_query_log
---send_eval KILL QUERY $k_thread;
-
-
-#
-# Let bf_kill continue
-#
---connection node_1
-SET DEBUG_SYNC='now SIGNAL bf_continue';
---connection node_1c
---reap
-SHOW CREATE TABLE t1;
-SELECT * FROM t1;
-
---connection node_1b
---reap
---enable_query_log
-
---connection node_1
-SET DEBUG_SYNC= 'RESET';
-DROP TABLE t1;
-
---disconnect node_1a
---disconnect node_1b
---disconnect node_1c
-
---echo #
---echo # Case 2: We execute bf kill to wsrep_innobase_kill_one_trx
---echo # function just after wsrep_thd_LOCK(thd) call. Then we
---echo # try to kill victim transaction by KILL QUERY
---echo #
-
-CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
-INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
-
-#
-# This will be victim transaction for both bf kill and
-# user KILL
-#
---connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
-begin;
-update t1 set b = b * 10 where id between 2 and 4;
-
-#
-# Take thread id for above query
-#
---connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
---let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
-
-#
-# Set DEBUG_SYNC and send conflicting DDL that will be TOI (bf) and
-# cause bf_kill
-#
---connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
-SET DEBUG_SYNC='wsrep_after_BF_victim_lock SIGNAL bf_kill WAIT_FOR bf_continue';
---send ALTER TABLE t1 ADD UNIQUE KEY b1(b);
-
-#
-# Wait until we have reached the sync point
-#
---connection node_1
-SET DEBUG_SYNC='now WAIT_FOR bf_kill';
-
-#
-# Try to kill update query
-#
---connection node_1b
---disable_query_log
---send_eval KILL QUERY $k_thread;
-
-#
-# Let bf_kill continue
-#
---connection node_1
-SET DEBUG_SYNC='now SIGNAL bf_continue';
---connection node_1c
---reap
-SHOW CREATE TABLE t1;
-SELECT * FROM t1;
-
---connection node_1b
---reap
---enable_query_log
-
---connection node_1
-SET DEBUG_SYNC= 'RESET';
-DROP TABLE t1;
-
---disconnect node_1a
---disconnect node_1b
---disconnect node_1c
-
---echo #
---echo # Case 3: Create victim transaction and try to send user KILL
---echo # from several threads
---echo #
-
-CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
-INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
-
-#
-# This will be victim transaction for user KILL
-#
---connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
-begin;
-update t1 set b = b * 10 where id between 2 and 4;
-
-#
-# Take thread id for above query
-#
---connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
---connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
---connect node_1d, 127.0.0.1, root, , test, $NODE_MYPORT_1
-
---connection node_1b
---let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
-
-#
-# Try to kill update query from several connections concurrently
-#
---disable_query_log
---send_eval KILL QUERY $k_thread;
-
---connection node_1c
---disable_query_log
---send_eval KILL QUERY $k_thread;
-
---connection node_1d
---disable_query_log
---send_eval KILL QUERY $k_thread;
-
-#
-# We do not know execution order so any of these could fail as KILL
-# has been already done
-#
---connection node_1b
---enable_query_log
---error 0,ER_KILL_DENIED_ERROR
---reap
---connection node_1c
---enable_query_log
---error 0,ER_KILL_DENIED_ERROR
---reap
---connection node_1d
---enable_query_log
---error 0,ER_KILL_DENIED_ERROR
---reap
-
---connection node_1
---disconnect node_1a
---disconnect node_1b
---disconnect node_1c
---disconnect node_1d
-DROP TABLE t1;
-
---echo #
---echo # Case 4: MDL-conflict, we execute ALTER until we hit gap in
---echo # wsrep_abort_transaction, while we are there we try to
---echo # manually KILL conflicting transaction (UPDATE) and
---echo # send conflicting transaction from other node to be executed
---echo # in this node by applier. As ALTER and KILL are TOI they
---echo # are not executed concurrently. Similarly UPDATE from other
---echo # node will wait for certification.
---echo #
-
-CREATE TABLE t1(id int not null primary key, b int) engine=innodb;
-INSERT INTO t1 values (1,1),(2,2),(3,3),(4,4),(5,5);
-
-#
-# This will be victim transaction for both bf kill and
-# user KILL, and should not have any effect on result
-#
---connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
-begin;
-update t1 set b = b * 10 where id between 2 and 4;
-
-#
-# Take thread id for above query
-#
---connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
---let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
-
-#
-# Set DEBUG_SYNC and send conflicting DDL that will be TOI (bf) and
-# cause bf_kill but let's execute it only to gap in wsrep_abort_transaction
-#
---connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
-SET DEBUG_SYNC='wsrep_abort_victim_unlocked SIGNAL bf_kill_unlocked WAIT_FOR bf_continue';
---send ALTER TABLE t1 ADD UNIQUE KEY b1(b);
-
-#
-# Wait until we have reached the sync point
-#
---connection node_1
-SET DEBUG_SYNC='now WAIT_FOR bf_kill_unlocked';
-
-#
-# Try to kill update query
-#
---connection node_1b
---disable_query_log
---send_eval KILL QUERY $k_thread;
-
-#
-# Send conflicting update from other node, this should be applied on both nodes
-# but should not kill ALTER
-#
---enable_query_log
---connection node_2
---send update t1 set b = b + 1000 where id between 2 and 4;
-
-#
-# Let bf_kill continue
-#
---connection node_1
-SET DEBUG_SYNC='now SIGNAL bf_continue';
---connection node_1c
---reap
-SHOW CREATE TABLE t1;
-SELECT * FROM t1;
-
---connection node_1b
---reap
---enable_query_log
-
---connection node_1
-SET DEBUG_SYNC= 'RESET';
-SELECT * FROM t1;
-
---connection node_2
---reap
-SHOW CREATE TABLE t1;
-SELECT * FROM t1;
-DROP TABLE t1;
-
---disconnect node_1a
---disconnect node_1c
-
diff --git a/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test b/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test
index 3b4b427f551..fadc94d78ff 100644
--- a/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test
+++ b/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test
@@ -54,11 +54,15 @@ INSERT INTO parent VALUES (1, 0);
 --connection node_2a
 --reap
 
-#
-# ALTER TABLE could bf kill one or more of INSERTs to parent, so
-# the actual number of rows in PARENT depends on whether
-# the INSERT is committed before ALTER TABLE is executed
-#
+--connection node_1
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) FROM parent;
+SELECT COUNT(*) FROM child;
+
+--connection node_2
+SET SESSION wsrep_sync_wait=15;
+SELECT COUNT(*) FROM parent;
+SELECT COUNT(*) FROM child;
 
 DROP TABLE child;
 DROP TABLE parent;
diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test
index 875315c0e7c..0cf13650ce0 100644
--- a/mysql-test/suite/wsrep/t/variables.test
+++ b/mysql-test/suite/wsrep/t/variables.test
@@ -66,7 +66,7 @@ call mtr.add_suppression("WSREP: Failed to get provider options");
 
 #evalp SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
 
---replace_regex /.*libgalera.*/libgalera_smm.so/
+--replace_regex /.*libgalera_smm.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_slave_threads;
 SELECT @@global.wsrep_cluster_address;
@@ -77,7 +77,7 @@ SHOW STATUS LIKE 'wsrep_thread_count';
 
 #evalp SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
 
---replace_regex /.*libgalera.*/libgalera_smm.so/
+--replace_regex /.*libgalera_smm.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_cluster_address;
 SELECT @@global.wsrep_on;
@@ -101,7 +101,7 @@ SELECT VARIABLE_VALUE AS EXPECT_1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VA
 SELECT VARIABLE_VALUE AS EXPECT_1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
 SELECT VARIABLE_VALUE AS EXPECT_2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
 
---replace_regex /.*libgalera.*/libgalera_smm.so/
+--replace_regex /.*libgalera_smm.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_cluster_address;
 SELECT @@global.wsrep_on;
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 269ad67165a..9afc701c6ba 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -2391,7 +2391,7 @@ static void clean_up_mutexes()
 static void set_ports()
 {
 }
-void close_connection(THD *thd, uint sql_errno, my_bool locked)
+void close_connection(THD *thd, uint sql_errno)
 {
 }
 #else
@@ -2867,7 +2867,7 @@ static void network_init(void)
     For the connection that is doing shutdown, this is called twice
 */
 
-void close_connection(THD *thd, uint sql_errno, my_bool locked)
+void close_connection(THD *thd, uint sql_errno)
 {
   DBUG_ENTER("close_connection");
 
@@ -2877,10 +2877,7 @@ void close_connection(THD *thd, uint sql_errno, my_bool locked)
   thd->print_aborted_warning(3, sql_errno ? ER_DEFAULT(sql_errno)
                                           : "CLOSE_CONNECTION");
 
-  if (locked)
-    thd->disconnect_mutexed();
-  else
-    thd->disconnect();
+  thd->disconnect();
 
   MYSQL_CONNECTION_DONE((int) sql_errno, thd->thread_id);
 
diff --git a/sql/mysqld.h b/sql/mysqld.h
index 1084173ae5f..44b0491f138 100644
--- a/sql/mysqld.h
+++ b/sql/mysqld.h
@@ -83,7 +83,7 @@ enum enum_slave_parallel_mode {
 
 /* Function prototypes */
 void kill_mysql(THD *thd= 0);
-void close_connection(THD *thd, uint sql_errno= 0, my_bool locked=false);
+void close_connection(THD *thd, uint sql_errno= 0);
 void handle_connection_in_main_thread(CONNECT *thd);
 void create_thread_to_handle_connection(CONNECT *connect);
 void signal_thd_deleted();
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index 05ec9a1c369..5ada018e540 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -1804,11 +1804,11 @@ void THD::awake(killed_state state_to_set)
           the Vio might be disassociated concurrently.
 */
 
-void THD::disconnect_mutexed()
+void THD::disconnect()
 {
   Vio *vio= NULL;
 
-  mysql_mutex_assert_owner(&LOCK_thd_data);
+  mysql_mutex_lock(&LOCK_thd_data);
 
   set_killed(KILL_CONNECTION);
 
@@ -1826,6 +1826,8 @@ void THD::disconnect_mutexed()
   if (net.vio != vio)
     vio_close(net.vio);
   net.thd= 0;                                   // Don't collect statistics
+
+  mysql_mutex_unlock(&LOCK_thd_data);
 }
 
 
@@ -1877,18 +1879,16 @@ bool THD::notify_shared_lock(MDL_context_owner *ctx_in_use,
           thread can see those instances (e.g. see partitioning code).
         */
         if (!thd_table->needs_reopen())
+        {
           signalled|= mysql_lock_abort_for_thread(this, thd_table);
+          if (WSREP(this) && wsrep_thd_is_BF(this, FALSE))
+          {
+            WSREP_DEBUG("remove_table_from_cache: %llu",
+                        (unsigned long long) this->real_id);
+            wsrep_abort_thd((void *)this, (void *)in_use, FALSE);
+          }
+        }
       }
-#ifdef WITH_WSREP
-      if (WSREP(this) && wsrep_thd_is_BF(this, false))
-      {
-        WSREP_DEBUG("notify_shared_lock: BF thread %llu query %s"
-                    " victim %llu query %s",
-                    this->real_id, wsrep_thd_query(this),
-                    in_use->real_id, wsrep_thd_query(in_use));
-        wsrep_abort_thd((void *)this, (void *)in_use, false);
-      }
-#endif /* WITH_WSREP */
     }
     mysql_mutex_unlock(&in_use->LOCK_thd_data);
   }
diff --git a/sql/sql_class.h b/sql/sql_class.h
index 7dbc81be770..5f871f9caf6 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -3231,13 +3231,8 @@ public:
   void awake(killed_state state_to_set);
  
   /** Disconnect the associated communication endpoint. */
-  inline void disconnect()
-  {
-    mysql_mutex_lock(&LOCK_thd_data);
-    disconnect_mutexed();
-    mysql_mutex_unlock(&LOCK_thd_data);
-  }
-  void disconnect_mutexed();
+  void disconnect();
+
 
   /*
     Allows this thread to serve as a target for others to schedule Async 
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index 2bec9c6b6cd..3e1f248b082 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2017, Oracle and/or its affiliates.
-   Copyright (c) 2008, 2021, MariaDB
+   Copyright (c) 2008, 2020, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -9069,18 +9069,6 @@ static
 void sql_kill(THD *thd, longlong id, killed_state state, killed_type type)
 {
   uint error;
-#ifdef WITH_WSREP
-  if (WSREP(thd))
-  {
-    WSREP_DEBUG("sql_kill called");
-    if (thd->wsrep_applier)
-    {
-      WSREP_DEBUG("KILL in applying, bailing out here");
-      return;
-    }
-    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
-  }
-#endif /* WITH_WSREP */
   if (!(error= kill_one_thread(thd, id, state, type)))
   {
     if (!thd->killed)
@@ -9090,11 +9078,6 @@ void sql_kill(THD *thd, longlong id, killed_state state, killed_type type)
   }
   else
     my_error(error, MYF(0), id);
-#ifdef WITH_WSREP
-  return;
- wsrep_error_label:
-  my_error(ER_CANNOT_USER, MYF(0), wsrep_thd_query(thd));
-#endif /* WITH_WSREP */
 }
 
 
@@ -9103,18 +9086,6 @@ void sql_kill_user(THD *thd, LEX_USER *user, killed_state state)
 {
   uint error;
   ha_rows rows;
-#ifdef WITH_WSREP
-  if (WSREP(thd))
-  {
-    WSREP_DEBUG("sql_kill_user called");
-    if (thd->wsrep_applier)
-    {
-      WSREP_DEBUG("KILL in applying, bailing out here");
-      return;
-    }
-    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
-  }
-#endif /* WITH_WSREP */
   if (!(error= kill_threads_for_user(thd, user, state, &rows)))
     my_ok(thd, rows);
   else
@@ -9125,11 +9096,6 @@ void sql_kill_user(THD *thd, LEX_USER *user, killed_state state)
     */
     my_error(error, MYF(0), user->host.str, user->user.str);
   }
-#ifdef WITH_WSREP
-  return;
- wsrep_error_label:
-  my_error(ER_CANNOT_USER, MYF(0), user->user.str);
-#endif /* WITH_WSREP */
 }
 
 
diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index 00a6dfe2f8a..f22d8bf0f5a 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -1,4 +1,4 @@
-/* Copyright 2008-2021 Codership Oy <http://www.codership.com>
+/* Copyright 2008-2015 Codership Oy <http://www.codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -835,25 +835,13 @@ void wsrep_thr_init()
   DBUG_VOID_RETURN;
 }
 
-/* This is wrapper for wsrep_break_lock in thr_lock.c */
-static int wsrep_thr_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
-{
-  THD* victim_thd= (THD *) victim_thd_ptr;
-  /* We need to lock THD::LOCK_thd_data to protect victim
-  from concurrent usage or disconnect or delete. */
-  mysql_mutex_lock(&victim_thd->LOCK_thd_data);
-  int res= wsrep_abort_thd(bf_thd_ptr, victim_thd_ptr, signal);
-  mysql_mutex_unlock(&victim_thd->LOCK_thd_data);
-  return res;
-}
-
 void wsrep_init_startup (bool first)
 {
   if (wsrep_init()) unireg_abort(1);
 
   wsrep_thr_lock_init(
      (wsrep_thd_is_brute_force_fun)wsrep_thd_is_BF,
-     (wsrep_abort_thd_fun)wsrep_thr_abort_thd,
+     (wsrep_abort_thd_fun)wsrep_abort_thd,
      wsrep_debug, wsrep_convert_LOCK_to_trx,
      (wsrep_on_fun)wsrep_on);
 
@@ -1706,11 +1694,6 @@ static int wsrep_TOI_begin(THD *thd, char *db_, char *table_,
   case SQLCOM_DROP_TABLE:
     buf_err= wsrep_drop_table_query(thd, &buf, &buf_len);
     break;
-  case SQLCOM_KILL:
-    WSREP_DEBUG("KILL as TOI: %s", thd->query());
-    buf_err= wsrep_to_buf_helper(thd, thd->query(), thd->query_length(),
-                                 &buf, &buf_len);
-    break;
   case SQLCOM_CREATE_ROLE:
     if (sp_process_definer(thd))
     {
@@ -2075,11 +2058,8 @@ bool wsrep_grant_mdl_exception(MDL_context *requestor_ctx,
         ticket->wsrep_report(true);
       }
 
-      /* This will call wsrep_abort_transaction so we should hold
-      THD::LOCK_thd_data to protect victim from concurrent usage
-      or disconnect or delete. */
-      wsrep_abort_thd((void *) request_thd, (void *) granted_thd, 1);
       mysql_mutex_unlock(&granted_thd->LOCK_thd_data);
+      wsrep_abort_thd((void *) request_thd, (void *) granted_thd, 1);
       ret= false;
     }
   }
@@ -2261,7 +2241,6 @@ error:
 static bool abort_replicated(THD *thd)
 {
   bool ret_code= false;
-  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
   if (thd->wsrep_query_state== QUERY_COMMITTING)
   {
     WSREP_DEBUG("aborting replicated trx: %llu", (ulonglong)(thd->real_id));
@@ -2276,7 +2255,6 @@ static bool abort_replicated(THD *thd)
 /**/
 static inline bool is_client_connection(THD *thd)
 {
-  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
   return (thd->wsrep_client_thread && thd->variables.wsrep_on);
 }
 
@@ -2285,8 +2263,9 @@ static inline bool is_replaying_connection(THD *thd)
 {
   bool ret;
 
-  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
+  mysql_mutex_lock(&thd->LOCK_thd_data);
   ret=  (thd->wsrep_conflict_state == REPLAYING) ? true : false;
+  mysql_mutex_unlock(&thd->LOCK_thd_data);
 
   return ret;
 }
@@ -2296,8 +2275,9 @@ static inline bool is_committing_connection(THD *thd)
 {
   bool ret;
 
-  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
+  mysql_mutex_lock(&thd->LOCK_thd_data);
   ret=  (thd->wsrep_query_state == QUERY_COMMITTING) ? true : false;
+  mysql_mutex_unlock(&thd->LOCK_thd_data);
 
   return ret;
 }
@@ -2310,17 +2290,13 @@ static bool have_client_connections()
   I_List_iterator<THD> it(threads);
   while ((tmp=it++))
   {
-    /* Protect thread from concurrent usage or disconnect or delete. */
-    mysql_mutex_lock(&tmp->LOCK_thd_data);
     DBUG_PRINT("quit",("Informing thread %lld that it's time to die",
                        (longlong) tmp->thread_id));
     if (is_client_connection(tmp) && tmp->killed == KILL_CONNECTION)
     {
       (void)abort_replicated(tmp);
-      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       return true;
     }
-    mysql_mutex_unlock(&tmp->LOCK_thd_data);
   }
   return false;
 }
@@ -2352,21 +2328,14 @@ static my_bool have_committing_connections()
   I_List_iterator<THD> it(threads);
   while ((tmp=it++))
   {
-    /* Protect from concurrent usage or disconnect or delete */
-    mysql_mutex_lock(&tmp->LOCK_thd_data);
     if (!is_client_connection(tmp))
-    {
-      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
-    }
 
     if (is_committing_connection(tmp))
     {
       mysql_mutex_unlock(&LOCK_thread_count);
-      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       return TRUE;
     }
-    mysql_mutex_unlock(&tmp->LOCK_thd_data);
   }
   mysql_mutex_unlock(&LOCK_thread_count);
   return FALSE;
@@ -2409,44 +2378,33 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD *except_caller_thd)
   {
     DBUG_PRINT("quit",("Informing thread %lld that it's time to die",
                        (longlong) tmp->thread_id));
-    /* Protect from concurrent usage or disconnect or delete */
-    mysql_mutex_lock(&tmp->LOCK_thd_data);
     /* We skip slave threads & scheduler on this first loop through. */
     if (!is_client_connection(tmp))
-    {
-      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
-    }
 
     if (tmp == except_caller_thd)
     {
       DBUG_ASSERT(is_client_connection(tmp));
-      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
     }
 
     if (is_replaying_connection(tmp))
     {
       tmp->set_killed(KILL_CONNECTION);
-      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
     }
 
-    /* replicated transactions must be skipped and aborted
-    with wsrep_abort_thd. */
+    /* replicated transactions must be skipped */
     if (abort_replicated(tmp))
-    {
-      mysql_mutex_unlock(&tmp->LOCK_thd_data);
       continue;
-    }
 
     WSREP_DEBUG("closing connection %lld", (longlong) tmp->thread_id);
 
     /*
-      instead of wsrep_close_thread() we do now  soft kill by
-      THD::awake(). Here also victim needs to be protected from
-      concurrent usage or disconnect or delete.
-    */
+      instead of wsrep_close_thread() we do now  soft kill by THD::awake
+     */
+    mysql_mutex_lock(&tmp->LOCK_thd_data);
+
     tmp->awake(KILL_CONNECTION);
 
     mysql_mutex_unlock(&tmp->LOCK_thd_data);
@@ -2465,19 +2423,16 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD *except_caller_thd)
   I_List_iterator<THD> it2(threads);
   while ((tmp=it2++))
   {
-    /* Protect from concurrent usage or disconnect or delete */
-    mysql_mutex_lock(&tmp->LOCK_thd_data);
-    if (is_client_connection(tmp))
+#ifndef __bsdi__				// Bug in BSDI kernel
+    if (is_client_connection(tmp) &&
+        !abort_replicated(tmp)    &&
+        !is_replaying_connection(tmp) &&
+        tmp != except_caller_thd)
     {
-      if (!abort_replicated(tmp) &&
-          !is_replaying_connection(tmp) &&
-          tmp != except_caller_thd)
-      {
-        WSREP_INFO("killing local connection: %lld", (longlong) tmp->thread_id);
-        close_connection(tmp,0, true);
-      }
+      WSREP_INFO("killing local connection: %lld", (longlong) tmp->thread_id);
+      close_connection(tmp,0);
     }
-    mysql_mutex_unlock(&tmp->LOCK_thd_data);
+#endif
   }
 
   DBUG_PRINT("quit",("Waiting for threads to die (count=%u)",thread_count));
@@ -2666,9 +2621,7 @@ extern "C" void wsrep_thd_set_query_state(
 
 void wsrep_thd_set_conflict_state(THD *thd, enum wsrep_conflict_state state)
 {
-  DBUG_ASSERT(thd);
-  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
-  thd->wsrep_conflict_state= state;
+  if (WSREP(thd)) thd->wsrep_conflict_state= state;
 }
 
 
@@ -2809,9 +2762,6 @@ extern "C" void wsrep_thd_awake(THD *thd, my_bool signal)
 {
   if (signal)
   {
-    /* Here we should hold THD::LOCK_thd_data to
-    protect from concurrent usage. */
-    mysql_mutex_assert_owner(&thd->LOCK_thd_data);
     thd->awake(KILL_QUERY);
   }
   else
diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc
index d849cd2cd78..ef8c0e132f7 100644
--- a/sql/wsrep_thd.cc
+++ b/sql/wsrep_thd.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2013-2021 Codership Oy <info@codership.com>
+/* Copyright (C) 2013 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -804,12 +804,10 @@ my_bool wsrep_thd_is_local(void *thd_ptr, my_bool sync)
 
 int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
 {
-  THD *victim_thd= (THD *) victim_thd_ptr;
-  THD *bf_thd= (THD *) bf_thd_ptr;
+  THD *victim_thd = (THD *) victim_thd_ptr;
+  THD *bf_thd     = (THD *) bf_thd_ptr;
   DBUG_ENTER("wsrep_abort_thd");
 
-  mysql_mutex_assert_owner(&victim_thd->LOCK_thd_data);
-
   if ( (WSREP(bf_thd) ||
          ( (WSREP_ON || bf_thd->variables.wsrep_OSU_method == WSREP_OSU_RSU) &&
            bf_thd->wsrep_exec_mode == TOTAL_ORDER) )                         &&
diff --git a/sql/wsrep_thd.h b/sql/wsrep_thd.h
index 1284cad1acf..46bc08a466a 100644
--- a/sql/wsrep_thd.h
+++ b/sql/wsrep_thd.h
@@ -32,6 +32,7 @@ void wsrep_create_rollbacker();
 
 int  wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr,
                                 my_bool signal);
+
 /*
   PA = Parallel Applying (on the slave side)
 */
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 921b8282e45..a42e8e3699c 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -60,6 +60,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
 
 #include <my_service_manager.h>
 #include <key.h>
+#include <sql_manager.h>
 
 /* Include necessary InnoDB headers */
 #include "btr0btr.h"
@@ -5233,21 +5234,17 @@ UNIV_INTERN void lock_cancel_waiting_and_release(lock_t* lock);
 @sa THD::awake() @sa ha_kill_query() */
 static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels)
 {
-  DBUG_ENTER("innobase_kill_query");
+	DBUG_ENTER("innobase_kill_query");
 #ifdef WITH_WSREP
-  if (wsrep_thd_get_conflict_state(thd) != NO_CONFLICT)
-  {
-    /* if victim has been signaled by BF thread and/or aborting
-    is already progressing, following query aborting is not necessary
-    any more. E.g. wsrep_innobase_kill_one_trx().
-    Also, BF thread should own trx mutex for the victim, which would
-    conflict with trx_mutex_enter() below
-    */
-    WSREP_DEBUG("Victim thread %ld bail out conflict_state %s query %s",
-		thd_get_thread_id(thd),
-		wsrep_thd_conflict_state_str(thd), wsrep_thd_query(thd));
-    DBUG_VOID_RETURN;
-  }
+	if (wsrep_thd_get_conflict_state(thd) != NO_CONFLICT) {
+		/* if victim has been signaled by BF thread and/or aborting
+		   is already progressing, following query aborting is not necessary
+		   any more.
+		   Also, BF thread should own trx mutex for the victim, which would
+		   conflict with trx_mutex_enter() below
+		*/
+		DBUG_VOID_RETURN;
+	}
 #endif /* WITH_WSREP */
 
   if (trx_t* trx= thd_to_trx(thd))
@@ -19502,9 +19499,9 @@ static struct st_mysql_storage_engine innobase_storage_engine=
 { MYSQL_HANDLERTON_INTERFACE_VERSION };
 
 #ifdef WITH_WSREP
-static
 void
 wsrep_abort_slave_trx(
+/*==================*/
 	wsrep_seqno_t bf_seqno,
 	wsrep_seqno_t victim_seqno)
 {
@@ -19514,97 +19511,93 @@ wsrep_abort_slave_trx(
 		"2) a bug in the code.\n\t"
 		"3) a database corruption.\n Node consistency compromized, "
 		"need to abort. Restart the node to resync with cluster.",
-		bf_seqno, victim_seqno);
+		(long long)bf_seqno, (long long)victim_seqno);
 	abort();
 }
-/*******************************************************************//**
-This function is used to kill one transaction in BF. */
-void
-wsrep_innobase_kill_one_trx(
-	MYSQL_THD const bf_thd,
-	const trx_t * const bf_trx,
-	trx_t *victim_trx,
-	my_bool signal)
+
+struct bg_wsrep_kill_trx_arg {
+	my_thread_id	thd_id;
+	trx_id_t	trx_id;
+	int64_t		bf_seqno;
+	ibool		signal;
+};
+
+static void bg_wsrep_kill_trx(
+	void *void_arg)
 {
-        ut_ad(bf_thd);
-        ut_ad(victim_trx);
-        ut_ad(lock_mutex_own());
-        ut_ad(trx_mutex_own(victim_trx));
+	bg_wsrep_kill_trx_arg *arg = (bg_wsrep_kill_trx_arg*)void_arg;
+	THD *thd 		   = find_thread_by_id(arg->thd_id, false);
+	trx_t *victim_trx 	   = NULL;
+	bool awake 		   = false;
+	DBUG_ENTER("bg_wsrep_kill_trx");
 
-	DBUG_ENTER("wsrep_innobase_kill_one_trx");
-	THD *thd= (THD *) victim_trx->mysql_thd;
-	int64_t bf_seqno= wsrep_thd_trx_seqno(bf_thd);
-
-	if (!thd) {
-		WSREP_WARN("no THD for trx: " TRX_ID_FMT, victim_trx->id);
-		DBUG_VOID_RETURN;
+	if (thd) {
+		victim_trx= thd_to_trx(thd);
+		/* Victim trx might not exist e.g. on MDL-conflict. */
+		if (victim_trx) {
+			lock_mutex_enter();
+			trx_mutex_enter(victim_trx);
+			if (victim_trx->id != arg->trx_id ||
+			    victim_trx->state == TRX_STATE_COMMITTED_IN_MEMORY)
+			{
+				/* Victim was meanwhile rolled back or
+				committed */
+				trx_mutex_exit(victim_trx);
+				lock_mutex_exit();
+				wsrep_thd_UNLOCK(thd);
+				victim_trx= NULL;
+			}
+		} else {
+			/* find_thread_by_id locked
+			THD::LOCK_thd_data */
+			wsrep_thd_UNLOCK(thd);
+		}
 	}
 
-	/* Here we need to lock THD::LOCK_thd_data to protect from
-	concurrent usage or disconnect or delete. */
-	DEBUG_SYNC(bf_thd, "wsrep_before_BF_victim_lock");
-	wsrep_thd_LOCK(thd);
-	DEBUG_SYNC(bf_thd, "wsrep_after_BF_victim_lock");
+	if (!victim_trx) {
+		/* Victim trx might not exist (MDL-conflict) or victim
+		was meanwhile rolled back or committed because of
+		a KILL statement or a disconnect. */
+		goto ret;
+	}
 
-	WSREP_DEBUG("Aborter %s trx_id: " TRX_ID_FMT " thread: %ld "
-		    "seqno: %lld query_state: %s conflict_state: %s query: %s",
-		    wsrep_thd_is_BF(bf_thd, false) ? "BF" : "normal",
-		    bf_trx ? bf_trx->id : TRX_ID_MAX,
-		    thd_get_thread_id(bf_thd),
-		    bf_seqno,
-		    wsrep_thd_query_state_str(bf_thd),
-		    wsrep_thd_conflict_state_str(bf_thd),
-		    wsrep_thd_query(bf_thd));
-
-	WSREP_DEBUG("Victim %s trx_id: " TRX_ID_FMT " thread: %ld "
-		    "seqno: %lld query_state: %s  conflict_state: %s query: %s",
-		    wsrep_thd_is_BF(thd, false) ? "BF" : "normal",
-		    victim_trx->id,
+	WSREP_DEBUG("BF kill (" ULINTPF ", seqno: " INT64PF
+		    "), victim: (%lu) trx: " TRX_ID_FMT,
+		    arg->signal, arg->bf_seqno,
 		    thd_get_thread_id(thd),
-		    wsrep_thd_trx_seqno(thd),
-		    wsrep_thd_query_state_str(thd),
-		    wsrep_thd_conflict_state_str(thd),
-		    wsrep_thd_query(thd));
+		    victim_trx->id);
 
-	WSREP_LOG_CONFLICT(bf_thd, thd, TRUE);
+	WSREP_DEBUG("Aborting query: %s conf %d trx: %" PRId64,
+		    (wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void",
+		    wsrep_thd_conflict_state(thd, FALSE),
+		    wsrep_thd_ws_handle(thd)->trx_id);
 
 	if (wsrep_thd_query_state(thd) == QUERY_EXITING) {
 		WSREP_DEBUG("kill trx EXITING for " TRX_ID_FMT,
 			    victim_trx->id);
-		wsrep_thd_UNLOCK(thd);
-		DBUG_VOID_RETURN;
+		goto ret_unlock;
 	}
 
 	if (wsrep_thd_exec_mode(thd) != LOCAL_STATE) {
-		WSREP_DEBUG("withdraw for BF trx: " TRX_ID_FMT
-			    ", state: %s exec %s",
+		WSREP_DEBUG("withdraw for BF trx: " TRX_ID_FMT ", state: %d",
 			    victim_trx->id,
-			    wsrep_thd_conflict_state_str(thd),
-			    wsrep_thd_exec_mode_str(thd));
+		wsrep_thd_get_conflict_state(thd));
 	}
 
 	switch (wsrep_thd_get_conflict_state(thd)) {
 	case NO_CONFLICT:
-		/* This will cause any call to innobase_kill_query()
-		for this thd to bail out. */
 		wsrep_thd_set_conflict_state(thd, MUST_ABORT);
 		break;
         case MUST_ABORT:
 		WSREP_DEBUG("victim " TRX_ID_FMT " in MUST ABORT state",
 			    victim_trx->id);
-		wsrep_thd_awake(thd, signal);
-		wsrep_thd_UNLOCK(thd);
-		DBUG_VOID_RETURN;
-		break;
+		goto ret_awake;
 	case ABORTED:
 	case ABORTING: // fall through
 	default:
-		WSREP_DEBUG("victim " TRX_ID_FMT " in state %s",
-			    victim_trx->id,
-			    wsrep_thd_conflict_state_str(thd));
-		wsrep_thd_UNLOCK(thd);
-		DBUG_VOID_RETURN;
-		break;
+		WSREP_DEBUG("victim " TRX_ID_FMT " in state %d",
+			    victim_trx->id, wsrep_thd_get_conflict_state(thd));
+		goto ret_unlock;
 	}
 
 	switch (wsrep_thd_query_state(thd)) {
@@ -19617,12 +19610,12 @@ wsrep_innobase_kill_one_trx(
 			    victim_trx->id);
 
 		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
-			wsrep_abort_slave_trx(bf_seqno,
+			wsrep_abort_slave_trx(arg->bf_seqno,
 					      wsrep_thd_trx_seqno(thd));
 		} else {
 			wsrep_t *wsrep= get_wsrep();
 			rcode = wsrep->abort_pre_commit(
-				wsrep, bf_seqno,
+				wsrep, arg->bf_seqno,
 				(wsrep_trx_id_t)wsrep_thd_ws_handle(thd)->trx_id
 			);
 
@@ -19631,10 +19624,7 @@ wsrep_innobase_kill_one_trx(
 				WSREP_DEBUG("cancel commit warning: "
 					    TRX_ID_FMT,
 					    victim_trx->id);
-				wsrep_thd_awake(thd, signal);
-				wsrep_thd_UNLOCK(thd);
-				DBUG_VOID_RETURN;
-				break;
+				goto ret_awake;
 			case WSREP_OK:
 				break;
 			default:
@@ -19647,12 +19637,9 @@ wsrep_innobase_kill_one_trx(
 				 * kill the lock holder first.
 				 */
 				abort();
-				break;
 			}
 		}
-		wsrep_thd_awake(thd, signal);
-		wsrep_thd_UNLOCK(thd);
-		break;
+		goto ret_awake;
 	case QUERY_EXEC:
 		/* it is possible that victim trx is itself waiting for some
 		 * other lock. We need to cancel this waiting
@@ -19673,65 +19660,118 @@ wsrep_innobase_kill_one_trx(
 				lock_cancel_waiting_and_release(wait_lock);
 			}
 
-			wsrep_thd_awake(thd, signal);
-			wsrep_thd_UNLOCK(thd);
 		} else {
 			/* abort currently executing query */
+			DBUG_PRINT("wsrep",("sending KILL_QUERY to: %lu",
+                                            thd_get_thread_id(thd)));
 			WSREP_DEBUG("kill query for: %ld",
 				thd_get_thread_id(thd));
-			wsrep_thd_awake(thd, signal);
-			wsrep_thd_UNLOCK(thd);
 
 			/* for BF thd, we need to prevent him from committing */
 			if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
-				wsrep_abort_slave_trx(bf_seqno,
-						      wsrep_thd_trx_seqno(thd));
+				wsrep_abort_slave_trx(arg->bf_seqno,
+						    wsrep_thd_trx_seqno(thd));
 			}
 		}
-		break;
+		goto ret_awake;
 	case QUERY_IDLE:
 	{
 		WSREP_DEBUG("kill IDLE for " TRX_ID_FMT, victim_trx->id);
 
 		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
 			WSREP_DEBUG("kill BF IDLE, seqno: %lld",
-				    wsrep_thd_trx_seqno(thd));
-			wsrep_thd_UNLOCK(thd);
-			wsrep_abort_slave_trx(bf_seqno,
+				    (long long)wsrep_thd_trx_seqno(thd));
+			wsrep_abort_slave_trx(arg->bf_seqno,
 					      wsrep_thd_trx_seqno(thd));
-			DBUG_VOID_RETURN;
+			goto ret_unlock;
 		}
-		/* This will lock thd from proceeding after net_read()
-		and innobase_kill_query to bail out for this thd. */
+                /* This will lock thd from proceeding after net_read() */
 		wsrep_thd_set_conflict_state(thd, ABORTING);
 
 		wsrep_lock_rollback();
 
 		if (wsrep_aborting_thd_contains(thd)) {
 			WSREP_WARN("duplicate thd aborter %lu",
-			           thd_get_thread_id(thd));
+			           (ulong) thd_get_thread_id(thd));
 		} else {
 			wsrep_aborting_thd_enqueue(thd);
+			DBUG_PRINT("wsrep",("enqueuing trx abort for %lu",
+			                    thd_get_thread_id(thd)));
 			WSREP_DEBUG("enqueuing trx abort for (%lu)",
-				    thd_get_thread_id(thd));
+			            thd_get_thread_id(thd));
 		}
 
+		DBUG_PRINT("wsrep",("signalling wsrep rollbacker"));
 		WSREP_DEBUG("signaling aborter");
 		wsrep_unlock_rollback();
-		wsrep_thd_UNLOCK(thd);
-
-		break;
+		goto ret_unlock;
 	}
 	default:
-		ut_error;
+		WSREP_WARN("bad wsrep query state: %d",
+			  wsrep_thd_query_state(thd));
+		goto ret_unlock;
 	}
 
+ret_awake:
+	awake= true;
+
+ret_unlock:
+	trx_mutex_exit(victim_trx);
+	lock_mutex_exit();
+	if (awake)
+		wsrep_thd_awake(thd, arg->signal);
+	wsrep_thd_UNLOCK(thd);
+
+ret:
+	free(arg);
+	DBUG_VOID_RETURN;
+
+}
+
+/*******************************************************************//**
+This function is used to kill one transaction in BF. */
+UNIV_INTERN
+void
+wsrep_innobase_kill_one_trx(
+/*========================*/
+	MYSQL_THD const bf_thd,
+	const trx_t * const bf_trx,
+	trx_t *victim_trx,
+	ibool signal)
+{
+	ut_ad(bf_thd);
+	ut_ad(victim_trx);
+	ut_ad(lock_mutex_own());
+	ut_ad(trx_mutex_own(victim_trx));
+
+	bg_wsrep_kill_trx_arg *arg = (bg_wsrep_kill_trx_arg*)malloc(sizeof(*arg));
+	arg->thd_id	= thd_get_thread_id(victim_trx->mysql_thd);
+	arg->trx_id	= victim_trx->id;
+	arg->bf_seqno	= wsrep_thd_trx_seqno((THD*)bf_thd);
+	arg->signal	= signal;
+
+	DBUG_ENTER("wsrep_innobase_kill_one_trx");
+
+	WSREP_LOG_CONFLICT(bf_thd, victim_trx->mysql_thd, TRUE);
+
+	DBUG_EXECUTE_IF("sync.wsrep_after_BF_victim_lock",
+		{
+		  const char act[]=
+		    "now "
+		    "wait_for signal.wsrep_after_BF_victim_lock";
+		  DBUG_ASSERT(!debug_sync_set_action(bf_thd,
+						     STRING_WITH_LEN(act)));
+		};);
+
+
+	mysql_manager_submit(bg_wsrep_kill_trx, arg);
 	DBUG_VOID_RETURN;
 }
 
 static
 void
 wsrep_abort_transaction(
+/*====================*/
 	handlerton* hton,
 	THD *bf_thd,
 	THD *victim_thd,
@@ -19739,66 +19779,28 @@ wsrep_abort_transaction(
 {
 	DBUG_ENTER("wsrep_abort_transaction");
 
-	ut_ad(bf_thd);
-	ut_ad(victim_thd);
-	trx_t* victim_trx= thd_to_trx(victim_thd);
-	trx_t* bf_trx= thd_to_trx(bf_thd);
+	trx_t* victim_trx	= thd_to_trx(victim_thd);
+	trx_t* bf_trx		= (bf_thd) ? thd_to_trx(bf_thd) : NULL;
 
-	/* Here we should hold THD::LOCK_thd_data to protect
-	victim from concurrent usage or disconnect or delete. */
-	WSREP_DEBUG("wsrep_abort_transaction: BF:"
-		    " thread %ld query_state %s conflict_state %s"
-		    " exec %s query %s trx " TRX_ID_FMT,
-		    thd_get_thread_id(bf_thd),
-		    wsrep_thd_query_state_str(bf_thd),
-		    wsrep_thd_conflict_state_str(bf_thd),
-		    wsrep_thd_exec_mode_str(bf_thd),
-		    wsrep_thd_query(bf_thd),
-		    bf_trx ? bf_trx->id : 0);
-
-	WSREP_DEBUG("wsrep_abort_transaction: victim:"
-		    " thread %ld query_state %s conflict_state %s"
-		    " exec %s query %s trx " TRX_ID_FMT,
-		    thd_get_thread_id(victim_thd),
-		    wsrep_thd_query_state_str(victim_thd),
-		    wsrep_thd_conflict_state_str(victim_thd),
-		    wsrep_thd_exec_mode_str(victim_thd),
-		    wsrep_thd_query(victim_thd),
-		    victim_trx ? victim_trx->id : 0);
+	WSREP_DEBUG("abort transaction: BF: %s victim: %s victim conf: %d",
+			wsrep_thd_query(bf_thd),
+			wsrep_thd_query(victim_thd),
+			wsrep_thd_conflict_state(victim_thd, FALSE));
 
 	if (victim_trx) {
-		WSREP_DEBUG("wsrep_abort_transaction: Victim thread %ld "
-			    "transaction " TRX_ID_FMT " trx_state %d",
-			    thd_get_thread_id(victim_thd),
-			    victim_trx->id,
-			    victim_trx->state);
-		/* This is necessary as correct mutexing order is
-		lock_sys -> trx -> THD::LOCK_thd_data and below
-		function assumes we have lock_sys and trx locked
-		and takes THD::LOCK_thd_data for THD state check. */
-		wsrep_thd_UNLOCK(victim_thd);
-		DEBUG_SYNC(bf_thd, "wsrep_abort_victim_unlocked");
-		DBUG_EXECUTE_IF("wsrep_abort_replicated_sleep",
-				WSREP_DEBUG("wsrep_abort_transaction: sleeping "
-					    "for thread %ld ",
-					    thd_get_thread_id(victim_thd));
-				my_sleep(100000););
 		lock_mutex_enter();
 		trx_mutex_enter(victim_trx);
 		wsrep_innobase_kill_one_trx(bf_thd, bf_trx, victim_trx, signal);
 		lock_mutex_exit();
 		trx_mutex_exit(victim_trx);
 		wsrep_srv_conc_cancel_wait(victim_trx);
-		wsrep_thd_LOCK(victim_thd);
 		DBUG_VOID_RETURN;
 	} else {
-		WSREP_DEBUG("wsrep_abort_transaction: Victim thread %ld "
-			    "no transaction",
-			    thd_get_thread_id(victim_thd));
-		/* This will cause any call to innobase_kill_query()
-		for this thd to bail out. */
+		WSREP_DEBUG("victim does not have transaction");
+		wsrep_thd_LOCK(victim_thd);
 		wsrep_thd_set_conflict_state(victim_thd, MUST_ABORT);
 		wsrep_thd_awake(victim_thd, signal);
+		wsrep_thd_UNLOCK(victim_thd);
 	}
 
 	DBUG_VOID_RETURN;
diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h
index 427e57f09d2..3eab2135969 100644
--- a/storage/innobase/include/ha_prototypes.h
+++ b/storage/innobase/include/ha_prototypes.h
@@ -233,11 +233,12 @@ innobase_casedn_str(
 	char*	a);	/*!< in/out: string to put in lower case */
 
 #ifdef WITH_WSREP
+UNIV_INTERN
 void
 wsrep_innobase_kill_one_trx(MYSQL_THD const thd_ptr,
                             const trx_t * const bf_trx,
                             trx_t *victim_trx,
-                            my_bool signal);
+                            ibool signal);
 int wsrep_innobase_mysql_sort(int mysql_type, uint charset_number,
                              unsigned char* str, unsigned int str_length,
                              unsigned int buf_length);

From 1a62c878970fea6d2013c432bbd5aae30dbaca89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Mon, 27 Sep 2021 08:25:22 +0300
Subject: [PATCH 12/31] Remove test from galera_fulltext until MDEV-24978 is
 fixed.

---
 .../suite/galera/r/galera_fulltext.result     | 28 ------------
 .../suite/galera/t/galera_fulltext.test       | 43 +++++++++----------
 2 files changed, 21 insertions(+), 50 deletions(-)

diff --git a/mysql-test/suite/galera/r/galera_fulltext.result b/mysql-test/suite/galera/r/galera_fulltext.result
index f52f5c996a3..18e3bff40fc 100644
--- a/mysql-test/suite/galera/r/galera_fulltext.result
+++ b/mysql-test/suite/galera/r/galera_fulltext.result
@@ -34,31 +34,3 @@ COUNT(f1) = 1000
 1
 DROP TABLE t1;
 DROP TABLE ten;
-connection node_1;
-SET @value=REPEAT (1,5001);
-CREATE TABLE t (a VARCHAR(5000),FULLTEXT (a)) engine=innodb DEFAULT CHARSET=utf8;
-INSERT IGNORE INTO t VALUES(@value);
-Warnings:
-Warning	1265	Data truncated for column 'a' at row 1
-SELECT COUNT(*) FROM t;
-COUNT(*)
-1
-connection node_2;
-SELECT COUNT(*) FROM t;
-COUNT(*)
-1
-connection node_1;
-DROP TABLE t;
-CREATE TABLE t (a VARCHAR(5000)) engine=innodb DEFAULT CHARSET=utf8;
-INSERT IGNORE INTO t VALUES(@value);
-Warnings:
-Warning	1265	Data truncated for column 'a' at row 1
-SELECT COUNT(*) FROM t;
-COUNT(*)
-1
-connection node_2;
-SELECT COUNT(*) FROM t;
-COUNT(*)
-1
-connection node_1;
-DROP TABLE t;
diff --git a/mysql-test/suite/galera/t/galera_fulltext.test b/mysql-test/suite/galera/t/galera_fulltext.test
index 76c29da4123..7e2fc5e581d 100644
--- a/mysql-test/suite/galera/t/galera_fulltext.test
+++ b/mysql-test/suite/galera/t/galera_fulltext.test
@@ -1,6 +1,4 @@
---source include/big_test.inc
 --source include/galera_cluster.inc
---source include/have_innodb.inc
 
 #
 # InnoDB FULLTEXT indexes
@@ -63,24 +61,25 @@ DROP TABLE ten;
 
 #
 # Case 2: UTF-8
+# TODO: MDEV-24978
 #
---connection node_1
-SET @value=REPEAT (1,5001);
-CREATE TABLE t (a VARCHAR(5000),FULLTEXT (a)) engine=innodb DEFAULT CHARSET=utf8;
-INSERT IGNORE INTO t VALUES(@value);
-SELECT COUNT(*) FROM t;
-
---connection node_2
-SELECT COUNT(*) FROM t;
-
---connection node_1
-DROP TABLE t;
-CREATE TABLE t (a VARCHAR(5000)) engine=innodb DEFAULT CHARSET=utf8;
-INSERT IGNORE INTO t VALUES(@value);
-SELECT COUNT(*) FROM t;
-
---connection node_2
-SELECT COUNT(*) FROM t;
-
---connection node_1
-DROP TABLE t;
+#--connection node_1
+#SET @value=REPEAT (1,5001);
+#CREATE TABLE t (a VARCHAR(5000),FULLTEXT (a)) engine=innodb DEFAULT CHARSET=utf8;
+#INSERT IGNORE INTO t VALUES(@value);
+#SELECT COUNT(*) FROM t;
+#
+#--connection node_2
+#SELECT COUNT(*) FROM t;
+#
+#--connection node_1
+#DROP TABLE t;
+#CREATE TABLE t (a VARCHAR(5000)) engine=innodb DEFAULT CHARSET=utf8;
+#INSERT IGNORE INTO t VALUES(@value);
+#SELECT COUNT(*) FROM t;
+#
+#--connection node_2
+#SELECT COUNT(*) FROM t;
+#
+#--connection node_1
+#DROP TABLE t;

From 3690c549c6e72646ba74f6b4c83813ee4ac3aea4 Mon Sep 17 00:00:00 2001
From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Fri, 23 Jul 2021 11:14:13 +0200
Subject: [PATCH 13/31] MDEV-24454 Crash at change_item_tree

Use in_sum_func (and so nest_level) only in LEX to which SELECT lex belong to

Reduce usage of current_select (because it does not always point on the correct
 SELECT_LEX, for example with prepare.

Change context for all classes inherited from Item_ident (was only for Item_field) in case of pushing down it to HAVING.

Now name resolution context have to have SELECT_LEX reference if the context is present.

Fixed feedback plugin stack usage.
---
 mysql-test/r/view.result                      |  43 ++++++++
 .../plugins/r/feedback_plugin_load.result     |  16 +++
 .../suite/plugins/t/feedback_plugin_load.test |   7 ++
 mysql-test/t/view.test                        |  49 +++++++++
 plugin/feedback/feedback.cc                   |  12 +-
 sql/item.cc                                   | 104 ++++++++++++------
 sql/item.h                                    |   1 +
 sql/item_subselect.cc                         |   3 +-
 sql/item_sum.cc                               |   8 +-
 sql/sql_base.cc                               |   1 +
 sql/sql_prepare.cc                            |   2 +-
 11 files changed, 202 insertions(+), 44 deletions(-)

diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result
index ef4f0a48534..bae415c17ea 100644
--- a/mysql-test/r/view.result
+++ b/mysql-test/r/view.result
@@ -6833,5 +6833,48 @@ sum(z)
 DROP TABLE t1;
 DROP VIEW v1;
 #
+# MDEV-24454: Crash at change_item_tree
+#
+CREATE TABLE t1(f0 INT);
+CREATE VIEW v1 AS
+SELECT
+f0 AS f1
+FROM t1;
+CREATE VIEW v2 AS
+SELECT
+(SELECT GROUP_CONCAT(v1.f1 SEPARATOR ', ')
+FROM v1 n) AS f2,
+GROUP_CONCAT('' SEPARATOR ', ') AS f3
+FROM v1;
+CREATE VIEW v3 AS
+SELECT 1 as f4 FROM v2;
+CREATE PROCEDURE p1()
+SELECT * FROM v3;
+CALL p1();
+f4
+1
+CALL p1();
+f4
+1
+drop procedure p1;
+drop view v1,v2,v3;
+drop table t1;
+#
+# MDEV-25631: Crash in st_select_lex::mark_as_dependent with
+# VIEW, aggregate and subquery
+#
+CREATE TABLE t1 (i1 int);
+insert into t1 values (1),(2),(3);
+CREATE VIEW v1 AS
+SELECT t1.i1 FROM (t1 a JOIN t1 ON (t1.i1 = (SELECT t1.i1 FROM t1 b)));
+SELECT 1 FROM (SELECT count(((SELECT i1 FROM v1))) FROM v1) dt ;
+ERROR 21000: Subquery returns more than 1 row
+delete from t1 where i1 > 1;
+SELECT 1 FROM (SELECT count(((SELECT i1 FROM v1))) FROM v1) dt ;
+1
+1
+drop view v1;
+drop table t1;
+#
 # End of 10.2 tests
 #
diff --git a/mysql-test/suite/plugins/r/feedback_plugin_load.result b/mysql-test/suite/plugins/r/feedback_plugin_load.result
index 843cd15ac94..8b2a561b9f0 100644
--- a/mysql-test/suite/plugins/r/feedback_plugin_load.result
+++ b/mysql-test/suite/plugins/r/feedback_plugin_load.result
@@ -24,3 +24,19 @@ VARIABLE_VALUE>0	VARIABLE_NAME
 1	Collation used latin1_swedish_ci
 1	Collation used utf8_bin
 1	Collation used utf8_general_ci
+prepare stmt from "SELECT VARIABLE_VALUE>0, VARIABLE_NAME FROM INFORMATION_SCHEMA.FEEDBACK WHERE VARIABLE_NAME LIKE 'Collation used %' ORDER BY VARIABLE_NAME";
+execute stmt;
+VARIABLE_VALUE>0	VARIABLE_NAME
+1	Collation used binary
+1	Collation used latin1_bin
+1	Collation used latin1_swedish_ci
+1	Collation used utf8_bin
+1	Collation used utf8_general_ci
+execute stmt;
+VARIABLE_VALUE>0	VARIABLE_NAME
+1	Collation used binary
+1	Collation used latin1_bin
+1	Collation used latin1_swedish_ci
+1	Collation used utf8_bin
+1	Collation used utf8_general_ci
+deallocate prepare stmt;
diff --git a/mysql-test/suite/plugins/t/feedback_plugin_load.test b/mysql-test/suite/plugins/t/feedback_plugin_load.test
index c0546cef0f9..9fbe523e5f3 100644
--- a/mysql-test/suite/plugins/t/feedback_plugin_load.test
+++ b/mysql-test/suite/plugins/t/feedback_plugin_load.test
@@ -42,3 +42,10 @@ if (`SELECT VERSION() LIKE '%embedded%'`)
 SELECT VARIABLE_VALUE>0, VARIABLE_NAME FROM INFORMATION_SCHEMA.FEEDBACK
 WHERE VARIABLE_NAME LIKE 'Collation used %'
 ORDER BY VARIABLE_NAME;
+
+prepare stmt from "SELECT VARIABLE_VALUE>0, VARIABLE_NAME FROM INFORMATION_SCHEMA.FEEDBACK WHERE VARIABLE_NAME LIKE 'Collation used %' ORDER BY VARIABLE_NAME";
+
+execute stmt;
+execute stmt;
+
+deallocate prepare stmt;
diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test
index 8cb00f7a6f4..128fa853e10 100644
--- a/mysql-test/t/view.test
+++ b/mysql-test/t/view.test
@@ -6559,6 +6559,55 @@ SELECT sum(z) FROM v1;
 DROP TABLE t1;
 DROP VIEW v1;
 
+--echo #
+--echo # MDEV-24454: Crash at change_item_tree
+--echo #
+
+CREATE TABLE t1(f0 INT);
+
+CREATE VIEW v1 AS
+SELECT
+  f0 AS f1
+FROM t1;
+
+CREATE VIEW v2 AS
+SELECT
+  (SELECT GROUP_CONCAT(v1.f1 SEPARATOR ', ')
+   FROM v1 n) AS f2,
+  GROUP_CONCAT('' SEPARATOR ', ') AS f3
+FROM v1;
+
+CREATE VIEW v3 AS
+SELECT 1 as f4 FROM v2;
+
+CREATE PROCEDURE p1()
+  SELECT * FROM v3;
+
+CALL p1();
+CALL p1();
+
+drop procedure p1;
+drop view v1,v2,v3;
+drop table t1;
+
+--echo #
+--echo # MDEV-25631: Crash in st_select_lex::mark_as_dependent with
+--echo # VIEW, aggregate and subquery
+--echo #
+
+CREATE TABLE t1 (i1 int);
+insert into t1 values (1),(2),(3); #not important
+CREATE VIEW v1 AS
+	SELECT t1.i1 FROM (t1 a JOIN t1 ON (t1.i1 = (SELECT t1.i1 FROM t1 b)));
+
+--error ER_SUBQUERY_NO_1_ROW
+SELECT 1 FROM (SELECT count(((SELECT i1 FROM v1))) FROM v1) dt ;
+delete from t1 where i1 > 1;
+SELECT 1 FROM (SELECT count(((SELECT i1 FROM v1))) FROM v1) dt ;
+
+drop view v1;
+drop table t1;
+
 --echo #
 --echo # End of 10.2 tests
 --echo #
diff --git a/plugin/feedback/feedback.cc b/plugin/feedback/feedback.cc
index c76828efa2b..3f6cc9dddbc 100644
--- a/plugin/feedback/feedback.cc
+++ b/plugin/feedback/feedback.cc
@@ -92,16 +92,18 @@ static COND * const OOM= (COND*)1;
 static COND* make_cond(THD *thd, TABLE_LIST *tables, LEX_STRING *filter)
 {
   Item_cond_or *res= NULL;
-  Name_resolution_context nrc;
+  /* A reference to this context will be stored in Item_field */
+  Name_resolution_context *nrc= new (thd->mem_root) Name_resolution_context;
   const char *db= tables->db, *table= tables->alias,
              *field= tables->table->field[0]->field_name;
   CHARSET_INFO *cs= &my_charset_latin1;
 
-  if (!filter->str)
+  if (!filter->str || !nrc)
     return 0;
 
-  nrc.init();
-  nrc.resolve_in_table_list_only(tables);
+  nrc->init();
+  nrc->resolve_in_table_list_only(tables);
+  nrc->select_lex= tables->select_lex;
 
   res= new (thd->mem_root) Item_cond_or(thd);
   if (!res)
@@ -109,7 +111,7 @@ static COND* make_cond(THD *thd, TABLE_LIST *tables, LEX_STRING *filter)
 
   for (; filter->str; filter++)
   {
-    Item_field  *fld= new (thd->mem_root) Item_field(thd, &nrc, db, table,
+    Item_field  *fld= new (thd->mem_root) Item_field(thd, nrc, db, table,
                                                      field);
     Item_string *pattern= new (thd->mem_root) Item_string(thd, filter->str,
                                                           (uint) filter->length, cs);
diff --git a/sql/item.cc b/sql/item.cc
index 96e118e6365..2a7c620b864 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -61,11 +61,12 @@ bool cmp_items(Item *a, Item *b)
 /**
   Set max_sum_func_level if it is needed
 */
-inline void set_max_sum_func_level(THD *thd, SELECT_LEX *select)
+inline void set_max_sum_func_level(SELECT_LEX *select)
 {
-  if (thd->lex->in_sum_func &&
-      thd->lex->in_sum_func->nest_level >= select->nest_level)
-    set_if_bigger(thd->lex->in_sum_func->max_sum_func_level,
+  LEX *lex_s= select->parent_lex;
+  if (lex_s->in_sum_func &&
+      lex_s->in_sum_func->nest_level >= select->nest_level)
+    set_if_bigger(lex_s->in_sum_func->max_sum_func_level,
                   select->nest_level - 1);
 }
 
@@ -780,6 +781,7 @@ Item_ident::Item_ident(THD *thd, Name_resolution_context *context_arg,
 {
   name = (char*) field_name_arg;
   name_length= name ? strlen(name) : 0;
+  DBUG_ASSERT(!context || context->select_lex);
 }
 
 
@@ -794,6 +796,7 @@ Item_ident::Item_ident(THD *thd, TABLE_LIST *view_arg, const char *field_name_ar
 {
   name = (char*) field_name_arg;
   name_length= name ? strlen(name) : 0;
+  DBUG_ASSERT(!context || context->select_lex);
 }
 
 
@@ -815,7 +818,9 @@ Item_ident::Item_ident(THD *thd, Item_ident *item)
    cached_table(item->cached_table),
    depended_from(item->depended_from),
    can_be_depended(item->can_be_depended)
-{}
+{
+  DBUG_ASSERT(!context || context->select_lex);
+}
 
 void Item_ident::cleanup()
 {
@@ -5117,7 +5122,14 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
   */
   Name_resolution_context *last_checked_context= context;
   Item **ref= (Item **) not_found_item;
-  SELECT_LEX *current_sel= thd->lex->current_select;
+  /*
+    There are cases when name resolution context is absent (when we are not
+    doing name resolution), but here the name resolution context should
+    be present because we are doing name resolution
+  */
+  DBUG_ASSERT(context);
+  SELECT_LEX *current_sel=  context->select_lex;
+  LEX *lex_s= context->select_lex->parent_lex;
   Name_resolution_context *outer_context= 0;
   SELECT_LEX *select= 0;
   /* Currently derived tables cannot be correlated */
@@ -5218,18 +5230,18 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
               return -1;
             thd->change_item_tree(reference, rf);
             select->inner_refs_list.push_back(rf, thd->mem_root);
-            rf->in_sum_func= thd->lex->in_sum_func;
+            rf->in_sum_func= lex_s->in_sum_func;
           }
           /*
             A reference is resolved to a nest level that's outer or the same as
             the nest level of the enclosing set function : adjust the value of
             max_arg_level for the function if it's needed.
           */
-          if (thd->lex->in_sum_func &&
-              thd->lex->in_sum_func->nest_level >= select->nest_level)
+          if (lex_s->in_sum_func &&
+              lex_s->in_sum_func->nest_level >= select->nest_level)
           {
             Item::Type ref_type= (*reference)->type();
-            set_if_bigger(thd->lex->in_sum_func->max_arg_level,
+            set_if_bigger(lex_s->in_sum_func->max_arg_level,
                           select->nest_level);
             set_field(*from_field);
             fixed= 1;
@@ -5250,10 +5262,10 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
                             ((ref_type == REF_ITEM || ref_type == FIELD_ITEM) ?
                              (Item_ident*) (*reference) :
                              0), false);
-          if (thd->lex->in_sum_func &&
-              thd->lex->in_sum_func->nest_level >= select->nest_level)
+          if (lex_s->in_sum_func &&
+              lex_s->in_sum_func->nest_level >= select->nest_level)
           {
-            set_if_bigger(thd->lex->in_sum_func->max_arg_level,
+            set_if_bigger(lex_s->in_sum_func->max_arg_level,
                           select->nest_level);
           }
           /*
@@ -5345,7 +5357,7 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
     {
       outer_context->select_lex->inner_refs_list.push_back((Item_outer_ref*)rf,
                                                            thd->mem_root);
-      ((Item_outer_ref*)rf)->in_sum_func= thd->lex->in_sum_func;
+      ((Item_outer_ref*)rf)->in_sum_func= lex_s->in_sum_func;
     }
     thd->change_item_tree(reference, rf);
     /*
@@ -5360,7 +5372,7 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
       We can not "move" aggregate function in the place where
       its arguments are not defined.
     */
-    set_max_sum_func_level(thd, select);
+    set_max_sum_func_level(select);
     mark_as_dependent(thd, last_checked_context->select_lex,
                       context->select_lex, rf,
                       rf, false);
@@ -5373,7 +5385,7 @@ Item_field::fix_outer_field(THD *thd, Field **from_field, Item **reference)
       We can not "move" aggregate function in the place where
       its arguments are not defined.
     */
-    set_max_sum_func_level(thd, select);
+    set_max_sum_func_level(select);
     mark_as_dependent(thd, last_checked_context->select_lex,
                       context->select_lex,
                       this, (Item_ident*)*reference, false);
@@ -5450,7 +5462,20 @@ bool Item_field::fix_fields(THD *thd, Item **reference)
   DBUG_ASSERT(fixed == 0);
   Field *from_field= (Field *)not_found_field;
   bool outer_fixed= false;
-  SELECT_LEX *select= thd->lex->current_select;
+  SELECT_LEX *select;
+  LEX *lex_s;
+  if (context)
+  {
+    select= context->select_lex;
+    lex_s= context->select_lex->parent_lex;
+  }
+  else
+  {
+    // No real name resolution, used somewhere in SP
+    DBUG_ASSERT(field);
+    select= NULL;
+    lex_s= NULL;
+  }
 
   if (!field)					// If field is not checked
   {
@@ -5511,7 +5536,7 @@ bool Item_field::fix_fields(THD *thd, Item **reference)
               We can not "move" aggregate function in the place where
               its arguments are not defined.
             */
-            set_max_sum_func_level(thd, select);
+            set_max_sum_func_level(select);
             set_field(new_field);
             depended_from= (*((Item_field**)res))->depended_from;
             return 0;
@@ -5540,7 +5565,7 @@ bool Item_field::fix_fields(THD *thd, Item **reference)
               We can not "move" aggregate function in the place where
               its arguments are not defined.
             */
-            set_max_sum_func_level(thd, select);
+            set_max_sum_func_level(select);
             return FALSE;
           }
         }
@@ -5577,10 +5602,11 @@ bool Item_field::fix_fields(THD *thd, Item **reference)
         goto mark_non_agg_field;
     }
 
-    if (thd->lex->in_sum_func &&
-        thd->lex->in_sum_func->nest_level == 
+    if (lex_s &&
+        lex_s->in_sum_func &&
+        lex_s->in_sum_func->nest_level ==
         select->nest_level)
-      set_if_bigger(thd->lex->in_sum_func->max_arg_level,
+      set_if_bigger(lex_s->in_sum_func->max_arg_level,
                     select->nest_level);
     /*
       if it is not expression from merged VIEW we will set this field.
@@ -5646,8 +5672,9 @@ bool Item_field::fix_fields(THD *thd, Item **reference)
   if (field->vcol_info)
     fix_session_vcol_expr_for_read(thd, field, field->vcol_info);
   if (thd->variables.sql_mode & MODE_ONLY_FULL_GROUP_BY &&
-      !outer_fixed && !thd->lex->in_sum_func &&
+      !outer_fixed &&
       select &&
+      !lex_s->in_sum_func &&
       select->cur_pos_in_select_list != UNDEF_POS &&
       select->join)
   {
@@ -5682,13 +5709,13 @@ mark_non_agg_field:
       */
       select_lex= context->select_lex;
     }
-    if (!thd->lex->in_sum_func)
+    if (!lex_s || !lex_s->in_sum_func)
       select_lex->set_non_agg_field_used(true);
     else
     {
       if (outer_fixed)
-        thd->lex->in_sum_func->outer_fields.push_back(this, thd->mem_root);
-      else if (thd->lex->in_sum_func->nest_level !=
+        lex_s->in_sum_func->outer_fields.push_back(this, thd->mem_root);
+      else if (lex_s->in_sum_func->nest_level !=
           select->nest_level)
         select_lex->set_non_agg_field_used(true);
     }
@@ -7181,6 +7208,12 @@ Item *get_field_item_for_having(THD *thd, Item *item, st_select_lex *sel)
   return NULL; 
 }
 
+Item *Item_ident::derived_field_transformer_for_having(THD *thd, uchar *arg)
+{
+  st_select_lex *sel= (st_select_lex *)arg;
+  context= &sel->context;
+  return this;
+}
 
 Item *Item_field::derived_field_transformer_for_having(THD *thd, uchar *arg)
 {
@@ -7200,12 +7233,13 @@ Item *Item_field::derived_field_transformer_for_having(THD *thd, uchar *arg)
 Item *Item_direct_view_ref::derived_field_transformer_for_having(THD *thd,
                                                                  uchar *arg)
 {
+  st_select_lex *sel= (st_select_lex *)arg;
+  context= &sel->context;
   if ((*ref)->marker & SUBSTITUTION_FL)
   {
     this->marker|= SUBSTITUTION_FL;
     return this;
   }
-  st_select_lex *sel= (st_select_lex *)arg;
   table_map tab_map= sel->master_unit()->derived->table->map;
   if ((item_equal && !(item_equal->used_tables() & tab_map)) ||
       !item_equal)
@@ -7501,7 +7535,9 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
 {
   enum_parsing_place place= NO_MATTER;
   DBUG_ASSERT(fixed == 0);
-  SELECT_LEX *current_sel= thd->lex->current_select;
+
+  SELECT_LEX *current_sel= context->select_lex;
+  LEX *lex_s= context->select_lex->parent_lex;
 
   if (set_properties_only)
   {
@@ -7662,10 +7698,10 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
           the nest level of the enclosing set function : adjust the value of
           max_arg_level for the function if it's needed.
         */
-        if (thd->lex->in_sum_func &&
-            thd->lex->in_sum_func->nest_level >= 
+        if (lex_s->in_sum_func &&
+            lex_s->in_sum_func->nest_level >=
             last_checked_context->select_lex->nest_level)
-          set_if_bigger(thd->lex->in_sum_func->max_arg_level,
+          set_if_bigger(lex_s->in_sum_func->max_arg_level,
                         last_checked_context->select_lex->nest_level);
         return FALSE;
       }
@@ -7685,10 +7721,10 @@ bool Item_ref::fix_fields(THD *thd, Item **reference)
         the nest level of the enclosing set function : adjust the value of
         max_arg_level for the function if it's needed.
       */
-      if (thd->lex->in_sum_func &&
-          thd->lex->in_sum_func->nest_level >= 
+      if (lex_s->in_sum_func &&
+          lex_s->in_sum_func->nest_level >=
           last_checked_context->select_lex->nest_level)
-        set_if_bigger(thd->lex->in_sum_func->max_arg_level,
+        set_if_bigger(lex_s->in_sum_func->max_arg_level,
                       last_checked_context->select_lex->nest_level);
     }
   }
diff --git a/sql/item.h b/sql/item.h
index 8be9a7fe417..35700129243 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -2630,6 +2630,7 @@ public:
     Collect outer references
   */
   virtual bool collect_outer_ref_processor(void *arg);
+  Item *derived_field_transformer_for_having(THD *thd, uchar *arg);
   friend bool insert_fields(THD *thd, Name_resolution_context *context,
                             const char *db_name,
                             const char *table_name, List_iterator<Item> *it,
diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc
index 25621dfe104..3f9a760ce09 100644
--- a/sql/item_subselect.cc
+++ b/sql/item_subselect.cc
@@ -5190,8 +5190,9 @@ bool subselect_hash_sj_engine::make_semi_join_conds()
                                 NULL, TL_READ);
   tmp_table_ref->table= tmp_table;
 
-  context= new Name_resolution_context;
+  context= new (thd->mem_root) Name_resolution_context;
   context->init();
+  context->select_lex= item_in->unit->first_select();
   context->first_name_resolution_table=
     context->last_name_resolution_table= tmp_table_ref;
   semi_join_conds_context= context;
diff --git a/sql/item_sum.cc b/sql/item_sum.cc
index dd65f04a652..f20be3b5226 100644
--- a/sql/item_sum.cc
+++ b/sql/item_sum.cc
@@ -68,6 +68,7 @@ size_t Item_sum::ram_limitation(THD *thd)
 bool Item_sum::init_sum_func_check(THD *thd)
 {
   SELECT_LEX *curr_sel= thd->lex->current_select;
+  LEX *lex_s= (curr_sel ? curr_sel->parent_lex : thd->lex);
   if (curr_sel && !curr_sel->name_visibility_map)
   {
     for (SELECT_LEX *sl= curr_sel; sl; sl= sl->context.outer_select())
@@ -82,9 +83,9 @@ bool Item_sum::init_sum_func_check(THD *thd)
     return TRUE;
   }
   /* Set a reference to the nesting set function if there is  any */
-  in_sum_func= thd->lex->in_sum_func;
+  in_sum_func= lex_s->in_sum_func;
   /* Save a pointer to object to be used in items for nested set functions */
-  thd->lex->in_sum_func= this;
+  lex_s->in_sum_func= this;
   nest_level= thd->lex->current_select->nest_level;
   ref_by= 0;
   aggr_level= -1;
@@ -151,6 +152,7 @@ bool Item_sum::init_sum_func_check(THD *thd)
 bool Item_sum::check_sum_func(THD *thd, Item **ref)
 {
   SELECT_LEX *curr_sel= thd->lex->current_select;
+  LEX *lex_s= curr_sel->parent_lex;
   nesting_map allow_sum_func= (thd->lex->allow_sum_func &
                                curr_sel->name_visibility_map);
   bool invalid= FALSE;
@@ -310,7 +312,7 @@ bool Item_sum::check_sum_func(THD *thd, Item **ref)
   }
   aggr_sel->set_agg_func_used(true);
   update_used_tables();
-  thd->lex->in_sum_func= in_sum_func;
+  lex_s->in_sum_func= in_sum_func;
   return FALSE;
 }
 
diff --git a/sql/sql_base.cc b/sql/sql_base.cc
index 9a66b27a454..5173df260d5 100644
--- a/sql/sql_base.cc
+++ b/sql/sql_base.cc
@@ -6440,6 +6440,7 @@ set_new_item_local_context(THD *thd, Item_ident *item, TABLE_LIST *table_ref)
   if (!(context= new (thd->mem_root) Name_resolution_context))
     return TRUE;
   context->init();
+  context->select_lex= table_ref->select_lex;
   context->first_name_resolution_table=
     context->last_name_resolution_table= table_ref;
   item->context= context;
diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc
index 624bcb90cd5..9339cb925e5 100644
--- a/sql/sql_prepare.cc
+++ b/sql/sql_prepare.cc
@@ -3060,6 +3060,7 @@ void reinit_stmt_before_use(THD *thd, LEX *lex)
   }
   for (; sl; sl= sl->next_select_in_list())
   {
+    sl->parent_lex->in_sum_func= NULL;
     if (sl->changed_elements & TOUCHED_SEL_COND)
     {
       /* remove option which was put by mysql_explain_union() */
@@ -3190,7 +3191,6 @@ void reinit_stmt_before_use(THD *thd, LEX *lex)
     lex->result->set_thd(thd);
   }
   lex->allow_sum_func= 0;
-  lex->in_sum_func= NULL;
   DBUG_VOID_RETURN;
 }
 

From e55c303cc5d955eb3467fc6a89d8537900fb31c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Fri, 24 Sep 2021 11:11:57 +0300
Subject: [PATCH 14/31] Add wait_condition before problematic select

---
 mysql-test/suite/galera/disabled.def          |  1 -
 .../t/galera_var_ignore_apply_errors.cnf      |  7 +++++++
 .../t/galera_var_ignore_apply_errors.test     | 19 ++++++++++++++++++-
 3 files changed, 25 insertions(+), 2 deletions(-)
 create mode 100644 mysql-test/suite/galera/t/galera_var_ignore_apply_errors.cnf

diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def
index d742879bb10..96005fccf54 100644
--- a/mysql-test/suite/galera/disabled.def
+++ b/mysql-test/suite/galera/disabled.def
@@ -33,7 +33,6 @@ galera_pc_recovery : MDEV-25199 cluster fails to start up
 galera_shutdown_nonprim : MDEV-21493 galera.galera_shutdown_nonprim
 galera_toi_ddl_nonconflicting : MDEV-21518 galera.galera_toi_ddl_nonconflicting
 galera_toi_truncate : MDEV-22996 Hang on galera_toi_truncate test case
-galera_var_ignore_apply_errors : MDEV-20451: Lock wait timeout exceeded in galera_var_ignore_apply_errors
 galera_var_node_address : MDEV-20485 Galera test failure
 galera_var_notify_cmd : MDEV-21905 Galera test galera_var_notify_cmd causes hang
 galera_var_reject_queries : assertion in inline_mysql_socket_send
diff --git a/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.cnf b/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.cnf
new file mode 100644
index 00000000000..545cc8147e0
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.cnf
@@ -0,0 +1,7 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep_debug=1
+
+[mysqld.2]
+wsrep_debug=1
diff --git a/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.test b/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.test
index 0f8efad5163..5a00048a90e 100644
--- a/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.test
+++ b/mysql-test/suite/galera/t/galera_var_ignore_apply_errors.test
@@ -3,7 +3,6 @@
 #
 
 --source include/galera_cluster.inc
---source include/have_innodb.inc
 
 
 #
@@ -73,11 +72,15 @@ SET GLOBAL wsrep_on = ON;
 DELETE FROM t1 WHERE f1 = 1;
 
 --connection node_1
+--let $wait_condition = SELECT COUNT(*) = 0 FROM t1;
+--source include/wait_condition.inc
 SELECT COUNT(*) as expect_0 FROM t1;
 --connection node_2
 --source include/galera_wait_ready.inc
 --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 --source include/wait_condition.inc
+--let $wait_condition = SELECT COUNT(*) = 0 FROM t1;
+--source include/wait_condition.inc
 SELECT COUNT(*) as expect_0 FROM t1;
 
 DROP TABLE t1;
@@ -97,11 +100,15 @@ DELETE FROM t1 WHERE f1 = 2;
 COMMIT;
 
 --connection node_1
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
+--source include/wait_condition.inc
 SELECT COUNT(*) as expect_1 FROM t1;
 --connection node_2
 --source include/galera_wait_ready.inc
 --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 --source include/wait_condition.inc
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1;
+--source include/wait_condition.inc
 SELECT COUNT(*) as expect_1 FROM t1;
 
 DROP TABLE t1;
@@ -132,6 +139,8 @@ SELECT COUNT(*) as expect_0 FROM t1;
 --let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
 --source include/wait_condition.inc
 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--let $wait_condition = SELECT COUNT(*) = 0 FROM t1;
+--source include/wait_condition.inc
 SELECT COUNT(*) as expect_0 FROM t1;
 DROP TABLE t1;
 
@@ -171,6 +180,8 @@ SELECT COUNT(*) as expect_0 FROM t1;
 --let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
 --source include/wait_condition.inc
 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--let $wait_condition = SELECT COUNT(*) = 0 FROM t1;
+--source include/wait_condition.inc
 SELECT COUNT(*) as expect_0 FROM t1;
 DROP TABLE t1;
 
@@ -205,6 +216,8 @@ SELECT COUNT(*) as expect_0 FROM t1;
 --let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
 --source include/wait_condition.inc
 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--let $wait_condition = SELECT COUNT(*) = 0 FROM t1;
+--source include/wait_condition.inc
 SELECT COUNT(*) as expect_0 FROM t1;
 DROP TABLE t1,t2;
 
@@ -239,6 +252,10 @@ SELECT COUNT(*) as expect_0 FROM child;
 --let $wait_condition = SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
 --source include/wait_condition.inc
 SELECT VARIABLE_VALUE = 'Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status';
+--let $wait_condition = SELECT COUNT(*) = 0 FROM parent;
+--source include/wait_condition.inc
+--let $wait_condition = SELECT COUNT(*) = 0 FROM child;
+--source include/wait_condition.inc
 SELECT COUNT(*) as expect_0 FROM parent;
 SELECT COUNT(*) as expect_0 FROM child;
 DROP TABLE child, parent;

From 05abcd7e600ddc2dee280b000439ded2855ff772 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Fri, 24 Sep 2021 12:28:15 +0300
Subject: [PATCH 15/31] MDEV-21806 : galera.galera_partition MTR failed: failed
 to recover from DONOR state

Add wait_condition to wait until all nodes are in cluster
---
 mysql-test/suite/galera/r/galera_partition.result | 2 ++
 mysql-test/suite/galera/t/galera_partition.cnf    | 4 ++++
 mysql-test/suite/galera/t/galera_partition.test   | 6 ++++++
 3 files changed, 12 insertions(+)

diff --git a/mysql-test/suite/galera/r/galera_partition.result b/mysql-test/suite/galera/r/galera_partition.result
index f09a0272eda..0e8894794d7 100644
--- a/mysql-test/suite/galera/r/galera_partition.result
+++ b/mysql-test/suite/galera/r/galera_partition.result
@@ -424,3 +424,5 @@ connection node_4;
 connection node_1;
 DROP TABLE t1;
 DROP PROCEDURE p1;
+disconnect node_3;
+disconnect node_4;
diff --git a/mysql-test/suite/galera/t/galera_partition.cnf b/mysql-test/suite/galera/t/galera_partition.cnf
index e6cb13ef523..525eece04ab 100644
--- a/mysql-test/suite/galera/t/galera_partition.cnf
+++ b/mysql-test/suite/galera/t/galera_partition.cnf
@@ -3,16 +3,20 @@
 [mysqld.1]
 wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=10M;gmcast.segment=1'
 wsrep_slave_threads=10
+wsrep_debug=1
 
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=10M;gmcast.segment=1'
 wsrep_slave_threads=10
+wsrep_debug=1
 
 [mysqld.3]
 wsrep_provider_options='base_port=@mysqld.3.#galera_port;gcache.size=10M;gmcast.segment=2'
 wsrep_slave_threads=10
+wsrep_debug=1
 
 [mysqld.4]
 wsrep_provider_options='base_port=@mysqld.4.#galera_port;gcache.size=10M;gmcast.segment=3'
 wsrep_slave_threads=10
+wsrep_debug=1
 
diff --git a/mysql-test/suite/galera/t/galera_partition.test b/mysql-test/suite/galera/t/galera_partition.test
index bf3f02eaa92..3de45d54000 100644
--- a/mysql-test/suite/galera/t/galera_partition.test
+++ b/mysql-test/suite/galera/t/galera_partition.test
@@ -11,6 +11,8 @@ call mtr.add_suppression("WSREP: ALTER TABLE isolation failure");
 --connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4
 
 --connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 4 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
 
 CREATE TABLE t1(
  id bigint unsigned NOT NULL AUTO_INCREMENT,
@@ -449,3 +451,7 @@ reap;
 DROP TABLE t1;
 DROP PROCEDURE p1;
 
+--disconnect node_3
+--disconnect node_4
+
+

From 690c4725914377914640d7d714a7e93127ee169d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Mon, 27 Sep 2021 15:09:50 +0300
Subject: [PATCH 16/31] MDEV-21613 : galera_sr.GCF-1018B MTR failed: Failed to
 open table mysql.wsrep_streaming_log for writing

Query can be bf aborted already earlier and then we should not
even try to open table.
---
 sql/wsrep_schema.cc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/sql/wsrep_schema.cc b/sql/wsrep_schema.cc
index 252f41cb380..fe45c4d1711 100644
--- a/sql/wsrep_schema.cc
+++ b/sql/wsrep_schema.cc
@@ -289,7 +289,14 @@ static int open_table(THD* thd,
                         NULL, lock_type);
   thd->lex->query_tables_own_last= 0;
 
-  if (!open_n_lock_single_table(thd, &tables, tables.lock_type, flags)) {
+  // No need to open table if the query was bf aborted,
+  // thd client will get ER_LOCK_DEADLOCK in the end.
+  const bool interrupted= thd->killed ||
+       (thd->is_error() &&
+       (thd->get_stmt_da()->sql_errno() == ER_QUERY_INTERRUPTED));
+
+  if (interrupted ||
+      !open_n_lock_single_table(thd, &tables, tables.lock_type, flags)) {
     close_thread_tables(thd);
     DBUG_RETURN(1);
   }

From 0d68b0a2d6e52cbbbd971cd66dab8989dd1e48ac Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Thu, 23 Sep 2021 18:46:37 +0400
Subject: [PATCH 17/31] MDEV-26669 Add MY_COLLATION_HANDLER functions min_str()
 and max_str()

---
 include/m_ctype.h            |  37 ++++-
 mysys/charset-def.c          |   4 +-
 strings/ctype-big5.c         |  17 ++-
 strings/ctype-bin.c          |  13 +-
 strings/ctype-cp932.c        |  17 ++-
 strings/ctype-czech.c        |  13 +-
 strings/ctype-euc_kr.c       |  17 ++-
 strings/ctype-eucjpms.c      |  17 ++-
 strings/ctype-gb2312.c       |  17 ++-
 strings/ctype-gbk.c          |  17 ++-
 strings/ctype-latin1.c       |   5 +-
 strings/ctype-mb.c           |  41 ++++++
 strings/ctype-mb.h           |  37 +++++
 strings/ctype-simple.c       |  38 ++++-
 strings/ctype-simple.h       |  37 +++++
 strings/ctype-sjis.c         |  17 ++-
 strings/ctype-tis620.c       |   9 +-
 strings/ctype-uca.ic         |  17 ++-
 strings/ctype-ucs2.c         |  65 ++++++---
 strings/ctype-ujis.c         |  17 ++-
 strings/ctype-utf8.c         |  45 ++++--
 strings/ctype-win1250ch.c    |  14 +-
 unittest/strings/strings-t.c | 259 ++++++++++++++++++++++++++++++++++-
 23 files changed, 683 insertions(+), 87 deletions(-)
 create mode 100644 strings/ctype-mb.h
 create mode 100644 strings/ctype-simple.h

diff --git a/include/m_ctype.h b/include/m_ctype.h
index 279e8a74ddc..41523913c10 100644
--- a/include/m_ctype.h
+++ b/include/m_ctype.h
@@ -377,6 +377,12 @@ struct my_collation_handler_st
   void (*hash_sort)(CHARSET_INFO *cs, const uchar *key, size_t len,
 		    ulong *nr1, ulong *nr2); 
   my_bool (*propagate)(CHARSET_INFO *cs, const uchar *str, size_t len);
+  /*
+    Make minimum and maximum strings for the collation.
+    Put not more than "nchars" characters.
+  */
+  size_t (*min_str)(CHARSET_INFO *cs, uchar *dst, size_t dstlen, size_t nchars);
+  size_t (*max_str)(CHARSET_INFO *cs, uchar *dst, size_t dstlen, size_t nchars);
 };
 
 extern MY_COLLATION_HANDLER my_collation_8bit_bin_handler;
@@ -589,8 +595,23 @@ struct charset_info_st
   uchar     casedn_multiply;
   uint      mbminlen;
   uint      mbmaxlen;
+  /*
+    min_sort_char and max_sort_char represent the minimum
+    and the maximum character in the collation respectively.
+
+    For Unicode collations, these numbers are Unicode code points.
+    For non-Unicode collations these numbers are native character codes.
+    For example, in all 8bit collations these numbers are
+    in the range 0x00..0xFF.
+
+    min_sort_char and max_sort_char normally should not be used directly.
+    They are used internally in the following virtual functions:
+    - MY_COLLATION_HANDLER::like_range()
+    - MY_COLLATION_HANDLER::min_str()
+    - MY_COLLATION_HANDLER::max_str()
+  */
   my_wc_t   min_sort_char;
-  my_wc_t   max_sort_char; /* For LIKE optimization */
+  my_wc_t   max_sort_char;
   uchar     pad_char;
   my_bool   escape_with_backslash_is_dangerous;
   uchar     levels_for_order;
@@ -852,6 +873,16 @@ struct charset_info_st
     return (coll->propagate)(this, str, len);
   }
 
+  size_t min_str(uchar *dst, size_t dstlen, size_t nchars) const
+  {
+    return (coll->min_str)(this, dst, dstlen, nchars);
+  }
+
+  size_t max_str(uchar *dst, size_t dstlen, size_t nchars) const
+  {
+    return (coll->max_str)(this, dst, dstlen, nchars);
+  }
+
 #endif /* __cplusplus */
 };
 
@@ -1110,7 +1141,7 @@ extern struct charset_info_st my_charset_big5_bin;
 extern struct charset_info_st my_charset_big5_chinese_ci;
 extern struct charset_info_st my_charset_big5_nopad_bin;
 extern struct charset_info_st my_charset_big5_chinese_nopad_ci;
-extern struct charset_info_st my_charset_cp1250_czech_ci;
+extern struct charset_info_st my_charset_cp1250_czech_cs;
 extern struct charset_info_st my_charset_cp932_bin;
 extern struct charset_info_st my_charset_cp932_japanese_ci;
 extern struct charset_info_st my_charset_cp932_nopad_bin;
@@ -1134,7 +1165,7 @@ extern struct charset_info_st my_charset_gbk_chinese_nopad_ci;
 extern struct charset_info_st my_charset_latin1_bin;
 extern struct charset_info_st my_charset_latin1_nopad_bin;
 extern struct charset_info_st my_charset_latin1_german2_ci;
-extern struct charset_info_st my_charset_latin2_czech_ci;
+extern struct charset_info_st my_charset_latin2_czech_cs;
 extern struct charset_info_st my_charset_sjis_bin;
 extern struct charset_info_st my_charset_sjis_japanese_ci;
 extern struct charset_info_st my_charset_sjis_nopad_bin;
diff --git a/mysys/charset-def.c b/mysys/charset-def.c
index 737a1c88a47..2e23adc5147 100644
--- a/mysys/charset-def.c
+++ b/mysys/charset-def.c
@@ -202,7 +202,7 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused)))
 #endif
 
 #ifdef HAVE_CHARSET_cp1250
-  add_compiled_collation(&my_charset_cp1250_czech_ci);
+  add_compiled_collation(&my_charset_cp1250_czech_cs);
 #endif
 
 #ifdef HAVE_CHARSET_cp932
@@ -213,7 +213,7 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused)))
 #endif
 
 #ifdef HAVE_CHARSET_latin2
-  add_compiled_collation(&my_charset_latin2_czech_ci);
+  add_compiled_collation(&my_charset_latin2_czech_cs);
 #endif
 
 #ifdef HAVE_CHARSET_eucjpms
diff --git a/strings/ctype-big5.c b/strings/ctype-big5.c
index 73e3d8ea306..989d6b5f1a8 100644
--- a/strings/ctype-big5.c
+++ b/strings/ctype-big5.c
@@ -29,6 +29,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifdef HAVE_CHARSET_big5
 
@@ -6721,7 +6722,9 @@ static MY_COLLATION_HANDLER my_collation_handler_big5_chinese_ci=
   my_strcasecmp_mb,
   my_instr_mb,
   my_hash_sort_simple,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -6737,7 +6740,9 @@ static MY_COLLATION_HANDLER my_collation_handler_big5_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -6753,7 +6758,9 @@ static MY_COLLATION_HANDLER my_collation_handler_big5_chinese_nopad_ci=
   my_strcasecmp_mb,
   my_instr_mb,
   my_hash_sort_simple_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -6769,7 +6776,9 @@ static MY_COLLATION_HANDLER my_collation_handler_big5_nopad_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-bin.c b/strings/ctype-bin.c
index 2902c2edda1..b142cab523a 100644
--- a/strings/ctype-bin.c
+++ b/strings/ctype-bin.c
@@ -22,6 +22,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-simple.h"
 
 const char charset_name_binary[]= "binary";
 #define charset_name_binary_length (sizeof(charset_name_binary)-1)
@@ -497,7 +498,9 @@ MY_COLLATION_HANDLER my_collation_8bit_bin_handler =
   my_strcasecmp_bin,
   my_instr_bin,
   my_hash_sort_8bit_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_8bit_simple,
+  my_max_str_8bit_simple
 };
 
 
@@ -513,7 +516,9 @@ MY_COLLATION_HANDLER my_collation_8bit_nopad_bin_handler =
   my_strcasecmp_bin,
   my_instr_bin,
   my_hash_sort_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_8bit_simple_nopad,
+  my_max_str_8bit_simple
 };
 
 
@@ -529,7 +534,9 @@ static MY_COLLATION_HANDLER my_collation_binary_handler =
   my_strcasecmp_bin,
   my_instr_bin,
   my_hash_sort_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_8bit_simple_nopad,
+  my_max_str_8bit_simple
 };
 
 
diff --git a/strings/ctype-cp932.c b/strings/ctype-cp932.c
index 859fd029744..e3c29ae1a32 100644
--- a/strings/ctype-cp932.c
+++ b/strings/ctype-cp932.c
@@ -19,6 +19,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifdef HAVE_CHARSET_cp932
 
@@ -34676,7 +34677,9 @@ static MY_COLLATION_HANDLER my_collation_handler_cp932_japanese_ci=
   my_strcasecmp_8bit,
   my_instr_mb,
   my_hash_sort_simple,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -34692,7 +34695,9 @@ static MY_COLLATION_HANDLER my_collation_handler_cp932_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -34708,7 +34713,9 @@ static MY_COLLATION_HANDLER my_collation_handler_cp932_japanese_nopad_ci=
   my_strcasecmp_8bit,
   my_instr_mb,
   my_hash_sort_simple_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -34724,7 +34731,9 @@ static MY_COLLATION_HANDLER my_collation_handler_cp932_nopad_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-czech.c b/strings/ctype-czech.c
index 40baae67dd3..ca331dc667f 100644
--- a/strings/ctype-czech.c
+++ b/strings/ctype-czech.c
@@ -69,6 +69,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-simple.h"
 
 #else
 
@@ -605,7 +606,7 @@ static MY_UNI_IDX idx_uni_8859_2[]={
 };
 
 
-static MY_COLLATION_HANDLER my_collation_latin2_czech_ci_handler =
+static MY_COLLATION_HANDLER my_collation_latin2_czech_cs_handler =
 {
   NULL,			/* init */
   my_strnncoll_czech,
@@ -617,10 +618,12 @@ static MY_COLLATION_HANDLER my_collation_latin2_czech_ci_handler =
   my_strcasecmp_8bit,
   my_instr_simple,
   my_hash_sort_simple,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_8bit_simple,
+  my_max_str_8bit_simple
 };
 
-struct charset_info_st my_charset_latin2_czech_ci =
+struct charset_info_st my_charset_latin2_czech_cs =
 {
     2,0,0,                                      /* number    */
     MY_CS_COMPILED|MY_CS_STRNXFRM|MY_CS_CSSORT|
@@ -645,12 +648,12 @@ struct charset_info_st my_charset_latin2_czech_ci =
     1,			/* mbminlen   */
     1,			/* mbmaxlen  */
     0,			/* min_sort_char */
-    0,			/* max_sort_char */
+    0xAE,               /* max_sort_char */
     ' ',                /* pad char      */
     0,                  /* escape_with_backslash_is_dangerous */
     4,                  /* levels_for_order   */
     &my_charset_8bit_handler,
-    &my_collation_latin2_czech_ci_handler
+    &my_collation_latin2_czech_cs_handler
 };
 
 
diff --git a/strings/ctype-euc_kr.c b/strings/ctype-euc_kr.c
index e152862c0ef..3c257acb460 100644
--- a/strings/ctype-euc_kr.c
+++ b/strings/ctype-euc_kr.c
@@ -28,6 +28,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifdef HAVE_CHARSET_euckr
 
@@ -9966,7 +9967,9 @@ static MY_COLLATION_HANDLER my_collation_handler_euckr_korean_ci=
   my_strcasecmp_mb,
   my_instr_mb,
   my_hash_sort_simple,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -9982,7 +9985,9 @@ static MY_COLLATION_HANDLER my_collation_handler_euckr_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -9998,7 +10003,9 @@ static MY_COLLATION_HANDLER my_collation_handler_euckr_korean_nopad_ci=
   my_strcasecmp_mb,
   my_instr_mb,
   my_hash_sort_simple_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -10014,7 +10021,9 @@ static MY_COLLATION_HANDLER my_collation_handler_euckr_nopad_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-eucjpms.c b/strings/ctype-eucjpms.c
index 4c365b4ff6a..ca790291a4a 100644
--- a/strings/ctype-eucjpms.c
+++ b/strings/ctype-eucjpms.c
@@ -30,6 +30,7 @@ ctype-ujis.c file.
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifdef HAVE_CHARSET_eucjpms
 
@@ -67504,7 +67505,9 @@ static MY_COLLATION_HANDLER my_collation_eucjpms_japanese_ci_handler =
     my_strcasecmp_mb,
     my_instr_mb,
     my_hash_sort_simple,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -67520,7 +67523,9 @@ static MY_COLLATION_HANDLER my_collation_eucjpms_bin_handler =
     my_strcasecmp_mb_bin,
     my_instr_mb,
     my_hash_sort_mb_bin,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -67536,7 +67541,9 @@ static MY_COLLATION_HANDLER my_collation_eucjpms_japanese_nopad_ci_handler =
     my_strcasecmp_mb,
     my_instr_mb,
     my_hash_sort_simple_nopad,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple_nopad,
+    my_max_str_mb_simple
 };
 
 
@@ -67552,7 +67559,9 @@ static MY_COLLATION_HANDLER my_collation_eucjpms_nopad_bin_handler =
     my_strcasecmp_mb_bin,
     my_instr_mb,
     my_hash_sort_mb_nopad_bin,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple_nopad,
+    my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-gb2312.c b/strings/ctype-gb2312.c
index 56709b06bf7..a8e42945259 100644
--- a/strings/ctype-gb2312.c
+++ b/strings/ctype-gb2312.c
@@ -26,6 +26,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifdef HAVE_CHARSET_gb2312
 
@@ -6372,7 +6373,9 @@ static MY_COLLATION_HANDLER my_collation_handler_gb2312_chinese_ci=
   my_strcasecmp_mb,     /* instr      */
   my_instr_mb,
   my_hash_sort_simple,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -6388,7 +6391,9 @@ static MY_COLLATION_HANDLER my_collation_handler_gb2312_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -6404,7 +6409,9 @@ static MY_COLLATION_HANDLER my_collation_handler_gb2312_chinese_nopad_ci=
   my_strcasecmp_mb,
   my_instr_mb,
   my_hash_sort_simple_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -6420,7 +6427,9 @@ static MY_COLLATION_HANDLER my_collation_handler_gb2312_nopad_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-gbk.c b/strings/ctype-gbk.c
index 95f59a18d99..c8aade2eb31 100644
--- a/strings/ctype-gbk.c
+++ b/strings/ctype-gbk.c
@@ -28,6 +28,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifdef HAVE_CHARSET_gbk
 
@@ -10654,7 +10655,9 @@ static MY_COLLATION_HANDLER my_collation_handler_gbk_chinese_ci=
   my_strcasecmp_mb,
   my_instr_mb,
   my_hash_sort_simple,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -10670,7 +10673,9 @@ static MY_COLLATION_HANDLER my_collation_handler_gbk_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -10686,7 +10691,9 @@ static MY_COLLATION_HANDLER my_collation_handler_gbk_chinese_nopad_ci=
   my_strcasecmp_mb,
   my_instr_mb,
   my_hash_sort_simple_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -10702,7 +10709,9 @@ static MY_COLLATION_HANDLER my_collation_handler_gbk_nopad_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 static MY_CHARSET_HANDLER my_charset_handler=
diff --git a/strings/ctype-latin1.c b/strings/ctype-latin1.c
index 2a0983ee16f..9412f212420 100644
--- a/strings/ctype-latin1.c
+++ b/strings/ctype-latin1.c
@@ -17,6 +17,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-simple.h"
 
 const char charset_name_latin1[]= "latin1";
 #define charset_name_latin1_length  sizeof(charset_name_latin1)-1
@@ -737,7 +738,9 @@ static MY_COLLATION_HANDLER my_collation_german2_ci_handler=
   my_strcasecmp_8bit,
   my_instr_simple,
   my_hash_sort_latin1_de,
-  my_propagate_complex
+  my_propagate_complex,
+  my_min_str_8bit_simple,
+  my_max_str_8bit_simple
 };
 
 
diff --git a/strings/ctype-mb.c b/strings/ctype-mb.c
index b84c91efe5f..66f82a1d09c 100644
--- a/strings/ctype-mb.c
+++ b/strings/ctype-mb.c
@@ -16,6 +16,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifdef USE_MB
 
@@ -640,6 +641,46 @@ my_hash_sort_mb_bin(CHARSET_INFO *cs __attribute__((unused)),
 }
 
 
+static inline size_t
+my_repeat_char_native(CHARSET_INFO *cs,
+                      uchar *dst, size_t dst_size, size_t nchars,
+                      my_wc_t native_code)
+{
+  uchar *dst0= dst;
+  uchar *dstend= dst + dst_size;
+  int chlen= my_ci_native_to_mb(cs, native_code, dst, dstend);
+  if (chlen < 1 /* Not enough space */ || !nchars)
+    return 0;
+  for (dst+= chlen, nchars--;
+       dst + chlen <= dstend && nchars > 0;
+       dst+= chlen, nchars--)
+    memcpy(dst, dst0, chlen);
+  return dst - dst0;
+}
+
+
+size_t my_min_str_mb_simple(CHARSET_INFO *cs,
+                            uchar *dst, size_t dst_size, size_t nchars)
+{
+  return my_repeat_char_native(cs, dst, dst_size, nchars, cs->min_sort_char);
+}
+
+
+size_t my_min_str_mb_simple_nopad(CHARSET_INFO *cs,
+                                  uchar *dst, size_t dst_size, size_t nchars)
+{
+  /* For NOPAD collations, the empty string is the smallest possible */
+  return 0;
+}
+
+
+size_t my_max_str_mb_simple(CHARSET_INFO *cs,
+                            uchar *dst, size_t dst_size, size_t nchars)
+{
+  return my_repeat_char_native(cs, dst, dst_size, nchars, cs->max_sort_char);
+}
+
+
 /* 
   Fill the given buffer with 'maximum character' for given charset
   SYNOPSIS
diff --git a/strings/ctype-mb.h b/strings/ctype-mb.h
new file mode 100644
index 00000000000..b6f06e8df0d
--- /dev/null
+++ b/strings/ctype-mb.h
@@ -0,0 +1,37 @@
+#ifndef CTYPE_MB_INCLUDED
+#define CTYPE_MB_INCLUDED
+/* Copyright (C) 2021 MariaDB Corporation
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */
+
+/* This file is to be include first in all files in the string directory */
+
+#undef DBUG_ASSERT_AS_PRINTF
+#include <my_global.h>		/* Define standard vars */
+#include "m_string.h"		/* Exernal definitions of string functions */
+
+
+size_t
+my_min_str_mb_simple(CHARSET_INFO *cs,
+                     uchar *dst, size_t dst_size, size_t nchars);
+
+size_t
+my_min_str_mb_simple_nopad(CHARSET_INFO *cs,
+                           uchar *dst, size_t dst_size, size_t nchars);
+
+size_t
+my_max_str_mb_simple(CHARSET_INFO *cs,
+                     uchar *dst, size_t dst_size, size_t nchars);
+
+#endif /*CTYPE_MB_INCLUDED */
diff --git a/strings/ctype-simple.c b/strings/ctype-simple.c
index f9471f35f79..a0ba4357d5e 100644
--- a/strings/ctype-simple.c
+++ b/strings/ctype-simple.c
@@ -16,6 +16,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-simple.h"
 #include "my_sys.h"  /* Needed for MY_ERRNO_ERANGE */
 #include <errno.h>
 
@@ -891,6 +892,35 @@ cnv:
 }
 
 
+size_t my_min_str_8bit_simple(CHARSET_INFO *cs,
+                              uchar *dst, size_t dst_size,
+                              size_t nchars)
+{
+  set_if_smaller(dst_size, nchars);
+  memset(dst, cs->min_sort_char, dst_size);
+  return dst_size;
+}
+
+
+size_t my_min_str_8bit_simple_nopad(CHARSET_INFO *cs,
+                                    uchar *dst, size_t dst_size,
+                                    size_t nchars)
+{
+  /* For NOPAD collations, the empty string is always the smallest */
+  return 0;
+}
+
+
+size_t my_max_str_8bit_simple(CHARSET_INFO *cs,
+                              uchar *dst, size_t dst_size,
+                              size_t nchars)
+{
+  set_if_smaller(dst_size, nchars);
+  memset(dst, cs->max_sort_char, dst_size);
+  return dst_size;
+}
+
+
 /*
 ** Compare string against string with wildcard
 **	0 if matched
@@ -2104,7 +2134,9 @@ MY_COLLATION_HANDLER my_collation_8bit_simple_ci_handler =
     my_strcasecmp_8bit,
     my_instr_simple,
     my_hash_sort_simple,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_8bit_simple,
+    my_max_str_8bit_simple
 };
 
 
@@ -2120,5 +2152,7 @@ MY_COLLATION_HANDLER my_collation_8bit_simple_nopad_ci_handler =
     my_strcasecmp_8bit,
     my_instr_simple,
     my_hash_sort_simple_nopad,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_8bit_simple_nopad,
+    my_max_str_8bit_simple
 };
diff --git a/strings/ctype-simple.h b/strings/ctype-simple.h
new file mode 100644
index 00000000000..321c107a61f
--- /dev/null
+++ b/strings/ctype-simple.h
@@ -0,0 +1,37 @@
+#ifndef CTYPE_SIMPLE_INCLUDED
+#define CTYPE_SIMPLE_INCLUDED
+/* Copyright (C) 2021 MariaDB Corporation
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 of the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */
+
+/* This file is to be include first in all files in the string directory */
+
+#undef DBUG_ASSERT_AS_PRINTF
+#include <my_global.h>		/* Define standard vars */
+#include "m_string.h"		/* Exernal definitions of string functions */
+
+
+size_t
+my_min_str_8bit_simple(CHARSET_INFO *cs,
+                       uchar *dst, size_t dst_size, size_t nchars);
+
+size_t
+my_min_str_8bit_simple_nopad(CHARSET_INFO *cs,
+                             uchar *dst, size_t dst_size, size_t nchars);
+
+size_t
+my_max_str_8bit_simple(CHARSET_INFO *cs,
+                       uchar *dst, size_t dst_size, size_t nchars);
+
+#endif /*CTYPE_SIMPLE_INCLUDED */
diff --git a/strings/ctype-sjis.c b/strings/ctype-sjis.c
index b6b212f2c0d..d26083202fc 100644
--- a/strings/ctype-sjis.c
+++ b/strings/ctype-sjis.c
@@ -19,6 +19,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifdef HAVE_CHARSET_sjis
 
@@ -34064,7 +34065,9 @@ static MY_COLLATION_HANDLER my_collation_handler_sjis_japanese_ci=
   my_strcasecmp_8bit,
   my_instr_mb,
   my_hash_sort_simple,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -34080,7 +34083,9 @@ static MY_COLLATION_HANDLER my_collation_handler_sjis_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -34096,7 +34101,9 @@ static MY_COLLATION_HANDLER my_collation_handler_sjis_japanese_nopad_ci=
   my_strcasecmp_8bit,
   my_instr_mb,
   my_hash_sort_simple_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -34112,7 +34119,9 @@ static MY_COLLATION_HANDLER my_collation_handler_sjis_nopad_bin=
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-tis620.c b/strings/ctype-tis620.c
index 849f4897231..dac44295f3c 100644
--- a/strings/ctype-tis620.c
+++ b/strings/ctype-tis620.c
@@ -35,6 +35,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-simple.h"
 #include "t_ctype.h"
 #include <my_sys.h>
 
@@ -862,7 +863,9 @@ static MY_COLLATION_HANDLER my_collation_ci_handler =
     my_strcasecmp_8bit,
     my_instr_simple,				/* QQ: To be fixed */
     my_hash_sort_simple,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_8bit_simple,
+    my_max_str_8bit_simple
 };
 
 static MY_COLLATION_HANDLER my_collation_nopad_ci_handler =
@@ -877,7 +880,9 @@ static MY_COLLATION_HANDLER my_collation_nopad_ci_handler =
     my_strcasecmp_8bit,
     my_instr_simple,				/* QQ: To be fixed */
     my_hash_sort_simple_nopad,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_8bit_simple_nopad,
+    my_max_str_8bit_simple
 };
 
 static MY_CHARSET_HANDLER my_charset_handler=
diff --git a/strings/ctype-uca.ic b/strings/ctype-uca.ic
index e47f1e1fd82..cee12cf4d7b 100644
--- a/strings/ctype-uca.ic
+++ b/strings/ctype-uca.ic
@@ -15,6 +15,7 @@
   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */
 
+#include "ctype-mb.h"
 
 #ifndef MY_FUNCTION_NAME
 #error MY_FUNCTION_NAME is not defined
@@ -759,7 +760,9 @@ MY_COLLATION_HANDLER MY_FUNCTION_NAME(collation_handler)=
   NULL,                                /* strcasecmp() */
   my_instr_mb,
   MY_FUNCTION_NAME(hash_sort),
-  my_propagate_complex
+  my_propagate_complex,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -780,7 +783,9 @@ MY_COLLATION_HANDLER MY_FUNCTION_NAME(collation_handler_nopad)=
   NULL,                                /* strcasecmp() */
   my_instr_mb,
   MY_FUNCTION_NAME(hash_sort_nopad),
-  my_propagate_complex
+  my_propagate_complex,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -799,7 +804,9 @@ MY_COLLATION_HANDLER MY_FUNCTION_NAME(collation_handler_multilevel)=
   NULL,                                /* strcasecmp() */
   my_instr_mb,
   MY_FUNCTION_NAME(hash_sort),
-  my_propagate_complex
+  my_propagate_complex,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -818,7 +825,9 @@ MY_COLLATION_HANDLER MY_FUNCTION_NAME(collation_handler_nopad_multilevel)=
   NULL,                                /* strcasecmp() */
   my_instr_mb,
   MY_FUNCTION_NAME(hash_sort),
-  my_propagate_complex
+  my_propagate_complex,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c
index dbf0d86dc05..c230c65d3d6 100644
--- a/strings/ctype-ucs2.c
+++ b/strings/ctype-ucs2.c
@@ -20,6 +20,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 #include <my_sys.h>
 #include <stdarg.h>
 
@@ -1513,7 +1514,9 @@ static MY_COLLATION_HANDLER my_collation_utf16_general_ci_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf16,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -1529,7 +1532,9 @@ static MY_COLLATION_HANDLER my_collation_utf16_bin_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf16_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -1545,7 +1550,9 @@ static MY_COLLATION_HANDLER my_collation_utf16_general_nopad_ci_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf16_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -1561,7 +1568,9 @@ static MY_COLLATION_HANDLER my_collation_utf16_nopad_bin_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf16_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -1854,7 +1863,9 @@ static MY_COLLATION_HANDLER my_collation_utf16le_general_ci_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf16,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -1870,7 +1881,9 @@ static MY_COLLATION_HANDLER my_collation_utf16le_bin_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf16_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -1886,7 +1899,9 @@ static MY_COLLATION_HANDLER my_collation_utf16le_general_nopad_ci_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf16_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -1902,7 +1917,9 @@ static MY_COLLATION_HANDLER my_collation_utf16le_nopad_bin_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf16_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -2680,7 +2697,9 @@ static MY_COLLATION_HANDLER my_collation_utf32_general_ci_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf32,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -2696,7 +2715,9 @@ static MY_COLLATION_HANDLER my_collation_utf32_bin_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf32,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -2712,7 +2733,9 @@ static MY_COLLATION_HANDLER my_collation_utf32_general_nopad_ci_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf32_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -2728,7 +2751,9 @@ static MY_COLLATION_HANDLER my_collation_utf32_nopad_bin_handler =
   my_strcasecmp_mb2_or_mb4,
   my_instr_mb,
   my_hash_sort_utf32_nopad,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -3271,7 +3296,9 @@ static MY_COLLATION_HANDLER my_collation_ucs2_general_ci_handler =
     my_strcasecmp_mb2_or_mb4,
     my_instr_mb,
     my_hash_sort_ucs2,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -3287,7 +3314,9 @@ static MY_COLLATION_HANDLER my_collation_ucs2_bin_handler =
     my_strcasecmp_mb2_or_mb4,
     my_instr_mb,
     my_hash_sort_ucs2_bin,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -3303,7 +3332,9 @@ static MY_COLLATION_HANDLER my_collation_ucs2_general_nopad_ci_handler =
     my_strcasecmp_mb2_or_mb4,
     my_instr_mb,
     my_hash_sort_ucs2_nopad,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple_nopad,
+    my_max_str_mb_simple
 };
 
 
@@ -3319,7 +3350,9 @@ static MY_COLLATION_HANDLER my_collation_ucs2_nopad_bin_handler =
     my_strcasecmp_mb2_or_mb4,
     my_instr_mb,
     my_hash_sort_ucs2_nopad_bin,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple_nopad,
+    my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-ujis.c b/strings/ctype-ujis.c
index fa5856ebc0d..c190496b364 100644
--- a/strings/ctype-ujis.c
+++ b/strings/ctype-ujis.c
@@ -29,6 +29,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifdef HAVE_CHARSET_ujis
 
@@ -67248,7 +67249,9 @@ static MY_COLLATION_HANDLER my_collation_ujis_japanese_ci_handler =
     my_strcasecmp_mb,
     my_instr_mb,
     my_hash_sort_simple,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -67264,7 +67267,9 @@ static MY_COLLATION_HANDLER my_collation_ujis_bin_handler =
     my_strcasecmp_mb_bin,
     my_instr_mb,
     my_hash_sort_mb_bin,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -67280,7 +67285,9 @@ static MY_COLLATION_HANDLER my_collation_ujis_japanese_nopad_ci_handler =
     my_strcasecmp_mb,
     my_instr_mb,
     my_hash_sort_simple_nopad,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple_nopad,
+    my_max_str_mb_simple
 };
 
 
@@ -67296,7 +67303,9 @@ static MY_COLLATION_HANDLER my_collation_ujis_nopad_bin_handler =
     my_strcasecmp_mb_bin,
     my_instr_mb,
     my_hash_sort_mb_nopad_bin,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple_nopad,
+    my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c
index ee50d53829e..7fdc88352b0 100644
--- a/strings/ctype-utf8.c
+++ b/strings/ctype-utf8.c
@@ -21,6 +21,7 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-mb.h"
 
 #ifndef EILSEQ
 #define EILSEQ ENOENT
@@ -5366,7 +5367,9 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_general_ci_handler =
     my_strcasecmp_utf8mb3,
     my_instr_mb,
     my_hash_sort_utf8mb3,
-    my_propagate_complex
+    my_propagate_complex,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -5382,7 +5385,9 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_general_mysql500_ci_handler =
     my_strcasecmp_utf8mb3,
     my_instr_mb,
     my_hash_sort_utf8mb3,
-    my_propagate_complex
+    my_propagate_complex,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -5398,7 +5403,9 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_bin_handler =
     my_strcasecmp_mb_bin,
     my_instr_mb,
     my_hash_sort_mb_bin,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -5414,7 +5421,9 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_general_nopad_ci_handler =
   my_strcasecmp_utf8mb3,
   my_instr_mb,
   my_hash_sort_utf8mb3_nopad,
-  my_propagate_complex
+  my_propagate_complex,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -5430,7 +5439,9 @@ static MY_COLLATION_HANDLER my_collation_utf8mb3_nopad_bin_handler =
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -5760,7 +5771,9 @@ static MY_COLLATION_HANDLER my_collation_cs_handler =
     my_strcasecmp_utf8mb3,
     my_instr_mb,
     my_hash_sort_utf8mb3,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 struct charset_info_st my_charset_utf8mb3_general_cs=
@@ -7078,7 +7091,9 @@ static MY_COLLATION_HANDLER my_collation_filename_handler =
     my_strcasecmp_utf8mb3,
     my_instr_mb,
     my_hash_sort_utf8mb3,
-    my_propagate_complex
+    my_propagate_complex,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 static MY_CHARSET_HANDLER my_charset_filename_handler=
@@ -7718,7 +7733,9 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_general_ci_handler=
   my_strcasecmp_utf8mb4,
   my_instr_mb,
   my_hash_sort_utf8mb4,
-  my_propagate_complex
+  my_propagate_complex,
+  my_min_str_mb_simple,
+  my_max_str_mb_simple
 };
 
 
@@ -7734,7 +7751,9 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_bin_handler =
     my_strcasecmp_mb_bin,
     my_instr_mb,
     my_hash_sort_mb_bin,
-    my_propagate_simple
+    my_propagate_simple,
+    my_min_str_mb_simple,
+    my_max_str_mb_simple
 };
 
 
@@ -7750,7 +7769,9 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_general_nopad_ci_handler=
   my_strcasecmp_utf8mb4,
   my_instr_mb,
   my_hash_sort_utf8mb4_nopad,
-  my_propagate_complex
+  my_propagate_complex,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
@@ -7766,7 +7787,9 @@ static MY_COLLATION_HANDLER my_collation_utf8mb4_nopad_bin_handler =
   my_strcasecmp_mb_bin,
   my_instr_mb,
   my_hash_sort_mb_nopad_bin,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_mb_simple_nopad,
+  my_max_str_mb_simple
 };
 
 
diff --git a/strings/ctype-win1250ch.c b/strings/ctype-win1250ch.c
index a53a9768fde..689b20b8936 100644
--- a/strings/ctype-win1250ch.c
+++ b/strings/ctype-win1250ch.c
@@ -42,6 +42,8 @@
 
 #include "strings_def.h"
 #include <m_ctype.h>
+#include "ctype-simple.h"
+
 
 #else
 
@@ -671,7 +673,7 @@ my_like_range_win1250ch(CHARSET_INFO *cs __attribute__((unused)),
 }
 
 
-static MY_COLLATION_HANDLER my_collation_czech_ci_handler =
+static MY_COLLATION_HANDLER my_collation_czech_cs_handler =
 {
   NULL,				/* init */
   my_strnncoll_win1250ch,
@@ -683,11 +685,13 @@ static MY_COLLATION_HANDLER my_collation_czech_ci_handler =
   my_strcasecmp_8bit,
   my_instr_simple,
   my_hash_sort_simple,
-  my_propagate_simple
+  my_propagate_simple,
+  my_min_str_8bit_simple,
+  my_max_str_8bit_simple
 };
 
 
-struct charset_info_st my_charset_cp1250_czech_ci =
+struct charset_info_st my_charset_cp1250_czech_cs =
 {
   34,0,0,                                     /* number    */
   MY_CS_COMPILED|MY_CS_STRNXFRM|MY_CS_CSSORT|
@@ -712,12 +716,12 @@ struct charset_info_st my_charset_cp1250_czech_ci =
   1,				/* mbminlen  */
   1,				/* mbmaxlen  */
   0,				/* min_sort_char */
-  0,				/* max_sort_char */
+  0xFF,                         /* max_sort_char */
   ' ',                          /* pad char      */
   0,                            /* escape_with_backslash_is_dangerous */
   2,                            /* levels_for_order   */
   &my_charset_8bit_handler,
-  &my_collation_czech_ci_handler
+  &my_collation_czech_cs_handler
 };
 
 
diff --git a/unittest/strings/strings-t.c b/unittest/strings/strings-t.c
index 39e48a33627..2141d09c949 100644
--- a/unittest/strings/strings-t.c
+++ b/unittest/strings/strings-t.c
@@ -760,11 +760,264 @@ test_strcollsp()
 }
 
 
+typedef struct
+{
+  size_t size;
+  size_t nchars;
+  LEX_CSTRING min;
+  LEX_CSTRING max;
+} MINMAX_PARAM;
+
+
+static MINMAX_PARAM minmax_param_latin1_swedish_ci[]=
+{
+  {0, 0, {CSTR("")},             {CSTR("")}},
+  {0, 1, {CSTR("")},             {CSTR("")}},
+  {0, 2, {CSTR("")},             {CSTR("")}},
+  {0, 3, {CSTR("")},             {CSTR("")}},
+  {1, 0, {CSTR("")},             {CSTR("")}},
+  {1, 1, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {1, 2, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {1, 3, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {2, 0, {CSTR("")},             {CSTR("")}},
+  {2, 1, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {2, 2, {CSTR("\x00\x00")},     {CSTR("\xFF\xFF")}},
+  {2, 3, {CSTR("\x00\x00")},     {CSTR("\xFF\xFF")}},
+  {3, 0, {CSTR("")},             {CSTR("")}},
+  {3, 1, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {3, 2, {CSTR("\x00\x00")},     {CSTR("\xFF\xFF")}},
+  {3, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xFF\xFF\xFF")}}
+};
+
+
+static MINMAX_PARAM minmax_param_latin1_nopad_bin[]=
+{
+  {0, 0, {CSTR("")},             {CSTR("")}},
+  {0, 1, {CSTR("")},             {CSTR("")}},
+  {0, 2, {CSTR("")},             {CSTR("")}},
+  {0, 3, {CSTR("")},             {CSTR("")}},
+  {1, 0, {CSTR("")},             {CSTR("")}},
+  {1, 1, {CSTR("")},             {CSTR("\xFF")}},
+  {1, 2, {CSTR("")},             {CSTR("\xFF")}},
+  {1, 3, {CSTR("")},             {CSTR("\xFF")}},
+  {2, 0, {CSTR("")},             {CSTR("")}},
+  {2, 1, {CSTR("")},             {CSTR("\xFF")}},
+  {2, 2, {CSTR("")},             {CSTR("\xFF\xFF")}},
+  {2, 3, {CSTR("")},             {CSTR("\xFF\xFF")}},
+  {3, 0, {CSTR("")},             {CSTR("")}},
+  {3, 1, {CSTR("")},             {CSTR("\xFF")}},
+  {3, 2, {CSTR("")},             {CSTR("\xFF\xFF")}},
+  {3, 3, {CSTR("")},             {CSTR("\xFF\xFF\xFF")}}
+};
+
+
+static MINMAX_PARAM minmax_param_utf8mb3_unicode_ci[]=
+{
+  {0, 0, {CSTR("")},             {CSTR("")}},
+  {0, 1, {CSTR("")},             {CSTR("")}},
+  {0, 2, {CSTR("")},             {CSTR("")}},
+  {0, 3, {CSTR("")},             {CSTR("")}},
+  {1, 0, {CSTR("")},             {CSTR("")}},
+  {1, 1, {CSTR("\x09")},         {CSTR("")}},
+  {1, 2, {CSTR("\x09")},         {CSTR("")}},
+  {1, 3, {CSTR("\x09")},         {CSTR("")}},
+  {2, 0, {CSTR("")},             {CSTR("")}},
+  {2, 1, {CSTR("\x09")},         {CSTR("")}},
+  {2, 2, {CSTR("\x09\x09")},     {CSTR("")}},
+  {2, 3, {CSTR("\x09\x09")},     {CSTR("")}},
+  {3, 0, {CSTR("")},             {CSTR("")}},
+  {3, 1, {CSTR("\x09")},         {CSTR("\xEF\xBF\xBF")}},
+  {3, 2, {CSTR("\x09\x09")},     {CSTR("\xEF\xBF\xBF")}},
+  {3, 3, {CSTR("\x09\x09\x09")}, {CSTR("\xEF\xBF\xBF")}},
+  {4, 0, {CSTR("")},             {CSTR("")}},
+  {4, 1, {CSTR("\x09")},         {CSTR("\xEF\xBF\xBF")}},
+  {4, 2, {CSTR("\x09\x09")},     {CSTR("\xEF\xBF\xBF")}},
+  {4, 3, {CSTR("\x09\x09\x09")}, {CSTR("\xEF\xBF\xBF")}},
+  {5, 0, {CSTR("")},             {CSTR("")}},
+  {5, 1, {CSTR("\x09")},         {CSTR("\xEF\xBF\xBF")}},
+  {5, 2, {CSTR("\x09\x09")},     {CSTR("\xEF\xBF\xBF")}},
+  {5, 3, {CSTR("\x09\x09\x09")}, {CSTR("\xEF\xBF\xBF")}},
+  {6, 0, {CSTR("")},             {CSTR("")}},
+  {6, 1, {CSTR("\x09")},         {CSTR("\xEF\xBF\xBF")}},
+  {6, 2, {CSTR("\x09\x09")},     {CSTR("\xEF\xBF\xBF\xEF\xBF\xBF")}},
+  {6, 3, {CSTR("\x09\x09\x09")}, {CSTR("\xEF\xBF\xBF\xEF\xBF\xBF")}},
+  {7, 0, {CSTR("")},             {CSTR("")}},
+  {7, 1, {CSTR("\x09")},         {CSTR("\xEF\xBF\xBF")}},
+  {7, 2, {CSTR("\x09\x09")},     {CSTR("\xEF\xBF\xBF\xEF\xBF\xBF")}},
+  {7, 3, {CSTR("\x09\x09\x09")}, {CSTR("\xEF\xBF\xBF\xEF\xBF\xBF")}},
+  {8, 0, {CSTR("")},             {CSTR("")}},
+  {8, 1, {CSTR("\x09")},         {CSTR("\xEF\xBF\xBF")}},
+  {8, 2, {CSTR("\x09\x09")},     {CSTR("\xEF\xBF\xBF\xEF\xBF\xBF")}},
+  {8, 3, {CSTR("\x09\x09\x09")}, {CSTR("\xEF\xBF\xBF\xEF\xBF\xBF")}},
+  {9, 0, {CSTR("")},             {CSTR("")}},
+  {9, 1, {CSTR("\x09")},         {CSTR("\xEF\xBF\xBF")}},
+  {9, 2, {CSTR("\x09\x09")},     {CSTR("\xEF\xBF\xBF\xEF\xBF\xBF")}},
+  {9, 3, {CSTR("\x09\x09\x09")}, {CSTR("\xEF\xBF\xBF\xEF\xBF\xBF\xEF\xBF\xBF")}},
+};
+
+
+#ifdef HAVE_CHARSET_big5
+static MINMAX_PARAM minmax_param_big5_chinese_ci[]=
+{
+  {0, 0, {CSTR("")},             {CSTR("")}},
+  {0, 1, {CSTR("")},             {CSTR("")}},
+  {0, 2, {CSTR("")},             {CSTR("")}},
+  {0, 3, {CSTR("")},             {CSTR("")}},
+  {1, 0, {CSTR("")},             {CSTR("")}},
+  {1, 1, {CSTR("\x00")},         {CSTR("")}},
+  {1, 2, {CSTR("\x00")},         {CSTR("")}},
+  {1, 3, {CSTR("\x00")},         {CSTR("")}},
+  {2, 0, {CSTR("")},             {CSTR("")}},
+  {2, 1, {CSTR("\x00")},         {CSTR("\xF9\xD5")}},
+  {2, 2, {CSTR("\x00\x00")},     {CSTR("\xF9\xD5")}},
+  {2, 3, {CSTR("\x00\x00")},     {CSTR("\xF9\xD5")}},
+  {3, 0, {CSTR("")},             {CSTR("")}},
+  {3, 1, {CSTR("\x00")},         {CSTR("\xF9\xD5")}},
+  {3, 2, {CSTR("\x00\x00")},     {CSTR("\xF9\xD5")}},
+  {3, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xF9\xD5")}},
+  {4, 0, {CSTR("")},             {CSTR("")}},
+  {4, 1, {CSTR("\x00")},         {CSTR("\xF9\xD5")}},
+  {4, 2, {CSTR("\x00\x00")},     {CSTR("\xF9\xD5\xF9\xD5")}},
+  {4, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xF9\xD5\xF9\xD5")}},
+  {5, 0, {CSTR("")},             {CSTR("")}},
+  {5, 1, {CSTR("\x00")},         {CSTR("\xF9\xD5")}},
+  {5, 2, {CSTR("\x00\x00")},     {CSTR("\xF9\xD5\xF9\xD5")}},
+  {5, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xF9\xD5\xF9\xD5")}},
+  {6, 0, {CSTR("")},             {CSTR("")}},
+  {6, 1, {CSTR("\x00")},         {CSTR("\xF9\xD5")}},
+  {6, 2, {CSTR("\x00\x00")},     {CSTR("\xF9\xD5\xF9\xD5")}},
+  {6, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xF9\xD5\xF9\xD5\xF9\xD5")}},
+  {7, 0, {CSTR("")},             {CSTR("")}},
+  {7, 1, {CSTR("\x00")},         {CSTR("\xF9\xD5")}},
+  {7, 2, {CSTR("\x00\x00")},     {CSTR("\xF9\xD5\xF9\xD5")}},
+  {7, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xF9\xD5\xF9\xD5\xF9\xD5")}},
+  {8, 0, {CSTR("")},             {CSTR("")}},
+  {8, 1, {CSTR("\x00")},         {CSTR("\xF9\xD5")}},
+  {8, 2, {CSTR("\x00\x00")},     {CSTR("\xF9\xD5\xF9\xD5")}},
+  {8, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xF9\xD5\xF9\xD5\xF9\xD5")}},
+  {9, 0, {CSTR("")},             {CSTR("")}},
+  {9, 1, {CSTR("\x00")},         {CSTR("\xF9\xD5")}},
+  {9, 2, {CSTR("\x00\x00")},     {CSTR("\xF9\xD5\xF9\xD5")}},
+  {9, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xF9\xD5\xF9\xD5\xF9\xD5")}},
+};
+#endif
+
+#ifdef HAVE_CHARSET_cp1250
+static MINMAX_PARAM minmax_param_cp1250_czech_cs[]=
+{
+  {0, 0, {CSTR("")},             {CSTR("")}},
+  {0, 1, {CSTR("")},             {CSTR("")}},
+  {0, 2, {CSTR("")},             {CSTR("")}},
+  {0, 3, {CSTR("")},             {CSTR("")}},
+  {1, 0, {CSTR("")},             {CSTR("")}},
+  {1, 1, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {1, 2, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {1, 3, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {2, 0, {CSTR("")},             {CSTR("")}},
+  {2, 1, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {2, 2, {CSTR("\x00\x00")},     {CSTR("\xFF\xFF")}},
+  {2, 3, {CSTR("\x00\x00")},     {CSTR("\xFF\xFF")}},
+  {3, 0, {CSTR("")},             {CSTR("")}},
+  {3, 1, {CSTR("\x00")},         {CSTR("\xFF")}},
+  {3, 2, {CSTR("\x00\x00")},     {CSTR("\xFF\xFF")}},
+  {3, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xFF\xFF\xFF")}}
+};
+#endif
+
+
+#ifdef HAVE_CHARSET_latin2
+static MINMAX_PARAM minmax_param_latin2_czech_cs[]=
+{
+  {0, 0, {CSTR("")},             {CSTR("")}},
+  {0, 1, {CSTR("")},             {CSTR("")}},
+  {0, 2, {CSTR("")},             {CSTR("")}},
+  {0, 3, {CSTR("")},             {CSTR("")}},
+  {1, 0, {CSTR("")},             {CSTR("")}},
+  {1, 1, {CSTR("\x00")},         {CSTR("\xAE")}},
+  {1, 2, {CSTR("\x00")},         {CSTR("\xAE")}},
+  {1, 3, {CSTR("\x00")},         {CSTR("\xAE")}},
+  {2, 0, {CSTR("")},             {CSTR("")}},
+  {2, 1, {CSTR("\x00")},         {CSTR("\xAE")}},
+  {2, 2, {CSTR("\x00\x00")},     {CSTR("\xAE\xAE")}},
+  {2, 3, {CSTR("\x00\x00")},     {CSTR("\xAE\xAE")}},
+  {3, 0, {CSTR("")},             {CSTR("")}},
+  {3, 1, {CSTR("\x00")},         {CSTR("\xAE")}},
+  {3, 2, {CSTR("\x00\x00")},     {CSTR("\xAE\xAE")}},
+  {3, 3, {CSTR("\x00\x00\x00")}, {CSTR("\xAE\xAE\xAE")}}
+};
+#endif
+
+
+static int test_minmax_str_one(CHARSET_INFO *cs,
+                               const MINMAX_PARAM *params, size_t count)
+{
+  size_t i;
+  int failed_total= 0;
+  for (i= 0; i < count; i++)
+  {
+    int failed;
+    char min[32], hmin[64];
+    char max[32], hmax[64];
+    const MINMAX_PARAM *prm= &params[i];
+    size_t minlen= cs->coll->min_str(cs, (uchar *) min, prm->size,
+                                                        prm->nchars);
+    size_t maxlen= cs->coll->max_str(cs, (uchar *) max, prm->size,
+                                                        prm->nchars);
+    failed= minlen != prm->min.length || memcmp(min, prm->min.str, minlen) ||
+            maxlen != prm->max.length || memcmp(max, prm->max.str, maxlen);
+
+    str2hex(hmin, sizeof(hmin), min, minlen);
+    str2hex(hmax, sizeof(hmax), max, maxlen);
+    diag("%-32s %2d %2d   %-10s   %-10s%s",
+         cs->coll_name.str, (int) prm->size, (int) prm->nchars, hmin, hmax,
+         failed ? " FAILED" : "");
+    if (failed)
+    {
+      str2hex(hmin, sizeof(hmin), prm->min.str, prm->min.length);
+      str2hex(hmax, sizeof(hmax), prm->max.str, prm->max.length);
+      diag("%-40s %-10s   %-10s EXPECTED", cs->coll_name.str, hmin, hmax);
+    }
+    failed_total+= failed;
+  }
+  return failed_total;
+}
+
+
+static int test_minmax_str()
+{
+  int failed= 0;
+  failed+= test_minmax_str_one(&my_charset_latin1_nopad_bin,
+                               minmax_param_latin1_nopad_bin,
+                               array_elements(minmax_param_latin1_nopad_bin));
+  failed+= test_minmax_str_one(&my_charset_latin1,
+                               minmax_param_latin1_swedish_ci,
+                               array_elements(minmax_param_latin1_swedish_ci));
+  failed+= test_minmax_str_one(&my_charset_utf8mb3_unicode_ci,
+                               minmax_param_utf8mb3_unicode_ci,
+                               array_elements(minmax_param_utf8mb3_unicode_ci));
+#ifdef HAVE_CHARSET_big5
+  failed+= test_minmax_str_one(&my_charset_big5_chinese_ci,
+                               minmax_param_big5_chinese_ci,
+                               array_elements(minmax_param_big5_chinese_ci));
+#endif
+#ifdef HAVE_CHARSET_cp1250
+  failed+= test_minmax_str_one(&my_charset_cp1250_czech_cs,
+                               minmax_param_cp1250_czech_cs,
+                               array_elements(minmax_param_cp1250_czech_cs));
+#endif
+#ifdef HAVE_CHARSET_latin2
+  failed+= test_minmax_str_one(&my_charset_latin2_czech_cs,
+                               minmax_param_latin2_czech_cs,
+                               array_elements(minmax_param_latin2_czech_cs));
+#endif
+  return failed;
+}
+
 int main()
 {
   size_t i, failed= 0;
   
-  plan(2);
+  plan(3);
   diag("Testing my_like_range_xxx() functions");
   
   for (i= 0; i < array_elements(charset_list); i++)
@@ -782,5 +1035,9 @@ int main()
   failed= test_strcollsp();
   ok(failed == 0, "Testing my_ci_strnncollsp()");
 
+  diag("Testing min_str() and max_str()");
+  failed= test_minmax_str();
+  ok(failed == 0, "Testing min_str() and max_str() functions");
+
   return exit_status();
 }

From 1f099418b6cce00b72f601288735bd7454d037cd Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Mon, 27 Sep 2021 17:43:36 +0200
Subject: [PATCH 18/31] MDEV-20699 mysqldump of routines causes MariaDB to get
 killed by oom-killer

The reason for this behavior is that SP get cached, per connection.
The stored_program_cache is size of this cache, which amounts to 256
routines by default. A compiled stored procedure can easily be several
megabytes in size. Thus calling SHOW CREATE PROCEDURE for all stored
procedures, like mysqldump does, can require significant amount of memory.

Fixed by bypassing the cache for "SHOW CREATE". This should normally be
fine also perfomance-wise, as cache is meant to be used for repeated
execution, not repeated SHOW CREATEs.

Added a test to verify that CREATE PROCEDURE + SHOW CREATE PROCEURE do not
cache, i.e amount of allocated memory does not change.

Note, there is a change in existing behavior in an edge case :
If "SHOW CREATE PROCEDURE p1" called from p1, after p1 was altered, now
this will now return altered code. Previour behavior - relied on caching
and would return old code. The previous behavior might was not necessarily
correct.
---
 mysql-test/main/sp-bugs.result |  4 ++--
 mysql-test/main/sp-lock.result |  8 +-------
 mysql-test/main/sp-lock.test   |  3 ---
 mysql-test/main/sp.result      | 22 ++++++++++++++++++++++
 mysql-test/main/sp.test        | 17 +++++++++++++++++
 sql/sp.cc                      | 16 +++++++---------
 6 files changed, 49 insertions(+), 21 deletions(-)

diff --git a/mysql-test/main/sp-bugs.result b/mysql-test/main/sp-bugs.result
index f88b3b137d3..d758981b6ba 100644
--- a/mysql-test/main/sp-bugs.result
+++ b/mysql-test/main/sp-bugs.result
@@ -182,7 +182,7 @@ RETURN 0;
 END	latin1	latin1_swedish_ci	latin1_swedish_ci
 SHOW CREATE FUNCTION TESTF_bug11763507;
 Function	sql_mode	Create Function	character_set_client	collation_connection	Database Collation
-testf_bug11763507		CREATE DEFINER=`root`@`localhost` FUNCTION `testf_bug11763507`() RETURNS int(11)
+TESTF_bug11763507		CREATE DEFINER=`root`@`localhost` FUNCTION `TESTF_bug11763507`() RETURNS int(11)
 BEGIN
 RETURN 0;
 END	latin1	latin1_swedish_ci	latin1_swedish_ci
@@ -212,7 +212,7 @@ SELECT "PROCEDURE testp_bug11763507";
 END	latin1	latin1_swedish_ci	latin1_swedish_ci
 SHOW CREATE PROCEDURE TESTP_bug11763507;
 Procedure	sql_mode	Create Procedure	character_set_client	collation_connection	Database Collation
-testp_bug11763507		CREATE DEFINER=`root`@`localhost` PROCEDURE `testp_bug11763507`()
+TESTP_bug11763507		CREATE DEFINER=`root`@`localhost` PROCEDURE `TESTP_bug11763507`()
 BEGIN
 SELECT "PROCEDURE testp_bug11763507";
 END	latin1	latin1_swedish_ci	latin1_swedish_ci
diff --git a/mysql-test/main/sp-lock.result b/mysql-test/main/sp-lock.result
index acf951f6676..ec8d8970ae3 100644
--- a/mysql-test/main/sp-lock.result
+++ b/mysql-test/main/sp-lock.result
@@ -703,9 +703,6 @@ connection default;
 #
 # SHOW CREATE PROCEDURE p1 called from p1, after p1 was altered
 #
-# We are just covering the existing behaviour with tests. The
-# results are not necessarily correct."
-#
 CREATE PROCEDURE p1()
 BEGIN
 SELECT get_lock("test", 10);
@@ -736,10 +733,7 @@ get_lock("test", 10)
 1
 Procedure	sql_mode	Create Procedure	character_set_client	collation_connection	Database Collation
 p1	STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION	CREATE DEFINER=`root`@`localhost` PROCEDURE `p1`()
-BEGIN
-SELECT get_lock("test", 10);
-SHOW CREATE PROCEDURE p1;
-END	latin1	latin1_swedish_ci	latin1_swedish_ci
+BEGIN END	latin1	latin1_swedish_ci	latin1_swedish_ci
 connection con3;
 disconnect con3;
 connection con2;
diff --git a/mysql-test/main/sp-lock.test b/mysql-test/main/sp-lock.test
index 83ea07d4bda..703c59e8f28 100644
--- a/mysql-test/main/sp-lock.test
+++ b/mysql-test/main/sp-lock.test
@@ -807,9 +807,6 @@ connection default;
 --echo #
 --echo # SHOW CREATE PROCEDURE p1 called from p1, after p1 was altered
 --echo #
---echo # We are just covering the existing behaviour with tests. The
---echo # results are not necessarily correct."
---echo #
 
 delimiter |;
 CREATE PROCEDURE p1()
diff --git a/mysql-test/main/sp.result b/mysql-test/main/sp.result
index 6510eaf77f5..694c7bcd20e 100644
--- a/mysql-test/main/sp.result
+++ b/mysql-test/main/sp.result
@@ -8893,4 +8893,26 @@ ERROR 42000: You have an error in your SQL syntax; check the manual that corresp
 BEGIN
 RETURN '';
 END' at line 2
+SELECT VARIABLE_VALUE into @global_mem_used FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+SELECT VARIABLE_VALUE into @local_mem_used FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+CREATE PROCEDURE sp0() SELECT 1;
+SHOW CREATE PROCEDURE sp0;
+Procedure	sql_mode	Create Procedure	character_set_client	collation_connection	Database Collation
+sp0	STRICT_ALL_TABLES	CREATE DEFINER=`root`@`localhost` PROCEDURE `sp0`()
+SELECT 1	latin1	latin1_swedish_ci	latin1_swedish_ci
+DROP PROCEDURE sp0;
+SELECT VARIABLE_VALUE into @global_mem_used FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+SELECT VARIABLE_VALUE into @local_mem_used FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+CREATE PROCEDURE sp1() SELECT 1;
+SHOW CREATE PROCEDURE sp1;
+Procedure	sql_mode	Create Procedure	character_set_client	collation_connection	Database Collation
+sp1	STRICT_ALL_TABLES	CREATE DEFINER=`root`@`localhost` PROCEDURE `sp1`()
+SELECT 1	latin1	latin1_swedish_ci	latin1_swedish_ci
+SELECT VARIABLE_VALUE-@local_mem_used FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+VARIABLE_VALUE-@local_mem_used
+0
+SELECT VARIABLE_VALUE-@global_mem_used FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+VARIABLE_VALUE-@global_mem_used
+0
+DROP PROCEDURE sp1;
 # End of 10.3 tests
diff --git a/mysql-test/main/sp.test b/mysql-test/main/sp.test
index d6fba77b854..50ae78c8431 100644
--- a/mysql-test/main/sp.test
+++ b/mysql-test/main/sp.test
@@ -10431,4 +10431,21 @@ END;
 $$
 DELIMITER ;$$
 
+# MDEV-20699 do not cache SP in SHOW CREATE
+# Warmup round, this might allocate some memory for session variable
+# and the output
+SELECT VARIABLE_VALUE into @global_mem_used FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+SELECT VARIABLE_VALUE into @local_mem_used FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+CREATE PROCEDURE sp0() SELECT 1;
+SHOW CREATE PROCEDURE sp0;
+DROP PROCEDURE sp0;
+
+#Check that CREATE/SHOW does not use memory in caches.
+SELECT VARIABLE_VALUE into @global_mem_used FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+SELECT VARIABLE_VALUE into @local_mem_used FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+CREATE PROCEDURE sp1() SELECT 1;
+SHOW CREATE PROCEDURE sp1;
+SELECT VARIABLE_VALUE-@local_mem_used FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+SELECT VARIABLE_VALUE-@global_mem_used FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='MEMORY_USED';
+DROP PROCEDURE sp1;
 --echo # End of 10.3 tests
diff --git a/sql/sp.cc b/sql/sp.cc
index a4c4ca58414..fdfd9a79fef 100644
--- a/sql/sp.cc
+++ b/sql/sp.cc
@@ -1890,8 +1890,6 @@ bool
 Sp_handler::sp_show_create_routine(THD *thd,
                                    const Database_qualified_name *name) const
 {
-  sp_head *sp;
-
   DBUG_ENTER("sp_show_create_routine");
   DBUG_PRINT("enter", ("type: %s name: %.*s",
                        type_str(),
@@ -1904,20 +1902,20 @@ Sp_handler::sp_show_create_routine(THD *thd,
     It is "safe" to do as long as it doesn't affect the results
     of the binary log or the query cache, which currently it does not.
   */
-  if (sp_cache_routine(thd, name, false, &sp))
-    DBUG_RETURN(TRUE);
-
-  if (sp == NULL || sp->show_create_routine(thd, this))
+  sp_head *sp= 0;
+  bool free_sp= db_find_routine(thd, name, &sp) == SP_OK;
+  bool ret= !sp || sp->show_create_routine(thd, this);
+  if (ret)
   {
     /*
       If we have insufficient privileges, pretend the routine
       does not exist.
     */
     my_error(ER_SP_DOES_NOT_EXIST, MYF(0), type_str(), name->m_name.str);
-    DBUG_RETURN(TRUE);
   }
-
-  DBUG_RETURN(FALSE);
+  if (free_sp)
+    sp_head::destroy(sp);
+  DBUG_RETURN(ret);
 }
 
 

From 35f59bc4e1ad8b54c080932387c582bf86b5a77f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 28 Sep 2021 17:17:59 +0300
Subject: [PATCH 19/31] MDEV-26467: More cache friendliness

srw_mutex_impl<bool>::wait_and_lock(): In
commit a73eedbf3fabd19ca7183b738056c30e3f7bbe35 we introduced
an std::atomic::fetch_or() in a loop. Alas, on the IA-32 and AMD64,
that was being translated into a loop around LOCK CMPXCHG.
To avoid a nested loop, it is better to explicitly invoke
std::atomic::compare_exchange_weak() in the loop, but only if
the attempt has a chance to succeed (the HOLDER flag is not set).

It is even more efficient to use LOCK BTS, but contemporary compilers
fail to translate std::atomic::fetch_or(x) & x into that when x is
a single-bit constant. On GCC-compatible compilers, we will use
inline assembler to achieve that.

On other ISA than IA-32 and AMD64, we will continue to use
std::atomic::fetch_or().

ssux_lock_impl<spinloop>::rd_wait(): Use rd_lock_try().
A loop around std::atomic::compare_exchange_weak() should be
cheaper than fetch_add(), fetch_sub() and a wakeup system call.

These deficiencies were pointed out and the use of LOCK BTS was
suggested by Thiago Macieira.
---
 storage/innobase/sync/srw_lock.cc | 101 +++++++++++++++++++++---------
 1 file changed, 72 insertions(+), 29 deletions(-)

diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc
index f3c0afd64e3..542037bc0a2 100644
--- a/storage/innobase/sync/srw_lock.cc
+++ b/storage/innobase/sync/srw_lock.cc
@@ -294,57 +294,107 @@ void srw_mutex_impl<true>::wait_and_lock()
     DBUG_ASSERT(~HOLDER & lk);
     if (lk & HOLDER)
       lk= lock.load(std::memory_order_relaxed);
+#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+#elif defined _M_IX86||defined _M_X64||defined __i386__||defined __x86_64__
+    else if (lock.compare_exchange_weak(lk, lk | HOLDER,
+                                        std::memory_order_acquire,
+                                        std::memory_order_relaxed))
+      return;
+#else
+    else if (!((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) &
+               HOLDER))
+      goto acquired;
+#endif
     else
     {
-      lk= lock.fetch_or(HOLDER, std::memory_order_relaxed);
-      if (!(lk & HOLDER))
-        goto acquired;
+#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+      static_assert(HOLDER == (1U << 31), "compatibility");
+      __asm__ goto("lock btsl $31, %0\n\t"
+                   "jnc %l1" : : "m" (*this) : "cc", "memory" : acquired);
+      lk|= HOLDER;
+#endif
+      srw_pause(delay);
     }
-    srw_pause(delay);
     if (!--spin)
       break;
   }
 
-  for (;; wait(lk))
+  for (;;)
   {
+    DBUG_ASSERT(~HOLDER & lk);
     if (lk & HOLDER)
     {
+      wait(lk);
+#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+reload:
+#endif
       lk= lock.load(std::memory_order_relaxed);
-      if (lk & HOLDER)
-        continue;
     }
-    lk= lock.fetch_or(HOLDER, std::memory_order_relaxed);
-    if (!(lk & HOLDER))
+#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+    else
+    {
+      static_assert(HOLDER == (1U << 31), "compatibility");
+      __asm__ goto("lock btsl $31, %0\n\t"
+                   "jc %l1" : : "m" (*this) : "cc", "memory" : reload);
+acquired:
+      std::atomic_thread_fence(std::memory_order_acquire);
+      return;
+    }
+#elif defined _M_IX86||defined _M_X64||defined __i386__||defined __x86_64__
+    else if (lock.compare_exchange_weak(lk, lk | HOLDER,
+                                        std::memory_order_acquire,
+                                        std::memory_order_relaxed))
+      return;
+#else
+    else if (!((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) &
+               HOLDER))
     {
 acquired:
       DBUG_ASSERT(lk);
       std::atomic_thread_fence(std::memory_order_acquire);
       return;
     }
-    DBUG_ASSERT(lk > HOLDER);
+#endif
   }
 }
 
 template<>
 void srw_mutex_impl<false>::wait_and_lock()
 {
-  uint32_t lk= 1 + lock.fetch_add(1, std::memory_order_relaxed);
-  for (;; wait(lk))
+  for (uint32_t lk= 1 + lock.fetch_add(1, std::memory_order_relaxed);;)
   {
+    DBUG_ASSERT(~HOLDER & lk);
     if (lk & HOLDER)
     {
+      wait(lk);
+#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+reload:
+#endif
       lk= lock.load(std::memory_order_relaxed);
-      if (lk & HOLDER)
-        continue;
     }
-    lk= lock.fetch_or(HOLDER, std::memory_order_relaxed);
-    if (!(lk & HOLDER))
+#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+    else
+    {
+      static_assert(HOLDER == (1U << 31), "compatibility");
+      __asm__ goto("lock btsl $31, %0\n\t"
+                   "jc %l1" : : "m" (*this) : "cc", "memory" : reload);
+      std::atomic_thread_fence(std::memory_order_acquire);
+      return;
+    }
+#elif defined _M_IX86||defined _M_X64||defined __i386__||defined __x86_64__
+    else if (lock.compare_exchange_weak(lk, lk | HOLDER,
+                                        std::memory_order_acquire,
+                                        std::memory_order_relaxed))
+      return;
+#else
+    else if (!((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) &
+               HOLDER))
     {
       DBUG_ASSERT(lk);
       std::atomic_thread_fence(std::memory_order_acquire);
       return;
     }
-    DBUG_ASSERT(lk > HOLDER);
+#endif
   }
 }
 
@@ -373,19 +423,12 @@ void ssux_lock_impl<spinloop>::rd_wait()
   for (;;)
   {
     writer.wr_lock();
-    uint32_t lk= readers.fetch_add(1, std::memory_order_acquire);
-    if (UNIV_UNLIKELY(lk == WRITER))
-    {
-      readers.fetch_sub(1, std::memory_order_relaxed);
-      wake();
-      writer.wr_unlock();
-      pthread_yield();
-      continue;
-    }
-    DBUG_ASSERT(!(lk & WRITER));
-    break;
+    bool acquired= rd_lock_try();
+    writer.wr_unlock();
+    if (acquired)
+      break;
+    std::this_thread::yield();
   }
-  writer.wr_unlock();
 }
 
 template void ssux_lock_impl<true>::rd_wait();

From d0d4ade918a0462cd6468e6999209071a865f188 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 28 Sep 2021 17:19:06 +0300
Subject: [PATCH 20/31] MDEV-26467: Universally implement spin loop

Previously, neither our wrapper of Microsoft Windows SRWLOCK
nor the futex-less implementation SUX_LOCK_GENERIC supported spin loops.

This was suggested by Vladislav Vaintroub.
---
 storage/innobase/include/srw_lock.h | 105 +++++++++++++++-------------
 storage/innobase/sync/srw_lock.cc   |  57 ++++++++++++---
 2 files changed, 104 insertions(+), 58 deletions(-)

diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h
index b24e0a30857..308f3c0a68e 100644
--- a/storage/innobase/include/srw_lock.h
+++ b/storage/innobase/include/srw_lock.h
@@ -22,17 +22,24 @@ this program; if not, write to the Free Software Foundation, Inc.,
 
 #ifdef SUX_LOCK_GENERIC
 /** An exclusive-only variant of srw_lock */
-class srw_mutex final
+template<bool spinloop>
+class srw_mutex_impl final
 {
   pthread_mutex_t lock;
+  void wr_wait();
 public:
   void init() { pthread_mutex_init(&lock, nullptr); }
   void destroy() { pthread_mutex_destroy(&lock); }
-  void wr_lock() { pthread_mutex_lock(&lock); }
+  inline void wr_lock();
   void wr_unlock() { pthread_mutex_unlock(&lock); }
   bool wr_lock_try() { return !pthread_mutex_trylock(&lock); }
 };
-typedef srw_mutex srw_spin_mutex;
+
+template<> void srw_mutex_impl<true>::wr_wait();
+template<>
+inline void srw_mutex_impl<false>::wr_lock() { pthread_mutex_lock(&lock); }
+template<>
+inline void srw_mutex_impl<true>::wr_lock() { if (!wr_lock_try()) wr_wait(); }
 #else
 /** Futex-based mutex */
 template<bool spinloop>
@@ -81,15 +88,12 @@ public:
     }
   }
 };
+#endif
 
 typedef srw_mutex_impl<true> srw_spin_mutex;
 typedef srw_mutex_impl<false> srw_mutex;
-#endif
 
-# if defined _WIN32 || defined SUX_LOCK_GENERIC
-# else
-template<bool spinlock> class srw_lock_impl;
-#endif
+template<bool spinloop> class srw_lock_impl;
 
 /** Slim shared-update-exclusive lock with no recursion */
 template<bool spinloop>
@@ -265,45 +269,54 @@ public:
 #endif
 };
 
-#ifdef _WIN32
+#if defined _WIN32 || defined SUX_LOCK_GENERIC
 /** Slim read-write lock */
-class srw_lock_low
+template<bool spinloop>
+class srw_lock_
 {
 # ifdef UNIV_PFS_RWLOCK
-  friend class srw_lock_impl;
+  friend srw_lock_impl<spinloop>;
 # endif
+# ifdef _WIN32
   SRWLOCK lock;
+# else
+  rw_lock_t lock;
+# endif
+
+  void rd_wait();
+  void wr_wait();
 public:
-  void init() {}
-  void destroy() {}
-  void rd_lock() { AcquireSRWLockShared(&lock); }
-  bool rd_lock_try() { return TryAcquireSRWLockShared(&lock); }
-  void rd_unlock() { ReleaseSRWLockShared(&lock); }
-  void wr_lock() { AcquireSRWLockExclusive(&lock); }
-  bool wr_lock_try() { return TryAcquireSRWLockExclusive(&lock); }
-  void wr_unlock() { ReleaseSRWLockExclusive(&lock); }
+  void init() { IF_WIN(,my_rwlock_init(&lock, nullptr)); }
+  void destroy() { IF_WIN(,rwlock_destroy(&lock)); }
+  inline void rd_lock();
+  inline void wr_lock();
+  bool rd_lock_try()
+  { return IF_WIN(TryAcquireSRWLockShared(&lock), !rw_tryrdlock(&lock)); }
+  void rd_unlock()
+  { IF_WIN(ReleaseSRWLockShared(&lock), rw_unlock(&lock)); }
+  bool wr_lock_try()
+  { return IF_WIN(TryAcquireSRWLockExclusive(&lock), !rw_trywrlock(&lock)); }
+  void wr_unlock()
+  { IF_WIN(ReleaseSRWLockExclusive(&lock), rw_unlock(&lock)); }
 };
 
-typedef srw_lock_low srw_spin_lock_low;
-#elif defined SUX_LOCK_GENERIC
-/** Slim read-write lock */
-class srw_lock_low
-{
-# ifdef UNIV_PFS_RWLOCK
-  friend class srw_lock_impl;
-# endif
-  rw_lock_t lock;
-public:
-  void init() { my_rwlock_init(&lock, nullptr); }
-  void destroy() { rwlock_destroy(&lock); }
-  void rd_lock() { rw_rdlock(&lock); }
-  bool rd_lock_try() { return !rw_tryrdlock(&lock); }
-  void rd_unlock() { rw_unlock(&lock); }
-  void wr_lock() { rw_wrlock(&lock); }
-  bool wr_lock_try() { return !rw_trywrlock(&lock); }
-  void wr_unlock() { rw_unlock(&lock); }
-};
-typedef srw_lock_low srw_spin_lock_low;
+template<> void srw_lock_<true>::rd_wait();
+template<> void srw_lock_<true>::wr_wait();
+
+template<>
+inline void srw_lock_<false>::rd_lock()
+{ IF_WIN(AcquireSRWLockShared(&lock), rw_rdlock(&lock)); }
+template<>
+inline void srw_lock_<false>::wr_lock()
+{ IF_WIN(AcquireSRWLockExclusive(&lock), rw_wrlock(&lock)); }
+
+template<>
+inline void srw_lock_<true>::rd_lock() { if (!rd_lock_try()) rd_wait(); }
+template<>
+inline void srw_lock_<true>::wr_lock() { if (!wr_lock_try()) wr_wait(); }
+
+typedef srw_lock_<false> srw_lock_low;
+typedef srw_lock_<true> srw_spin_lock_low;
 #else
 typedef ssux_lock_impl<false> srw_lock_low;
 typedef ssux_lock_impl<true> srw_spin_lock_low;
@@ -398,17 +411,14 @@ public:
 };
 
 /** Slim reader-writer lock with PERFORMANCE_SCHEMA instrumentation */
-# if defined _WIN32 || defined SUX_LOCK_GENERIC
-# else
-template<bool spinlock>
-# endif
+template<bool spinloop>
 class srw_lock_impl
 {
   PSI_rwlock *pfs_psi;
 # if defined _WIN32 || defined SUX_LOCK_GENERIC
-  srw_lock_low lock;
+  srw_lock_<spinloop> lock;
 # else
-  ssux_lock_impl<spinlock> lock;
+  ssux_lock_impl<spinloop> lock;
 # endif
 
   ATTRIBUTE_NOINLINE void psi_rd_lock(const char *file, unsigned line);
@@ -458,12 +468,7 @@ public:
   bool wr_lock_try() { return lock.wr_lock_try(); }
 };
 
-# if defined _WIN32 || defined SUX_LOCK_GENERIC
-typedef srw_lock_impl srw_lock;
-typedef srw_lock_impl srw_spin_lock;
-# else
 typedef srw_lock_impl<false> srw_lock;
 typedef srw_lock_impl<true> srw_spin_lock;
-# endif
 
 #endif
diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc
index 542037bc0a2..44d16983635 100644
--- a/storage/innobase/sync/srw_lock.cc
+++ b/storage/innobase/sync/srw_lock.cc
@@ -36,6 +36,20 @@ static inline void srw_pause(unsigned delay)
 }
 
 #ifdef SUX_LOCK_GENERIC
+template<> void srw_mutex_impl<true>::wr_wait()
+{
+  const unsigned delay= srw_pause_delay();
+
+  for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
+  {
+    srw_pause(delay);
+    if (wr_lock_try())
+      return;
+  }
+
+  pthread_mutex_lock(&lock);
+}
+
 template<bool spinloop>
 void ssux_lock_impl<spinloop>::init()
 {
@@ -247,7 +261,6 @@ inline void ssux_lock_impl<spinloop>::wait(uint32_t lk)
 { WaitOnAddress(&readers, &lk, 4, INFINITE); }
 template<bool spinloop>
 void ssux_lock_impl<spinloop>::wake() { WakeByAddressSingle(&readers); }
-
 # else
 #  ifdef __linux__
 #   include <linux/futex.h>
@@ -435,17 +448,44 @@ template void ssux_lock_impl<true>::rd_wait();
 template void ssux_lock_impl<false>::rd_wait();
 #endif /* SUX_LOCK_GENERIC */
 
+#if defined _WIN32 || defined SUX_LOCK_GENERIC
+template<> void srw_lock_<true>::rd_wait()
+{
+  const unsigned delay= srw_pause_delay();
+
+  for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
+  {
+    srw_pause(delay);
+    if (rd_lock_try())
+      return;
+  }
+
+  IF_WIN(AcquireSRWLockShared(&lock), rw_rdlock(&lock));
+}
+
+template<> void srw_lock_<true>::wr_wait()
+{
+  const unsigned delay= srw_pause_delay();
+
+  for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
+  {
+    srw_pause(delay);
+    if (wr_lock_try())
+      return;
+  }
+
+  IF_WIN(AcquireSRWLockExclusive(&lock), rw_wrlock(&lock));
+}
+#endif
+
 #ifdef UNIV_PFS_RWLOCK
-# if defined _WIN32 || defined SUX_LOCK_GENERIC
-#  define void_srw_lock void srw_lock_impl
-# else
-#  define void_srw_lock template<bool spinloop> void srw_lock_impl<spinloop>
 template void srw_lock_impl<false>::psi_rd_lock(const char*, unsigned);
 template void srw_lock_impl<false>::psi_wr_lock(const char*, unsigned);
 template void srw_lock_impl<true>::psi_rd_lock(const char*, unsigned);
 template void srw_lock_impl<true>::psi_wr_lock(const char*, unsigned);
-# endif
-void_srw_lock::psi_rd_lock(const char *file, unsigned line)
+
+template<bool spinloop>
+void srw_lock_impl<spinloop>::psi_rd_lock(const char *file, unsigned line)
 {
   PSI_rwlock_locker_state state;
   const bool nowait= lock.rd_lock_try();
@@ -461,7 +501,8 @@ void_srw_lock::psi_rd_lock(const char *file, unsigned line)
     lock.rd_lock();
 }
 
-void_srw_lock::psi_wr_lock(const char *file, unsigned line)
+template<bool spinloop>
+void srw_lock_impl<spinloop>::psi_wr_lock(const char *file, unsigned line)
 {
   PSI_rwlock_locker_state state;
   const bool nowait= lock.wr_lock_try();

From e79fa9f542ea362577c78b9b0f4def2fc8d3be4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 28 Sep 2021 17:19:26 +0300
Subject: [PATCH 21/31] MDEV-26467: Actually use spinloop on block_lock

In commit 277ba134ad1c994bda50de574a87a06a071fbecb
we accidentally omitted this.
---
 storage/innobase/include/srw_lock.h |  2 +-
 storage/innobase/sync/srw_lock.cc   | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h
index 308f3c0a68e..3d0d82b81df 100644
--- a/storage/innobase/include/srw_lock.h
+++ b/storage/innobase/include/srw_lock.h
@@ -151,7 +151,7 @@ public:
   void wr_unlock();
 #else
   /** mutex for synchronization; held by U or X lock holders */
-  srw_mutex writer;
+  srw_mutex_impl<spinloop> writer;
   /** S or U holders, and WRITER flag for X holder or waiter */
   std::atomic<uint32_t> readers;
   /** indicates an X request; readers=WRITER indicates granted X lock */
diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc
index 44d16983635..f38847f852b 100644
--- a/storage/innobase/sync/srw_lock.cc
+++ b/storage/innobase/sync/srw_lock.cc
@@ -417,7 +417,23 @@ void ssux_lock_impl<spinloop>::wr_wait(uint32_t lk)
   DBUG_ASSERT(writer.is_locked());
   DBUG_ASSERT(lk);
   DBUG_ASSERT(lk < WRITER);
+
+  if (spinloop)
+  {
+    const unsigned delay= srw_pause_delay();
+
+    for (auto spin= srv_n_spin_wait_rounds; spin; spin--)
+    {
+      srw_pause(delay);
+      lk= readers.load(std::memory_order_acquire);
+      if (lk == WRITER)
+        return;
+      DBUG_ASSERT(lk > WRITER);
+    }
+  }
+
   lk|= WRITER;
+
   do
   {
     DBUG_ASSERT(lk > WRITER);

From 27738bd774a5435026602ef94a3abc492f1f72e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vicen=C8=9Biu=20Ciorbaru?= <cvicentiu@gmail.com>
Date: Tue, 28 Sep 2021 18:14:11 +0300
Subject: [PATCH 22/31] MTR: loops should divide milliseconds by milliseconds

timeout is in seconds, sleeptime is in miliseconds. Otherwise we sleep
for 10 times longer than the timeout.
---
 mysql-test/lib/mtr_process.pl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mysql-test/lib/mtr_process.pl b/mysql-test/lib/mtr_process.pl
index a384aa584c4..42254bd29b7 100644
--- a/mysql-test/lib/mtr_process.pl
+++ b/mysql-test/lib/mtr_process.pl
@@ -109,7 +109,7 @@ sub sleep_until_file_created ($$$$$) {
   my $proc=     shift;
   my $warn_seconds = shift;
   my $sleeptime= 10; # Milliseconds
-  my $loops= ($timeout * 10000) / $sleeptime;
+  my $loops= ($timeout * 1000) / $sleeptime;
   my $message_time= 60;
 
   for ( my $loop= 0; $loop <= $loops; $loop++ )

From e5a9dcfda20e174525ad63a30e12f592aa9c96d3 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Tue, 28 Sep 2021 22:25:47 +0200
Subject: [PATCH 23/31] MDEV-23306 Fix build dependency

---
 libmysqld/CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libmysqld/CMakeLists.txt b/libmysqld/CMakeLists.txt
index c133e5a4378..30ddf9ec1c3 100644
--- a/libmysqld/CMakeLists.txt
+++ b/libmysqld/CMakeLists.txt
@@ -142,6 +142,9 @@ SET(SQL_EMBEDDED_SOURCES emb_qcache.cc libmysqld.c lib_sql.cc
 ADD_CONVENIENCE_LIBRARY(sql_embedded ${SQL_EMBEDDED_SOURCES})
 DTRACE_INSTRUMENT(sql_embedded)
 ADD_DEPENDENCIES(sql_embedded GenError GenServerSource)
+IF(TARGET pcre2)
+  ADD_DEPENDENCIES(sql_embedded pcre2)
+ENDIF()
 
 # On Windows, static embedded server library is called mysqlserver.lib
 # On Unix, it is libmysqld.a

From 309209c51c211d0733a6c7a67fb70a523a1706f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Wed, 29 Sep 2021 10:15:07 +0300
Subject: [PATCH 24/31] MDEV-26467: Use LOCK BTS also with the Microsoft
 compiler

Microsoft does not define inline assembler syntax for AMD64,
but it defines the intrinsic function _interlockedbittestandset()
that we can use.

ssux_lock_impl<bool>::rd_wait(): Remove a call to yield,
because writer.wr_lock() will take care of context switches
between loop iterations.

This addresses suggestions by Vladislav Vaintroub.
---
 storage/innobase/sync/srw_lock.cc | 96 +++++++++++++++----------------
 1 file changed, 47 insertions(+), 49 deletions(-)

diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc
index f38847f852b..7d26dd2b590 100644
--- a/storage/innobase/sync/srw_lock.cc
+++ b/storage/innobase/sync/srw_lock.cc
@@ -295,6 +295,35 @@ template void ssux_lock_impl<false>::wake();
 template void srw_mutex_impl<true>::wake();
 template void ssux_lock_impl<true>::wake();
 
+/*
+
+Unfortunately, compilers targeting IA-32 or AMD64 currently cannot
+translate the following single-bit operations into Intel 80386 instructions:
+
+     m.fetch_or(1<<b) & 1<<b       LOCK BTS b, m
+     m.fetch_and(~(1<<b)) & 1<<b   LOCK BTR b, m
+     m.fetch_xor(1<<b) & 1<<b      LOCK BTC b, m
+
+Hence, we will manually translate fetch_or() using GCC-style inline
+assembler code or a Microsoft intrinsic function.
+
+*/
+#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+# define IF_FETCH_OR_GOTO(mem, bit, label)				\
+  __asm__ goto("lock btsl $" #bit ", %0\n\t"				\
+               "jc %l1" : : "m" (mem) : "cc", "memory" : label);
+# define IF_NOT_FETCH_OR_GOTO(mem, bit, label)				\
+  __asm__ goto("lock btsl $" #bit ", %0\n\t"				\
+               "jnc %l1" : : "m" (mem) : "cc", "memory" : label);
+#elif defined _MSC_VER && (defined _M_IX86 || defined _M_IX64)
+# define IF_FETCH_OR_GOTO(mem, bit, label)				\
+  if (_interlockedbittestandset(reinterpret_cast<volatile long*>(&mem), bit)) \
+    goto label;
+# define IF_NOT_FETCH_OR_GOTO(mem, bit, label)				\
+  if (!_interlockedbittestandset(reinterpret_cast<volatile long*>(&mem), bit))\
+    goto label;
+#endif
+
 template<>
 void srw_mutex_impl<true>::wait_and_lock()
 {
@@ -307,24 +336,15 @@ void srw_mutex_impl<true>::wait_and_lock()
     DBUG_ASSERT(~HOLDER & lk);
     if (lk & HOLDER)
       lk= lock.load(std::memory_order_relaxed);
-#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
-#elif defined _M_IX86||defined _M_X64||defined __i386__||defined __x86_64__
-    else if (lock.compare_exchange_weak(lk, lk | HOLDER,
-                                        std::memory_order_acquire,
-                                        std::memory_order_relaxed))
-      return;
-#else
-    else if (!((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) &
-               HOLDER))
-      goto acquired;
-#endif
     else
     {
-#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+#ifdef IF_NOT_FETCH_OR_GOTO
       static_assert(HOLDER == (1U << 31), "compatibility");
-      __asm__ goto("lock btsl $31, %0\n\t"
-                   "jnc %l1" : : "m" (*this) : "cc", "memory" : acquired);
+      IF_NOT_FETCH_OR_GOTO(*this, 31, acquired);
       lk|= HOLDER;
+#else
+      if (!((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) & HOLDER))
+        goto acquired;
 #endif
       srw_pause(delay);
     }
@@ -338,36 +358,25 @@ void srw_mutex_impl<true>::wait_and_lock()
     if (lk & HOLDER)
     {
       wait(lk);
-#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+#ifdef IF_FETCH_OR_GOTO
 reload:
 #endif
       lk= lock.load(std::memory_order_relaxed);
     }
-#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
     else
     {
+#ifdef IF_FETCH_OR_GOTO
       static_assert(HOLDER == (1U << 31), "compatibility");
-      __asm__ goto("lock btsl $31, %0\n\t"
-                   "jc %l1" : : "m" (*this) : "cc", "memory" : reload);
-acquired:
-      std::atomic_thread_fence(std::memory_order_acquire);
-      return;
-    }
-#elif defined _M_IX86||defined _M_X64||defined __i386__||defined __x86_64__
-    else if (lock.compare_exchange_weak(lk, lk | HOLDER,
-                                        std::memory_order_acquire,
-                                        std::memory_order_relaxed))
-      return;
+      IF_FETCH_OR_GOTO(*this, 31, reload);
 #else
-    else if (!((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) &
-               HOLDER))
-    {
-acquired:
+      if ((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) & HOLDER)
+        continue;
       DBUG_ASSERT(lk);
+#endif
+acquired:
       std::atomic_thread_fence(std::memory_order_acquire);
       return;
     }
-#endif
   }
 }
 
@@ -380,34 +389,24 @@ void srw_mutex_impl<false>::wait_and_lock()
     if (lk & HOLDER)
     {
       wait(lk);
-#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
+#ifdef IF_FETCH_OR_GOTO
 reload:
 #endif
       lk= lock.load(std::memory_order_relaxed);
     }
-#if defined __GNUC__ && (defined __i386__ || defined __x86_64__)
     else
     {
+#ifdef IF_FETCH_OR_GOTO
       static_assert(HOLDER == (1U << 31), "compatibility");
-      __asm__ goto("lock btsl $31, %0\n\t"
-                   "jc %l1" : : "m" (*this) : "cc", "memory" : reload);
-      std::atomic_thread_fence(std::memory_order_acquire);
-      return;
-    }
-#elif defined _M_IX86||defined _M_X64||defined __i386__||defined __x86_64__
-    else if (lock.compare_exchange_weak(lk, lk | HOLDER,
-                                        std::memory_order_acquire,
-                                        std::memory_order_relaxed))
-      return;
+      IF_FETCH_OR_GOTO(*this, 31, reload);
 #else
-    else if (!((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) &
-               HOLDER))
-    {
+      if ((lk= lock.fetch_or(HOLDER, std::memory_order_relaxed)) & HOLDER)
+        continue;
       DBUG_ASSERT(lk);
+#endif
       std::atomic_thread_fence(std::memory_order_acquire);
       return;
     }
-#endif
   }
 }
 
@@ -456,7 +455,6 @@ void ssux_lock_impl<spinloop>::rd_wait()
     writer.wr_unlock();
     if (acquired)
       break;
-    std::this_thread::yield();
   }
 }
 

From b2a5e0f28232b56c5c36e65a457d41d819b279bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Wed, 29 Sep 2021 12:13:11 +0300
Subject: [PATCH 25/31] Make innodb.innodb_defrag_stats more deterministic

Let us mask the actual values of the defragmentation-related fields,
because they may vary. Also, remove the dependency on purge,
and instead delete records by a ROLLBACK of INSERT.
---
 .../suite/innodb/r/innodb_defrag_stats.result | 227 +++++++++---------
 .../suite/innodb/t/innodb_defrag_stats.test   |  76 +++---
 2 files changed, 142 insertions(+), 161 deletions(-)

diff --git a/mysql-test/suite/innodb/r/innodb_defrag_stats.result b/mysql-test/suite/innodb/r/innodb_defrag_stats.result
index 6c5fe1817e2..d1e9e2e78ae 100644
--- a/mysql-test/suite/innodb/r/innodb_defrag_stats.result
+++ b/mysql-test/suite/innodb/r/innodb_defrag_stats.result
@@ -1,123 +1,118 @@
 SET GLOBAL innodb_defragment_stats_accuracy = 20;
 DELETE FROM mysql.innodb_index_stats;
 # Create table.
-CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB;
-INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1_to_1024;
+CREATE TABLE t1 (a INT PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256),
+KEY SECOND(a, b)) ENGINE=INNODB STATS_PERSISTENT=0;
+INSERT INTO t1 SELECT 100*FLOOR(seq/70)+seq%70, REPEAT('A', 256)
+FROM seq_1_to_1024;
 # Not enough page splits to trigger persistent stats write yet.
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-count(stat_value) = 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-count(stat_value) = 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) = 0
-1
-INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1025_to_2048;
-# Persistent stats recorded.
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-count(stat_value) > 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-count(stat_value) = 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) > 0
-1
-# Delete some rows.
+SELECT * FROM mysql.innodb_index_stats;
+database_name	table_name	index_name	last_update	stat_name	stat_value	sample_size	stat_description
+INSERT INTO t1 SELECT 100*FLOOR(seq/70)+seq%70, REPEAT('A', 256)
+FROM seq_1025_to_1433;
 BEGIN;
-delete from t1 where a between 100 * 20 and 100 * 20 + 30;
-delete from t1 where a between 100 * 19 and 100 * 19 + 30;
-delete from t1 where a between 100 * 18 and 100 * 18 + 30;
-delete from t1 where a between 100 * 17 and 100 * 17 + 30;
-delete from t1 where a between 100 * 16 and 100 * 16 + 30;
-delete from t1 where a between 100 * 15 and 100 * 15 + 30;
-delete from t1 where a between 100 * 14 and 100 * 14 + 30;
-delete from t1 where a between 100 * 13 and 100 * 13 + 30;
-delete from t1 where a between 100 * 12 and 100 * 12 + 30;
-delete from t1 where a between 100 * 11 and 100 * 11 + 30;
-delete from t1 where a between 100 * 10 and 100 * 10 + 30;
-delete from t1 where a between 100 * 9 and 100 * 9 + 30;
-delete from t1 where a between 100 * 8 and 100 * 8 + 30;
-delete from t1 where a between 100 * 7 and 100 * 7 + 30;
-delete from t1 where a between 100 * 6 and 100 * 6 + 30;
-delete from t1 where a between 100 * 5 and 100 * 5 + 30;
-delete from t1 where a between 100 * 4 and 100 * 4 + 30;
-delete from t1 where a between 100 * 3 and 100 * 3 + 30;
-delete from t1 where a between 100 * 2 and 100 * 2 + 30;
-delete from t1 where a between 100 * 1 and 100 * 1 + 30;
-COMMIT;
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-count(stat_value) > 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-count(stat_value) = 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) > 0
+INSERT INTO t1 SELECT 100*20+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*19+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*18+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*17+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*16+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*15+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*14+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*13+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*12+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*11+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*10+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*9+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*8+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*7+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*6+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*5+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*4+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*3+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*2+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+INSERT INTO t1 SELECT 100*1+seq, REPEAT('A', 256)
+FROM seq_70_to_99;
+ROLLBACK;
+SELECT @@GLOBAL.innodb_force_recovery<2 "have background defragmentation";
+have background defragmentation
 1
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
+table_name	index_name	stat_name
+t1	PRIMARY	n_leaf_pages_defrag
+t1	PRIMARY	n_leaf_pages_reserved
+t1	PRIMARY	n_page_split
+t1	SECOND	n_leaf_pages_defrag
+t1	SECOND	n_leaf_pages_reserved
+t1	SECOND	n_page_split
 optimize table t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	status	OK
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) > 0
-1
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
+table_name	index_name	stat_name
+t1	PRIMARY	n_leaf_pages_defrag
+t1	PRIMARY	n_leaf_pages_reserved
+t1	PRIMARY	n_page_split
+t1	PRIMARY	n_pages_freed
+t1	SECOND	n_leaf_pages_defrag
+t1	SECOND	n_leaf_pages_reserved
+t1	SECOND	n_page_split
+t1	SECOND	n_pages_freed
 set global innodb_defragment_stats_accuracy = 40;
 INSERT INTO t1 (b) SELECT b from t1;
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) > 0
-1
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
+table_name	index_name	stat_name
+t1	PRIMARY	n_leaf_pages_defrag
+t1	PRIMARY	n_leaf_pages_reserved
+t1	PRIMARY	n_page_split
+t1	PRIMARY	n_pages_freed
+t1	SECOND	n_leaf_pages_defrag
+t1	SECOND	n_leaf_pages_reserved
+t1	SECOND	n_page_split
+t1	SECOND	n_pages_freed
 INSERT INTO t1 (b) SELECT b from t1;
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) > 0
-1
+SELECT stat_name FROM mysql.innodb_index_stats WHERE table_name='t1';
+stat_name
+n_leaf_pages_defrag
+n_leaf_pages_defrag
+n_leaf_pages_reserved
+n_leaf_pages_reserved
+n_page_split
+n_page_split
+n_pages_freed
+n_pages_freed
 # Table rename should cause stats rename.
 rename table t1 to t2;
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-count(stat_value) = 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-count(stat_value) = 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) = 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_page_split');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_pages_freed');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) > 0
-1
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
+table_name	index_name	stat_name
+t2	PRIMARY	n_leaf_pages_defrag
+t2	PRIMARY	n_leaf_pages_reserved
+t2	PRIMARY	n_page_split
+t2	PRIMARY	n_pages_freed
+t2	SECOND	n_leaf_pages_defrag
+t2	SECOND	n_leaf_pages_reserved
+t2	SECOND	n_page_split
+t2	SECOND	n_pages_freed
 # Drop index should cause stats drop, but will not.
 drop index SECOND on t2;
-SELECT stat_name, stat_value>0 FROM mysql.innodb_index_stats
-WHERE table_name like '%t2%' AND index_name='SECOND';
-stat_name	stat_value>0
-n_leaf_pages_defrag	1
-n_leaf_pages_reserved	1
-n_page_split	1
-n_pages_freed	1
 #
 # MDEV-26636: Statistics must not be written for temporary tables
 #
@@ -125,20 +120,18 @@ SET GLOBAL innodb_defragment_stats_accuracy = 1;
 CREATE TEMPORARY TABLE t (a INT PRIMARY KEY, c CHAR(255) NOT NULL)
 ENGINE=InnoDB;
 INSERT INTO t SELECT seq, '' FROM seq_1_to_100;
-SELECT * FROM mysql.innodb_index_stats where table_name like '%t1%';
-database_name	table_name	index_name	last_update	stat_name	stat_value	sample_size	stat_description
-SELECT table_name, index_name, stat_name, stat_value>0
-FROM mysql.innodb_index_stats;
-table_name	index_name	stat_name	stat_value>0
-t2	PRIMARY	n_leaf_pages_defrag	1
-t2	PRIMARY	n_leaf_pages_reserved	1
-t2	PRIMARY	n_page_split	1
-t2	PRIMARY	n_pages_freed	1
-t2	SECOND	n_leaf_pages_defrag	1
-t2	SECOND	n_leaf_pages_reserved	1
-t2	SECOND	n_page_split	1
-t2	SECOND	n_pages_freed	1
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
+table_name	index_name	stat_name
+t2	PRIMARY	n_leaf_pages_defrag
+t2	PRIMARY	n_leaf_pages_reserved
+t2	PRIMARY	n_page_split
+t2	PRIMARY	n_pages_freed
+t2	SECOND	n_leaf_pages_defrag
+t2	SECOND	n_leaf_pages_reserved
+t2	SECOND	n_page_split
+t2	SECOND	n_pages_freed
 # Clean up
+ALTER TABLE t2 STATS_PERSISTENT=1;
 DROP TABLE t2;
 SELECT * FROM mysql.innodb_index_stats;
 database_name	table_name	index_name	last_update	stat_name	stat_value	sample_size	stat_description
diff --git a/mysql-test/suite/innodb/t/innodb_defrag_stats.test b/mysql-test/suite/innodb/t/innodb_defrag_stats.test
index e1e88a07477..799faa93ff0 100644
--- a/mysql-test/suite/innodb/t/innodb_defrag_stats.test
+++ b/mysql-test/suite/innodb/t/innodb_defrag_stats.test
@@ -8,77 +8,65 @@ SET GLOBAL innodb_defragment_stats_accuracy = 20;
 DELETE FROM mysql.innodb_index_stats;
 
 --echo # Create table.
-CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB;
+CREATE TABLE t1 (a INT PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256),
+                 KEY SECOND(a, b)) ENGINE=INNODB STATS_PERSISTENT=0;
 
-INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1_to_1024;
+INSERT INTO t1 SELECT 100*FLOOR(seq/70)+seq%70, REPEAT('A', 256)
+FROM seq_1_to_1024;
 
 --echo # Not enough page splits to trigger persistent stats write yet.
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
+SELECT * FROM mysql.innodb_index_stats;
 
-INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1025_to_2048;
+INSERT INTO t1 SELECT 100*FLOOR(seq/70)+seq%70, REPEAT('A', 256)
+FROM seq_1025_to_1433;
 
---echo # Persistent stats recorded.
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-
---echo # Delete some rows.
 BEGIN;
 let $num_delete = 20;
 while ($num_delete)
 {
-    let $j = 100 * $num_delete;
-    eval delete from t1 where a between $j and $j + 30;
+    eval INSERT INTO t1 SELECT 100*$num_delete+seq, REPEAT('A', 256)
+    FROM seq_70_to_99;
     dec $num_delete;
 }
-COMMIT;
+ROLLBACK;
 
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
+SELECT @@GLOBAL.innodb_force_recovery<2 "have background defragmentation";
+
+# Wait for defrag_pool to be processed.
+
+let $wait_timeout=30;
+let $wait_condition = SELECT COUNT(*)>0 FROM mysql.innodb_index_stats;
+--source include/wait_condition.inc
+
+--sorted_result
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
 
 optimize table t1;
 
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
+--sorted_result
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
 
 set global innodb_defragment_stats_accuracy = 40;
 
 INSERT INTO t1 (b) SELECT b from t1;
 
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-
+--sorted_result
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
 
 INSERT INTO t1 (b) SELECT b from t1;
 
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-
+--sorted_result
+SELECT stat_name FROM mysql.innodb_index_stats WHERE table_name='t1';
 
 --echo # Table rename should cause stats rename.
 rename table t1 to t2;
 
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_page_split');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_pages_freed');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_leaf_pages_defrag');
+--sorted_result
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
 
 --echo # Drop index should cause stats drop, but will not.
 drop index SECOND on t2;
 
---sorted_result
-SELECT stat_name, stat_value>0 FROM mysql.innodb_index_stats
-WHERE table_name like '%t2%' AND index_name='SECOND';
-
 --echo #
 --echo # MDEV-26636: Statistics must not be written for temporary tables
 --echo #
@@ -89,13 +77,13 @@ INSERT INTO t SELECT seq, '' FROM seq_1_to_100;
 
 --source include/restart_mysqld.inc
 
-SELECT * FROM mysql.innodb_index_stats where table_name like '%t1%';
-
 --sorted_result
-SELECT table_name, index_name, stat_name, stat_value>0
-FROM mysql.innodb_index_stats;
+SELECT table_name, index_name, stat_name FROM mysql.innodb_index_stats;
 
 --echo # Clean up
+# Starting with 10.6, DROP TABLE will not touch persistent statistics
+# (not defragmentation statistics either) if the table has none!
+ALTER TABLE t2 STATS_PERSISTENT=1;
 DROP TABLE t2;
 
 SELECT * FROM mysql.innodb_index_stats;

From a1352870a21aceb004d5d4677c5a107e43c589b3 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Wed, 29 Sep 2021 12:29:17 +0200
Subject: [PATCH 26/31] MDEV-26717
 mysql_upgrade_service/mariadb-upgrade-service -avoid slow shutdown

Apparently, slow shutdown is not necessary anymore after MDEV-15912 fix
---
 sql/mysql_upgrade_service.cc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/sql/mysql_upgrade_service.cc b/sql/mysql_upgrade_service.cc
index b3683618e6e..8e2b394594b 100644
--- a/sql/mysql_upgrade_service.cc
+++ b/sql/mysql_upgrade_service.cc
@@ -271,7 +271,7 @@ void stop_mysqld_service()
     }
 
     /*
-      Remeber initial state of the service, we will restore it on
+      Remember initial state of the service, we will restore it on
       exit.
     */
     if(initial_service_state == UINT_MAX)
@@ -492,8 +492,10 @@ int main(int argc, char **argv)
   CopyFile(service_properties.inifile, my_ini_bck, FALSE);
   upgrade_config_file(service_properties.inifile);
 
-  log("Phase %d/%d: Ensuring innodb slow shutdown%s", ++phase, max_phases,
-    old_mysqld_exe_exists?",this can take some time":"(skipped)");
+  bool do_start_stop_server = old_mysqld_exe_exists && initial_service_state != SERVICE_RUNNING;
+
+  log("Phase %d/%d: Start and stop server in the old version, to avoid crash recovery %s", ++phase, max_phases,
+    do_start_stop_server?",this can take some time":"(skipped)");
 
   char socket_param[FN_REFLEN];
   sprintf_s(socket_param, "--socket=mysql_upgrade_service_%d",
@@ -501,11 +503,11 @@ int main(int argc, char **argv)
 
   DWORD start_duration_ms = 0;
 
-  if (old_mysqld_exe_exists)
+  if (do_start_stop_server)
   {
-    /* Start/stop server with  --loose-innodb-fast-shutdown=0 */
+    /* Start/stop server with  --loose-innodb-fast-shutdown=1 */
     mysqld_process = (HANDLE)run_tool(P_NOWAIT, service_properties.mysqld_exe,
-      defaults_file_param, "--loose-innodb-fast-shutdown=0", "--skip-networking",
+      defaults_file_param, "--loose-innodb-fast-shutdown=1", "--skip-networking",
       "--enable-named-pipe", socket_param, "--skip-slave-start", NULL);
 
     if (mysqld_process == INVALID_HANDLE_VALUE)

From 4e9366df7b097ae38db8ead75a4f4e5d58ad8dca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Wed, 29 Sep 2021 14:50:38 +0300
Subject: [PATCH 27/31] MDEV-26672 test fixup

Occasionally, after restart, additional transactions will have been
executed, possibly related to innodb_stats_auto_recalc.

We should only care that the transaction ID sequence does
not go backwards.
---
 mysql-test/suite/innodb/t/undo_truncate.test | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mysql-test/suite/innodb/t/undo_truncate.test b/mysql-test/suite/innodb/t/undo_truncate.test
index 32697e59c91..51bb4f4b9fc 100644
--- a/mysql-test/suite/innodb/t/undo_truncate.test
+++ b/mysql-test/suite/innodb/t/undo_truncate.test
@@ -53,7 +53,7 @@ let $trx_after= `select substr('$trx_after',9)`;
 
 drop table t1, t2;
 
-if ($trx_before != $trx_after)
+if ($trx_before > $trx_after)
 {
-  echo Transaction sequence mismatch: $trx_before != $trx_after;
+  echo Transaction sequence mismatch: $trx_before > $trx_after;
 }

From f3bc4f49f7c02018cac2c721837f9d1f52e9fff0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Wed, 29 Sep 2021 15:16:04 +0300
Subject: [PATCH 28/31] MDEV-20699 fixup: Re-record compat/oracle.sp-package
 result

---
 mysql-test/suite/compat/oracle/r/sp-package.result | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/mysql-test/suite/compat/oracle/r/sp-package.result b/mysql-test/suite/compat/oracle/r/sp-package.result
index 9a53b04d4ad..598c766c808 100644
--- a/mysql-test/suite/compat/oracle/r/sp-package.result
+++ b/mysql-test/suite/compat/oracle/r/sp-package.result
@@ -699,6 +699,10 @@ END
 character_set_client	latin1
 collation_connection	latin1_swedish_ci
 Database Collation	latin1_swedish_ci
+Warnings:
+Level	Note
+Code	1585
+Message	This function 'concat' has the same name as a native function
 SHOW CREATE PACKAGE BODY test2;
 Package body	test2
 sql_mode	PIPES_AS_CONCAT,ANSI_QUOTES,IGNORE_SPACE,ORACLE,NO_KEY_OPTIONS,NO_TABLE_OPTIONS,NO_FIELD_OPTIONS,NO_AUTO_CREATE_USER,SIMULTANEOUS_ASSIGNMENT
@@ -719,6 +723,10 @@ END
 character_set_client	latin1
 collation_connection	latin1_swedish_ci
 Database Collation	latin1_swedish_ci
+Warnings:
+Level	Note
+Code	1585
+Message	This function 'concat' has the same name as a native function
 DROP PACKAGE BODY test2;
 SELECT test2.f1();
 ERROR 42000: FUNCTION test.test2.f1 does not exist

From 333d6c30f87a7862a2d6ca379c49e2ea46451ebc Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Wed, 29 Sep 2021 20:40:00 +0200
Subject: [PATCH 29/31] MDEV-20699 followup.

Normally we disable caching of routines in "SHOW CREATE".
Introduce an exception, if debug_dbug="+d,cache_sp_in_show_create".
lock_sync.test needs a way to populate the cache without side effects,
or else it runs into debug_sync timeouts.

So, this possibility to cache will be remain only for very special tests.
---
 mysql-test/main/lock_sync.result | 3 +++
 mysql-test/main/lock_sync.test   | 3 +++
 sql/sp.cc                        | 8 ++++++++
 3 files changed, 14 insertions(+)

diff --git a/mysql-test/main/lock_sync.result b/mysql-test/main/lock_sync.result
index d017cf90cb8..55ceaedd04f 100644
--- a/mysql-test/main/lock_sync.result
+++ b/mysql-test/main/lock_sync.result
@@ -205,6 +205,8 @@ end|
 # called below.
 #
 connection con1;
+set @save_dbug=@@debug_dbug;
+set debug_dbug="+d,cache_sp_in_show_create";
 # Cache all functions used in the tests below so statements
 # calling them won't need to open and lock mysql.proc table
 # and we can assume that each statement locks its tables
@@ -229,6 +231,7 @@ show create function f14;
 show create function f15;
 show create function f16;
 show create function f17;
+set debug_dbug=@save_dbug;
 connection default;
 #
 # 1. Statements that read tables and do not use subqueries.
diff --git a/mysql-test/main/lock_sync.test b/mysql-test/main/lock_sync.test
index af8435f7fbb..b576f211792 100644
--- a/mysql-test/main/lock_sync.test
+++ b/mysql-test/main/lock_sync.test
@@ -232,6 +232,8 @@ let $con_aux2= con2;
 let $table= t1;
 
 connection con1;
+set @save_dbug=@@debug_dbug;
+set debug_dbug="+d,cache_sp_in_show_create";
 --echo # Cache all functions used in the tests below so statements
 --echo # calling them won't need to open and lock mysql.proc table
 --echo # and we can assume that each statement locks its tables
@@ -257,6 +259,7 @@ show create function f14;
 show create function f15;
 show create function f16;
 show create function f17;
+set debug_dbug=@save_dbug;
 --enable_result_log
 connection default;
 
diff --git a/sql/sp.cc b/sql/sp.cc
index fdfd9a79fef..8bf3c450941 100644
--- a/sql/sp.cc
+++ b/sql/sp.cc
@@ -1903,6 +1903,14 @@ Sp_handler::sp_show_create_routine(THD *thd,
     of the binary log or the query cache, which currently it does not.
   */
   sp_head *sp= 0;
+
+  DBUG_EXECUTE_IF("cache_sp_in_show_create",
+    /* Some tests need just need a way to cache SP without other side-effects.*/
+    sp_cache_routine(thd, name, false, &sp);
+    sp->show_create_routine(thd, this);
+    DBUG_RETURN(false);
+  );
+
   bool free_sp= db_find_routine(thd, name, &sp) == SP_OK;
   bool ret= !sp || sp->show_create_routine(thd, this);
   if (ret)

From 260649de0472e90be665ae2c442c4435e125b022 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= <otto@kekalainen.net>
Date: Wed, 4 Aug 2021 21:13:04 -0700
Subject: [PATCH 30/31] Misc improvements to the Gitlab-CI pipeline for MariaDB

- Add new Ninja and Clang build jobs. This helps to ensure those
  toolchains also work in addition to default CMake/gcc.

- Generate dependencies.dot/png to illustrate the CMake/Make/Ninja
  build dependencies. Viewing this image and identifying bottle necks
  in parallelism can help make the build run faster.

- Enable CUnit tests now as they are fixed on 10.6 (MDEV-25820).

- Limit parallel builds to 2 CPUs (full parallelism needs MDEV-25968) on
  CMake/Make. Now only the Ninja builds run full parallel builds as only
  Ninja is smart enough to prevent builds failing on resource
  over-consumption.

- Enable Gitlab-CI cache for job 'centos8' for ccache so that it builds
  faster. Don't use Gitlab-CI cache for other jobs, as it would too easily
  use up all free tier storage on Gitlab.com and force users to get a paid
  account just for MariaDB builds.

- On other jobs clean away ccache, as it only had a 5% hit rate on single
  builds with no downloaded cache.

- Dump full database contents during the test install so that one can
  use diff to compare the database contents at different stages and thus
  track/debug potential bugs in mariadb-install-db and mariadb-upgrade
  code.

Bugfixes:

- Zero out ccache stats before each run so that 'ccache -s' would actually
  show the stats for the latest run.
---
 .gitlab-ci.yml        | 201 ++++++++++++++++++++++++++++++++----------
 cmake/cpack_rpm.cmake |   4 +-
 2 files changed, 155 insertions(+), 50 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0a1bf5bff71..cd22fc5dd8c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,8 +37,22 @@ image: fedora:33
 # many components that are otherwise slow to build.
 variables:
   CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_MROONGA=NO -DPLUGIN_TOKUDB=NO -DPLUGIN_PERFSCHEMA=NO -DWITH_WSREP=OFF"
-  # Major version dictates which branches share the same ccache
+  # Major version dictates which branches share the same ccache. E.g. 10.6-abc
+  # and 10.6-xyz will have the same cache.
   MARIADB_MAJOR_VERSION: "10.6"
+  # NOTE! Currently ccache is only used on the Centos8 build. As each job has
+  # sufficiently different environments they are unable to benefit from each
+  # other's ccaches. As each build generates about 1 GB of ccache, having
+  # multiple caches would quickly consume all free storage on Gitlab-CI and
+  # grind all builds to a halt. Also the network overhead of download/upload
+  # decreases the benefit of ccache in Gitlab-CI, and current cache:when and
+  # cache:policy are not flexible enough to have a system where the cache is
+  # uploaded only once a week and not on every build. Having ccache on at least
+  # one build still helps ensure that ccache compatibility is at least tested
+  # and if the Centos 8 build is always significantly faster than all other
+  # builds (e.g. on self-hosted Gitlab instances) then users would at least be
+  # able to discover it.
+  #
   # Most steps don't need the source code, only artifacts
   GIT_STRATEGY: none
   # Hack to satisfy directory name length requirement by CPackRPM in CMake 3.x
@@ -65,23 +79,21 @@ fedora:
     GIT_STRATEGY: fetch
     GIT_SUBMODULE_STRATEGY: normal
   script:
-    - yum install -y yum-utils rpm-build ccache openssl-devel
-    - source /etc/profile.d/ccache.sh
-    - export CCACHE_DIR="$(pwd)/.ccache"; ccache -s
+    - yum install -y yum-utils rpm-build openssl-devel graphviz
     # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
     - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
     # This repository does not have any .spec files, so install dependencies based on Fedora spec file
     - yum-builddep -y mariadb-server
     - mkdir builddir; cd builddir
     - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    - eatmydata make package 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    # @TODO: Don't use -j on Gitlab.com as builds just get stuck when running
-    # multi-proc, needs more debugging
-    - make test || true  # Unit tests constantly fail, see https://jira.mariadb.org/browse/MDEV-25820
-    # - make test-force || true  # mysql-test-runner takes too long, run it in a separate job instead
+    - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
+    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run MTR in a separate job instead
     - *rpm_listfiles
     - mkdir ../rpm; mv *.rpm ../rpm
-    - ccache -s
   artifacts:
     when: always  # Must be able to see logs
     paths:
@@ -89,14 +101,74 @@ fedora:
       - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
       - rpm
       - builddir/_CPack_Packages/Linux/RPM/SPECS/
-  cache:
-    key: $MARIADB_MAJOR_VERSION-$CI_JOB_NAME
+      - dependencies.dot
+      - dependencies.png
+
+fedora-ninja:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel graphviz ninja-build
+    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - cmake -DRPM=generic $CMAKE_FLAGS -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -G Ninja .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - ninja -t graph > ../dependencies.dot && dot -Tpng -o ../dependencies.png ../dependencies.dot
+    - eatmydata ninja package --verbose 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # Ninja builds are not affected by bug https://jira.mariadb.org/browse/MDEV-25968
+    - ninja test
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
     paths:
-      - .ccache
-    # policy: pull
-    # @TODO: It would be enough to only download the cache. There is no need for
-    # every job to upload a new cache every time. A monthly or weekly scheduled
-    # run could update the cache for all other builds to us.
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+      - dependencies.dot
+      - dependencies.png
+
+fedora-clang:
+  stage: build
+  variables:
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: normal
+  script:
+    - yum install -y yum-utils rpm-build openssl-devel graphviz clang
+    # Accelerate builds with unsafe disk access, as we can afford to loose the entire build anyway
+    - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
+    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    - yum-builddep -y mariadb-server
+    - mkdir builddir; cd builddir
+    - export CXX=${CXX:-clang++}
+    - export CC=${CC:-clang}
+    - export CXX_FOR_BUILD=${CXX_FOR_BUILD:-clang++}
+    - export CC_FOR_BUILD=${CC_FOR_BUILD:-clang}
+    - export CFLAGS='-Wno-unused-command-line-argument'
+    - export CXXFLAGS='-Wno-unused-command-line-argument'
+    - cmake -DRPM=generic $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - cmake --graphviz=../dependencies.dot .. && dot -Tpng -o ../dependencies.png ../dependencies.dot
+    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run MTr in a separate job instead
+    - *rpm_listfiles
+    - mkdir ../rpm; mv *.rpm ../rpm
+  artifacts:
+    when: always  # Must be able to see logs
+    paths:
+      - build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpmlist-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+      - rpm
+      - builddir/_CPack_Packages/Linux/RPM/SPECS/
+      - dependencies.dot
+      - dependencies.png
 
 centos8:
   stage: build
@@ -119,16 +191,16 @@ centos8:
     - yum install -y https://github.com/stewartsmith/libeatmydata/releases/download/v129/libeatmydata-129-1.fc33.x86_64.rpm
     - yum install -y ccache  # From EPEL
     - source /etc/profile.d/ccache.sh
-    - export CCACHE_DIR="$(pwd)/.ccache"; ccache -s
+    - export CCACHE_DIR="$(pwd)/.ccache"; ccache --zero-stats
     # This repository does not have any .spec files, so install dependencies based on CentOS spec file
     - yum-builddep -y mariadb-server
     - mkdir builddir; cd builddir
     - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    - eatmydata make package 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    # @TODO: Don't use -j on Gitlab.com as builds just get stuck when running
-    # multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
-    - make test || true  # Unit tests constantly fail, see https://jira.mariadb.org/browse/MDEV-25820
-    # - make test-force || true  # mysql-test-runner takes too long, run it in a separate job instead
+    - eatmydata make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run it MTR a separate job instead
     - *rpm_listfiles
     - mkdir ../rpm; mv *.rpm ../rpm
     - ccache -s
@@ -140,7 +212,7 @@ centos8:
       - rpm
       - builddir/_CPack_Packages/Linux/RPM/SPECS/
   cache:
-    key: $MARIADB_MAJOR_VERSION-$CI_JOB_NAME
+    key: $MARIADB_MAJOR_VERSION
     paths:
       - .ccache
 
@@ -157,11 +229,11 @@ centos7:
     - yum install -y yum-utils rpm-build gcc gcc-c++ bison libxml2-devel libevent-devel openssl-devel
     - mkdir builddir; cd builddir
     - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    - make package 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
-    # @TODO: Don't use -j on Gitlab.com as builds just get stuck when running
-    # multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
-    - make test || true  # Unit tests constantly fail, see https://jira.mariadb.org/browse/MDEV-25820
-    # - make test-force || true  # mysql-test-runner takes too long, run it in a separate job instead
+    - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
+    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
+    - make test
+    # - make test-force  # mysql-test-runner takes too long, run it in a separate job instead
     - *rpm_listfiles
     - mkdir ../rpm; mv *.rpm ../rpm
   artifacts:
@@ -178,20 +250,20 @@ mysql-test-run:
     - fedora
   script:
     # Install packages so tests and the dependencies install
-    # @TODO: RPM missing 'patch' as dependency, so installing it manually for now
-    - yum install -y rpm/*.rpm patch
+    # @TODO: RPM missing 'patch' and 'diff' as dependency, so installing it manually for now
+    - yum install -y rpm/*.rpm patch diffutils
     # @TODO: Fix on packaging level for /usr/share/mariadb to work and errormsg.sys be found
     - rm -rf /usr/share/mariadb; ln -s /usr/share/mysql /usr/share/mariadb
     # mtr expects to be launched in-place and with write access to it's own directories
     - cd /usr/share/mysql-test
+    # Skip failing tests
     - |
       echo "
-      main.mysqldump : flaky on Gitlab-CI
-      main.flush_logs_not_windows : flaky in containers in general
-      main.mysql_upgrade_noengine : requires diff but diffutils is not a dependency
+      main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE'
+      main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
+      main.mysql_upgrade_noengine : upgrade output order does not match the expected
       " > skiplist
-    # @TODO: Flaky tests are skipped for now, but should be fixed
-    - ./mtr --suite=main --force --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist
+    - ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist
   artifacts:
     when: always  # Also show results when tests fail
     reports:
@@ -226,11 +298,27 @@ fedora install:
     # Fedora does not support running services in Docker (like Debian packages do) so start it manually
     - /usr/bin/mariadb-install-db -u mysql
     - sudo -u mysql /usr/sbin/mariadbd & sleep 10
-    # @TODO: Since we did a manual start, we also need to run upgrade manually
-    - /usr/bin/mariadb-upgrade -u root --socket /var/lib/mysql/mysql.sock
+    # Dump database contents as is before upgrade
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > installed-database.sql
+    # Since we did a manual start, we also need to run upgrade manually
+    - /usr/bin/mariadb-upgrade -u root
+    # Dump database contents as is after upgrade
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > upgraded-database.sql
     - |
-      mysql --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
-      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly"
+      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
+      grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't install properly"
+    - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
+    - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
+    - mariadb -e "SHUTDOWN;"
+    - rm -rf /var/lib/mysql/*  # Clear datadir before next run
+    # Start database without install-db step
+    - sudo -u mysql /usr/sbin/mariadbd --skip-network --skip-grant & sleep 10
+    # Dump database contents in initial state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > empty-database.sql
+  artifacts:
+    paths:
+      - installed-database.sql
+      - upgraded-database.sql
 
 fedora upgrade:
   stage: test
@@ -242,24 +330,39 @@ fedora upgrade:
     - /usr/libexec/mysql-check-socket
     - /usr/libexec/mysql-prepare-db-dir
     - sudo -u mysql /usr/libexec/mysqld --basedir=/usr & sleep 10
+    # Dump database contents in installed state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-installed-database.sql
     - /usr/libexec/mysql-check-upgrade
-    - mysql --skip-column-names -e "SELECT @@version, @@version_comment"  # Show version
+    # Dump database contents in upgraded state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > old-upgraded-database.sql
+    - mariadb --skip-column-names -e "SELECT @@version, @@version_comment"  # Show version
     # @TODO: Upgrade from Fedora 33 MariaDB 10.4 to MariaDB.org latest does not work
     # so do this manual step to remove conflicts until packaging is fixed
-    - >
-      yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check
-      mariadb-backup mariadb-connector-c-config
+    - yum remove -y mariadb-server-utils mariadb-gssapi-server mariadb-cracklib-password-check mariadb-backup mariadb-connector-c-config
     - rm -f rpm/*debuginfo*  # Not relevant in this test
-    - yum install -y rpm/*.rpm procps  # procps provides pkill
+    - yum install -y rpm/*.rpm
     # nothing provides galera-4 on Fedora, so this step fails if built with wsrep
-    - pkill mysqld || true; sleep 5; pkill mysqld || true; sleep 5
-    - /usr/bin/mariadb-install-db -u mysql
+    - mysql -e "SHUTDOWN;"
+    - /usr/bin/mariadb-install-db  # This step should not do anything on upgrades, just exit
     - sudo -u mysql /usr/sbin/mariadbd & sleep 10
+    # Dump database contents in installed state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-installed-database.sql || true
+    # The step above fails on: mariadb-dump: Couldn't execute 'show events': Cannot proceed, because event scheduler is disabled (1577)
     # @TODO: Since we did a manual start, we also need to run upgrade manually
-    - /usr/bin/mariadb-upgrade -u root --socket /var/lib/mysql/mysql.sock
+    - /usr/bin/mariadb-upgrade
+    # Dump database contents in upgraded state
+    - mariadb-dump --all-databases --all-tablespaces --triggers --routines --events --skip-extended-insert > new-upgraded-database.sql
     - |
-      mysql --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
+      mariadb --skip-column-names -e "SELECT @@version, @@version_comment" | tee /tmp/version
       grep $MARIADB_MAJOR_VERSION /tmp/version || echo "MariaDB didn't upgrade properly"
+    - mariadb --table -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
+    - mariadb --table -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
+  artifacts:
+    paths:
+      - old-installed-database.sql
+      - old-upgraded-database.sql
+      - new-installed-database.sql
+      - new-upgraded-database.sql
 
 # Once all RPM builds and tests have passed, also run the DEB builds and tests
 # @NOTE: This is likely to work well only on salsa.debian.org as the Gitlab.com
diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake
index bba9238664b..91322b975bf 100644
--- a/cmake/cpack_rpm.cmake
+++ b/cmake/cpack_rpm.cmake
@@ -1,5 +1,7 @@
 IF(RPM)
 
+MESSAGE(STATUS "CPackRPM building with RPM configuration: ${RPM}")
+
 SET(CPACK_GENERATOR "RPM")
 SET(CPACK_RPM_PACKAGE_DEBUG 1)
 SET(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
@@ -274,7 +276,7 @@ FILE(GLOB compat101 RELATIVE ${CMAKE_SOURCE_DIR}
     "${CMAKE_SOURCE_DIR}/../MariaDB-shared-10.1.*.rpm")
 IF(compat53 AND compat101)
   FOREACH(compat_rpm "${compat53}" "${compat101}")
-    MESSAGE("Using ${compat_rpm} to build MariaDB-compat")
+    MESSAGE(STATUS "Using ${compat_rpm} to build MariaDB-compat")
     INSTALL(CODE "EXECUTE_PROCESS(
                    COMMAND rpm2cpio ${CMAKE_SOURCE_DIR}/${compat_rpm}
                    COMMAND cpio --extract --make-directories */libmysqlclient*.so.* -

From be803f037f98812410f24b67af61aa7857969dcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Thu, 30 Sep 2021 10:01:10 +0300
Subject: [PATCH 31/31] MDEV-25215 Excessive logging "InnoDB: Cannot close
 file"

In commit 45ed9dd957eebc7fc84feb2509f4aa6baa908a95 (MDEV-23855)
when removing fil_system.LRU we failed to rate-limit the output
for reporting violations of innodb_open_files or open_files_limit.

If the server is run with a small limit of open files that is
well below the number of .ibd files that are being accessed by
the workload, and if at the same time innodb_log_file_size is
very small so that log checkpoints will occur frequently,
the process of enforcing the open files limit may be run very often.

fil_space_t::try_to_close(): Display at most one message per call,
and only if at least 5 seconds have elapsed since the last time a
message was output.

fil_node_open_file(): Only output a summary message if
fil_space_t::try_to_close() displayed a message during this run.
(Note: multiple threads may execute fil_node_open_file() on
different files at the same time.)

fil_space_t::get(): Do not dereference a null pointer if n & STOPPING.
This was caught by the test case below.

Unfortunately, it is not possible to create a fully deterministic
test case (expecting exactly 1 message to be emitted). The following with
--innodb-open-files=10 --innodb-log-file-size=4m
would occasionally fail to find the message in the log:

--source include/have_innodb.inc
--source include/have_partition.inc
--source include/have_sequence.inc

call mtr.add_suppression("InnoDB: innodb_open_files=10 is exceeded");

CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY) ENGINE=InnoDB
PARTITION BY key (pk) PARTITIONS 100;

INSERT INTO t1 SELECT * FROM seq_1_to_100;
--disable_query_log
let $n=400;
while ($n)
{
BEGIN; DELETE FROM t1; ROLLBACK;
dec $n;
}
--enable_query_log

let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err;
let SEARCH_PATTERN= \[Note\] InnoDB: Cannot close file;
-- source include/search_pattern_in_file.inc

DROP TABLE t1;
---
 storage/innobase/fil/fil0fil.cc    | 35 ++++++++++++++++++++----------
 storage/innobase/include/fil0fil.h |  2 ++
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index b9421df3ffb..39049f237e1 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -97,13 +97,22 @@ bool fil_space_t::try_to_close(bool print_info)
 
     if (const auto n= space->set_closing())
     {
-      if (print_info)
-        ib::info() << "Cannot close file " << node->name
-                   << " because of "
-                   << (n & PENDING)
-                   << ((n & NEEDS_FSYNC)
-                       ? " pending operations and pending fsync"
-                       : " pending operations");
+      if (!print_info)
+        continue;
+      print_info= false;
+      const time_t now= time(nullptr);
+      if (now - fil_system.n_open_exceeded_time < 5)
+        continue; /* We display messages at most once in 5 seconds. */
+      fil_system.n_open_exceeded_time= now;
+
+      if (n & PENDING)
+        sql_print_information("InnoDB: Cannot close file %s because of "
+                              UINT32PF " pending operations%s", node->name,
+                              n & PENDING,
+                              (n & NEEDS_FSYNC) ? " and pending fsync" : "");
+      else if (n & NEEDS_FSYNC)
+        sql_print_information("InnoDB: Cannot close file %s because of "
+                              "pending fsync", node->name);
       continue;
     }
 
@@ -424,15 +433,18 @@ static bool fil_node_open_file(fil_node_t *node)
   ut_ad(node->space->purpose != FIL_TYPE_TEMPORARY);
   ut_ad(node->space->referenced());
 
+  const auto old_time= fil_system.n_open_exceeded_time;
+
   for (ulint count= 0; fil_system.n_open >= srv_max_n_open_files; count++)
   {
     if (fil_space_t::try_to_close(count > 1))
       count= 0;
     else if (count >= 2)
     {
-      ib::warn() << "innodb_open_files=" << srv_max_n_open_files
-                 << " is exceeded (" << fil_system.n_open
-                 << ") files stay open)";
+      if (old_time != fil_system.n_open_exceeded_time)
+        sql_print_warning("InnoDB: innodb_open_files=" ULINTPF
+                          " is exceeded (" ULINTPF " files stay open)",
+                          srv_max_n_open_files, fil_system.n_open);
       break;
     }
     else
@@ -1470,8 +1482,7 @@ fil_space_t *fil_space_t::get(ulint id)
 
   if (n & STOPPING)
     space= nullptr;
-
-  if ((n & CLOSING) && !space->prepare())
+  else if ((n & CLOSING) && !space->prepare())
     space= nullptr;
 
   return space;
diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h
index 57b103518d5..10fec4d0871 100644
--- a/storage/innobase/include/fil0fil.h
+++ b/storage/innobase/include/fil0fil.h
@@ -1387,6 +1387,8 @@ public:
   sized_ilist<fil_space_t, unflushed_spaces_tag_t> unflushed_spaces;
   /** number of currently open files; protected by mutex */
   ulint n_open;
+  /** last time we noted n_open exceeding the limit; protected by mutex */
+  time_t n_open_exceeded_time;
 	ulint		max_assigned_id;/*!< maximum space id in the existing
 					tables, or assigned during the time
 					mysqld has been up; at an InnoDB