From ce870b2a2a54a1cab844be40a3fabdc8a2a53ce2 Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Mon, 26 Jul 2021 14:19:26 +0530
Subject: [PATCH] MDEV-25998 InnoDB removes the tablespace from default encrypt
 list early

Problem:
=========
As a part of MDEV-14398 patch, InnoDB added and removed
the tablespace from default encrypt list. But InnoDB removes
the tablespace from the default encrypt list too early due to
i) other encryption thread working on the tablespace
ii) When tablespace is being flushed at the end of
key rotation

InnoDB fails to decrypt/encrypt the tablespace since
the tablespace removed too early and it leads to
test case failure.

Solution:
=========
Avoid the removal of tablespace from default_encrypt_list
only when
1) Another active encryption thread working on tablespace
2) Eligible for tablespace key rotation
3) Tablespace is in flushing phase

Removed the workaround in encryption.innodb_encryption_filekeys test case.
---
 .../t/innodb_encryption_filekeys.test         | 18 -----------
 storage/innobase/fil/fil0crypt.cc             | 30 +++++++++++++++++--
 2 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test b/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test
index cd5724638b1..03447bbcfa6 100644
--- a/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test
+++ b/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test
@@ -48,12 +48,6 @@ while ($cnt)
     {
         real_sleep 1;
         dec $cnt;
-        if ($cnt == 200)
-	{
-	  --disable_query_log
-	  set global innodb_encrypt_tables = on;
-	  --enable_query_log
-	}
     }
 }
 if (!$success)
@@ -84,12 +78,6 @@ while ($cnt)
     {
         real_sleep 1;
         dec $cnt;
-        if ($cnt == 200)
-	{
-	  --disable_query_log
-	  set global innodb_encrypt_tables = off;
-	  --enable_query_log
-	}
     }
 }
 if (!$success)
@@ -119,12 +107,6 @@ while ($cnt)
     {
         real_sleep 1;
         dec $cnt;
-        if ($cnt == 200)
-	{
-	  --disable_query_log
-	  set global innodb_encrypt_tables=on;
-	  --enable_query_log
-	}
     }
 }
 if (!$success)
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 94899ed0f65..4e710b4bb4f 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -1092,6 +1092,33 @@ struct rotate_thread_t {
 	}
 };
 
+/** Avoid the removal of the tablespace from
+default_encrypt_list only when
+1) Another active encryption thread working on tablespace
+2) Eligible for tablespace key rotation
+3) Tablespace is in flushing phase
+@return true if tablespace should be removed from
+default encrypt */
+static bool fil_crypt_must_remove(const fil_space_t &space)
+{
+  ut_ad(space.purpose == FIL_TYPE_TABLESPACE);
+  fil_space_crypt_t *crypt_data = space.crypt_data;
+  mutex_own(&fil_system->mutex);
+  const ulong encrypt_tables= srv_encrypt_tables;
+  if (!crypt_data)
+    return !encrypt_tables;
+  if (!crypt_data->is_key_found())
+    return true;
+
+  mutex_enter(&crypt_data->mutex);
+  const bool remove= (space.is_stopping() || crypt_data->not_encrypted()) &&
+    (!crypt_data->rotate_state.flushing &&
+     !encrypt_tables == !!crypt_data->min_key_version &&
+     !crypt_data->rotate_state.active_threads);
+  mutex_exit(&crypt_data->mutex);
+  return remove;
+}
+
 /***********************************************************************
 Check if space needs rotation given a key_state
 @param[in,out]		state		Key rotation state
@@ -1429,8 +1456,7 @@ inline fil_space_t *fil_system_t::default_encrypt_next(
       If there is a change in innodb_encrypt_tables variables
       value then don't remove the last processed tablespace
       from the default encrypt list. */
-      if (released && (!recheck || space->crypt_data) &&
-          !encrypt == !srv_encrypt_tables)
+      if (released && !recheck && fil_crypt_must_remove(*space))
       {
         ut_a(!default_encrypt_tables.empty());
         default_encrypt_tables.remove(*space);