MDEV-8588: Assertion failure in file ha_innodb.cc line 21140 if at least one encrypted table exists and encryption service is not available

Analysis: Problem was that in fil_read_first_page we do find that
table has encryption information and that encryption service
or used key_id is not available. But, then we just printed
fatal error message that causes above assertion.

Fix: When we open single table tablespace if it has encryption
information (crypt_data) store this crypt data to the table
structure. When we open a table and we find out that tablespace
is not available, check has table a encryption information
and from there is encryption service or used key_id is not available.
If it is, add additional warning for SQL-layer.

mysql-test/mysql-test-run.pl

@@ -4435,6 +4435,20 @@ sub extract_warning_lines ($$) {
      qr|SSL error: Failed to set ciphers to use|,
      qr/Plugin 'InnoDB' will be forced to shutdown/,
      qr|Could not increase number of max_open_files to more than|,
+     qr/InnoDB: Error table encrypted but encryption service not available.*/,
+     qr/InnoDB: Could not find a valid tablespace file for*/,
+     qr/InnoDB: Tablespace open failed for*/,
+     qr/InnoDB: Failed to find tablespace for table*/,
+     qr/InnoDB: Space */,
+     qr|InnoDB: You may have to recover from a backup|,
+     qr|InnoDB: It is also possible that your operatingsystem has corrupted its own file cache|,
+     qr|InnoDB: and rebooting your computer removes the error|,
+     qr|InnoDB: If the corrupt page is an index page you can also try to|,
+     qr|nnoDB: fix the corruption by dumping, dropping, and reimporting|,
+     qr|InnoDB: the corrupt table. You can use CHECK|,
+     qr|InnoDB: TABLE to scan your table for corruption|,
+     qr/InnoDB: See also */
+
     );
 
   my $matched_lines= [];

mysql-test/suite/encryption/r/innodb-bad-key-change.result

@@ -0,0 +1,35 @@
+call mtr.add_suppression("Plugin 'file_key_management' init function returned error");
+call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*");
+call mtr.add_suppression("Plugin 'file_key_management' registration.*failed");
+
+# Start server with keys2.txt
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+CREATE TABLE t1 (c VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
+INSERT INTO t1 VALUES ('foobar');
+ALTER TABLE t1 ADD COLUMN c2 INT;
+INSERT INTO t1 VALUES ('foobar',2);
+SELECT * FROM t1;
+c	c2
+foobar	NULL
+foobar	2
+TRUNCATE TABLE t1;
+SELECT * FROM t1;
+c	c2
+INSERT INTO t1 VALUES ('foobar',1);
+INSERT INTO t1 VALUES ('foobar',2);
+FLUSH TABLE WITH READ LOCK;
+SELECT * FROM t1;
+c	c2
+foobar	1
+foobar	2
+
+# Restart server with keysbad3.txt
+SELECT * FROM t1;
+ERROR 42S02: Table 'test.t1' doesn't exist in engine
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1812	Tablespace is missing for table 'test/t1'
+Warning	155	Table test/t1 is encrypted but encryption service or used key_id 2 is not available.  Can't continue reading table.
+Error	1932	Table 'test.t1' doesn't exist in engine
+DROP TABLE t1;

mysql-test/suite/encryption/t/innodb-bad-key-change.test

@@ -0,0 +1,46 @@
+-- source include/have_innodb.inc
+-- source include/have_file_key_management_plugin.inc
+# embedded does not support restart
+-- source include/not_embedded.inc
+
+call mtr.add_suppression("Plugin 'file_key_management' init function returned error");
+call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*");
+call mtr.add_suppression("Plugin 'file_key_management' registration.*failed");
+
+--echo
+--echo # Start server with keys2.txt
+-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt
+-- source include/restart_mysqld.inc
+
+--disable_query_log
+let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
+let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
+--enable_query_log
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+
+CREATE TABLE t1 (c VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2;
+INSERT INTO t1 VALUES ('foobar');
+ALTER TABLE t1 ADD COLUMN c2 INT;
+INSERT INTO t1 VALUES ('foobar',2);
+SELECT * FROM t1;
+TRUNCATE TABLE t1;
+SELECT * FROM t1;
+INSERT INTO t1 VALUES ('foobar',1);
+INSERT INTO t1 VALUES ('foobar',2);
+FLUSH TABLE WITH READ LOCK;
+SELECT * FROM t1;
+
+--echo
+--echo # Restart server with keysbad3.txt
+-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keysbad3.txt
+-- source include/restart_mysqld.inc
+
+--error 1932
+SELECT * FROM t1;
+SHOW WARNINGS;
+
+-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keysbad3.txt
+-- source include/restart_mysqld.inc
+DROP TABLE t1;

mysql-test/suite/innodb/r/innodb_bug14147491.result

@@ -1,3 +1,4 @@
+call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*");
 CALL mtr.add_suppression("InnoDB: Error: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts");
 CALL mtr.add_suppression("InnoDB: Warning: database page corruption or a failed");
 CALL mtr.add_suppression("InnoDB: Database page corruption on disk or a failed");

mysql-test/suite/innodb/t/innodb_bug14147491.test

@@ -4,6 +4,8 @@
 
 -- source include/not_encrypted.inc
 
+call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*");
+
 # Don't test under valgrind, memory leaks will occur
 source include/not_valgrind.inc;
 # Avoid CrashReporter popup on Mac

storage/innobase/dict/dict0load.cc

@@ -1168,7 +1168,7 @@ loop:
 			dberr_t	err = fil_open_single_table_tablespace(
 				read_page_0, srv_read_only_mode ? false : true,
 				space_id, dict_tf_to_fsp_flags(flags),
-				name, filepath);
+				name, filepath, NULL);
 
 			if (err != DB_SUCCESS) {
 				ib_logf(IB_LOG_LEVEL_ERROR,
@@ -2412,7 +2412,7 @@ err_exit:
 			err = fil_open_single_table_tablespace(
 				true, false, table->space,
 				dict_tf_to_fsp_flags(table->flags),
-				name, filepath);
+				name, filepath, table);
 
 			if (err != DB_SUCCESS) {
 				/* We failed to find a sensible

storage/innobase/fil/fil0fil.cc

@@ -1986,6 +1986,7 @@ fil_read_first_page(
 	const char*	check_msg = NULL;
 	fil_space_crypt_t* cdata;
 
+
 	buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
 
 	/* Align the memory for a possible read from a raw device */
@@ -2015,6 +2016,10 @@ fil_read_first_page(
 		fsp_flags_get_zip_size(*flags), NULL);
 	cdata = fil_space_read_crypt_data(space, page, offset);
 
+	if (crypt_data) {
+		*crypt_data = cdata;
+	}
+
 	/* If file space is encrypted we need to have at least some
 	encryption service available where to get keys */
 	if ((cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_ON) ||
@@ -2022,16 +2027,14 @@ fil_read_first_page(
 			cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
 
 		if (!encryption_key_id_exists(cdata->key_id)) {
-			ib_logf(IB_LOG_LEVEL_FATAL,
+			ib_logf(IB_LOG_LEVEL_ERROR,
-				"Tablespace id %ld encrypted but encryption service"
+				"Tablespace id %ld is encrypted but encryption service"
-				" not available. Can't continue opening tablespace.\n",
+				" or used key_id %u is not available. Can't continue opening tablespace.",
-				space);
+				space, cdata->key_id);
-			ut_error;
-		}
-	}
 
-	if (crypt_data) {
+			return ("table encrypted but encryption service not available.");
-		*crypt_data = cdata;
+
+		}
 	}
 
 	ut_free(buf);
@@ -3621,7 +3624,8 @@ fil_open_single_table_tablespace(
 	ulint		flags,		/*!< in: tablespace flags */
 	const char*	tablename,	/*!< in: table name in the
 					databasename/tablename format */
-	const char*	path_in)	/*!< in: tablespace filepath */
+	const char*	path_in,	/*!< in: tablespace filepath */
+	dict_table_t*	table)		/*!< in: table */
 {
 	dberr_t		err = DB_SUCCESS;
 	bool		dict_filepath_same_as_default = false;
@@ -3738,6 +3742,10 @@ fil_open_single_table_tablespace(
 			&def.lsn, &def.lsn, &def.crypt_data);
 		def.valid = !def.check_msg;
 
+		if (table) {
+			table->crypt_data = def.crypt_data;
+		}
+
 		/* Validate this single-table-tablespace with SYS_TABLES,
 		but do not compare the DATA_DIR flag, in case the
 		tablespace was relocated. */
@@ -3763,6 +3771,10 @@ fil_open_single_table_tablespace(
 			&remote.lsn, &remote.lsn, &remote.crypt_data);
 		remote.valid = !remote.check_msg;
 
+		if (table) {
+			table->crypt_data = remote.crypt_data;
+		}
+
 		/* Validate this single-table-tablespace with SYS_TABLES,
 		but do not compare the DATA_DIR flag, in case the
 		tablespace was relocated. */
@@ -3789,6 +3801,10 @@ fil_open_single_table_tablespace(
 			&dict.lsn, &dict.lsn, &dict.crypt_data);
 		dict.valid = !dict.check_msg;
 
+		if (table) {
+			table->crypt_data = dict.crypt_data;
+		}
+
 		/* Validate this single-table-tablespace with SYS_TABLES,
 		but do not compare the DATA_DIR flag, in case the
 		tablespace was relocated. */
@@ -3970,8 +3986,10 @@ cleanup_and_exit:
 		mem_free(remote.filepath);
 	}
 	if (remote.crypt_data && remote.crypt_data != crypt_data) {
+		if (err == DB_SUCCESS) {
 			fil_space_destroy_crypt_data(&remote.crypt_data);
 		}
+	}
 	if (dict.success) {
 		os_file_close(dict.file);
 	}
@@ -3985,8 +4003,10 @@ cleanup_and_exit:
 		os_file_close(def.file);
 	}
 	if (def.crypt_data && def.crypt_data != crypt_data) {
+		if (err == DB_SUCCESS) {
 			fil_space_destroy_crypt_data(&def.crypt_data);
 		}
+	}
 
 	mem_free(def.filepath);
 

storage/innobase/handler/ha_innodb.cc

@@ -5629,6 +5629,26 @@ table_opened:
 		free_share(share);
 		my_errno = ENOENT;
 
+		/* If table has no talespace but it has crypt data, check
+		is tablespace made unaccessible because encryption service
+		or used key_id is not available. */
+		if (ib_table && ib_table->crypt_data) {
+			fil_space_crypt_t* crypt_data = ib_table->crypt_data;
+			if ((crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
+				(srv_encrypt_tables &&
+					crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
+
+				if (!encryption_key_id_exists(crypt_data->key_id)) {
+					push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
+						HA_ERR_NO_SUCH_TABLE,
+						"Table %s is encrypted but encryption service or"
+						" used key_id %u is not available. "
+						" Can't continue reading table.",
+						ib_table->name, crypt_data->key_id);
+				}
+			}
+		}
+
 		dict_table_close(ib_table, FALSE, FALSE);
 
 		DBUG_RETURN(HA_ERR_NO_SUCH_TABLE);

storage/innobase/include/dict0mem.h

@@ -48,6 +48,9 @@ Created 1/8/1996 Heikki Tuuri
 #include "trx0types.h"
 #include "fts0fts.h"
 #include "os0once.h"
+#include "fil0fil.h"
+#include <my_crypt.h>
+#include "fil0crypt.h"
 #include <set>
 #include <algorithm>
 #include <iterator>
@@ -1014,6 +1017,7 @@ struct dict_table_t{
 	table_id_t	id;	/*!< id of the table */
 	mem_heap_t*	heap;	/*!< memory heap */
 	char*		name;	/*!< table name */
+	fil_space_crypt_t *crypt_data; /*!< crypt data if present */
 	const char*	dir_path_of_temp_table;/*!< NULL or the directory path
 				where a TEMPORARY table that was explicitly
 				created by a user should be placed if

storage/innobase/include/fil0fil.h

@@ -214,6 +214,7 @@ struct fsp_open_info {
 	ulint		arch_log_no;	/*!< latest archived log file number */
 #endif /* UNIV_LOG_ARCHIVE */
 	fil_space_crypt_t* crypt_data;	/*!< crypt data */
+	dict_table_t*	table;		/*!< table */
 };
 
 struct fil_space_t;
@@ -833,7 +834,8 @@ fil_open_single_table_tablespace(
 	ulint		flags,		/*!< in: tablespace flags */
 	const char*	tablename,	/*!< in: table name in the
 					databasename/tablename format */
-	const char*	filepath)	/*!< in: tablespace filepath */
+	const char*	filepath,	/*!< in: tablespace filepath */
+	dict_table_t*	table)		/*!< in: table */
 	__attribute__((nonnull(5), warn_unused_result));
 
 #endif /* !UNIV_HOTBACKUP */

storage/innobase/row/row0import.cc

@@ -3628,7 +3628,7 @@ row_import_for_mysql(
 	err = fil_open_single_table_tablespace(
 		true, true, table->space,
 		dict_tf_to_fsp_flags(table->flags),
-		table->name, filepath);
+		table->name, filepath, table);
 
 	DBUG_EXECUTE_IF("ib_import_open_tablespace_failure",
 			err = DB_TABLESPACE_NOT_FOUND;);

storage/xtradb/dict/dict0load.cc

@@ -1168,7 +1168,7 @@ loop:
 			dberr_t	err = fil_open_single_table_tablespace(
 				read_page_0, srv_read_only_mode ? false : true,
 				space_id, dict_tf_to_fsp_flags(flags),
-				name, filepath);
+				name, filepath, NULL);
 
 			if (err != DB_SUCCESS) {
 				ib_logf(IB_LOG_LEVEL_ERROR,
@@ -2413,7 +2413,7 @@ err_exit:
 			err = fil_open_single_table_tablespace(
 				true, false, table->space,
 				dict_tf_to_fsp_flags(table->flags),
-				name, filepath);
+				name, filepath, table);
 
 			if (err != DB_SUCCESS) {
 				/* We failed to find a sensible

storage/xtradb/fil/fil0fil.cc

@@ -2023,6 +2023,7 @@ fil_read_first_page(
 	const char*	check_msg = NULL;
 	fil_space_crypt_t* cdata;
 
+
 	buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
 
 	/* Align the memory for a possible read from a raw device */
@@ -2053,6 +2054,10 @@ fil_read_first_page(
 		fsp_flags_get_zip_size(*flags), NULL);
 	cdata = fil_space_read_crypt_data(space, page, offset);
 
+	if (crypt_data) {
+		*crypt_data = cdata;
+	}
+
 	/* If file space is encrypted we need to have at least some
 	encryption service available where to get keys */
 	if ((cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_ON) ||
@@ -2060,16 +2065,14 @@ fil_read_first_page(
 			cdata && cdata->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
 
 		if (!encryption_key_id_exists(cdata->key_id)) {
-			ib_logf(IB_LOG_LEVEL_FATAL,
+			ib_logf(IB_LOG_LEVEL_ERROR,
-				"Tablespace id %ld encrypted but encryption service"
+				"Tablespace id %ld is encrypted but encryption service"
-				" not available. Can't continue opening tablespace.\n",
+				" or used key_id %u is not available. Can't continue opening tablespace.",
-				space);
+				space, cdata->key_id);
-			ut_error;
-		}
-	}
 
-	if (crypt_data) {
+			return ("table encrypted but encryption service not available.");
-		*crypt_data = cdata;
+
+		}
 	}
 
 	ut_free(buf);
@@ -3655,7 +3658,8 @@ fil_open_single_table_tablespace(
 	ulint		flags,		/*!< in: tablespace flags */
 	const char*	tablename,	/*!< in: table name in the
 					databasename/tablename format */
-	const char*	path_in)	/*!< in: tablespace filepath */
+	const char*	path_in,	/*!< in: tablespace filepath */
+	dict_table_t*	table)		/*!< in: table */
 {
 	dberr_t		err = DB_SUCCESS;
 	bool		dict_filepath_same_as_default = false;
@@ -3769,6 +3773,10 @@ fil_open_single_table_tablespace(
 			&def.lsn, &def.lsn, &def.crypt_data);
 		def.valid = !def.check_msg;
 
+		if (table) {
+			table->crypt_data = def.crypt_data;
+		}
+
 		/* Validate this single-table-tablespace with SYS_TABLES,
 		but do not compare the DATA_DIR flag, in case the
 		tablespace was relocated. */
@@ -3791,6 +3799,10 @@ fil_open_single_table_tablespace(
 			&remote.lsn, &remote.lsn, &remote.crypt_data);
 		remote.valid = !remote.check_msg;
 
+		if (table) {
+			table->crypt_data = remote.crypt_data;
+		}
+
 		/* Validate this single-table-tablespace with SYS_TABLES,
 		but do not compare the DATA_DIR flag, in case the
 		tablespace was relocated. */
@@ -3814,6 +3826,10 @@ fil_open_single_table_tablespace(
 			&dict.lsn, &dict.lsn, &dict.crypt_data);
 		dict.valid = !dict.check_msg;
 
+		if (table) {
+			table->crypt_data = dict.crypt_data;
+		}
+
 		/* Validate this single-table-tablespace with SYS_TABLES,
 		but do not compare the DATA_DIR flag, in case the
 		tablespace was relocated. */
@@ -3995,8 +4011,10 @@ cleanup_and_exit:
 		mem_free(remote.filepath);
 	}
 	if (remote.crypt_data && remote.crypt_data != crypt_data) {
+		if (err == DB_SUCCESS) {
 			fil_space_destroy_crypt_data(&remote.crypt_data);
 		}
+	}
 	if (dict.success) {
 		os_file_close(dict.file);
 	}
@@ -4010,8 +4028,10 @@ cleanup_and_exit:
 		os_file_close(def.file);
 	}
 	if (def.crypt_data && def.crypt_data != crypt_data) {
+		if (err == DB_SUCCESS) {
 			fil_space_destroy_crypt_data(&def.crypt_data);
 		}
+	}
 
 	mem_free(def.filepath);
 

storage/xtradb/handler/ha_innodb.cc

@@ -6080,6 +6080,26 @@ table_opened:
 		free_share(share);
 		my_errno = ENOENT;
 
+		/* If table has no talespace but it has crypt data, check
+		is tablespace made unaccessible because encryption service
+		or used key_id is not available. */
+		if (ib_table && ib_table->crypt_data) {
+			fil_space_crypt_t* crypt_data = ib_table->crypt_data;
+			if ((crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
+				(srv_encrypt_tables &&
+					crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
+
+				if (!encryption_key_id_exists(crypt_data->key_id)) {
+					push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
+						HA_ERR_NO_SUCH_TABLE,
+						"Table %s is encrypted but encryption service or"
+						" used key_id %u is not available. "
+						" Can't continue reading table.",
+						ib_table->name, crypt_data->key_id);
+				}
+			}
+		}
+
 		dict_table_close(ib_table, FALSE, FALSE);
 
 		DBUG_RETURN(HA_ERR_NO_SUCH_TABLE);

storage/xtradb/include/dict0mem.h

@@ -51,6 +51,9 @@ Created 1/8/1996 Heikki Tuuri
 #include "trx0types.h"
 #include "fts0fts.h"
 #include "os0once.h"
+#include "fil0fil.h"
+#include <my_crypt.h>
+#include "fil0crypt.h"
 #include <set>
 #include <algorithm>
 #include <iterator>
@@ -1030,6 +1033,7 @@ struct dict_table_t{
 	table_id_t	id;	/*!< id of the table */
 	mem_heap_t*	heap;	/*!< memory heap */
 	char*		name;	/*!< table name */
+	fil_space_crypt_t *crypt_data; /*!< crypt data if present */
 	const char*	dir_path_of_temp_table;/*!< NULL or the directory path
 				where a TEMPORARY table that was explicitly
 				created by a user should be placed if

storage/xtradb/include/fil0fil.h

@@ -207,6 +207,7 @@ struct fsp_open_info {
 	ulint		flags;		/*!< Tablespace flags */
 	ulint		encryption_error; /*!< if an encryption error occurs */
 	fil_space_crypt_t* crypt_data;  /*!< crypt data */
+	dict_table_t*	table;		/*!< table */
 };
 
 struct fil_space_t;
@@ -828,7 +829,8 @@ fil_open_single_table_tablespace(
 	ulint		flags,		/*!< in: tablespace flags */
 	const char*	tablename,	/*!< in: table name in the
 					databasename/tablename format */
-	const char*	filepath)	/*!< in: tablespace filepath */
+	const char*	filepath,	/*!< in: tablespace filepath */
+	dict_table_t*	table)		/*!< in: table */
 	__attribute__((nonnull(5), warn_unused_result));
 
 #endif /* !UNIV_HOTBACKUP */

storage/xtradb/row/row0import.cc

@@ -3651,7 +3651,7 @@ row_import_for_mysql(
 	err = fil_open_single_table_tablespace(
 		true, true, table->space,
 		dict_tf_to_fsp_flags(table->flags),
-		table->name, filepath);
+		table->name, filepath, table);
 
 	DBUG_EXECUTE_IF("ib_import_open_tablespace_failure",
 			err = DB_TABLESPACE_NOT_FOUND;);