MDEV-8238: Tables with encryption=yes using file_key_management plugin are not encrypted

Analysis: Problem was that encryption was skipped.

Fixed by making sure that tables with ENCRYPTED=YES are encrypted.

mysql-test/std_data/keys2.txt

@@ -0,0 +1,6 @@
+1;593E580927F47AB530D3B1237CDEF6D6
+2;352E42F1B9DB5CB915C3262FE745520A
+3;CFE065600F5EB57481075C65180C3F8A
+4;205379930183490D3BECA139BDF4DB5B
+5;E2D944D5D837A1DCB22FF7FD397892EE
+6;BAFE99B0BB87F2CD33A6AF26A11F6BD1

mysql-test/std_data/keys3.txt

@@ -0,0 +1,7 @@
+1;593E580927F47AB530D3B1237CDEF6D6
+2;E4B00A45BF775B4E07D634EC5CA5912B
+3;6E35ACB162B29D1FB9E178021DAF16ED
+4;971A664A88EE0022D408E40BFAB17E79
+5;C4FF86FD89879380DA97EAC0BA3057B7
+6;BAFE99B0BB87F2CD33A6AF26A11F6BD1
+

mysql-test/suite/encryption/r/innodb-page_encryption.result

@@ -247,11 +247,11 @@ select count(*) from innodb_redundant t1, innodb_normal t2 where
 t1.c1 = t2.c1 and t1.b = t2.b;
 count(*)
 2000
-SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
-variable_value = 0
+variable_value >= 0
 1
-SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
-variable_value = 0
+variable_value >= 0
 1
 drop procedure innodb_insert_proc;
 drop table innodb_normal;

mysql-test/suite/encryption/r/innodb_page_encryption_key_change.result

@@ -0,0 +1,143 @@
+# Restart mysqld --loose-file-key-management-filename=/home/jan/mysql/10.1-bugs/mysql-test/std_data/keys2.txt
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb;
+create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encrypted=yes encryption_key_id=2;
+create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=3;
+create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encrypted=yes encryption_key_id=4;
+create table innodb_redundant(c1 bigint not null, b char(200))  engine=innodb row_format=redundant encrypted=yes encryption_key_id=5;
+insert into innodb_normal values (1,'test1'),(2,'foo'),(3,'bar'),(4,'mariadb');
+insert into innodb_compact select * from innodb_normal;
+insert into innodb_compressed select * from innodb_normal;
+insert into innodb_dynamic select * from innodb_normal;
+insert into innodb_redundant select * from innodb_normal;
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+variable_value >= 0
+1
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
+variable_value >= 0
+1
+alter table innodb_compact engine=innodb encryption_key_id = 6;
+alter table innodb_compressed engine=innodb encryption_key_id = 6;
+alter table innodb_dynamic engine=innodb encryption_key_id = 6;
+alter table innodb_redundant engine=innodb encryption_key_id = 6;
+select * from innodb_normal;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_compact;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_compressed;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_dynamic;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_redundant;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+variable_value >= 0
+1
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
+variable_value >= 0
+1
+# Restart mysqld --loose-file-key-management-filename=/home/jan/mysql/10.1-bugs/mysql-test/std_data/keys3.txt
+select * from innodb_normal;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_compact;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_compressed;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_dynamic;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_redundant;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+variable_value >= 0
+1
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
+variable_value >= 0
+1
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+alter table innodb_compact engine=innodb encryption_key_id = 2;
+alter table innodb_compressed engine=innodb encryption_key_id = 3;
+alter table innodb_dynamic engine=innodb encryption_key_id = 4;
+alter table innodb_redundant engine=innodb encryption_key_id = 5;
+select * from innodb_normal;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_compact;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_compressed;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_dynamic;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+select * from innodb_redundant;
+c1	b
+1	test1
+2	foo
+3	bar
+4	mariadb
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+variable_value >= 0
+1
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
+variable_value >= 0
+1
+drop table innodb_normal;
+drop table innodb_compact;
+drop table innodb_compressed;
+drop table innodb_dynamic;
+drop table innodb_redundant;

mysql-test/suite/encryption/t/innodb-page_encryption.test

@@ -145,9 +145,8 @@ t1.c1 = t2.c1 and t1.b = t2.b;
 select count(*) from innodb_redundant t1, innodb_normal t2 where
 t1.c1 = t2.c1 and t1.b = t2.b;
 
-# After alter+restart these should be 0
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
-SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
-SELECT variable_value = 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
 
 drop procedure innodb_insert_proc;
 drop table innodb_normal;

mysql-test/suite/encryption/t/innodb_page_encryption_key_change.test

@@ -0,0 +1,91 @@
+-- source include/have_innodb.inc
+-- source include/have_file_key_management_plugin.inc
+# embedded does not support restart
+-- source include/not_embedded.inc
+
+--echo # Restart mysqld --loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt
+-- let $restart_parameters=--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt
+-- source include/restart_mysqld.inc
+
+--disable_query_log
+let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
+let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
+--enable_query_log
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+
+create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb;
+create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encrypted=yes encryption_key_id=2;
+create table innodb_compressed(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=3;
+create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encrypted=yes encryption_key_id=4;
+create table innodb_redundant(c1 bigint not null, b char(200))  engine=innodb row_format=redundant encrypted=yes encryption_key_id=5;
+
+insert into innodb_normal values (1,'test1'),(2,'foo'),(3,'bar'),(4,'mariadb');
+insert into innodb_compact select * from innodb_normal;
+insert into innodb_compressed select * from innodb_normal;
+insert into innodb_dynamic select * from innodb_normal;
+insert into innodb_redundant select * from innodb_normal;
+
+# Note there that these variables are updated only when real I/O is done, thus they are not reliable
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
+
+alter table innodb_compact engine=innodb encryption_key_id = 6;
+alter table innodb_compressed engine=innodb encryption_key_id = 6;
+alter table innodb_dynamic engine=innodb encryption_key_id = 6;
+alter table innodb_redundant engine=innodb encryption_key_id = 6;
+
+select * from innodb_normal;
+select * from innodb_compact;
+select * from innodb_compressed;
+select * from innodb_dynamic;
+select * from innodb_redundant;
+
+# Note there that these variables are updated only when real I/O is done, thus they are not reliable
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
+
+--echo # Restart mysqld --loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt
+-- let $restart_parameters=--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt
+-- source include/restart_mysqld.inc
+
+select * from innodb_normal;
+select * from innodb_compact;
+select * from innodb_compressed;
+select * from innodb_dynamic;
+select * from innodb_redundant;
+
+# Note there that these variables are updated only when real I/O is done, thus they are not reliable
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+
+alter table innodb_compact engine=innodb encryption_key_id = 2;
+alter table innodb_compressed engine=innodb encryption_key_id = 3;
+alter table innodb_dynamic engine=innodb encryption_key_id = 4;
+alter table innodb_redundant engine=innodb encryption_key_id = 5;
+
+select * from innodb_normal;
+select * from innodb_compact;
+select * from innodb_compressed;
+select * from innodb_dynamic;
+select * from innodb_redundant;
+
+# Note there that these variables are updated only when real I/O is done, thus they are not reliable
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_encrypted';
+SELECT variable_value >= 0 FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_num_pages_decrypted';
+
+drop table innodb_normal;
+drop table innodb_compact;
+drop table innodb_compressed;
+drop table innodb_dynamic;
+drop table innodb_redundant;
+
+# reset system
+--disable_query_log
+EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
+EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
+--enable_query_log

storage/innobase/buf/buf0buf.cc

@@ -5728,7 +5728,8 @@ buf_page_encrypt_before_write(
 		return const_cast<byte*>(src_frame);
 	}
 
-	if (fil_space_check_encryption_write(bpage->space) == false) {
+	/* Is encryption needed? */
+	if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) {
 		/* An unencrypted table */
 		bpage->key_version = 0;
 		return const_cast<byte*>(src_frame);

storage/innobase/fil/fil0crypt.cc

@@ -220,7 +220,7 @@ fil_space_create_crypt_data(
 		&crypt_data->mutex, SYNC_NO_ORDER_CHECK);
 	crypt_data->locker = crypt_data_scheme_locker;
 	my_random_bytes(crypt_data->iv, sizeof(crypt_data->iv));
-	crypt_data->encryption = FIL_SPACE_ENCRYPTION_DEFAULT;
+	crypt_data->encryption = encrypt_mode;
 	crypt_data->key_id = key_id;
 	return crypt_data;
 }
@@ -542,32 +542,6 @@ fil_space_clear_crypt_data(
 	memset(page + offset, 0, size);
 }
 
-/*********************************************************************
-Check if page shall be encrypted before write
-@return true if page should be encrypted, false if not */
-UNIV_INTERN
-bool
-fil_space_check_encryption_write(
-/*=============================*/
-	ulint	space)	/*!< in: tablespace id */
-{
-	if (!srv_encrypt_tables) {
-		return false;
-	}
-
-	fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space);
-
-	if (crypt_data == NULL) {
-		return false;
-	}
-
-	if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) {
-		return false;
-	}
-
-	return true;
-}
-
 /******************************************************************
 Encrypt a page */
 UNIV_INTERN

storage/innobase/include/fil0crypt.h

@@ -198,14 +198,6 @@ fil_space_check_encryption_read(
 /*============================*/
 	ulint space);          /*!< in: tablespace id */
 
-/*********************************************************************
-Check if page shall be encrypted before write */
-UNIV_INTERN
-bool
-fil_space_check_encryption_write(
-/*=============================*/
-	ulint space);          /*!< in: tablespace id */
-
 /*********************************************************************
 Encrypt buffer page */
 UNIV_INTERN

storage/xtradb/buf/buf0buf.cc

@@ -5887,7 +5887,8 @@ buf_page_encrypt_before_write(
 		return const_cast<byte*>(src_frame);
 	}
 
-	if (fil_space_check_encryption_write(bpage->space) == false) {
+	/* Is encryption needed? */
+	if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) {
 		/* An unencrypted table */
 		bpage->key_version = 0;
 		return const_cast<byte*>(src_frame);

storage/xtradb/fil/fil0crypt.cc

@@ -220,7 +220,7 @@ fil_space_create_crypt_data(
 		&crypt_data->mutex, SYNC_NO_ORDER_CHECK);
 	crypt_data->locker = crypt_data_scheme_locker;
 	my_random_bytes(crypt_data->iv, sizeof(crypt_data->iv));
-	crypt_data->encryption = FIL_SPACE_ENCRYPTION_DEFAULT;
+	crypt_data->encryption = encrypt_mode;
 	crypt_data->key_id = key_id;
 	return crypt_data;
 }
@@ -542,32 +542,6 @@ fil_space_clear_crypt_data(
 	memset(page + offset, 0, size);
 }
 
-/*********************************************************************
-Check if page shall be encrypted before write
-@return true if page should be encrypted, false if not */
-UNIV_INTERN
-bool
-fil_space_check_encryption_write(
-/*=============================*/
-	ulint	space)	/*!< in: tablespace id */
-{
-	if (!srv_encrypt_tables) {
-		return false;
-	}
-
-	fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space);
-
-	if (crypt_data == NULL) {
-		return false;
-	}
-
-	if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) {
-		return false;
-	}
-
-	return true;
-}
-
 /******************************************************************
 Encrypt a page */
 UNIV_INTERN

storage/xtradb/include/fil0crypt.h

@@ -198,14 +198,6 @@ fil_space_check_encryption_read(
 /*============================*/
 	ulint space);          /*!< in: tablespace id */
 
-/*********************************************************************
-Check if page shall be encrypted before write */
-UNIV_INTERN
-bool
-fil_space_check_encryption_write(
-/*=============================*/
-	ulint space);          /*!< in: tablespace id */
-
 /*********************************************************************
 Encrypt buffer page */
 UNIV_INTERN