MDEV-11939: innochecksum mistakes a file for an encrypted one (page 0 invalid)

Always read full page 0 to determine does tablespace contain
encryption metadata. Tablespaces that are page compressed or
page compressed and encrypted do not compare checksum as
it does not exists. For encrypted tables use checksum
verification written for encrypted tables and normal tables
use normal method.

buf_page_is_checksum_valid_crc32
buf_page_is_checksum_valid_innodb
buf_page_is_checksum_valid_none
	Add Innochecksum logging to file

buf_page_is_corrupted
        Remove ib_logf and page_warn_strict_checksum
        calls in innochecksum compilation. Add innochecksum
        logging to file.

fil0crypt.cc fil0crypt.h
        Modify to be able to use in innochecksum compilation and
	move fil_space_verify_crypt_checksum to end of the file.
	Add innochecksum logging to file.

univ.i
        Add innochecksum strict_verify, log_file and cur_page_num
        variables as extern.

page_zip_verify_checksum
        Add innochecksum logging to file.

innochecksum.cc
        Lot of changes most notable able to read encryption
        metadata from page 0 of the tablespace.

Added test case where we corrupt intentionally
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION (encryption key version)
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum)
FIL_DATA+10 (data)

extra/CMakeLists.txt

@@ -77,7 +77,9 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE)
       ../storage/innobase/buf/buf0checksum.cc
       ../storage/innobase/ut/ut0crc32.cc
       ../storage/innobase/ut/ut0ut.cc
+      ../storage/innobase/buf/buf0buf.cc
       ../storage/innobase/page/page0zip.cc
+      ../storage/innobase/fil/fil0crypt.cc
       )
 
   IF(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")

mysql-test/suite/encryption/r/innochecksum.result

@@ -7,6 +7,7 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR
 CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
 CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1;
 CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
 # Write file to make mysql-test-run.pl expect the "crash", but don't
 # start it until it's told to
 # We give 30 seconds to do a clean shutdown because we do not want
@@ -18,6 +19,27 @@ CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_CO
 # Run innochecksum on t3
 # Run innochecksum on t4
 # Run innochecksum on t4
+# Run innochecksum on t5
+# Run innochecksum on t6
+# Backup tables before corrupting
+# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+# Run innochecksum on t2
+# Run innochecksum on t3
+# no encryption corrupting the field should not have effect
+# Run innochecksum on t6
+# no encryption corrupting the field should not have effect
+# Restore the original tables
+# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum)
+# Run innochecksum on t2
+# Run innochecksum on t3
+# Run innochecksum on t6
+# no encryption corrupting the field should not have effect
+# Restore the original tables
+# Corrupt FIL_DATA+10 (data)
+# Run innochecksum on t2
+# Run innochecksum on t3
+# Run innochecksum on t6
+# Restore the original tables
 # Write file to make mysql-test-run.pl start up the server again
 # Cleanup
-DROP TABLE t1, t2, t3, t4, t5;
+DROP TABLE t1, t2, t3, t4, t5, t6;

mysql-test/suite/encryption/t/innochecksum.test

@@ -2,7 +2,7 @@
 # MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables
 #
 
-# Don't test under embedded
+# Don't test under embedded as we restart server
 -- source include/not_embedded.inc
 # Require InnoDB
 -- source include/have_innodb.inc
@@ -13,16 +13,12 @@ if (!$INNOCHECKSUM) {
   --die Need innochecksum binary
 }
 
---disable_query_log
-let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`;
-let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
-let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
---enable_query_log
-
+--disable_warnings
 SET GLOBAL innodb_file_format = `Barracuda`;
 SET GLOBAL innodb_file_per_table = ON;
 # zlib
 set global innodb_compression_algorithm = 1;
+--enable_warnings
 
 --echo # Create and populate a tables
 CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
@@ -30,9 +26,11 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR
 CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
 CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1;
 CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
 
 --disable_query_log
 --let $i = 1000
+begin;
 while ($i)
 {
   INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100));
@@ -42,6 +40,8 @@ INSERT INTO t2 SELECT * FROM t1;
 INSERT INTO t3 SELECT * FROM t1;
 INSERT INTO t4 SELECT * FROM t1;
 INSERT INTO t5 SELECT * FROM t1;
+INSERT INTO t6 SELECT * FROM t1;
+commit;
 --enable_query_log
 
 let $MYSQLD_DATADIR=`select @@datadir`;
@@ -50,6 +50,10 @@ let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd;
 let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd;
 let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd;
 let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd;
+let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd;
+
+let INNODB_PAGE_SIZE=`select @@innodb_page_size`;
+let MYSQLD_DATADIR=`select @@datadir`;
 
 --echo # Write file to make mysql-test-run.pl expect the "crash", but don't
 --echo # start it until it's told to
@@ -83,19 +87,198 @@ shutdown_server 30;
 
 --exec $INNOCHECKSUM $t4_IBD
 
+--echo # Run innochecksum on t5
+
+--exec $INNOCHECKSUM $t5_IBD
+
+--echo # Run innochecksum on t6
+
+--exec $INNOCHECKSUM $t6_IBD
+
 --enable_result_log
 
+--echo # Backup tables before corrupting
+--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd.backup
+--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t2.ibd.backup
+--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t3.ibd.backup
+--copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_DATADIR/test/t4.ibd.backup
+--copy_file $MYSQLD_DATADIR/test/t5.ibd $MYSQLD_DATADIR/test/t5.ibd.backup
+--copy_file $MYSQLD_DATADIR/test/t6.ibd $MYSQLD_DATADIR/test/t6.ibd.backup
+
+#
+# MDEV-11939: innochecksum mistakes a file for an encrypted one
+#
+
+--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+
+perl;
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+EOF
+
+-- disable_result_log
+--error 1
+--exec $INNOCHECKSUM $t1_IBD
+
+--echo # Run innochecksum on t2
+
+--error 1
+--exec $INNOCHECKSUM $t2_IBD
+
+--echo # Run innochecksum on t3
+--echo # no encryption corrupting the field should not have effect
+--exec $INNOCHECKSUM $t3_IBD
+
+--echo # Run innochecksum on t6
+--echo # no encryption corrupting the field should not have effect
+--exec $INNOCHECKSUM $t6_IBD
+
+--enable_result_log
+
+--echo # Restore the original tables
+--remove_file $MYSQLD_DATADIR/test/t1.ibd
+--remove_file $MYSQLD_DATADIR/test/t2.ibd
+--remove_file $MYSQLD_DATADIR/test/t3.ibd
+--remove_file $MYSQLD_DATADIR/test/t4.ibd
+--remove_file $MYSQLD_DATADIR/test/t5.ibd
+--remove_file $MYSQLD_DATADIR/test/t6.ibd
+--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd
+--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd
+--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd
+--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd
+--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd
+--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd
+
+--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum)
+
+perl;
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+EOF
+
+-- disable_result_log
+--error 1
+--exec $INNOCHECKSUM $t1_IBD
+
+--echo # Run innochecksum on t2
+--error 1
+--exec $INNOCHECKSUM $t2_IBD
+
+--echo # Run innochecksum on t3
+--error 1
+--exec $INNOCHECKSUM $t3_IBD
+
+--echo # Run innochecksum on t6
+--echo # no encryption corrupting the field should not have effect
+--exec $INNOCHECKSUM $t6_IBD
+
+--enable_result_log
+
+--echo # Restore the original tables
+--remove_file $MYSQLD_DATADIR/test/t1.ibd
+--remove_file $MYSQLD_DATADIR/test/t2.ibd
+--remove_file $MYSQLD_DATADIR/test/t3.ibd
+--remove_file $MYSQLD_DATADIR/test/t4.ibd
+--remove_file $MYSQLD_DATADIR/test/t5.ibd
+--remove_file $MYSQLD_DATADIR/test/t6.ibd
+--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd
+--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd
+--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd
+--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd
+--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd
+--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd
+
+--echo # Corrupt FIL_DATA+10 (data)
+
+perl;
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open";
+binmode FILE;
+seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek";
+print FILE pack("H*", "c00lcafedeadb017");
+close FILE or die "close";
+EOF
+
+-- disable_result_log
+--error 1
+--exec $INNOCHECKSUM $t1_IBD
+
+--echo # Run innochecksum on t2
+--error 1
+--exec $INNOCHECKSUM $t2_IBD
+
+--echo # Run innochecksum on t3
+--error 1
+--exec $INNOCHECKSUM $t3_IBD
+
+--echo # Run innochecksum on t6
+--error 1
+--exec $INNOCHECKSUM $t6_IBD
+
+--enable_result_log
+
+--echo # Restore the original tables
+--move_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd
+--move_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd
+--move_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd
+--move_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd
+--move_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd
+--move_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd
+
 --echo # Write file to make mysql-test-run.pl start up the server again
 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
 --enable_reconnect
 --source include/wait_until_connected_again.inc
 
 --echo # Cleanup
-DROP TABLE t1, t2, t3, t4, t5;
-
-# reset system
---disable_query_log
-EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig;
-EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
-EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
---enable_query_log
+DROP TABLE t1, t2, t3, t4, t5, t6;

storage/innobase/buf/buf0buf.cc

@@ -31,12 +31,16 @@ The database buffer buf_pool
 Created 11/5/1995 Heikki Tuuri
 *******************************************************/
 
+#include "univ.i"
+#include "mach0data.h"
 #include "buf0buf.h"
+#include <string.h>
 
 #ifdef UNIV_NONINL
 #include "buf0buf.ic"
 #endif
 
+#ifndef UNIV_INNOCHECKSUM
 #include "mem0mem.h"
 #include "btr0btr.h"
 #include "fil0fil.h"
@@ -52,13 +56,15 @@ Created 11/5/1995 Heikki Tuuri
 #include "srv0srv.h"
 #include "dict0dict.h"
 #include "log0recv.h"
-#include "page0zip.h"
 #include "srv0mon.h"
-#include "buf0checksum.h"
 #ifdef HAVE_LIBNUMA
 #include <numa.h>
 #include <numaif.h>
 #endif // HAVE_LIBNUMA
+#endif /* !UNIV_INNOCHECKSUM */
+#include "page0zip.h"
+#include "buf0checksum.h"
+#ifndef UNIV_INNOCHECKSUM
 #include "fil0pagecompress.h"
 #include "ha_prototypes.h"
 #include "ut0byte.h"
@@ -528,6 +534,7 @@ buf_block_alloc(
 	return(block);
 }
 #endif /* !UNIV_HOTBACKUP */
+#endif /* !UNIV_INNOCHECKSUM */
 
 /** Check if a page is all zeroes.
 @param[in]	read_buf	database page
@@ -561,6 +568,17 @@ buf_page_is_checksum_valid_crc32(
 {
 	ib_uint32_t	crc32 = buf_calc_page_crc32(read_buf);
 
+#ifdef UNIV_INNOCHECKSUM
+	if (log_file
+	    && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) {
+		fprintf(log_file, "page::%lu;"
+			" crc32 calculated = %u;"
+			" recorded checksum field1 = %lu recorded"
+			" checksum field2 =%lu\n", cur_page_num,
+			crc32, checksum_field1, checksum_field2);
+	}
+#endif /* UNIV_INNOCHECKSUM */
+
 	if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) {
 		DBUG_PRINT("buf_checksum",
 			("Page checksum crc32 not valid field1 " ULINTPF
@@ -595,12 +613,45 @@ buf_page_is_checksum_valid_innodb(
 	2. Newer InnoDB versions store the old formula checksum
 	(buf_calc_page_old_checksum()). */
 
+	ulint	old_checksum = buf_calc_page_old_checksum(read_buf);
+	ulint	new_checksum = buf_calc_page_new_checksum(read_buf);
+
+#ifdef UNIV_INNOCHECKSUM
+	if (log_file
+	    && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) {
+		fprintf(log_file, "page::%lu;"
+			" old style: calculated ="
+			" %lu; recorded = %lu\n",
+			cur_page_num, old_checksum,
+			checksum_field2);
+		fprintf(log_file, "page::%lu;"
+			" new style: calculated ="
+			" %lu; crc32 = %u; recorded = %lu\n",
+			cur_page_num, new_checksum,
+			buf_calc_page_crc32(read_buf), checksum_field1);
+	}
+
+	if (log_file
+	    && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) {
+		fprintf(log_file, "page::%lu;"
+			" old style: calculated ="
+			" %lu; recorded checksum = %lu\n",
+			cur_page_num, old_checksum,
+			checksum_field2);
+		fprintf(log_file, "page::%lu;"
+			" new style: calculated ="
+			" %lu; recorded checksum  = %lu\n",
+			cur_page_num, new_checksum,
+			checksum_field1);
+	}
+#endif /* UNIV_INNOCHECKSUM */
+
 	if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN)
-	    && checksum_field2 != buf_calc_page_old_checksum(read_buf)) {
+	    && checksum_field2 != old_checksum) {
 		DBUG_PRINT("buf_checksum",
 			("Page checksum innodb not valid field1 " ULINTPF
 				" field2 " ULINTPF "crc32 " ULINTPF " lsn " ULINTPF ".",
-			checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf),
+			checksum_field1, checksum_field2, old_checksum,
 				mach_read_from_4(read_buf + FIL_PAGE_LSN)));
 
 		return(false);
@@ -612,11 +663,11 @@ buf_page_is_checksum_valid_innodb(
 	(always equal to 0), to FIL_PAGE_SPACE_OR_CHKSUM */
 
 	if (checksum_field1 != 0
-	    && checksum_field1 != buf_calc_page_new_checksum(read_buf)) {
+	    && checksum_field1 != new_checksum) {
 		DBUG_PRINT("buf_checksum",
 			("Page checksum innodb not valid field1 " ULINTPF
 				" field2 " ULINTPF "crc32 " ULINTPF " lsn " ULINTPF ".",
-			checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf),
+			checksum_field1, checksum_field2, new_checksum,
 				mach_read_from_4(read_buf + FIL_PAGE_LSN)));
 
 		return(false);
@@ -646,6 +697,18 @@ buf_page_is_checksum_valid_none(
 				mach_read_from_4(read_buf + FIL_PAGE_LSN)));
 	}
 
+#ifdef UNIV_INNOCHECKSUM
+	if (log_file
+	    && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) {
+		fprintf(log_file,
+			"page::%lu; none checksum: calculated"
+			" = %lu; recorded checksum_field1 = %lu"
+			" recorded checksum_field2 = %lu\n",
+			cur_page_num, BUF_NO_CHECKSUM_MAGIC,
+			checksum_field1, checksum_field2);
+	}
+#endif /* UNIV_INNOCHECKSUM */
+
 	return(checksum_field1 == checksum_field2
 	       && checksum_field1 == BUF_NO_CHECKSUM_MAGIC);
 }
@@ -662,14 +725,18 @@ buf_page_is_corrupted(
 	bool			check_lsn,
 	const byte*		read_buf,
 	ulint			zip_size,
+#ifndef UNIV_INNOCHECKSUM
 	const fil_space_t* 	space)
+#else
+	const void* 	 	space)
+#endif
 {
-	ulint		checksum_field1;
-	ulint		checksum_field2;
-	ulint 		space_id = mach_read_from_4(
-		read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
-	ulint page_type = mach_read_from_2(
-		read_buf + FIL_PAGE_TYPE);
+	ulint checksum_field1 = 0;
+	ulint checksum_field2 = 0;
+#ifndef UNIV_INNOCHECKSUM
+	ulint space_id = mach_read_from_4(read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
+#endif
+	ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE);
 
 	/* We can trust page type if page compression is set on tablespace
 	flags because page compression flag means file must have been
@@ -682,7 +749,10 @@ buf_page_is_corrupted(
 	decompressed at this stage). */
 	if ((page_type == FIL_PAGE_PAGE_COMPRESSED ||
 	     page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)
-	    && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) {
+#ifndef UNIV_INNOCHECKSUM
+	    && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)
+#endif
+	) {
 		return (false);
 	}
 
@@ -693,16 +763,17 @@ buf_page_is_corrupted(
 
 		/* Stored log sequence numbers at the start and the end
 		of page do not match */
-
+#ifndef UNIV_INNOCHECKSUM
 		ib_logf(IB_LOG_LEVEL_INFO,
 			"Log sequence number at the start %lu and the end %lu do not match.",
 			mach_read_from_4(read_buf + FIL_PAGE_LSN + 4),
 			mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4));
+#endif /* UNIV_INNOCHECKSUM */
 
 		return(true);
 	}
 
-#ifndef UNIV_HOTBACKUP
+#if !defined(UNIV_HOTBACKUP) && !defined(UNIV_INNOCHECKSUM)
 	if (check_lsn && recv_lsn_checks_on) {
 		lsn_t	current_lsn;
 
@@ -742,7 +813,7 @@ buf_page_is_corrupted(
 	}
 
 	if (zip_size) {
-		return(!page_zip_verify_checksum(read_buf, zip_size));
+		return(!page_zip_verify_checksum((const void *)read_buf, zip_size));
 	}
 
 	checksum_field1 = mach_read_from_4(
@@ -762,9 +833,10 @@ buf_page_is_corrupted(
 		/* make sure that the page is really empty */
 		for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) {
 			if (read_buf[i] != 0) {
+#ifndef UNIV_INNOCHECKSUM
 				ib_logf(IB_LOG_LEVEL_INFO,
 					"Checksum fields zero but page is not empty.");
-
+#endif
 				return(true);
 			}
 		}
@@ -774,7 +846,9 @@ buf_page_is_corrupted(
 
 	DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(true); );
 
+#ifndef UNIV_INNOCHECKSUM
 	ulint	page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET);
+#endif
 
 	const srv_checksum_algorithm_t	curr_algo =
 		static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
@@ -792,12 +866,32 @@ buf_page_is_corrupted(
 			checksum_field1, checksum_field2)) {
 			if (curr_algo
 			    == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) {
+#ifndef UNIV_INNOCHECKSUM
 				page_warn_strict_checksum(
 					curr_algo,
 					SRV_CHECKSUM_ALGORITHM_NONE,
 					space_id, page_no);
+#endif /* !UNIV_INNOCHECKSUM */
 			}
 
+#ifdef UNIV_INNOCHECKSUM
+			if (log_file) {
+				fprintf(log_file, "page::" ULINTPF ";"
+					" old style: calculated = " ULINTPF ";"
+					" recorded = " ULINTPF "\n",
+					cur_page_num,
+					buf_calc_page_old_checksum(read_buf),
+					checksum_field2);
+				fprintf(log_file, "page::" ULINTPF ";"
+					" new style: calculated = " ULINTPF ";"
+					" crc32 = %u; recorded = " ULINTPF "\n",
+					cur_page_num,
+					buf_calc_page_new_checksum(read_buf),
+					buf_calc_page_crc32(read_buf),
+					checksum_field1);
+			}
+#endif /* UNIV_INNOCHECKSUM */
+
 			return(false);
 		}
 
@@ -805,15 +899,24 @@ buf_page_is_corrupted(
 			checksum_field1, checksum_field2)) {
 			if (curr_algo
 			    == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) {
+#ifndef UNIV_INNOCHECKSUM
 				page_warn_strict_checksum(
 					curr_algo,
 					SRV_CHECKSUM_ALGORITHM_INNODB,
 					space_id, page_no);
+#endif
 			}
 
 			return(false);
 		}
 
+#ifdef UNIV_INNOCHECKSUM
+		if (log_file) {
+			fprintf(log_file, "Fail; page " ULINTPF
+				" invalid (fails crc32 checksum)\n",
+				cur_page_num);
+		}
+#endif /* UNIV_INNOCHECKSUM */
 		return(true);
 
 	case SRV_CHECKSUM_ALGORITHM_INNODB:
@@ -828,11 +931,29 @@ buf_page_is_corrupted(
 			checksum_field1, checksum_field2)) {
 			if (curr_algo
 			    == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) {
+#ifndef UNIV_INNOCHECKSUM
 				page_warn_strict_checksum(
 					curr_algo,
 					SRV_CHECKSUM_ALGORITHM_NONE,
 					space_id, page_no);
+#endif
 			}
+#ifdef UNIV_INNOCHECKSUM
+			if (log_file) {
+				fprintf(log_file, "page::" ULINTPF ";"
+					" old style: calculated = " ULINTPF ";"
+					" recorded = " ULINTPF "\n", cur_page_num,
+					buf_calc_page_old_checksum(read_buf),
+					checksum_field2);
+				fprintf(log_file, "page::" ULINTPF ";"
+					" new style: calculated = " ULINTPF ";"
+					" crc32 = %u; recorded = " ULINTPF "\n",
+					cur_page_num,
+					buf_calc_page_new_checksum(read_buf),
+					buf_calc_page_crc32(read_buf),
+					checksum_field1);
+			}
+#endif /* UNIV_INNOCHECKSUM */
 
 			return(false);
 		}
@@ -841,15 +962,25 @@ buf_page_is_corrupted(
 			checksum_field1, checksum_field2)) {
 			if (curr_algo
 			    == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) {
+#ifndef UNIV_INNOCHECKSUM
 				page_warn_strict_checksum(
 					curr_algo,
 					SRV_CHECKSUM_ALGORITHM_CRC32,
 					space_id, page_no);
+#endif
 			}
 
 			return(false);
 		}
 
+#ifdef UNIV_INNOCHECKSUM
+		if (log_file) {
+			fprintf(log_file, "Fail; page " ULINTPF
+				" invalid (fails innodb checksum)\n",
+				cur_page_num);
+		}
+#endif /* UNIV_INNOCHECKSUM */
+
 		return(true);
 
 	case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
@@ -861,22 +992,34 @@ buf_page_is_corrupted(
 
 		if (buf_page_is_checksum_valid_crc32(read_buf,
 			checksum_field1, checksum_field2)) {
+#ifndef UNIV_INNOCHECKSUM
 			page_warn_strict_checksum(
 				curr_algo,
 				SRV_CHECKSUM_ALGORITHM_CRC32,
 				space_id, page_no);
+#endif
 			return(false);
 		}
 
 		if (buf_page_is_checksum_valid_innodb(read_buf,
 			checksum_field1, checksum_field2)) {
+#ifndef UNIV_INNOCHECKSUM
 			page_warn_strict_checksum(
 				curr_algo,
 				SRV_CHECKSUM_ALGORITHM_INNODB,
 				space_id, page_no);
+#endif
 			return(false);
 		}
 
+#ifdef UNIV_INNOCHECKSUM
+		if (log_file) {
+			fprintf(log_file, "Fail; page " ULINTPF
+				" invalid (fails none checksum)\n",
+				cur_page_num);
+		}
+#endif /* UNIV_INNOCHECKSUM */
+
 		return(true);
 
 	case SRV_CHECKSUM_ALGORITHM_NONE:
@@ -890,6 +1033,7 @@ buf_page_is_corrupted(
 	return(false);
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /********************************************************************//**
 Prints a page to stderr. */
 UNIV_INTERN
@@ -6352,3 +6496,4 @@ buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space)
 	ut_ad(space->n_pending_ios > 0);
 	return (success);
 }
+#endif /* !UNIV_INNOCHECKSUM */

storage/innobase/buf/buf0checksum.cc

@@ -128,8 +128,6 @@ buf_calc_page_old_checksum(
 	return(checksum);
 }
 
-#ifndef UNIV_INNOCHECKSUM
-
 /********************************************************************//**
 Return a printable string describing the checksum algorithm.
 @return	algorithm name */
@@ -158,4 +156,3 @@ buf_checksum_algorithm_name(
 	return(NULL);
 }
 
-#endif /* !UNIV_INNOCHECKSUM */

storage/innobase/fil/fil0crypt.cc

@@ -25,13 +25,18 @@ Modified           Jan Lindström jan.lindstrom@mariadb.com
 
 #include "fil0fil.h"
 #include "fil0crypt.h"
+#include "mach0data.h"
+#include "page0zip.h"
+#include "buf0buf.h"
+#include "buf0checksum.h"
+
+#ifndef UNIV_INNOCHECKSUM
+
 #include "srv0srv.h"
 #include "srv0start.h"
-#include "mach0data.h"
 #include "log0recv.h"
 #include "mtr0mtr.h"
 #include "mtr0log.h"
-#include "page0zip.h"
 #include "ut0ut.h"
 #include "btr0scrub.h"
 #include "fsp0fsp.h"
@@ -107,13 +112,20 @@ UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key;
 extern my_bool srv_background_scrub_data_uncompressed;
 extern my_bool srv_background_scrub_data_compressed;
 
+/***********************************************************************
+Check if a key needs rotation given a key_state
+@param[in]	encrypt_mode		Encryption mode
+@param[in]	key_version		Current key version
+@param[in]	latest_key_version	Latest key version
+@param[in]	rotate_key_age		when to rotate
+@return true if key needs rotation, false if not */
 static bool
 fil_crypt_needs_rotation(
-	fil_encryption_t        encrypt_mode,           /*!< in: Encryption
-							mode */
-	uint			key_version,		/*!< in: Key version */
-	uint			latest_key_version,	/*!< in: Latest key version */
-	uint			rotate_key_age);	/*!< in: When to rotate */
+	fil_encryption_t	encrypt_mode,
+	uint			key_version,
+	uint			latest_key_version,
+	uint			rotate_key_age)
+	MY_ATTRIBUTE((warn_unused_result));
 
 /*********************************************************************
 Init space crypt */
@@ -908,137 +920,6 @@ fil_crypt_calculate_checksum(
 	return checksum;
 }
 
-/*********************************************************************
-Verify that post encryption checksum match calculated checksum.
-This function should be called only if tablespace contains crypt_data
-metadata (this is strong indication that tablespace is encrypted).
-Function also verifies that traditional checksum does not match
-calculated checksum as if it does page could be valid unencrypted,
-encrypted, or corrupted.
-
-@param[in]	page		Page to verify
-@param[in]	zip_size	zip size
-@param[in]	space		Tablespace
-@param[in]	pageno		Page no
-@return true if page is encrypted AND OK, false otherwise */
-UNIV_INTERN
-bool
-fil_space_verify_crypt_checksum(
-	byte* 			page,
-	ulint			zip_size,
-	const fil_space_t*	space,
-	ulint			pageno)
-{
-	uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
-
-	/* If page is not encrypted, return false */
-	if (key_version == 0) {
-		return(false);
-	}
-
-	/* Read stored post encryption checksum. */
-	ib_uint32_t checksum = mach_read_from_4(
-		page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
-
-	/* Declare empty pages non-corrupted */
-	if (checksum == 0
-	    && *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
-	    && buf_page_is_zeroes(page, zip_size)) {
-		return(true);
-	}
-
-	/* Compressed and encrypted pages do not have checksum. Assume not
-	corrupted. Page verification happens after decompression in
-	buf_page_io_complete() using buf_page_is_corrupted(). */
-	if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
-		return (true);
-	}
-
-	ib_uint32_t cchecksum1 = 0;
-	ib_uint32_t cchecksum2 = 0;
-
-	/* Calculate checksums */
-	if (zip_size) {
-		cchecksum1 = page_zip_calc_checksum(
-			page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
-
-		if(cchecksum1 != checksum) {
-			cchecksum2 = page_zip_calc_checksum(
-				page, zip_size,
-				SRV_CHECKSUM_ALGORITHM_INNODB);
-		}
-	} else {
-		cchecksum1 = buf_calc_page_crc32(page);
-
-		if (cchecksum1 != checksum) {
-			cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum(
-				page);
-		}
-	}
-
-	/* If stored checksum matches one of the calculated checksums
-	page is not corrupted. */
-
-	bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
-		|| checksum == BUF_NO_CHECKSUM_MAGIC);
-
-	/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
-	first page of each system tablespace file at
-	FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
-	the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
-	(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
-	than page 0 of the system tablespace.
-
-	Starting from MariaDB 10.1 the field has been repurposed for
-	encryption key_version.
-
-	Starting with MySQL 5.7 (and MariaDB Server 10.2), the
-	field has been repurposed for SPATIAL INDEX pages for
-	FIL_RTREE_SPLIT_SEQ_NUM.
-
-	Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
-	checksum.
-
-	Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
-	field would usually be 0 for pages that are not encrypted, we cannot
-	assume that a nonzero value means that the page is encrypted.
-	Therefore we must validate the page both as encrypted and unencrypted
-	when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
-	*/
-
-	ulint checksum1 = mach_read_from_4(
-		page + FIL_PAGE_SPACE_OR_CHKSUM);
-
-	ulint checksum2 = checksum1;
-
-	bool valid;
-
-	if (zip_size) {
-		valid = (checksum1 == cchecksum1);
-	} else {
-		checksum2 = mach_read_from_4(
-			page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
-		valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
-		|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
-	}
-
-	if (encrypted && valid) {
-		/* If page is encrypted and traditional checksums match,
-		page could be still encrypted, or not encrypted and valid or
-		corrupted. */
-		ib_logf(IB_LOG_LEVEL_ERROR,
-			" Page %lu in space %s (%lu) maybe corrupted."
-			" Post encryption checksum %u stored [%lu:%lu] key_version %u",
-			pageno,
-			space ? space->name : "N/A",
-			mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
-			checksum, checksum1, checksum2, key_version);
-		encrypted = false;
-	}
-
-	return(encrypted);
-}
-
 /***********************************************************************/
 
 /** A copy of global key state */
@@ -2555,8 +2436,9 @@ fil_space_crypt_close_tablespace(
 
 		if (now >= last + 30) {
 			ib_logf(IB_LOG_LEVEL_WARN,
-				"Waited %ld seconds to drop space: %s(" ULINTPF ").",
-				now - start, space->name, space->id);
+				"Waited %ld seconds to drop space: %s (" ULINTPF
+				") active threads %u flushing=%d.",
+				now - start, space->name, space->id, cnt, flushing);
 			last = now;
 		}
 	}
@@ -2659,3 +2541,159 @@ fil_space_get_scrub_status(
 		mutex_exit(&crypt_data->mutex);
 	}
 }
+
+#endif /* !UNIV_INNOCHECKSUM */
+
+/*********************************************************************
+Verify that post encryption checksum match calculated checksum.
+This function should be called only if tablespace contains crypt_data
+metadata (this is strong indication that tablespace is encrypted).
+Function also verifies that traditional checksum does not match
+calculated checksum as if it does page could be valid unencrypted,
+encrypted, or corrupted.
+
+@param[in]	page		Page to verify
+@param[in]	zip_size	zip size
+@param[in]	space		Tablespace
+@param[in]	pageno		Page no
+@return true if page is encrypted AND OK, false otherwise */
+UNIV_INTERN
+bool
+fil_space_verify_crypt_checksum(
+	byte* 			page,
+	ulint			zip_size,
+#ifndef UNIV_INNOCHECKSUM
+	const fil_space_t*	space,
+#else
+	const void*		space,
+#endif
+	ulint			pageno)
+{
+	uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+
+	/* If page is not encrypted, return false */
+	if (key_version == 0) {
+		return(false);
+	}
+
+	srv_checksum_algorithm_t algorithm =
+			static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
+
+	/* If no checksum is used, can't continue checking. */
+	if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
+		return(true);
+	}
+
+	/* Read stored post encryption checksum. */
+	ib_uint32_t checksum = mach_read_from_4(
+		page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
+
+	/* Declare empty pages non-corrupted */
+	if (checksum == 0
+	    && *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
+	    && buf_page_is_zeroes(page, zip_size)) {
+		return(true);
+	}
+
+	/* Compressed and encrypted pages do not have checksum. Assume not
+	corrupted. Page verification happens after decompression in
+	buf_page_io_complete() using buf_page_is_corrupted(). */
+	if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+		return (true);
+	}
+
+	ib_uint32_t cchecksum1 = 0;
+	ib_uint32_t cchecksum2 = 0;
+
+	/* Calculate checksums */
+	if (zip_size) {
+		cchecksum1 = page_zip_calc_checksum(
+			page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
+
+		cchecksum2 = (cchecksum1 == checksum)
+			? 0
+			: page_zip_calc_checksum(
+				page, zip_size,
+				SRV_CHECKSUM_ALGORITHM_INNODB);
+	} else {
+		cchecksum1 = buf_calc_page_crc32(page);
+		cchecksum2 = (cchecksum1 == checksum)
+			? 0
+			: buf_calc_page_new_checksum(page);
+	}
+
+	/* If stored checksum matches one of the calculated checksums
+	page is not corrupted. */
+
+	bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
+		|| checksum == BUF_NO_CHECKSUM_MAGIC);
+
+	/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
+	first page of each system tablespace file at
+	FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
+	the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
+	(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
+	than page 0 of the system tablespace.
+
+	Starting from MariaDB 10.1 the field has been repurposed for
+	encryption key_version.
+
+	Starting with MySQL 5.7 (and MariaDB Server 10.2), the
+	field has been repurposed for SPATIAL INDEX pages for
+	FIL_RTREE_SPLIT_SEQ_NUM.
+
+	Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
+	checksum.
+
+	Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
+	field would usually be 0 for pages that are not encrypted, we cannot
+	assume that a nonzero value means that the page is encrypted.
+	Therefore we must validate the page both as encrypted and unencrypted
+	when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
+	*/
+
+	uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM);
+	uint32_t checksum2;
+
+	bool valid;
+
+	if (zip_size) {
+		valid = (checksum1 == cchecksum1);
+		checksum2 = checksum1;
+	} else {
+		checksum2 = mach_read_from_4(
+			page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
+		valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
+		|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
+	}
+
+	if (encrypted && valid) {
+		/* If page is encrypted and traditional checksums match,
+		page could be still encrypted, or not encrypted and valid or
+		corrupted. */
+#ifndef UNIV_INNOCHECKSUM
+		ib_logf(IB_LOG_LEVEL_ERROR,
+			" Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted."
+			" Post encryption checksum %u stored [%u:%u] key_version %u",
+			pageno,
+			space ? space->name : "N/A",
+			mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
+			checksum, checksum1, checksum2, key_version);
+#else
+		if (log_file) {
+			fprintf(log_file,
+				"Page " ULINTPF ":" ULINTPF " may be corrupted."
+				" Post encryption checksum %u"
+				" stored [%u:%u] key_version %u\n",
+				pageno,
+				mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
+				checksum, checksum1, checksum2,
+				key_version);
+		}
+#endif /* UNIV_INNOCHECKSUM */
+
+		encrypted = false;
+	}
+
+	return(encrypted);
+}

storage/innobase/include/buf0buf.h

@@ -31,6 +31,7 @@ Created 11/5/1995 Heikki Tuuri
 #include "fil0fil.h"
 #include "mtr0types.h"
 #include "buf0types.h"
+#ifndef UNIV_INNOCHECKSUM
 #include "hash0hash.h"
 #include "ut0byte.h"
 #include "page0types.h"
@@ -643,6 +644,8 @@ buf_block_unfix(
 # define buf_block_modify_clock_inc(block) ((void) 0)
 #endif /* !UNIV_HOTBACKUP */
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 /** Checks if the page is in crc32 checksum format.
 @param[in]	read_buf	database page
 @param[in]	checksum_field1	new checksum field
@@ -691,8 +694,13 @@ buf_page_is_corrupted(
 	bool			check_lsn,
 	const byte*		read_buf,
 	ulint			zip_size,
+#ifndef UNIV_INNOCHECKSUM
 	const fil_space_t* 	space)
+#else
+	const void* 	 	space = NULL)
+#endif
 	MY_ATTRIBUTE((warn_unused_result));
+
 /** Check if a page is all zeroes.
 @param[in]	read_buf	database page
 @param[in]	zip_size	ROW_FORMAT=COMPRESSED page size, or 0
@@ -700,6 +708,9 @@ buf_page_is_corrupted(
 UNIV_INTERN
 bool
 buf_page_is_zeroes(const byte* read_buf, ulint zip_size);
+
+#ifndef UNIV_INNOCHECKSUM
+
 #ifndef UNIV_HOTBACKUP
 /**********************************************************************//**
 Gets the space id, page offset, and byte offset within page of a
@@ -2470,4 +2481,5 @@ struct	CheckUnzipLRUAndLRUList {
 #include "buf0buf.ic"
 #endif
 
+#endif /*! UNIV_INNOCHECKSUM */
 #endif

storage/innobase/include/fil0crypt.h

@@ -71,9 +71,11 @@ struct key_struct
 /** is encryption enabled */
 extern ulong	srv_encrypt_tables;
 
+#ifndef UNIV_INNOCHECKSUM
 #ifdef UNIV_PFS_MUTEX
 extern mysql_pfs_key_t fil_crypt_data_mutex_key;
 #endif
+#endif /* !UNIV_INNOCHECKSUM */
 
 /** Mutex helper for crypt_data->scheme
 @param[in, out]	schme	encryption scheme
@@ -102,6 +104,8 @@ struct fil_space_rotate_state_t
 	} scrubbing;
 };
 
+#ifndef UNIV_INNOCHECKSUM
+
 struct fil_space_crypt_t : st_encryption_scheme
 {
  public:
@@ -399,6 +403,8 @@ fil_crypt_calculate_checksum(
 	const byte*	dst_frame)
 	MY_ATTRIBUTE((warn_unused_result));
 
+#endif /* UNIV_INNOCHECKSUM */
+
 /*********************************************************************
 Verify that post encryption checksum match calculated checksum.
 This function should be called only if tablespace contains crypt_data
@@ -417,10 +423,16 @@ bool
 fil_space_verify_crypt_checksum(
 	byte* 			page,
 	ulint			zip_size,
+#ifndef UNIV_INNOCHECKSUM
 	const fil_space_t*	space,
+#else
+	const void*		space,
+#endif
 	ulint			pageno)
 	MY_ATTRIBUTE((warn_unused_result));
 
+#ifndef UNIV_INNOCHECKSUM
+
 /*********************************************************************
 Adjust thread count for key rotation
 @param[in]	enw_cnt		Number of threads to be used */
@@ -508,4 +520,5 @@ fil_space_get_scrub_status(
 #include "fil0crypt.ic"
 #endif
 
+#endif /* !UNIV_INNOCHECKSUM */
 #endif /* fil0crypt_h */

storage/innobase/include/fil0fil.h

@@ -183,8 +183,6 @@ extern fil_addr_t	fil_addr_null;
 #define FIL_LOG			502	/*!< redo log */
 /* @} */
 
-#ifndef UNIV_INNOCHECKSUM
-
 /** Structure containing encryption specification */
 struct fil_space_crypt_t;
 
@@ -209,6 +207,10 @@ extern ulint	fil_n_pending_tablespace_flushes;
 /** Number of files currently open */
 extern ulint	fil_n_file_opened;
 
+#ifndef UNIV_INNOCHECKSUM
+
+struct fil_space_t;
+
 struct fsp_open_info {
 	ibool		success;	/*!< Has the tablespace been opened? */
 	const char*	check_msg;	/*!< fil_check_first_page() message */
@@ -225,8 +227,6 @@ struct fsp_open_info {
 	dict_table_t*	table;		/*!< table */
 };
 
-struct fil_space_t;
-
 /** File node of a tablespace or the log data space */
 struct fil_node_t {
 	fil_space_t*	space;	/*!< backpointer to the space where this node

storage/innobase/include/mach0data.ic

@@ -154,6 +154,8 @@ mach_read_from_3(
 		);
 }
 
+#endif /* !UNIV_INNOCHECKSUM */
+
 /*******************************************************//**
 The following function is used to store data in four consecutive
 bytes. We store the most significant byte to the lowest address. */
@@ -172,8 +174,6 @@ mach_write_to_4(
 	b[3] = (byte) n;
 }
 
-#endif /* !UNIV_INNOCHECKSUM */
-
 /********************************************************//**
 The following function is used to fetch data from 4 consecutive
 bytes. The most significant byte is at the lowest address.

storage/innobase/include/univ.i

@@ -482,6 +482,12 @@ typedef long int		lint;
 typedef unsigned long long int	ullint;
 #endif /* UNIV_HOTBACKUP */
 
+#ifdef UNIV_INNOCHECKSUM
+extern bool 		strict_verify;
+extern FILE* 		log_file;
+extern ulint		cur_page_num;
+#endif /* UNIV_INNOCHECKSUM */
+
 #ifndef __WIN__
 #if SIZEOF_LONG != SIZEOF_VOIDP
 #error "Error: InnoDB's ulint must be of the same size as void*"

storage/innobase/page/page0zip.cc

@@ -4937,26 +4937,26 @@ page_zip_verify_checksum(
 #error "FIL_PAGE_LSN must be 64 bit aligned"
 #endif
 
-#ifndef UNIV_INNOCHECKSUM
-	/* innochecksum doesn't compile with ut_d. Since we don't
-	need to check for empty pages when running innochecksum,
-	just don't include this code. */
 	/* Check if page is empty */
 	if (stored == 0
 	    && *reinterpret_cast<const ib_uint64_t*>(static_cast<const char*>(
 		data)
 		+ FIL_PAGE_LSN) == 0) {
 		/* make sure that the page is really empty */
-		ulint i;
-		for (i = 0; i < size; i++) {
+		for (ulint i = 0; i < size; i++) {
 			if (*((const char*) data + i) != 0) {
 				return(FALSE);
 			}
 		}
+#ifdef UNIV_INNOCHECKSUM
+		if (log_file) {
+			fprintf(log_file, "Page::%lu is empty and"
+				" uncorrupted\n", cur_page_num);
+		}
+#endif /* UNIV_INNOCHECKSUM */
 		/* Empty page */
 		return(TRUE);
 	}
-#endif
 
 	const srv_checksum_algorithm_t	curr_algo =
 		static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
@@ -4968,6 +4968,33 @@ page_zip_verify_checksum(
 	calc = static_cast<ib_uint32_t>(page_zip_calc_checksum(
 		data, size, curr_algo));
 
+#ifdef UNIV_INNOCHECKSUM
+	if (log_file) {
+		fprintf(log_file, "page::%lu;"
+			" %s checksum: calculated = %u;"
+			" recorded = %u\n", cur_page_num,
+			buf_checksum_algorithm_name(
+				static_cast<srv_checksum_algorithm_t>(
+				srv_checksum_algorithm)),
+			calc, stored);
+	}
+
+	if (!strict_verify) {
+
+		const uint32_t	crc32 = page_zip_calc_checksum(
+			data, size, SRV_CHECKSUM_ALGORITHM_CRC32);
+
+		if (log_file) {
+			fprintf(log_file, "page::%lu: crc32 checksum:"
+				" calculated = %u; recorded = %u\n",
+				cur_page_num, crc32, stored);
+			fprintf(log_file, "page::%lu: none checksum:"
+				" calculated = %lu; recorded = %u\n",
+				cur_page_num, BUF_NO_CHECKSUM_MAGIC, stored);
+		}
+	}
+#endif /* UNIV_INNOCHECKSUM */
+
 	if (stored == calc) {
 		return(TRUE);
 	}

storage/xtradb/fil/fil0crypt.cc

@@ -25,13 +25,18 @@ Modified           Jan Lindström jan.lindstrom@mariadb.com
 
 #include "fil0fil.h"
 #include "fil0crypt.h"
+#include "mach0data.h"
+#include "page0zip.h"
+#include "buf0buf.h"
+#include "buf0checksum.h"
+
+#ifndef UNIV_INNOCHECKSUM
+
 #include "srv0srv.h"
 #include "srv0start.h"
-#include "mach0data.h"
 #include "log0recv.h"
 #include "mtr0mtr.h"
 #include "mtr0log.h"
-#include "page0zip.h"
 #include "ut0ut.h"
 #include "btr0scrub.h"
 #include "fsp0fsp.h"
@@ -107,13 +112,20 @@ UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key;
 extern my_bool srv_background_scrub_data_uncompressed;
 extern my_bool srv_background_scrub_data_compressed;
 
+/***********************************************************************
+Check if a key needs rotation given a key_state
+@param[in]	encrypt_mode		Encryption mode
+@param[in]	key_version		Current key version
+@param[in]	latest_key_version	Latest key version
+@param[in]	rotate_key_age		when to rotate
+@return true if key needs rotation, false if not */
 static bool
 fil_crypt_needs_rotation(
-	fil_encryption_t        encrypt_mode,           /*!< in: Encryption
-							mode */
-	uint			key_version,		/*!< in: Key version */
-	uint			latest_key_version,	/*!< in: Latest key version */
-	uint			rotate_key_age);	/*!< in: When to rotate */
+	fil_encryption_t	encrypt_mode,
+	uint			key_version,
+	uint			latest_key_version,
+	uint			rotate_key_age)
+	MY_ATTRIBUTE((warn_unused_result));
 
 /*********************************************************************
 Init space crypt */
@@ -908,137 +920,6 @@ fil_crypt_calculate_checksum(
 	return checksum;
 }
 
-/*********************************************************************
-Verify that post encryption checksum match calculated checksum.
-This function should be called only if tablespace contains crypt_data
-metadata (this is strong indication that tablespace is encrypted).
-Function also verifies that traditional checksum does not match
-calculated checksum as if it does page could be valid unencrypted,
-encrypted, or corrupted.
-
-@param[in]	page		Page to verify
-@param[in]	zip_size	zip size
-@param[in]	space		Tablespace
-@param[in]	pageno		Page no
-@return true if page is encrypted AND OK, false otherwise */
-UNIV_INTERN
-bool
-fil_space_verify_crypt_checksum(
-	byte* 			page,
-	ulint			zip_size,
-	const fil_space_t*	space,
-	ulint			pageno)
-{
-	uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
-
-	/* If page is not encrypted, return false */
-	if (key_version == 0) {
-		return(false);
-	}
-
-	/* Read stored post encryption checksum. */
-	ib_uint32_t checksum = mach_read_from_4(
-		page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
-
-	/* Declare empty pages non-corrupted */
-	if (checksum == 0
-	    && *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
-	    && buf_page_is_zeroes(page, zip_size)) {
-		return(true);
-	}
-
-	/* Compressed and encrypted pages do not have checksum. Assume not
-	corrupted. Page verification happens after decompression in
-	buf_page_io_complete() using buf_page_is_corrupted(). */
-	if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
-		return (true);
-	}
-
-	ib_uint32_t cchecksum1 = 0;
-	ib_uint32_t cchecksum2 = 0;
-
-	/* Calculate checksums */
-	if (zip_size) {
-		cchecksum1 = page_zip_calc_checksum(
-			page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
-
-		if(cchecksum1 != checksum) {
-			cchecksum2 = page_zip_calc_checksum(
-				page, zip_size,
-				SRV_CHECKSUM_ALGORITHM_INNODB);
-		}
-	} else {
-		cchecksum1 = buf_calc_page_crc32(page);
-
-		if (cchecksum1 != checksum) {
-			cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum(
-				page);
-		}
-	}
-
-	/* If stored checksum matches one of the calculated checksums
-	page is not corrupted. */
-
-	bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
-		|| checksum == BUF_NO_CHECKSUM_MAGIC);
-
-	/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
-	first page of each system tablespace file at
-	FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
-	the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
-	(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
-	than page 0 of the system tablespace.
-
-	Starting from MariaDB 10.1 the field has been repurposed for
-	encryption key_version.
-
-	Starting with MySQL 5.7 (and MariaDB Server 10.2), the
-	field has been repurposed for SPATIAL INDEX pages for
-	FIL_RTREE_SPLIT_SEQ_NUM.
-
-	Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
-	checksum.
-
-	Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
-	field would usually be 0 for pages that are not encrypted, we cannot
-	assume that a nonzero value means that the page is encrypted.
-	Therefore we must validate the page both as encrypted and unencrypted
-	when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
-	*/
-
-	ulint checksum1 = mach_read_from_4(
-		page + FIL_PAGE_SPACE_OR_CHKSUM);
-
-	ulint checksum2 = checksum1;
-
-	bool valid;
-
-	if (zip_size) {
-		valid = (checksum1 == cchecksum1);
-	} else {
-		checksum1 = mach_read_from_4(
-			page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
-		valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
-		|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
-	}
-
-	if (encrypted && valid) {
-		/* If page is encrypted and traditional checksums match,
-		page could be still encrypted, or not encrypted and valid or
-		corrupted. */
-		ib_logf(IB_LOG_LEVEL_ERROR,
-			" Page %lu in space %s (%lu) maybe corrupted."
-			" Post encryption checksum %u stored [%lu:%lu] key_version %u",
-			pageno,
-			space ? space->name : "N/A",
-			mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
-			checksum, checksum1, checksum2, key_version);
-		encrypted = false;
-	}
-
-	return(encrypted);
-}
-
 /***********************************************************************/
 
 /** A copy of global key state */
@@ -2660,3 +2541,159 @@ fil_space_get_scrub_status(
 		mutex_exit(&crypt_data->mutex);
 	}
 }
+
+#endif /* !UNIV_INNOCHECKSUM */
+
+/*********************************************************************
+Verify that post encryption checksum match calculated checksum.
+This function should be called only if tablespace contains crypt_data
+metadata (this is strong indication that tablespace is encrypted).
+Function also verifies that traditional checksum does not match
+calculated checksum as if it does page could be valid unencrypted,
+encrypted, or corrupted.
+
+@param[in]	page		Page to verify
+@param[in]	zip_size	zip size
+@param[in]	space		Tablespace
+@param[in]	pageno		Page no
+@return true if page is encrypted AND OK, false otherwise */
+UNIV_INTERN
+bool
+fil_space_verify_crypt_checksum(
+	byte* 			page,
+	ulint			zip_size,
+#ifndef UNIV_INNOCHECKSUM
+	const fil_space_t*	space,
+#else
+	const void*		space,
+#endif
+	ulint			pageno)
+{
+	uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+
+	/* If page is not encrypted, return false */
+	if (key_version == 0) {
+		return(false);
+	}
+
+	srv_checksum_algorithm_t algorithm =
+			static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
+
+	/* If no checksum is used, can't continue checking. */
+	if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
+		return(true);
+	}
+
+	/* Read stored post encryption checksum. */
+	ib_uint32_t checksum = mach_read_from_4(
+		page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
+
+	/* Declare empty pages non-corrupted */
+	if (checksum == 0
+	    && *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
+	    && buf_page_is_zeroes(page, zip_size)) {
+		return(true);
+	}
+
+	/* Compressed and encrypted pages do not have checksum. Assume not
+	corrupted. Page verification happens after decompression in
+	buf_page_io_complete() using buf_page_is_corrupted(). */
+	if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+		return (true);
+	}
+
+	ib_uint32_t cchecksum1 = 0;
+	ib_uint32_t cchecksum2 = 0;
+
+	/* Calculate checksums */
+	if (zip_size) {
+		cchecksum1 = page_zip_calc_checksum(
+			page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
+
+		cchecksum2 = (cchecksum1 == checksum)
+			? 0
+			: page_zip_calc_checksum(
+				page, zip_size,
+				SRV_CHECKSUM_ALGORITHM_INNODB);
+	} else {
+		cchecksum1 = buf_calc_page_crc32(page);
+		cchecksum2 = (cchecksum1 == checksum)
+			? 0
+			: buf_calc_page_new_checksum(page);
+	}
+
+	/* If stored checksum matches one of the calculated checksums
+	page is not corrupted. */
+
+	bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
+		|| checksum == BUF_NO_CHECKSUM_MAGIC);
+
+	/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
+	first page of each system tablespace file at
+	FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
+	the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
+	(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
+	than page 0 of the system tablespace.
+
+	Starting from MariaDB 10.1 the field has been repurposed for
+	encryption key_version.
+
+	Starting with MySQL 5.7 (and MariaDB Server 10.2), the
+	field has been repurposed for SPATIAL INDEX pages for
+	FIL_RTREE_SPLIT_SEQ_NUM.
+
+	Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
+	checksum.
+
+	Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
+	field would usually be 0 for pages that are not encrypted, we cannot
+	assume that a nonzero value means that the page is encrypted.
+	Therefore we must validate the page both as encrypted and unencrypted
+	when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
+	*/
+
+	uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM);
+	uint32_t checksum2;
+
+	bool valid;
+
+	if (zip_size) {
+		valid = (checksum1 == cchecksum1);
+		checksum2 = checksum1;
+	} else {
+		checksum2 = mach_read_from_4(
+			page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
+		valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
+		|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
+	}
+
+	if (encrypted && valid) {
+		/* If page is encrypted and traditional checksums match,
+		page could be still encrypted, or not encrypted and valid or
+		corrupted. */
+#ifndef UNIV_INNOCHECKSUM
+		ib_logf(IB_LOG_LEVEL_ERROR,
+			" Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted."
+			" Post encryption checksum %u stored [%u:%u] key_version %u",
+			pageno,
+			space ? space->name : "N/A",
+			mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
+			checksum, checksum1, checksum2, key_version);
+#else
+		if (log_file) {
+			fprintf(log_file,
+				"Page " ULINTPF ":" ULINTPF " may be corrupted."
+				" Post encryption checksum %u"
+				" stored [%u:%u] key_version %u\n",
+				pageno,
+				mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
+				checksum, checksum1, checksum2,
+				key_version);
+		}
+#endif /* UNIV_INNOCHECKSUM */
+
+		encrypted = false;
+	}
+
+	return(encrypted);
+}