Merge 10.1 into 10.2

extra/innochecksum.cc

@@ -523,7 +523,16 @@ is_page_corrupted(
 	normal method. */
 	if (is_encrypted && key_version != 0) {
 		is_corrupted = !fil_space_verify_crypt_checksum(buf,
-			page_size, space_id, (ulint)cur_page_num);
+								page_size);
+		if (is_corrupted && log_file) {
+			fprintf(log_file,
+				"Page " ULINTPF ":%llu may be corrupted;"
+				" key_version=%u\n",
+				space_id, cur_page_num,
+				mach_read_from_4(
+					FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+					+ buf));
+		}
 	} else {
 		is_corrupted = true;
 	}

extra/mariabackup/fil_cur.cc

@@ -362,9 +362,14 @@ read_retry:
                     page_no >= FSP_EXTENT_SIZE &&
                     page_no < FSP_EXTENT_SIZE * 3) {
                         /* We ignore the doublewrite buffer pages */
-                } else if (fil_space_verify_crypt_checksum(
+                } else if (mach_read_from_4(
-                                   page, cursor->page_size,
+				   page
-				   space->id, page_no)) {
+				   + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
+			   && space->crypt_data
+			   && space->crypt_data->type
+			   != CRYPT_SCHEME_UNENCRYPTED
+			   && fil_space_verify_crypt_checksum(
+                                   page, cursor->page_size)) {
 			ut_ad(mach_read_from_4(page + FIL_PAGE_SPACE_ID)
 			      == space->id);
 

mysql-test/include/innodb_encrypt_tables.combinations

@@ -3,12 +3,12 @@ innodb_encrypt_tables=ON
 plugin-load-add=$FILE_KEY_MANAGEMENT_SO
 loose-file-key-management
 loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt
-file-key-management-encryption-algorithm=aes_ctr
+file-key-management-encryption-algorithm=aes_cbc
 
 [clear]
 innodb_encrypt_tables=OFF
 plugin-load-add=$FILE_KEY_MANAGEMENT_SO
 loose-file-key-management
 loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt
-file-key-management-encryption-algorithm=aes_ctr
+file-key-management-encryption-algorithm=aes_cbc
 

mysql-test/suite/encryption/r/innodb-force-corrupt.result

@@ -1,5 +1,4 @@
-call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\.");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version=3221342974");
-call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
 SET GLOBAL innodb_file_per_table = ON;
 set global innodb_compression_algorithm = 1;
 # Create and populate tables to be corrupted

mysql-test/suite/encryption/t/innodb-force-corrupt.test

@@ -7,8 +7,7 @@
 # Don't test under embedded
 -- source include/not_embedded.inc
 
-call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\.");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version=3221342974");
-call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
 
 SET GLOBAL innodb_file_per_table = ON;
 set global innodb_compression_algorithm = 1;
@@ -50,17 +49,17 @@ perl;
 open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open";
 binmode FILE;
 seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
-print FILE pack("H*", "c00lcafedeadb017");
+print FILE pack("H*", "c001cafedeadb017");
 close FILE or die "close";
 open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open";
 binmode FILE;
 seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
-print FILE pack("H*", "c00lcafedeadb017");
+print FILE pack("H*", "c001cafedeadb017");
 close FILE or die "close";
 open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open";
 binmode FILE;
 seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
-print FILE pack("H*", "c00lcafedeadb017");
+print FILE pack("H*", "c001cafedeadb017");
 close FILE or die "close";
 EOF
 

mysql-test/suite/galera/include/have_filekeymanagement.inc

@@ -0,0 +1,3 @@
+#
+# Used in galera/suite.pm to check file key management plugin
+#

mysql-test/suite/galera/r/galera_defaults.result

@@ -52,4 +52,3 @@ WSREP_SST_DONOR
 WSREP_SST_DONOR_REJECTS_QUERIES	OFF
 WSREP_SST_METHOD	rsync
 WSREP_SYNC_WAIT	15
-<BASE_DIR>; <BASE_HOST>; <BASE_PORT>; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT30S; evs.info_log_mask = 0; evs.install_timeout = PT15S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT10S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; <GCACHE_DIR>; gcache.keep_pages_size = 0; gcache.mem_size = 0; <GCACHE_NAME>; gcache.page_size = 128M; gcache.recover = no; gcache.size = 10M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; <GCS_RECV_Q_HARD_LIMIT>; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; <GMCAST_LISTEN_ADDR>; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; <IST_RECV_ADDR>; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT90S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; <REPL_PROTO_MAX>;socket.checksum = 2; socket.recv_buf_size = 212992; 

mysql-test/suite/galera/suite.pm

@@ -96,6 +96,8 @@ if (which(socat)) {
 
 sub skip_combinations {
   my %skip = ();
+  $skip{'include/have_filekeymanagement.inc'} = 'needs file_key_management plugin'
+             unless $ENV{FILE_KEY_MANAGEMENT_SO};
   $skip{'include/have_xtrabackup.inc'} = 'Need innobackupex'
              unless which(innobackupex);
   $skip{'include/have_xtrabackup.inc'} = 'Need socat or nc'

mysql-test/suite/galera/t/galera_defaults.test

@@ -36,26 +36,3 @@ AND VARIABLE_NAME NOT IN (
         'WSREP_PATCH_VERSION'
 )
 ORDER BY VARIABLE_NAME;
-
-# wsrep_provider_options
-#
-# We replace the ones that vary from run to run with placeholders
-
---let _WSREP_PROVIDER_OPTIONS = `SELECT @@wsrep_provider_options`
---perl
-	use strict;
-	my $wsrep_provider_options = $ENV{'_WSREP_PROVIDER_OPTIONS'};
-	$wsrep_provider_options =~ s/base_dir = .*?;/<BASE_DIR>;/sgio;
-	$wsrep_provider_options =~ s/base_host = .*?;/<BASE_HOST>;/sgio;
-	$wsrep_provider_options =~ s/base_port = .*?;/<BASE_PORT>;/sgio;
-	$wsrep_provider_options =~ s/gcache\.dir = .*?;/<GCACHE_DIR>;/sgio;
-	$wsrep_provider_options =~ s/gcache\.name = .*?;/<GCACHE_NAME>;/sgio;
-	$wsrep_provider_options =~ s/gmcast\.listen_addr = .*?;/<GMCAST_LISTEN_ADDR>;/sgio;
-	$wsrep_provider_options =~ s/gcs\.recv_q_hard_limit = .*?;/<GCS_RECV_Q_HARD_LIMIT>;/sgio;
-	$wsrep_provider_options =~ s/ist\.recv_addr = .*?;/<IST_RECV_ADDR>;/sgio;
-	$wsrep_provider_options =~ s/evs\.evict = .*?;/<EVS_EVICT>;/sgio;
-	$wsrep_provider_options =~ s/signal = .*?;\s*//sgio;
-	$wsrep_provider_options =~ s/dbug = .*?;\s*//sgio;
-	$wsrep_provider_options =~ s/repl.proto_max = .*?;\s*/<REPL_PROTO_MAX>;/sgio;
-	print $wsrep_provider_options."\n";
-EOF

mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test

@@ -1,5 +1,6 @@
 --source include/big_test.inc
 --source include/galera_cluster.inc
+--source include/have_filekeymanagement.inc
 --source include/innodb_encrypt_tables.inc
 --source include/innodb_page_size_small.inc
 --source include/have_mariabackup.inc

storage/innobase/buf/buf0buf.cc

@@ -525,9 +525,13 @@ decompress_with_slot:
 			       + dst_frame)) {
 		/* Verify encryption checksum before we even try to
 		decrypt. */
-		if (!fil_space_verify_crypt_checksum(
+		if (!fil_space_verify_crypt_checksum(dst_frame, bpage->size)) {
-			    dst_frame, bpage->size, bpage->id.space(),
+			ib::error() << "Encrypted page " << bpage->id
-			    bpage->id.page_no())) {
+				    << " in file " << space->chain.start->name
+				    << " looks corrupted; key_version="
+				    << mach_read_from_4(
+					    FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+					    + dst_frame);
 decrypt_failed:
 			/* Mark page encrypted in case it should be. */
 			if (space->crypt_data->type
@@ -735,23 +739,6 @@ buf_block_alloc(
 }
 #endif /* !UNIV_INNOCHECKSUM */
 
-/** Checks if a page contains only zeroes.
-@param[in]	read_buf	database page
-@param[in]	page_size	page size
-@return true if page is filled with zeroes */
-bool
-buf_page_is_zeroes(
-	const byte*		read_buf,
-	const page_size_t&	page_size)
-{
-	for (ulint i = 0; i < page_size.logical(); i++) {
-		if (read_buf[i] != 0) {
-			return(false);
-		}
-	}
-	return(true);
-}
-
 /** Checks if the page is in crc32 checksum format.
 @param[in]	read_buf		database page
 @param[in]	checksum_field1		new checksum field
@@ -5843,18 +5830,14 @@ or decrypt/decompress just failed.
 @retval	DB_DECRYPTION_FAILED	if page post encryption checksum matches but
 after decryption normal page checksum does not match.
 @retval	DB_TABLESPACE_DELETED	if accessed tablespace is not found */
-static
+static dberr_t buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
-dberr_t
-buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
 {
 	ut_ad(space->n_pending_ios > 0);
 
 	byte* dst_frame = (bpage->zip.data) ? bpage->zip.data :
 		((buf_block_t*) bpage)->frame;
-	bool still_encrypted = false;
 	dberr_t err = DB_SUCCESS;
 	bool corrupted = false;
-	fil_space_crypt_t* crypt_data = space->crypt_data;
 
 	/* In buf_decrypt_after_read we have either decrypted the page if
 	page post encryption checksum matches and used key_id is found
@@ -5862,12 +5845,12 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
 	not decrypted and it could be either encrypted and corrupted
 	or corrupted or good page. If we decrypted, there page could
 	still be corrupted if used key does not match. */
-	still_encrypted = crypt_data
+	const bool still_encrypted = mach_read_from_4(
-		&& crypt_data->type != CRYPT_SCHEME_UNENCRYPTED
+		dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
+		&& space->crypt_data
+		&& space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED
 		&& !bpage->encrypted
-		&& fil_space_verify_crypt_checksum(
+		&& fil_space_verify_crypt_checksum(dst_frame, bpage->size);
-			dst_frame, bpage->size,
-			bpage->id.space(), bpage->id.page_no());
 
 	if (!still_encrypted) {
 		/* If traditional checksums match, we assume that page is

storage/innobase/buf/buf0dblwr.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2013, 2017, MariaDB Corporation.
+Copyright (c) 2013, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -338,6 +338,20 @@ too_small:
 	goto start_again;
 }
 
+/** Check if a page is all zeroes.
+@param[in]	read_buf	database page
+@param[in]	page_size	page frame size
+@return	whether the page is all zeroes */
+static bool buf_page_is_zeroes(const byte* read_buf, size_t page_size)
+{
+	for (ulint i = 0; i < page_size; i++) {
+		if (read_buf[i] != 0) {
+			return false;
+		}
+	}
+	return true;
+}
+
 /**
 At database startup initializes the doublewrite buffer memory structure if
 we already have a doublewrite buffer created in the data files. If we are
@@ -573,7 +587,7 @@ buf_dblwr_process()
 		}
 
 		const page_size_t	page_size(space->flags);
-		ut_ad(!buf_page_is_zeroes(page, page_size));
+		ut_ad(!buf_page_is_zeroes(page, page_size.physical()));
 
 		/* We want to ensure that for partial reads the
 		unread portion of the page is NUL. */
@@ -597,7 +611,9 @@ buf_dblwr_process()
 		}
 
 		const bool is_all_zero = buf_page_is_zeroes(
-			read_buf, page_size);
+			read_buf, page_size.physical());
+		const bool expect_encrypted = space->crypt_data
+			&& space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED;
 
 		if (is_all_zero) {
 			/* We will check if the copy in the
@@ -613,10 +629,13 @@ buf_dblwr_process()
 				goto bad;
 			}
 
-			if (fil_space_verify_crypt_checksum(
+			if (expect_encrypted && mach_read_from_4(
-				    read_buf, page_size, space_id, page_no)
+				    read_buf
-			    || !buf_page_is_corrupted(
+				    + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
-				    true, read_buf, page_size, space)) {
+			    ? fil_space_verify_crypt_checksum(read_buf,
+							      page_size)
+			    : !buf_page_is_corrupted(true, read_buf,
+						     page_size, space)) {
 				/* The page is good; there is no need
 				to consult the doublewrite buffer. */
 				continue;
@@ -635,9 +654,11 @@ bad:
 				&& page_size.is_compressed())) {
 			goto bad_doublewrite;
 		}
-		if (!fil_space_verify_crypt_checksum(page, page_size,
+
-						     space_id, page_no)
+		if (expect_encrypted && mach_read_from_4(
-		    && buf_page_is_corrupted(true, page, page_size, space)) {
+			    page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
+		    ? !fil_space_verify_crypt_checksum(page, page_size)
+		    : buf_page_is_corrupted(true, page, page_size, space)) {
 			if (!is_all_zero) {
 bad_doublewrite:
 				ib::warn() << "A doublewrite copy of page "

storage/innobase/fil/fil0crypt.cc

@@ -616,8 +616,7 @@ fil_encrypt_buf(
 	// store the post-encryption checksum after the key-version
 	mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum);
 
-	ut_ad(fil_space_verify_crypt_checksum(dst_frame, page_size,
+	ut_ad(fil_space_verify_crypt_checksum(dst_frame, page_size));
-					      space, offset));
 
 	srv_stats.pages_encrypted.inc();
 
@@ -2533,164 +2532,68 @@ encrypted, or corrupted.
 
 @param[in,out]	page		page frame (checksum is temporarily modified)
 @param[in]	page_size	page size
-@param[in]	space		tablespace identifier
+@return whether the encrypted page is OK */
-@param[in]	offset		page number
-@return true if page is encrypted AND OK, false otherwise */
-UNIV_INTERN
 bool
-fil_space_verify_crypt_checksum(
+fil_space_verify_crypt_checksum(const byte* page, const page_size_t& page_size)
-	byte* 			page,
-	const page_size_t&	page_size,
-	ulint			space,
-	ulint			offset)
 {
-	uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+	ut_ad(mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION));
-
-	/* If page is not encrypted, return false */
-	if (key_version == 0) {
-		return false;
-	}
-
-	/* Read stored post encryption checksum. */
-	uint32_t checksum = mach_read_from_4(
-		page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
-
-	/* Declare empty pages non-corrupted */
-	if (checksum == 0
-	    && *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
-	    && buf_page_is_zeroes(page, page_size)) {
-		return(true);
-	}
 
 	/* Compressed and encrypted pages do not have checksum. Assume not
 	corrupted. Page verification happens after decompression in
 	buf_page_io_complete() using buf_page_is_corrupted(). */
-	if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+	if (mach_read_from_2(page + FIL_PAGE_TYPE)
-		return (true);
+	    == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+		return true;
 	}
 
-	uint32_t cchecksum1, cchecksum2;
+	/* Read stored post encryption checksum. */
-
+	const ib_uint32_t checksum = mach_read_from_4(
-	/* Calculate checksums */
+		page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
-	if (page_size.is_compressed()) {
-		cchecksum1 = page_zip_calc_checksum(
-			page, page_size.physical(),
-			SRV_CHECKSUM_ALGORITHM_CRC32);
-
-		cchecksum2 = (cchecksum1 == checksum)
-			? 0
-			: page_zip_calc_checksum(
-				page, page_size.physical(),
-				SRV_CHECKSUM_ALGORITHM_INNODB);
-	} else {
-		cchecksum1 = buf_calc_page_crc32(page);
-		cchecksum2 = (cchecksum1 == checksum)
-			? 0
-			: buf_calc_page_new_checksum(page);
-	}
 
 	/* If stored checksum matches one of the calculated checksums
 	page is not corrupted. */
+	srv_checksum_algorithm_t algorithm = srv_checksum_algorithm_t(
+		srv_checksum_algorithm);
 
-	bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
+	switch (algorithm) {
-		|| checksum == BUF_NO_CHECKSUM_MAGIC);
+	case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
-
+		if (page_size.is_compressed()) {
-	/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
+			return checksum == page_zip_calc_checksum(
-	first page of each system tablespace file at
+				page, page_size.physical(),
-	FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
+				SRV_CHECKSUM_ALGORITHM_CRC32);
-	the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
-	(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
-	than page 0 of the system tablespace.
-
-	Starting from MariaDB 10.1 the field has been repurposed for
-	encryption key_version.
-
-	Starting with MySQL 5.7 (and MariaDB Server 10.2), the
-	field has been repurposed for SPATIAL INDEX pages for
-	FIL_RTREE_SPLIT_SEQ_NUM.
-
-	Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
-	checksum.
-
-	Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
-	field would usually be 0 for pages that are not encrypted, we cannot
-	assume that a nonzero value means that the page is encrypted.
-	Therefore we must validate the page both as encrypted and unencrypted
-	when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
-	*/
-
-	uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM);
-	uint32_t checksum2;
-
-	bool valid = false;
-
-	if (page_size.is_compressed()) {
-		valid = checksum1 == cchecksum1;
-		checksum2 = checksum1;
-	} else {
-		checksum2 = mach_read_from_4(
-			page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
-
-		srv_checksum_algorithm_t algorithm =
-			static_cast<srv_checksum_algorithm_t>(
-					srv_checksum_algorithm);
-		switch (algorithm) {
-		case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
-			valid = buf_page_is_checksum_valid_crc32(
-				page, checksum1, checksum2);
-			break;
-		case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
-			valid = buf_page_is_checksum_valid_innodb(
-				page, checksum1, checksum2);
-			break;
-		case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
-		case SRV_CHECKSUM_ALGORITHM_CRC32:
-		case SRV_CHECKSUM_ALGORITHM_INNODB:
-		case SRV_CHECKSUM_ALGORITHM_NONE:
-			/* never supported
-			innodb_checksum_algorithm=none or strict_none
-			for encrypted pages. */
-			valid = buf_page_is_checksum_valid_crc32(
-				page, checksum1, checksum2)
-				|| buf_page_is_checksum_valid_innodb(
-					page, checksum1, checksum2);
-			break;
 		}
-	}
 
-	if (encrypted && valid) {
+		return checksum == buf_calc_page_crc32(page);
-		/* If page is encrypted and traditional checksums match,
+	case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
-		page could be still encrypted, or not encrypted and valid or
+		if (page_size.is_compressed()) {
-		corrupted. */
+			return checksum == page_zip_calc_checksum(
-#ifdef UNIV_INNOCHECKSUM
+				page, page_size.physical(),
-		fprintf(log_file ? log_file : stderr,
+				SRV_CHECKSUM_ALGORITHM_INNODB);
-			"Page " ULINTPF ":" ULINTPF " may be corrupted."
+		}
-			" Post encryption checksum %u"
+		return checksum == buf_calc_page_new_checksum(page);
-			" stored [%u:%u] key_version %u\n",
+	case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
-			space, offset, checksum, checksum1, checksum2,
+		return checksum == BUF_NO_CHECKSUM_MAGIC;
-			key_version);
+	case SRV_CHECKSUM_ALGORITHM_NONE:
-#else /* UNIV_INNOCHECKSUM */
+		return true;
-		ib::error()
+	case SRV_CHECKSUM_ALGORITHM_INNODB:
-			<< " Page " << space << ":" << offset
+	case SRV_CHECKSUM_ALGORITHM_CRC32:
-			<< " may be corrupted.";
+		if (checksum == BUF_NO_CHECKSUM_MAGIC) {
-		ib::info()
+			return true;
-			<< "If encrypted:  stored checksum" << checksum
+		}
-			<< " calculated checksum [" << cchecksum1 << ":" << cchecksum2
+		if (page_size.is_compressed()) {
-			<< "] key_version " << key_version;
+			if (checksum == page_zip_calc_checksum(
-		ib::info()
+				    page, page_size.physical(), algorithm)) {
-			<< "If unencrypted: stored checksum [" << checksum1
+				return true;
-			<< ":" << checksum2 << "] calculated crc32 ["
+			}
-			<< buf_calc_page_crc32(page)
-# ifdef INNODB_BUG_ENDIAN_CRC32
-			<< ":" << buf_calc_page_crc32(page, true)
-# endif /* INNODB_BUG_ENDIAN_CRC32 */
-			<< "] innodb ["
-			<< buf_calc_page_old_checksum(page) << ":"
-			<< buf_calc_page_new_checksum(page) << "] LSN "
-			<< mach_read_from_4(page + FIL_PAGE_LSN);
-#endif
-		encrypted = false;
-	}
 
-	return(encrypted);
+			algorithm = algorithm == SRV_CHECKSUM_ALGORITHM_INNODB
+				? SRV_CHECKSUM_ALGORITHM_CRC32
+				: SRV_CHECKSUM_ALGORITHM_INNODB;
+			return checksum == page_zip_calc_checksum(
+				page, page_size.physical(), algorithm);
+		}
+
+		return checksum == buf_calc_page_crc32(page)
+			|| checksum == buf_calc_page_new_checksum(page);
+	}
 }

storage/innobase/ibuf/ibuf0ibuf.cc

@@ -4944,6 +4944,20 @@ ibuf_print(
 	mutex_exit(&ibuf_mutex);
 }
 
+/** Check if a page is all zeroes.
+@param[in]	read_buf	database page
+@param[in]	size		page size
+@return	whether the page is all zeroes */
+static bool buf_page_is_zeroes(const byte* read_buf, const page_size_t& size)
+{
+	for (ulint i = 0; i < size.physical(); i++) {
+		if (read_buf[i] != 0) {
+			return false;
+		}
+	}
+	return true;
+}
+
 /******************************************************************//**
 Checks the insert buffer bitmaps on IMPORT TABLESPACE.
 @return DB_SUCCESS or error code */

storage/innobase/include/buf0buf.h

@@ -748,15 +748,6 @@ buf_page_is_checksum_valid_none(
 	ulint				checksum_field2)
 	MY_ATTRIBUTE((nonnull(1), warn_unused_result));
 
-/** Checks if a page contains only zeroes.
-@param[in]	read_buf	database page
-@param[in]	page_size	page size
-@return true if page is filled with zeroes */
-bool
-buf_page_is_zeroes(
-	const byte*		read_buf,
-	const page_size_t&	page_size);
-
 /** Check if a page is corrupt.
 @param[in]	check_lsn	whether the LSN should be checked
 @param[in]	read_buf	database page
@@ -775,9 +766,7 @@ buf_page_is_corrupted(
 #endif
 	MY_ATTRIBUTE((warn_unused_result));
 
-
 #ifndef UNIV_INNOCHECKSUM
-
 /**********************************************************************//**
 Gets the space id, page offset, and byte offset within page of a
 pointer pointing to a buffer frame containing a file page. */

storage/innobase/include/fil0crypt.h

@@ -1,6 +1,6 @@
 /*****************************************************************************
 Copyright (C) 2013, 2015, Google Inc. All Rights Reserved.
-Copyright (c) 2015, 2017, MariaDB Corporation.
+Copyright (c) 2015, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -490,16 +490,9 @@ encrypted, or corrupted.
 
 @param[in,out]	page		page frame (checksum is temporarily modified)
 @param[in]	page_size	page size
-@param[in]	space		tablespace identifier
-@param[in]	offset		page number
 @return true if page is encrypted AND OK, false otherwise */
-UNIV_INTERN
 bool
-fil_space_verify_crypt_checksum(
+fil_space_verify_crypt_checksum(const byte* page, const page_size_t& page_size)
-	byte* 			page,
-	const page_size_t&	page_size,
-	ulint			space,
-	ulint			offset)
 	MY_ATTRIBUTE((warn_unused_result));
 
 #endif /* fil0crypt_h */

storage/innobase/row/row0import.cc

@@ -3447,9 +3447,7 @@ not_encrypted:
 				}
 			} else {
 				if (!fil_space_verify_crypt_checksum(
-					    src, callback.get_page_size(),
+					    src, callback.get_page_size())) {
-					    block->page.id.space(),
-					    block->page.id.page_no())) {
 					goto page_corrupted;
 				}
 

storage/xtradb/buf/buf0buf.cc

@@ -481,8 +481,15 @@ decompress_with_slot:
 		/* Verify encryption checksum before we even try to
 		decrypt. */
 		if (!fil_space_verify_crypt_checksum(
-			    dst_frame, buf_page_get_zip_size(bpage), NULL,
+			    dst_frame, buf_page_get_zip_size(bpage))) {
-			    bpage->offset)) {
+			ib_logf(IB_LOG_LEVEL_ERROR,
+				"Encrypted page %u:%u in file %s"
+				" looks corrupted; key_version=" ULINTPF,
+				bpage->space, bpage->offset,
+				space->chain.start->name,
+				mach_read_from_4(
+					FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+					+ dst_frame));
 decrypt_failed:
 			/* Mark page encrypted in case it should be. */
 			if (space->crypt_data->type
@@ -728,24 +735,6 @@ buf_block_alloc(
 }
 #endif /* !UNIV_HOTBACKUP */
 
-/** Check if a page is all zeroes.
-@param[in]	read_buf	database page
-@param[in]	zip_size	ROW_FORMAT=COMPRESSED page size, or 0
-@return	whether the page is all zeroes */
-UNIV_INTERN
-bool
-buf_page_is_zeroes(const byte* read_buf, ulint zip_size)
-{
-	const ulint page_size = zip_size ? zip_size : UNIV_PAGE_SIZE;
-
-	for (ulint i = 0; i < page_size; i++) {
-		if (read_buf[i] != 0) {
-			return(false);
-		}
-	}
-	return(true);
-}
-
 /** Checks if the page is in crc32 checksum format.
 @param[in]	read_buf	database page
 @param[in]	checksum_field1	new checksum field
@@ -4780,19 +4769,15 @@ or decrypt/decompress just failed.
 @retval	DB_DECRYPTION_FAILED	if page post encryption checksum matches but
 after decryption normal page checksum does not match.
 @retval	DB_TABLESPACE_DELETED	if accessed tablespace is not found */
-static
+static dberr_t buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
-dberr_t
-buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
 {
 	ut_ad(space->n_pending_ios > 0);
 
 	ulint zip_size = buf_page_get_zip_size(bpage);
 	byte* dst_frame = (zip_size) ? bpage->zip.data :
 		((buf_block_t*) bpage)->frame;
-	bool still_encrypted = false;
 	dberr_t err = DB_SUCCESS;
 	bool corrupted = false;
-	fil_space_crypt_t* crypt_data = space->crypt_data;
 
 	/* In buf_decrypt_after_read we have either decrypted the page if
 	page post encryption checksum matches and used key_id is found
@@ -4800,11 +4785,12 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
 	not decrypted and it could be either encrypted and corrupted
 	or corrupted or good page. If we decrypted, there page could
 	still be corrupted if used key does not match. */
-	still_encrypted = (crypt_data &&
+	const bool still_encrypted = mach_read_from_4(
-		crypt_data->type != CRYPT_SCHEME_UNENCRYPTED &&
+		dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
-		!bpage->encrypted &&
+		&& space->crypt_data
-		fil_space_verify_crypt_checksum(dst_frame, zip_size,
+		&& space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED
-			space, bpage->offset));
+		&& !bpage->encrypted
+		&& fil_space_verify_crypt_checksum(dst_frame, zip_size);
 
 	if (!still_encrypted) {
 		/* If traditional checksums match, we assume that page is

storage/xtradb/buf/buf0dblwr.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2013, 2017, MariaDB Corporation.
+Copyright (c) 2013, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -362,6 +362,22 @@ too_small:
 	goto start_again;
 }
 
+/** Check if a page is all zeroes.
+@param[in]	read_buf	database page
+@param[in]	zip_size	ROW_FORMAT=COMPRESSED page size, or 0
+@return	whether the page is all zeroes */
+static bool buf_page_is_zeroes(const byte* read_buf, ulint zip_size)
+{
+	const ulint page_size = zip_size ? zip_size : UNIV_PAGE_SIZE;
+
+	for (ulint i = 0; i < page_size; i++) {
+		if (read_buf[i] != 0) {
+			return false;
+		}
+	}
+	return true;
+}
+
 /****************************************************************//**
 At a database startup initializes the doublewrite buffer memory structure if
 we already have a doublewrite buffer created in the data files. If we are
@@ -556,6 +572,9 @@ buf_dblwr_process()
 
 		const bool is_all_zero = buf_page_is_zeroes(
 			read_buf, zip_size);
+		const bool expect_encrypted = space()->crypt_data
+			&& space()->crypt_data->type
+			!= CRYPT_SCHEME_UNENCRYPTED;
 
 		if (is_all_zero) {
 			/* We will check if the copy in the
@@ -566,17 +585,17 @@ buf_dblwr_process()
 			/* Decompress the page before
 			validating the checksum. */
 			ulint decomp = fil_page_decompress(buf, read_buf);
-			if (!decomp) {
-				goto bad;
-			}
 			if (!decomp || (decomp != srv_page_size && zip_size)) {
 				goto bad;
 			}
 
-			if (fil_space_verify_crypt_checksum(
+			if (expect_encrypted && mach_read_from_4(
-				    read_buf, zip_size, NULL, page_no)
+				    read_buf
-			    || !buf_page_is_corrupted(
+				    + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
-				    true, read_buf, zip_size, space())) {
+			    ? fil_space_verify_crypt_checksum(read_buf,
+							      zip_size)
+			    : !buf_page_is_corrupted(true, read_buf,
+						     zip_size, space())) {
 				/* The page is good; there is no need
 				to consult the doublewrite buffer. */
 				continue;
@@ -595,9 +614,11 @@ bad:
 		if (!decomp || (decomp != srv_page_size && zip_size)) {
 			goto bad_doublewrite;
 		}
-		if (!fil_space_verify_crypt_checksum(page, zip_size, NULL,
+
-						     page_no)
+		if (expect_encrypted && mach_read_from_4(
-		    && buf_page_is_corrupted(true, page, zip_size, space)) {
+			    page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
+		    ? !fil_space_verify_crypt_checksum(page, zip_size)
+		    : buf_page_is_corrupted(true, page, zip_size, space())) {
 			if (!is_all_zero) {
 bad_doublewrite:
 				ib_logf(IB_LOG_LEVEL_WARN,

storage/xtradb/fil/fil0crypt.cc

@@ -662,7 +662,7 @@ fil_encrypt_buf(
 	// store the post-encryption checksum after the key-version
 	mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum);
 
-	ut_ad(fil_space_verify_crypt_checksum(dst_frame, zip_size, NULL, offset));
+	ut_ad(fil_space_verify_crypt_checksum(dst_frame, zip_size));
 
 	srv_stats.pages_encrypted.inc();
 
@@ -2568,167 +2568,66 @@ encrypted, or corrupted.
 
 @param[in]	page		Page to verify
 @param[in]	zip_size	zip size
-@param[in]	space		Tablespace
+@return whether the encrypted page is OK */
-@param[in]	pageno		Page no
-@return true if page is encrypted AND OK, false otherwise */
 UNIV_INTERN
-bool
+bool fil_space_verify_crypt_checksum(const byte* page, ulint zip_size)
-fil_space_verify_crypt_checksum(
-	byte* 			page,
-	ulint			zip_size,
-#ifndef UNIV_INNOCHECKSUM
-	const fil_space_t*	space,
-#else
-	const void*		space,
-#endif
-	ulint			pageno)
 {
-	uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+	ut_ad(mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION));
-
-	/* If page is not encrypted, return false */
-	if (key_version == 0) {
-		return(false);
-	}
-
-	srv_checksum_algorithm_t algorithm =
-			static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
-
-	/* If no checksum is used, can't continue checking. */
-	if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
-		return(true);
-	}
-
-	/* Read stored post encryption checksum. */
-	ib_uint32_t checksum = mach_read_from_4(
-		page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
-
-	/* Declare empty pages non-corrupted */
-	if (checksum == 0
-	    && *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
-	    && buf_page_is_zeroes(page, zip_size)) {
-		return(true);
-	}
 
 	/* Compressed and encrypted pages do not have checksum. Assume not
 	corrupted. Page verification happens after decompression in
 	buf_page_io_complete() using buf_page_is_corrupted(). */
-	if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+	if (mach_read_from_2(page + FIL_PAGE_TYPE)
-		return (true);
+	    == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+		return true;
 	}
 
-	ib_uint32_t cchecksum1 = 0;
+	/* Read stored post encryption checksum. */
-	ib_uint32_t cchecksum2 = 0;
+	const ib_uint32_t checksum = mach_read_from_4(
-
+		page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
-	/* Calculate checksums */
-	if (zip_size) {
-		cchecksum1 = page_zip_calc_checksum(
-			page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
-
-		cchecksum2 = (cchecksum1 == checksum)
-			? 0
-			: page_zip_calc_checksum(
-				page, zip_size,
-				SRV_CHECKSUM_ALGORITHM_INNODB);
-	} else {
-		cchecksum1 = buf_calc_page_crc32(page);
-		cchecksum2 = (cchecksum1 == checksum)
-			? 0
-			: buf_calc_page_new_checksum(page);
-	}
 
 	/* If stored checksum matches one of the calculated checksums
 	page is not corrupted. */
+	srv_checksum_algorithm_t algorithm = srv_checksum_algorithm_t(
+		srv_checksum_algorithm);
 
-	bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
+	switch (algorithm) {
-		|| checksum == BUF_NO_CHECKSUM_MAGIC);
+	case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
-
+		if (zip_size) {
-	/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
+			return checksum == page_zip_calc_checksum(
-	first page of each system tablespace file at
+				page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
-	FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
-	the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
-	(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
-	than page 0 of the system tablespace.
-
-	Starting from MariaDB 10.1 the field has been repurposed for
-	encryption key_version.
-
-	Starting with MySQL 5.7 (and MariaDB Server 10.2), the
-	field has been repurposed for SPATIAL INDEX pages for
-	FIL_RTREE_SPLIT_SEQ_NUM.
-
-	Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
-	checksum.
-
-	Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
-	field would usually be 0 for pages that are not encrypted, we cannot
-	assume that a nonzero value means that the page is encrypted.
-	Therefore we must validate the page both as encrypted and unencrypted
-	when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
-	*/
-
-	uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM);
-	uint32_t checksum2;
-
-	bool valid = false;
-
-	if (zip_size) {
-		valid = (checksum1 == cchecksum1);
-		checksum2 = checksum1;
-	} else {
-		checksum2 = mach_read_from_4(
-			page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
-		switch (algorithm) {
-		case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
-			valid = buf_page_is_checksum_valid_crc32(page, checksum1,
-								 checksum2);
-			break;
-		case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
-			valid = buf_page_is_checksum_valid_innodb(page, checksum1,
-								  checksum2);
-			break;
-		case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
-			valid = buf_page_is_checksum_valid_none(page, checksum1,
-								checksum2);
-			break;
-		case SRV_CHECKSUM_ALGORITHM_CRC32:
-		case SRV_CHECKSUM_ALGORITHM_INNODB:
-			valid = buf_page_is_checksum_valid_crc32(
-					page, checksum1, checksum2)
-				|| buf_page_is_checksum_valid_innodb(
-					page, checksum1, checksum2);
-			break;
-		case SRV_CHECKSUM_ALGORITHM_NONE:
-			ut_error;
 		}
-	}
 
-	if (encrypted && valid) {
+		return checksum == buf_calc_page_crc32(page);
-		/* If page is encrypted and traditional checksums match,
+	case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
-		page could be still encrypted, or not encrypted and valid or
+		if (zip_size) {
-		corrupted. */
+			return checksum == page_zip_calc_checksum(
-#ifndef UNIV_INNOCHECKSUM
+				page, zip_size, SRV_CHECKSUM_ALGORITHM_INNODB);
-		ib_logf(IB_LOG_LEVEL_ERROR,
-			" Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted."
-			" Post encryption checksum %u stored [%u:%u] key_version %u",
-			pageno,
-			space ? space->name : "N/A",
-			mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
-			checksum, checksum1, checksum2, key_version);
-#else
-		if (log_file) {
-			fprintf(log_file,
-				"Page " ULINTPF ":" ULINTPF " may be corrupted."
-				" Post encryption checksum %u"
-				" stored [%u:%u] key_version %u\n",
-				pageno,
-				mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
-				checksum, checksum1, checksum2,
-				key_version);
 		}
-#endif /* UNIV_INNOCHECKSUM */
+		return checksum == buf_calc_page_new_checksum(page);
+	case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
+		return checksum == BUF_NO_CHECKSUM_MAGIC;
+	case SRV_CHECKSUM_ALGORITHM_NONE:
+		return true;
+	case SRV_CHECKSUM_ALGORITHM_INNODB:
+	case SRV_CHECKSUM_ALGORITHM_CRC32:
+		if (checksum == BUF_NO_CHECKSUM_MAGIC) {
+			return true;
+		}
+		if (zip_size) {
+			if (checksum == page_zip_calc_checksum(
+				    page, zip_size, algorithm)) {
+				return true;
+			}
 
-		encrypted = false;
+			algorithm = algorithm == SRV_CHECKSUM_ALGORITHM_INNODB
+				? SRV_CHECKSUM_ALGORITHM_CRC32
+				: SRV_CHECKSUM_ALGORITHM_INNODB;
+			return checksum == page_zip_calc_checksum(
+				page, zip_size, algorithm);
+		}
+
+		return checksum == buf_calc_page_crc32(page)
+			|| checksum == buf_calc_page_new_checksum(page);
 	}
-
-	return(encrypted);
 }

storage/xtradb/include/buf0buf.h

@@ -692,13 +692,6 @@ buf_page_is_corrupted(
 	ulint			zip_size,
 	const fil_space_t* 	space)
 	MY_ATTRIBUTE((warn_unused_result));
-/** Check if a page is all zeroes.
-@param[in]	read_buf	database page
-@param[in]	zip_size	ROW_FORMAT=COMPRESSED page size, or 0
-@return	whether the page is all zeroes */
-UNIV_INTERN
-bool
-buf_page_is_zeroes(const byte* read_buf, ulint zip_size);
 #ifndef UNIV_HOTBACKUP
 /**********************************************************************//**
 Gets the space id, page offset, and byte offset within page of a

storage/xtradb/include/fil0crypt.h

@@ -409,16 +409,9 @@ encrypted, or corrupted.
 
 @param[in]	page		Page to verify
 @param[in]	zip_size	zip size
-@param[in]	space		Tablespace
+@return whether the encrypted page is OK */
-@param[in]	pageno		Page no
-@return true if page is encrypted AND OK, false otherwise */
 UNIV_INTERN
-bool
+bool fil_space_verify_crypt_checksum(const byte* page, ulint zip_size)
-fil_space_verify_crypt_checksum(
-	byte* 			page,
-	ulint			zip_size,
-	const fil_space_t*	space,
-	ulint			pageno)
 	MY_ATTRIBUTE((warn_unused_result));
 
 /*********************************************************************

storage/xtradb/row/row0import.cc

@@ -3537,8 +3537,7 @@ not_encrypted:
 				}
 			} else {
 				if (!fil_space_verify_crypt_checksum(
-					    src, callback.get_zip_size(),
+					    src, callback.get_zip_size())) {
-					    NULL, block->page.offset)) {
 					goto page_corrupted;
 				}