database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-01 03:47:19 +03:00
Code Activity

MDEV-11759: Encryption code in MariaDB 10.1/10.2 causes

Browse Source 
compatibility problems

Pages that are encrypted contain post encryption checksum on
different location that normal checksum fields. Therefore,
we should before decryption check this checksum to avoid
unencrypting corrupted pages. After decryption we can use
traditional checksum check to detect if page is corrupted
or unencryption was done using incorrect key.

Pages that are page compressed do not contain any checksum,
here we need to fist unencrypt, decompress and finally
use tradional checksum check to detect page corruption
or that we used incorrect key in unencryption.

buf0buf.cc: buf_page_is_corrupted() mofified so that
compressed pages are skipped.

buf0buf.h, buf_block_init(), buf_page_init_low():
removed unnecessary page_encrypted, page_compressed,
stored_checksum, valculated_checksum fields from
buf_page_t

buf_page_get_gen(): use new buf_page_check_corrupt() function
to detect corrupted pages.

buf_page_check_corrupt(): If page was not yet decrypted
check if post encryption checksum still matches.
If page is not anymore encrypted, use buf_page_is_corrupted()
traditional checksum method.

If page is detected as corrupted and it is not encrypted
we print corruption message to error log.
If page is still encrypted or it was encrypted and now
corrupted, we will print message that page is
encrypted to error log.

buf_page_io_complete(): use new buf_page_check_corrupt()
function to detect corrupted pages.

buf_page_decrypt_after_read(): Verify post encryption
checksum before tring to decrypt.

fil0crypt.cc: fil_encrypt_buf() verify post encryption
checksum and ind fil_space_decrypt() return true
if we really decrypted the page.

fil_space_verify_crypt_checksum(): rewrite to use
the method used when calculating post encryption
checksum. We also check if post encryption checksum
matches that traditional checksum check does not
match.

fil0fil.ic: Add missed page type encrypted and page
compressed to fil_get_page_type_name()

Note that this change does not yet fix innochecksum tool,
that will be done in separate MDEV.

Fix test failures caused by buf page corruption injection.
This commit is contained in:
Jan Lindström
2017-02-06 10:47:55 +02:00
parent bc4686f0f4
commit ddf2fac733
36 changed files with 935 additions and 818 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
mysql-test/suite/encryption/r/innodb-bad-key-change.result
View File

@ -2,7 +2,7 @@ call mtr.add_suppression("Plugin 'file_key_management' init function returned er
call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*");
call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed");
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/r/innodb-bad-key-change2.result
View File

@ -1,5 +1,5 @@
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/r/innodb-bad-key-change4.result
View File

@ -1,5 +1,5 @@
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/r/innodb-bad-key-change5.result
View File

@ -1,5 +1,5 @@
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result
View File

@ -1,5 +1,5 @@
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/r/innodb-encryption-disable.result
View File

@ -1,7 +1,7 @@
SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_file_per_table = ON;
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/r/innodb-missing-key.result
View File

@ -1,5 +1,5 @@
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
# Start server with keys2.txt # Start server with keys2.txt

2
mysql-test/suite/encryption/t/innodb-bad-key-change.test
View File

@ -16,7 +16,7 @@ call mtr.add_suppression("Plugin 'file_key_management' init function returned er
call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*");
call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed");
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/t/innodb-bad-key-change2.test
View File

@ -9,7 +9,7 @@
# MDEV-8750: Server crashes in page_cur_is_after_last on altering table using a wrong encryption key # MDEV-8750: Server crashes in page_cur_is_after_last on altering table using a wrong encryption key
# #
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/t/innodb-bad-key-change4.test
View File

@ -9,7 +9,7 @@
# MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys # MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys
# #
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/t/innodb-bad-key-change5.test
View File

@ -9,7 +9,7 @@
# MDEV-8769: Server crash at file btr0btr.ic line 122 when defragmenting encrypted table using incorrect keys # MDEV-8769: Server crash at file btr0btr.ic line 122 when defragmenting encrypted table using incorrect keys
# #
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test
View File

@ -3,7 +3,7 @@
# #
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/t/innodb-encryption-disable.test
View File

@ -18,7 +18,7 @@ SET GLOBAL innodb_file_per_table = ON;
# MDEV-9559: Server without encryption configs crashes if selecting from an implicitly encrypted table # MDEV-9559: Server without encryption configs crashes if selecting from an implicitly encrypted table
# #
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem.");
call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*");

2
mysql-test/suite/encryption/t/innodb-missing-key.test
View File

@ -10,7 +10,7 @@
# MDEV-11004: Unable to start (Segfault or os error 2) when encryption key missing # MDEV-11004: Unable to start (Segfault or os error 2) when encryption key missing
# #
call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted");
call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match.");
call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file.");
--echo --echo

2
mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result
View File

@ -1,5 +1,7 @@
call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Page for tablespace ");
call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x"); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x");
call mtr.add_suppression("InnoDB: Corruption: Block in space_id .* in file .* corrupted");
call mtr.add_suppression("InnoDB: Based on page type .*");
FLUSH TABLES; FLUSH TABLES;
SET GLOBAL innodb_file_per_table = 1; SET GLOBAL innodb_file_per_table = 1;
SELECT @@innodb_file_per_table; SELECT @@innodb_file_per_table;

3
mysql-test/suite/innodb/r/innodb_bug14147491.result
View File

@ -1,4 +1,6 @@
call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*");
call mtr.add_suppression("InnoDB: Corruption: Block in space_id .* in file .* corrupted.");
call mtr.add_suppression("InnoDB: Based on page type .*");
CALL mtr.add_suppression("InnoDB: Error: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts"); CALL mtr.add_suppression("InnoDB: Error: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts");
CALL mtr.add_suppression("InnoDB: Database page corruption on disk or a failed"); CALL mtr.add_suppression("InnoDB: Database page corruption on disk or a failed");
CALL mtr.add_suppression("InnoDB: Space .* file test/t1 read of page .*"); CALL mtr.add_suppression("InnoDB: Space .* file test/t1 read of page .*");
@ -10,6 +12,7 @@ CALL mtr.add_suppression("InnoDB: fix the corruption by dumping, dropping, and r
CALL mtr.add_suppression("InnoDB: the corrupt table. You can use CHECK"); CALL mtr.add_suppression("InnoDB: the corrupt table. You can use CHECK");
CALL mtr.add_suppression("InnoDB: TABLE to scan your table for corruption."); CALL mtr.add_suppression("InnoDB: TABLE to scan your table for corruption.");
CALL mtr.add_suppression("InnoDB: See also .* about forcing recovery."); CALL mtr.add_suppression("InnoDB: See also .* about forcing recovery.");
flush tables;
# Create and populate the table to be corrupted # Create and populate the table to be corrupted
CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
INSERT INTO t1 (b) VALUES ('corrupt me'); INSERT INTO t1 (b) VALUES ('corrupt me');

2
mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test
View File

@ -19,6 +19,8 @@
call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Page for tablespace ");
call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x"); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x");
call mtr.add_suppression("InnoDB: Corruption: Block in space_id .* in file .* corrupted");
call mtr.add_suppression("InnoDB: Based on page type .*");
FLUSH TABLES; FLUSH TABLES;
let MYSQLD_DATADIR =`SELECT @@datadir`; let MYSQLD_DATADIR =`SELECT @@datadir`;

3
mysql-test/suite/innodb/t/innodb_bug14147491.test
View File

@ -5,6 +5,8 @@
-- source include/not_encrypted.inc -- source include/not_encrypted.inc
call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*");
call mtr.add_suppression("InnoDB: Corruption: Block in space_id .* in file .* corrupted.");
call mtr.add_suppression("InnoDB: Based on page type .*");
# Don't test under valgrind, memory leaks will occur # Don't test under valgrind, memory leaks will occur
source include/not_valgrind.inc; source include/not_valgrind.inc;
@ -31,6 +33,7 @@ CALL mtr.add_suppression("InnoDB: fix the corruption by dumping, dropping, and r
CALL mtr.add_suppression("InnoDB: the corrupt table. You can use CHECK"); CALL mtr.add_suppression("InnoDB: the corrupt table. You can use CHECK");
CALL mtr.add_suppression("InnoDB: TABLE to scan your table for corruption."); CALL mtr.add_suppression("InnoDB: TABLE to scan your table for corruption.");
CALL mtr.add_suppression("InnoDB: See also .* about forcing recovery."); CALL mtr.add_suppression("InnoDB: See also .* about forcing recovery.");
flush tables;
--echo # Create and populate the table to be corrupted --echo # Create and populate the table to be corrupted

426
storage/innobase/buf/buf0buf.cc
View File

@ -68,11 +68,6 @@ Created 11/5/1995 Heikki Tuuri
#include "lzo/lzo1x.h" #include "lzo/lzo1x.h"
#endif #endif
/* Enable this for checksum error messages. */
//#ifdef UNIV_DEBUG
//#define UNIV_DEBUG_LEVEL2 1
//#endif
/* /*
IMPLEMENTATION OF THE BUFFER POOL IMPLEMENTATION OF THE BUFFER POOL
================================= =================================
@ -325,18 +320,6 @@ on the io_type */
/* prototypes for new functions added to ha_innodb.cc */ /* prototypes for new functions added to ha_innodb.cc */
trx_t* innobase_get_trx(); trx_t* innobase_get_trx();
/********************************************************************//**
Check if page is maybe compressed, encrypted or both when we encounter
corrupted page. Note that we can't be 100% sure if page is corrupted
or decrypt/decompress just failed.
*/
static
ibool
buf_page_check_corrupt(
/*===================*/
buf_page_t* bpage); /*!< in/out: buffer page read from
disk */
/********************************************************************//** /********************************************************************//**
Gets the smallest oldest_modification lsn for any page in the pool. Returns Gets the smallest oldest_modification lsn for any page in the pool. Returns
zero if all modified pages have been flushed to disk. zero if all modified pages have been flushed to disk.
@ -503,6 +486,7 @@ buf_block_alloc(
/********************************************************************//** /********************************************************************//**
Checks if a page is all zeroes. Checks if a page is all zeroes.
@return TRUE if the page is all zeroes */ @return TRUE if the page is all zeroes */
UNIV_INTERN
bool bool
buf_page_is_zeroes( buf_page_is_zeroes(
/*===============*/ /*===============*/
@ -525,7 +509,7 @@ buf_page_is_zeroes(
@param[in] checksum_field1 new checksum field @param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field @param[in] checksum_field2 old checksum field
@return true if the page is in crc32 checksum format */ @return true if the page is in crc32 checksum format */
UNIV_INLINE UNIV_INTERN
bool bool
buf_page_is_checksum_valid_crc32( buf_page_is_checksum_valid_crc32(
const byte* read_buf, const byte* read_buf,
@ -534,13 +518,12 @@ buf_page_is_checksum_valid_crc32(
{ {
ib_uint32_t crc32 = buf_calc_page_crc32(read_buf); ib_uint32_t crc32 = buf_calc_page_crc32(read_buf);
#ifdef UNIV_DEBUG_LEVEL2
if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) {
ib_logf(IB_LOG_LEVEL_INFO, DBUG_PRINT("buf_checksum",
"Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", ("Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.",
checksum_field1, checksum_field2, (ulint)crc32); checksum_field1, checksum_field2, (ulint)crc32));
} }
#endif
return(checksum_field1 == crc32 && checksum_field2 == crc32); return(checksum_field1 == crc32 && checksum_field2 == crc32);
} }
@ -550,7 +533,7 @@ buf_page_is_checksum_valid_crc32(
@param[in] checksum_field1 new checksum field @param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field @param[in] checksum_field2 old checksum field
@return true if the page is in innodb checksum format */ @return true if the page is in innodb checksum format */
UNIV_INLINE UNIV_INTERN
bool bool
buf_page_is_checksum_valid_innodb( buf_page_is_checksum_valid_innodb(
const byte* read_buf, const byte* read_buf,
@ -569,13 +552,11 @@ buf_page_is_checksum_valid_innodb(
if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN)
&& checksum_field2 != buf_calc_page_old_checksum(read_buf)) { && checksum_field2 != buf_calc_page_old_checksum(read_buf)) {
#ifdef UNIV_DEBUG_LEVEL2 DBUG_PRINT("buf_checksum",
ib_logf(IB_LOG_LEVEL_INFO, ("Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.",
"Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.",
checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf),
mach_read_from_4(read_buf + FIL_PAGE_LSN) mach_read_from_4(read_buf + FIL_PAGE_LSN)));
);
#endif
return(false); return(false);
} }
@ -586,13 +567,11 @@ buf_page_is_checksum_valid_innodb(
if (checksum_field1 != 0 if (checksum_field1 != 0
&& checksum_field1 != buf_calc_page_new_checksum(read_buf)) { && checksum_field1 != buf_calc_page_new_checksum(read_buf)) {
#ifdef UNIV_DEBUG_LEVEL2 DBUG_PRINT("buf_checksum",
ib_logf(IB_LOG_LEVEL_INFO, ("Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.",
"Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.",
checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf), checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf),
mach_read_from_4(read_buf + FIL_PAGE_LSN) mach_read_from_4(read_buf + FIL_PAGE_LSN)));
);
#endif
return(false); return(false);
} }
@ -604,22 +583,20 @@ buf_page_is_checksum_valid_innodb(
@param[in] checksum_field1 new checksum field @param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field @param[in] checksum_field2 old checksum field
@return true if the page is in none checksum format */ @return true if the page is in none checksum format */
UNIV_INLINE UNIV_INTERN
bool bool
buf_page_is_checksum_valid_none( buf_page_is_checksum_valid_none(
const byte* read_buf, const byte* read_buf,
ulint checksum_field1, ulint checksum_field1,
ulint checksum_field2) ulint checksum_field2)
{ {
#ifdef UNIV_DEBUG_LEVEL2
if (!(checksum_field1 == checksum_field2 || checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { if (!(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) {
ib_logf(IB_LOG_LEVEL_INFO, DBUG_PRINT("buf_checksum",
"Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", ("Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.",
checksum_field1, checksum_field2, BUF_NO_CHECKSUM_MAGIC, checksum_field1, checksum_field2, BUF_NO_CHECKSUM_MAGIC,
mach_read_from_4(read_buf + FIL_PAGE_LSN) mach_read_from_4(read_buf + FIL_PAGE_LSN)));
);
} }
#endif
return(checksum_field1 == checksum_field2 return(checksum_field1 == checksum_field2
&& checksum_field1 == BUF_NO_CHECKSUM_MAGIC); && checksum_field1 == BUF_NO_CHECKSUM_MAGIC);
@ -627,16 +604,18 @@ buf_page_is_checksum_valid_none(
/********************************************************************//** /********************************************************************//**
Checks if a page is corrupt. Checks if a page is corrupt.
@return TRUE if corrupted */ @param[in] check_lsn true if LSN should be checked
@param[in] read_buf Page to be checked
@param[in] zip_size compressed size or 0
@param[in] space Pointer to tablespace
@return true if corrupted, false if not */
UNIV_INTERN UNIV_INTERN
ibool bool
buf_page_is_corrupted( buf_page_is_corrupted(
/*==================*/ bool check_lsn,
bool check_lsn, /*!< in: true if we need to check const byte* read_buf,
and complain about the LSN */ ulint zip_size,
const byte* read_buf, /*!< in: a database page */ const fil_space_t* space)
ulint zip_size) /*!< in: size of compressed page;
0 for uncompressed pages */
{ {
ulint checksum_field1; ulint checksum_field1;
ulint checksum_field2; ulint checksum_field2;
@ -644,26 +623,21 @@ buf_page_is_corrupted(
read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint page_type = mach_read_from_4( ulint page_type = mach_read_from_4(
read_buf + FIL_PAGE_TYPE); read_buf + FIL_PAGE_TYPE);
bool no_checksum = (page_type == FIL_PAGE_PAGE_COMPRESSED ||
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
/* We can trust page type if page compression is set on tablespace
/* Page is encrypted if encryption information is found from flags because page compression flag means file must have been
tablespace and page contains used key_version. This is true created with 10.1 (later than 5.5 code base). In 10.1 page
also for pages first compressed and then encrypted. */ compressed tables do not contain post compression checksum and
if (crypt_data && FIL_PAGE_END_LSN_OLD_CHKSUM field stored. Note that space can
crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && be null if we are in fil_check_first_page() and first page
fil_page_is_encrypted(read_buf)) { is not compressed or encrypted. */
no_checksum = true; if ((page_type == FIL_PAGE_PAGE_COMPRESSED ||
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)
&& space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) {
return (false);
} }
/* Return early if there is no checksum or END_LSN */ if (!zip_size
if (no_checksum) {
return (FALSE);
}
if (!no_checksum && !zip_size
&& memcmp(read_buf + FIL_PAGE_LSN + 4, && memcmp(read_buf + FIL_PAGE_LSN + 4,
read_buf + UNIV_PAGE_SIZE read_buf + UNIV_PAGE_SIZE
- FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) { - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) {
@ -676,7 +650,7 @@ buf_page_is_corrupted(
mach_read_from_4(read_buf + FIL_PAGE_LSN + 4), mach_read_from_4(read_buf + FIL_PAGE_LSN + 4),
mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4)); mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4));
return(TRUE); return(true);
} }
#ifndef UNIV_HOTBACKUP #ifndef UNIV_HOTBACKUP
@ -715,7 +689,7 @@ buf_page_is_corrupted(
/* Check whether the checksum fields have correct values */ /* Check whether the checksum fields have correct values */
if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
return(FALSE); return(false);
} }
if (zip_size) { if (zip_size) {
@ -742,14 +716,14 @@ buf_page_is_corrupted(
ib_logf(IB_LOG_LEVEL_INFO, ib_logf(IB_LOG_LEVEL_INFO,
"Checksum fields zero but page is not empty."); "Checksum fields zero but page is not empty.");
return(TRUE); return(true);
} }
} }
return(FALSE); return(false);
} }
DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); ); DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(true); );
ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET); ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET);
@ -762,7 +736,7 @@ buf_page_is_corrupted(
if (buf_page_is_checksum_valid_crc32(read_buf, if (buf_page_is_checksum_valid_crc32(read_buf,
checksum_field1, checksum_field2)) { checksum_field1, checksum_field2)) {
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_none(read_buf, if (buf_page_is_checksum_valid_none(read_buf,
@ -775,7 +749,7 @@ buf_page_is_corrupted(
space_id, page_no); space_id, page_no);
} }
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_innodb(read_buf, if (buf_page_is_checksum_valid_innodb(read_buf,
@ -788,17 +762,17 @@ buf_page_is_corrupted(
space_id, page_no); space_id, page_no);
} }
return(FALSE); return(false);
} }
return(TRUE); return(true);
case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
if (buf_page_is_checksum_valid_innodb(read_buf, if (buf_page_is_checksum_valid_innodb(read_buf,
checksum_field1, checksum_field2)) { checksum_field1, checksum_field2)) {
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_none(read_buf, if (buf_page_is_checksum_valid_none(read_buf,
@ -811,7 +785,7 @@ buf_page_is_corrupted(
space_id, page_no); space_id, page_no);
} }
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_crc32(read_buf, if (buf_page_is_checksum_valid_crc32(read_buf,
@ -824,16 +798,16 @@ buf_page_is_corrupted(
space_id, page_no); space_id, page_no);
} }
return(FALSE); return(false);
} }
return(TRUE); return(true);
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
if (buf_page_is_checksum_valid_none(read_buf, if (buf_page_is_checksum_valid_none(read_buf,
checksum_field1, checksum_field2)) { checksum_field1, checksum_field2)) {
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_crc32(read_buf, if (buf_page_is_checksum_valid_crc32(read_buf,
@ -842,7 +816,7 @@ buf_page_is_corrupted(
curr_algo, curr_algo,
SRV_CHECKSUM_ALGORITHM_CRC32, SRV_CHECKSUM_ALGORITHM_CRC32,
space_id, page_no); space_id, page_no);
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_innodb(read_buf, if (buf_page_is_checksum_valid_innodb(read_buf,
@ -851,10 +825,10 @@ buf_page_is_corrupted(
curr_algo, curr_algo,
SRV_CHECKSUM_ALGORITHM_INNODB, SRV_CHECKSUM_ALGORITHM_INNODB,
space_id, page_no); space_id, page_no);
return(FALSE); return(false);
} }
return(TRUE); return(true);
case SRV_CHECKSUM_ALGORITHM_NONE: case SRV_CHECKSUM_ALGORITHM_NONE:
/* should have returned FALSE earlier */ /* should have returned FALSE earlier */
@ -864,7 +838,7 @@ buf_page_is_corrupted(
} }
ut_error; ut_error;
return(FALSE); return(false);
} }
/********************************************************************//** /********************************************************************//**
@ -1133,12 +1107,9 @@ buf_block_init(
block->page.state = BUF_BLOCK_NOT_USED; block->page.state = BUF_BLOCK_NOT_USED;
block->page.buf_fix_count = 0; block->page.buf_fix_count = 0;
block->page.io_fix = BUF_IO_NONE; block->page.io_fix = BUF_IO_NONE;
block->page.corrupted = false;
block->page.key_version = 0; block->page.key_version = 0;
block->page.page_encrypted = false;
block->page.page_compressed = false;
block->page.encrypted = false; block->page.encrypted = false;
block->page.stored_checksum = BUF_NO_CHECKSUM_MAGIC;
block->page.calculated_checksum = BUF_NO_CHECKSUM_MAGIC;
block->page.real_size = 0; block->page.real_size = 0;
block->page.write_size = 0; block->page.write_size = 0;
block->modify_clock = 0; block->modify_clock = 0;
@ -2981,14 +2952,14 @@ loop:
} else if (retries < BUF_PAGE_READ_MAX_RETRIES) { } else if (retries < BUF_PAGE_READ_MAX_RETRIES) {
++retries; ++retries;
bool corrupted = true; bool corrupted = false;
if (bpage) { if (bpage) {
corrupted = buf_page_check_corrupt(bpage); corrupted = buf_page_check_corrupt(bpage);
} }
/* Do not try again for encrypted pages */ /* Do not try again for encrypted pages */
if (!corrupted) { if (corrupted && bpage->encrypted) {
ib_mutex_t* pmutex = buf_page_get_mutex(bpage); ib_mutex_t* pmutex = buf_page_get_mutex(bpage);
buf_pool = buf_pool_from_bpage(bpage); buf_pool = buf_pool_from_bpage(bpage);
@ -3016,14 +2987,14 @@ loop:
retries = BUF_PAGE_READ_MAX_RETRIES; retries = BUF_PAGE_READ_MAX_RETRIES;
); );
} else { } else {
bool corrupted = true; bool corrupted = false;
if (bpage) { if (bpage) {
corrupted = buf_page_check_corrupt(bpage); corrupted = buf_page_check_corrupt(bpage);
} }
if (corrupted) { if (corrupted && !bpage->encrypted) {
fprintf(stderr, "InnoDB: Error: Unable" ib_logf(IB_LOG_LEVEL_ERROR, "Unable"
" to read tablespace %lu page no" " to read tablespace %lu page no"
" %lu into the buffer pool after" " %lu into the buffer pool after"
" %lu attempts\n" " %lu attempts\n"
@ -3797,11 +3768,8 @@ buf_page_init_low(
bpage->oldest_modification = 0; bpage->oldest_modification = 0;
bpage->write_size = 0; bpage->write_size = 0;
bpage->key_version = 0; bpage->key_version = 0;
bpage->stored_checksum = BUF_NO_CHECKSUM_MAGIC;
bpage->calculated_checksum = BUF_NO_CHECKSUM_MAGIC;
bpage->page_encrypted = false;
bpage->page_compressed = false;
bpage->encrypted = false; bpage->encrypted = false;
bpage->corrupted = false;
bpage->real_size = 0; bpage->real_size = 0;
bpage->slot = NULL; bpage->slot = NULL;
@ -4496,72 +4464,69 @@ buf_mark_space_corrupt(
Check if page is maybe compressed, encrypted or both when we encounter Check if page is maybe compressed, encrypted or both when we encounter
corrupted page. Note that we can't be 100% sure if page is corrupted corrupted page. Note that we can't be 100% sure if page is corrupted
or decrypt/decompress just failed. or decrypt/decompress just failed.
*/ @param[in,out] bpage Page
static @return true if page corrupted, false if not */
ibool UNIV_INTERN
bool
buf_page_check_corrupt( buf_page_check_corrupt(
/*===================*/ buf_page_t* bpage)
buf_page_t* bpage) /*!< in/out: buffer page read from disk */
{ {
ulint zip_size = buf_page_get_zip_size(bpage); ulint zip_size = buf_page_get_zip_size(bpage);
byte* dst_frame = (zip_size) ? bpage->zip.data : byte* dst_frame = (zip_size) ? bpage->zip.data :
((buf_block_t*) bpage)->frame; ((buf_block_t*) bpage)->frame;
bool page_compressed = bpage->page_encrypted;
ulint stored_checksum = bpage->stored_checksum;
ulint calculated_checksum = bpage->calculated_checksum;
bool page_compressed_encrypted = bpage->page_compressed;
ulint space_id = mach_read_from_4( ulint space_id = mach_read_from_4(
dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
fil_space_t* space = fil_space_found_by_id(space_id); fil_space_t* space = fil_space_found_by_id(space_id);
bool corrupted = true; fil_space_crypt_t* crypt_data = space->crypt_data;
ulint key_version = bpage->key_version; bool still_encrypted = false;
bool corrupted = false;
ulint page_type = mach_read_from_2(dst_frame + FIL_PAGE_TYPE);
if (key_version != 0 || page_compressed_encrypted) { /* In buf_decrypt_after_read we have either decrypted the page if
bpage->encrypted = true; page post encryption checksum matches and used key_id is found
from the encryption plugin. If checksum did not match page was
not decrypted and it could be either encrypted and corrupted
or corrupted or good page. If we decrypted, there page could
still be corrupted if used key does not match. */
still_encrypted = (crypt_data &&
crypt_data->type != CRYPT_SCHEME_UNENCRYPTED &&
!bpage->encrypted &&
fil_space_verify_crypt_checksum(dst_frame, zip_size,
space, bpage->offset));
if (!still_encrypted) {
/* If traditional checksums match, we assume that page is
not anymore encrypted. */
corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space);
if (!corrupted) {
bpage->encrypted = false;
}
} }
if (key_version != 0 || /* Pages that we think are unencrypted but do not match the checksum
(crypt_data && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) || checks could be corrupted or encrypted or both. */
page_compressed || page_compressed_encrypted) { if (corrupted && !bpage->encrypted) {
bpage->corrupted = true;
ib_logf(IB_LOG_LEVEL_ERROR,
"%s: Block in space_id %lu in file %s corrupted.",
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ? "Maybe corruption" : "Corruption",
space_id, space ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"Based on page type %s (" ULINTPF ")",
fil_get_page_type_name(page_type), page_type);
} else if (still_encrypted || (bpage->encrypted && corrupted)) {
bpage->encrypted = true;
bpage->corrupted = false;
corrupted = true;
/* Page is really corrupted if post encryption stored ib_logf(IB_LOG_LEVEL_ERROR,
checksum does not match calculated checksum after page was "Block in space_id %lu in file %s encrypted.",
read. For pages compressed and then encrypted, there is no space_id, space ? space->name : "NULL");
checksum. */ ib_logf(IB_LOG_LEVEL_ERROR,
corrupted = (!page_compressed_encrypted && stored_checksum != calculated_checksum); "However key management plugin or used key_version %u is not found or"
if (corrupted) {
ib_logf(IB_LOG_LEVEL_ERROR,
"%s: Block in space_id %lu in file %s corrupted.",
page_compressed_encrypted ? "Maybe corruption" : "Corruption",
space_id, space ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"Page based on contents %s encrypted.",
(key_version == 0 && page_compressed_encrypted == false) ? "not" : "maybe");
if (stored_checksum != BUF_NO_CHECKSUM_MAGIC || calculated_checksum != BUF_NO_CHECKSUM_MAGIC) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Page stored checksum %lu but calculated checksum %lu.",
stored_checksum, calculated_checksum);
}
ib_logf(IB_LOG_LEVEL_ERROR,
"Reason could be that key_version %lu in page "
"or in crypt_data %p could not be found.",
key_version, crypt_data);
ib_logf(IB_LOG_LEVEL_ERROR,
"Reason could be also that key management plugin is not found or"
" used encryption algorithm or method does not match.");
ib_logf(IB_LOG_LEVEL_ERROR,
"Based on page page compressed %d, compressed and encrypted %d.",
page_compressed, page_compressed_encrypted);
} else {
ib_logf(IB_LOG_LEVEL_ERROR,
"Block in space_id %lu in file %s encrypted.",
space_id, space ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"However key management plugin or used key_id %lu is not found or"
" used encryption algorithm or method does not match.", " used encryption algorithm or method does not match.",
key_version); bpage->key_version);
if (space_id > TRX_SYS_SPACE) {
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"Marking tablespace as missing. You may drop this table or" "Marking tablespace as missing. You may drop this table or"
" install correct key management plugin and key file."); " install correct key management plugin and key file.");
@ -4589,6 +4554,8 @@ buf_page_io_complete(
const ibool uncompressed = (buf_page_get_state(bpage) const ibool uncompressed = (buf_page_get_state(bpage)
== BUF_BLOCK_FILE_PAGE); == BUF_BLOCK_FILE_PAGE);
fil_space_t* space = NULL; fil_space_t* space = NULL;
byte* frame = NULL;
bool corrupted = false;
ut_a(buf_page_in_file(bpage)); ut_a(buf_page_in_file(bpage));
@ -4604,21 +4571,13 @@ buf_page_io_complete(
if (io_type == BUF_IO_READ) { if (io_type == BUF_IO_READ) {
ulint read_page_no; ulint read_page_no;
ulint read_space_id; ulint read_space_id;
byte* frame = NULL;
if (!buf_page_decrypt_after_read(bpage)) { buf_page_decrypt_after_read(bpage);
/* encryption error! */
if (buf_page_get_zip_size(bpage)) {
frame = bpage->zip.data;
} else {
frame = ((buf_block_t*) bpage)->frame;
}
ib_logf(IB_LOG_LEVEL_INFO, if (buf_page_get_zip_size(bpage)) {
"Page %u in tablespace %u encryption error key_version %u.", frame = bpage->zip.data;
bpage->offset, bpage->space, bpage->key_version); } else {
frame = ((buf_block_t*) bpage)->frame;
goto database_corrupted;
} }
if (buf_page_get_zip_size(bpage)) { if (buf_page_get_zip_size(bpage)) {
@ -4680,27 +4639,25 @@ buf_page_io_complete(
bpage->offset); bpage->offset);
} }
/* From version 3.23.38 up we store the page checksum corrupted = buf_page_check_corrupt(bpage);
to the 4 first bytes of the page end lsn field */
if (buf_page_is_corrupted(true, frame, database_corrupted:
buf_page_get_zip_size(bpage))) {
if (corrupted) {
/* Not a real corruption if it was triggered by /* Not a real corruption if it was triggered by
error injection */ error injection */
DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", DBUG_EXECUTE_IF("buf_page_is_corrupt_failure",
if (bpage->space > TRX_SYS_SPACE if (bpage->space > TRX_SYS_SPACE
&& buf_mark_space_corrupt(bpage)) { && buf_mark_space_corrupt(bpage)) {
ib_logf(IB_LOG_LEVEL_INFO, ib_logf(IB_LOG_LEVEL_INFO,
"Simulated page corruption"); "Simulated page corruption");
return(true); return(true);
} }
goto page_not_corrupt; goto page_not_corrupt;
;); );
database_corrupted:
bool corrupted = buf_page_check_corrupt(bpage);
if (corrupted) { if (!bpage->encrypted) {
fil_system_enter(); fil_system_enter();
space = fil_space_get_by_id(bpage->space); space = fil_space_get_by_id(bpage->space);
fil_system_exit(); fil_system_exit();
@ -4708,15 +4665,14 @@ database_corrupted:
"Database page corruption on disk" "Database page corruption on disk"
" or a failed"); " or a failed");
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"Space %lu file %s read of page %u.", "Space %u file %s read of page %u.",
(ulint)bpage->space, bpage->space,
space ? space->name : "NULL", space ? space->name : "NULL",
bpage->offset); bpage->offset);
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"You may have to recover" "You may have to recover"
" from a backup."); " from a backup.");
buf_page_print(frame, buf_page_get_zip_size(bpage), buf_page_print(frame, buf_page_get_zip_size(bpage),
BUF_PAGE_PRINT_NO_CRASH); BUF_PAGE_PRINT_NO_CRASH);
@ -4747,10 +4703,7 @@ database_corrupted:
&& buf_mark_space_corrupt(bpage)) { && buf_mark_space_corrupt(bpage)) {
return(false); return(false);
} else { } else {
corrupted = buf_page_check_corrupt(bpage); if (!bpage->encrypted) {
ulint key_version = bpage->key_version;
if (corrupted) {
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"Ending processing because of a corrupt database page."); "Ending processing because of a corrupt database page.");
@ -4762,15 +4715,14 @@ database_corrupted:
"However key management plugin or used key_id %lu is not found or" "However key management plugin or used key_id %lu is not found or"
" used encryption algorithm or method does not match." " used encryption algorithm or method does not match."
" Can't continue opening the table.", " Can't continue opening the table.",
(ulint)bpage->space, key_version); bpage->space, bpage->key_version);
if (bpage->space > TRX_SYS_SPACE) { if (bpage->encrypted && bpage->space > TRX_SYS_SPACE) {
if (corrupted) { buf_mark_space_corrupt(bpage);
buf_mark_space_corrupt(bpage);
}
} else { } else {
ut_error; ut_error;
} }
return(false); return(false);
} }
} }
@ -4785,20 +4737,31 @@ database_corrupted:
recv_recover_page(TRUE, (buf_block_t*) bpage); recv_recover_page(TRUE, (buf_block_t*) bpage);
} }
if (uncompressed && !recv_no_ibuf_operations) { if (uncompressed && !recv_no_ibuf_operations
&& fil_page_get_type(frame) == FIL_PAGE_INDEX
&& page_is_leaf(frame)) {
buf_block_t* block;
ibool update_ibuf_bitmap;
block = (buf_block_t *) bpage;
update_ibuf_bitmap = TRUE;
if (bpage && bpage->encrypted) { if (bpage && bpage->encrypted) {
fprintf(stderr, ib_logf(IB_LOG_LEVEL_WARN,
"InnoDB: Warning: Table in tablespace %lu encrypted." "Table in tablespace %lu encrypted."
"However key management plugin or used key_id %u is not found or" "However key management plugin or used key_version %u is not found or"
" used encryption algorithm or method does not match." " used encryption algorithm or method does not match."
" Can't continue opening the table.\n", " Can't continue opening the table.\n",
(ulint)bpage->space, bpage->key_version); (ulint)bpage->space, bpage->key_version);
} else { } else {
ibuf_merge_or_delete_for_page( ibuf_merge_or_delete_for_page(
(buf_block_t*) bpage, bpage->space, block, bpage->space,
bpage->offset, buf_page_get_zip_size(bpage), bpage->offset, buf_page_get_zip_size(bpage),
TRUE); update_ibuf_bitmap);
} }
} }
} else { } else {
/* io_type == BUF_IO_WRITE */ /* io_type == BUF_IO_WRITE */
@ -6119,13 +6082,17 @@ buf_pool_reserve_tmp_slot(
/********************************************************************//** /********************************************************************//**
Encrypts a buffer page right before it's flushed to disk Encrypts a buffer page right before it's flushed to disk
@param[in,out] bpage Page control block
@param[in,out] src_frame Source page
@param[in] space_id Tablespace id
@return either unencrypted source page or decrypted page.
*/ */
UNIV_INTERN
byte* byte*
buf_page_encrypt_before_write( buf_page_encrypt_before_write(
/*==========================*/ buf_page_t* bpage,
buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ byte* src_frame,
byte* src_frame, /*!< in: src frame */ ulint space_id)
ulint space_id) /*!< in: space id */
{ {
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
ulint zip_size = buf_page_get_zip_size(bpage); ulint zip_size = buf_page_get_zip_size(bpage);
@ -6244,11 +6211,13 @@ buf_page_encrypt_before_write(
/********************************************************************//** /********************************************************************//**
Decrypt page after it has been read from disk Decrypt page after it has been read from disk
@param[in,out] bpage Page control block
@return true if successfull, false if something went wrong
*/ */
ibool UNIV_INTERN
bool
buf_page_decrypt_after_read( buf_page_decrypt_after_read(
/*========================*/ buf_page_t* bpage)
buf_page_t* bpage) /*!< in/out: buffer page read from disk */
{ {
ulint zip_size = buf_page_get_zip_size(bpage); ulint zip_size = buf_page_get_zip_size(bpage);
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
@ -6265,49 +6234,21 @@ buf_page_decrypt_after_read(
dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
/* Page is encrypted if encryption information is found from /* Page is encrypted if encryption information is found from
tablespace and page contains used key_version. This is true tablespace and page contains used key_version. This is true
also for pages first compressed and then encrypted. */ also for pages first compressed and then encrypted. */
if (!crypt_data || if (!crypt_data && key_version != 0) {
(crypt_data && key_version = 0;
crypt_data->type == CRYPT_SCHEME_UNENCRYPTED &&
key_version != 0)) {
byte* frame = NULL;
if (buf_page_get_zip_size(bpage)) {
frame = bpage->zip.data;
} else {
frame = ((buf_block_t*) bpage)->frame;
}
/* If page is not corrupted at this point, page can't be
encrypted, thus set key_version to 0. If page is corrupted,
we assume at this point that it is encrypted as page
contained key_version != 0. Note that page could still be
really corrupted. This we will find out after decrypt by
checking page checksums. */
if (!buf_page_is_corrupted(false, frame, buf_page_get_zip_size(bpage))) {
key_version = 0;
}
} }
/* If page is encrypted read post-encryption checksum */ bpage->key_version = key_version;
if (!page_compressed_encrypted && key_version != 0) {
bpage->stored_checksum = mach_read_from_4(dst_frame + + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
}
ut_ad(bpage->key_version == 0);
if (bpage->offset == 0) { if (bpage->offset == 0) {
/* File header pages are not encrypted/compressed */ /* File header pages are not encrypted/compressed */
return (TRUE); return (true);
} }
/* Store these for corruption check */
bpage->key_version = key_version;
bpage->page_encrypted = page_compressed_encrypted;
bpage->page_compressed = page_compressed;
if (page_compressed) { if (page_compressed) {
/* the page we read is unencrypted */ /* the page we read is unencrypted */
/* Find free slot from temporary memory array */ /* Find free slot from temporary memory array */
@ -6334,6 +6275,13 @@ buf_page_decrypt_after_read(
buf_tmp_buffer_t* slot = NULL; buf_tmp_buffer_t* slot = NULL;
if (key_version) { if (key_version) {
/* Verify encryption checksum before we even try to
decrypt. */
if (!fil_space_verify_crypt_checksum(dst_frame,
zip_size, NULL, bpage->offset)) {
return (false);
}
/* Find free slot from temporary memory array */ /* Find free slot from temporary memory array */
slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
@ -6341,20 +6289,14 @@ buf_page_decrypt_after_read(
fil_page_type_validate(dst_frame); fil_page_type_validate(dst_frame);
#endif #endif
/* Calculate checksum before decrypt, this will be
used later to find out if incorrect key was used. */
if (!page_compressed_encrypted) {
bpage->calculated_checksum = fil_crypt_calculate_checksum(zip_size, dst_frame);
}
/* decrypt using crypt_buf to dst_frame */ /* decrypt using crypt_buf to dst_frame */
byte* res = fil_space_decrypt(bpage->space, byte* res = fil_space_decrypt(bpage->space,
slot->crypt_buf, slot->crypt_buf,
size, size,
dst_frame); dst_frame,
&bpage->encrypted);
if (!res) { if (!res) {
bpage->encrypted = true;
success = false; success = false;
} }
#ifdef UNIV_DEBUG #ifdef UNIV_DEBUG
@ -6387,7 +6329,5 @@ buf_page_decrypt_after_read(
} }
} }
bpage->key_version = key_version;
return (success); return (success);
} }

16
storage/innobase/buf/buf0dblwr.cc
View File

@ -383,10 +383,11 @@ buf_dblwr_init_or_load_pages(
doublewrite = read_buf + TRX_SYS_DOUBLEWRITE; doublewrite = read_buf + TRX_SYS_DOUBLEWRITE;
if (mach_read_from_4(read_buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) { if (mach_read_from_4(read_buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) {
bool decrypted = false;
byte* tmp = fil_space_decrypt((ulint)TRX_SYS_SPACE, byte* tmp = fil_space_decrypt((ulint)TRX_SYS_SPACE,
read_buf + UNIV_PAGE_SIZE, read_buf + UNIV_PAGE_SIZE,
UNIV_PAGE_SIZE, /* page size */ UNIV_PAGE_SIZE, /* page size */
read_buf); read_buf, &decrypted);
doublewrite = tmp + TRX_SYS_DOUBLEWRITE; doublewrite = tmp + TRX_SYS_DOUBLEWRITE;
} }
@ -487,6 +488,7 @@ buf_dblwr_process()
byte* read_buf; byte* read_buf;
byte* unaligned_read_buf; byte* unaligned_read_buf;
recv_dblwr_t& recv_dblwr = recv_sys->dblwr; recv_dblwr_t& recv_dblwr = recv_sys->dblwr;
fil_space_t* space=NULL;
unaligned_read_buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE)); unaligned_read_buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
@ -514,6 +516,10 @@ buf_dblwr_process()
continue; continue;
} }
if (!space) {
space = fil_space_found_by_id(space_id);
}
ulint zip_size = fil_space_get_zip_size(space_id); ulint zip_size = fil_space_get_zip_size(space_id);
ut_ad(!buf_page_is_zeroes(page, zip_size)); ut_ad(!buf_page_is_zeroes(page, zip_size));
@ -548,9 +554,9 @@ buf_dblwr_process()
} }
if (fil_space_verify_crypt_checksum( if (fil_space_verify_crypt_checksum(
read_buf, zip_size) read_buf, zip_size, NULL, page_no)
|| !buf_page_is_corrupted( || !buf_page_is_corrupted(
true, read_buf, zip_size)) { true, read_buf, zip_size, space)) {
/* The page is good; there is no need /* The page is good; there is no need
to consult the doublewrite buffer. */ to consult the doublewrite buffer. */
continue; continue;
@ -573,8 +579,8 @@ buf_dblwr_process()
NULL, page, UNIV_PAGE_SIZE, NULL, true); NULL, page, UNIV_PAGE_SIZE, NULL, true);
} }
if (!fil_space_verify_crypt_checksum(page, zip_size) if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no)
&& buf_page_is_corrupted(true, page, zip_size)) { && buf_page_is_corrupted(true, page, zip_size, space)) {
if (!is_all_zero) { if (!is_all_zero) {
ib_logf(IB_LOG_LEVEL_WARN, ib_logf(IB_LOG_LEVEL_WARN,
"A doublewrite copy of page " "A doublewrite copy of page "

187
storage/innobase/fil/fil0crypt.cc
View File

@ -625,6 +625,8 @@ fil_encrypt_buf(
// store the post-encryption checksum after the key-version // store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum); mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum);
ut_ad(fil_space_verify_crypt_checksum(dst_frame, zip_size, NULL, offset));
srv_stats.pages_encrypted.inc(); srv_stats.pages_encrypted.inc();
return dst_frame; return dst_frame;
@ -676,6 +678,7 @@ fil_space_encrypt(
byte* comp_mem = NULL; byte* comp_mem = NULL;
byte* uncomp_mem = NULL; byte* uncomp_mem = NULL;
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
fil_space_t* tspace = fil_space_found_by_id(space);
if (page_compressed_encrypted) { if (page_compressed_encrypted) {
comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); comp_mem = (byte *)malloc(UNIV_PAGE_SIZE);
@ -685,7 +688,7 @@ fil_space_encrypt(
src = uncomp_mem; src = uncomp_mem;
} }
bool corrupted1 = buf_page_is_corrupted(true, src, zip_size); bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, tspace);
bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err);
/* Need to decompress the page if it was also compressed */ /* Need to decompress the page if it was also compressed */
@ -694,7 +697,7 @@ fil_space_encrypt(
fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL);
} }
bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size); bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, tspace);
bool different = memcmp(src, tmp_mem, size); bool different = memcmp(src, tmp_mem, size);
if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) {
@ -858,19 +861,25 @@ fil_space_decrypt(
/****************************************************************** /******************************************************************
Decrypt a page Decrypt a page
@return encrypted page, or original not encrypted page if encryption is @param[in] space Tablespace id
not needed. */ @param[in] tmp_frame Temporary buffer used for decrypting
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] decrypted true if page was decrypted
@return decrypted page, or original not encrypted page if decryption is
not needed.*/
UNIV_INTERN UNIV_INTERN
byte* byte*
fil_space_decrypt( fil_space_decrypt(
/*==============*/ ulint space,
ulint space, /*!< in: Fil space id */ byte* tmp_frame,
byte* tmp_frame, /*!< in: temporary buffer */ ulint page_size,
ulint page_size, /*!< in: page size */ byte* src_frame,
byte* src_frame) /*!< in/out: page buffer */ bool* decrypted)
{ {
dberr_t err = DB_SUCCESS; dberr_t err = DB_SUCCESS;
byte* res = NULL; byte* res = NULL;
*decrypted = false;
bool encrypted = fil_space_decrypt( bool encrypted = fil_space_decrypt(
fil_space_get_crypt_data(space), fil_space_get_crypt_data(space),
@ -881,6 +890,7 @@ fil_space_decrypt(
if (err == DB_SUCCESS) { if (err == DB_SUCCESS) {
if (encrypted) { if (encrypted) {
*decrypted = true;
/* Copy the decrypted page back to page buffer, not /* Copy the decrypted page back to page buffer, not
really any other options. */ really any other options. */
memcpy(src_frame, tmp_frame, page_size); memcpy(src_frame, tmp_frame, page_size);
@ -934,83 +944,114 @@ fil_crypt_calculate_checksum(
} }
/********************************************************************* /*********************************************************************
Verify checksum for a page (iff it's encrypted) Verify that post encryption checksum match calculated checksum.
NOTE: currently this function can only be run in single threaded mode This function should be called only if tablespace contains crypt_data
as it modifies srv_checksum_algorithm (temporarily) metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */ @return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN UNIV_INTERN
bool bool
fil_space_verify_crypt_checksum( fil_space_verify_crypt_checksum(
/*============================*/ byte* page,
const byte* src_frame, /*!< in: page the verify */ ulint zip_size,
ulint zip_size) /*!< in: compressed size if const fil_space_t* space,
row_format compressed */ ulint pageno)
{ {
// key version uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
uint key_version = mach_read_from_4(
src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* If page is not encrypted, return false */
if (key_version == 0) { if (key_version == 0) {
return false; // unencrypted page return false;
} }
/* "trick" the normal checksum routines by storing the post-encryption /* If no checksum is used, can't continue checking. */
* checksum into the normal checksum field allowing for reuse of if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
* the normal routines */ return(true);
// post encryption checksum
ib_uint32_t stored_post_encryption = mach_read_from_4(
src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
// save pre encryption checksum for restore in end of this function
ib_uint32_t stored_pre_encryption = mach_read_from_4(
src_frame + FIL_PAGE_SPACE_OR_CHKSUM);
ib_uint32_t checksum_field2 = mach_read_from_4(
src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
/** prepare frame for usage of normal checksum routines */
mach_write_to_4(const_cast<byte*>(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM,
stored_post_encryption);
/* NOTE: this function is (currently) only run when restoring
* dblwr-buffer, server is single threaded so it's safe to modify
* srv_checksum_algorithm */
srv_checksum_algorithm_t save_checksum_algorithm =
(srv_checksum_algorithm_t)srv_checksum_algorithm;
if (zip_size == 0 &&
(save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB ||
save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) {
/* handle ALGORITHM_INNODB specially,
* "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC
* checksum_field2 is sort of pointless anyway...
*/
srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
mach_write_to_4(const_cast<byte*>(src_frame) +
UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
BUF_NO_CHECKSUM_MAGIC);
} }
/* verify checksums */ /* Read stored post encryption checksum. */
ibool corrupted = buf_page_is_corrupted(false, src_frame, zip_size); ib_uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/** restore frame & algorithm */ /* Declare empty pages non-corrupted */
srv_checksum_algorithm = save_checksum_algorithm; if (checksum == 0
&& *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0) {
mach_write_to_4(const_cast<byte*>(src_frame) + return (true);
FIL_PAGE_SPACE_OR_CHKSUM,
stored_pre_encryption);
mach_write_to_4(const_cast<byte*>(src_frame) +
UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
checksum_field2);
if (!corrupted) {
return true; // page was encrypted and checksum matched
} else {
return false; // page was encrypted but checksum didn't match
} }
/* Compressed and encrypted pages do not have checksum. Assume not
corrupted. */
if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return (true);
}
/* Compressed pages use different checksum method. We first store
the post encryption checksum on checksum location and after function
restore the original. */
if (zip_size) {
ib_uint32_t old = static_cast<ib_uint32_t>(mach_read_from_4(
page + FIL_PAGE_SPACE_OR_CHKSUM));
mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
bool valid = page_zip_verify_checksum(page, zip_size);
mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old);
return (valid);
}
/* If stored checksum matches one of the calculated checksums
page is not corrupted. */
ib_uint32_t cchecksum1 = buf_calc_page_crc32(page);
ib_uint32_t cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum(
page);
bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
|| checksum == BUF_NO_CHECKSUM_MAGIC);
/* Old InnoDB versions did not initialize
FIL_PAGE_FILE_FLUSH_LSN field so there could be garbage
and above checksum check could produce false positive.
Thus we also check does the traditional stored
checksum fields match the calculated one. Both of these
could naturally produce false positive but then
we just decrypt the page and after that corrupted
pages very probable stay corrupted and valid
pages very probable stay valid.
*/
ulint checksum1 = mach_read_from_4(
page + FIL_PAGE_SPACE_OR_CHKSUM);
ulint checksum2 = mach_read_from_4(
page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
bool valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
|| buf_page_is_checksum_valid_none(page,checksum1,checksum2)
|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
if (encrypted && valid) {
/* If page is encrypted and traditional checksums match,
page could be still encrypted, or not encrypted and valid or
corrupted. */
ib_logf(IB_LOG_LEVEL_ERROR,
" Page %lu in space %s (%lu) maybe corrupted."
" Post encryption checksum %u stored [%lu:%lu] key_version %u",
pageno,
space ? space->name : "N/A",
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2, key_version);
encrypted = false;
}
return(encrypted);
} }
/***********************************************************************/ /***********************************************************************/

6
storage/innobase/fil/fil0fil.cc
View File

@ -2257,7 +2257,7 @@ fil_check_first_page(const page_t* page, ulint space_id, ulint flags)
} }
if (buf_page_is_corrupted( if (buf_page_is_corrupted(
false, page, fsp_flags_get_zip_size(flags))) { false, page, fsp_flags_get_zip_size(flags), NULL)) {
return("checksum mismatch"); return("checksum mismatch");
} }
@ -4515,13 +4515,13 @@ fil_user_tablespace_find_space_id(
to UNIV_PAGE_SIZE. */ to UNIV_PAGE_SIZE. */
if (page_size == UNIV_PAGE_SIZE) { if (page_size == UNIV_PAGE_SIZE) {
uncompressed_ok = !buf_page_is_corrupted( uncompressed_ok = !buf_page_is_corrupted(
false, page, 0); false, page, 0, NULL);
} }
bool compressed_ok = false; bool compressed_ok = false;
if (page_size <= UNIV_PAGE_SIZE_DEF) { if (page_size <= UNIV_PAGE_SIZE_DEF) {
compressed_ok = !buf_page_is_corrupted( compressed_ok = !buf_page_is_corrupted(
false, page, page_size); false, page, page_size, NULL);
} }
if (uncompressed_ok || compressed_ok) { if (uncompressed_ok || compressed_ok) {

2
storage/innobase/fil/fil0pagecompress.cc
View File

@ -366,7 +366,7 @@ fil_compress_page(
fil_decompress_page(uncomp_page, comp_page, len, NULL); fil_decompress_page(uncomp_page, comp_page, len, NULL);
if(buf_page_is_corrupted(false, uncomp_page, 0)) { if(buf_page_is_corrupted(false, uncomp_page, 0, space)) {
buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH);
ut_error; ut_error;
} }

86
storage/innobase/include/buf0buf.h
View File

@ -1,7 +1,7 @@
/***************************************************************************** /*****************************************************************************
Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2013, 2016, MariaDB Corporation. Copyright (c) 2013, 2017, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
@ -642,19 +642,69 @@ buf_block_unfix(
#else /* !UNIV_HOTBACKUP */ #else /* !UNIV_HOTBACKUP */
# define buf_block_modify_clock_inc(block) ((void) 0) # define buf_block_modify_clock_inc(block) ((void) 0)
#endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_HOTBACKUP */
/** Checks if the page is in crc32 checksum format.
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field
@return true if the page is in crc32 checksum format */
bool
buf_page_is_checksum_valid_crc32(
const byte* read_buf,
ulint checksum_field1,
ulint checksum_field2)
MY_ATTRIBUTE((warn_unused_result));
/** Checks if the page is in innodb checksum format.
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field
@return true if the page is in innodb checksum format */
bool
buf_page_is_checksum_valid_innodb(
const byte* read_buf,
ulint checksum_field1,
ulint checksum_field2)
MY_ATTRIBUTE((warn_unused_result));
/** Checks if the page is in none checksum format.
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field
@return true if the page is in none checksum format */
bool
buf_page_is_checksum_valid_none(
const byte* read_buf,
ulint checksum_field1,
ulint checksum_field2)
MY_ATTRIBUTE((warn_unused_result));
/********************************************************************//** /********************************************************************//**
Checks if a page is corrupt. Checks if a page is corrupt.
@return TRUE if corrupted */ @param[in] check_lsn true if LSN should be checked
UNIV_INTERN @param[in] read_buf Page to be checked
ibool @param[in] zip_size compressed size or 0
@param[in] space Pointer to tablespace
@return true if corrupted, false if not */
bool
buf_page_is_corrupted( buf_page_is_corrupted(
/*==================*/ bool check_lsn,
bool check_lsn, /*!< in: true if we need to check the const byte* read_buf,
and complain about the LSN */ ulint zip_size,
const byte* read_buf, /*!< in: a database page */ const fil_space_t* space)
ulint zip_size) /*!< in: size of compressed page; MY_ATTRIBUTE((warn_unused_result));
0 for uncompressed pages */
MY_ATTRIBUTE((nonnull, warn_unused_result)); /********************************************************************//**
Check if page is maybe compressed, encrypted or both when we encounter
corrupted page. Note that we can't be 100% sure if page is corrupted
or decrypt/decompress just failed.
@param[in] bpage Page
@return true if page corrupted, false if not */
bool
buf_page_check_corrupt(
buf_page_t* bpage) /*!< in/out: buffer page read from disk */
MY_ATTRIBUTE(( warn_unused_result));
/********************************************************************//** /********************************************************************//**
Checks if a page is all zeroes. Checks if a page is all zeroes.
@return TRUE if the page is all zeroes */ @return TRUE if the page is all zeroes */
@ -1490,7 +1540,7 @@ The hook that is called just after a page is read from disk.
The function decrypt disk content into buf_page_t and releases the The function decrypt disk content into buf_page_t and releases the
temporary buffer that was allocated in buf_page_decrypt_before_read */ temporary buffer that was allocated in buf_page_decrypt_before_read */
UNIV_INTERN UNIV_INTERN
ibool bool
buf_page_decrypt_after_read( buf_page_decrypt_after_read(
/*========================*/ /*========================*/
buf_page_t* page); /*!< in/out: buffer page read from disk */ buf_page_t* page); /*!< in/out: buffer page read from disk */
@ -1593,14 +1643,9 @@ struct buf_page_t{
operation needed. */ operation needed. */
unsigned key_version; /*!< key version for this block */ unsigned key_version; /*!< key version for this block */
bool page_encrypted; /*!< page is page encrypted */ bool encrypted; /*!< page is still encrypted */
bool page_compressed;/*!< page is page compressed */ bool corrupted; /*!< page is corrupted */
ulint stored_checksum;/*!< stored page checksum if page
encrypted */
bool encrypted; /*!< page is still encrypted */
ulint calculated_checksum;
/*!< calculated checksum if page
encrypted */
ulint real_size; /*!< Real size of the page ulint real_size; /*!< Real size of the page
Normal pages == UNIV_PAGE_SIZE Normal pages == UNIV_PAGE_SIZE
page compressed pages, payload page compressed pages, payload
@ -1714,6 +1759,7 @@ struct buf_page_t{
0 if the block was never accessed 0 if the block was never accessed
in the buffer pool. Protected by in the buffer pool. Protected by
block mutex */ block mutex */
ibool is_corrupt;
# if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG # if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG
ibool file_page_was_freed; ibool file_page_was_freed;
/*!< this is set to TRUE when /*!< this is set to TRUE when

48
storage/innobase/include/fil0crypt.h
View File

@ -328,7 +328,7 @@ fil_space_check_encryption_read(
/****************************************************************** /******************************************************************
Decrypt a page Decrypt a page
@return true if page is decrypted, false if not. */ @return true if page decrypted, false if not.*/
UNIV_INTERN UNIV_INTERN
bool bool
fil_space_decrypt( fil_space_decrypt(
@ -336,9 +336,10 @@ fil_space_decrypt(
fil_space_crypt_t* crypt_data, /*!< in: crypt data */ fil_space_crypt_t* crypt_data, /*!< in: crypt data */
byte* tmp_frame, /*!< in: temporary buffer */ byte* tmp_frame, /*!< in: temporary buffer */
ulint page_size, /*!< in: page size */ ulint page_size, /*!< in: page size */
byte* src_frame, /*!< in:out: page buffer */ byte* src_frame, /*!< in: out: page buffer */
dberr_t* err); /*!< in: out: DB_SUCCESS or dberr_t* err) /*!< in: out: DB_SUCCESS or
error code */ error code */
MY_ATTRIBUTE((warn_unused_result));
/********************************************************************* /*********************************************************************
Encrypt buffer page Encrypt buffer page
@ -355,31 +356,46 @@ fil_space_encrypt(
ulint size, /*!< in: size of data to encrypt */ ulint size, /*!< in: size of data to encrypt */
byte* dst_frame); /*!< in: where to encrypt to */ byte* dst_frame); /*!< in: where to encrypt to */
/********************************************************************* /******************************************************************
Decrypt buffer page Decrypt a page
@return decrypted page, or original not encrypted page if decrypt is @param[in] space Tablespace id
@param[in] tmp_frame Temporary buffer used for decrypting
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] decrypted true if page was decrypted
@return decrypted page, or original not encrypted page if decryption is
not needed.*/ not needed.*/
UNIV_INTERN UNIV_INTERN
byte* byte*
fil_space_decrypt( fil_space_decrypt(
/*==============*/ /*==============*/
ulint space, /*!< in: tablespace id */ ulint space,
byte* src_frame, /*!< in: page frame */ byte* src_frame,
ulint page_size, /*!< in: size of data to encrypt */ ulint page_size,
byte* dst_frame) /*!< in: where to decrypt to */ byte* dst_frame,
bool* decrypted)
__attribute__((warn_unused_result)); __attribute__((warn_unused_result));
/********************************************************************* /*********************************************************************
fil_space_verify_crypt_checksum Verify that post encryption checksum match calculated checksum.
NOTE: currently this function can only be run in single threaded mode This function should be called only if tablespace contains crypt_data
as it modifies srv_checksum_algorithm (temporarily) metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */ @return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN UNIV_INTERN
bool bool
fil_space_verify_crypt_checksum( fil_space_verify_crypt_checksum(
/*============================*/ byte* page,
const byte* src_frame,/*!< in: page frame */ ulint zip_size,
ulint zip_size); /*!< in: size of data to encrypt */ const fil_space_t* space,
ulint pageno)
__attribute__((warn_unused_result));
/********************************************************************* /*********************************************************************
Init threads for key rotation */ Init threads for key rotation */

10
storage/innobase/include/fil0fil.ic
View File

@ -59,6 +59,8 @@ fil_get_page_type_name(
switch(page_type) { switch(page_type) {
case FIL_PAGE_PAGE_COMPRESSED: case FIL_PAGE_PAGE_COMPRESSED:
return (const char*)"PAGE_COMPRESSED"; return (const char*)"PAGE_COMPRESSED";
case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
return (const char*)"PAGE_COMPRESSED_ENCRYPTED";
case FIL_PAGE_INDEX: case FIL_PAGE_INDEX:
return (const char*)"INDEX"; return (const char*)"INDEX";
case FIL_PAGE_UNDO_LOG: case FIL_PAGE_UNDO_LOG:
@ -87,9 +89,13 @@ fil_get_page_type_name(
return (const char*)"ZBLOB2"; return (const char*)"ZBLOB2";
case FIL_PAGE_TYPE_COMPRESSED: case FIL_PAGE_TYPE_COMPRESSED:
return (const char*)"ORACLE PAGE COMPRESSED"; return (const char*)"ORACLE PAGE COMPRESSED";
default:
return (const char*)"PAGE TYPE CORRUPTED"; /* No default to make compiler generate warning if
new page type is added but not handled here. */
} }
return (const char*)"PAGE TYPE CORRUPTED";
} }
/****************************************************************//** /****************************************************************//**

5
storage/innobase/row/row0import.cc
View File

@ -2033,12 +2033,15 @@ PageConverter::validate(
buf_block_t* block) UNIV_NOTHROW buf_block_t* block) UNIV_NOTHROW
{ {
buf_frame_t* page = get_frame(block); buf_frame_t* page = get_frame(block);
ulint space_id = mach_read_from_4(
page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
fil_space_t* space = fil_space_found_by_id(space_id);
/* Check that the page number corresponds to the offset in /* Check that the page number corresponds to the offset in
the file. Flag as corrupt if it doesn't. Disable the check the file. Flag as corrupt if it doesn't. Disable the check
for LSN in buf_page_is_corrupted() */ for LSN in buf_page_is_corrupted() */
if (buf_page_is_corrupted(false, page, get_zip_size()) if (buf_page_is_corrupted(false, page, get_zip_size(), space)
|| (page_get_page_no(page) != offset / m_page_size || (page_get_page_no(page) != offset / m_page_size
&& page_get_page_no(page) != 0)) { && page_get_page_no(page) != 0)) {

568
storage/xtradb/buf/buf0buf.cc
View File

@ -65,26 +65,9 @@ Created 11/5/1995 Heikki Tuuri
#include "fil0pagecompress.h" #include "fil0pagecompress.h"
#include "ha_prototypes.h" #include "ha_prototypes.h"
/* Enable this for checksum error messages. */
//#ifdef UNIV_DEBUG
//#define UNIV_DEBUG_LEVEL2 1
//#endif
/* prototypes for new functions added to ha_innodb.cc */ /* prototypes for new functions added to ha_innodb.cc */
trx_t* innobase_get_trx(); trx_t* innobase_get_trx();
/********************************************************************//**
Check if page is maybe compressed, encrypted or both when we encounter
corrupted page. Note that we can't be 100% sure if page is corrupted
or decrypt/decompress just failed.
*/
static
ibool
buf_page_check_corrupt(
/*===================*/
buf_page_t* bpage); /*!< in/out: buffer page read from
disk */
static inline static inline
void void
_increment_page_get_statistics(buf_block_t* block, trx_t* trx) _increment_page_get_statistics(buf_block_t* block, trx_t* trx)
@ -568,6 +551,7 @@ buf_block_alloc(
/********************************************************************//** /********************************************************************//**
Checks if a page is all zeroes. Checks if a page is all zeroes.
@return TRUE if the page is all zeroes */ @return TRUE if the page is all zeroes */
UNIV_INTERN
bool bool
buf_page_is_zeroes( buf_page_is_zeroes(
/*===============*/ /*===============*/
@ -590,7 +574,7 @@ buf_page_is_zeroes(
@param[in] checksum_field1 new checksum field @param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field @param[in] checksum_field2 old checksum field
@return true if the page is in crc32 checksum format */ @return true if the page is in crc32 checksum format */
UNIV_INLINE UNIV_INTERN
bool bool
buf_page_is_checksum_valid_crc32( buf_page_is_checksum_valid_crc32(
const byte* read_buf, const byte* read_buf,
@ -599,13 +583,11 @@ buf_page_is_checksum_valid_crc32(
{ {
ib_uint32_t crc32 = buf_calc_page_crc32(read_buf); ib_uint32_t crc32 = buf_calc_page_crc32(read_buf);
#ifdef UNIV_DEBUG_LEVEL2
if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) {
ib_logf(IB_LOG_LEVEL_INFO, DBUG_PRINT("buf_checksum",
"Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", ("Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.",
checksum_field1, checksum_field2, (ulint)crc32); checksum_field1, checksum_field2, (ulint)crc32));
} }
#endif
return(checksum_field1 == crc32 && checksum_field2 == crc32); return(checksum_field1 == crc32 && checksum_field2 == crc32);
} }
@ -615,7 +597,7 @@ buf_page_is_checksum_valid_crc32(
@param[in] checksum_field1 new checksum field @param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field @param[in] checksum_field2 old checksum field
@return true if the page is in innodb checksum format */ @return true if the page is in innodb checksum format */
UNIV_INLINE UNIV_INTERN
bool bool
buf_page_is_checksum_valid_innodb( buf_page_is_checksum_valid_innodb(
const byte* read_buf, const byte* read_buf,
@ -634,13 +616,12 @@ buf_page_is_checksum_valid_innodb(
if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN)
&& checksum_field2 != buf_calc_page_old_checksum(read_buf)) { && checksum_field2 != buf_calc_page_old_checksum(read_buf)) {
#ifdef UNIV_DEBUG_LEVEL2
ib_logf(IB_LOG_LEVEL_INFO, DBUG_PRINT("buf_checksum",
"Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", ("Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.",
checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf),
mach_read_from_4(read_buf + FIL_PAGE_LSN) mach_read_from_4(read_buf + FIL_PAGE_LSN)));
);
#endif
return(false); return(false);
} }
@ -651,13 +632,12 @@ buf_page_is_checksum_valid_innodb(
if (checksum_field1 != 0 if (checksum_field1 != 0
&& checksum_field1 != buf_calc_page_new_checksum(read_buf)) { && checksum_field1 != buf_calc_page_new_checksum(read_buf)) {
#ifdef UNIV_DEBUG_LEVEL2
ib_logf(IB_LOG_LEVEL_INFO, DBUG_PRINT("buf_checksum",
"Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", ("Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.",
checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf), checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf),
mach_read_from_4(read_buf + FIL_PAGE_LSN) mach_read_from_4(read_buf + FIL_PAGE_LSN)));
);
#endif
return(false); return(false);
} }
@ -669,22 +649,20 @@ buf_page_is_checksum_valid_innodb(
@param[in] checksum_field1 new checksum field @param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field @param[in] checksum_field2 old checksum field
@return true if the page is in none checksum format */ @return true if the page is in none checksum format */
UNIV_INLINE UNIV_INTERN
bool bool
buf_page_is_checksum_valid_none( buf_page_is_checksum_valid_none(
const byte* read_buf, const byte* read_buf,
ulint checksum_field1, ulint checksum_field1,
ulint checksum_field2) ulint checksum_field2)
{ {
#ifdef UNIV_DEBUG_LEVEL2
if (!(checksum_field1 == checksum_field2 || checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { if (!(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) {
ib_logf(IB_LOG_LEVEL_INFO, DBUG_PRINT("buf_checksum",
"Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", ("Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.",
checksum_field1, checksum_field2, BUF_NO_CHECKSUM_MAGIC, checksum_field1, checksum_field2, BUF_NO_CHECKSUM_MAGIC,
mach_read_from_4(read_buf + FIL_PAGE_LSN) mach_read_from_4(read_buf + FIL_PAGE_LSN)));
);
} }
#endif
return(checksum_field1 == checksum_field2 return(checksum_field1 == checksum_field2
&& checksum_field1 == BUF_NO_CHECKSUM_MAGIC); && checksum_field1 == BUF_NO_CHECKSUM_MAGIC);
@ -692,16 +670,18 @@ buf_page_is_checksum_valid_none(
/********************************************************************//** /********************************************************************//**
Checks if a page is corrupt. Checks if a page is corrupt.
@return TRUE if corrupted */ @param[in] check_lsn true if LSN should be checked
@param[in] read_buf Page to be checked
@param[in] zip_size compressed size or 0
@param[in] space Pointer to tablespace
@return true if corrupted, false if not */
UNIV_INTERN UNIV_INTERN
ibool bool
buf_page_is_corrupted( buf_page_is_corrupted(
/*==================*/ bool check_lsn,
bool check_lsn, /*!< in: true if we need to check const byte* read_buf,
and complain about the LSN */ ulint zip_size,
const byte* read_buf, /*!< in: a database page */ const fil_space_t* space)
ulint zip_size) /*!< in: size of compressed page;
0 for uncompressed pages */
{ {
ulint checksum_field1; ulint checksum_field1;
ulint checksum_field2; ulint checksum_field2;
@ -709,26 +689,21 @@ buf_page_is_corrupted(
read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint page_type = mach_read_from_4( ulint page_type = mach_read_from_4(
read_buf + FIL_PAGE_TYPE); read_buf + FIL_PAGE_TYPE);
bool no_checksum = (page_type == FIL_PAGE_PAGE_COMPRESSED ||
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
/* We can trust page type if page compression is set on tablespace
/* Page is encrypted if encryption information is found from flags because page compression flag means file must have been
tablespace and page contains used key_version. This is true created with 10.1 (later than 5.5 code base). In 10.1 page
also for pages first compressed and then encrypted. */ compressed tables do not contain post compression checksum and
if (crypt_data && FIL_PAGE_END_LSN_OLD_CHKSUM field stored. Note that space can
crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && be null if we are in fil_check_first_page() and first page
fil_page_is_encrypted(read_buf)) { is not compressed or encrypted. */
no_checksum = true; if ((page_type == FIL_PAGE_PAGE_COMPRESSED ||
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)
&& space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) {
return (false);
} }
/* Return early if there is no checksum or END_LSN */ if (!zip_size
if (no_checksum) {
return (FALSE);
}
if (!no_checksum && !zip_size
&& memcmp(read_buf + FIL_PAGE_LSN + 4, && memcmp(read_buf + FIL_PAGE_LSN + 4,
read_buf + UNIV_PAGE_SIZE read_buf + UNIV_PAGE_SIZE
- FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) { - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) {
@ -780,7 +755,7 @@ buf_page_is_corrupted(
/* Check whether the checksum fields have correct values */ /* Check whether the checksum fields have correct values */
if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
return(FALSE); return(false);
} }
if (zip_size) { if (zip_size) {
@ -807,14 +782,14 @@ buf_page_is_corrupted(
ib_logf(IB_LOG_LEVEL_INFO, ib_logf(IB_LOG_LEVEL_INFO,
"Checksum fields zero but page is not empty."); "Checksum fields zero but page is not empty.");
return(TRUE); return(true);
} }
} }
return(FALSE); return(false);
} }
DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); ); DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(true); );
ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET); ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET);
@ -827,7 +802,7 @@ buf_page_is_corrupted(
if (buf_page_is_checksum_valid_crc32(read_buf, if (buf_page_is_checksum_valid_crc32(read_buf,
checksum_field1, checksum_field2)) { checksum_field1, checksum_field2)) {
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_none(read_buf, if (buf_page_is_checksum_valid_none(read_buf,
@ -840,7 +815,7 @@ buf_page_is_corrupted(
space_id, page_no); space_id, page_no);
} }
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_innodb(read_buf, if (buf_page_is_checksum_valid_innodb(read_buf,
@ -853,17 +828,17 @@ buf_page_is_corrupted(
space_id, page_no); space_id, page_no);
} }
return(FALSE); return(false);
} }
return(TRUE); return(true);
case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
if (buf_page_is_checksum_valid_innodb(read_buf, if (buf_page_is_checksum_valid_innodb(read_buf,
checksum_field1, checksum_field2)) { checksum_field1, checksum_field2)) {
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_none(read_buf, if (buf_page_is_checksum_valid_none(read_buf,
@ -876,7 +851,7 @@ buf_page_is_corrupted(
space_id, page_no); space_id, page_no);
} }
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_crc32(read_buf, if (buf_page_is_checksum_valid_crc32(read_buf,
@ -889,16 +864,16 @@ buf_page_is_corrupted(
space_id, page_no); space_id, page_no);
} }
return(FALSE); return(false);
} }
return(TRUE); return(true);
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
if (buf_page_is_checksum_valid_none(read_buf, if (buf_page_is_checksum_valid_none(read_buf,
checksum_field1, checksum_field2)) { checksum_field1, checksum_field2)) {
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_crc32(read_buf, if (buf_page_is_checksum_valid_crc32(read_buf,
@ -907,7 +882,7 @@ buf_page_is_corrupted(
curr_algo, curr_algo,
SRV_CHECKSUM_ALGORITHM_CRC32, SRV_CHECKSUM_ALGORITHM_CRC32,
space_id, page_no); space_id, page_no);
return(FALSE); return(false);
} }
if (buf_page_is_checksum_valid_innodb(read_buf, if (buf_page_is_checksum_valid_innodb(read_buf,
@ -916,10 +891,10 @@ buf_page_is_corrupted(
curr_algo, curr_algo,
SRV_CHECKSUM_ALGORITHM_INNODB, SRV_CHECKSUM_ALGORITHM_INNODB,
space_id, page_no); space_id, page_no);
return(FALSE); return(false);
} }
return(TRUE); return(true);
case SRV_CHECKSUM_ALGORITHM_NONE: case SRV_CHECKSUM_ALGORITHM_NONE:
/* should have returned FALSE earlier */ /* should have returned FALSE earlier */
@ -929,7 +904,7 @@ buf_page_is_corrupted(
} }
ut_error; ut_error;
return(FALSE); return(false);
} }
/********************************************************************//** /********************************************************************//**
@ -1198,12 +1173,9 @@ buf_block_init(
block->page.state = BUF_BLOCK_NOT_USED; block->page.state = BUF_BLOCK_NOT_USED;
block->page.buf_fix_count = 0; block->page.buf_fix_count = 0;
block->page.io_fix = BUF_IO_NONE; block->page.io_fix = BUF_IO_NONE;
block->page.key_version = 0;
block->page.page_encrypted = false;
block->page.page_compressed = false;
block->page.encrypted = false; block->page.encrypted = false;
block->page.stored_checksum = BUF_NO_CHECKSUM_MAGIC; block->page.corrupted = false;
block->page.calculated_checksum = BUF_NO_CHECKSUM_MAGIC; block->page.key_version = 0;
block->page.real_size = 0; block->page.real_size = 0;
block->page.write_size = 0; block->page.write_size = 0;
block->modify_clock = 0; block->modify_clock = 0;
@ -3023,14 +2995,14 @@ loop:
} else if (retries < BUF_PAGE_READ_MAX_RETRIES) { } else if (retries < BUF_PAGE_READ_MAX_RETRIES) {
++retries; ++retries;
bool corrupted = true; bool corrupted = false;
if (bpage) { if (bpage) {
corrupted = buf_page_check_corrupt(bpage); corrupted = buf_page_check_corrupt(bpage);
} }
/* Do not try again for encrypted pages */ /* Do not try again for encrypted pages */
if (!corrupted) { if (corrupted && bpage->encrypted) {
ib_mutex_t* pmutex = buf_page_get_mutex(bpage); ib_mutex_t* pmutex = buf_page_get_mutex(bpage);
mutex_enter(&buf_pool->LRU_list_mutex); mutex_enter(&buf_pool->LRU_list_mutex);
mutex_enter(pmutex); mutex_enter(pmutex);
@ -3059,14 +3031,14 @@ loop:
retries = BUF_PAGE_READ_MAX_RETRIES; retries = BUF_PAGE_READ_MAX_RETRIES;
); );
} else { } else {
bool corrupted = true; bool corrupted = false;
if (bpage) { if (bpage) {
corrupted = buf_page_check_corrupt(bpage); corrupted = buf_page_check_corrupt(bpage);
} }
if (corrupted) { if (corrupted && !bpage->encrypted) {
fprintf(stderr, "InnoDB: Error: Unable" ib_logf(IB_LOG_LEVEL_ERROR, "Unable"
" to read tablespace %lu page no" " to read tablespace %lu page no"
" %lu into the buffer pool after" " %lu into the buffer pool after"
" %lu attempts\n" " %lu attempts\n"
@ -3877,12 +3849,9 @@ buf_page_init_low(
bpage->newest_modification = 0; bpage->newest_modification = 0;
bpage->oldest_modification = 0; bpage->oldest_modification = 0;
bpage->write_size = 0; bpage->write_size = 0;
bpage->key_version = 0;
bpage->stored_checksum = BUF_NO_CHECKSUM_MAGIC;
bpage->calculated_checksum = BUF_NO_CHECKSUM_MAGIC;
bpage->page_encrypted = false;
bpage->page_compressed = false;
bpage->encrypted = false; bpage->encrypted = false;
bpage->corrupted = false;
bpage->key_version = 0;
bpage->real_size = 0; bpage->real_size = 0;
HASH_INVALIDATE(bpage, hash); HASH_INVALIDATE(bpage, hash);
@ -4595,72 +4564,70 @@ buf_mark_space_corrupt(
Check if page is maybe compressed, encrypted or both when we encounter Check if page is maybe compressed, encrypted or both when we encounter
corrupted page. Note that we can't be 100% sure if page is corrupted corrupted page. Note that we can't be 100% sure if page is corrupted
or decrypt/decompress just failed. or decrypt/decompress just failed.
*/ @param[in,out] bpage Page
static @return true if page corrupted, false if not */
ibool UNIV_INTERN
bool
buf_page_check_corrupt( buf_page_check_corrupt(
/*===================*/ buf_page_t* bpage)
buf_page_t* bpage) /*!< in/out: buffer page read from disk */
{ {
ulint zip_size = buf_page_get_zip_size(bpage); ulint zip_size = buf_page_get_zip_size(bpage);
byte* dst_frame = (zip_size) ? bpage->zip.data : byte* dst_frame = (zip_size) ? bpage->zip.data :
((buf_block_t*) bpage)->frame; ((buf_block_t*) bpage)->frame;
bool page_compressed = bpage->page_encrypted;
ulint stored_checksum = bpage->stored_checksum;
ulint calculated_checksum = bpage->calculated_checksum;
bool page_compressed_encrypted = bpage->page_compressed;
ulint space_id = mach_read_from_4( ulint space_id = mach_read_from_4(
dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
fil_space_t* space = fil_space_found_by_id(space_id); fil_space_t* space = fil_space_found_by_id(space_id);
bool corrupted = true; fil_space_crypt_t* crypt_data = space->crypt_data;
ulint key_version = bpage->key_version; bool still_encrypted = false;
bool corrupted = false;
ulint page_type = mach_read_from_2(dst_frame + FIL_PAGE_TYPE);
if (key_version != 0 || page_compressed_encrypted) { /* In buf_decrypt_after_read we have either decrypted the page if
bpage->encrypted = true; page post encryption checksum matches and used key_id is found
from the encryption plugin. If checksum did not match page was
not decrypted and it could be either encrypted and corrupted
or corrupted or good page. If we decrypted, there page could
still be corrupted if used key does not match. */
still_encrypted = (crypt_data &&
crypt_data->type != CRYPT_SCHEME_UNENCRYPTED &&
!bpage->encrypted &&
fil_space_verify_crypt_checksum(dst_frame, zip_size,
space, bpage->offset));
if (!still_encrypted) {
/* If traditional checksums match, we assume that page is
not anymore encrypted. */
corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space);
if (!corrupted) {
bpage->encrypted = false;
}
} }
if (key_version != 0 || /* Pages that we think are unencrypted but do not match the checksum
(crypt_data && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) || checks could be corrupted or encrypted or both. */
page_compressed || page_compressed_encrypted) { if (corrupted && !bpage->encrypted) {
bpage->corrupted = true;
ib_logf(IB_LOG_LEVEL_ERROR,
"%s: Block in space_id %lu in file %s corrupted.",
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ? "Maybe corruption" : "Corruption",
space_id, space ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"Based on page type %s (" ULINTPF ")",
fil_get_page_type_name(page_type), page_type);
} else if (still_encrypted || (bpage->encrypted && corrupted)) {
bpage->encrypted = true;
bpage->corrupted = false;
corrupted = true;
/* Page is really corrupted if post encryption stored ib_logf(IB_LOG_LEVEL_ERROR,
checksum does not match calculated checksum after page was "Block in space_id %lu in file %s encrypted.",
read. For pages compressed and then encrypted, there is no space_id, space ? space->name : "NULL");
checksum. */ ib_logf(IB_LOG_LEVEL_ERROR,
corrupted = (!page_compressed_encrypted && stored_checksum != calculated_checksum); "However key management plugin or used key_version %u is not found or"
if (corrupted) {
ib_logf(IB_LOG_LEVEL_ERROR,
"%s: Block in space_id %lu in file %s corrupted.",
page_compressed_encrypted ? "Maybe corruption" : "Corruption",
space_id, space ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"Page based on contents %s encrypted.",
(key_version == 0 && page_compressed_encrypted == false) ? "not" : "maybe");
if (stored_checksum != BUF_NO_CHECKSUM_MAGIC || calculated_checksum != BUF_NO_CHECKSUM_MAGIC) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Page stored checksum %lu but calculated checksum %lu.",
stored_checksum, calculated_checksum);
}
ib_logf(IB_LOG_LEVEL_ERROR,
"Reason could be that key_version %lu in page "
"or in crypt_data %p could not be found.",
key_version, crypt_data);
ib_logf(IB_LOG_LEVEL_ERROR,
"Reason could be also that key management plugin is not found or"
" used encryption algorithm or method does not match.");
ib_logf(IB_LOG_LEVEL_ERROR,
"Based on page page compressed %d, compressed and encrypted %d.",
page_compressed, page_compressed_encrypted);
} else {
ib_logf(IB_LOG_LEVEL_ERROR,
"Block in space_id %lu in file %s encrypted.",
space_id, space ? space->name : "NULL");
ib_logf(IB_LOG_LEVEL_ERROR,
"However key management plugin or used key_id %lu is not found or"
" used encryption algorithm or method does not match.", " used encryption algorithm or method does not match.",
key_version); bpage->key_version);
if (space_id > TRX_SYS_SPACE) {
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"Marking tablespace as missing. You may drop this table or" "Marking tablespace as missing. You may drop this table or"
" install correct key management plugin and key file."); " install correct key management plugin and key file.");
@ -4686,6 +4653,8 @@ buf_page_io_complete(
== BUF_BLOCK_FILE_PAGE); == BUF_BLOCK_FILE_PAGE);
bool have_LRU_mutex = false; bool have_LRU_mutex = false;
fil_space_t* space = NULL; fil_space_t* space = NULL;
byte* frame = NULL;
bool corrupted = false;
ut_a(buf_page_in_file(bpage)); ut_a(buf_page_in_file(bpage));
@ -4701,21 +4670,13 @@ buf_page_io_complete(
if (io_type == BUF_IO_READ) { if (io_type == BUF_IO_READ) {
ulint read_page_no; ulint read_page_no;
ulint read_space_id; ulint read_space_id;
byte* frame;
if (!buf_page_decrypt_after_read(bpage)) { buf_page_decrypt_after_read(bpage);
/* encryption error! */
if (buf_page_get_zip_size(bpage)) {
frame = bpage->zip.data;
} else {
frame = ((buf_block_t*) bpage)->frame;
}
ib_logf(IB_LOG_LEVEL_INFO, if (buf_page_get_zip_size(bpage)) {
"Page %u in tablespace %u encryption error key_version %u.", frame = bpage->zip.data;
bpage->offset, bpage->space, bpage->key_version); } else {
frame = ((buf_block_t*) bpage)->frame;
goto database_corrupted;
} }
if (buf_page_get_zip_size(bpage)) { if (buf_page_get_zip_size(bpage)) {
@ -4732,6 +4693,9 @@ buf_page_io_complete(
"Page %u in tablespace %u zip_decompress failure.", "Page %u in tablespace %u zip_decompress failure.",
bpage->offset, bpage->space); bpage->offset, bpage->space);
corrupted = true;
bpage->corrupted = true;
goto database_corrupted; goto database_corrupted;
} }
os_atomic_decrement_ulint(&buf_pool->n_pend_unzip, 1); os_atomic_decrement_ulint(&buf_pool->n_pend_unzip, 1);
@ -4780,121 +4744,116 @@ buf_page_io_complete(
if (UNIV_LIKELY(!bpage->is_corrupt || if (UNIV_LIKELY(!bpage->is_corrupt ||
!srv_pass_corrupt_table)) { !srv_pass_corrupt_table)) {
/* From version 3.23.38 up we store the page checksum corrupted = buf_page_check_corrupt(bpage);
to the 4 first bytes of the page end lsn field */
if (buf_page_is_corrupted(true, frame, }
buf_page_get_zip_size(bpage))) {
/* Not a real corruption if it was triggered by
error injection */
DBUG_EXECUTE_IF("buf_page_is_corrupt_failure",
if (bpage->space > TRX_SYS_SPACE
&& buf_mark_space_corrupt(bpage)) {
ib_logf(IB_LOG_LEVEL_INFO,
"Simulated page corruption");
return(true);
}
goto page_not_corrupt;
;);
database_corrupted: database_corrupted:
bool corrupted = buf_page_check_corrupt(bpage); if (corrupted) {
/* Not a real corruption if it was triggered by
error injection */
if (corrupted) { DBUG_EXECUTE_IF("buf_page_is_corrupt_failure",
fil_system_enter(); if (bpage->space > TRX_SYS_SPACE
space = fil_space_get_by_id(bpage->space); && buf_mark_space_corrupt(bpage)) {
fil_system_exit(); ib_logf(IB_LOG_LEVEL_INFO,
ib_logf(IB_LOG_LEVEL_ERROR, "Simulated page corruption");
"Database page corruption on disk" return(true);
" or a failed"); }
ib_logf(IB_LOG_LEVEL_ERROR, goto page_not_corrupt;
"Space %u file %s read of page %u.", );
bpage->space,
space ? space->name : "NULL",
bpage->offset);
ib_logf(IB_LOG_LEVEL_ERROR,
"You may have to recover"
" from a backup.");
if (!bpage->encrypted) {
fil_system_enter();
space = fil_space_get_by_id(bpage->space);
fil_system_exit();
ib_logf(IB_LOG_LEVEL_ERROR,
"Database page corruption on disk"
" or a failed");
ib_logf(IB_LOG_LEVEL_ERROR,
"Space %u file %s read of page %u.",
bpage->space,
space ? space->name : "NULL",
bpage->offset);
ib_logf(IB_LOG_LEVEL_ERROR,
"You may have to recover"
" from a backup.");
buf_page_print(frame, buf_page_get_zip_size(bpage), buf_page_print(frame, buf_page_get_zip_size(bpage),
BUF_PAGE_PRINT_NO_CRASH); BUF_PAGE_PRINT_NO_CRASH);
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"It is also possible that your operating" "It is also possible that your operating"
"system has corrupted its own file cache."); "system has corrupted its own file cache.");
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"and rebooting your computer removes the error."); "and rebooting your computer removes the error.");
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"If the corrupt page is an index page you can also try to"); "If the corrupt page is an index page you can also try to");
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"fix the corruption by dumping, dropping, and reimporting"); "fix the corruption by dumping, dropping, and reimporting");
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"the corrupt table. You can use CHECK"); "the corrupt table. You can use CHECK");
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"TABLE to scan your table for corruption."); "TABLE to scan your table for corruption.");
ib_logf(IB_LOG_LEVEL_ERROR, ib_logf(IB_LOG_LEVEL_ERROR,
"See also " "See also "
REFMAN "forcing-innodb-recovery.html" REFMAN "forcing-innodb-recovery.html"
" about forcing recovery."); " about forcing recovery.");
}
if (srv_pass_corrupt_table && bpage->space != 0
&& bpage->space < SRV_LOG_SPACE_FIRST_ID) {
trx_t* trx;
fprintf(stderr,
"InnoDB: space %u will be treated as corrupt.\n",
bpage->space);
fil_space_set_corrupt(bpage->space);
trx = innobase_get_trx();
if (trx && trx->dict_operation_lock_mode == RW_X_LATCH) {
dict_table_set_corrupt_by_space(bpage->space, FALSE);
} else {
dict_table_set_corrupt_by_space(bpage->space, TRUE);
} }
if (srv_pass_corrupt_table && bpage->space != 0 bpage->is_corrupt = TRUE;
&& bpage->space < SRV_LOG_SPACE_FIRST_ID) { }
trx_t* trx;
fprintf(stderr, if (srv_force_recovery < SRV_FORCE_IGNORE_CORRUPT) {
"InnoDB: space %u will be treated as corrupt.\n", /* If page space id is larger than TRX_SYS_SPACE
bpage->space); (0), we will attempt to mark the corresponding
fil_space_set_corrupt(bpage->space); table as corrupted instead of crashing server */
if (bpage->space > TRX_SYS_SPACE
&& buf_mark_space_corrupt(bpage)) {
return(false);
} else {
if (!bpage->encrypted) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Ending processing because of a corrupt database page.");
trx = innobase_get_trx(); ut_error;
if (trx && trx->dict_operation_lock_mode == RW_X_LATCH) {
dict_table_set_corrupt_by_space(bpage->space, FALSE);
} else {
dict_table_set_corrupt_by_space(bpage->space, TRUE);
} }
bpage->is_corrupt = TRUE;
}
if (srv_force_recovery < SRV_FORCE_IGNORE_CORRUPT) { ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED,
/* If page space id is larger than TRX_SYS_SPACE "Table in tablespace %lu encrypted."
(0), we will attempt to mark the corresponding "However key management plugin or used key_id %lu is not found or"
table as corrupted instead of crashing server */ " used encryption algorithm or method does not match."
if (bpage->space > TRX_SYS_SPACE " Can't continue opening the table.",
&& buf_mark_space_corrupt(bpage)) { bpage->space, bpage->key_version);
return(false);
if (bpage->encrypted && bpage->space > TRX_SYS_SPACE) {
buf_mark_space_corrupt(bpage);
} else { } else {
corrupted = buf_page_check_corrupt(bpage); ut_error;
ulint key_version = bpage->key_version;
if (corrupted) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Ending processing because of a corrupt database page.");
ut_error;
}
ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED,
"Table in tablespace %lu encrypted."
"However key management plugin or used key_id %lu is not found or"
" used encryption algorithm or method does not match."
" Can't continue opening the table.",
(ulint)bpage->space, key_version);
if (bpage->space > TRX_SYS_SPACE) {
if (corrupted) {
buf_mark_space_corrupt(bpage);
}
} else {
ut_error;
}
return(false);
} }
return(false);
} }
} }
} /**/ }
DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", DBUG_EXECUTE_IF("buf_page_is_corrupt_failure",
page_not_corrupt: bpage = bpage; ); page_not_corrupt: bpage = bpage; );
@ -4924,9 +4883,9 @@ database_corrupted:
} }
if (bpage && bpage->encrypted) { if (bpage && bpage->encrypted) {
fprintf(stderr, ib_logf(IB_LOG_LEVEL_WARN,
"InnoDB: Warning: Table in tablespace %lu encrypted." "Table in tablespace %lu encrypted."
"However key management plugin or used key_id %u is not found or" "However key management plugin or used key_version %u is not found or"
" used encryption algorithm or method does not match." " used encryption algorithm or method does not match."
" Can't continue opening the table.\n", " Can't continue opening the table.\n",
(ulint)bpage->space, bpage->key_version); (ulint)bpage->space, bpage->key_version);
@ -6301,13 +6260,16 @@ buf_pool_reserve_tmp_slot(
/********************************************************************//** /********************************************************************//**
Encrypts a buffer page right before it's flushed to disk Encrypts a buffer page right before it's flushed to disk
@param[in,out] bpage Page control block
@param[in,out] src_frame Source page
@param[in] space_id Tablespace id
@return either unencrypted source page or decrypted page.
*/ */
byte* byte*
buf_page_encrypt_before_write( buf_page_encrypt_before_write(
/*==========================*/ buf_page_t* bpage,
buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ byte* src_frame,
byte* src_frame, /*!< in: src frame */ ulint space_id)
ulint space_id) /*!< in: space id */
{ {
fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id);
ulint zip_size = buf_page_get_zip_size(bpage); ulint zip_size = buf_page_get_zip_size(bpage);
@ -6426,11 +6388,13 @@ buf_page_encrypt_before_write(
/********************************************************************//** /********************************************************************//**
Decrypt page after it has been read from disk Decrypt page after it has been read from disk
@param[in,out] bpage Page control block
@return true if successfull, false if something went wrong
*/ */
ibool UNIV_INTERN
bool
buf_page_decrypt_after_read( buf_page_decrypt_after_read(
/*========================*/ buf_page_t* bpage)
buf_page_t* bpage) /*!< in/out: buffer page read from disk */
{ {
ulint zip_size = buf_page_get_zip_size(bpage); ulint zip_size = buf_page_get_zip_size(bpage);
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
@ -6450,46 +6414,17 @@ buf_page_decrypt_after_read(
/* Page is encrypted if encryption information is found from /* Page is encrypted if encryption information is found from
tablespace and page contains used key_version. This is true tablespace and page contains used key_version. This is true
also for pages first compressed and then encrypted. */ also for pages first compressed and then encrypted. */
if (!crypt_data || if (!crypt_data && key_version != 0) {
(crypt_data && key_version = 0;
crypt_data->type == CRYPT_SCHEME_UNENCRYPTED &&
key_version != 0)) {
byte* frame = NULL;
if (buf_page_get_zip_size(bpage)) {
frame = bpage->zip.data;
} else {
frame = ((buf_block_t*) bpage)->frame;
}
/* If page is not corrupted at this point, page can't be
encrypted, thus set key_version to 0. If page is corrupted,
we assume at this point that it is encrypted as page
contained key_version != 0. Note that page could still be
really corrupted. This we will find out after decrypt by
checking page checksums. */
if (!buf_page_is_corrupted(false, frame, buf_page_get_zip_size(bpage))) {
key_version = 0;
}
} }
/* If page is encrypted read post-encryption checksum */ bpage->key_version = key_version;
if (!page_compressed_encrypted && key_version != 0) {
bpage->stored_checksum = mach_read_from_4(dst_frame + + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
}
ut_ad(bpage->key_version == 0);
if (bpage->offset == 0) { if (bpage->offset == 0) {
/* File header pages are not encrypted/compressed */ /* File header pages are not encrypted/compressed */
return (TRUE); return (true);
} }
/* Store these for corruption check */
bpage->key_version = key_version;
bpage->page_encrypted = page_compressed_encrypted;
bpage->page_compressed = page_compressed;
if (page_compressed) { if (page_compressed) {
/* the page we read is unencrypted */ /* the page we read is unencrypted */
/* Find free slot from temporary memory array */ /* Find free slot from temporary memory array */
@ -6516,6 +6451,13 @@ buf_page_decrypt_after_read(
buf_tmp_buffer_t* slot = NULL; buf_tmp_buffer_t* slot = NULL;
if (key_version) { if (key_version) {
/* Verify encryption checksum before we even try to
decrypt. */
if (!fil_space_verify_crypt_checksum(dst_frame,
zip_size, NULL, bpage->offset)) {
return (false);
}
/* Find free slot from temporary memory array */ /* Find free slot from temporary memory array */
slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed);
@ -6523,20 +6465,14 @@ buf_page_decrypt_after_read(
fil_page_type_validate(dst_frame); fil_page_type_validate(dst_frame);
#endif #endif
/* Calculate checksum before decrypt, this will be
used later to find out if incorrect key was used. */
if (!page_compressed_encrypted) {
bpage->calculated_checksum = fil_crypt_calculate_checksum(zip_size, dst_frame);
}
/* decrypt using crypt_buf to dst_frame */ /* decrypt using crypt_buf to dst_frame */
byte* res = fil_space_decrypt(bpage->space, byte* res = fil_space_decrypt(bpage->space,
slot->crypt_buf, slot->crypt_buf,
size, size,
dst_frame); dst_frame,
&bpage->encrypted);
if (!res) { if (!res) {
bpage->encrypted = true;
success = false; success = false;
} }
#ifdef UNIV_DEBUG #ifdef UNIV_DEBUG
@ -6569,7 +6505,5 @@ buf_page_decrypt_after_read(
} }
} }
bpage->key_version = key_version;
return (success); return (success);
} }

16
storage/xtradb/buf/buf0dblwr.cc
View File

@ -383,10 +383,11 @@ buf_dblwr_init_or_load_pages(
doublewrite = read_buf + TRX_SYS_DOUBLEWRITE; doublewrite = read_buf + TRX_SYS_DOUBLEWRITE;
if (mach_read_from_4(read_buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) { if (mach_read_from_4(read_buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) {
bool decrypted = false;
byte* tmp = fil_space_decrypt((ulint)TRX_SYS_SPACE, byte* tmp = fil_space_decrypt((ulint)TRX_SYS_SPACE,
read_buf + UNIV_PAGE_SIZE, read_buf + UNIV_PAGE_SIZE,
UNIV_PAGE_SIZE, /* page size */ UNIV_PAGE_SIZE, /* page size */
read_buf); read_buf, &decrypted);
doublewrite = tmp + TRX_SYS_DOUBLEWRITE; doublewrite = tmp + TRX_SYS_DOUBLEWRITE;
} }
@ -487,6 +488,7 @@ buf_dblwr_process()
byte* read_buf; byte* read_buf;
byte* unaligned_read_buf; byte* unaligned_read_buf;
recv_dblwr_t& recv_dblwr = recv_sys->dblwr; recv_dblwr_t& recv_dblwr = recv_sys->dblwr;
fil_space_t* space=NULL;
unaligned_read_buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE)); unaligned_read_buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
@ -514,6 +516,10 @@ buf_dblwr_process()
continue; continue;
} }
if (!space) {
space = fil_space_found_by_id(space_id);
}
ulint zip_size = fil_space_get_zip_size(space_id); ulint zip_size = fil_space_get_zip_size(space_id);
ut_ad(!buf_page_is_zeroes(page, zip_size)); ut_ad(!buf_page_is_zeroes(page, zip_size));
@ -548,9 +554,9 @@ buf_dblwr_process()
} }
if (fil_space_verify_crypt_checksum( if (fil_space_verify_crypt_checksum(
read_buf, zip_size) read_buf, zip_size, NULL, page_no)
|| !buf_page_is_corrupted( || !buf_page_is_corrupted(
true, read_buf, zip_size)) { true, read_buf, zip_size, space)) {
/* The page is good; there is no need /* The page is good; there is no need
to consult the doublewrite buffer. */ to consult the doublewrite buffer. */
continue; continue;
@ -573,8 +579,8 @@ buf_dblwr_process()
NULL, page, UNIV_PAGE_SIZE, NULL, true); NULL, page, UNIV_PAGE_SIZE, NULL, true);
} }
if (!fil_space_verify_crypt_checksum(page, zip_size) if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no)
&& buf_page_is_corrupted(true, page, zip_size)) { && buf_page_is_corrupted(true, page, zip_size, space)) {
if (!is_all_zero) { if (!is_all_zero) {
ib_logf(IB_LOG_LEVEL_WARN, ib_logf(IB_LOG_LEVEL_WARN,
"A doublewrite copy of page " "A doublewrite copy of page "

187
storage/xtradb/fil/fil0crypt.cc
View File

@ -625,6 +625,8 @@ fil_encrypt_buf(
// store the post-encryption checksum after the key-version // store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum); mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum);
ut_ad(fil_space_verify_crypt_checksum(dst_frame, zip_size, NULL, offset));
srv_stats.pages_encrypted.inc(); srv_stats.pages_encrypted.inc();
return dst_frame; return dst_frame;
@ -676,6 +678,7 @@ fil_space_encrypt(
byte* comp_mem = NULL; byte* comp_mem = NULL;
byte* uncomp_mem = NULL; byte* uncomp_mem = NULL;
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
fil_space_t* tspace = fil_space_found_by_id(space);
if (page_compressed_encrypted) { if (page_compressed_encrypted) {
comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); comp_mem = (byte *)malloc(UNIV_PAGE_SIZE);
@ -685,7 +688,7 @@ fil_space_encrypt(
src = uncomp_mem; src = uncomp_mem;
} }
bool corrupted1 = buf_page_is_corrupted(true, src, zip_size); bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, tspace);
bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err);
/* Need to decompress the page if it was also compressed */ /* Need to decompress the page if it was also compressed */
@ -694,7 +697,7 @@ fil_space_encrypt(
fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL);
} }
bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size); bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, tspace);
bool different = memcmp(src, tmp_mem, size); bool different = memcmp(src, tmp_mem, size);
if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) {
@ -858,19 +861,25 @@ fil_space_decrypt(
/****************************************************************** /******************************************************************
Decrypt a page Decrypt a page
@return encrypted page, or original not encrypted page if encryption is @param[in] space Tablespace id
not needed. */ @param[in] tmp_frame Temporary buffer used for decrypting
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] decrypted true if page was decrypted
@return decrypted page, or original not encrypted page if decryption is
not needed.*/
UNIV_INTERN UNIV_INTERN
byte* byte*
fil_space_decrypt( fil_space_decrypt(
/*==============*/ ulint space,
ulint space, /*!< in: Fil space id */ byte* tmp_frame,
byte* tmp_frame, /*!< in: temporary buffer */ ulint page_size,
ulint page_size, /*!< in: page size */ byte* src_frame,
byte* src_frame) /*!< in/out: page buffer */ bool* decrypted)
{ {
dberr_t err = DB_SUCCESS; dberr_t err = DB_SUCCESS;
byte* res = NULL; byte* res = NULL;
*decrypted = false;
bool encrypted = fil_space_decrypt( bool encrypted = fil_space_decrypt(
fil_space_get_crypt_data(space), fil_space_get_crypt_data(space),
@ -881,6 +890,7 @@ fil_space_decrypt(
if (err == DB_SUCCESS) { if (err == DB_SUCCESS) {
if (encrypted) { if (encrypted) {
*decrypted = true;
/* Copy the decrypted page back to page buffer, not /* Copy the decrypted page back to page buffer, not
really any other options. */ really any other options. */
memcpy(src_frame, tmp_frame, page_size); memcpy(src_frame, tmp_frame, page_size);
@ -934,83 +944,114 @@ fil_crypt_calculate_checksum(
} }
/********************************************************************* /*********************************************************************
Verify checksum for a page (iff it's encrypted) Verify that post encryption checksum match calculated checksum.
NOTE: currently this function can only be run in single threaded mode This function should be called only if tablespace contains crypt_data
as it modifies srv_checksum_algorithm (temporarily) metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */ @return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN UNIV_INTERN
bool bool
fil_space_verify_crypt_checksum( fil_space_verify_crypt_checksum(
/*============================*/ byte* page,
const byte* src_frame, /*!< in: page the verify */ ulint zip_size,
ulint zip_size) /*!< in: compressed size if const fil_space_t* space,
row_format compressed */ ulint pageno)
{ {
// key version uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
uint key_version = mach_read_from_4(
src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* If page is not encrypted, return false */
if (key_version == 0) { if (key_version == 0) {
return false; // unencrypted page return false;
} }
/* "trick" the normal checksum routines by storing the post-encryption /* If no checksum is used, can't continue checking. */
* checksum into the normal checksum field allowing for reuse of if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
* the normal routines */ return(true);
// post encryption checksum
ib_uint32_t stored_post_encryption = mach_read_from_4(
src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
// save pre encryption checksum for restore in end of this function
ib_uint32_t stored_pre_encryption = mach_read_from_4(
src_frame + FIL_PAGE_SPACE_OR_CHKSUM);
ib_uint32_t checksum_field2 = mach_read_from_4(
src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
/** prepare frame for usage of normal checksum routines */
mach_write_to_4(const_cast<byte*>(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM,
stored_post_encryption);
/* NOTE: this function is (currently) only run when restoring
* dblwr-buffer, server is single threaded so it's safe to modify
* srv_checksum_algorithm */
srv_checksum_algorithm_t save_checksum_algorithm =
(srv_checksum_algorithm_t)srv_checksum_algorithm;
if (zip_size == 0 &&
(save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB ||
save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) {
/* handle ALGORITHM_INNODB specially,
* "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC
* checksum_field2 is sort of pointless anyway...
*/
srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
mach_write_to_4(const_cast<byte*>(src_frame) +
UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
BUF_NO_CHECKSUM_MAGIC);
} }
/* verify checksums */ /* Read stored post encryption checksum. */
ibool corrupted = buf_page_is_corrupted(false, src_frame, zip_size); ib_uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/** restore frame & algorithm */ /* Declare empty pages non-corrupted */
srv_checksum_algorithm = save_checksum_algorithm; if (checksum == 0
&& *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0) {
mach_write_to_4(const_cast<byte*>(src_frame) + return (true);
FIL_PAGE_SPACE_OR_CHKSUM,
stored_pre_encryption);
mach_write_to_4(const_cast<byte*>(src_frame) +
UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM,
checksum_field2);
if (!corrupted) {
return true; // page was encrypted and checksum matched
} else {
return false; // page was encrypted but checksum didn't match
} }
/* Compressed and encrypted pages do not have checksum. Assume not
corrupted. */
if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return (true);
}
/* Compressed pages use different checksum method. We first store
the post encryption checksum on checksum location and after function
restore the original. */
if (zip_size) {
ib_uint32_t old = static_cast<ib_uint32_t>(mach_read_from_4(
page + FIL_PAGE_SPACE_OR_CHKSUM));
mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
bool valid = page_zip_verify_checksum(page, zip_size);
mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old);
return (valid);
}
/* If stored checksum matches one of the calculated checksums
page is not corrupted. */
ib_uint32_t cchecksum1 = buf_calc_page_crc32(page);
ib_uint32_t cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum(
page);
bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
|| checksum == BUF_NO_CHECKSUM_MAGIC);
/* Old InnoDB versions did not initialize
FIL_PAGE_FILE_FLUSH_LSN field so there could be garbage
and above checksum check could produce false positive.
Thus we also check does the traditional stored
checksum fields match the calculated one. Both of these
could naturally produce false positive but then
we just decrypt the page and after that corrupted
pages very probable stay corrupted and valid
pages very probable stay valid.
*/
ulint checksum1 = mach_read_from_4(
page + FIL_PAGE_SPACE_OR_CHKSUM);
ulint checksum2 = mach_read_from_4(
page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
bool valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
|| buf_page_is_checksum_valid_none(page,checksum1,checksum2)
|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
if (encrypted && valid) {
/* If page is encrypted and traditional checksums match,
page could be still encrypted, or not encrypted and valid or
corrupted. */
ib_logf(IB_LOG_LEVEL_ERROR,
" Page %lu in space %s (%lu) maybe corrupted."
" Post encryption checksum %u stored [%lu:%lu] key_version %u",
pageno,
space ? space->name : "N/A",
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2, key_version);
encrypted = false;
}
return(encrypted);
} }
/***********************************************************************/ /***********************************************************************/

6
storage/xtradb/fil/fil0fil.cc
View File

@ -2309,7 +2309,7 @@ fil_check_first_page(const page_t* page, ulint space_id, ulint flags)
} }
if (buf_page_is_corrupted( if (buf_page_is_corrupted(
false, page, fsp_flags_get_zip_size(flags))) { false, page, fsp_flags_get_zip_size(flags), NULL)) {
return("checksum mismatch"); return("checksum mismatch");
} }
@ -4547,13 +4547,13 @@ fil_user_tablespace_find_space_id(
to UNIV_PAGE_SIZE. */ to UNIV_PAGE_SIZE. */
if (page_size == UNIV_PAGE_SIZE) { if (page_size == UNIV_PAGE_SIZE) {
uncompressed_ok = !buf_page_is_corrupted( uncompressed_ok = !buf_page_is_corrupted(
false, page, 0); false, page, 0, NULL);
} }
bool compressed_ok = false; bool compressed_ok = false;
if (page_size <= UNIV_PAGE_SIZE_DEF) { if (page_size <= UNIV_PAGE_SIZE_DEF) {
compressed_ok = !buf_page_is_corrupted( compressed_ok = !buf_page_is_corrupted(
false, page, page_size); false, page, page_size, NULL);
} }
if (uncompressed_ok || compressed_ok) { if (uncompressed_ok || compressed_ok) {

2
storage/xtradb/fil/fil0pagecompress.cc
View File

@ -366,7 +366,7 @@ fil_compress_page(
fil_decompress_page(uncomp_page, comp_page, len, NULL); fil_decompress_page(uncomp_page, comp_page, len, NULL);
if(buf_page_is_corrupted(false, uncomp_page, 0)) { if(buf_page_is_corrupted(false, uncomp_page, 0, space)) {
buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH);
ut_error; ut_error;
} }

86
storage/xtradb/include/buf0buf.h
View File

@ -1,7 +1,7 @@
/***************************************************************************** /*****************************************************************************
Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2013, 2016, MariaDB Corporation. Copyright (c) 2013, 2017, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software the terms of the GNU General Public License as published by the Free Software
@ -639,19 +639,68 @@ buf_block_unfix(
#else /* !UNIV_HOTBACKUP */ #else /* !UNIV_HOTBACKUP */
# define buf_block_modify_clock_inc(block) ((void) 0) # define buf_block_modify_clock_inc(block) ((void) 0)
#endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_HOTBACKUP */
/** Checks if the page is in crc32 checksum format.
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field
@return true if the page is in crc32 checksum format */
bool
buf_page_is_checksum_valid_crc32(
const byte* read_buf,
ulint checksum_field1,
ulint checksum_field2)
MY_ATTRIBUTE((warn_unused_result));
/** Checks if the page is in innodb checksum format.
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field
@return true if the page is in innodb checksum format */
bool
buf_page_is_checksum_valid_innodb(
const byte* read_buf,
ulint checksum_field1,
ulint checksum_field2)
MY_ATTRIBUTE((warn_unused_result));
/** Checks if the page is in none checksum format.
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field
@return true if the page is in none checksum format */
bool
buf_page_is_checksum_valid_none(
const byte* read_buf,
ulint checksum_field1,
ulint checksum_field2)
MY_ATTRIBUTE((warn_unused_result));
/********************************************************************//** /********************************************************************//**
Checks if a page is corrupt. Checks if a page is corrupt.
@return TRUE if corrupted */ @param[in] check_lsn true if LSN should be checked
UNIV_INTERN @param[in] read_buf Page to be checked
ibool @param[in] zip_size compressed size or 0
@param[in] space Pointer to tablespace
@return true if corrupted, false if not */
bool
buf_page_is_corrupted( buf_page_is_corrupted(
/*==================*/ bool check_lsn,
bool check_lsn, /*!< in: true if we need to check the const byte* read_buf,
and complain about the LSN */ ulint zip_size,
const byte* read_buf, /*!< in: a database page */ const fil_space_t* space)
ulint zip_size) /*!< in: size of compressed page; MY_ATTRIBUTE((warn_unused_result));
0 for uncompressed pages */ /********************************************************************//**
MY_ATTRIBUTE((nonnull, warn_unused_result)); Check if page is maybe compressed, encrypted or both when we encounter
corrupted page. Note that we can't be 100% sure if page is corrupted
or decrypt/decompress just failed.
@param[in] bpage Page
@return true if page corrupted, false if not */
bool
buf_page_check_corrupt(
buf_page_t* bpage) /*!< in/out: buffer page read from disk */
MY_ATTRIBUTE(( warn_unused_result));
/********************************************************************//** /********************************************************************//**
Checks if a page is all zeroes. Checks if a page is all zeroes.
@return TRUE if the page is all zeroes */ @return TRUE if the page is all zeroes */
@ -1524,7 +1573,7 @@ The hook that is called just after a page is read from disk.
The function decrypt disk content into buf_page_t and releases the The function decrypt disk content into buf_page_t and releases the
temporary buffer that was allocated in buf_page_decrypt_before_read */ temporary buffer that was allocated in buf_page_decrypt_before_read */
UNIV_INTERN UNIV_INTERN
ibool bool
buf_page_decrypt_after_read( buf_page_decrypt_after_read(
/*========================*/ /*========================*/
buf_page_t* page); /*!< in/out: buffer page read from disk */ buf_page_t* page); /*!< in/out: buffer page read from disk */
@ -1630,15 +1679,9 @@ struct buf_page_t{
if written again we check is TRIM if written again we check is TRIM
operation needed. */ operation needed. */
unsigned key_version; /*!< key version for this block */ unsigned key_version; /*!< key version for this block */
bool page_encrypted; /*!< page is page encrypted */ bool encrypted; /*!< page is still encrypted */
bool page_compressed;/*!< page is page compressed */ bool corrupted; /*!< page is corrupted */
ulint stored_checksum;/*!< stored page checksum if page
encrypted */
bool encrypted; /*!< page is still encrypted */
ulint calculated_checksum;
/*!< calculated checksum if page
encrypted */
ulint real_size; /*!< Real size of the page ulint real_size; /*!< Real size of the page
Normal pages == UNIV_PAGE_SIZE Normal pages == UNIV_PAGE_SIZE
@ -2318,7 +2361,6 @@ buf_pool_mutex_exit(
/*================*/ /*================*/
buf_pool_t* buf_pool); /*!< in: buffer pool */ buf_pool_t* buf_pool); /*!< in: buffer pool */
#ifndef UNIV_NONINL #ifndef UNIV_NONINL
#include "buf0buf.ic" #include "buf0buf.ic"
#endif #endif

48
storage/xtradb/include/fil0crypt.h
View File

@ -328,7 +328,7 @@ fil_space_check_encryption_read(
/****************************************************************** /******************************************************************
Decrypt a page Decrypt a page
@return true if page is decrypted, false if not. */ @return true if page decrypted, false if not.*/
UNIV_INTERN UNIV_INTERN
bool bool
fil_space_decrypt( fil_space_decrypt(
@ -336,9 +336,10 @@ fil_space_decrypt(
fil_space_crypt_t* crypt_data, /*!< in: crypt data */ fil_space_crypt_t* crypt_data, /*!< in: crypt data */
byte* tmp_frame, /*!< in: temporary buffer */ byte* tmp_frame, /*!< in: temporary buffer */
ulint page_size, /*!< in: page size */ ulint page_size, /*!< in: page size */
byte* src_frame, /*!< in:out: page buffer */ byte* src_frame, /*!< in: out: page buffer */
dberr_t* err); /*!< in: out: DB_SUCCESS or dberr_t* err) /*!< in: out: DB_SUCCESS or
error code */ error code */
MY_ATTRIBUTE((warn_unused_result));
/********************************************************************* /*********************************************************************
Encrypt buffer page Encrypt buffer page
@ -355,31 +356,46 @@ fil_space_encrypt(
ulint size, /*!< in: size of data to encrypt */ ulint size, /*!< in: size of data to encrypt */
byte* dst_frame); /*!< in: where to encrypt to */ byte* dst_frame); /*!< in: where to encrypt to */
/********************************************************************* /******************************************************************
Decrypt buffer page Decrypt a page
@return decrypted page, or original not encrypted page if decrypt is @param[in] space Tablespace id
@param[in] tmp_frame Temporary buffer used for decrypting
@param[in] page_size Page size
@param[in,out] src_frame Page to decrypt
@param[out] decrypted true if page was decrypted
@return decrypted page, or original not encrypted page if decryption is
not needed.*/ not needed.*/
UNIV_INTERN UNIV_INTERN
byte* byte*
fil_space_decrypt( fil_space_decrypt(
/*==============*/ /*==============*/
ulint space, /*!< in: tablespace id */ ulint space,
byte* src_frame, /*!< in: page frame */ byte* src_frame,
ulint page_size, /*!< in: size of data to encrypt */ ulint page_size,
byte* dst_frame) /*!< in: where to decrypt to */ byte* dst_frame,
bool* decrypted)
__attribute__((warn_unused_result)); __attribute__((warn_unused_result));
/********************************************************************* /*********************************************************************
fil_space_verify_crypt_checksum Verify that post encryption checksum match calculated checksum.
NOTE: currently this function can only be run in single threaded mode This function should be called only if tablespace contains crypt_data
as it modifies srv_checksum_algorithm (temporarily) metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */ @return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN UNIV_INTERN
bool bool
fil_space_verify_crypt_checksum( fil_space_verify_crypt_checksum(
/*============================*/ byte* page,
const byte* src_frame,/*!< in: page frame */ ulint zip_size,
ulint zip_size); /*!< in: size of data to encrypt */ const fil_space_t* space,
ulint pageno)
__attribute__((warn_unused_result));
/********************************************************************* /*********************************************************************
Init threads for key rotation */ Init threads for key rotation */

10
storage/xtradb/include/fil0fil.ic
View File

@ -59,6 +59,8 @@ fil_get_page_type_name(
switch(page_type) { switch(page_type) {
case FIL_PAGE_PAGE_COMPRESSED: case FIL_PAGE_PAGE_COMPRESSED:
return (const char*)"PAGE_COMPRESSED"; return (const char*)"PAGE_COMPRESSED";
case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
return (const char*)"PAGE_COMPRESSED_ENCRYPTED";
case FIL_PAGE_INDEX: case FIL_PAGE_INDEX:
return (const char*)"INDEX"; return (const char*)"INDEX";
case FIL_PAGE_UNDO_LOG: case FIL_PAGE_UNDO_LOG:
@ -87,9 +89,13 @@ fil_get_page_type_name(
return (const char*)"ZBLOB2"; return (const char*)"ZBLOB2";
case FIL_PAGE_TYPE_COMPRESSED: case FIL_PAGE_TYPE_COMPRESSED:
return (const char*)"ORACLE PAGE COMPRESSED"; return (const char*)"ORACLE PAGE COMPRESSED";
default:
return (const char*)"PAGE TYPE CORRUPTED"; /* No default to make compiler generate warning if
new page type is added but not handled here. */
} }
return (const char*)"PAGE TYPE CORRUPTED";
} }
/****************************************************************//** /****************************************************************//**

6
storage/xtradb/row/row0import.cc
View File

@ -40,6 +40,7 @@ Created 2012-02-08 by Sunny Bains.
#include "row0mysql.h" #include "row0mysql.h"
#include "srv0start.h" #include "srv0start.h"
#include "row0quiesce.h" #include "row0quiesce.h"
#include "buf0buf.h"
#include <vector> #include <vector>
@ -2036,12 +2037,15 @@ PageConverter::validate(
buf_block_t* block) UNIV_NOTHROW buf_block_t* block) UNIV_NOTHROW
{ {
buf_frame_t* page = get_frame(block); buf_frame_t* page = get_frame(block);
ulint space_id = mach_read_from_4(
page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
fil_space_t* space = fil_space_found_by_id(space_id);
/* Check that the page number corresponds to the offset in /* Check that the page number corresponds to the offset in
the file. Flag as corrupt if it doesn't. Disable the check the file. Flag as corrupt if it doesn't. Disable the check
for LSN in buf_page_is_corrupted() */ for LSN in buf_page_is_corrupted() */
if (buf_page_is_corrupted(false, page, get_zip_size()) if (buf_page_is_corrupted(false, page, get_zip_size(), space)
|| (page_get_page_no(page) != offset / m_page_size || (page_get_page_no(page) != offset / m_page_size
&& page_get_page_no(page) != 0)) { && page_get_page_no(page) != 0)) {