diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index b613cca47be..82bbd60f59f 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -2,7 +2,7 @@ call mtr.add_suppression("Plugin 'file_key_management' init function returned er call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result index ea760e7a213..deb0a5e8cc3 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result @@ -1,5 +1,5 @@ call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result index 531ba4063a4..36de9729a9b 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result @@ -1,5 +1,5 @@ call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change5.result b/mysql-test/suite/encryption/r/innodb-bad-key-change5.result index 11130a7a20b..cb1073cbaae 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change5.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change5.result @@ -1,5 +1,5 @@ call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result b/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result index 630fba146bf..ded4ad698b0 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result @@ -1,5 +1,5 @@ call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/r/innodb-encryption-disable.result b/mysql-test/suite/encryption/r/innodb-encryption-disable.result index 63ff1dcda71..74c568bdf3e 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-disable.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-disable.result @@ -1,7 +1,7 @@ SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/r/innodb-missing-key.result b/mysql-test/suite/encryption/r/innodb-missing-key.result index b59ec158273..ed9f83770b6 100644 --- a/mysql-test/suite/encryption/r/innodb-missing-key.result +++ b/mysql-test/suite/encryption/r/innodb-missing-key.result @@ -1,5 +1,5 @@ call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); # Start server with keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index f4f3ecf69ea..7886529241f 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -16,7 +16,7 @@ call mtr.add_suppression("Plugin 'file_key_management' init function returned er call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test index 7c61c34ec59..7a401853296 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test @@ -9,7 +9,7 @@ # MDEV-8750: Server crashes in page_cur_is_after_last on altering table using a wrong encryption key # call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test index 96f9563fb3b..77333b0666b 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test @@ -9,7 +9,7 @@ # MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys # call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change5.test b/mysql-test/suite/encryption/t/innodb-bad-key-change5.test index c6c0c963818..39b6ad2fee0 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change5.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change5.test @@ -9,7 +9,7 @@ # MDEV-8769: Server crash at file btr0btr.ic line 122 when defragmenting encrypted table using incorrect keys # call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test b/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test index d27402edaa8..a301951ea7f 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test @@ -3,7 +3,7 @@ # call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/t/innodb-encryption-disable.test b/mysql-test/suite/encryption/t/innodb-encryption-disable.test index 42d8008d1aa..b5996a51eb6 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-disable.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-disable.test @@ -18,7 +18,7 @@ SET GLOBAL innodb_file_per_table = ON; # MDEV-9559: Server without encryption configs crashes if selecting from an implicitly encrypted table # call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); call mtr.add_suppression(".*InnoDB: Cannot open table test/.* from the internal data dictionary of InnoDB though the .frm file for the table exists. See .* for how you can resolve the problem."); call mtr.add_suppression("InnoDB: .ibd file is missing for table test/.*"); diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.test b/mysql-test/suite/encryption/t/innodb-missing-key.test index 8fcfb766117..205d3fa2061 100644 --- a/mysql-test/suite/encryption/t/innodb-missing-key.test +++ b/mysql-test/suite/encryption/t/innodb-missing-key.test @@ -10,7 +10,7 @@ # MDEV-11004: Unable to start (Segfault or os error 2) when encryption key missing # call mtr.add_suppression("InnoDB: Block in space_id .* in file test/.* encrypted"); -call mtr.add_suppression("InnoDB: However key management plugin or used key_id .* is not found or used encryption algorithm or method does not match."); +call mtr.add_suppression("InnoDB: However key management plugin or used key_version .* is not found or used encryption algorithm or method does not match."); call mtr.add_suppression("InnoDB: Marking tablespace as missing. You may drop this table or install correct key management plugin and key file."); --echo diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result index 58721a8732a..05ecfe9f8ef 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result @@ -1,5 +1,7 @@ call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x"); +call mtr.add_suppression("InnoDB: Corruption: Block in space_id .* in file .* corrupted"); +call mtr.add_suppression("InnoDB: Based on page type .*"); FLUSH TABLES; SET GLOBAL innodb_file_per_table = 1; SELECT @@innodb_file_per_table; diff --git a/mysql-test/suite/innodb/r/innodb_bug14147491.result b/mysql-test/suite/innodb/r/innodb_bug14147491.result index 5ff6f652f9e..1a8249f9e54 100644 --- a/mysql-test/suite/innodb/r/innodb_bug14147491.result +++ b/mysql-test/suite/innodb/r/innodb_bug14147491.result @@ -1,4 +1,6 @@ call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); +call mtr.add_suppression("InnoDB: Corruption: Block in space_id .* in file .* corrupted."); +call mtr.add_suppression("InnoDB: Based on page type .*"); CALL mtr.add_suppression("InnoDB: Error: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts"); CALL mtr.add_suppression("InnoDB: Database page corruption on disk or a failed"); CALL mtr.add_suppression("InnoDB: Space .* file test/t1 read of page .*"); @@ -10,6 +12,7 @@ CALL mtr.add_suppression("InnoDB: fix the corruption by dumping, dropping, and r CALL mtr.add_suppression("InnoDB: the corrupt table. You can use CHECK"); CALL mtr.add_suppression("InnoDB: TABLE to scan your table for corruption."); CALL mtr.add_suppression("InnoDB: See also .* about forcing recovery."); +flush tables; # Create and populate the table to be corrupted CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; INSERT INTO t1 (b) VALUES ('corrupt me'); diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test index 0bfe67c06bc..f5e220ad4aa 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test @@ -19,6 +19,8 @@ call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x"); +call mtr.add_suppression("InnoDB: Corruption: Block in space_id .* in file .* corrupted"); +call mtr.add_suppression("InnoDB: Based on page type .*"); FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491.test b/mysql-test/suite/innodb/t/innodb_bug14147491.test index d48fa074147..853c7e63f35 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491.test +++ b/mysql-test/suite/innodb/t/innodb_bug14147491.test @@ -5,6 +5,8 @@ -- source include/not_encrypted.inc call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); +call mtr.add_suppression("InnoDB: Corruption: Block in space_id .* in file .* corrupted."); +call mtr.add_suppression("InnoDB: Based on page type .*"); # Don't test under valgrind, memory leaks will occur source include/not_valgrind.inc; @@ -31,6 +33,7 @@ CALL mtr.add_suppression("InnoDB: fix the corruption by dumping, dropping, and r CALL mtr.add_suppression("InnoDB: the corrupt table. You can use CHECK"); CALL mtr.add_suppression("InnoDB: TABLE to scan your table for corruption."); CALL mtr.add_suppression("InnoDB: See also .* about forcing recovery."); +flush tables; --echo # Create and populate the table to be corrupted diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ef120b8409f..0afbc21af38 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -68,11 +68,6 @@ Created 11/5/1995 Heikki Tuuri #include "lzo/lzo1x.h" #endif -/* Enable this for checksum error messages. */ -//#ifdef UNIV_DEBUG -//#define UNIV_DEBUG_LEVEL2 1 -//#endif - /* IMPLEMENTATION OF THE BUFFER POOL ================================= @@ -325,18 +320,6 @@ on the io_type */ /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); -/********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter -corrupted page. Note that we can't be 100% sure if page is corrupted -or decrypt/decompress just failed. -*/ -static -ibool -buf_page_check_corrupt( -/*===================*/ - buf_page_t* bpage); /*!< in/out: buffer page read from - disk */ - /********************************************************************//** Gets the smallest oldest_modification lsn for any page in the pool. Returns zero if all modified pages have been flushed to disk. @@ -503,6 +486,7 @@ buf_block_alloc( /********************************************************************//** Checks if a page is all zeroes. @return TRUE if the page is all zeroes */ +UNIV_INTERN bool buf_page_is_zeroes( /*===============*/ @@ -525,7 +509,7 @@ buf_page_is_zeroes( @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field @return true if the page is in crc32 checksum format */ -UNIV_INLINE +UNIV_INTERN bool buf_page_is_checksum_valid_crc32( const byte* read_buf, @@ -534,13 +518,12 @@ buf_page_is_checksum_valid_crc32( { ib_uint32_t crc32 = buf_calc_page_crc32(read_buf); -#ifdef UNIV_DEBUG_LEVEL2 if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { - ib_logf(IB_LOG_LEVEL_INFO, - "Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", - checksum_field1, checksum_field2, (ulint)crc32); + DBUG_PRINT("buf_checksum", + ("Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", + checksum_field1, checksum_field2, (ulint)crc32)); } -#endif + return(checksum_field1 == crc32 && checksum_field2 == crc32); } @@ -550,7 +533,7 @@ buf_page_is_checksum_valid_crc32( @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field @return true if the page is in innodb checksum format */ -UNIV_INLINE +UNIV_INTERN bool buf_page_is_checksum_valid_innodb( const byte* read_buf, @@ -569,13 +552,11 @@ buf_page_is_checksum_valid_innodb( if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { -#ifdef UNIV_DEBUG_LEVEL2 - ib_logf(IB_LOG_LEVEL_INFO, - "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + DBUG_PRINT("buf_checksum", + ("Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), - mach_read_from_4(read_buf + FIL_PAGE_LSN) - ); -#endif + mach_read_from_4(read_buf + FIL_PAGE_LSN))); + return(false); } @@ -586,13 +567,11 @@ buf_page_is_checksum_valid_innodb( if (checksum_field1 != 0 && checksum_field1 != buf_calc_page_new_checksum(read_buf)) { -#ifdef UNIV_DEBUG_LEVEL2 - ib_logf(IB_LOG_LEVEL_INFO, - "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + DBUG_PRINT("buf_checksum", + ("Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf), - mach_read_from_4(read_buf + FIL_PAGE_LSN) - ); -#endif + mach_read_from_4(read_buf + FIL_PAGE_LSN))); + return(false); } @@ -604,22 +583,20 @@ buf_page_is_checksum_valid_innodb( @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field @return true if the page is in none checksum format */ -UNIV_INLINE +UNIV_INTERN bool buf_page_is_checksum_valid_none( const byte* read_buf, ulint checksum_field1, ulint checksum_field2) { -#ifdef UNIV_DEBUG_LEVEL2 - if (!(checksum_field1 == checksum_field2 || checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { - ib_logf(IB_LOG_LEVEL_INFO, - "Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + + if (!(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { + DBUG_PRINT("buf_checksum", + ("Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", checksum_field1, checksum_field2, BUF_NO_CHECKSUM_MAGIC, - mach_read_from_4(read_buf + FIL_PAGE_LSN) - ); + mach_read_from_4(read_buf + FIL_PAGE_LSN))); } -#endif return(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); @@ -627,16 +604,18 @@ buf_page_is_checksum_valid_none( /********************************************************************//** Checks if a page is corrupt. -@return TRUE if corrupted */ +@param[in] check_lsn true if LSN should be checked +@param[in] read_buf Page to be checked +@param[in] zip_size compressed size or 0 +@param[in] space Pointer to tablespace +@return true if corrupted, false if not */ UNIV_INTERN -ibool +bool buf_page_is_corrupted( -/*==================*/ - bool check_lsn, /*!< in: true if we need to check - and complain about the LSN */ - const byte* read_buf, /*!< in: a database page */ - ulint zip_size) /*!< in: size of compressed page; - 0 for uncompressed pages */ + bool check_lsn, + const byte* read_buf, + ulint zip_size, + const fil_space_t* space) { ulint checksum_field1; ulint checksum_field2; @@ -644,26 +623,21 @@ buf_page_is_corrupted( read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint page_type = mach_read_from_4( read_buf + FIL_PAGE_TYPE); - bool no_checksum = (page_type == FIL_PAGE_PAGE_COMPRESSED || - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); - - /* Page is encrypted if encryption information is found from - tablespace and page contains used key_version. This is true - also for pages first compressed and then encrypted. */ - if (crypt_data && - crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && - fil_page_is_encrypted(read_buf)) { - no_checksum = true; + /* We can trust page type if page compression is set on tablespace + flags because page compression flag means file must have been + created with 10.1 (later than 5.5 code base). In 10.1 page + compressed tables do not contain post compression checksum and + FIL_PAGE_END_LSN_OLD_CHKSUM field stored. Note that space can + be null if we are in fil_check_first_page() and first page + is not compressed or encrypted. */ + if ((page_type == FIL_PAGE_PAGE_COMPRESSED || + page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) + && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) { + return (false); } - /* Return early if there is no checksum or END_LSN */ - if (no_checksum) { - return (FALSE); - } - - if (!no_checksum && !zip_size + if (!zip_size && memcmp(read_buf + FIL_PAGE_LSN + 4, read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) { @@ -676,7 +650,7 @@ buf_page_is_corrupted( mach_read_from_4(read_buf + FIL_PAGE_LSN + 4), mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4)); - return(TRUE); + return(true); } #ifndef UNIV_HOTBACKUP @@ -715,7 +689,7 @@ buf_page_is_corrupted( /* Check whether the checksum fields have correct values */ if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { - return(FALSE); + return(false); } if (zip_size) { @@ -742,14 +716,14 @@ buf_page_is_corrupted( ib_logf(IB_LOG_LEVEL_INFO, "Checksum fields zero but page is not empty."); - return(TRUE); + return(true); } } - return(FALSE); + return(false); } - DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); ); + DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(true); ); ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET); @@ -762,7 +736,7 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2)) { - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_none(read_buf, @@ -775,7 +749,7 @@ buf_page_is_corrupted( space_id, page_no); } - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_innodb(read_buf, @@ -788,17 +762,17 @@ buf_page_is_corrupted( space_id, page_no); } - return(FALSE); + return(false); } - return(TRUE); + return(true); case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: if (buf_page_is_checksum_valid_innodb(read_buf, checksum_field1, checksum_field2)) { - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_none(read_buf, @@ -811,7 +785,7 @@ buf_page_is_corrupted( space_id, page_no); } - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_crc32(read_buf, @@ -824,16 +798,16 @@ buf_page_is_corrupted( space_id, page_no); } - return(FALSE); + return(false); } - return(TRUE); + return(true); case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_crc32(read_buf, @@ -842,7 +816,7 @@ buf_page_is_corrupted( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, space_id, page_no); - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_innodb(read_buf, @@ -851,10 +825,10 @@ buf_page_is_corrupted( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, space_id, page_no); - return(FALSE); + return(false); } - return(TRUE); + return(true); case SRV_CHECKSUM_ALGORITHM_NONE: /* should have returned FALSE earlier */ @@ -864,7 +838,7 @@ buf_page_is_corrupted( } ut_error; - return(FALSE); + return(false); } /********************************************************************//** @@ -1133,12 +1107,9 @@ buf_block_init( block->page.state = BUF_BLOCK_NOT_USED; block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; + block->page.corrupted = false; block->page.key_version = 0; - block->page.page_encrypted = false; - block->page.page_compressed = false; block->page.encrypted = false; - block->page.stored_checksum = BUF_NO_CHECKSUM_MAGIC; - block->page.calculated_checksum = BUF_NO_CHECKSUM_MAGIC; block->page.real_size = 0; block->page.write_size = 0; block->modify_clock = 0; @@ -2981,14 +2952,14 @@ loop: } else if (retries < BUF_PAGE_READ_MAX_RETRIES) { ++retries; - bool corrupted = true; + bool corrupted = false; if (bpage) { corrupted = buf_page_check_corrupt(bpage); } /* Do not try again for encrypted pages */ - if (!corrupted) { + if (corrupted && bpage->encrypted) { ib_mutex_t* pmutex = buf_page_get_mutex(bpage); buf_pool = buf_pool_from_bpage(bpage); @@ -3016,14 +2987,14 @@ loop: retries = BUF_PAGE_READ_MAX_RETRIES; ); } else { - bool corrupted = true; + bool corrupted = false; if (bpage) { corrupted = buf_page_check_corrupt(bpage); } - if (corrupted) { - fprintf(stderr, "InnoDB: Error: Unable" + if (corrupted && !bpage->encrypted) { + ib_logf(IB_LOG_LEVEL_ERROR, "Unable" " to read tablespace %lu page no" " %lu into the buffer pool after" " %lu attempts\n" @@ -3797,11 +3768,8 @@ buf_page_init_low( bpage->oldest_modification = 0; bpage->write_size = 0; bpage->key_version = 0; - bpage->stored_checksum = BUF_NO_CHECKSUM_MAGIC; - bpage->calculated_checksum = BUF_NO_CHECKSUM_MAGIC; - bpage->page_encrypted = false; - bpage->page_compressed = false; bpage->encrypted = false; + bpage->corrupted = false; bpage->real_size = 0; bpage->slot = NULL; @@ -4496,72 +4464,69 @@ buf_mark_space_corrupt( Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. -*/ -static -ibool +@param[in,out] bpage Page +@return true if page corrupted, false if not */ +UNIV_INTERN +bool buf_page_check_corrupt( -/*===================*/ - buf_page_t* bpage) /*!< in/out: buffer page read from disk */ + buf_page_t* bpage) { ulint zip_size = buf_page_get_zip_size(bpage); byte* dst_frame = (zip_size) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - bool page_compressed = bpage->page_encrypted; - ulint stored_checksum = bpage->stored_checksum; - ulint calculated_checksum = bpage->calculated_checksum; - bool page_compressed_encrypted = bpage->page_compressed; ulint space_id = mach_read_from_4( dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_t* space = fil_space_found_by_id(space_id); - bool corrupted = true; - ulint key_version = bpage->key_version; + fil_space_crypt_t* crypt_data = space->crypt_data; + bool still_encrypted = false; + bool corrupted = false; + ulint page_type = mach_read_from_2(dst_frame + FIL_PAGE_TYPE); - if (key_version != 0 || page_compressed_encrypted) { - bpage->encrypted = true; + /* In buf_decrypt_after_read we have either decrypted the page if + page post encryption checksum matches and used key_id is found + from the encryption plugin. If checksum did not match page was + not decrypted and it could be either encrypted and corrupted + or corrupted or good page. If we decrypted, there page could + still be corrupted if used key does not match. */ + still_encrypted = (crypt_data && + crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && + !bpage->encrypted && + fil_space_verify_crypt_checksum(dst_frame, zip_size, + space, bpage->offset)); + if (!still_encrypted) { + /* If traditional checksums match, we assume that page is + not anymore encrypted. */ + corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space); + + if (!corrupted) { + bpage->encrypted = false; + } } - if (key_version != 0 || - (crypt_data && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) || - page_compressed || page_compressed_encrypted) { + /* Pages that we think are unencrypted but do not match the checksum + checks could be corrupted or encrypted or both. */ + if (corrupted && !bpage->encrypted) { + bpage->corrupted = true; + ib_logf(IB_LOG_LEVEL_ERROR, + "%s: Block in space_id %lu in file %s corrupted.", + page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ? "Maybe corruption" : "Corruption", + space_id, space ? space->name : "NULL"); + ib_logf(IB_LOG_LEVEL_ERROR, + "Based on page type %s (" ULINTPF ")", + fil_get_page_type_name(page_type), page_type); + } else if (still_encrypted || (bpage->encrypted && corrupted)) { + bpage->encrypted = true; + bpage->corrupted = false; + corrupted = true; - /* Page is really corrupted if post encryption stored - checksum does not match calculated checksum after page was - read. For pages compressed and then encrypted, there is no - checksum. */ - corrupted = (!page_compressed_encrypted && stored_checksum != calculated_checksum); - - if (corrupted) { - ib_logf(IB_LOG_LEVEL_ERROR, - "%s: Block in space_id %lu in file %s corrupted.", - page_compressed_encrypted ? "Maybe corruption" : "Corruption", - space_id, space ? space->name : "NULL"); - ib_logf(IB_LOG_LEVEL_ERROR, - "Page based on contents %s encrypted.", - (key_version == 0 && page_compressed_encrypted == false) ? "not" : "maybe"); - if (stored_checksum != BUF_NO_CHECKSUM_MAGIC || calculated_checksum != BUF_NO_CHECKSUM_MAGIC) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Page stored checksum %lu but calculated checksum %lu.", - stored_checksum, calculated_checksum); - } - ib_logf(IB_LOG_LEVEL_ERROR, - "Reason could be that key_version %lu in page " - "or in crypt_data %p could not be found.", - key_version, crypt_data); - ib_logf(IB_LOG_LEVEL_ERROR, - "Reason could be also that key management plugin is not found or" - " used encryption algorithm or method does not match."); - ib_logf(IB_LOG_LEVEL_ERROR, - "Based on page page compressed %d, compressed and encrypted %d.", - page_compressed, page_compressed_encrypted); - } else { - ib_logf(IB_LOG_LEVEL_ERROR, - "Block in space_id %lu in file %s encrypted.", - space_id, space ? space->name : "NULL"); - ib_logf(IB_LOG_LEVEL_ERROR, - "However key management plugin or used key_id %lu is not found or" + ib_logf(IB_LOG_LEVEL_ERROR, + "Block in space_id %lu in file %s encrypted.", + space_id, space ? space->name : "NULL"); + ib_logf(IB_LOG_LEVEL_ERROR, + "However key management plugin or used key_version %u is not found or" " used encryption algorithm or method does not match.", - key_version); + bpage->key_version); + if (space_id > TRX_SYS_SPACE) { ib_logf(IB_LOG_LEVEL_ERROR, "Marking tablespace as missing. You may drop this table or" " install correct key management plugin and key file."); @@ -4589,6 +4554,8 @@ buf_page_io_complete( const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); fil_space_t* space = NULL; + byte* frame = NULL; + bool corrupted = false; ut_a(buf_page_in_file(bpage)); @@ -4604,21 +4571,13 @@ buf_page_io_complete( if (io_type == BUF_IO_READ) { ulint read_page_no; ulint read_space_id; - byte* frame = NULL; - if (!buf_page_decrypt_after_read(bpage)) { - /* encryption error! */ - if (buf_page_get_zip_size(bpage)) { - frame = bpage->zip.data; - } else { - frame = ((buf_block_t*) bpage)->frame; - } + buf_page_decrypt_after_read(bpage); - ib_logf(IB_LOG_LEVEL_INFO, - "Page %u in tablespace %u encryption error key_version %u.", - bpage->offset, bpage->space, bpage->key_version); - - goto database_corrupted; + if (buf_page_get_zip_size(bpage)) { + frame = bpage->zip.data; + } else { + frame = ((buf_block_t*) bpage)->frame; } if (buf_page_get_zip_size(bpage)) { @@ -4680,27 +4639,25 @@ buf_page_io_complete( bpage->offset); } - /* From version 3.23.38 up we store the page checksum - to the 4 first bytes of the page end lsn field */ + corrupted = buf_page_check_corrupt(bpage); - if (buf_page_is_corrupted(true, frame, - buf_page_get_zip_size(bpage))) { +database_corrupted: + + if (corrupted) { /* Not a real corruption if it was triggered by error injection */ DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", if (bpage->space > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { + && buf_mark_space_corrupt(bpage)) { ib_logf(IB_LOG_LEVEL_INFO, "Simulated page corruption"); return(true); } goto page_not_corrupt; - ;); -database_corrupted: - bool corrupted = buf_page_check_corrupt(bpage); + ); - if (corrupted) { + if (!bpage->encrypted) { fil_system_enter(); space = fil_space_get_by_id(bpage->space); fil_system_exit(); @@ -4708,15 +4665,14 @@ database_corrupted: "Database page corruption on disk" " or a failed"); ib_logf(IB_LOG_LEVEL_ERROR, - "Space %lu file %s read of page %u.", - (ulint)bpage->space, + "Space %u file %s read of page %u.", + bpage->space, space ? space->name : "NULL", bpage->offset); ib_logf(IB_LOG_LEVEL_ERROR, "You may have to recover" " from a backup."); - buf_page_print(frame, buf_page_get_zip_size(bpage), BUF_PAGE_PRINT_NO_CRASH); @@ -4747,10 +4703,7 @@ database_corrupted: && buf_mark_space_corrupt(bpage)) { return(false); } else { - corrupted = buf_page_check_corrupt(bpage); - ulint key_version = bpage->key_version; - - if (corrupted) { + if (!bpage->encrypted) { ib_logf(IB_LOG_LEVEL_ERROR, "Ending processing because of a corrupt database page."); @@ -4762,15 +4715,14 @@ database_corrupted: "However key management plugin or used key_id %lu is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table.", - (ulint)bpage->space, key_version); + bpage->space, bpage->key_version); - if (bpage->space > TRX_SYS_SPACE) { - if (corrupted) { - buf_mark_space_corrupt(bpage); - } + if (bpage->encrypted && bpage->space > TRX_SYS_SPACE) { + buf_mark_space_corrupt(bpage); } else { ut_error; } + return(false); } } @@ -4785,20 +4737,31 @@ database_corrupted: recv_recover_page(TRUE, (buf_block_t*) bpage); } - if (uncompressed && !recv_no_ibuf_operations) { + if (uncompressed && !recv_no_ibuf_operations + && fil_page_get_type(frame) == FIL_PAGE_INDEX + && page_is_leaf(frame)) { + + buf_block_t* block; + ibool update_ibuf_bitmap; + + block = (buf_block_t *) bpage; + + update_ibuf_bitmap = TRUE; + if (bpage && bpage->encrypted) { - fprintf(stderr, - "InnoDB: Warning: Table in tablespace %lu encrypted." - "However key management plugin or used key_id %u is not found or" + ib_logf(IB_LOG_LEVEL_WARN, + "Table in tablespace %lu encrypted." + "However key management plugin or used key_version %u is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table.\n", (ulint)bpage->space, bpage->key_version); } else { ibuf_merge_or_delete_for_page( - (buf_block_t*) bpage, bpage->space, + block, bpage->space, bpage->offset, buf_page_get_zip_size(bpage), - TRUE); + update_ibuf_bitmap); } + } } else { /* io_type == BUF_IO_WRITE */ @@ -6119,13 +6082,17 @@ buf_pool_reserve_tmp_slot( /********************************************************************//** Encrypts a buffer page right before it's flushed to disk +@param[in,out] bpage Page control block +@param[in,out] src_frame Source page +@param[in] space_id Tablespace id +@return either unencrypted source page or decrypted page. */ +UNIV_INTERN byte* buf_page_encrypt_before_write( -/*==========================*/ - buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ - byte* src_frame, /*!< in: src frame */ - ulint space_id) /*!< in: space id */ + buf_page_t* bpage, + byte* src_frame, + ulint space_id) { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); ulint zip_size = buf_page_get_zip_size(bpage); @@ -6244,11 +6211,13 @@ buf_page_encrypt_before_write( /********************************************************************//** Decrypt page after it has been read from disk +@param[in,out] bpage Page control block +@return true if successfull, false if something went wrong */ -ibool +UNIV_INTERN +bool buf_page_decrypt_after_read( -/*========================*/ - buf_page_t* bpage) /*!< in/out: buffer page read from disk */ + buf_page_t* bpage) { ulint zip_size = buf_page_get_zip_size(bpage); ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; @@ -6265,49 +6234,21 @@ buf_page_decrypt_after_read( dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); + /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true also for pages first compressed and then encrypted. */ - if (!crypt_data || - (crypt_data && - crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && - key_version != 0)) { - byte* frame = NULL; - - if (buf_page_get_zip_size(bpage)) { - frame = bpage->zip.data; - } else { - frame = ((buf_block_t*) bpage)->frame; - } - - /* If page is not corrupted at this point, page can't be - encrypted, thus set key_version to 0. If page is corrupted, - we assume at this point that it is encrypted as page - contained key_version != 0. Note that page could still be - really corrupted. This we will find out after decrypt by - checking page checksums. */ - if (!buf_page_is_corrupted(false, frame, buf_page_get_zip_size(bpage))) { - key_version = 0; - } + if (!crypt_data && key_version != 0) { + key_version = 0; } - /* If page is encrypted read post-encryption checksum */ - if (!page_compressed_encrypted && key_version != 0) { - bpage->stored_checksum = mach_read_from_4(dst_frame + + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - } - - ut_ad(bpage->key_version == 0); + bpage->key_version = key_version; if (bpage->offset == 0) { /* File header pages are not encrypted/compressed */ - return (TRUE); + return (true); } - /* Store these for corruption check */ - bpage->key_version = key_version; - bpage->page_encrypted = page_compressed_encrypted; - bpage->page_compressed = page_compressed; - if (page_compressed) { /* the page we read is unencrypted */ /* Find free slot from temporary memory array */ @@ -6334,6 +6275,13 @@ buf_page_decrypt_after_read( buf_tmp_buffer_t* slot = NULL; if (key_version) { + /* Verify encryption checksum before we even try to + decrypt. */ + if (!fil_space_verify_crypt_checksum(dst_frame, + zip_size, NULL, bpage->offset)) { + return (false); + } + /* Find free slot from temporary memory array */ slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); @@ -6341,20 +6289,14 @@ buf_page_decrypt_after_read( fil_page_type_validate(dst_frame); #endif - /* Calculate checksum before decrypt, this will be - used later to find out if incorrect key was used. */ - if (!page_compressed_encrypted) { - bpage->calculated_checksum = fil_crypt_calculate_checksum(zip_size, dst_frame); - } - /* decrypt using crypt_buf to dst_frame */ byte* res = fil_space_decrypt(bpage->space, slot->crypt_buf, size, - dst_frame); + dst_frame, + &bpage->encrypted); if (!res) { - bpage->encrypted = true; success = false; } #ifdef UNIV_DEBUG @@ -6387,7 +6329,5 @@ buf_page_decrypt_after_read( } } - bpage->key_version = key_version; - return (success); } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index e49185f6a1f..02980c80112 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -383,10 +383,11 @@ buf_dblwr_init_or_load_pages( doublewrite = read_buf + TRX_SYS_DOUBLEWRITE; if (mach_read_from_4(read_buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) { + bool decrypted = false; byte* tmp = fil_space_decrypt((ulint)TRX_SYS_SPACE, read_buf + UNIV_PAGE_SIZE, UNIV_PAGE_SIZE, /* page size */ - read_buf); + read_buf, &decrypted); doublewrite = tmp + TRX_SYS_DOUBLEWRITE; } @@ -487,6 +488,7 @@ buf_dblwr_process() byte* read_buf; byte* unaligned_read_buf; recv_dblwr_t& recv_dblwr = recv_sys->dblwr; + fil_space_t* space=NULL; unaligned_read_buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); @@ -514,6 +516,10 @@ buf_dblwr_process() continue; } + if (!space) { + space = fil_space_found_by_id(space_id); + } + ulint zip_size = fil_space_get_zip_size(space_id); ut_ad(!buf_page_is_zeroes(page, zip_size)); @@ -548,9 +554,9 @@ buf_dblwr_process() } if (fil_space_verify_crypt_checksum( - read_buf, zip_size) + read_buf, zip_size, NULL, page_no) || !buf_page_is_corrupted( - true, read_buf, zip_size)) { + true, read_buf, zip_size, space)) { /* The page is good; there is no need to consult the doublewrite buffer. */ continue; @@ -573,8 +579,8 @@ buf_dblwr_process() NULL, page, UNIV_PAGE_SIZE, NULL, true); } - if (!fil_space_verify_crypt_checksum(page, zip_size) - && buf_page_is_corrupted(true, page, zip_size)) { + if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no) + && buf_page_is_corrupted(true, page, zip_size, space)) { if (!is_all_zero) { ib_logf(IB_LOG_LEVEL_WARN, "A doublewrite copy of page " diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 2999bea2765..fed1cf94d84 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -625,6 +625,8 @@ fil_encrypt_buf( // store the post-encryption checksum after the key-version mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum); + ut_ad(fil_space_verify_crypt_checksum(dst_frame, zip_size, NULL, offset)); + srv_stats.pages_encrypted.inc(); return dst_frame; @@ -676,6 +678,7 @@ fil_space_encrypt( byte* comp_mem = NULL; byte* uncomp_mem = NULL; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + fil_space_t* tspace = fil_space_found_by_id(space); if (page_compressed_encrypted) { comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); @@ -685,7 +688,7 @@ fil_space_encrypt( src = uncomp_mem; } - bool corrupted1 = buf_page_is_corrupted(true, src, zip_size); + bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, tspace); bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); /* Need to decompress the page if it was also compressed */ @@ -694,7 +697,7 @@ fil_space_encrypt( fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); } - bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size); + bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, tspace); bool different = memcmp(src, tmp_mem, size); if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { @@ -858,19 +861,25 @@ fil_space_decrypt( /****************************************************************** Decrypt a page -@return encrypted page, or original not encrypted page if encryption is -not needed. */ +@param[in] space Tablespace id +@param[in] tmp_frame Temporary buffer used for decrypting +@param[in] page_size Page size +@param[in,out] src_frame Page to decrypt +@param[out] decrypted true if page was decrypted +@return decrypted page, or original not encrypted page if decryption is +not needed.*/ UNIV_INTERN byte* fil_space_decrypt( -/*==============*/ - ulint space, /*!< in: Fil space id */ - byte* tmp_frame, /*!< in: temporary buffer */ - ulint page_size, /*!< in: page size */ - byte* src_frame) /*!< in/out: page buffer */ + ulint space, + byte* tmp_frame, + ulint page_size, + byte* src_frame, + bool* decrypted) { dberr_t err = DB_SUCCESS; byte* res = NULL; + *decrypted = false; bool encrypted = fil_space_decrypt( fil_space_get_crypt_data(space), @@ -881,6 +890,7 @@ fil_space_decrypt( if (err == DB_SUCCESS) { if (encrypted) { + *decrypted = true; /* Copy the decrypted page back to page buffer, not really any other options. */ memcpy(src_frame, tmp_frame, page_size); @@ -934,83 +944,114 @@ fil_crypt_calculate_checksum( } /********************************************************************* -Verify checksum for a page (iff it's encrypted) -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) +Verify that post encryption checksum match calculated checksum. +This function should be called only if tablespace contains crypt_data +metadata (this is strong indication that tablespace is encrypted). +Function also verifies that traditional checksum does not match +calculated checksum as if it does page could be valid unencrypted, +encrypted, or corrupted. +@param[in] page Page to verify +@param[in] zip_size zip size +@param[in] space Tablespace +@param[in] pageno Page no @return true if page is encrypted AND OK, false otherwise */ UNIV_INTERN bool fil_space_verify_crypt_checksum( -/*============================*/ - const byte* src_frame, /*!< in: page the verify */ - ulint zip_size) /*!< in: compressed size if - row_format compressed */ + byte* page, + ulint zip_size, + const fil_space_t* space, + ulint pageno) { - // key version - uint key_version = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + /* If page is not encrypted, return false */ if (key_version == 0) { - return false; // unencrypted page + return false; } - /* "trick" the normal checksum routines by storing the post-encryption - * checksum into the normal checksum field allowing for reuse of - * the normal routines */ - - // post encryption checksum - ib_uint32_t stored_post_encryption = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - // save pre encryption checksum for restore in end of this function - ib_uint32_t stored_pre_encryption = mach_read_from_4( - src_frame + FIL_PAGE_SPACE_OR_CHKSUM); - - ib_uint32_t checksum_field2 = mach_read_from_4( - src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); - - /** prepare frame for usage of normal checksum routines */ - mach_write_to_4(const_cast(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM, - stored_post_encryption); - - /* NOTE: this function is (currently) only run when restoring - * dblwr-buffer, server is single threaded so it's safe to modify - * srv_checksum_algorithm */ - srv_checksum_algorithm_t save_checksum_algorithm = - (srv_checksum_algorithm_t)srv_checksum_algorithm; - - if (zip_size == 0 && - (save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || - save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) { - /* handle ALGORITHM_INNODB specially, - * "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC - * checksum_field2 is sort of pointless anyway... - */ - srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; - mach_write_to_4(const_cast(src_frame) + - UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, - BUF_NO_CHECKSUM_MAGIC); + /* If no checksum is used, can't continue checking. */ + if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { + return(true); } - /* verify checksums */ - ibool corrupted = buf_page_is_corrupted(false, src_frame, zip_size); + /* Read stored post encryption checksum. */ + ib_uint32_t checksum = mach_read_from_4( + page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - /** restore frame & algorithm */ - srv_checksum_algorithm = save_checksum_algorithm; - - mach_write_to_4(const_cast(src_frame) + - FIL_PAGE_SPACE_OR_CHKSUM, - stored_pre_encryption); - - mach_write_to_4(const_cast(src_frame) + - UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, - checksum_field2); - - if (!corrupted) { - return true; // page was encrypted and checksum matched - } else { - return false; // page was encrypted but checksum didn't match + /* Declare empty pages non-corrupted */ + if (checksum == 0 + && *reinterpret_cast(page + FIL_PAGE_LSN) == 0) { + return (true); } + + /* Compressed and encrypted pages do not have checksum. Assume not + corrupted. */ + if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + return (true); + } + + /* Compressed pages use different checksum method. We first store + the post encryption checksum on checksum location and after function + restore the original. */ + if (zip_size) { + ib_uint32_t old = static_cast(mach_read_from_4( + page + FIL_PAGE_SPACE_OR_CHKSUM)); + + mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + + bool valid = page_zip_verify_checksum(page, zip_size); + + mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old); + + return (valid); + } + + /* If stored checksum matches one of the calculated checksums + page is not corrupted. */ + + ib_uint32_t cchecksum1 = buf_calc_page_crc32(page); + ib_uint32_t cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( + page); + bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 + || checksum == BUF_NO_CHECKSUM_MAGIC); + + /* Old InnoDB versions did not initialize + FIL_PAGE_FILE_FLUSH_LSN field so there could be garbage + and above checksum check could produce false positive. + Thus we also check does the traditional stored + checksum fields match the calculated one. Both of these + could naturally produce false positive but then + we just decrypt the page and after that corrupted + pages very probable stay corrupted and valid + pages very probable stay valid. + */ + ulint checksum1 = mach_read_from_4( + page + FIL_PAGE_SPACE_OR_CHKSUM); + + ulint checksum2 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + + + bool valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) + || buf_page_is_checksum_valid_none(page,checksum1,checksum2) + || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); + + if (encrypted && valid) { + /* If page is encrypted and traditional checksums match, + page could be still encrypted, or not encrypted and valid or + corrupted. */ + ib_logf(IB_LOG_LEVEL_ERROR, + " Page %lu in space %s (%lu) maybe corrupted." + " Post encryption checksum %u stored [%lu:%lu] key_version %u", + pageno, + space ? space->name : "N/A", + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, key_version); + encrypted = false; + } + + return(encrypted); } /***********************************************************************/ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index fe6a2922b35..97e70645636 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2257,7 +2257,7 @@ fil_check_first_page(const page_t* page, ulint space_id, ulint flags) } if (buf_page_is_corrupted( - false, page, fsp_flags_get_zip_size(flags))) { + false, page, fsp_flags_get_zip_size(flags), NULL)) { return("checksum mismatch"); } @@ -4515,13 +4515,13 @@ fil_user_tablespace_find_space_id( to UNIV_PAGE_SIZE. */ if (page_size == UNIV_PAGE_SIZE) { uncompressed_ok = !buf_page_is_corrupted( - false, page, 0); + false, page, 0, NULL); } bool compressed_ok = false; if (page_size <= UNIV_PAGE_SIZE_DEF) { compressed_ok = !buf_page_is_corrupted( - false, page, page_size); + false, page, page_size, NULL); } if (uncompressed_ok || compressed_ok) { diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 5c6ef3bfd0d..bfc054701a4 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -366,7 +366,7 @@ fil_compress_page( fil_decompress_page(uncomp_page, comp_page, len, NULL); - if(buf_page_is_corrupted(false, uncomp_page, 0)) { + if(buf_page_is_corrupted(false, uncomp_page, 0, space)) { buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); ut_error; } diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index d7de9d81ca2..25f6fae76e9 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -642,19 +642,69 @@ buf_block_unfix( #else /* !UNIV_HOTBACKUP */ # define buf_block_modify_clock_inc(block) ((void) 0) #endif /* !UNIV_HOTBACKUP */ + +/** Checks if the page is in crc32 checksum format. +@param[in] read_buf database page +@param[in] checksum_field1 new checksum field +@param[in] checksum_field2 old checksum field +@return true if the page is in crc32 checksum format */ +bool +buf_page_is_checksum_valid_crc32( + const byte* read_buf, + ulint checksum_field1, + ulint checksum_field2) + MY_ATTRIBUTE((warn_unused_result)); + +/** Checks if the page is in innodb checksum format. +@param[in] read_buf database page +@param[in] checksum_field1 new checksum field +@param[in] checksum_field2 old checksum field +@return true if the page is in innodb checksum format */ +bool +buf_page_is_checksum_valid_innodb( + const byte* read_buf, + ulint checksum_field1, + ulint checksum_field2) + MY_ATTRIBUTE((warn_unused_result)); + +/** Checks if the page is in none checksum format. +@param[in] read_buf database page +@param[in] checksum_field1 new checksum field +@param[in] checksum_field2 old checksum field +@return true if the page is in none checksum format */ +bool +buf_page_is_checksum_valid_none( + const byte* read_buf, + ulint checksum_field1, + ulint checksum_field2) + MY_ATTRIBUTE((warn_unused_result)); + /********************************************************************//** Checks if a page is corrupt. -@return TRUE if corrupted */ -UNIV_INTERN -ibool +@param[in] check_lsn true if LSN should be checked +@param[in] read_buf Page to be checked +@param[in] zip_size compressed size or 0 +@param[in] space Pointer to tablespace +@return true if corrupted, false if not */ +bool buf_page_is_corrupted( -/*==================*/ - bool check_lsn, /*!< in: true if we need to check the - and complain about the LSN */ - const byte* read_buf, /*!< in: a database page */ - ulint zip_size) /*!< in: size of compressed page; - 0 for uncompressed pages */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + bool check_lsn, + const byte* read_buf, + ulint zip_size, + const fil_space_t* space) + MY_ATTRIBUTE((warn_unused_result)); + +/********************************************************************//** +Check if page is maybe compressed, encrypted or both when we encounter +corrupted page. Note that we can't be 100% sure if page is corrupted +or decrypt/decompress just failed. +@param[in] bpage Page +@return true if page corrupted, false if not */ +bool +buf_page_check_corrupt( + buf_page_t* bpage) /*!< in/out: buffer page read from disk */ + MY_ATTRIBUTE(( warn_unused_result)); + /********************************************************************//** Checks if a page is all zeroes. @return TRUE if the page is all zeroes */ @@ -1490,7 +1540,7 @@ The hook that is called just after a page is read from disk. The function decrypt disk content into buf_page_t and releases the temporary buffer that was allocated in buf_page_decrypt_before_read */ UNIV_INTERN -ibool +bool buf_page_decrypt_after_read( /*========================*/ buf_page_t* page); /*!< in/out: buffer page read from disk */ @@ -1593,14 +1643,9 @@ struct buf_page_t{ operation needed. */ unsigned key_version; /*!< key version for this block */ - bool page_encrypted; /*!< page is page encrypted */ - bool page_compressed;/*!< page is page compressed */ - ulint stored_checksum;/*!< stored page checksum if page - encrypted */ - bool encrypted; /*!< page is still encrypted */ - ulint calculated_checksum; - /*!< calculated checksum if page - encrypted */ + bool encrypted; /*!< page is still encrypted */ + bool corrupted; /*!< page is corrupted */ + ulint real_size; /*!< Real size of the page Normal pages == UNIV_PAGE_SIZE page compressed pages, payload @@ -1714,6 +1759,7 @@ struct buf_page_t{ 0 if the block was never accessed in the buffer pool. Protected by block mutex */ + ibool is_corrupt; # if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG ibool file_page_was_freed; /*!< this is set to TRUE when diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 42cdafde4d0..6aae71ba12d 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -328,7 +328,7 @@ fil_space_check_encryption_read( /****************************************************************** Decrypt a page -@return true if page is decrypted, false if not. */ +@return true if page decrypted, false if not.*/ UNIV_INTERN bool fil_space_decrypt( @@ -336,9 +336,10 @@ fil_space_decrypt( fil_space_crypt_t* crypt_data, /*!< in: crypt data */ byte* tmp_frame, /*!< in: temporary buffer */ ulint page_size, /*!< in: page size */ - byte* src_frame, /*!< in:out: page buffer */ - dberr_t* err); /*!< in: out: DB_SUCCESS or + byte* src_frame, /*!< in: out: page buffer */ + dberr_t* err) /*!< in: out: DB_SUCCESS or error code */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************* Encrypt buffer page @@ -355,31 +356,46 @@ fil_space_encrypt( ulint size, /*!< in: size of data to encrypt */ byte* dst_frame); /*!< in: where to encrypt to */ -/********************************************************************* -Decrypt buffer page -@return decrypted page, or original not encrypted page if decrypt is +/****************************************************************** +Decrypt a page +@param[in] space Tablespace id +@param[in] tmp_frame Temporary buffer used for decrypting +@param[in] page_size Page size +@param[in,out] src_frame Page to decrypt +@param[out] decrypted true if page was decrypted +@return decrypted page, or original not encrypted page if decryption is not needed.*/ UNIV_INTERN byte* fil_space_decrypt( /*==============*/ - ulint space, /*!< in: tablespace id */ - byte* src_frame, /*!< in: page frame */ - ulint page_size, /*!< in: size of data to encrypt */ - byte* dst_frame) /*!< in: where to decrypt to */ + ulint space, + byte* src_frame, + ulint page_size, + byte* dst_frame, + bool* decrypted) __attribute__((warn_unused_result)); /********************************************************************* -fil_space_verify_crypt_checksum -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) +Verify that post encryption checksum match calculated checksum. +This function should be called only if tablespace contains crypt_data +metadata (this is strong indication that tablespace is encrypted). +Function also verifies that traditional checksum does not match +calculated checksum as if it does page could be valid unencrypted, +encrypted, or corrupted. +@param[in] page Page to verify +@param[in] zip_size zip size +@param[in] space Tablespace +@param[in] pageno Page no @return true if page is encrypted AND OK, false otherwise */ UNIV_INTERN bool fil_space_verify_crypt_checksum( -/*============================*/ - const byte* src_frame,/*!< in: page frame */ - ulint zip_size); /*!< in: size of data to encrypt */ + byte* page, + ulint zip_size, + const fil_space_t* space, + ulint pageno) + __attribute__((warn_unused_result)); /********************************************************************* Init threads for key rotation */ diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index ceebf6c1ab3..772255bb75b 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -59,6 +59,8 @@ fil_get_page_type_name( switch(page_type) { case FIL_PAGE_PAGE_COMPRESSED: return (const char*)"PAGE_COMPRESSED"; + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + return (const char*)"PAGE_COMPRESSED_ENCRYPTED"; case FIL_PAGE_INDEX: return (const char*)"INDEX"; case FIL_PAGE_UNDO_LOG: @@ -87,9 +89,13 @@ fil_get_page_type_name( return (const char*)"ZBLOB2"; case FIL_PAGE_TYPE_COMPRESSED: return (const char*)"ORACLE PAGE COMPRESSED"; - default: - return (const char*)"PAGE TYPE CORRUPTED"; + + /* No default to make compiler generate warning if + new page type is added but not handled here. */ } + + return (const char*)"PAGE TYPE CORRUPTED"; + } /****************************************************************//** diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index f54c9d589c2..e9cfa126df0 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -2033,12 +2033,15 @@ PageConverter::validate( buf_block_t* block) UNIV_NOTHROW { buf_frame_t* page = get_frame(block); + ulint space_id = mach_read_from_4( + page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + fil_space_t* space = fil_space_found_by_id(space_id); /* Check that the page number corresponds to the offset in the file. Flag as corrupt if it doesn't. Disable the check for LSN in buf_page_is_corrupted() */ - if (buf_page_is_corrupted(false, page, get_zip_size()) + if (buf_page_is_corrupted(false, page, get_zip_size(), space) || (page_get_page_no(page) != offset / m_page_size && page_get_page_no(page) != 0)) { diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 65c4e734968..d0bfa2de344 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -65,26 +65,9 @@ Created 11/5/1995 Heikki Tuuri #include "fil0pagecompress.h" #include "ha_prototypes.h" -/* Enable this for checksum error messages. */ -//#ifdef UNIV_DEBUG -//#define UNIV_DEBUG_LEVEL2 1 -//#endif - /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); -/********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter -corrupted page. Note that we can't be 100% sure if page is corrupted -or decrypt/decompress just failed. -*/ -static -ibool -buf_page_check_corrupt( -/*===================*/ - buf_page_t* bpage); /*!< in/out: buffer page read from - disk */ - static inline void _increment_page_get_statistics(buf_block_t* block, trx_t* trx) @@ -568,6 +551,7 @@ buf_block_alloc( /********************************************************************//** Checks if a page is all zeroes. @return TRUE if the page is all zeroes */ +UNIV_INTERN bool buf_page_is_zeroes( /*===============*/ @@ -590,7 +574,7 @@ buf_page_is_zeroes( @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field @return true if the page is in crc32 checksum format */ -UNIV_INLINE +UNIV_INTERN bool buf_page_is_checksum_valid_crc32( const byte* read_buf, @@ -599,13 +583,11 @@ buf_page_is_checksum_valid_crc32( { ib_uint32_t crc32 = buf_calc_page_crc32(read_buf); -#ifdef UNIV_DEBUG_LEVEL2 if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) { - ib_logf(IB_LOG_LEVEL_INFO, - "Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", - checksum_field1, checksum_field2, (ulint)crc32); + DBUG_PRINT("buf_checksum", + ("Page checksum crc32 not valid field1 %lu field2 %lu crc32 %lu.", + checksum_field1, checksum_field2, (ulint)crc32)); } -#endif return(checksum_field1 == crc32 && checksum_field2 == crc32); } @@ -615,7 +597,7 @@ buf_page_is_checksum_valid_crc32( @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field @return true if the page is in innodb checksum format */ -UNIV_INLINE +UNIV_INTERN bool buf_page_is_checksum_valid_innodb( const byte* read_buf, @@ -634,13 +616,12 @@ buf_page_is_checksum_valid_innodb( if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { -#ifdef UNIV_DEBUG_LEVEL2 - ib_logf(IB_LOG_LEVEL_INFO, - "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + + DBUG_PRINT("buf_checksum", + ("Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf), - mach_read_from_4(read_buf + FIL_PAGE_LSN) - ); -#endif + mach_read_from_4(read_buf + FIL_PAGE_LSN))); + return(false); } @@ -651,13 +632,12 @@ buf_page_is_checksum_valid_innodb( if (checksum_field1 != 0 && checksum_field1 != buf_calc_page_new_checksum(read_buf)) { -#ifdef UNIV_DEBUG_LEVEL2 - ib_logf(IB_LOG_LEVEL_INFO, - "Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + + DBUG_PRINT("buf_checksum", + ("Page checksum innodb not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf), - mach_read_from_4(read_buf + FIL_PAGE_LSN) - ); -#endif + mach_read_from_4(read_buf + FIL_PAGE_LSN))); + return(false); } @@ -669,22 +649,20 @@ buf_page_is_checksum_valid_innodb( @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field @return true if the page is in none checksum format */ -UNIV_INLINE +UNIV_INTERN bool buf_page_is_checksum_valid_none( const byte* read_buf, ulint checksum_field1, ulint checksum_field2) { -#ifdef UNIV_DEBUG_LEVEL2 - if (!(checksum_field1 == checksum_field2 || checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { - ib_logf(IB_LOG_LEVEL_INFO, - "Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", + + if (!(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { + DBUG_PRINT("buf_checksum", + ("Page checksum none not valid field1 %lu field2 %lu crc32 %lu lsn %lu.", checksum_field1, checksum_field2, BUF_NO_CHECKSUM_MAGIC, - mach_read_from_4(read_buf + FIL_PAGE_LSN) - ); + mach_read_from_4(read_buf + FIL_PAGE_LSN))); } -#endif return(checksum_field1 == checksum_field2 && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); @@ -692,16 +670,18 @@ buf_page_is_checksum_valid_none( /********************************************************************//** Checks if a page is corrupt. -@return TRUE if corrupted */ +@param[in] check_lsn true if LSN should be checked +@param[in] read_buf Page to be checked +@param[in] zip_size compressed size or 0 +@param[in] space Pointer to tablespace +@return true if corrupted, false if not */ UNIV_INTERN -ibool +bool buf_page_is_corrupted( -/*==================*/ - bool check_lsn, /*!< in: true if we need to check - and complain about the LSN */ - const byte* read_buf, /*!< in: a database page */ - ulint zip_size) /*!< in: size of compressed page; - 0 for uncompressed pages */ + bool check_lsn, + const byte* read_buf, + ulint zip_size, + const fil_space_t* space) { ulint checksum_field1; ulint checksum_field2; @@ -709,26 +689,21 @@ buf_page_is_corrupted( read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint page_type = mach_read_from_4( read_buf + FIL_PAGE_TYPE); - bool no_checksum = (page_type == FIL_PAGE_PAGE_COMPRESSED || - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); - - /* Page is encrypted if encryption information is found from - tablespace and page contains used key_version. This is true - also for pages first compressed and then encrypted. */ - if (crypt_data && - crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && - fil_page_is_encrypted(read_buf)) { - no_checksum = true; + /* We can trust page type if page compression is set on tablespace + flags because page compression flag means file must have been + created with 10.1 (later than 5.5 code base). In 10.1 page + compressed tables do not contain post compression checksum and + FIL_PAGE_END_LSN_OLD_CHKSUM field stored. Note that space can + be null if we are in fil_check_first_page() and first page + is not compressed or encrypted. */ + if ((page_type == FIL_PAGE_PAGE_COMPRESSED || + page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) + && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) { + return (false); } - /* Return early if there is no checksum or END_LSN */ - if (no_checksum) { - return (FALSE); - } - - if (!no_checksum && !zip_size + if (!zip_size && memcmp(read_buf + FIL_PAGE_LSN + 4, read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) { @@ -780,7 +755,7 @@ buf_page_is_corrupted( /* Check whether the checksum fields have correct values */ if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { - return(FALSE); + return(false); } if (zip_size) { @@ -807,14 +782,14 @@ buf_page_is_corrupted( ib_logf(IB_LOG_LEVEL_INFO, "Checksum fields zero but page is not empty."); - return(TRUE); + return(true); } } - return(FALSE); + return(false); } - DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); ); + DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(true); ); ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET); @@ -827,7 +802,7 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2)) { - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_none(read_buf, @@ -840,7 +815,7 @@ buf_page_is_corrupted( space_id, page_no); } - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_innodb(read_buf, @@ -853,17 +828,17 @@ buf_page_is_corrupted( space_id, page_no); } - return(FALSE); + return(false); } - return(TRUE); + return(true); case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: if (buf_page_is_checksum_valid_innodb(read_buf, checksum_field1, checksum_field2)) { - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_none(read_buf, @@ -876,7 +851,7 @@ buf_page_is_corrupted( space_id, page_no); } - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_crc32(read_buf, @@ -889,16 +864,16 @@ buf_page_is_corrupted( space_id, page_no); } - return(FALSE); + return(false); } - return(TRUE); + return(true); case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_crc32(read_buf, @@ -907,7 +882,7 @@ buf_page_is_corrupted( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, space_id, page_no); - return(FALSE); + return(false); } if (buf_page_is_checksum_valid_innodb(read_buf, @@ -916,10 +891,10 @@ buf_page_is_corrupted( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, space_id, page_no); - return(FALSE); + return(false); } - return(TRUE); + return(true); case SRV_CHECKSUM_ALGORITHM_NONE: /* should have returned FALSE earlier */ @@ -929,7 +904,7 @@ buf_page_is_corrupted( } ut_error; - return(FALSE); + return(false); } /********************************************************************//** @@ -1198,12 +1173,9 @@ buf_block_init( block->page.state = BUF_BLOCK_NOT_USED; block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; - block->page.key_version = 0; - block->page.page_encrypted = false; - block->page.page_compressed = false; block->page.encrypted = false; - block->page.stored_checksum = BUF_NO_CHECKSUM_MAGIC; - block->page.calculated_checksum = BUF_NO_CHECKSUM_MAGIC; + block->page.corrupted = false; + block->page.key_version = 0; block->page.real_size = 0; block->page.write_size = 0; block->modify_clock = 0; @@ -3023,14 +2995,14 @@ loop: } else if (retries < BUF_PAGE_READ_MAX_RETRIES) { ++retries; - bool corrupted = true; + bool corrupted = false; if (bpage) { corrupted = buf_page_check_corrupt(bpage); } /* Do not try again for encrypted pages */ - if (!corrupted) { + if (corrupted && bpage->encrypted) { ib_mutex_t* pmutex = buf_page_get_mutex(bpage); mutex_enter(&buf_pool->LRU_list_mutex); mutex_enter(pmutex); @@ -3059,14 +3031,14 @@ loop: retries = BUF_PAGE_READ_MAX_RETRIES; ); } else { - bool corrupted = true; + bool corrupted = false; if (bpage) { corrupted = buf_page_check_corrupt(bpage); } - if (corrupted) { - fprintf(stderr, "InnoDB: Error: Unable" + if (corrupted && !bpage->encrypted) { + ib_logf(IB_LOG_LEVEL_ERROR, "Unable" " to read tablespace %lu page no" " %lu into the buffer pool after" " %lu attempts\n" @@ -3877,12 +3849,9 @@ buf_page_init_low( bpage->newest_modification = 0; bpage->oldest_modification = 0; bpage->write_size = 0; - bpage->key_version = 0; - bpage->stored_checksum = BUF_NO_CHECKSUM_MAGIC; - bpage->calculated_checksum = BUF_NO_CHECKSUM_MAGIC; - bpage->page_encrypted = false; - bpage->page_compressed = false; bpage->encrypted = false; + bpage->corrupted = false; + bpage->key_version = 0; bpage->real_size = 0; HASH_INVALIDATE(bpage, hash); @@ -4595,72 +4564,70 @@ buf_mark_space_corrupt( Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. -*/ -static -ibool +@param[in,out] bpage Page +@return true if page corrupted, false if not */ +UNIV_INTERN +bool buf_page_check_corrupt( -/*===================*/ - buf_page_t* bpage) /*!< in/out: buffer page read from disk */ + buf_page_t* bpage) { ulint zip_size = buf_page_get_zip_size(bpage); byte* dst_frame = (zip_size) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - bool page_compressed = bpage->page_encrypted; - ulint stored_checksum = bpage->stored_checksum; - ulint calculated_checksum = bpage->calculated_checksum; - bool page_compressed_encrypted = bpage->page_compressed; ulint space_id = mach_read_from_4( dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_t* space = fil_space_found_by_id(space_id); - bool corrupted = true; - ulint key_version = bpage->key_version; + fil_space_crypt_t* crypt_data = space->crypt_data; + bool still_encrypted = false; + bool corrupted = false; + ulint page_type = mach_read_from_2(dst_frame + FIL_PAGE_TYPE); - if (key_version != 0 || page_compressed_encrypted) { - bpage->encrypted = true; + /* In buf_decrypt_after_read we have either decrypted the page if + page post encryption checksum matches and used key_id is found + from the encryption plugin. If checksum did not match page was + not decrypted and it could be either encrypted and corrupted + or corrupted or good page. If we decrypted, there page could + still be corrupted if used key does not match. */ + still_encrypted = (crypt_data && + crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && + !bpage->encrypted && + fil_space_verify_crypt_checksum(dst_frame, zip_size, + space, bpage->offset)); + + if (!still_encrypted) { + /* If traditional checksums match, we assume that page is + not anymore encrypted. */ + corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space); + + if (!corrupted) { + bpage->encrypted = false; + } } - if (key_version != 0 || - (crypt_data && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) || - page_compressed || page_compressed_encrypted) { + /* Pages that we think are unencrypted but do not match the checksum + checks could be corrupted or encrypted or both. */ + if (corrupted && !bpage->encrypted) { + bpage->corrupted = true; + ib_logf(IB_LOG_LEVEL_ERROR, + "%s: Block in space_id %lu in file %s corrupted.", + page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ? "Maybe corruption" : "Corruption", + space_id, space ? space->name : "NULL"); + ib_logf(IB_LOG_LEVEL_ERROR, + "Based on page type %s (" ULINTPF ")", + fil_get_page_type_name(page_type), page_type); + } else if (still_encrypted || (bpage->encrypted && corrupted)) { + bpage->encrypted = true; + bpage->corrupted = false; + corrupted = true; - /* Page is really corrupted if post encryption stored - checksum does not match calculated checksum after page was - read. For pages compressed and then encrypted, there is no - checksum. */ - corrupted = (!page_compressed_encrypted && stored_checksum != calculated_checksum); - - if (corrupted) { - ib_logf(IB_LOG_LEVEL_ERROR, - "%s: Block in space_id %lu in file %s corrupted.", - page_compressed_encrypted ? "Maybe corruption" : "Corruption", - space_id, space ? space->name : "NULL"); - ib_logf(IB_LOG_LEVEL_ERROR, - "Page based on contents %s encrypted.", - (key_version == 0 && page_compressed_encrypted == false) ? "not" : "maybe"); - if (stored_checksum != BUF_NO_CHECKSUM_MAGIC || calculated_checksum != BUF_NO_CHECKSUM_MAGIC) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Page stored checksum %lu but calculated checksum %lu.", - stored_checksum, calculated_checksum); - } - ib_logf(IB_LOG_LEVEL_ERROR, - "Reason could be that key_version %lu in page " - "or in crypt_data %p could not be found.", - key_version, crypt_data); - ib_logf(IB_LOG_LEVEL_ERROR, - "Reason could be also that key management plugin is not found or" - " used encryption algorithm or method does not match."); - ib_logf(IB_LOG_LEVEL_ERROR, - "Based on page page compressed %d, compressed and encrypted %d.", - page_compressed, page_compressed_encrypted); - } else { - ib_logf(IB_LOG_LEVEL_ERROR, - "Block in space_id %lu in file %s encrypted.", - space_id, space ? space->name : "NULL"); - ib_logf(IB_LOG_LEVEL_ERROR, - "However key management plugin or used key_id %lu is not found or" + ib_logf(IB_LOG_LEVEL_ERROR, + "Block in space_id %lu in file %s encrypted.", + space_id, space ? space->name : "NULL"); + ib_logf(IB_LOG_LEVEL_ERROR, + "However key management plugin or used key_version %u is not found or" " used encryption algorithm or method does not match.", - key_version); + bpage->key_version); + if (space_id > TRX_SYS_SPACE) { ib_logf(IB_LOG_LEVEL_ERROR, "Marking tablespace as missing. You may drop this table or" " install correct key management plugin and key file."); @@ -4686,6 +4653,8 @@ buf_page_io_complete( == BUF_BLOCK_FILE_PAGE); bool have_LRU_mutex = false; fil_space_t* space = NULL; + byte* frame = NULL; + bool corrupted = false; ut_a(buf_page_in_file(bpage)); @@ -4701,21 +4670,13 @@ buf_page_io_complete( if (io_type == BUF_IO_READ) { ulint read_page_no; ulint read_space_id; - byte* frame; - if (!buf_page_decrypt_after_read(bpage)) { - /* encryption error! */ - if (buf_page_get_zip_size(bpage)) { - frame = bpage->zip.data; - } else { - frame = ((buf_block_t*) bpage)->frame; - } + buf_page_decrypt_after_read(bpage); - ib_logf(IB_LOG_LEVEL_INFO, - "Page %u in tablespace %u encryption error key_version %u.", - bpage->offset, bpage->space, bpage->key_version); - - goto database_corrupted; + if (buf_page_get_zip_size(bpage)) { + frame = bpage->zip.data; + } else { + frame = ((buf_block_t*) bpage)->frame; } if (buf_page_get_zip_size(bpage)) { @@ -4732,6 +4693,9 @@ buf_page_io_complete( "Page %u in tablespace %u zip_decompress failure.", bpage->offset, bpage->space); + corrupted = true; + bpage->corrupted = true; + goto database_corrupted; } os_atomic_decrement_ulint(&buf_pool->n_pend_unzip, 1); @@ -4780,121 +4744,116 @@ buf_page_io_complete( if (UNIV_LIKELY(!bpage->is_corrupt || !srv_pass_corrupt_table)) { - /* From version 3.23.38 up we store the page checksum - to the 4 first bytes of the page end lsn field */ + corrupted = buf_page_check_corrupt(bpage); - if (buf_page_is_corrupted(true, frame, - buf_page_get_zip_size(bpage))) { + } - /* Not a real corruption if it was triggered by - error injection */ - DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", - if (bpage->space > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { - ib_logf(IB_LOG_LEVEL_INFO, - "Simulated page corruption"); - return(true); - } - goto page_not_corrupt; - ;); database_corrupted: - bool corrupted = buf_page_check_corrupt(bpage); + if (corrupted) { + /* Not a real corruption if it was triggered by + error injection */ - if (corrupted) { - fil_system_enter(); - space = fil_space_get_by_id(bpage->space); - fil_system_exit(); - ib_logf(IB_LOG_LEVEL_ERROR, - "Database page corruption on disk" - " or a failed"); - ib_logf(IB_LOG_LEVEL_ERROR, - "Space %u file %s read of page %u.", - bpage->space, - space ? space->name : "NULL", - bpage->offset); - ib_logf(IB_LOG_LEVEL_ERROR, - "You may have to recover" - " from a backup."); + DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", + if (bpage->space > TRX_SYS_SPACE + && buf_mark_space_corrupt(bpage)) { + ib_logf(IB_LOG_LEVEL_INFO, + "Simulated page corruption"); + return(true); + } + goto page_not_corrupt; + ); + if (!bpage->encrypted) { + fil_system_enter(); + space = fil_space_get_by_id(bpage->space); + fil_system_exit(); + ib_logf(IB_LOG_LEVEL_ERROR, + "Database page corruption on disk" + " or a failed"); + ib_logf(IB_LOG_LEVEL_ERROR, + "Space %u file %s read of page %u.", + bpage->space, + space ? space->name : "NULL", + bpage->offset); + ib_logf(IB_LOG_LEVEL_ERROR, + "You may have to recover" + " from a backup."); - buf_page_print(frame, buf_page_get_zip_size(bpage), - BUF_PAGE_PRINT_NO_CRASH); + buf_page_print(frame, buf_page_get_zip_size(bpage), + BUF_PAGE_PRINT_NO_CRASH); - ib_logf(IB_LOG_LEVEL_ERROR, - "It is also possible that your operating" - "system has corrupted its own file cache."); - ib_logf(IB_LOG_LEVEL_ERROR, - "and rebooting your computer removes the error."); - ib_logf(IB_LOG_LEVEL_ERROR, - "If the corrupt page is an index page you can also try to"); - ib_logf(IB_LOG_LEVEL_ERROR, - "fix the corruption by dumping, dropping, and reimporting"); - ib_logf(IB_LOG_LEVEL_ERROR, - "the corrupt table. You can use CHECK"); - ib_logf(IB_LOG_LEVEL_ERROR, - "TABLE to scan your table for corruption."); - ib_logf(IB_LOG_LEVEL_ERROR, - "See also " - REFMAN "forcing-innodb-recovery.html" - " about forcing recovery."); + ib_logf(IB_LOG_LEVEL_ERROR, + "It is also possible that your operating" + "system has corrupted its own file cache."); + ib_logf(IB_LOG_LEVEL_ERROR, + "and rebooting your computer removes the error."); + ib_logf(IB_LOG_LEVEL_ERROR, + "If the corrupt page is an index page you can also try to"); + ib_logf(IB_LOG_LEVEL_ERROR, + "fix the corruption by dumping, dropping, and reimporting"); + ib_logf(IB_LOG_LEVEL_ERROR, + "the corrupt table. You can use CHECK"); + ib_logf(IB_LOG_LEVEL_ERROR, + "TABLE to scan your table for corruption."); + ib_logf(IB_LOG_LEVEL_ERROR, + "See also " + REFMAN "forcing-innodb-recovery.html" + " about forcing recovery."); + } + + if (srv_pass_corrupt_table && bpage->space != 0 + && bpage->space < SRV_LOG_SPACE_FIRST_ID) { + trx_t* trx; + + fprintf(stderr, + "InnoDB: space %u will be treated as corrupt.\n", + bpage->space); + fil_space_set_corrupt(bpage->space); + + trx = innobase_get_trx(); + + if (trx && trx->dict_operation_lock_mode == RW_X_LATCH) { + dict_table_set_corrupt_by_space(bpage->space, FALSE); + } else { + dict_table_set_corrupt_by_space(bpage->space, TRUE); } - if (srv_pass_corrupt_table && bpage->space != 0 - && bpage->space < SRV_LOG_SPACE_FIRST_ID) { - trx_t* trx; + bpage->is_corrupt = TRUE; + } - fprintf(stderr, - "InnoDB: space %u will be treated as corrupt.\n", - bpage->space); - fil_space_set_corrupt(bpage->space); + if (srv_force_recovery < SRV_FORCE_IGNORE_CORRUPT) { + /* If page space id is larger than TRX_SYS_SPACE + (0), we will attempt to mark the corresponding + table as corrupted instead of crashing server */ + if (bpage->space > TRX_SYS_SPACE + && buf_mark_space_corrupt(bpage)) { + return(false); + } else { + if (!bpage->encrypted) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Ending processing because of a corrupt database page."); - trx = innobase_get_trx(); - if (trx && trx->dict_operation_lock_mode == RW_X_LATCH) { - dict_table_set_corrupt_by_space(bpage->space, FALSE); - } else { - dict_table_set_corrupt_by_space(bpage->space, TRUE); + ut_error; } - bpage->is_corrupt = TRUE; - } - if (srv_force_recovery < SRV_FORCE_IGNORE_CORRUPT) { - /* If page space id is larger than TRX_SYS_SPACE - (0), we will attempt to mark the corresponding - table as corrupted instead of crashing server */ - if (bpage->space > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { - return(false); + ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, + "Table in tablespace %lu encrypted." + "However key management plugin or used key_id %lu is not found or" + " used encryption algorithm or method does not match." + " Can't continue opening the table.", + bpage->space, bpage->key_version); + + if (bpage->encrypted && bpage->space > TRX_SYS_SPACE) { + buf_mark_space_corrupt(bpage); } else { - corrupted = buf_page_check_corrupt(bpage); - ulint key_version = bpage->key_version; - - if (corrupted) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Ending processing because of a corrupt database page."); - - ut_error; - } - - ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, - "Table in tablespace %lu encrypted." - "However key management plugin or used key_id %lu is not found or" - " used encryption algorithm or method does not match." - " Can't continue opening the table.", - (ulint)bpage->space, key_version); - - if (bpage->space > TRX_SYS_SPACE) { - if (corrupted) { - buf_mark_space_corrupt(bpage); - } - } else { - ut_error; - } - return(false); + ut_error; } + + return(false); } } - } /**/ + } DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", page_not_corrupt: bpage = bpage; ); @@ -4924,9 +4883,9 @@ database_corrupted: } if (bpage && bpage->encrypted) { - fprintf(stderr, - "InnoDB: Warning: Table in tablespace %lu encrypted." - "However key management plugin or used key_id %u is not found or" + ib_logf(IB_LOG_LEVEL_WARN, + "Table in tablespace %lu encrypted." + "However key management plugin or used key_version %u is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table.\n", (ulint)bpage->space, bpage->key_version); @@ -6301,13 +6260,16 @@ buf_pool_reserve_tmp_slot( /********************************************************************//** Encrypts a buffer page right before it's flushed to disk +@param[in,out] bpage Page control block +@param[in,out] src_frame Source page +@param[in] space_id Tablespace id +@return either unencrypted source page or decrypted page. */ byte* buf_page_encrypt_before_write( -/*==========================*/ - buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ - byte* src_frame, /*!< in: src frame */ - ulint space_id) /*!< in: space id */ + buf_page_t* bpage, + byte* src_frame, + ulint space_id) { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); ulint zip_size = buf_page_get_zip_size(bpage); @@ -6426,11 +6388,13 @@ buf_page_encrypt_before_write( /********************************************************************//** Decrypt page after it has been read from disk +@param[in,out] bpage Page control block +@return true if successfull, false if something went wrong */ -ibool +UNIV_INTERN +bool buf_page_decrypt_after_read( -/*========================*/ - buf_page_t* bpage) /*!< in/out: buffer page read from disk */ + buf_page_t* bpage) { ulint zip_size = buf_page_get_zip_size(bpage); ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; @@ -6450,46 +6414,17 @@ buf_page_decrypt_after_read( /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true also for pages first compressed and then encrypted. */ - if (!crypt_data || - (crypt_data && - crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && - key_version != 0)) { - byte* frame = NULL; - - if (buf_page_get_zip_size(bpage)) { - frame = bpage->zip.data; - } else { - frame = ((buf_block_t*) bpage)->frame; - } - - /* If page is not corrupted at this point, page can't be - encrypted, thus set key_version to 0. If page is corrupted, - we assume at this point that it is encrypted as page - contained key_version != 0. Note that page could still be - really corrupted. This we will find out after decrypt by - checking page checksums. */ - if (!buf_page_is_corrupted(false, frame, buf_page_get_zip_size(bpage))) { - key_version = 0; - } + if (!crypt_data && key_version != 0) { + key_version = 0; } - /* If page is encrypted read post-encryption checksum */ - if (!page_compressed_encrypted && key_version != 0) { - bpage->stored_checksum = mach_read_from_4(dst_frame + + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - } - - ut_ad(bpage->key_version == 0); + bpage->key_version = key_version; if (bpage->offset == 0) { /* File header pages are not encrypted/compressed */ - return (TRUE); + return (true); } - /* Store these for corruption check */ - bpage->key_version = key_version; - bpage->page_encrypted = page_compressed_encrypted; - bpage->page_compressed = page_compressed; - if (page_compressed) { /* the page we read is unencrypted */ /* Find free slot from temporary memory array */ @@ -6516,6 +6451,13 @@ buf_page_decrypt_after_read( buf_tmp_buffer_t* slot = NULL; if (key_version) { + /* Verify encryption checksum before we even try to + decrypt. */ + if (!fil_space_verify_crypt_checksum(dst_frame, + zip_size, NULL, bpage->offset)) { + return (false); + } + /* Find free slot from temporary memory array */ slot = buf_pool_reserve_tmp_slot(buf_pool, page_compressed); @@ -6523,20 +6465,14 @@ buf_page_decrypt_after_read( fil_page_type_validate(dst_frame); #endif - /* Calculate checksum before decrypt, this will be - used later to find out if incorrect key was used. */ - if (!page_compressed_encrypted) { - bpage->calculated_checksum = fil_crypt_calculate_checksum(zip_size, dst_frame); - } - /* decrypt using crypt_buf to dst_frame */ byte* res = fil_space_decrypt(bpage->space, slot->crypt_buf, size, - dst_frame); + dst_frame, + &bpage->encrypted); if (!res) { - bpage->encrypted = true; success = false; } #ifdef UNIV_DEBUG @@ -6569,7 +6505,5 @@ buf_page_decrypt_after_read( } } - bpage->key_version = key_version; - return (success); } diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 68bb83e4903..a60d3f2ed12 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -383,10 +383,11 @@ buf_dblwr_init_or_load_pages( doublewrite = read_buf + TRX_SYS_DOUBLEWRITE; if (mach_read_from_4(read_buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) { + bool decrypted = false; byte* tmp = fil_space_decrypt((ulint)TRX_SYS_SPACE, read_buf + UNIV_PAGE_SIZE, UNIV_PAGE_SIZE, /* page size */ - read_buf); + read_buf, &decrypted); doublewrite = tmp + TRX_SYS_DOUBLEWRITE; } @@ -487,6 +488,7 @@ buf_dblwr_process() byte* read_buf; byte* unaligned_read_buf; recv_dblwr_t& recv_dblwr = recv_sys->dblwr; + fil_space_t* space=NULL; unaligned_read_buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); @@ -514,6 +516,10 @@ buf_dblwr_process() continue; } + if (!space) { + space = fil_space_found_by_id(space_id); + } + ulint zip_size = fil_space_get_zip_size(space_id); ut_ad(!buf_page_is_zeroes(page, zip_size)); @@ -548,9 +554,9 @@ buf_dblwr_process() } if (fil_space_verify_crypt_checksum( - read_buf, zip_size) + read_buf, zip_size, NULL, page_no) || !buf_page_is_corrupted( - true, read_buf, zip_size)) { + true, read_buf, zip_size, space)) { /* The page is good; there is no need to consult the doublewrite buffer. */ continue; @@ -573,8 +579,8 @@ buf_dblwr_process() NULL, page, UNIV_PAGE_SIZE, NULL, true); } - if (!fil_space_verify_crypt_checksum(page, zip_size) - && buf_page_is_corrupted(true, page, zip_size)) { + if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no) + && buf_page_is_corrupted(true, page, zip_size, space)) { if (!is_all_zero) { ib_logf(IB_LOG_LEVEL_WARN, "A doublewrite copy of page " diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 2999bea2765..fed1cf94d84 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -625,6 +625,8 @@ fil_encrypt_buf( // store the post-encryption checksum after the key-version mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum); + ut_ad(fil_space_verify_crypt_checksum(dst_frame, zip_size, NULL, offset)); + srv_stats.pages_encrypted.inc(); return dst_frame; @@ -676,6 +678,7 @@ fil_space_encrypt( byte* comp_mem = NULL; byte* uncomp_mem = NULL; ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + fil_space_t* tspace = fil_space_found_by_id(space); if (page_compressed_encrypted) { comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); @@ -685,7 +688,7 @@ fil_space_encrypt( src = uncomp_mem; } - bool corrupted1 = buf_page_is_corrupted(true, src, zip_size); + bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, tspace); bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); /* Need to decompress the page if it was also compressed */ @@ -694,7 +697,7 @@ fil_space_encrypt( fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); } - bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size); + bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, tspace); bool different = memcmp(src, tmp_mem, size); if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { @@ -858,19 +861,25 @@ fil_space_decrypt( /****************************************************************** Decrypt a page -@return encrypted page, or original not encrypted page if encryption is -not needed. */ +@param[in] space Tablespace id +@param[in] tmp_frame Temporary buffer used for decrypting +@param[in] page_size Page size +@param[in,out] src_frame Page to decrypt +@param[out] decrypted true if page was decrypted +@return decrypted page, or original not encrypted page if decryption is +not needed.*/ UNIV_INTERN byte* fil_space_decrypt( -/*==============*/ - ulint space, /*!< in: Fil space id */ - byte* tmp_frame, /*!< in: temporary buffer */ - ulint page_size, /*!< in: page size */ - byte* src_frame) /*!< in/out: page buffer */ + ulint space, + byte* tmp_frame, + ulint page_size, + byte* src_frame, + bool* decrypted) { dberr_t err = DB_SUCCESS; byte* res = NULL; + *decrypted = false; bool encrypted = fil_space_decrypt( fil_space_get_crypt_data(space), @@ -881,6 +890,7 @@ fil_space_decrypt( if (err == DB_SUCCESS) { if (encrypted) { + *decrypted = true; /* Copy the decrypted page back to page buffer, not really any other options. */ memcpy(src_frame, tmp_frame, page_size); @@ -934,83 +944,114 @@ fil_crypt_calculate_checksum( } /********************************************************************* -Verify checksum for a page (iff it's encrypted) -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) +Verify that post encryption checksum match calculated checksum. +This function should be called only if tablespace contains crypt_data +metadata (this is strong indication that tablespace is encrypted). +Function also verifies that traditional checksum does not match +calculated checksum as if it does page could be valid unencrypted, +encrypted, or corrupted. +@param[in] page Page to verify +@param[in] zip_size zip size +@param[in] space Tablespace +@param[in] pageno Page no @return true if page is encrypted AND OK, false otherwise */ UNIV_INTERN bool fil_space_verify_crypt_checksum( -/*============================*/ - const byte* src_frame, /*!< in: page the verify */ - ulint zip_size) /*!< in: compressed size if - row_format compressed */ + byte* page, + ulint zip_size, + const fil_space_t* space, + ulint pageno) { - // key version - uint key_version = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + /* If page is not encrypted, return false */ if (key_version == 0) { - return false; // unencrypted page + return false; } - /* "trick" the normal checksum routines by storing the post-encryption - * checksum into the normal checksum field allowing for reuse of - * the normal routines */ - - // post encryption checksum - ib_uint32_t stored_post_encryption = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - // save pre encryption checksum for restore in end of this function - ib_uint32_t stored_pre_encryption = mach_read_from_4( - src_frame + FIL_PAGE_SPACE_OR_CHKSUM); - - ib_uint32_t checksum_field2 = mach_read_from_4( - src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); - - /** prepare frame for usage of normal checksum routines */ - mach_write_to_4(const_cast(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM, - stored_post_encryption); - - /* NOTE: this function is (currently) only run when restoring - * dblwr-buffer, server is single threaded so it's safe to modify - * srv_checksum_algorithm */ - srv_checksum_algorithm_t save_checksum_algorithm = - (srv_checksum_algorithm_t)srv_checksum_algorithm; - - if (zip_size == 0 && - (save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || - save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) { - /* handle ALGORITHM_INNODB specially, - * "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC - * checksum_field2 is sort of pointless anyway... - */ - srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; - mach_write_to_4(const_cast(src_frame) + - UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, - BUF_NO_CHECKSUM_MAGIC); + /* If no checksum is used, can't continue checking. */ + if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { + return(true); } - /* verify checksums */ - ibool corrupted = buf_page_is_corrupted(false, src_frame, zip_size); + /* Read stored post encryption checksum. */ + ib_uint32_t checksum = mach_read_from_4( + page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - /** restore frame & algorithm */ - srv_checksum_algorithm = save_checksum_algorithm; - - mach_write_to_4(const_cast(src_frame) + - FIL_PAGE_SPACE_OR_CHKSUM, - stored_pre_encryption); - - mach_write_to_4(const_cast(src_frame) + - UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, - checksum_field2); - - if (!corrupted) { - return true; // page was encrypted and checksum matched - } else { - return false; // page was encrypted but checksum didn't match + /* Declare empty pages non-corrupted */ + if (checksum == 0 + && *reinterpret_cast(page + FIL_PAGE_LSN) == 0) { + return (true); } + + /* Compressed and encrypted pages do not have checksum. Assume not + corrupted. */ + if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + return (true); + } + + /* Compressed pages use different checksum method. We first store + the post encryption checksum on checksum location and after function + restore the original. */ + if (zip_size) { + ib_uint32_t old = static_cast(mach_read_from_4( + page + FIL_PAGE_SPACE_OR_CHKSUM)); + + mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + + bool valid = page_zip_verify_checksum(page, zip_size); + + mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old); + + return (valid); + } + + /* If stored checksum matches one of the calculated checksums + page is not corrupted. */ + + ib_uint32_t cchecksum1 = buf_calc_page_crc32(page); + ib_uint32_t cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( + page); + bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 + || checksum == BUF_NO_CHECKSUM_MAGIC); + + /* Old InnoDB versions did not initialize + FIL_PAGE_FILE_FLUSH_LSN field so there could be garbage + and above checksum check could produce false positive. + Thus we also check does the traditional stored + checksum fields match the calculated one. Both of these + could naturally produce false positive but then + we just decrypt the page and after that corrupted + pages very probable stay corrupted and valid + pages very probable stay valid. + */ + ulint checksum1 = mach_read_from_4( + page + FIL_PAGE_SPACE_OR_CHKSUM); + + ulint checksum2 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + + + bool valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) + || buf_page_is_checksum_valid_none(page,checksum1,checksum2) + || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); + + if (encrypted && valid) { + /* If page is encrypted and traditional checksums match, + page could be still encrypted, or not encrypted and valid or + corrupted. */ + ib_logf(IB_LOG_LEVEL_ERROR, + " Page %lu in space %s (%lu) maybe corrupted." + " Post encryption checksum %u stored [%lu:%lu] key_version %u", + pageno, + space ? space->name : "N/A", + mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), + checksum, checksum1, checksum2, key_version); + encrypted = false; + } + + return(encrypted); } /***********************************************************************/ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index f4301d47028..aae804432c6 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -2309,7 +2309,7 @@ fil_check_first_page(const page_t* page, ulint space_id, ulint flags) } if (buf_page_is_corrupted( - false, page, fsp_flags_get_zip_size(flags))) { + false, page, fsp_flags_get_zip_size(flags), NULL)) { return("checksum mismatch"); } @@ -4547,13 +4547,13 @@ fil_user_tablespace_find_space_id( to UNIV_PAGE_SIZE. */ if (page_size == UNIV_PAGE_SIZE) { uncompressed_ok = !buf_page_is_corrupted( - false, page, 0); + false, page, 0, NULL); } bool compressed_ok = false; if (page_size <= UNIV_PAGE_SIZE_DEF) { compressed_ok = !buf_page_is_corrupted( - false, page, page_size); + false, page, page_size, NULL); } if (uncompressed_ok || compressed_ok) { diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index fe4f64d88ca..303ab5102fb 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -366,7 +366,7 @@ fil_compress_page( fil_decompress_page(uncomp_page, comp_page, len, NULL); - if(buf_page_is_corrupted(false, uncomp_page, 0)) { + if(buf_page_is_corrupted(false, uncomp_page, 0, space)) { buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); ut_error; } diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 6924481af49..7a39d544a24 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -639,19 +639,68 @@ buf_block_unfix( #else /* !UNIV_HOTBACKUP */ # define buf_block_modify_clock_inc(block) ((void) 0) #endif /* !UNIV_HOTBACKUP */ + +/** Checks if the page is in crc32 checksum format. +@param[in] read_buf database page +@param[in] checksum_field1 new checksum field +@param[in] checksum_field2 old checksum field +@return true if the page is in crc32 checksum format */ +bool +buf_page_is_checksum_valid_crc32( + const byte* read_buf, + ulint checksum_field1, + ulint checksum_field2) + MY_ATTRIBUTE((warn_unused_result)); + +/** Checks if the page is in innodb checksum format. +@param[in] read_buf database page +@param[in] checksum_field1 new checksum field +@param[in] checksum_field2 old checksum field +@return true if the page is in innodb checksum format */ +bool +buf_page_is_checksum_valid_innodb( + const byte* read_buf, + ulint checksum_field1, + ulint checksum_field2) + MY_ATTRIBUTE((warn_unused_result)); + +/** Checks if the page is in none checksum format. +@param[in] read_buf database page +@param[in] checksum_field1 new checksum field +@param[in] checksum_field2 old checksum field +@return true if the page is in none checksum format */ +bool +buf_page_is_checksum_valid_none( + const byte* read_buf, + ulint checksum_field1, + ulint checksum_field2) + MY_ATTRIBUTE((warn_unused_result)); + /********************************************************************//** Checks if a page is corrupt. -@return TRUE if corrupted */ -UNIV_INTERN -ibool +@param[in] check_lsn true if LSN should be checked +@param[in] read_buf Page to be checked +@param[in] zip_size compressed size or 0 +@param[in] space Pointer to tablespace +@return true if corrupted, false if not */ +bool buf_page_is_corrupted( -/*==================*/ - bool check_lsn, /*!< in: true if we need to check the - and complain about the LSN */ - const byte* read_buf, /*!< in: a database page */ - ulint zip_size) /*!< in: size of compressed page; - 0 for uncompressed pages */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + bool check_lsn, + const byte* read_buf, + ulint zip_size, + const fil_space_t* space) + MY_ATTRIBUTE((warn_unused_result)); +/********************************************************************//** +Check if page is maybe compressed, encrypted or both when we encounter +corrupted page. Note that we can't be 100% sure if page is corrupted +or decrypt/decompress just failed. +@param[in] bpage Page +@return true if page corrupted, false if not */ +bool +buf_page_check_corrupt( + buf_page_t* bpage) /*!< in/out: buffer page read from disk */ + MY_ATTRIBUTE(( warn_unused_result)); + /********************************************************************//** Checks if a page is all zeroes. @return TRUE if the page is all zeroes */ @@ -1524,7 +1573,7 @@ The hook that is called just after a page is read from disk. The function decrypt disk content into buf_page_t and releases the temporary buffer that was allocated in buf_page_decrypt_before_read */ UNIV_INTERN -ibool +bool buf_page_decrypt_after_read( /*========================*/ buf_page_t* page); /*!< in/out: buffer page read from disk */ @@ -1630,15 +1679,9 @@ struct buf_page_t{ if written again we check is TRIM operation needed. */ - unsigned key_version; /*!< key version for this block */ - bool page_encrypted; /*!< page is page encrypted */ - bool page_compressed;/*!< page is page compressed */ - ulint stored_checksum;/*!< stored page checksum if page - encrypted */ - bool encrypted; /*!< page is still encrypted */ - ulint calculated_checksum; - /*!< calculated checksum if page - encrypted */ + unsigned key_version; /*!< key version for this block */ + bool encrypted; /*!< page is still encrypted */ + bool corrupted; /*!< page is corrupted */ ulint real_size; /*!< Real size of the page Normal pages == UNIV_PAGE_SIZE @@ -2318,7 +2361,6 @@ buf_pool_mutex_exit( /*================*/ buf_pool_t* buf_pool); /*!< in: buffer pool */ - #ifndef UNIV_NONINL #include "buf0buf.ic" #endif diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index 42cdafde4d0..6aae71ba12d 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -328,7 +328,7 @@ fil_space_check_encryption_read( /****************************************************************** Decrypt a page -@return true if page is decrypted, false if not. */ +@return true if page decrypted, false if not.*/ UNIV_INTERN bool fil_space_decrypt( @@ -336,9 +336,10 @@ fil_space_decrypt( fil_space_crypt_t* crypt_data, /*!< in: crypt data */ byte* tmp_frame, /*!< in: temporary buffer */ ulint page_size, /*!< in: page size */ - byte* src_frame, /*!< in:out: page buffer */ - dberr_t* err); /*!< in: out: DB_SUCCESS or + byte* src_frame, /*!< in: out: page buffer */ + dberr_t* err) /*!< in: out: DB_SUCCESS or error code */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************* Encrypt buffer page @@ -355,31 +356,46 @@ fil_space_encrypt( ulint size, /*!< in: size of data to encrypt */ byte* dst_frame); /*!< in: where to encrypt to */ -/********************************************************************* -Decrypt buffer page -@return decrypted page, or original not encrypted page if decrypt is +/****************************************************************** +Decrypt a page +@param[in] space Tablespace id +@param[in] tmp_frame Temporary buffer used for decrypting +@param[in] page_size Page size +@param[in,out] src_frame Page to decrypt +@param[out] decrypted true if page was decrypted +@return decrypted page, or original not encrypted page if decryption is not needed.*/ UNIV_INTERN byte* fil_space_decrypt( /*==============*/ - ulint space, /*!< in: tablespace id */ - byte* src_frame, /*!< in: page frame */ - ulint page_size, /*!< in: size of data to encrypt */ - byte* dst_frame) /*!< in: where to decrypt to */ + ulint space, + byte* src_frame, + ulint page_size, + byte* dst_frame, + bool* decrypted) __attribute__((warn_unused_result)); /********************************************************************* -fil_space_verify_crypt_checksum -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) +Verify that post encryption checksum match calculated checksum. +This function should be called only if tablespace contains crypt_data +metadata (this is strong indication that tablespace is encrypted). +Function also verifies that traditional checksum does not match +calculated checksum as if it does page could be valid unencrypted, +encrypted, or corrupted. +@param[in] page Page to verify +@param[in] zip_size zip size +@param[in] space Tablespace +@param[in] pageno Page no @return true if page is encrypted AND OK, false otherwise */ UNIV_INTERN bool fil_space_verify_crypt_checksum( -/*============================*/ - const byte* src_frame,/*!< in: page frame */ - ulint zip_size); /*!< in: size of data to encrypt */ + byte* page, + ulint zip_size, + const fil_space_t* space, + ulint pageno) + __attribute__((warn_unused_result)); /********************************************************************* Init threads for key rotation */ diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic index 23614a6567a..2b3ddc04b3b 100644 --- a/storage/xtradb/include/fil0fil.ic +++ b/storage/xtradb/include/fil0fil.ic @@ -59,6 +59,8 @@ fil_get_page_type_name( switch(page_type) { case FIL_PAGE_PAGE_COMPRESSED: return (const char*)"PAGE_COMPRESSED"; + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + return (const char*)"PAGE_COMPRESSED_ENCRYPTED"; case FIL_PAGE_INDEX: return (const char*)"INDEX"; case FIL_PAGE_UNDO_LOG: @@ -87,9 +89,13 @@ fil_get_page_type_name( return (const char*)"ZBLOB2"; case FIL_PAGE_TYPE_COMPRESSED: return (const char*)"ORACLE PAGE COMPRESSED"; - default: - return (const char*)"PAGE TYPE CORRUPTED"; + + /* No default to make compiler generate warning if + new page type is added but not handled here. */ } + + return (const char*)"PAGE TYPE CORRUPTED"; + } /****************************************************************//** diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index 6170eb66195..4e8488f88a8 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -40,6 +40,7 @@ Created 2012-02-08 by Sunny Bains. #include "row0mysql.h" #include "srv0start.h" #include "row0quiesce.h" +#include "buf0buf.h" #include @@ -2036,12 +2037,15 @@ PageConverter::validate( buf_block_t* block) UNIV_NOTHROW { buf_frame_t* page = get_frame(block); + ulint space_id = mach_read_from_4( + page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + fil_space_t* space = fil_space_found_by_id(space_id); /* Check that the page number corresponds to the offset in the file. Flag as corrupt if it doesn't. Disable the check for LSN in buf_page_is_corrupted() */ - if (buf_page_is_corrupted(false, page, get_zip_size()) + if (buf_page_is_corrupted(false, page, get_zip_size(), space) || (page_get_page_no(page) != offset / m_page_size && page_get_page_no(page) != 0)) {