diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index e67bcc13f0b..74df94180e9 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -523,7 +523,7 @@ is_page_corrupted( normal method. */ if (is_encrypted && key_version != 0) { is_corrupted = use_full_crc32 - ? buf_page_is_corrupted(true, buf, flags) + ? !!buf_page_is_corrupted(false, buf, flags) : !fil_space_verify_crypt_checksum(buf, zip_size); if (is_corrupted && log_file) { diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index 6f31583072b..871cc7508c0 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -291,7 +291,7 @@ static bool page_is_corrupted(const byte *page, ulint page_no, } if (space->full_crc32()) { - return buf_page_is_corrupted(true, page, space->flags); + return buf_page_is_corrupted(false, page, space->flags); } /* Validate encrypted pages. The first page is never encrypted. @@ -324,7 +324,7 @@ static bool page_is_corrupted(const byte *page, ulint page_no, } if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return buf_page_is_corrupted(true, tmp_page, + return buf_page_is_corrupted(false, tmp_page, space->flags); } } @@ -344,11 +344,11 @@ static bool page_is_corrupted(const byte *page, ulint page_no, && cursor->zip_size) || page_type == FIL_PAGE_PAGE_COMPRESSED || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || buf_page_is_corrupted(true, tmp_page, + || buf_page_is_corrupted(false, tmp_page, space->flags)); } - return buf_page_is_corrupted(true, page, space->flags); + return buf_page_is_corrupted(false, page, space->flags); } PRAGMA_REENABLE_CHECK_STACK_FRAME diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index fea69b6dbb0..13d56020b7c 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -3472,7 +3472,6 @@ static void xb_load_single_table_tablespace(const char *dirname, size_t dirlen = dirname == NULL ? 0 : strlen(dirname); size_t namelen = strlen(filname); ulint pathlen = dirname == NULL ? namelen + 1: dirlen + namelen + 2; - lsn_t flush_lsn; dberr_t err; fil_space_t *space; bool defer = false; @@ -3507,7 +3506,7 @@ static void xb_load_single_table_tablespace(const char *dirname, for (int i = 0; i < 10; i++) { file->m_defer = false; - err = file->validate_first_page(&flush_lsn); + err = file->validate_first_page(file->get_first_page()); if (file->m_defer) { if (defer_space_id) { @@ -3549,7 +3548,7 @@ static void xb_load_single_table_tablespace(const char *dirname, skip_node_page0 ? file->detach() : pfs_os_file_t(), 0, false, false); node->deferred= defer; - if (!space->read_page0()) + if (!space->read_page0(nullptr, true)) err = DB_CANNOT_OPEN_FILE; mysql_mutex_unlock(&fil_system.mutex); @@ -6176,8 +6175,10 @@ static bool xtrabackup_prepare_func(char** argv) goto error_cleanup; } + mysql_mutex_lock(&recv_sys.mutex); ok = fil_system.sys_space->open(false) && xtrabackup_apply_deltas(); + mysql_mutex_unlock(&recv_sys.mutex); xb_data_files_close(); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index 1ca9d784d57..2efccbe9e9a 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -2,8 +2,9 @@ call mtr.add_suppression("Plugin 'file_key_management' init function returned er call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` (has an unreadable root page|is corrupted)"); call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted; key_version=1"); -call mtr.add_suppression("failed to read \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]"); +call mtr.add_suppression("InnoDB: Failed to read page [1-9][0-9]* from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t1.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` is corrupted"); call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result index b4552651ae6..064b878fe69 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result @@ -3,6 +3,8 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page n call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded\\."); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("InnoDB: Cannot delete tablespace .* because it is not found in the tablespace memory cache"); call mtr.add_suppression("InnoDB: ALTER TABLE `test`\\.`t1` DISCARD TABLESPACE failed to find tablespace"); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result index 5a3e275dda9..f2f652ab730 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result @@ -2,6 +2,8 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` (has an unreadable root pa call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t1\\.ibd' cannot be decrypted; key_version=1"); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); # restart: --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt diff --git a/mysql-test/suite/encryption/r/innodb-compressed-blob.result b/mysql-test/suite/encryption/r/innodb-compressed-blob.result index f1b75b05219..41c9b3e58d5 100644 --- a/mysql-test/suite/encryption/r/innodb-compressed-blob.result +++ b/mysql-test/suite/encryption/r/innodb-compressed-blob.result @@ -1,7 +1,7 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted; key_version=1"); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Unable to decompress ..test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test/t[12]\\.ibd' page \\[page id: space=[1-9][0-9]*, page number=3\\]"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` is corrupted"); # Restart mysqld --file-key-management-filename=keys2.txt diff --git a/mysql-test/suite/encryption/r/innodb-encryption-disable.result b/mysql-test/suite/encryption/r/innodb-encryption-disable.result index 179bc550617..1df5d7e504b 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-disable.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-disable.result @@ -2,6 +2,8 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t[15]` (has an unreadable root call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[15].ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[15]\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t[15]\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[15]` is corrupted"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); # restart: --innodb-encrypt-tables=ON --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt diff --git a/mysql-test/suite/encryption/r/innodb-force-corrupt.result b/mysql-test/suite/encryption/r/innodb-force-corrupt.result index ad75df952b3..0c1b52e9e37 100644 --- a/mysql-test/suite/encryption/r/innodb-force-corrupt.result +++ b/mysql-test/suite/encryption/r/innodb-force-corrupt.result @@ -1,7 +1,10 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t[13]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("InnoDB: File '.*test/t[123]\\.ibd' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[13]\\.ibd'"); +call mtr.add_suppression("InnoDB: Failed to read page 6 from file '.*test/t2\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version="); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption in an InnoDB type table"); -call mtr.add_suppression("\\[ERROR\\] (mysqld|mariadbd).*: Index for table 't2' is corrupt; try to repair it"); +call mtr.add_suppression("\\[ERROR\\] mariadbd.*: Index for table 't2' is corrupt; try to repair it"); SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; # Create and populate tables to be corrupted diff --git a/mysql-test/suite/encryption/r/innodb-missing-key.result b/mysql-test/suite/encryption/r/innodb-missing-key.result index 6bacbfe78b2..a19056c37ed 100644 --- a/mysql-test/suite/encryption/r/innodb-missing-key.result +++ b/mysql-test/suite/encryption/r/innodb-missing-key.result @@ -2,6 +2,8 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` (has an unreadable root pa call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file .*test.t[12].ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted"); # Start server with keys2.txt # restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt diff --git a/mysql-test/suite/encryption/r/innodb-redo-badkey.result b/mysql-test/suite/encryption/r/innodb-redo-badkey.result index 34fd043a7bd..16035bd3b15 100644 --- a/mysql-test/suite/encryption/r/innodb-redo-badkey.result +++ b/mysql-test/suite/encryption/r/innodb-redo-badkey.result @@ -24,6 +24,7 @@ insert into t2 select * from t1; insert into t3 select * from t1; insert into t4 select * from t1; commit; +set global innodb_log_checkpoint_now=on; SET GLOBAL innodb_flush_log_at_trx_commit=1; begin; update t1 set c = repeat('secret3', 20); diff --git a/mysql-test/suite/encryption/t/corrupted_during_recovery.test b/mysql-test/suite/encryption/t/corrupted_during_recovery.test index d4048997a13..ea7ad218936 100644 --- a/mysql-test/suite/encryption/t/corrupted_during_recovery.test +++ b/mysql-test/suite/encryption/t/corrupted_during_recovery.test @@ -5,11 +5,11 @@ call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); call mtr.add_suppression("InnoDB: Failed to read page [123] from file '.*test.t1\\.ibd': Table is encrypted but decrypt failed"); call mtr.add_suppression("InnoDB: The page \\[page id: space=\\d+, page number=3\\] in file '.*test.t1\\.ibd' cannot be decrypted"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table in tablespace \\d+ encrypted. However key management plugin or used key_version \\d+ is not found or used encryption algorithm or method does not match. Can't continue opening the table."); -call mtr.add_suppression("InnoDB: (Unable to apply log to|Discarding log for) corrupted page "); +call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page "); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index 30beff4bd85..f08c40f67c3 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -12,8 +12,9 @@ call mtr.add_suppression("Plugin 'file_key_management' init function returned er call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` (has an unreadable root page|is corrupted)"); call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted; key_version=1"); -call mtr.add_suppression("failed to read \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]"); +call mtr.add_suppression("InnoDB: Failed to read page [1-9][0-9]* from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t1.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` is corrupted"); call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found"); # for innodb_checksum_algorithm=full_crc32 only diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test index ad252b6c92f..b0378585e00 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test @@ -14,6 +14,8 @@ call mtr.add_suppression("InnoDB: Recovery failed to read page"); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded\\."); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("InnoDB: Cannot delete tablespace .* because it is not found in the tablespace memory cache"); call mtr.add_suppression("InnoDB: ALTER TABLE `test`\\.`t1` DISCARD TABLESPACE failed to find tablespace"); diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test index 759f4e58280..23f726d3d58 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test @@ -12,6 +12,8 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page n call mtr.add_suppression("InnoDB: Recovery failed to read page"); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted"); # for innodb_checksum_algorithm=full_crc32 only call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); diff --git a/mysql-test/suite/encryption/t/innodb-compressed-blob.test b/mysql-test/suite/encryption/t/innodb-compressed-blob.test index a47db8e64e2..097b0fb4602 100644 --- a/mysql-test/suite/encryption/t/innodb-compressed-blob.test +++ b/mysql-test/suite/encryption/t/innodb-compressed-blob.test @@ -7,7 +7,7 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted; key_version=1"); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Unable to decompress ..test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test/t[12]\\.ibd' page \\[page id: space=[1-9][0-9]*, page number=3\\]"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` is corrupted"); diff --git a/mysql-test/suite/encryption/t/innodb-encryption-disable.test b/mysql-test/suite/encryption/t/innodb-encryption-disable.test index 939ad2b5547..20624ba9fd7 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-disable.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-disable.test @@ -11,6 +11,8 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t[15]` (has an unreadable root call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[15].ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[15]\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t[15]\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[15]` is corrupted"); # Suppression for builds where file_key_management plugin is linked statically diff --git a/mysql-test/suite/encryption/t/innodb-force-corrupt.test b/mysql-test/suite/encryption/t/innodb-force-corrupt.test index 51771f1e14b..e0a344e16c2 100644 --- a/mysql-test/suite/encryption/t/innodb-force-corrupt.test +++ b/mysql-test/suite/encryption/t/innodb-force-corrupt.test @@ -8,9 +8,12 @@ -- source include/not_embedded.inc call mtr.add_suppression("InnoDB: Table `test`\\.`t[13]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("InnoDB: File '.*test/t[123]\\.ibd' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[13]\\.ibd'"); +call mtr.add_suppression("InnoDB: Failed to read page 6 from file '.*test/t2\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version="); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption in an InnoDB type table"); -call mtr.add_suppression("\\[ERROR\\] (mysqld|mariadbd).*: Index for table 't2' is corrupt; try to repair it"); +call mtr.add_suppression("\\[ERROR\\] mariadbd.*: Index for table 't2' is corrupt; try to repair it"); SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.test b/mysql-test/suite/encryption/t/innodb-missing-key.test index e5bee99fee2..a0367b8f211 100644 --- a/mysql-test/suite/encryption/t/innodb-missing-key.test +++ b/mysql-test/suite/encryption/t/innodb-missing-key.test @@ -11,6 +11,8 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` (has an unreadable root pa call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file .*test.t[12].ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted"); --echo # Start server with keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-redo-badkey.test b/mysql-test/suite/encryption/t/innodb-redo-badkey.test index 393ca4ad375..ed92a267a91 100644 --- a/mysql-test/suite/encryption/t/innodb-redo-badkey.test +++ b/mysql-test/suite/encryption/t/innodb-redo-badkey.test @@ -54,6 +54,8 @@ insert into t3 select * from t1; insert into t4 select * from t1; commit; +set global innodb_log_checkpoint_now=on; + --source ../../suite/innodb/include/no_checkpoint_start.inc # diff --git a/mysql-test/suite/innodb/r/corrupted_during_recovery.result b/mysql-test/suite/innodb/r/corrupted_during_recovery.result index 2cab795f6a1..593943b4951 100644 --- a/mysql-test/suite/innodb/r/corrupted_during_recovery.result +++ b/mysql-test/suite/innodb/r/corrupted_during_recovery.result @@ -8,8 +8,10 @@ INSERT INTO t2 VALUES(1); # Corrupt the pages SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' +FOUND 1 /InnoDB: Page \[page id: space=[1-9][0-9]*, page number=3\] log sequence number 1311768467463790320 is in the future!/ in mysqld.1.err SELECT * FROM t1; -Got one of the listed errors +a +1 SELECT * FROM t2; a 1 @@ -27,7 +29,7 @@ SET GLOBAL innodb_flush_log_at_trx_commit=1; DELETE FROM t1 WHERE pk=3; # Kill the server disconnect con1; -# Corrupt the pages +# Corrupt the page SELECT * FROM t1; pk 1 diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result index 65ff203a28e..667c65d7b7b 100644 --- a/mysql-test/suite/innodb/r/doublewrite.result +++ b/mysql-test/suite/innodb/r/doublewrite.result @@ -21,8 +21,8 @@ connection default; flush table t1 for export; # Kill the server # restart -FOUND 1 /InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile/ in mysqld.1.err -FOUND 1 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=3\]/ in mysqld.1.err +FOUND 1 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=0\]/ in mysqld.1.err +# restart XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text @@ -44,7 +44,7 @@ connection default; flush table t1 for export; # Kill the server # restart -FOUND 1 /InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile/ in mysqld.1.err +FOUND 4 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=[03]\]/ in mysqld.1.err XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text diff --git a/mysql-test/suite/innodb/r/doublewrite_debug.result b/mysql-test/suite/innodb/r/doublewrite_debug.result index aa141c18a6e..a743217f34e 100644 --- a/mysql-test/suite/innodb/r/doublewrite_debug.result +++ b/mysql-test/suite/innodb/r/doublewrite_debug.result @@ -37,7 +37,7 @@ set global innodb_buf_flush_list_now = 1; # Make the 1st page (page_no=0) and 2nd page (page_no=1) # of the system tablespace all zeroes. # restart -FOUND 1 /InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile/ in mysqld.1.err +FOUND 1 /InnoDB: Recovered page \[page id: space=0, page number=0\]/ in mysqld.1.err FOUND 1 /InnoDB: Recovered page \[page id: space=0, page number=1\]/ in mysqld.1.err check table t1; Table Op Msg_type Msg_text @@ -66,7 +66,7 @@ set global innodb_buf_flush_list_now = 1; # Kill the server # Corrupt the 1st page (page_no=0) and 2nd page of the system tablespace. # restart -FOUND 2 /InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile/ in mysqld.1.err +FOUND 2 /InnoDB: Recovered page \[page id: space=0, page number=0\]/ in mysqld.1.err FOUND 2 /InnoDB: Recovered page \[page id: space=0, page number=1\]/ in mysqld.1.err check table t1; Table Op Msg_type Msg_text diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result index 5770ebb27db..6b7bb9b9584 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result @@ -9,8 +9,8 @@ call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS="); call mtr.add_suppression("InnoDB: Unknown index id .* on page"); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; SET GLOBAL innodb_file_per_table = 1; CREATE TABLE t1 (c1 INT) ENGINE = InnoDB; diff --git a/mysql-test/suite/innodb/r/page_id_innochecksum.result b/mysql-test/suite/innodb/r/page_id_innochecksum.result index bde986c07ef..4ffbf2f1453 100644 --- a/mysql-test/suite/innodb/r/page_id_innochecksum.result +++ b/mysql-test/suite/innodb/r/page_id_innochecksum.result @@ -6,3 +6,4 @@ FOUND 1 /page id mismatch/ in result.log InnoDB 0 transactions not purged drop table t1; call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd': Page read from tablespace is corrupted\\."); +call mtr.add_suppression("InnoDB: File '.*t1\\.ibd' is corrupted"); diff --git a/mysql-test/suite/innodb/r/undo_space_dblwr.result b/mysql-test/suite/innodb/r/undo_space_dblwr.result index 2172ce53cb7..d7c1cb125a9 100644 --- a/mysql-test/suite/innodb/r/undo_space_dblwr.result +++ b/mysql-test/suite/innodb/r/undo_space_dblwr.result @@ -12,7 +12,7 @@ set global innodb_fil_make_page_dirty_debug = 1; SET GLOBAL innodb_buf_flush_list_now = 1; # Kill the server # restart: --debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0 -FOUND 1 /Restoring page \[page id: space=1, page number=0\] of datafile '.*undo001' from the doublewrite buffer./ in mysqld.1.err +FOUND 1 /Recovered page \[page id: space=1, page number=0\] to '.*undo001' from the doublewrite buffer\./ in mysqld.1.err check table t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb/t/corrupted_during_recovery.test b/mysql-test/suite/innodb/t/corrupted_during_recovery.test index 5b67bc3df1d..42524976dd7 100644 --- a/mysql-test/suite/innodb/t/corrupted_during_recovery.test +++ b/mysql-test/suite/innodb/t/corrupted_during_recovery.test @@ -4,15 +4,18 @@ call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); -call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test.t1\\.ibd': Page read from tablespace is corrupted."); -call mtr.add_suppression("InnoDB: (Unable to apply log to|Discarding log for) corrupted page .*, page number=3\\]"); +call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page 3 in file .*test.t1\\.ibd"); call mtr.add_suppression("InnoDB: Table `test`.`t1` is corrupted. Please drop the table and recreate."); call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: A long wait .* was observed for dict_sys"); +call mtr.add_suppression("InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 1311768467463790320 is in the future!"); +call mtr.add_suppression("InnoDB: Your database may be corrupt"); +call mtr.add_suppression("InnoDB: MySQL-8\\.0 tablespace in .*test/t2\\.ibd"); +call mtr.add_suppression("InnoDB: Restart in MySQL for migration/recovery\\."); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let ALGO=`select @@innodb_checksum_algorithm`; CREATE TABLE t1(a BIGINT PRIMARY KEY) ENGINE=InnoDB; INSERT INTO t1 VALUES(1); # Force a redo log checkpoint. @@ -30,15 +33,32 @@ INSERT INTO t2 VALUES(1); --echo # Corrupt the pages perl; +do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; +my $polynomial = 0x82f63b78; # CRC-32C +my $algo = $ENV{ALGO}; my $ps = $ENV{INNODB_PAGE_SIZE}; - my $file = "$ENV{MYSQLD_DATADIR}/test/t1.ibd"; open(FILE, "+<$file") || die "Unable to open $file"; binmode FILE; sysseek(FILE, 3*$ps, 0) || die "Unable to seek $file\n"; die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps; -# Replace the a=1 with a=0. -$page =~ s/\x80\x0\x0\x0\x0\x0\x0\x1/\x80\x0\x0\x0\x0\x0\x0\x0/; +# Assign a future FIL_PAGE_LSN +substr($page, 16, 8) = pack("NN", 0x12345678, 0x9abcdef0); +substr($page, $ps - 8, 8) = pack("NN", 0x9abcdef0, 0x9abcdef0); +if ($algo =~ /full_crc32/) +{ + my $ck = mycrc32(substr($page, 0, $ps - 4), 0, $polynomial); + substr($page, $ps - 4, 4) = pack("N", $ck); +} +else +{ + # Replace the innodb_checksum_algorithm=crc32 checksum + my $ck= pack("N", + mycrc32(substr($page, 4, 22), 0, $polynomial) ^ + mycrc32(substr($page, 38, $ps - 38 - 8), 0, $polynomial)); + substr ($page, 0, 4) = $ck; + substr ($page, $ps - 8, 4) = $ck; +} sysseek(FILE, 3*$ps, 0) || die "Unable to seek $file\n"; syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n"; close FILE or die "close"; @@ -46,20 +66,23 @@ close FILE or die "close"; $file = "$ENV{MYSQLD_DATADIR}/test/t2.ibd"; open(FILE, "+<$file") || die "Unable to open $file"; binmode FILE; -# Corrupt pages 1 to 3. MLOG_INIT_FILE_PAGE2 should protect us! -# Unfortunately, we are not immune to page 0 corruption. -seek (FILE, $ps, SEEK_SET) or die "seek"; -print FILE chr(0xff) x ($ps * 3); +# Corrupt pages 0 to 3. INIT_PAGE should protect us! +print FILE chr(0xff) x ($ps * 4); close FILE or die "close"; EOF --source include/start_mysqld.inc --error ER_UNKNOWN_STORAGE_ENGINE SELECT * FROM t1; + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 1311768467463790320 is in the future!; +--source include/search_pattern_in_file.inc + let $restart_parameters=--innodb_force_recovery=1; --source include/restart_mysqld.inc ---error ER_NO_SUCH_TABLE_IN_ENGINE,ER_TABLE_CORRUPT +--error 0,ER_NO_SUCH_TABLE_IN_ENGINE SELECT * FROM t1; SELECT * FROM t2; CHECK TABLE t2; @@ -81,13 +104,36 @@ DELETE FROM t1 WHERE pk=3; --source ../include/no_checkpoint_end.inc disconnect con1; ---echo # Corrupt the pages +--echo # Corrupt the page perl; +do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; +my $polynomial = 0x82f63b78; # CRC-32C +my $algo = $ENV{ALGO}; +my $ps = $ENV{INNODB_PAGE_SIZE}; my $file = "$ENV{MYSQLD_DATADIR}/test/t1.ibd"; open(FILE, "+<$file") || die "Unable to open $file"; binmode FILE; -seek (FILE, $ENV{INNODB_PAGE_SIZE} * 3, SEEK_SET) or die "seek"; -print FILE "junk"; +sysseek(FILE, $ps * 3, SEEK_SET) or die "seek"; +sysread(FILE, $page, $ps)==$ps||die "Unable to read $file\n"; +# Set FIL_PAGE_LSN to the maximum +substr($page, 16, 8) = chr(255) x 8; +substr($page, $ps - 8, 8) = chr(255) x 8; +if ($algo =~ /full_crc32/) +{ + my $ck = mycrc32(substr($page, 0, $ps - 4), 0, $polynomial); + substr($page, $ps - 4, 4) = pack("N", $ck); +} +else +{ + # Replace the innodb_checksum_algorithm=crc32 checksum + my $ck= pack("N", + mycrc32(substr($page, 4, 22), 0, $polynomial) ^ + mycrc32(substr($page_, 38, $ps - 38 - 8), 0, $polynomial)); + substr ($page, 0, 4) = $ck; + substr ($page, $ps - 8, 4) = $ck; +} +sysseek(FILE, $ps * 3, SEEK_SET) or die "seek"; +syswrite(FILE, $page); close FILE or die "close"; EOF --source include/start_mysqld.inc diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index 7e38851facb..faafbb7cbac 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -18,6 +18,9 @@ call mtr.add_suppression("InnoDB: Checksum mismatch in datafile: "); call mtr.add_suppression("InnoDB: Inconsistent tablespace ID in .*t1\\.ibd"); call mtr.add_suppression("\\[Warning\\] Found 1 prepared XA transactions"); call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile:"); +call mtr.add_suppression("InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 18446744073709551615 is in the future!"); +call mtr.add_suppression("InnoDB: Your database may be corrupt or you may have copied the InnoDB tablespace but not the ib_logfile0"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; @@ -74,7 +77,26 @@ syswrite(FILE, chr(0) x ($page_size/2)); sysseek(FILE, 3*$page_size, 0); sysread(FILE, $page, $page_size)==$page_size||die "Unable to read $name\n"; sysseek(FILE, 3*$page_size, 0)||die "Unable to seek $fname\n"; -syswrite(FILE, chr(0) x ($page_size/2)); +my $corrupted = $page; +# Set FIL_PAGE_LSN to the maximum +substr($corrupted, 16, 8) = chr(255) x 8; +substr($corrupted, $page_size - 8, 8) = chr(255) x 8; +if ($algo =~ /full_crc32/) +{ + my $ck = mycrc32(substr($corrupted, 0, $page_size - 4), 0, $polynomial); + substr($corrupted, $page_size - 4, 4) = pack("N", $ck); +} +else +{ + # Replace the innodb_checksum_algorithm=crc32 checksum + my $ck= pack("N", + mycrc32(substr($corrupted, 4, 22), 0, $polynomial) ^ + mycrc32(substr($corrupted_, 38, $page_size - 38 - 8), 0, + $polynomial)); + substr ($corrupted, 0, 4) = $ck; + substr ($corrupted, $page_size - 8, 4) = $ck; +} +syswrite(FILE, $corrupted); close FILE; # Change the flag offset of page 0 in doublewrite buffer @@ -116,10 +138,28 @@ die "Did not find the page in the doublewrite buffer ($d1,$d2)\n"; EOF --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=[1-9][0-9]*, page number=0\\]; --source include/search_pattern_in_file.inc -let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=3\]; +let SEARCH_PATTERN=InnoDB: The log was only scanned up to \\d+, while the current LSN at the time of the latest checkpoint \\d+ was 0 and the maximum LSN on a data page was 18446744073709551615! --source include/search_pattern_in_file.inc +--error ER_XAER_NOTA +XA ROLLBACK 'x'; +let $shutdown_timeout=0; +--source include/shutdown_mysqld.inc +let $shutdown_timeout=; +# Corrupt the file in a better way. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; +my $page_size = $ENV{INNODB_PAGE_SIZE}; +open(FILE, "+<", $fname) or die; +sysseek(FILE, ($page_size/2), 0); +syswrite(FILE, chr(0) x ($page_size/2)); +sysseek(FILE, 3*$page_size, 0); +syswrite(FILE, chr(0) x ($page_size/2)); +close FILE; +EOF +--source include/start_mysqld.inc XA ROLLBACK 'x'; check table t1; select f1, f2 from t1; @@ -147,7 +187,7 @@ close FILE; EOF --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=[1-9][0-9]*, page number=[03]\\]; --source include/search_pattern_in_file.inc XA ROLLBACK 'x'; check table t1; diff --git a/mysql-test/suite/innodb/t/doublewrite_debug.test b/mysql-test/suite/innodb/t/doublewrite_debug.test index b8dcd5068ef..b207823e3d1 100644 --- a/mysql-test/suite/innodb/t/doublewrite_debug.test +++ b/mysql-test/suite/innodb/t/doublewrite_debug.test @@ -81,10 +81,10 @@ EOF let $restart_parameters=; --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=0\\]; --source include/search_pattern_in_file.inc -let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=0, page number=1\]; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=1\\]; --source include/search_pattern_in_file.inc check table t1; @@ -129,10 +129,10 @@ EOF let $restart_parameters=; --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=0\\]; --source include/search_pattern_in_file.inc -let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=0, page number=1\]; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=1\\]; --source include/search_pattern_in_file.inc check table t1; diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test index 34f36034353..04e7b053ce0 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test @@ -29,8 +29,8 @@ call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS="); call mtr.add_suppression("InnoDB: Unknown index id .* on page"); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491.test b/mysql-test/suite/innodb/t/innodb_bug14147491.test index 10f3c98be9e..d954f2fadea 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491.test +++ b/mysql-test/suite/innodb/t/innodb_bug14147491.test @@ -9,7 +9,7 @@ --disable_query_log call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted\\. Please drop the table and recreate\\."); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); +call mtr.add_suppression("InnoDB: Failed to read page [1-9][0-9]* from file '.*test.t1\\.ibd'"); call mtr.add_suppression("InnoDB: We detected index corruption in an InnoDB type table"); call mtr.add_suppression("Index for table 't1' is corrupt; try to repair it"); call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); diff --git a/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test b/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test index 21d5336528a..026be74373f 100644 --- a/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test +++ b/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test @@ -2,9 +2,8 @@ --source include/have_debug.inc --disable_query_log -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 19 from file '.*test.t1\\.ibd': Page read from tablespace is corrupted\\."); -call mtr.add_suppression("InnoDB: (Unable to apply log to|Discarding log for) corrupted page .*, page number=19\\]"); +call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page 19 in file .*t1\\.ibd"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Plugin initialization aborted at srv0start\\.cc.* with error Data structure corruption"); call mtr.add_suppression("\\[ERROR\\] Plugin 'InnoDB' (init function|registration)"); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption"); diff --git a/mysql-test/suite/innodb/t/page_id_innochecksum.test b/mysql-test/suite/innodb/t/page_id_innochecksum.test index 9d8114d1720..09523cccd1f 100644 --- a/mysql-test/suite/innodb/t/page_id_innochecksum.test +++ b/mysql-test/suite/innodb/t/page_id_innochecksum.test @@ -66,5 +66,6 @@ let $restart_parameters=--innodb-force-recovery=1; --source include/wait_all_purged.inc drop table t1; call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd': Page read from tablespace is corrupted\\."); +call mtr.add_suppression("InnoDB: File '.*t1\\.ibd' is corrupted"); let $restart_parameters=; --source include/restart_mysqld.inc diff --git a/mysql-test/suite/innodb/t/undo_space_dblwr.test b/mysql-test/suite/innodb/t/undo_space_dblwr.test index 4cf4d3b8b6d..33e8ed9d651 100644 --- a/mysql-test/suite/innodb/t/undo_space_dblwr.test +++ b/mysql-test/suite/innodb/t/undo_space_dblwr.test @@ -39,7 +39,7 @@ EOF --source include/start_mysqld.inc let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; -let SEARCH_PATTERN= Restoring page \[page id: space=1, page number=0\] of datafile '.*undo001' from the doublewrite buffer.; +let SEARCH_PATTERN= Recovered page \\[page id: space=1, page number=0\\] to '.*undo001' from the doublewrite buffer\\.; --source include/search_pattern_in_file.inc check table t1; diff --git a/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result b/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result index dadf1117f7e..5ce6affeb0c 100644 --- a/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result +++ b/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result @@ -9,8 +9,8 @@ call mtr.add_suppression("InnoDB: The error means"); call mtr.add_suppression("InnoDB: Cannot open datafile .*t1\\.ibd"); call mtr.add_suppression("InnoDB: Ignoring tablespace for test/t1 "); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; SET SESSION innodb_strict_mode=1; CREATE TABLE t1 (c1 INT) ENGINE = Innodb diff --git a/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test b/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test index b46cc3a2b06..235f74bcc1d 100644 --- a/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test +++ b/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test @@ -25,8 +25,8 @@ call mtr.add_suppression("InnoDB: The error means"); call mtr.add_suppression("InnoDB: Cannot open datafile .*t1\\.ibd"); call mtr.add_suppression("InnoDB: Ignoring tablespace for test/t1 "); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; diff --git a/mysql-test/suite/mariabackup/encrypted_page_compressed.result b/mysql-test/suite/mariabackup/encrypted_page_compressed.result index de4c966caf4..31098e45675 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_compressed.result +++ b/mysql-test/suite/mariabackup/encrypted_page_compressed.result @@ -1,5 +1,7 @@ call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); call mtr.add_suppression("InnoDB: Encrypted page .* in file .*test.t1\\.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes; insert into t1(b, c) values("mariadb", "mariabackup"); # Corrupt the table diff --git a/mysql-test/suite/mariabackup/encrypted_page_compressed.test b/mysql-test/suite/mariabackup/encrypted_page_compressed.test index 54fffb7d08f..2138afd80cc 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/encrypted_page_compressed.test @@ -1,6 +1,8 @@ source include/have_file_key_management.inc; call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); call mtr.add_suppression("InnoDB: Encrypted page .* in file .*test.t1\\.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes; insert into t1(b, c) values("mariadb", "mariabackup"); diff --git a/mysql-test/suite/mariabackup/encrypted_page_corruption.result b/mysql-test/suite/mariabackup/encrypted_page_corruption.result index b328d361cd6..62f39b561da 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_corruption.result +++ b/mysql-test/suite/mariabackup/encrypted_page_corruption.result @@ -1,4 +1,6 @@ call mtr.add_suppression("\\[ERROR\\] InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=3\\] in file '.*test.t1\\.ibd' cannot be decrypted."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Table `test`\\.`t1` has an unreadable root page"); CREATE TABLE t1(c VARCHAR(128)) ENGINE INNODB, encrypted=yes; insert into t1 select repeat('a',100); diff --git a/mysql-test/suite/mariabackup/encrypted_page_corruption.test b/mysql-test/suite/mariabackup/encrypted_page_corruption.test index 1beb020b463..696eccf8270 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_corruption.test +++ b/mysql-test/suite/mariabackup/encrypted_page_corruption.test @@ -2,6 +2,8 @@ --source include/innodb_page_size.inc call mtr.add_suppression("\\[ERROR\\] InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=3\\] in file '.*test.t1\\.ibd' cannot be decrypted."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Table `test`\\.`t1` has an unreadable root page"); CREATE TABLE t1(c VARCHAR(128)) ENGINE INNODB, encrypted=yes; insert into t1 select repeat('a',100); diff --git a/mysql-test/suite/mariabackup/innodb_redo_overwrite.result b/mysql-test/suite/mariabackup/innodb_redo_overwrite.result index 9076dbaa57a..fb83f530fcc 100644 --- a/mysql-test/suite/mariabackup/innodb_redo_overwrite.result +++ b/mysql-test/suite/mariabackup/innodb_redo_overwrite.result @@ -22,6 +22,5 @@ INSERT INTO t SELECT * FROM t; INSERT INTO t SELECT * FROM t; INSERT INTO t SELECT * FROM t; # xtrabackup backup -FOUND 1 /failed: redo log block is overwritten/ in backup.log FOUND 1 /failed: redo log block checksum does not match/ in backup.log DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/innodb_redo_overwrite.test b/mysql-test/suite/mariabackup/innodb_redo_overwrite.test index 1ed75607be8..02f30af07cf 100644 --- a/mysql-test/suite/mariabackup/innodb_redo_overwrite.test +++ b/mysql-test/suite/mariabackup/innodb_redo_overwrite.test @@ -31,13 +31,9 @@ INSERT INTO t SELECT * FROM t; --disable_result_log --error 1 ---exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events > $backuplog +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events --enable_result_log ---let SEARCH_PATTERN=failed: redo log block is overwritten ---let SEARCH_FILE=$backuplog ---source include/search_pattern_in_file.inc ---remove_file $backuplog --rmdir $targetdir --let before_innodb_log_copy_thread_started=INSERT INTO test.t VALUES (0), (1), (2), (3), (4), (5), (6), (7), (8), (9) diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.result b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result index dfcf19b6c2b..ededfbb8e8d 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.result +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result @@ -1,3 +1,5 @@ +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB PAGE_COMPRESSED=YES STATS_PERSISTENT=0; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test index 700c4dd2034..5f75bb98332 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test @@ -1,3 +1,5 @@ +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB PAGE_COMPRESSED=YES STATS_PERSISTENT=0; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index dcebd9414ba..db046675d1e 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -510,43 +510,46 @@ buf_page_is_checksum_valid_crc32( return checksum_field1 == crc32; } +#ifndef UNIV_INNOCHECKSUM /** Checks whether the lsn present in the page is lesser than the peek current lsn. -@param[in] check_lsn lsn to check -@param[in] read_buf page. */ -static void buf_page_check_lsn(bool check_lsn, const byte* read_buf) +@param check_lsn lsn to check +@param read_buf page frame +@return whether the FIL_PAGE_LSN is invalid */ +static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) { -#ifndef UNIV_INNOCHECKSUM - if (check_lsn && recv_lsn_checks_on) { - const lsn_t current_lsn = log_sys.get_lsn(); - const lsn_t page_lsn - = mach_read_from_8(read_buf + FIL_PAGE_LSN); + if (!check_lsn) + return false; + lsn_t current_lsn= log_sys.get_lsn(); + if (UNIV_UNLIKELY(current_lsn == LOG_START_LSN + LOG_BLOCK_HDR_SIZE) && + srv_force_recovery == SRV_FORCE_NO_LOG_REDO) + return false; + const lsn_t page_lsn= mach_read_from_8(read_buf + FIL_PAGE_LSN); - /* Since we are going to reset the page LSN during the import - phase it makes no sense to spam the log with error messages. */ - if (current_lsn < page_lsn) { + if (UNIV_LIKELY(current_lsn >= page_lsn)) + return false; - const uint32_t space_id = mach_read_from_4( - read_buf + FIL_PAGE_SPACE_ID); - const uint32_t page_no = mach_read_from_4( - read_buf + FIL_PAGE_OFFSET); + const uint32_t space_id= mach_read_from_4(read_buf + FIL_PAGE_SPACE_ID); + const uint32_t page_no= mach_read_from_4(read_buf + FIL_PAGE_OFFSET); - ib::error() << "Page " << page_id_t(space_id, page_no) - << " log sequence number " << page_lsn - << " is in the future! Current system" - << " log sequence number " - << current_lsn << "."; + sql_print_error("InnoDB: Page " + "[page id: space=" UINT32PF ", page number=" UINT32PF "]" + " log sequence number " LSN_PF + " is in the future! Current system log sequence number " + LSN_PF ".", + space_id, page_no, page_lsn, current_lsn); - ib::error() << "Your database may be corrupt or" - " you may have copied the InnoDB" - " tablespace but not the InnoDB" - " log files. " - << FORCE_RECOVERY_MSG; + if (srv_force_recovery) + return false; - } - } -#endif /* !UNIV_INNOCHECKSUM */ + sql_print_error("InnoDB: Your database may be corrupt or" + " you may have copied the InnoDB" + " tablespace but not the ib_logfile0. %s", + FORCE_RECOVERY_MSG); + + return true; } +#endif /** Check if a buffer is all zeroes. @@ -563,7 +566,7 @@ bool buf_is_zeroes(span buf) @param[in] read_buf database page @param[in] fsp_flags tablespace flags @return whether the page is corrupted */ -bool +buf_page_is_corrupted_reason buf_page_is_corrupted( bool check_lsn, const byte* read_buf, @@ -574,27 +577,28 @@ buf_page_is_corrupted( const uint size = buf_page_full_crc32_size( read_buf, &compressed, &corrupted); if (corrupted) { - return true; + return CORRUPTED_OTHER; } const byte* end = read_buf + (size - FIL_PAGE_FCRC32_CHECKSUM); uint crc32 = mach_read_from_4(end); if (!crc32 && size == srv_page_size && buf_is_zeroes(span(read_buf, size))) { - return false; + return NOT_CORRUPTED; } DBUG_EXECUTE_IF( "page_intermittent_checksum_mismatch", { static int page_counter; - if (page_counter++ == 3) { + if (mach_read_from_4(FIL_PAGE_OFFSET + read_buf) + && page_counter++ == 3) { crc32++; } }); if (crc32 != ut_crc32(read_buf, size - FIL_PAGE_FCRC32_CHECKSUM)) { - return true; + return CORRUPTED_OTHER; } static_assert(FIL_PAGE_FCRC32_KEY_VERSION == 0, "alignment"); static_assert(FIL_PAGE_LSN % 4 == 0, "alignment"); @@ -606,11 +610,15 @@ buf_page_is_corrupted( end - (FIL_PAGE_FCRC32_END_LSN - FIL_PAGE_FCRC32_CHECKSUM), 4)) { - return true; + return CORRUPTED_OTHER; } - buf_page_check_lsn(check_lsn, read_buf); - return false; + return +#ifndef UNIV_INNOCHECKSUM + buf_page_check_lsn(check_lsn, read_buf) + ? CORRUPTED_FUTURE_LSN : +#endif + NOT_CORRUPTED; } const ulint zip_size = fil_space_t::zip_size(fsp_flags); @@ -631,7 +639,13 @@ buf_page_is_corrupted( && FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp_flags) #endif ) { - return(false); + check_lsn: + return +#ifndef UNIV_INNOCHECKSUM + buf_page_check_lsn(check_lsn, read_buf) + ? CORRUPTED_FUTURE_LSN : +#endif + NOT_CORRUPTED; } static_assert(FIL_PAGE_LSN % 4 == 0, "alignment"); @@ -644,15 +658,16 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ - return(true); + return CORRUPTED_OTHER; } - buf_page_check_lsn(check_lsn, read_buf); - /* Check whether the checksum fields have correct values */ if (zip_size) { - return !page_zip_verify_checksum(read_buf, zip_size); + if (!page_zip_verify_checksum(read_buf, zip_size)) { + return CORRUPTED_OTHER; + } + goto check_lsn; } const uint32_t checksum_field1 = mach_read_from_4( @@ -689,7 +704,7 @@ buf_page_is_corrupted( } if (all_zeroes) { - return false; + return NOT_CORRUPTED; } } @@ -698,13 +713,17 @@ buf_page_is_corrupted( case SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: #endif /* !UNIV_INNOCHECKSUM */ - return !buf_page_is_checksum_valid_crc32( - read_buf, checksum_field1, checksum_field2); + if (!buf_page_is_checksum_valid_crc32(read_buf, + checksum_field1, + checksum_field2)) { + return CORRUPTED_OTHER; + } + goto check_lsn; #ifndef UNIV_INNOCHECKSUM default: if (checksum_field1 == BUF_NO_CHECKSUM_MAGIC && checksum_field2 == BUF_NO_CHECKSUM_MAGIC) { - return false; + goto check_lsn; } const uint32_t crc32 = buf_calc_page_crc32(read_buf); @@ -722,27 +741,35 @@ buf_page_is_corrupted( DBUG_EXECUTE_IF( "page_intermittent_checksum_mismatch", { static int page_counter; - if (page_counter++ == 3) return true; + if (mach_read_from_4(FIL_PAGE_OFFSET + + read_buf) + && page_counter++ == 3) + return CORRUPTED_OTHER; }); if ((checksum_field1 != crc32 || checksum_field2 != crc32) && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { - return true; + return CORRUPTED_OTHER; } } switch (checksum_field1) { case 0: case BUF_NO_CHECKSUM_MAGIC: - return false; + break; + default: + if ((checksum_field1 != crc32 + || checksum_field2 != crc32) + && checksum_field1 + != buf_calc_page_new_checksum(read_buf)) { + return CORRUPTED_OTHER; + } } - return (checksum_field1 != crc32 || checksum_field2 != crc32) - && checksum_field1 - != buf_calc_page_new_checksum(read_buf); } #endif /* !UNIV_INNOCHECKSUM */ + goto check_lsn; } #ifndef UNIV_INNOCHECKSUM @@ -3628,6 +3655,7 @@ or decrypt/decompress just failed. @return whether the operation succeeded @retval DB_SUCCESS if page has been read and is not corrupted @retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted +@retval DB_CORRUPTION if the page LSN is in the future @retval DB_DECRYPTION_FAILED if page post encryption checksum matches but after decryption normal page checksum does not match. */ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, @@ -3664,8 +3692,18 @@ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, node.space->is_compressed())) { err = DB_PAGE_CORRUPTED; } - } else if (buf_page_is_corrupted(true, dst_frame, node.space->flags)) { - err = DB_PAGE_CORRUPTED; + } else { + switch (buf_page_is_corrupted(true, dst_frame, + node.space->flags)) { + case NOT_CORRUPTED: + break; + case CORRUPTED_OTHER: + err = DB_PAGE_CORRUPTED; + break; + case CORRUPTED_FUTURE_LSN: + err = DB_CORRUPTION; + break; + } } if (seems_encrypted && err == DB_PAGE_CORRUPTED @@ -3685,9 +3723,8 @@ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, /** Complete a read of a page. @param node data file @return whether the operation succeeded -@retval DB_PAGE_CORRUPTED if the checksum fails -@retval DB_DECRYPTION_FAILED if the page cannot be decrypted -@retval DB_FAIL if the page contains the wrong ID */ +@retval DB_PAGE_CORRUPTED if the checksum or the page ID is incorrect +@retval DB_DECRYPTION_FAILED if the page cannot be decrypted */ dberr_t buf_page_t::read_complete(const fil_node_t &node) { const page_id_t expected_id{id()}; @@ -3715,9 +3752,8 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node) if (!ok) { - ib::info() << "Page " << expected_id << " zip_decompress failure."; err= DB_PAGE_CORRUPTED; - goto database_corrupted; + goto database_corrupted_compressed; } } @@ -3742,15 +3778,21 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node) node.space->crypt_data && node.space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { - ib::error() << "Cannot decrypt " << expected_id; err= DB_DECRYPTION_FAILED; goto release_page; } else { - ib::error() << "Space id and page no stored in the page, read in are " - << read_id << ", should be " << expected_id; - err= DB_PAGE_CORRUPTED; + sql_print_error("InnoDB: Space id and page no stored in the page," + " read in from %s are " + "[page id: space=" UINT32PF ", page number=" UINT32PF + "], should be " + "[page id: space=" UINT32PF ", page number=" UINT32PF + "]", + node.name, + read_id.space(), read_id.page_no(), + expected_id.space(), expected_id.page_no()); + err= DB_FAIL; goto release_page; } } @@ -3760,24 +3802,9 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node) { database_corrupted: if (belongs_to_unzip_LRU()) +database_corrupted_compressed: memset_aligned(frame, 0, srv_page_size); - if (err == DB_PAGE_CORRUPTED) - { - ib::error() << "Database page corruption on disk" - " or a failed read of file '" - << node.name << "' page " << expected_id - << ". You may have to recover from a backup."; - - buf_page_print(read_frame, zip_size()); - - node.space->set_corrupted(); - - ib::info() << " You can use CHECK TABLE to scan" - " your table for corruption. " - << FORCE_RECOVERY_MSG; - } - if (!srv_force_recovery) goto release_page; } @@ -3785,6 +3812,24 @@ database_corrupted: if (err == DB_PAGE_CORRUPTED || err == DB_DECRYPTION_FAILED) { release_page: + if (recv_sys.free_corrupted_page(expected_id, node)); + else if (err == DB_FAIL) + err= DB_PAGE_CORRUPTED; + else + { + sql_print_error("InnoDB: Failed to read page " UINT32PF + " from file '%s': %s", expected_id.page_no(), + node.name, ut_strerr(err)); + + buf_page_print(read_frame, zip_size()); + + if (node.space->set_corrupted() && + !is_predefined_tablespace(node.space->id)) + sql_print_information("InnoDB: You can use CHECK TABLE to scan" + " your table for corruption. %s", + FORCE_RECOVERY_MSG); + } + buf_pool.corrupted_evict(this, buf_page_t::READ_FIX); return err; } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index c1b4fdd4bf3..31036bf978d 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -33,6 +33,7 @@ Created 2011/12/19 #include "trx0sys.h" #include "fil0crypt.h" #include "fil0pagecompress.h" +#include "log.h" using st_::span; @@ -356,6 +357,13 @@ void buf_dblwr_t::recover() ut_ad(recv_sys.parse_start_lsn); if (!is_created()) return; + const lsn_t max_lsn{log_sys.get_lsn()}; + /* The recv_sys.scanned_lsn may include some "padding" after the + last log record, depending on the value of + innodb_log_write_ahead_size. After MDEV-14425 eliminated + OS_FILE_LOG_BLOCK_SIZE, these two LSN must be equal. */ + ut_ad(recv_sys.scanned_lsn >= max_lsn); + ut_ad(recv_sys.scanned_lsn < max_lsn + 32 * OS_FILE_LOG_BLOCK_SIZE); uint32_t page_no_dblwr= 0; byte *read_buf= static_cast(aligned_malloc(3 * srv_page_size, @@ -365,25 +373,15 @@ void buf_dblwr_t::recover() for (recv_dblwr_t::list::iterator i= recv_sys.dblwr.pages.begin(); i != recv_sys.dblwr.pages.end(); ++i, ++page_no_dblwr) { - byte *page= *i; + const page_t *const page= *i; const uint32_t page_no= page_get_page_no(page); - if (!page_no) /* recovered via recv_dblwr_t::restore_first_page() */ - continue; - const lsn_t lsn= mach_read_from_8(page + FIL_PAGE_LSN); - if (recv_sys.parse_start_lsn > lsn) - /* Pages written before the checkpoint are not useful for recovery. */ + if (recv_sys.parse_start_lsn > lsn || lsn > recv_sys.scanned_lsn) + /* Pages written before or after the recovery range are not usable. */ continue; - const ulint space_id= page_get_space_id(page); + const uint32_t space_id= page_get_space_id(page); const page_id_t page_id(space_id, page_no); - if (recv_sys.scanned_lsn < lsn) - { - ib::info() << "Ignoring a doublewrite copy of page " << page_id - << " with future log sequence number " << lsn; - continue; - } - fil_space_t *space= fil_space_t::get(space_id); if (!space) @@ -395,10 +393,14 @@ void buf_dblwr_t::recover() /* Do not report the warning for undo tablespaces, because they can be truncated in place. */ if (!srv_is_undo_tablespace(space_id)) - ib::warn() << "A copy of page " << page_no - << " in the doublewrite buffer slot " << page_no_dblwr - << " is beyond the end of " << space->chain.start->name - << " (" << space->size << " pages)"; + sql_print_warning("InnoDB: A copy of page " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " in the doublewrite buffer slot " UINT32PF + " is beyond the end of %s (" UINT32PF " pages)", + page_id.space(), page_id.page_no(), + page_no_dblwr, space->chain.start->name, + space->size); next_page: space->release(); continue; @@ -417,41 +419,48 @@ next_page: physical_size, read_buf); if (UNIV_UNLIKELY(fio.err != DB_SUCCESS)) - { - ib::warn() << "Double write buffer recovery: " << page_id - << " ('" << space->chain.start->name - << "') read failed with error: " << fio.err; - continue; - } - - if (buf_is_zeroes(span(read_buf, physical_size))) + sql_print_warning("InnoDB: Double write buffer recovery: " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " ('%s') read failed with error: %s", + page_id.space(), page_id.page_no(), fio.node->name, + ut_strerr(fio.err)); + else if (buf_is_zeroes(span(read_buf, physical_size))) { /* We will check if the copy in the doublewrite buffer is valid. If not, we will ignore this page (there should be redo log records to initialize it). */ } - else if (recv_sys.dblwr.validate_page(page_id, read_buf, space, buf)) + else if (recv_sys.dblwr.validate_page(page_id, max_lsn, space, + read_buf, buf)) goto next_page; else /* We intentionally skip this message for all-zero pages. */ - ib::info() << "Trying to recover page " << page_id - << " from the doublewrite buffer."; + sql_print_information("InnoDB: Trying to recover page " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " from the doublewrite buffer.", + page_id.space(), page_id.page_no()); - page= recv_sys.dblwr.find_page(page_id, space, buf); + if (const byte *page= + recv_sys.dblwr.find_page(page_id, max_lsn, space, buf)) + { + /* Write the good page from the doublewrite buffer to the intended + position. */ + space->reacquire(); + fio= space->io(IORequestWrite, + os_offset_t{page_id.page_no()} * physical_size, + physical_size, const_cast(page)); - if (!page) - goto next_page; + if (fio.err == DB_SUCCESS) + sql_print_information("InnoDB: Recovered page " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " to '%s' from the doublewrite buffer.", + page_id.space(), page_id.page_no(), + fio.node->name); + } - /* Write the good page from the doublewrite buffer to the intended - position. */ - space->reacquire(); - fio= space->io(IORequestWrite, - os_offset_t{page_id.page_no()} * physical_size, - physical_size, page); - - if (fio.err == DB_SUCCESS) - ib::info() << "Recovered page " << page_id << " to '" << fio.node->name - << "' from the doublewrite buffer."; goto next_page; } diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 49a088b425f..d664637fcdc 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1024,7 +1024,7 @@ buf_LRU_block_free_non_file_page( } /** Release a memory block to the buffer pool. */ -ATTRIBUTE_COLD void buf_pool_t::free_block(buf_block_t *block) +ATTRIBUTE_COLD void buf_pool_t::free_block(buf_block_t *block) noexcept { ut_ad(this == &buf_pool); mysql_mutex_lock(&mutex); @@ -1175,13 +1175,12 @@ static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, const page_id_t id, @param bpage x-latched page that was found corrupted @param state expected current state of the page */ ATTRIBUTE_COLD -void buf_pool_t::corrupted_evict(buf_page_t *bpage, uint32_t state) +void buf_pool_t::corrupted_evict(buf_page_t *bpage, uint32_t state) noexcept { const page_id_t id{bpage->id()}; buf_pool_t::hash_chain &chain= buf_pool.page_hash.cell_get(id.fold()); page_hash_latch &hash_lock= buf_pool.page_hash.lock_get(chain); - recv_sys.free_corrupted_page(id); mysql_mutex_lock(&mutex); hash_lock.lock(); diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 6e509b77141..f2ffee0651f 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -326,15 +326,13 @@ buf_read_page_low( dst, bpage); if (UNIV_UNLIKELY(fio.err != DB_SUCCESS)) { + recv_sys.free_corrupted_page(page_id, *space->chain.start); buf_pool.corrupted_evict(bpage, buf_page_t::READ_FIX); } else if (sync) { thd_wait_end(nullptr); /* The i/o was already completed in space->io() */ fio.err = bpage->read_complete(*fio.node); space->release(); - if (fio.err == DB_FAIL) { - fio.err = DB_PAGE_CORRUPTED; - } if (mariadb_timer) { mariadb_increment_pages_read_time(mariadb_timer); } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 38cdaa6aec2..91574035568 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -53,10 +53,14 @@ Created 10/25/1995 Heikki Tuuri # include #endif -ATTRIBUTE_COLD void fil_space_t::set_corrupted() const +ATTRIBUTE_COLD bool fil_space_t::set_corrupted() const noexcept { if (!is_stopping() && !is_corrupted.test_and_set()) + { sql_print_error("InnoDB: File '%s' is corrupted", chain.start->name); + return true; + } + return false; } /** Determine if the space id is a user tablespace id or not. @@ -351,11 +355,14 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, return node; } -__attribute__((warn_unused_result, nonnull)) +__attribute__((warn_unused_result, nonnull(1))) /** Open a tablespace file. @param node data file +@param page first page of the tablespace, or nullptr +@param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the file was successfully opened */ -static bool fil_node_open_file_low(fil_node_t *node) +static bool fil_node_open_file_low(fil_node_t *node, const byte *page, + bool no_lsn) { ut_ad(!node->is_open()); ut_ad(node->space->is_closing()); @@ -405,7 +412,8 @@ static bool fil_node_open_file_low(fil_node_t *node) } if (node->size); - else if (!node->read_page0() || !fil_comp_algo_validate(node->space)) + else if (!node->read_page0(page, no_lsn) || + !fil_comp_algo_validate(node->space)) { #ifndef _WIN32 fail: @@ -425,8 +433,10 @@ static bool fil_node_open_file_low(fil_node_t *node) /** Open a tablespace file. @param node data file +@param page first page of the tablespace, or nullptr +@param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the file was successfully opened */ -static bool fil_node_open_file(fil_node_t *node) +static bool fil_node_open_file(fil_node_t *node, const byte *page, bool no_lsn) { mysql_mutex_assert_owner(&fil_system.mutex); ut_ad(!node->is_open()); @@ -465,7 +475,7 @@ static bool fil_node_open_file(fil_node_t *node) /* The node can be opened beween releasing and acquiring fil_system.mutex in the above code */ - return node->is_open() || fil_node_open_file_low(node); + return node->is_open() || fil_node_open_file_low(node, page, no_lsn); } /** Close the file handle. */ @@ -676,7 +686,8 @@ ATTRIBUTE_COLD bool fil_space_t::prepare_acquired() ut_ad(!id || purpose == FIL_TYPE_TEMPORARY || node == UT_LIST_GET_FIRST(chain)); - const bool is_open= node && (node->is_open() || fil_node_open_file(node)); + const bool is_open= node && + (node->is_open() || fil_node_open_file(node, nullptr, false)); if (!is_open) release(); @@ -1109,8 +1120,10 @@ fil_assign_new_space_id( } /** Read the first page of a data file. +@param dpage copy of a first page, from the doublewrite buffer, or nullptr +@param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the page was found valid */ -bool fil_space_t::read_page0() +bool fil_space_t::read_page0(const byte *dpage, bool no_lsn) noexcept { ut_ad(fil_system.is_initialised()); mysql_mutex_assert_owner(&fil_system.mutex); @@ -1127,31 +1140,25 @@ bool fil_space_t::read_page0() ut_ad("this should not happen" == 0); return false; } - const bool ok= node->is_open() || fil_node_open_file(node); + const bool ok= node->is_open() || fil_node_open_file(node, dpage, no_lsn); release(); return ok; } -/** Look up a tablespace and ensure that its first page has been validated. */ -static fil_space_t *fil_space_get_space(ulint id) -{ - if (fil_space_t *space= fil_space_get_by_id(id)) - if (space->read_page0()) - return space; - return nullptr; -} - void fil_space_set_recv_size_and_flags(ulint id, uint32_t size, uint32_t flags) { ut_ad(id < SRV_SPACE_ID_UPPER_BOUND); + mysql_mutex_assert_owner(&recv_sys.mutex); mysql_mutex_lock(&fil_system.mutex); - if (fil_space_t *space= fil_space_get_space(id)) - { - if (size) - space->recv_size= size; - if (flags != FSP_FLAGS_FCRC32_MASK_MARKER) - space->flags= flags; - } + if (fil_space_t *space= fil_space_get_by_id(id)) + if (space->read_page0(recv_sys.dblwr.find_page(page_id_t(id, 0), LSN_MAX), + true)) + { + if (size) + space->recv_size= size; + if (flags != FSP_FLAGS_FCRC32_MASK_MARKER) + space->flags= flags; + } mysql_mutex_unlock(&fil_system.mutex); } @@ -1166,12 +1173,16 @@ bool fil_space_t::open(bool create_new_db) bool success= true; bool skip_read= create_new_db; + const page_t *page= skip_read + ? nullptr + : recv_sys.dblwr.find_page(page_id_t{id, 0}, LSN_MAX); + mysql_mutex_lock(&fil_system.mutex); for (fil_node_t *node= UT_LIST_GET_FIRST(chain); node; node= UT_LIST_GET_NEXT(chain, node)) { - if (!node->is_open() && !fil_node_open_file_low(node)) + if (!node->is_open() && !fil_node_open_file_low(node, page, page)) { err_exit: success= false; @@ -1189,7 +1200,7 @@ err_exit: continue; } - if (!node->read_page0()) + if (!node->read_page0(page, true)) { fil_system.n_open--; os_file_close(node->handle); @@ -1198,6 +1209,7 @@ err_exit: } skip_read= true; + page= nullptr; } if (!create_new_db) @@ -2930,6 +2942,7 @@ void IORequest::read_complete(int io_error) const { sql_print_error("InnoDB: Read error %d of page " UINT32PF " in file %s", io_error, id.page_no(), node->name); + recv_sys.free_corrupted_page(id, *node); buf_pool.corrupted_evict(bpage, buf_page_t::READ_FIX); corrupted: if (recv_recovery_is_on() && !srv_force_recovery) @@ -2939,13 +2952,8 @@ void IORequest::read_complete(int io_error) const mysql_mutex_unlock(&recv_sys.mutex); } } - else if (dberr_t err= bpage->read_complete(*node)) - { - if (err != DB_FAIL) - ib::error() << "Failed to read page " << id.page_no() - << " from file '" << node->name << "': " << err; + else if (bpage->read_complete(*node)) goto corrupted; - } node->space->release(); } diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index e7f9184c221..8aa3f3dba73 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -234,6 +234,46 @@ Datafile::same_as( #endif /* WIN32 */ } +dberr_t Datafile::read_first_page_flags(const page_t *page) noexcept +{ + ut_ad(m_order == 0); + + if (memcmp_aligned<2>(FIL_PAGE_SPACE_ID + page, + FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4)) + { + sql_print_error("InnoDB: Inconsistent tablespace ID in %s", m_filepath); + return DB_CORRUPTION; + } + + m_space_id= mach_read_from_4(FIL_PAGE_SPACE_ID + page); + m_flags= fsp_header_get_flags(page); + if (!fil_space_t::is_valid_flags(m_flags, m_space_id)) + { + ulint cflags= fsp_flags_convert_from_101(m_flags); + if (unsigned(cflags) == ~0U) + switch (fsp_flags_is_incompatible_mysql(m_flags)) { + case 0: + sql_print_error("InnoDB: Invalid flags 0x%zx in %s", + m_flags, m_filepath); + return DB_CORRUPTION; + case 3: + case 2: + sql_print_error("InnoDB: MySQL-8.0 tablespace in %s", m_filepath); + goto unsupported; + case 1: + sql_print_error("InnoDB: MySQL Encrypted tablespace in %s", + m_filepath); + unsupported: + sql_print_error("InnoDB: Restart in MySQL for migration/recovery."); + return DB_UNSUPPORTED; + } + else + m_flags= cflags; + } + + return DB_SUCCESS; +} + /** Reads a few significant fields from the first page of the first datafile. The Datafile must already be open. @param[in] read_only_mode If true, then readonly mode checks are enforced. @@ -286,56 +326,16 @@ Datafile::read_first_page(bool read_only_mode) } } - if (err != DB_SUCCESS) { - return(err); + if (err == DB_SUCCESS && m_order == 0) { + err = read_first_page_flags(m_first_page); } - if (m_order == 0) { - if (memcmp_aligned<2>(FIL_PAGE_SPACE_ID + m_first_page, - FSP_HEADER_OFFSET + FSP_SPACE_ID - + m_first_page, 4)) { - ib::error() - << "Inconsistent tablespace ID in " - << m_filepath; - return DB_CORRUPTION; - } - - m_space_id = mach_read_from_4(FIL_PAGE_SPACE_ID - + m_first_page); - m_flags = fsp_header_get_flags(m_first_page); - if (!fil_space_t::is_valid_flags(m_flags, m_space_id)) { - ulint cflags = fsp_flags_convert_from_101(m_flags); - if (cflags == ULINT_UNDEFINED) { - switch (fsp_flags_is_incompatible_mysql(m_flags)) { - case 0: - sql_print_error("InnoDB: Invalid flags 0x%zx in %s", - m_flags, m_filepath); - return(DB_CORRUPTION); - case 3: - case 2: - sql_print_error("InnoDB: MySQL-8.0 tablespace in %s", - m_filepath); - break; - case 1: - sql_print_error("InnoDB: MySQL Encrypted tablespace in %s", - m_filepath); - break; - } - sql_print_error("InnoDB: Restart in MySQL for migration/recovery."); - return(DB_UNSUPPORTED); - } else { - m_flags = cflags; - } - } - } - - const size_t physical_size = fil_space_t::physical_size(m_flags); - - if (physical_size > page_size) { + if (err == DB_SUCCESS + && fil_space_t::physical_size(m_flags) > page_size) { ib::error() << "File " << m_filepath << " should be longer than " << page_size << " bytes"; - return(DB_CORRUPTION); + err = DB_CORRUPTION; } return(err); @@ -367,7 +367,7 @@ Datafile::validate_to_dd(ulint space_id, ulint flags) /* Validate this single-table-tablespace with the data dictionary, but do not compare the DATA_DIR flag, in case the tablespace was remotely located. */ - err = validate_first_page(0); + err = validate_first_page(m_first_page); if (err != DB_SUCCESS) { return(err); } @@ -395,6 +395,70 @@ Datafile::validate_to_dd(ulint space_id, ulint flags) return(DB_ERROR); } +uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file) +{ + os_offset_t file_size= os_file_get_size(file); + if (file_size != (os_offset_t) -1) + { + for (const page_t *page : pages) + { + uint32_t space_id= page_get_space_id(page); + byte *read_page= nullptr; + if (page_get_page_no(page) > 0 || space_id == 0) + { +next_page: + aligned_free(read_page); + continue; + } + uint32_t flags= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + page); + size_t page_size= fil_space_t::physical_size(flags); + if (file_size < 4 * page_size) + goto next_page; + read_page= + static_cast(aligned_malloc(3 * page_size, page_size)); + /* Read 3 pages from the file and match the space id + with the space id which is stored in + doublewrite buffer page. */ + if (os_file_read(IORequestRead, file, read_page, page_size, + 3 * page_size, nullptr) != DB_SUCCESS) + goto next_page; + for (ulint j= 0; j <= 2; j++) + { + byte *cur_page= read_page + j * page_size; + if (buf_is_zeroes(span(cur_page, page_size))) + { + aligned_free(read_page); + return 0; + } + if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 || + memcmp(cur_page + FIL_PAGE_SPACE_ID, + page + FIL_PAGE_SPACE_ID, 4) || + buf_page_is_corrupted(false, cur_page, flags)) + goto next_page; + } + + aligned_free(read_page); + page= find_page(page_id_t{space_id, 0}, LSN_MAX); + + if (!page) + { + /* If the first page of the given user tablespace is not there + in the doublewrite buffer, then the recovery is going to fail + now. Report error only when doublewrite buffer is not empty */ + sql_print_error("InnoDB: Corrupted page " + "[page id: space=" UINT32PF ", page number=0]" + " of datafile '%s' could not be found" + " in the doublewrite buffer", space_id, name); + break; + } + + return space_id; + } + } + return 0; +} + /** Validates this datafile for the purpose of recovery. The file should exist and be successfully opened. We initially open it in read-only mode because we just want to read the SpaceID. However, if the first page is @@ -410,7 +474,7 @@ Datafile::validate_for_recovery() ut_ad(is_open()); ut_ad(!srv_read_only_mode); - err = validate_first_page(0); + err = validate_first_page(m_first_page); switch (err) { case DB_TABLESPACE_EXISTS: @@ -427,14 +491,7 @@ Datafile::validate_for_recovery() m_space_id is set in read_first_page(). */ /* fall through */ default: - /* Re-open the file in read-write mode Attempt to restore - page 0 from doublewrite and read the space ID from a survey - of the first few pages. */ - close(); - err = open_read_write(); - if (err != DB_SUCCESS) { - return(err); - } + const page_t *first_page = nullptr; if (!m_space_id) { m_space_id = recv_sys.dblwr.find_first_page( @@ -460,15 +517,17 @@ Datafile::validate_for_recovery() return DB_SUCCESS; /* empty file */ } - if (recv_sys.dblwr.restore_first_page( - m_space_id, m_filepath, m_handle)) { + first_page = recv_sys.dblwr.find_page( + page_id_t(m_space_id, 0), LSN_MAX); + + if (!first_page) { return m_defer ? err : DB_CORRUPTION; } free_first_page: /* Free the previously read first page and then re-validate. */ free_first_page(); m_defer = false; - err = validate_first_page(0); + err = validate_first_page(first_page); } return(err); @@ -478,31 +537,24 @@ free_first_page: tablespace is opened. This occurs before the fil_space_t is created so the Space ID found here must not already be open. m_is_valid is set true on success, else false. -@param[out] flush_lsn contents of FIL_PAGE_FILE_FLUSH_LSN +@param[in] first_page the contents of the first page @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ -dberr_t Datafile::validate_first_page(lsn_t *flush_lsn) +dberr_t Datafile::validate_first_page(const page_t *first_page) { const char* error_txt = NULL; m_is_valid = true; + ut_ad(first_page || !m_first_page); - if (m_first_page == NULL - && read_first_page(srv_read_only_mode) != DB_SUCCESS) { - - error_txt = "Cannot read first page"; - } else { - ut_ad(m_first_page); - - if (flush_lsn != NULL) { - - *flush_lsn = mach_read_from_8( - m_first_page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + if (first_page) { + if (dberr_t err = read_first_page_flags(first_page)) { + m_is_valid = false; + return err; } - } - - if (error_txt != NULL) { + } else if (read_first_page(srv_read_only_mode) != DB_SUCCESS) { + error_txt = "Cannot read first page"; err_exit: free_first_page(); @@ -518,11 +570,14 @@ err_exit: m_space_id, m_flags); m_is_valid = false; return DB_CORRUPTION; + } else { + first_page = m_first_page; + ut_ad(first_page); } /* Check if the whole page is blank. */ if (!m_space_id && !m_flags) { - const byte* b = m_first_page; + const byte* b = first_page; ulint nonzero_bytes = srv_page_size; while (*b == '\0' && --nonzero_bytes != 0) { @@ -560,7 +615,7 @@ err_exit: return(DB_ERROR); } - if (page_get_page_no(m_first_page) != 0) { + if (page_get_page_no(first_page) != 0) { /* First page must be number 0 */ error_txt = "Header page contains inconsistent data"; goto err_exit; @@ -571,8 +626,13 @@ err_exit: goto err_exit; } - if (buf_page_is_corrupted(false, m_first_page, m_flags)) { - /* Look for checksum and other corruptions. */ + switch (buf_page_is_corrupted(false, first_page, m_flags)) { + case NOT_CORRUPTED: + break; + case CORRUPTED_FUTURE_LSN: + error_txt = "LSN is in the future"; + goto err_exit; + case CORRUPTED_OTHER: error_txt = "Checksum mismatch"; goto err_exit; } diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index 9ede0a070f6..2a60f60b43d 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -587,37 +587,43 @@ SysTablespace::read_lsn_and_check_flags(lsn_t* flushed_lsn) /* Check the contents of the first page of the first datafile. */ - err = it->validate_first_page(flushed_lsn); + err = it->validate_first_page(it->m_first_page); + const page_t *first_page = it->m_first_page; if (err != DB_SUCCESS) { - if (recv_sys.dblwr.restore_first_page( - it->m_space_id, it->m_filepath, - it->handle())) { - it->close(); - return(err); + mysql_mutex_lock(&recv_sys.mutex); + first_page = recv_sys.dblwr.find_page( + page_id_t(space_id(), 0), LSN_MAX); + mysql_mutex_unlock(&recv_sys.mutex); + if (!first_page) { + err = DB_CORRUPTION; + } else { + err = it->read_first_page_flags(first_page); + if (err == DB_SUCCESS) { + err = it->validate_first_page(first_page); + } } - err = it->read_first_page( - m_ignore_read_only && srv_read_only_mode); } /* Make sure the tablespace space ID matches the space ID on the first page of the first datafile. */ - if (space_id() != it->m_space_id) { + if (err != DB_SUCCESS) { + } else if (space_id() != it->m_space_id) { ib::error() << "The data file '" << it->filepath() << "' has the wrong space ID. It should be " << space_id() << ", but " << it->m_space_id << " was found"; - - it->close(); - - return(err); + err = DB_CORRUPTION; + } else { + *flushed_lsn = mach_read_from_8( + first_page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); } it->close(); - return(DB_SUCCESS); + return err; } /** Check if a file can be opened in the correct mode. diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 82179e6d646..19e8782395c 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -308,12 +308,20 @@ buf_block_modify_clock_inc( @return whether the buffer is all zeroes */ bool buf_is_zeroes(st_::span buf); +/** Reason why buf_page_is_corrupted() fails */ +enum buf_page_is_corrupted_reason +{ + CORRUPTED_FUTURE_LSN= -1, + NOT_CORRUPTED= 0, + CORRUPTED_OTHER +}; + /** Check if a page is corrupt. @param[in] check_lsn whether the LSN should be checked @param[in] read_buf database page @param[in] fsp_flags tablespace flags @return whether the page is corrupted */ -bool +buf_page_is_corrupted_reason buf_page_is_corrupted( bool check_lsn, const byte* read_buf, @@ -773,9 +781,8 @@ public: /** Complete a read of a page. @param node data file @return whether the operation succeeded - @retval DB_PAGE_CORRUPTED if the checksum fails - @retval DB_DECRYPTION_FAILED if the page cannot be decrypted - @retval DB_FAIL if the page contains the wrong ID */ + @retval DB_PAGE_CORRUPTED if the checksum or the page ID is incorrect + @retval DB_DECRYPTION_FAILED if the page cannot be decrypted */ dberr_t read_complete(const fil_node_t &node); /** Release a write fix after a page write was completed. @@ -1352,10 +1359,11 @@ public: /** Release and evict a corrupted page. @param bpage x-latched page that was found corrupted @param state expected current state of the page */ - ATTRIBUTE_COLD void corrupted_evict(buf_page_t *bpage, uint32_t state); + ATTRIBUTE_COLD void corrupted_evict(buf_page_t *bpage, uint32_t state) + noexcept; /** Release a memory block to the buffer pool. */ - ATTRIBUTE_COLD void free_block(buf_block_t *block); + ATTRIBUTE_COLD void free_block(buf_block_t *block) noexcept; #ifdef UNIV_DEBUG /** Find a block that points to a ROW_FORMAT=COMPRESSED page diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index ecdd386ffc1..7e000147b2f 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -515,8 +515,9 @@ public: written while the space ID is being updated in each page. */ inline void set_imported(); - /** Report the tablespace as corrupted */ - ATTRIBUTE_COLD void set_corrupted() const; + /** Report the tablespace as corrupted + @return whether this was the first call */ + ATTRIBUTE_COLD bool set_corrupted() const noexcept; /** @return whether the storage device is rotational (HDD, not SSD) */ inline bool is_rotational() const; @@ -1030,8 +1031,10 @@ public: void flush_low(); /** Read the first page of a data file. + @param dpage copy of a first page, from the doublewrite buffer, or nullptr + @param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the page was found valid */ - bool read_page0(); + bool read_page0(const byte *dpage, bool no_lsn) noexcept; /** Determine the next tablespace for encryption key rotation. @param space current tablespace (nullptr to start from the beginning) @@ -1128,8 +1131,10 @@ struct fil_node_t final bool is_open() const { return handle != OS_FILE_CLOSED; } /** Read the first page of a data file. + @param dpage copy of a first page, from the doublewrite buffer, or nullptr + @param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the page was found valid */ - bool read_page0(); + bool read_page0(const byte *dpage, bool no_lsn) noexcept; /** Determine some file metadata when creating or reading the file. @param file the file that is being created, or OS_FILE_CLOSED */ @@ -1613,7 +1618,7 @@ inline uint32_t fil_space_t::get_size() if (!size) { mysql_mutex_lock(&fil_system.mutex); - read_page0(); + read_page0(nullptr, false); mysql_mutex_unlock(&fil_system.mutex); } return size; diff --git a/storage/innobase/include/fsp0file.h b/storage/innobase/include/fsp0file.h index 2af2abc1e85..cfea359af17 100644 --- a/storage/innobase/include/fsp0file.h +++ b/storage/innobase/include/fsp0file.h @@ -216,11 +216,11 @@ public: tablespace is opened. This occurs before the fil_space_t is created so the Space ID found here must not already be open. m_is_valid is set true on success, else false. - @param[out] flush_lsn contents of FIL_PAGE_FILE_FLUSH_LSN + @param[in] first_page the contents of the first page @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ - dberr_t validate_first_page(lsn_t* flush_lsn) + dberr_t validate_first_page(const byte *first_page) MY_ATTRIBUTE((warn_unused_result)); /** Get Datafile::m_filepath. @@ -368,6 +368,13 @@ private: dberr_t read_first_page(bool read_only_mode) MY_ATTRIBUTE((warn_unused_result)); + /** Read m_space_id, m_flags from a page frame. + @param page a copy of the first page of the tablespace + @retval DB_SUCCESS if the page seems to be valid + @retval DB_CORRUPTION if the page looks corrupted + @retval DB_UNSUPPORTED if the page is in an unsupported format */ + dberr_t read_first_page_flags(const byte *page) noexcept; + /** Free the first page from memory when it is no longer needed. */ void free_first_page(); diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 542563997cf..773bd4c9f86 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -127,32 +127,28 @@ struct recv_dblwr_t /** Validate the page. @param page_id page identifier - @param page page contents + @param max_lsn the maximum allowed LSN @param space the tablespace of the page (not available for page 0) + @param page page contents @param tmp_buf 2*srv_page_size for decrypting and decompressing any page_compressed or encrypted pages @return whether the page is valid */ - bool validate_page(const page_id_t page_id, const byte *page, - const fil_space_t *space, byte *tmp_buf); + bool validate_page(const page_id_t page_id, lsn_t max_lsn, + const fil_space_t *space, + const byte *page, byte *tmp_buf) const noexcept; - /** Find a doublewrite copy of a page. + /** Find a doublewrite copy of a page with the smallest FIL_PAGE_LSN + that is large enough for recovery. @param page_id page identifier - @param space tablespace (not available for page_id.page_no()==0) + @param max_lsn the maximum allowed LSN + @param space tablespace (nullptr for page_id.page_no()==0) @param tmp_buf 2*srv_page_size for decrypting and decompressing any page_compressed or encrypted pages @return page frame - @retval NULL if no valid page for page_id was found */ - byte* find_page(const page_id_t page_id, const fil_space_t *space= NULL, - byte *tmp_buf= NULL); - - /** Restore the first page of the given tablespace from - doublewrite buffer. - @param space_id tablespace identifier - @param name tablespace filepath - @param file tablespace file handle - @return whether the operation failed */ - bool restore_first_page( - ulint space_id, const char *name, pfs_os_file_t file); + @retval nullptr if no valid page for page_id was found */ + const byte *find_page(const page_id_t page_id, lsn_t max_lsn, + const fil_space_t *space= nullptr, + byte *tmp_buf= nullptr) const noexcept; /** Restore the first page of the given tablespace from doublewrite buffer. @@ -456,20 +452,28 @@ public: inline void free(const void *data); /** Remove records for a corrupted page. - This function should only be called when innodb_force_recovery is set. - @param page_id corrupted page identifier */ - ATTRIBUTE_COLD void free_corrupted_page(page_id_t page_id); + @param page_id corrupted page identifier + @param node file for which an error is to be reported + @return whether an error message was reported */ + ATTRIBUTE_COLD bool free_corrupted_page(page_id_t page_id, + const fil_node_t &node) noexcept; /** Flag data file corruption during recovery. */ - ATTRIBUTE_COLD void set_corrupt_fs(); + ATTRIBUTE_COLD void set_corrupt_fs() noexcept; /** Flag log file corruption during recovery. */ - ATTRIBUTE_COLD void set_corrupt_log(); + ATTRIBUTE_COLD void set_corrupt_log() noexcept; /** @return whether data file corruption was found */ bool is_corrupt_fs() const { return UNIV_UNLIKELY(found_corrupt_fs); } /** @return whether log file corruption was found */ bool is_corrupt_log() const { return UNIV_UNLIKELY(found_corrupt_log); } + /** Check if recovery reached a consistent log sequence number. + @param start_lsn the checkpoint LSN + @param end_lsn the end LSN of the FILE_CHECKPOINT mini-transaction + @return whether the recovery failed to process enough log */ + inline bool validate_checkpoint(lsn_t start_lsn, lsn_t end_lsn) const; + /** Attempt to initialize a page based on redo log records. @param page_id page identifier @return the recovered block @@ -512,11 +516,6 @@ Protected by log_sys.mutex. */ extern bool recv_no_log_write; #endif /* UNIV_DEBUG */ -/** TRUE if buf_page_is_corrupted() should check if the log sequence -number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by -recv_recovery_from_checkpoint_start(). */ -extern bool recv_lsn_checks_on; - /** Size of the parsing buffer; it must accommodate RECV_SCAN_SIZE many times! */ #define RECV_PARSING_BUF_SIZE (2U << 20) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 00b92285677..558316c2e50 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -66,11 +66,6 @@ Protected by log_sys.mutex. */ bool recv_no_log_write = false; #endif /* UNIV_DEBUG */ -/** TRUE if buf_page_is_corrupted() should check if the log sequence -number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by -recv_recovery_from_checkpoint_start(). */ -bool recv_lsn_checks_on; - /** If the following is TRUE, the buffer pool file pages must be invalidated after recovery and no ibuf operations are allowed; this becomes TRUE if the log record hash table becomes too full, and log records must be merged @@ -82,11 +77,6 @@ true means that recovery is running and no operations on the log file are allowed yet: the variable name is misleading. */ bool recv_no_ibuf_operations; -/** The maximum lsn we see for a page during the recovery process. If this -is bigger than the lsn we are able to scan up to, that is an indication that -the recovery failed and the database may be corrupt. */ -static lsn_t recv_max_page_lsn; - /** Stored physical log record */ struct log_phys_t : public log_rec_t { @@ -880,7 +870,7 @@ processed: This is invoked if we found neither a valid first page in the data file nor redo log records that would initialize the first page. */ - void deferred_dblwr() + void deferred_dblwr(lsn_t max_lsn) { for (auto d= defers.begin(); d != defers.end(); ) { @@ -891,7 +881,7 @@ processed: continue; } const page_id_t page_id{d->first, 0}; - const byte *page= recv_sys.dblwr.find_page(page_id); + const byte *page= recv_sys.dblwr.find_page(page_id, max_lsn); if (!page) goto next_item; const uint32_t space_id= mach_read_from_4(page + FIL_PAGE_SPACE_ID); @@ -1477,7 +1467,6 @@ void recv_sys_t::create() progress_time = time(NULL); ut_ad(pages.empty()); pages_it = pages.end(); - recv_max_page_lsn = 0; memset(truncated_undo_spaces, 0, sizeof truncated_undo_spaces); last_stored_lsn = 1; @@ -2787,8 +2776,11 @@ same_page: rewind(l + rlen, log); if (*store == STORE_IF_EXISTS) { - log_sys.set_lsn(recovered_lsn); - log_sys.set_flushed_lsn(recovered_lsn); + if (log_sys.get_lsn() < start_lsn) + { + log_sys.set_lsn(start_lsn); + log_sys.set_flushed_lsn(start_lsn); + } mysql_mutex_unlock(&mutex); this->apply(false); mysql_mutex_lock(&mutex); @@ -2797,9 +2789,9 @@ same_page: } else { - last_stored_lsn= recovered_lsn; + last_stored_lsn= start_lsn; sql_print_information("InnoDB: Multi-batch recovery needed at LSN " - LSN_PF, recovered_lsn); + LSN_PF, start_lsn); *store= STORE_NO; } goto restart; @@ -3114,11 +3106,18 @@ set_start_lsn: mtr.discard_modifications(); mtr.commit(); + fil_space_t* s = space + ? space + : fil_space_t::get(block->page.id().space()); + buf_pool.corrupted_evict(&block->page, block->page.state() & buf_page_t::LRU_MASK); - block = nullptr; - goto done; + if (!space) { + s->release(); + } + + return nullptr; } if (!start_lsn) { @@ -3157,29 +3156,26 @@ set_start_lsn: mtr.discard_modifications(); mtr.commit(); -done: - /* FIXME: do this in page read, protected with recv_sys.mutex! */ - if (recv_max_page_lsn < page_lsn) { - recv_max_page_lsn = page_lsn; - } - return block; } /** Remove records for a corrupted page. -This function should only be called when innodb_force_recovery is set. -@param page_id corrupted page identifier */ -ATTRIBUTE_COLD void recv_sys_t::free_corrupted_page(page_id_t page_id) +@param page_id corrupted page identifier +@param node file for which an error is to be reported +@return whether an error message was reported */ +ATTRIBUTE_COLD +bool recv_sys_t::free_corrupted_page(page_id_t page_id, + const fil_node_t &node) noexcept { if (!recovery_on) - return; + return false; mysql_mutex_lock(&mutex); map::iterator p= pages.find(page_id); if (p == pages.end()) { mysql_mutex_unlock(&mutex); - return; + return false; } p->second.being_processed= -1; @@ -3187,18 +3183,20 @@ ATTRIBUTE_COLD void recv_sys_t::free_corrupted_page(page_id_t page_id) set_corrupt_fs(); mysql_mutex_unlock(&mutex); - ib::error_or_warn(!srv_force_recovery) - << "Unable to apply log to corrupted page " << page_id; + (srv_force_recovery ? sql_print_warning : sql_print_error) + ("InnoDB: Unable to apply log to corrupted page " UINT32PF + " in file %s", page_id.page_no(), node.name); + return true; } -ATTRIBUTE_COLD void recv_sys_t::set_corrupt_log() +ATTRIBUTE_COLD void recv_sys_t::set_corrupt_log() noexcept { mysql_mutex_lock(&mutex); found_corrupt_log= true; mysql_mutex_unlock(&mutex); } -ATTRIBUTE_COLD void recv_sys_t::set_corrupt_fs() +ATTRIBUTE_COLD void recv_sys_t::set_corrupt_fs() noexcept { mysql_mutex_assert_owner(&mutex); if (!srv_force_recovery) @@ -4111,7 +4109,6 @@ recv_group_scan_log_recs( recv_sys.scanned_lsn = *contiguous_lsn; recv_sys.recovered_lsn = *contiguous_lsn; recv_sys.scanned_checkpoint_no = 0; - ut_ad(recv_max_page_lsn == 0); mysql_mutex_unlock(&recv_sys.mutex); lsn_t start_lsn; @@ -4448,6 +4445,19 @@ dberr_t recv_recovery_read_max_checkpoint() return err; } +inline +bool recv_sys_t::validate_checkpoint(lsn_t start_lsn, lsn_t end_lsn) const +{ + if (recovered_lsn >= start_lsn && recovered_lsn >= end_lsn) + return false; + sql_print_error("InnoDB: The log was only scanned up to " + LSN_PF ", while the current LSN at the " + "time of the latest checkpoint " LSN_PF + " was " LSN_PF "!", + recovered_lsn, start_lsn, end_lsn); + return true; +} + /** Start recovering from a redo log checkpoint. @param[in] flush_lsn FIL_PAGE_FILE_FLUSH_LSN of first system tablespace page @@ -4657,11 +4667,15 @@ completed: rescan = true; } + ut_ad(log_sys.get_lsn() >= recv_sys.scanned_lsn); + recv_sys.parse_start_lsn = checkpoint_lsn; if (srv_operation <= SRV_OPERATION_EXPORT_RESTORED) { - deferred_spaces.deferred_dblwr(); + mysql_mutex_lock(&recv_sys.mutex); + deferred_spaces.deferred_dblwr(log_sys.get_lsn()); buf_dblwr.recover(); + mysql_mutex_unlock(&recv_sys.mutex); } ut_ad(srv_force_recovery <= SRV_FORCE_NO_UNDO_LOG_SCAN); @@ -4679,10 +4693,10 @@ completed: } ut_ad(contiguous_lsn <= recv_sys.recovered_lsn); - ut_ad(recv_sys.scanned_lsn == recv_sys.scanned_lsn); - log_sys.set_lsn(recv_sys.recovered_lsn); - log_sys.set_flushed_lsn(recv_sys.recovered_lsn); + ut_ad(log_sys.get_lsn() >= recv_sys.recovered_lsn); + ut_ad(log_sys.get_flushed_lsn() + >= recv_sys.recovered_lsn); /* In case of multi-batch recovery, redo log for the last batch is not @@ -4694,25 +4708,8 @@ completed: } if (!log_sys.is_physical()) { - } else if (recv_sys.recovered_lsn < checkpoint_lsn - || recv_sys.recovered_lsn < end_lsn) { - sql_print_error("InnoDB: The log was only scanned up to " - LSN_PF ", while the current LSN at the " - "time of the latest checkpoint " LSN_PF - " was " LSN_PF "!", - recv_sys.recovered_lsn, - checkpoint_lsn, end_lsn); + } else if (recv_sys.validate_checkpoint(checkpoint_lsn, end_lsn)) { goto err_exit; - } else if (log_sys.log.scanned_lsn < checkpoint_lsn - || log_sys.log.scanned_lsn < end_lsn - || log_sys.log.scanned_lsn < recv_max_page_lsn) { - sql_print_error("InnoDB: We scanned the log up to " LSN_PF - ". A checkpoint was at " LSN_PF - " and the maximum LSN on a database page was " - LSN_PF ". It is possible that the" - " database is now corrupt!", - log_sys.log.scanned_lsn, checkpoint_lsn, - recv_max_page_lsn); } log_sys.next_checkpoint_lsn = checkpoint_lsn; @@ -4755,7 +4752,6 @@ completed: err = recv_rename_files(); } - recv_lsn_checks_on = true; mysql_mutex_unlock(&recv_sys.mutex); /* The database is now ready to start almost normal processing of user @@ -4768,14 +4764,17 @@ completed: goto func_exit; } -bool recv_dblwr_t::validate_page(const page_id_t page_id, - const byte *page, +bool recv_dblwr_t::validate_page(const page_id_t page_id, lsn_t max_lsn, const fil_space_t *space, - byte *tmp_buf) + const byte *page, byte *tmp_buf) + const noexcept { + mysql_mutex_assert_owner(&recv_sys.mutex); + ulint flags; + if (page_id.page_no() == 0) { - ulint flags= fsp_header_get_flags(page); + flags= fsp_header_get_flags(page); if (!fil_space_t::is_valid_flags(flags, page_id.space())) { ulint cflags= fsp_flags_convert_from_101(flags); @@ -4790,9 +4789,15 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, } /* Page 0 is never page_compressed or encrypted. */ - return !buf_page_is_corrupted(true, page, flags); + goto check_if_corrupted; } + flags= space->flags; + + if (space->full_crc32()) + check_if_corrupted: + return !buf_page_is_corrupted(max_lsn < LSN_MAX, page, flags); + ut_ad(tmp_buf); byte *tmp_frame= tmp_buf; byte *tmp_page= tmp_buf + srv_page_size; @@ -4800,9 +4805,6 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, const bool expect_encrypted= space->crypt_data && space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED; - if (space->full_crc32()) - return !buf_page_is_corrupted(true, page, space->flags); - if (expect_encrypted && mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)) { @@ -4829,127 +4831,45 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, return false; /* decompression failed */ if (decomp == srv_page_size) return false; /* the page was not compressed (invalid page type) */ - return !buf_page_is_corrupted(true, tmp_page, space->flags); + page= tmp_page; } - return !buf_page_is_corrupted(true, page, space->flags); + goto check_if_corrupted; } -byte *recv_dblwr_t::find_page(const page_id_t page_id, - const fil_space_t *space, byte *tmp_buf) +const byte *recv_dblwr_t::find_page(const page_id_t page_id, lsn_t max_lsn, + const fil_space_t *space, byte *tmp_buf) + const noexcept { - byte *result= NULL; - lsn_t max_lsn= 0; + mysql_mutex_assert_owner(&recv_sys.mutex); + ut_ad(recv_sys.scanned_lsn <= max_lsn); for (byte *page : pages) { if (page_get_page_no(page) != page_id.page_no() || page_get_space_id(page) != page_id.space()) continue; + const lsn_t lsn= mach_read_from_8(page + FIL_PAGE_LSN); if (page_id.page_no() == 0) { + if (!lsn) + continue; uint32_t flags= mach_read_from_4( FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); if (!fil_space_t::is_valid_flags(flags, page_id.space())) continue; } - const lsn_t lsn= mach_read_from_8(page + FIL_PAGE_LSN); - if (lsn <= max_lsn || - !validate_page(page_id, page, space, tmp_buf)) + if (lsn > max_lsn || lsn < recv_sys.parse_start_lsn || + !validate_page(page_id, max_lsn, space, page, tmp_buf)) { /* Mark processed for subsequent iterations in buf_dblwr_t::recover() */ - memset(page + FIL_PAGE_LSN, 0, 8); + memset_aligned<8>(page + FIL_PAGE_LSN, 0, 8); continue; } - ut_a(page_get_page_no(page) == page_id.page_no()); - max_lsn= lsn; - result= page; + return page; } - return result; -} - -uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file) -{ - os_offset_t file_size= os_file_get_size(file); - if (file_size != (os_offset_t) -1) - { - for (const page_t *page : pages) - { - uint32_t space_id= page_get_space_id(page); - byte *read_page= nullptr; - if (page_get_page_no(page) > 0 || space_id == 0) - { -next_page: - aligned_free(read_page); - continue; - } - uint32_t flags= mach_read_from_4( - FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - page_id_t page_id(space_id, 0); - size_t page_size= fil_space_t::physical_size(flags); - if (file_size < 4 * page_size) - goto next_page; - read_page= - static_cast(aligned_malloc(3 * page_size, page_size)); - /* Read 3 pages from the file and match the space id - with the space id which is stored in - doublewrite buffer page. */ - if (os_file_read(IORequestRead, file, read_page, page_size, - 3 * page_size, nullptr) != DB_SUCCESS) - goto next_page; - for (ulint j= 0; j <= 2; j++) - { - byte *cur_page= read_page + j * page_size; - if (buf_is_zeroes(span(cur_page, page_size))) - { - space_id= 0; - goto early_exit; - } - if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 || - memcmp(cur_page + FIL_PAGE_SPACE_ID, - page + FIL_PAGE_SPACE_ID, 4) || - buf_page_is_corrupted(false, cur_page, flags)) - goto next_page; - } - if (!restore_first_page(space_id, name, file)) - { -early_exit: - aligned_free(read_page); - return space_id; - } - break; - } - } - return 0; -} - -bool recv_dblwr_t::restore_first_page(ulint space_id, const char *name, - pfs_os_file_t file) -{ - const page_id_t page_id(space_id, 0); - const byte* page= find_page(page_id); - if (!page) - { - /* If the first page of the given user tablespace is not there - in the doublewrite buffer, then the recovery is going to fail - now. Report error only when doublewrite buffer is not empty */ - if (pages.size()) - ib::error() << "Corrupted page " << page_id << " of datafile '" - << name <<"' could not be found in the " - <<"doublewrite buffer."; - return true; - } - - ulint physical_size= fil_space_t::physical_size( - mach_read_from_4(page + FSP_HEADER_OFFSET + FSP_SPACE_FLAGS)); - ib::info() << "Restoring page " << page_id << " of datafile '" - << name << "' from the doublewrite buffer. Writing " - << physical_size << " bytes into file '" << name << "'"; - - return os_file_write( - IORequestWrite, name, file, page, 0, physical_size) != - DB_SUCCESS; + return nullptr; } diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 45cb24c251c..90b11f17ae5 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -4118,9 +4118,10 @@ void fil_node_t::find_metadata(os_file_t file /** Read the first page of a data file. @return whether the page was found valid */ -bool fil_node_t::read_page0() +bool fil_node_t::read_page0(const byte *dpage, bool no_lsn) noexcept { mysql_mutex_assert_owner(&fil_system.mutex); + ut_ad(!dpage || no_lsn); const unsigned psize= space->physical_size(); #ifndef _WIN32 struct stat statbuf; @@ -4144,15 +4145,18 @@ bool fil_node_t::read_page0() if (!deferred) { - page_t *page= static_cast(aligned_malloc(psize, psize)); - if (os_file_read(IORequestRead, handle, page, 0, psize, nullptr) - != DB_SUCCESS) + page_t *apage= static_cast(aligned_malloc(psize, psize)); + if (os_file_read(IORequestRead, handle, apage, 0, psize, nullptr) != + DB_SUCCESS) { sql_print_error("InnoDB: Unable to read first page of file %s", name); - aligned_free(page); + err_exit: + aligned_free(apage); return false; } + const page_t *page= apage; + retry: const ulint space_id= memcmp_aligned<2> (FIL_PAGE_SPACE_ID + page, FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4) @@ -4160,8 +4164,16 @@ bool fil_node_t::read_page0() : mach_read_from_4(FIL_PAGE_SPACE_ID + page); ulint flags= fsp_header_get_flags(page); const uint32_t size= fsp_header_get_field(page, FSP_SIZE); + if (!space_id && !flags && !size && dpage) + { + retry_dpage: + page= dpage; + dpage= nullptr; + goto retry; + } const uint32_t free_limit= fsp_header_get_field(page, FSP_FREE_LIMIT); const uint32_t free_len= flst_get_len(FSP_HEADER_OFFSET + FSP_FREE + page); + if (!fil_space_t::is_valid_flags(flags, space->id)) { ulint cflags= fsp_flags_convert_from_101(flags); @@ -4178,7 +4190,6 @@ bool fil_node_t::read_page0() } } - aligned_free(page); goto invalid; } @@ -4187,28 +4198,40 @@ bool fil_node_t::read_page0() !fil_space_t::is_flags_equal((space->flags & ~FSP_FLAGS_MEM_MASK), (flags & ~FSP_FLAGS_MEM_MASK))) { -invalid: + invalid: + if (dpage) + goto retry_dpage; sql_print_error("InnoDB: Expected tablespace flags 0x%zx but found 0x%zx" " in the file %s", space->flags, flags, name); - return false; + goto err_exit; } flags_ok: ut_ad(!(flags & FSP_FLAGS_MEM_MASK)); + if (buf_page_is_corrupted(!no_lsn, page, flags) != NOT_CORRUPTED) + { + if (dpage) + goto retry_dpage; + sql_print_error("InnoDB: The first page of file %s is corrupted", name); + goto err_exit; + } + + if (UNIV_UNLIKELY(space_id != space->id)) + { + if (dpage) + goto retry_dpage; + sql_print_error("InnoDB: Expected tablespace id %zu but found %zu" + " in the file %s", ulint{space->id}, ulint{space_id}, + name); + goto err_exit; + } + /* Try to read crypt_data from page 0 if it is not yet read. */ if (!space->crypt_data) space->crypt_data= fil_space_read_crypt_data( fil_space_t::zip_size(flags), page); - aligned_free(page); - - if (UNIV_UNLIKELY(space_id != space->id)) - { - ib::error() << "Expected tablespace id " << space->id - << " but found " << space_id - << " in the file " << name; - return false; - } + aligned_free(apage); space->flags= (space->flags & FSP_FLAGS_MEM_MASK) | flags; ut_ad(space->free_limit == 0 || space->free_limit == free_limit); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 4d826bb0412..303c4a44f77 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -509,13 +509,14 @@ static ulint srv_undo_tablespace_open(bool create, const char* name, ulint i) ulint n_retries = 5; os_offset_t size= os_file_get_size(fh); ut_a(size != os_offset_t(-1)); + page_t *apage= nullptr; + const page_t *page= nullptr; if (!create) { - page_t *page= static_cast(aligned_malloc(srv_page_size, - srv_page_size)); + apage= static_cast(aligned_malloc(srv_page_size, srv_page_size)); undo_retry: - if (os_file_read(IORequestRead, fh, page, 0, srv_page_size, nullptr) != + if (os_file_read(IORequestRead, fh, apage, 0, srv_page_size, nullptr) != DB_SUCCESS) { err_exit: @@ -526,11 +527,12 @@ err_exit: n_retries--; goto undo_retry; } - ib::error() << "Unable to read first page of file " << name; - aligned_free(page); + sql_print_error("InnoDB: Unable to read first page of file %s", name); + aligned_free(apage); return ULINT_UNDEFINED; } + page= apage; DBUG_EXECUTE_IF("undo_space_read_fail", goto err_exit;); uint32_t id= mach_read_from_4(FIL_PAGE_SPACE_ID + page); @@ -538,18 +540,30 @@ err_exit: memcmp_aligned<2>(FIL_PAGE_SPACE_ID + page, FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4)) { - ib::error() << "Inconsistent tablespace ID in file " << name; + sql_print_error("InnoDB: Inconsistent tablespace ID in file %s", name); goto err_exit; } space_id= id; fsp_flags= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - if (buf_page_is_corrupted(false, page, fsp_flags) && - recv_sys.dblwr.restore_first_page(space_id, name, fh)) - goto err_exit; + if (buf_page_is_corrupted(false, page, fsp_flags)) + { + page= recv_sys.dblwr.find_page(page_id_t{space_id, 0}, LSN_MAX); + if (!page) + { + /* If the first page of the given user tablespace is not there + in the doublewrite buffer, then the recovery is going to fail + now. Report error only when doublewrite buffer is not empty */ + sql_print_error("InnoDB: Corrupted page " + "[page id: space=" UINT32PF ", page number=0]" + " of datafile '%s' could not be found" + " in the doublewrite buffer", space_id, name); + goto err_exit; + } - aligned_free(page); + fsp_flags= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + } } /* Load the tablespace into InnoDB's internal data structures. */ @@ -572,7 +586,7 @@ err_exit: space->set_sizes(SRV_UNDO_TABLESPACE_SIZE_IN_PAGES); space->size= file->size= uint32_t(size >> srv_page_size_shift); } - else if (!file->read_page0()) + else if (!file->read_page0(page, true)) { os_file_close(file->handle); file->handle= OS_FILE_CLOSED; @@ -581,6 +595,7 @@ err_exit: } mysql_mutex_unlock(&fil_system.mutex); + aligned_free(apage); return space_id; } @@ -743,29 +758,30 @@ srv_undo_tablespaces_init(bool create_new_db) srv_operation == SRV_OPERATION_RESTORE_DELTA) ? srv_undo_tablespaces : TRX_SYS_N_RSEGS; - if (dberr_t err= srv_all_undo_tablespaces_open(create_new_db, n_undo)) - return err; + mysql_mutex_lock(&recv_sys.mutex); + dberr_t err= srv_all_undo_tablespaces_open(create_new_db, n_undo); + mysql_mutex_unlock(&recv_sys.mutex); /* Initialize srv_undo_space_id_start=0 when there are no dedicated undo tablespaces. */ if (srv_undo_tablespaces_open == 0) srv_undo_space_id_start= 0; - if (create_new_db) + if (err == DB_SUCCESS && create_new_db) { mtr_t mtr; for (ulint i= 0; i < srv_undo_tablespaces; ++i) { mtr.start(); - dberr_t err= fsp_header_init(fil_space_get(srv_undo_space_id_start + i), - SRV_UNDO_TABLESPACE_SIZE_IN_PAGES, &mtr); + err= fsp_header_init(fil_space_get(srv_undo_space_id_start + i), + SRV_UNDO_TABLESPACE_SIZE_IN_PAGES, &mtr); mtr.commit(); if (err) - return err; + break; } } - return DB_SUCCESS; + return err; } /** Create the temporary file tablespace. @@ -1278,13 +1294,17 @@ dberr_t srv_start(bool create_new_db) return srv_init_abort(err); } - /* Open data files for system tablespace. */ - if (!fil_system.sys_space->open(false)) { - return srv_init_abort(DB_ERROR); - } + mysql_mutex_lock(&recv_sys.mutex); + bool all_opened = fil_system.sys_space->open(false); + mysql_mutex_unlock(&recv_sys.mutex); - /* Open undo tablespaces. */ - err = srv_undo_tablespaces_init(false); + /* Open data files for system tablespace. */ + if (!all_opened) { + err = DB_ERROR; + } else { + /* Open undo tablespaces. */ + err = srv_undo_tablespaces_init(false); + } if (err != DB_SUCCESS) { return srv_init_abort(err); } @@ -1393,13 +1413,17 @@ dberr_t srv_start(bool create_new_db) flushed_lsn = log_sys.get_lsn(); } - /* Open data files in the systemtablespace: we keep - them open until database shutdown */ + /* Open data files in the system tablespace: we keep + them open until database shutdown */ + mysql_mutex_lock(&recv_sys.mutex); ut_d(fil_system.sys_space->recv_size = srv_sys_space_size_debug); - err = fil_system.sys_space->open(create_new_db) - ? srv_undo_tablespaces_init(create_new_db) - : DB_ERROR; + ? DB_SUCCESS : DB_ERROR; + mysql_mutex_unlock(&recv_sys.mutex); + + if (err == DB_SUCCESS) { + err = srv_undo_tablespaces_init(create_new_db); + } /* If the force recovery is set very high then we carry on regardless of all errors. Basically this is fingers crossed mode. */