diff --git a/client/mysqldump.c b/client/mysqldump.c index a09720dac72..9d3e9d046cb 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -4270,7 +4270,7 @@ static void dump_table(const char *table, const char *db, const uchar *hash_key, if (versioned && !opt_xml && opt_dump_history) { - fprintf(md_result_file,"/*!101100 SET @old_system_versioning_insert_history=@@session.system_versioning_insert_history, @@session.system_versioning_insert_history=1 */;\n"); + fprintf(md_result_file,"/*M!101100 SET @old_system_versioning_insert_history=@@session.system_versioning_insert_history, @@session.system_versioning_insert_history=1 */;\n"); check_io(md_result_file); } if (opt_lock) @@ -4572,7 +4572,7 @@ static void dump_table(const char *table, const char *db, const uchar *hash_key, } if (versioned && !opt_xml && opt_dump_history) { - fprintf(md_result_file,"/*!101100 SET system_versioning_insert_history=@old_system_versioning_insert_history */;\n"); + fprintf(md_result_file,"/*M!101100 SET system_versioning_insert_history=@old_system_versioning_insert_history */;\n"); check_io(md_result_file); } mysql_free_result(res); diff --git a/cmake/libfmt.cmake b/cmake/libfmt.cmake index 78a0df6f911..a9dc6937b86 100644 --- a/cmake/libfmt.cmake +++ b/cmake/libfmt.cmake @@ -15,8 +15,8 @@ MACRO(BUNDLE_LIBFMT) ExternalProject_Add( libfmt PREFIX "${dir}" - URL "https://github.com/fmtlib/fmt/releases/download/11.0.1/fmt-11.0.1.zip" - URL_MD5 5f3915e2eff60e7f70c558120592100d + URL "https://github.com/fmtlib/fmt/releases/download/11.0.2/fmt-11.0.2.zip" + URL_MD5 c622dca45ec3fc95254c48370a9f7a1d INSTALL_COMMAND "" CONFIGURE_COMMAND "" BUILD_COMMAND "" diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index 94c21544365..ec2b2e25db9 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -519,7 +519,7 @@ static bool is_page_corrupted(byte *buf, bool is_encrypted, uint32_t flags) normal method. */ if (is_encrypted && key_version != 0) { is_corrupted = use_full_crc32 - ? buf_page_is_corrupted(true, buf, flags) + ? !!buf_page_is_corrupted(false, buf, flags) : !fil_space_verify_crypt_checksum(buf, zip_size); if (is_corrupted && log_file) { diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc index be871e4a65f..647432b15d4 100644 --- a/extra/mariabackup/fil_cur.cc +++ b/extra/mariabackup/fil_cur.cc @@ -283,7 +283,7 @@ static bool page_is_corrupted(const byte *page, ulint page_no, } if (space->full_crc32()) { - return buf_page_is_corrupted(true, page, space->flags); + return buf_page_is_corrupted(false, page, space->flags); } /* Validate encrypted pages. The first page is never encrypted. @@ -316,7 +316,7 @@ static bool page_is_corrupted(const byte *page, ulint page_no, } if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - return buf_page_is_corrupted(true, tmp_page, + return buf_page_is_corrupted(false, tmp_page, space->flags); } } @@ -336,11 +336,11 @@ static bool page_is_corrupted(const byte *page, ulint page_no, && cursor->zip_size) || page_type == FIL_PAGE_PAGE_COMPRESSED || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED - || buf_page_is_corrupted(true, tmp_page, + || buf_page_is_corrupted(false, tmp_page, space->flags)); } - return buf_page_is_corrupted(true, page, space->flags); + return buf_page_is_corrupted(false, page, space->flags); } PRAGMA_REENABLE_CHECK_STACK_FRAME diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 3ae7be0337b..9adf7ef43c3 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -3426,7 +3426,8 @@ static bool xtrabackup_copy_mmap_logfile() recv_sys_t::parse_mtr_result r; const byte *start= &log_sys.buf[recv_sys.offset]; - if (recv_sys.parse_mmap(false) == recv_sys_t::OK) + if (recv_sys.parse_mmap(false) == + recv_sys_t::OK) { const byte *end; @@ -3446,7 +3447,8 @@ static bool xtrabackup_copy_mmap_logfile() start = seq + 1; } } - while ((r= recv_sys.parse_mmap(false)) == recv_sys_t::OK); + while ((r= recv_sys.parse_mmap(false)) == + recv_sys_t::OK); end= &log_sys.buf[recv_sys.offset]; @@ -3551,7 +3553,8 @@ static bool xtrabackup_copy_logfile() if (log_sys.buf[recv_sys.offset] <= 1) break; - if (recv_sys.parse_mtr(false) == recv_sys_t::OK) + if (recv_sys.parse_mtr(false) == + recv_sys_t::OK) { do { @@ -3561,7 +3564,8 @@ static bool xtrabackup_copy_logfile() sequence_offset)); *seq= 1; } - while ((r= recv_sys.parse_mtr(false)) == recv_sys_t::OK); + while ((r= recv_sys.parse_mtr(false)) == + recv_sys_t::OK); if (ds_write(dst_log_file, log_sys.buf + start_offset, recv_sys.offset - start_offset)) @@ -3929,7 +3933,7 @@ static void xb_load_single_table_tablespace(const char *dirname, for (int i = 0; i < 10; i++) { file->m_defer = false; - err = file->validate_first_page(); + err = file->validate_first_page(file->get_first_page()); if (file->m_defer) { if (defer_space_id) { @@ -3971,7 +3975,7 @@ static void xb_load_single_table_tablespace(const char *dirname, skip_node_page0 ? file->detach() : pfs_os_file_t(), 0, false, false); node->deferred= defer; - if (!space->read_page0()) + if (!space->read_page0(nullptr, true)) err = DB_CANNOT_OPEN_FILE; mysql_mutex_unlock(&fil_system.mutex); @@ -6863,8 +6867,10 @@ error: goto error; } - ok = fil_system.sys_space->open(false) - && xtrabackup_apply_deltas(); + mysql_mutex_lock(&recv_sys.mutex); + ok = fil_system.sys_space->open(false); + mysql_mutex_unlock(&recv_sys.mutex); + if (ok) ok = xtrabackup_apply_deltas(); xb_data_files_close(); diff --git a/mysql-test/main/ctype_utf32.result b/mysql-test/main/ctype_utf32.result index 321d6a98683..17f9391667d 100644 --- a/mysql-test/main/ctype_utf32.result +++ b/mysql-test/main/ctype_utf32.result @@ -3025,6 +3025,15 @@ HEX(DATE_FORMAT(TIME'11:22:33',@format)) # End of 10.4 tests # # +# MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32 +# +SELECT CAST(CONVERT('-9223372036854775808' USING utf32) AS SIGNED) AS c1; +c1 +-9223372036854775808 +# +# End of 10.5 tests +# +# # Start of 10.11 tests # # @@ -3109,12 +3118,3 @@ SET NAMES utf8mb4; # # End of 10.11 tests # -# -# MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32 -# -SELECT CAST(CONVERT('-9223372036854775808' USING utf32) AS SIGNED) AS c1; -c1 --9223372036854775808 -# -# End of 10.5 tests -# diff --git a/mysql-test/main/ctype_utf32.test b/mysql-test/main/ctype_utf32.test index 75c9e3e84f8..1542cbaee62 100644 --- a/mysql-test/main/ctype_utf32.test +++ b/mysql-test/main/ctype_utf32.test @@ -1161,6 +1161,16 @@ SELECT HEX(DATE_FORMAT(TIME'11:22:33',@format)); --echo # End of 10.4 tests --echo # +--echo # +--echo # MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32 +--echo # + +SELECT CAST(CONVERT('-9223372036854775808' USING utf32) AS SIGNED) AS c1; + +--echo # +--echo # End of 10.5 tests +--echo # + --echo # --echo # Start of 10.11 tests --echo # @@ -1238,14 +1248,3 @@ SET NAMES utf8mb4; --echo # --enable_service_connection - - ---echo # ---echo # MDEV-31221 UBSAN runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int' in my_strtoll10_utf32 ---echo # - -SELECT CAST(CONVERT('-9223372036854775808' USING utf32) AS SIGNED) AS c1; - ---echo # ---echo # End of 10.5 tests ---echo # diff --git a/mysql-test/main/delete.test b/mysql-test/main/delete.test index 5bc2b4812eb..9e206041090 100644 --- a/mysql-test/main/delete.test +++ b/mysql-test/main/delete.test @@ -633,7 +633,9 @@ drop table t1,t2,t3; --echo # SET @sort_buffer_size_save= @@sort_buffer_size; +--disable_warnings SET sort_buffer_size=1125899906842624; +--enable_warnings CREATE TABLE t1 (a INT,b CHAR,KEY(a,b)); DELETE a1 FROM t1 AS a1,t1 AS a2 WHERE a1.a=a2.a; DROP TABLE t1; diff --git a/mysql-test/main/mysql57nopart.result b/mysql-test/main/mysql57nopart.result new file mode 100644 index 00000000000..2b902f27947 --- /dev/null +++ b/mysql-test/main/mysql57nopart.result @@ -0,0 +1,13 @@ +# +# MDEV-35079 Migrate MySQL5.7 to MariaDB 10.4, then to MariaDB 10.11 Failed +# +select table_name, concat('<',table_comment,'>') from information_schema.tables +where table_schema='test'; +table_name concat('<',table_comment,'>') +mysql57part +Warnings: +Warning 1290 The MariaDB server is running with the --skip-partition option so it cannot execute this statement +drop table mysql57part; +Warnings: +Warning 1932 Table 'test.mysql57part' doesn't exist in engine +# End of 10.11 tests diff --git a/mysql-test/main/mysql57nopart.test b/mysql-test/main/mysql57nopart.test new file mode 100644 index 00000000000..93df68cee5f --- /dev/null +++ b/mysql-test/main/mysql57nopart.test @@ -0,0 +1,11 @@ +source include/have_innodb.inc; +--echo # +--echo # MDEV-35079 Migrate MySQL5.7 to MariaDB 10.4, then to MariaDB 10.11 Failed +--echo # +let $datadir=`select @@datadir`; +copy_file std_data/mysql57part.frm $datadir/test/mysql57part.frm; +select table_name, concat('<',table_comment,'>') from information_schema.tables + where table_schema='test'; +drop table mysql57part; + +--echo # End of 10.11 tests diff --git a/mysql-test/std_data/mysql57part.frm b/mysql-test/std_data/mysql57part.frm new file mode 100644 index 00000000000..9a2e5779973 Binary files /dev/null and b/mysql-test/std_data/mysql57part.frm differ diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index d2ad5fb9909..5fe5eb27fc5 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -3,8 +3,10 @@ call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` has an unreadable root page"); call mtr.add_suppression("Table .*t[12].* is corrupted"); call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted; key_version=1"); -call mtr.add_suppression("failed to read \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]"); +call mtr.add_suppression("InnoDB: Failed to read page [1-9][0-9]* from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t1.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t[12]` is corrupted"); call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found"); call mtr.add_suppression("File '.*mariadb-test.std_data.keysbad3\\.txt' not found"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result index 129de6e9c46..47bb9bf97d0 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result @@ -3,7 +3,9 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page n call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded\\."); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("InnoDB: Cannot delete tablespace .* because it is not found in the tablespace memory cache"); call mtr.add_suppression("InnoDB: ALTER TABLE `test`\\.`t1` DISCARD TABLESPACE failed to find tablespace"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot decrypt \\[page id: space="); diff --git a/mysql-test/suite/encryption/r/innodb-compressed-blob.result b/mysql-test/suite/encryption/r/innodb-compressed-blob.result index 3fc44be95aa..c8294763f7e 100644 --- a/mysql-test/suite/encryption/r/innodb-compressed-blob.result +++ b/mysql-test/suite/encryption/r/innodb-compressed-blob.result @@ -1,7 +1,7 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted; key_version=1"); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Unable to decompress ..test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test/t[12]\\.ibd' page \\[page id: space=[1-9][0-9]*, page number=3\\]"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); call mtr.add_suppression("Table `test`\\.`t[12]` is corrupted"); # Restart mysqld --file-key-management-filename=keys2.txt diff --git a/mysql-test/suite/encryption/r/innodb-encryption-disable.result b/mysql-test/suite/encryption/r/innodb-encryption-disable.result index 86c6d63649f..13bd8ac4a34 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-disable.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-disable.result @@ -2,6 +2,8 @@ call mtr.add_suppression("Table `test`\\.`t[15]` (has an unreadable root page|is call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[15].ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[15]\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t[15]\\.ibd' is corrupted"); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); # restart: --innodb-encrypt-tables=ON --plugin-load-add=file_key_management --file-key-management --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt create table t5 ( diff --git a/mysql-test/suite/encryption/r/innodb-force-corrupt.result b/mysql-test/suite/encryption/r/innodb-force-corrupt.result index 40b85bebabb..e559ddc066f 100644 --- a/mysql-test/suite/encryption/r/innodb-force-corrupt.result +++ b/mysql-test/suite/encryption/r/innodb-force-corrupt.result @@ -1,7 +1,10 @@ -call mtr.add_suppression("Table `test`\\.`t[13]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("Table `test`\\.`t[123]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("InnoDB: File '.*test/t[123]\\.ibd' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[13]\\.ibd'"); +call mtr.add_suppression("InnoDB: Failed to read page 6 from file '.*test/t2\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version="); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption in an InnoDB type table"); -call mtr.add_suppression("\\[ERROR\\] (mysqld|mariadbd).*: Index for table 't2' is corrupt; try to repair it"); +call mtr.add_suppression("\\[ERROR\\] mariadbd.*: Index for table 't2' is corrupt; try to repair it"); set global innodb_compression_algorithm = 1; # Create and populate tables to be corrupted CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT,c char(200)) ENGINE=InnoDB encrypted=yes; diff --git a/mysql-test/suite/encryption/r/innodb-missing-key.result b/mysql-test/suite/encryption/r/innodb-missing-key.result index 83c9166d05b..14fb0275f1c 100644 --- a/mysql-test/suite/encryption/r/innodb-missing-key.result +++ b/mysql-test/suite/encryption/r/innodb-missing-key.result @@ -2,7 +2,9 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` has an unreadable root pag call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file .*test.t[12].ibd looks corrupted; key_version=1"); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); # Start server with keys2.txt # restart: --file-key-management-filename=MYSQL_TEST_DIR/std_data/keys2.txt CREATE TABLE t1(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=19; diff --git a/mysql-test/suite/encryption/r/innodb-redo-badkey.result b/mysql-test/suite/encryption/r/innodb-redo-badkey.result index 6b8f6aa11da..e2c2e8423c8 100644 --- a/mysql-test/suite/encryption/r/innodb-redo-badkey.result +++ b/mysql-test/suite/encryption/r/innodb-redo-badkey.result @@ -23,6 +23,7 @@ insert into t2 select * from t1; insert into t3 select * from t1; insert into t4 select * from t1; commit; +set global innodb_log_checkpoint_now=on; SET GLOBAL innodb_flush_log_at_trx_commit=1; begin; update t1 set c = repeat('secret3', 20); diff --git a/mysql-test/suite/encryption/t/corrupted_during_recovery.test b/mysql-test/suite/encryption/t/corrupted_during_recovery.test index e4a31a0b478..dabf06dd047 100644 --- a/mysql-test/suite/encryption/t/corrupted_during_recovery.test +++ b/mysql-test/suite/encryption/t/corrupted_during_recovery.test @@ -5,11 +5,11 @@ call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); call mtr.add_suppression("InnoDB: Failed to read page [123] from file '.*test.t1\\.ibd': Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("InnoDB: The page \\[page id: space=\\d+, page number=3\\] in file '.*test.t1\\.ibd' cannot be decrypted"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table in tablespace \\d+ encrypted. However key management plugin or used key_version \\d+ is not found or used encryption algorithm or method does not match. Can't continue opening the table."); -call mtr.add_suppression("InnoDB: (Unable to apply log to|Discarding log for) corrupted page "); +call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page "); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index a5060cb8195..3ca97596840 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -13,8 +13,10 @@ call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); call mtr.add_suppression("InnoDB: Table `test`\\.`t[12]` has an unreadable root page"); call mtr.add_suppression("Table .*t[12].* is corrupted"); call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted; key_version=1"); -call mtr.add_suppression("failed to read \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]"); +call mtr.add_suppression("InnoDB: Failed to read page [1-9][0-9]* from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t1.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t[12]` is corrupted"); call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found"); call mtr.add_suppression("File '.*mariadb-test.std_data.keysbad3\\.txt' not found"); # for innodb_checksum_algorithm=full_crc32 only diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test index ac8e18b4980..eec4872c0a0 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test @@ -14,7 +14,9 @@ call mtr.add_suppression("InnoDB: Recovery failed to read page"); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded\\."); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); call mtr.add_suppression("InnoDB: Cannot delete tablespace .* because it is not found in the tablespace memory cache"); call mtr.add_suppression("InnoDB: ALTER TABLE `test`\\.`t1` DISCARD TABLESPACE failed to find tablespace"); # for innodb_checksum_algorithm=full_crc32 only diff --git a/mysql-test/suite/encryption/t/innodb-compressed-blob.test b/mysql-test/suite/encryption/t/innodb-compressed-blob.test index 83aac811f7c..091ed4003a8 100644 --- a/mysql-test/suite/encryption/t/innodb-compressed-blob.test +++ b/mysql-test/suite/encryption/t/innodb-compressed-blob.test @@ -7,7 +7,7 @@ call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted; key_version=1"); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Unable to decompress ..test.t[12]\\.ibd\\[page id: space=[1-9][0-9]*, page number=[0-9]+\\]"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test/t[12]\\.ibd' page \\[page id: space=[1-9][0-9]*, page number=3\\]"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[12]\\.ibd'"); call mtr.add_suppression("InnoDB: File '.*test/t[12]\\.ibd' is corrupted"); call mtr.add_suppression("Table `test`\\.`t[12]` is corrupted"); diff --git a/mysql-test/suite/encryption/t/innodb-encryption-disable.test b/mysql-test/suite/encryption/t/innodb-encryption-disable.test index 7a7e590e304..676edd54f98 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-disable.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-disable.test @@ -11,6 +11,8 @@ call mtr.add_suppression("Table `test`\\.`t[15]` (has an unreadable root page|is call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[15].ibd looks corrupted; key_version=1"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[15]\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t[15]\\.ibd' is corrupted"); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); diff --git a/mysql-test/suite/encryption/t/innodb-force-corrupt.test b/mysql-test/suite/encryption/t/innodb-force-corrupt.test index 32205decf09..3cab1081990 100644 --- a/mysql-test/suite/encryption/t/innodb-force-corrupt.test +++ b/mysql-test/suite/encryption/t/innodb-force-corrupt.test @@ -7,10 +7,13 @@ # Don't test under embedded -- source include/not_embedded.inc -call mtr.add_suppression("Table `test`\\.`t[13]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("Table `test`\\.`t[123]` (has an unreadable root page|is corrupted)"); +call mtr.add_suppression("InnoDB: File '.*test/t[123]\\.ibd' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t[13]\\.ibd'"); +call mtr.add_suppression("InnoDB: Failed to read page 6 from file '.*test/t2\\.ibd'"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version="); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption in an InnoDB type table"); -call mtr.add_suppression("\\[ERROR\\] (mysqld|mariadbd).*: Index for table 't2' is corrupt; try to repair it"); +call mtr.add_suppression("\\[ERROR\\] mariadbd.*: Index for table 't2' is corrupt; try to repair it"); set global innodb_compression_algorithm = 1; diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.test b/mysql-test/suite/encryption/t/innodb-missing-key.test index 0c7a1df9ae2..07e22484113 100644 --- a/mysql-test/suite/encryption/t/innodb-missing-key.test +++ b/mysql-test/suite/encryption/t/innodb-missing-key.test @@ -11,7 +11,9 @@ call mtr.add_suppression("InnoDB: Table `test`\\.`t1` has an unreadable root pag call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("InnoDB: Recovery failed to read page"); call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file .*test.t[12].ibd looks corrupted; key_version=1"); -call mtr.add_suppression("Table .*t1.* is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); +call mtr.add_suppression("Table `test`\\.`t1` is corrupted"); --echo # Start server with keys2.txt -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-redo-badkey.test b/mysql-test/suite/encryption/t/innodb-redo-badkey.test index de6d7f2f253..bacc71dd2c8 100644 --- a/mysql-test/suite/encryption/t/innodb-redo-badkey.test +++ b/mysql-test/suite/encryption/t/innodb-redo-badkey.test @@ -52,6 +52,8 @@ insert into t3 select * from t1; insert into t4 select * from t1; commit; +set global innodb_log_checkpoint_now=on; + --source ../../suite/innodb/include/no_checkpoint_start.inc # diff --git a/mysql-test/suite/innodb/r/corrupted_during_recovery.result b/mysql-test/suite/innodb/r/corrupted_during_recovery.result index 2cab795f6a1..593943b4951 100644 --- a/mysql-test/suite/innodb/r/corrupted_during_recovery.result +++ b/mysql-test/suite/innodb/r/corrupted_during_recovery.result @@ -8,8 +8,10 @@ INSERT INTO t2 VALUES(1); # Corrupt the pages SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' +FOUND 1 /InnoDB: Page \[page id: space=[1-9][0-9]*, page number=3\] log sequence number 1311768467463790320 is in the future!/ in mysqld.1.err SELECT * FROM t1; -Got one of the listed errors +a +1 SELECT * FROM t2; a 1 @@ -27,7 +29,7 @@ SET GLOBAL innodb_flush_log_at_trx_commit=1; DELETE FROM t1 WHERE pk=3; # Kill the server disconnect con1; -# Corrupt the pages +# Corrupt the page SELECT * FROM t1; pk 1 diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result index 65ff203a28e..667c65d7b7b 100644 --- a/mysql-test/suite/innodb/r/doublewrite.result +++ b/mysql-test/suite/innodb/r/doublewrite.result @@ -21,8 +21,8 @@ connection default; flush table t1 for export; # Kill the server # restart -FOUND 1 /InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile/ in mysqld.1.err -FOUND 1 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=3\]/ in mysqld.1.err +FOUND 1 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=0\]/ in mysqld.1.err +# restart XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text @@ -44,7 +44,7 @@ connection default; flush table t1 for export; # Kill the server # restart -FOUND 1 /InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile/ in mysqld.1.err +FOUND 4 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=[03]\]/ in mysqld.1.err XA ROLLBACK 'x'; check table t1; Table Op Msg_type Msg_text diff --git a/mysql-test/suite/innodb/r/doublewrite_debug.result b/mysql-test/suite/innodb/r/doublewrite_debug.result index aa141c18a6e..a743217f34e 100644 --- a/mysql-test/suite/innodb/r/doublewrite_debug.result +++ b/mysql-test/suite/innodb/r/doublewrite_debug.result @@ -37,7 +37,7 @@ set global innodb_buf_flush_list_now = 1; # Make the 1st page (page_no=0) and 2nd page (page_no=1) # of the system tablespace all zeroes. # restart -FOUND 1 /InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile/ in mysqld.1.err +FOUND 1 /InnoDB: Recovered page \[page id: space=0, page number=0\]/ in mysqld.1.err FOUND 1 /InnoDB: Recovered page \[page id: space=0, page number=1\]/ in mysqld.1.err check table t1; Table Op Msg_type Msg_text @@ -66,7 +66,7 @@ set global innodb_buf_flush_list_now = 1; # Kill the server # Corrupt the 1st page (page_no=0) and 2nd page of the system tablespace. # restart -FOUND 2 /InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile/ in mysqld.1.err +FOUND 2 /InnoDB: Recovered page \[page id: space=0, page number=0\]/ in mysqld.1.err FOUND 2 /InnoDB: Recovered page \[page id: space=0, page number=1\]/ in mysqld.1.err check table t1; Table Op Msg_type Msg_text diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result index 1c3b10c0597..1c6761dbca1 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result @@ -9,8 +9,8 @@ call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS="); call mtr.add_suppression("InnoDB: Unknown index id .* on page"); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; CREATE TABLE t1 (c1 INT) ENGINE = InnoDB; INSERT INTO t1 VALUES(1),(2),(3); diff --git a/mysql-test/suite/innodb/r/page_id_innochecksum.result b/mysql-test/suite/innodb/r/page_id_innochecksum.result index bde986c07ef..4ffbf2f1453 100644 --- a/mysql-test/suite/innodb/r/page_id_innochecksum.result +++ b/mysql-test/suite/innodb/r/page_id_innochecksum.result @@ -6,3 +6,4 @@ FOUND 1 /page id mismatch/ in result.log InnoDB 0 transactions not purged drop table t1; call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd': Page read from tablespace is corrupted\\."); +call mtr.add_suppression("InnoDB: File '.*t1\\.ibd' is corrupted"); diff --git a/mysql-test/suite/innodb/r/undo_space_dblwr.result b/mysql-test/suite/innodb/r/undo_space_dblwr.result index 2172ce53cb7..d7c1cb125a9 100644 --- a/mysql-test/suite/innodb/r/undo_space_dblwr.result +++ b/mysql-test/suite/innodb/r/undo_space_dblwr.result @@ -12,7 +12,7 @@ set global innodb_fil_make_page_dirty_debug = 1; SET GLOBAL innodb_buf_flush_list_now = 1; # Kill the server # restart: --debug_dbug=+d,ib_log_checkpoint_avoid_hard --innodb_flush_sync=0 -FOUND 1 /Restoring page \[page id: space=1, page number=0\] of datafile '.*undo001' from the doublewrite buffer./ in mysqld.1.err +FOUND 1 /Recovered page \[page id: space=1, page number=0\] to '.*undo001' from the doublewrite buffer\./ in mysqld.1.err check table t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb/t/corrupted_during_recovery.test b/mysql-test/suite/innodb/t/corrupted_during_recovery.test index 8324054992c..f62215a7207 100644 --- a/mysql-test/suite/innodb/t/corrupted_during_recovery.test +++ b/mysql-test/suite/innodb/t/corrupted_during_recovery.test @@ -4,15 +4,18 @@ call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); -call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test.t1\\.ibd': Page read from tablespace is corrupted."); -call mtr.add_suppression("InnoDB: (Unable to apply log to|Discarding log for) corrupted page .*, page number=3\\]"); -call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate."); +call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page 3 in file .*test.t1\\.ibd"); +call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate\\."); call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: A long wait .* was observed for dict_sys"); +call mtr.add_suppression("InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 1311768467463790320 is in the future!"); +call mtr.add_suppression("InnoDB: Your database may be corrupt"); +call mtr.add_suppression("InnoDB: MySQL-8\\.0 tablespace in .*test/t2\\.ibd"); +call mtr.add_suppression("InnoDB: Restart in MySQL for migration/recovery\\."); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let ALGO=`select @@innodb_checksum_algorithm`; CREATE TABLE t1(a BIGINT PRIMARY KEY) ENGINE=InnoDB; INSERT INTO t1 VALUES(1); # Force a redo log checkpoint. @@ -30,15 +33,32 @@ INSERT INTO t2 VALUES(1); --echo # Corrupt the pages perl; +do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; +my $polynomial = 0x82f63b78; # CRC-32C +my $algo = $ENV{ALGO}; my $ps = $ENV{INNODB_PAGE_SIZE}; - my $file = "$ENV{MYSQLD_DATADIR}/test/t1.ibd"; open(FILE, "+<$file") || die "Unable to open $file"; binmode FILE; sysseek(FILE, 3*$ps, 0) || die "Unable to seek $file\n"; die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps; -# Replace the a=1 with a=0. -$page =~ s/\x80\x0\x0\x0\x0\x0\x0\x1/\x80\x0\x0\x0\x0\x0\x0\x0/; +# Assign a future FIL_PAGE_LSN +substr($page, 16, 8) = pack("NN", 0x12345678, 0x9abcdef0); +substr($page, $ps - 8, 8) = pack("NN", 0x9abcdef0, 0x9abcdef0); +if ($algo =~ /full_crc32/) +{ + my $ck = mycrc32(substr($page, 0, $ps - 4), 0, $polynomial); + substr($page, $ps - 4, 4) = pack("N", $ck); +} +else +{ + # Replace the innodb_checksum_algorithm=crc32 checksum + my $ck= pack("N", + mycrc32(substr($page, 4, 22), 0, $polynomial) ^ + mycrc32(substr($page, 38, $ps - 38 - 8), 0, $polynomial)); + substr ($page, 0, 4) = $ck; + substr ($page, $ps - 8, 4) = $ck; +} sysseek(FILE, 3*$ps, 0) || die "Unable to seek $file\n"; syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n"; close FILE or die "close"; @@ -46,20 +66,23 @@ close FILE or die "close"; $file = "$ENV{MYSQLD_DATADIR}/test/t2.ibd"; open(FILE, "+<$file") || die "Unable to open $file"; binmode FILE; -# Corrupt pages 1 to 3. MLOG_INIT_FILE_PAGE2 should protect us! -# Unfortunately, we are not immune to page 0 corruption. -seek (FILE, $ps, SEEK_SET) or die "seek"; -print FILE chr(0xff) x ($ps * 3); +# Corrupt pages 0 to 3. INIT_PAGE should protect us! +print FILE chr(0xff) x ($ps * 4); close FILE or die "close"; EOF --source include/start_mysqld.inc --error ER_UNKNOWN_STORAGE_ENGINE SELECT * FROM t1; + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +let SEARCH_PATTERN=InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 1311768467463790320 is in the future!; +--source include/search_pattern_in_file.inc + let $restart_parameters=--innodb_force_recovery=1; --source include/restart_mysqld.inc ---error ER_NO_SUCH_TABLE_IN_ENGINE,ER_TABLE_CORRUPT +--error 0,ER_NO_SUCH_TABLE_IN_ENGINE SELECT * FROM t1; SELECT * FROM t2; CHECK TABLE t2; @@ -81,13 +104,36 @@ DELETE FROM t1 WHERE pk=3; --source ../include/no_checkpoint_end.inc disconnect con1; ---echo # Corrupt the pages +--echo # Corrupt the page perl; +do "$ENV{MTR_SUITE_DIR}/include/crc32.pl"; +my $polynomial = 0x82f63b78; # CRC-32C +my $algo = $ENV{ALGO}; +my $ps = $ENV{INNODB_PAGE_SIZE}; my $file = "$ENV{MYSQLD_DATADIR}/test/t1.ibd"; open(FILE, "+<$file") || die "Unable to open $file"; binmode FILE; -seek (FILE, $ENV{INNODB_PAGE_SIZE} * 3, SEEK_SET) or die "seek"; -print FILE "junk"; +sysseek(FILE, $ps * 3, SEEK_SET) or die "seek"; +sysread(FILE, $page, $ps)==$ps||die "Unable to read $file\n"; +# Set FIL_PAGE_LSN to the maximum +substr($page, 16, 8) = chr(255) x 8; +substr($page, $ps - 8, 8) = chr(255) x 8; +if ($algo =~ /full_crc32/) +{ + my $ck = mycrc32(substr($page, 0, $ps - 4), 0, $polynomial); + substr($page, $ps - 4, 4) = pack("N", $ck); +} +else +{ + # Replace the innodb_checksum_algorithm=crc32 checksum + my $ck= pack("N", + mycrc32(substr($page, 4, 22), 0, $polynomial) ^ + mycrc32(substr($page_, 38, $ps - 38 - 8), 0, $polynomial)); + substr ($page, 0, 4) = $ck; + substr ($page, $ps - 8, 4) = $ck; +} +sysseek(FILE, $ps * 3, SEEK_SET) or die "seek"; +syswrite(FILE, $page); close FILE or die "close"; EOF --source include/start_mysqld.inc diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index d73009908db..70832584228 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -18,6 +18,9 @@ call mtr.add_suppression("InnoDB: Checksum mismatch in datafile: "); call mtr.add_suppression("InnoDB: Inconsistent tablespace ID in .*t1\\.ibd"); call mtr.add_suppression("\\[Warning\\] Found 1 prepared XA transactions"); call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile:"); +call mtr.add_suppression("InnoDB: Page \\[page id: space=[1-9][0-9]*, page number=3\\] log sequence number 18446744073709551615 is in the future!"); +call mtr.add_suppression("InnoDB: Your database may be corrupt or you may have copied the InnoDB tablespace but not the ib_logfile0"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted"); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; @@ -71,7 +74,26 @@ syswrite(FILE, chr(0) x ($page_size/2)); sysseek(FILE, 3*$page_size, 0); sysread(FILE, $page, $page_size)==$page_size||die "Unable to read $name\n"; sysseek(FILE, 3*$page_size, 0)||die "Unable to seek $fname\n"; -syswrite(FILE, chr(0) x ($page_size/2)); +my $corrupted = $page; +# Set FIL_PAGE_LSN to the maximum +substr($corrupted, 16, 8) = chr(255) x 8; +substr($corrupted, $page_size - 8, 8) = chr(255) x 8; +if ($algo =~ /full_crc32/) +{ + my $ck = mycrc32(substr($corrupted, 0, $page_size - 4), 0, $polynomial); + substr($corrupted, $page_size - 4, 4) = pack("N", $ck); +} +else +{ + # Replace the innodb_checksum_algorithm=crc32 checksum + my $ck= pack("N", + mycrc32(substr($corrupted, 4, 22), 0, $polynomial) ^ + mycrc32(substr($corrupted_, 38, $page_size - 38 - 8), 0, + $polynomial)); + substr ($corrupted, 0, 4) = $ck; + substr ($corrupted, $page_size - 8, 4) = $ck; +} +syswrite(FILE, $corrupted); close FILE; # Change the flag offset of page 0 in doublewrite buffer @@ -113,10 +135,28 @@ die "Did not find the page in the doublewrite buffer ($d1,$d2)\n"; EOF --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=[1-9][0-9]*, page number=0\\]; --source include/search_pattern_in_file.inc -let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=3\]; +let SEARCH_PATTERN=InnoDB: The log was only scanned up to \\d+, while the current LSN at the time of the latest checkpoint \\d+ was 0 and the maximum LSN on a data page was 18446744073709551615! --source include/search_pattern_in_file.inc +--error ER_XAER_NOTA +XA ROLLBACK 'x'; +let $shutdown_timeout=0; +--source include/shutdown_mysqld.inc +let $shutdown_timeout=; +# Corrupt the file in a better way. +perl; +use IO::Handle; +my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; +my $page_size = $ENV{INNODB_PAGE_SIZE}; +open(FILE, "+<", $fname) or die; +sysseek(FILE, ($page_size/2), 0); +syswrite(FILE, chr(0) x ($page_size/2)); +sysseek(FILE, 3*$page_size, 0); +syswrite(FILE, chr(0) x ($page_size/2)); +close FILE; +EOF +--source include/start_mysqld.inc XA ROLLBACK 'x'; check table t1; select f1, f2 from t1; @@ -144,7 +184,7 @@ close FILE; EOF --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=[1-9][0-9]*, page number=[03]\\]; --source include/search_pattern_in_file.inc XA ROLLBACK 'x'; check table t1; diff --git a/mysql-test/suite/innodb/t/doublewrite_debug.test b/mysql-test/suite/innodb/t/doublewrite_debug.test index b8dcd5068ef..b207823e3d1 100644 --- a/mysql-test/suite/innodb/t/doublewrite_debug.test +++ b/mysql-test/suite/innodb/t/doublewrite_debug.test @@ -81,10 +81,10 @@ EOF let $restart_parameters=; --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=0\\]; --source include/search_pattern_in_file.inc -let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=0, page number=1\]; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=1\\]; --source include/search_pattern_in_file.inc check table t1; @@ -129,10 +129,10 @@ EOF let $restart_parameters=; --source include/start_mysqld.inc -let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=0, page number=0\] of datafile; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=0\\]; --source include/search_pattern_in_file.inc -let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=0, page number=1\]; +let SEARCH_PATTERN=InnoDB: Recovered page \\[page id: space=0, page number=1\\]; --source include/search_pattern_in_file.inc check table t1; diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test index f83866b0f4c..db5a0525f8b 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test @@ -29,8 +29,8 @@ call mtr.add_suppression("InnoDB: Page for tablespace "); call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS="); call mtr.add_suppression("InnoDB: Unknown index id .* on page"); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491.test b/mysql-test/suite/innodb/t/innodb_bug14147491.test index 6fe4b319e34..cff1b0c78af 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491.test +++ b/mysql-test/suite/innodb/t/innodb_bug14147491.test @@ -9,7 +9,7 @@ --disable_query_log call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted\\. Please drop the table and recreate\\."); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); +call mtr.add_suppression("InnoDB: Failed to read page [1-9][0-9]* from file '.*test.t1\\.ibd'"); call mtr.add_suppression("InnoDB: We detected index corruption in an InnoDB type table"); call mtr.add_suppression("Index for table 't1' is corrupt; try to repair it"); call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); diff --git a/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test b/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test index 21d5336528a..026be74373f 100644 --- a/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test +++ b/mysql-test/suite/innodb/t/leaf_page_corrupted_during_recovery.test @@ -2,9 +2,8 @@ --source include/have_debug.inc --disable_query_log -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*test.t1\\.ibd' page"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 19 from file '.*test.t1\\.ibd': Page read from tablespace is corrupted\\."); -call mtr.add_suppression("InnoDB: (Unable to apply log to|Discarding log for) corrupted page .*, page number=19\\]"); +call mtr.add_suppression("InnoDB: Unable to apply log to corrupted page 19 in file .*t1\\.ibd"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Plugin initialization aborted at srv0start\\.cc.* with error Data structure corruption"); call mtr.add_suppression("\\[ERROR\\] Plugin 'InnoDB' (init function|registration)"); call mtr.add_suppression("\\[ERROR\\] InnoDB: We detected index corruption"); diff --git a/mysql-test/suite/innodb/t/page_id_innochecksum.test b/mysql-test/suite/innodb/t/page_id_innochecksum.test index 9d8114d1720..09523cccd1f 100644 --- a/mysql-test/suite/innodb/t/page_id_innochecksum.test +++ b/mysql-test/suite/innodb/t/page_id_innochecksum.test @@ -66,5 +66,6 @@ let $restart_parameters=--innodb-force-recovery=1; --source include/wait_all_purged.inc drop table t1; call mtr.add_suppression("InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd': Page read from tablespace is corrupted\\."); +call mtr.add_suppression("InnoDB: File '.*t1\\.ibd' is corrupted"); let $restart_parameters=; --source include/restart_mysqld.inc diff --git a/mysql-test/suite/innodb/t/undo_space_dblwr.test b/mysql-test/suite/innodb/t/undo_space_dblwr.test index 4cf4d3b8b6d..33e8ed9d651 100644 --- a/mysql-test/suite/innodb/t/undo_space_dblwr.test +++ b/mysql-test/suite/innodb/t/undo_space_dblwr.test @@ -39,7 +39,7 @@ EOF --source include/start_mysqld.inc let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; -let SEARCH_PATTERN= Restoring page \[page id: space=1, page number=0\] of datafile '.*undo001' from the doublewrite buffer.; +let SEARCH_PATTERN= Recovered page \\[page id: space=1, page number=0\\] to '.*undo001' from the doublewrite buffer\\.; --source include/search_pattern_in_file.inc check table t1; diff --git a/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result b/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result index f44190680c0..0d9fdbde86f 100644 --- a/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result +++ b/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result @@ -9,8 +9,8 @@ call mtr.add_suppression("InnoDB: Error number \\d+ means"); call mtr.add_suppression("InnoDB: Cannot open datafile .*t1\\.ibd"); call mtr.add_suppression("InnoDB: Ignoring tablespace for test/t1 "); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; SET SESSION innodb_strict_mode=1; CREATE TABLE t1 (c1 INT) ENGINE = Innodb diff --git a/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test b/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test index 97a1909a875..79e87fc65ec 100644 --- a/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test +++ b/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test @@ -25,8 +25,8 @@ call mtr.add_suppression("InnoDB: Error number \\d+ means"); call mtr.add_suppression("InnoDB: Cannot open datafile .*t1\\.ibd"); call mtr.add_suppression("InnoDB: Ignoring tablespace for test/t1 "); call mtr.add_suppression("InnoDB: Cannot save statistics for table `test`\\.`t1` because the \\.ibd file is missing"); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed read of file '.*ibdata1' page"); -call mtr.add_suppression("InnoDB: File '.*ibdata1' is corrupted"); +call mtr.add_suppression("InnoDB: Failed to read page \\d+ from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("InnoDB: File '.*(ibdata1|t1\\.ibd)' is corrupted"); FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; diff --git a/mysql-test/suite/mariabackup/compression_providers_unloaded.result b/mysql-test/suite/mariabackup/compression_providers_unloaded.result index ccf3e0355a4..89cd6ed2158 100644 --- a/mysql-test/suite/mariabackup/compression_providers_unloaded.result +++ b/mysql-test/suite/mariabackup/compression_providers_unloaded.result @@ -1,6 +1,7 @@ call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("Refusing to load corrupted table"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); # # Testing mariabackup with bzip2 compression # diff --git a/mysql-test/suite/mariabackup/compression_providers_unloaded.test b/mysql-test/suite/mariabackup/compression_providers_unloaded.test index 673c16d03cf..1c2b5fa1db8 100644 --- a/mysql-test/suite/mariabackup/compression_providers_unloaded.test +++ b/mysql-test/suite/mariabackup/compression_providers_unloaded.test @@ -2,6 +2,7 @@ let $alg = $MTR_COMBINATIONS; call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("Refusing to load corrupted table"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); if (`select count(*) = 0 from information_schema.plugins where plugin_name = 'provider_$alg' and plugin_status='active'`) { diff --git a/mysql-test/suite/mariabackup/encrypted_page_compressed.result b/mysql-test/suite/mariabackup/encrypted_page_compressed.result index de4c966caf4..31098e45675 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_compressed.result +++ b/mysql-test/suite/mariabackup/encrypted_page_compressed.result @@ -1,5 +1,7 @@ call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); call mtr.add_suppression("InnoDB: Encrypted page .* in file .*test.t1\\.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes; insert into t1(b, c) values("mariadb", "mariabackup"); # Corrupt the table diff --git a/mysql-test/suite/mariabackup/encrypted_page_compressed.test b/mysql-test/suite/mariabackup/encrypted_page_compressed.test index 245fcc31c0d..9faf654d334 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/encrypted_page_compressed.test @@ -1,6 +1,8 @@ source include/have_file_key_management.inc; call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); call mtr.add_suppression("InnoDB: Encrypted page .* in file .*test.t1\\.ibd looks corrupted; key_version=1"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes; insert into t1(b, c) values("mariadb", "mariabackup"); diff --git a/mysql-test/suite/mariabackup/encrypted_page_corruption.result b/mysql-test/suite/mariabackup/encrypted_page_corruption.result index b328d361cd6..62f39b561da 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_corruption.result +++ b/mysql-test/suite/mariabackup/encrypted_page_corruption.result @@ -1,4 +1,6 @@ call mtr.add_suppression("\\[ERROR\\] InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=3\\] in file '.*test.t1\\.ibd' cannot be decrypted."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Table `test`\\.`t1` has an unreadable root page"); CREATE TABLE t1(c VARCHAR(128)) ENGINE INNODB, encrypted=yes; insert into t1 select repeat('a',100); diff --git a/mysql-test/suite/mariabackup/encrypted_page_corruption.test b/mysql-test/suite/mariabackup/encrypted_page_corruption.test index 9ba958c68a0..10e4b8c98fc 100644 --- a/mysql-test/suite/mariabackup/encrypted_page_corruption.test +++ b/mysql-test/suite/mariabackup/encrypted_page_corruption.test @@ -2,6 +2,8 @@ --source include/innodb_page_size.inc call mtr.add_suppression("\\[ERROR\\] InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=3\\] in file '.*test.t1\\.ibd' cannot be decrypted."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("\\[ERROR\\] InnoDB: Table `test`\\.`t1` has an unreadable root page"); CREATE TABLE t1(c VARCHAR(128)) ENGINE INNODB, encrypted=yes; insert into t1 select repeat('a',100); diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.result b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result index dfcf19b6c2b..ededfbb8e8d 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.result +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.result @@ -1,3 +1,5 @@ +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB PAGE_COMPRESSED=YES STATS_PERSISTENT=0; diff --git a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test index 31e8323b8b6..68f22e69e03 100644 --- a/mysql-test/suite/mariabackup/unencrypted_page_compressed.test +++ b/mysql-test/suite/mariabackup/unencrypted_page_compressed.test @@ -1,3 +1,5 @@ +call mtr.add_suppression("\\[ERROR\\] InnoDB: Failed to read page 3 from file '.*test/t1\\.ibd'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("InnoDB: Table `test`.`t1` has an unreadable root page"); CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB PAGE_COMPRESSED=YES STATS_PERSISTENT=0; diff --git a/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff b/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff index 3f449220df5..e48fb5eaa8f 100644 --- a/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff +++ b/mysql-test/suite/plugins/r/compression,mroonga-lz4.rdiff @@ -1,6 +1,6 @@ --- suite/plugins/r/compression.result +++ suite/plugins/r/compression.reject -@@ -1,13 +1,8 @@ +@@ -1,14 +1,8 @@ # -# Testing bzip2 compression provider with innodb +# Testing lz4 compression provider with mroonga @@ -10,13 +10,14 @@ -call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); -call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); -call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate"); +-call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); -call mtr.add_suppression("Table .*t1.* is compressed with (\\w+), which is not currently loaded. Please load the \\1 provider plugin to open the table"); -create table t1 (a int, b text ) engine = innodb page_compressed = 1; +create table t1 (a int, b text COMMENT 'FLAGS "COLUMN_SCALAR|COMPRESS_LZ4"') engine = mroonga charset = utf8; insert t1 (a, b) values (0, repeat("abc", 100)); insert t1 (a, b) values (1, repeat("def", 1000)); insert t1 (a, b) values (2, repeat("ghi", 10000)); -@@ -16,12 +11,20 @@ +@@ -17,12 +11,20 @@ 0 abcabcabc 300 1 defdefdef 3000 2 ghighighi 30000 diff --git a/mysql-test/suite/plugins/r/compression.result b/mysql-test/suite/plugins/r/compression.result index 07bfbc0b9bb..2319754960c 100644 --- a/mysql-test/suite/plugins/r/compression.result +++ b/mysql-test/suite/plugins/r/compression.result @@ -6,6 +6,7 @@ set global innodb_compression_algorithm = bzip2; call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate"); +call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("Table .*t1.* is compressed with (\\w+), which is not currently loaded. Please load the \\1 provider plugin to open the table"); create table t1 (a int, b text ) engine = innodb page_compressed = 1; insert t1 (a, b) values (0, repeat("abc", 100)); diff --git a/mysql-test/suite/plugins/t/compression.test b/mysql-test/suite/plugins/t/compression.test index 95ae2df9462..e05750f9e99 100644 --- a/mysql-test/suite/plugins/t/compression.test +++ b/mysql-test/suite/plugins/t/compression.test @@ -22,6 +22,7 @@ if ($engine == "innodb") { call mtr.add_suppression("Background Page read failed to read, uncompress, or decrypt"); call mtr.add_suppression("Table is compressed or encrypted but uncompress or decrypt failed"); call mtr.add_suppression("Table `test`.`t1` is corrupted. Please drop the table and recreate"); + call mtr.add_suppression("InnoDB: File '.*test/t1\\.ibd' is corrupted"); call mtr.add_suppression("Table .*t1.* is compressed with (\\w+), which is not currently loaded. Please load the \\1 provider plugin to open the table"); } if ($engine == "mroonga") { diff --git a/mysql-test/suite/versioning/r/data.result b/mysql-test/suite/versioning/r/data.result index 9832292a7e1..b21e44f391f 100644 --- a/mysql-test/suite/versioning/r/data.result +++ b/mysql-test/suite/versioning/r/data.result @@ -98,10 +98,10 @@ CREATE TABLE `t1` ( `x` int(11) DEFAULT NULL ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci WITH SYSTEM VERSIONING; /*!40101 SET character_set_client = @saved_cs_client */; -/*!101100 SET @old_system_versioning_insert_history=@@session.system_versioning_insert_history, @@session.system_versioning_insert_history=1 */; +/*M!101100 SET @old_system_versioning_insert_history=@@session.system_versioning_insert_history, @@session.system_versioning_insert_history=1 */; INSERT INTO `t1` (`x`, row_start, row_end) VALUES (1,'2010-10-10 10:10:10.101010','2011-11-11 11:11:11.111111'), (2,'2010-10-10 10:10:10.101010','2038-01-19 03:14:07.999999'); -/*!101100 SET system_versioning_insert_history=@old_system_versioning_insert_history */; +/*M!101100 SET system_versioning_insert_history=@old_system_versioning_insert_history */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; CREATE TABLE `t2` ( @@ -111,10 +111,10 @@ CREATE TABLE `t2` ( PERIOD FOR SYSTEM_TIME (`row_start`, `row_end`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci WITH SYSTEM VERSIONING; /*!40101 SET character_set_client = @saved_cs_client */; -/*!101100 SET @old_system_versioning_insert_history=@@session.system_versioning_insert_history, @@session.system_versioning_insert_history=1 */; +/*M!101100 SET @old_system_versioning_insert_history=@@session.system_versioning_insert_history, @@session.system_versioning_insert_history=1 */; INSERT INTO `t2` (`x`, `row_start`, `row_end`) VALUES (1,'2010-10-10 10:10:10.101010','2011-11-11 11:11:11.111111'), (2,'2010-10-10 10:10:10.101010','2038-01-19 03:14:07.999999'); -/*!101100 SET system_versioning_insert_history=@old_system_versioning_insert_history */; +/*M!101100 SET system_versioning_insert_history=@old_system_versioning_insert_history */; mariadb-dump: --dump-history can't be used with --as-of. mariadb-dump: --dump-history can't be used with --replace. mariadb-dump: --xml can't be used with --dump-history. diff --git a/sql/table.cc b/sql/table.cc index 2e597583f07..768253a1c72 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -1736,9 +1736,9 @@ public: */ #ifdef WITH_PARTITION_STORAGE_ENGINE -static bool change_to_partiton_engine(LEX_CSTRING *name, - plugin_ref *se_plugin) +static bool change_to_partiton_engine(plugin_ref *se_plugin) { + LEX_CSTRING name= { STRING_WITH_LEN("partition") }; /* Use partition handler tmp_plugin is locked with a local lock. @@ -1746,10 +1746,9 @@ static bool change_to_partiton_engine(LEX_CSTRING *name, replacing it with a globally locked version of tmp_plugin */ /* Check if the partitioning engine is ready */ - if (!plugin_is_ready(name, MYSQL_STORAGE_ENGINE_PLUGIN)) + if (!plugin_is_ready(&name, MYSQL_STORAGE_ENGINE_PLUGIN)) { - my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), - "--skip-partition"); + my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), "--skip-partition"); return 1; } plugin_unlock(NULL, *se_plugin); @@ -2047,7 +2046,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, else if (str_db_type_length == 9 && !strncmp((char *) next_chunk + 2, "partition", 9)) { - if (change_to_partiton_engine(&se_name, &se_plugin)) + if (change_to_partiton_engine(&se_plugin)) goto err; } #endif @@ -2091,7 +2090,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, share->mysql_version >= 50600 && share->mysql_version <= 50799) { share->keep_original_mysql_version= 1; - if (change_to_partiton_engine(&se_name, &se_plugin)) + if (change_to_partiton_engine(&se_plugin)) goto err; } } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 5af4ba0ea37..7828b0ffa39 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -509,43 +509,46 @@ buf_page_is_checksum_valid_crc32( return checksum_field1 == crc32; } +#ifndef UNIV_INNOCHECKSUM /** Checks whether the lsn present in the page is lesser than the peek current lsn. -@param[in] check_lsn lsn to check -@param[in] read_buf page. */ -static void buf_page_check_lsn(bool check_lsn, const byte* read_buf) +@param check_lsn lsn to check +@param read_buf page frame +@return whether the FIL_PAGE_LSN is invalid */ +static bool buf_page_check_lsn(bool check_lsn, const byte *read_buf) { -#ifndef UNIV_INNOCHECKSUM - if (check_lsn && recv_lsn_checks_on) { - const lsn_t current_lsn = log_sys.get_lsn(); - const lsn_t page_lsn - = mach_read_from_8(read_buf + FIL_PAGE_LSN); + if (!check_lsn) + return false; + lsn_t current_lsn= log_sys.get_lsn(); + if (UNIV_UNLIKELY(current_lsn == log_sys.FIRST_LSN) && + srv_force_recovery == SRV_FORCE_NO_LOG_REDO) + return false; + const lsn_t page_lsn= mach_read_from_8(read_buf + FIL_PAGE_LSN); - /* Since we are going to reset the page LSN during the import - phase it makes no sense to spam the log with error messages. */ - if (current_lsn < page_lsn) { + if (UNIV_LIKELY(current_lsn >= page_lsn)) + return false; - const uint32_t space_id = mach_read_from_4( - read_buf + FIL_PAGE_SPACE_ID); - const uint32_t page_no = mach_read_from_4( - read_buf + FIL_PAGE_OFFSET); + const uint32_t space_id= mach_read_from_4(read_buf + FIL_PAGE_SPACE_ID); + const uint32_t page_no= mach_read_from_4(read_buf + FIL_PAGE_OFFSET); - ib::error() << "Page " << page_id_t(space_id, page_no) - << " log sequence number " << page_lsn - << " is in the future! Current system" - << " log sequence number " - << current_lsn << "."; + sql_print_error("InnoDB: Page " + "[page id: space=" UINT32PF ", page number=" UINT32PF "]" + " log sequence number " LSN_PF + " is in the future! Current system log sequence number " + LSN_PF ".", + space_id, page_no, page_lsn, current_lsn); - ib::error() << "Your database may be corrupt or" - " you may have copied the InnoDB" - " tablespace but not the InnoDB" - " log files. " - << FORCE_RECOVERY_MSG; + if (srv_force_recovery) + return false; - } - } -#endif /* !UNIV_INNOCHECKSUM */ + sql_print_error("InnoDB: Your database may be corrupt or" + " you may have copied the InnoDB" + " tablespace but not the ib_logfile0. %s", + FORCE_RECOVERY_MSG); + + return true; } +#endif /** Check if a buffer is all zeroes. @@ -562,35 +565,36 @@ bool buf_is_zeroes(span buf) @param read_buf database page @param fsp_flags contents of FIL_SPACE_FLAGS @return whether the page is corrupted */ -bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, - uint32_t fsp_flags) +buf_page_is_corrupted_reason +buf_page_is_corrupted(bool check_lsn, const byte *read_buf, uint32_t fsp_flags) { if (fil_space_t::full_crc32(fsp_flags)) { bool compressed = false, corrupted = false; const uint size = buf_page_full_crc32_size( read_buf, &compressed, &corrupted); if (corrupted) { - return true; + return CORRUPTED_OTHER; } const byte* end = read_buf + (size - FIL_PAGE_FCRC32_CHECKSUM); uint crc32 = mach_read_from_4(end); if (!crc32 && size == srv_page_size && buf_is_zeroes(span(read_buf, size))) { - return false; + return NOT_CORRUPTED; } DBUG_EXECUTE_IF( "page_intermittent_checksum_mismatch", { static int page_counter; - if (page_counter++ == 6) { + if (mach_read_from_4(FIL_PAGE_OFFSET + read_buf) + && page_counter++ == 6) { crc32++; } }); if (crc32 != my_crc32c(0, read_buf, size - FIL_PAGE_FCRC32_CHECKSUM)) { - return true; + return CORRUPTED_OTHER; } static_assert(FIL_PAGE_FCRC32_KEY_VERSION == 0, "alignment"); static_assert(FIL_PAGE_LSN % 4 == 0, "alignment"); @@ -602,11 +606,15 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, end - (FIL_PAGE_FCRC32_END_LSN - FIL_PAGE_FCRC32_CHECKSUM), 4)) { - return true; + return CORRUPTED_OTHER; } - buf_page_check_lsn(check_lsn, read_buf); - return false; + return +#ifndef UNIV_INNOCHECKSUM + buf_page_check_lsn(check_lsn, read_buf) + ? CORRUPTED_FUTURE_LSN : +#endif + NOT_CORRUPTED; } const ulint zip_size = fil_space_t::zip_size(fsp_flags); @@ -627,7 +635,13 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, && FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp_flags) #endif ) { - return(false); + check_lsn: + return +#ifndef UNIV_INNOCHECKSUM + buf_page_check_lsn(check_lsn, read_buf) + ? CORRUPTED_FUTURE_LSN : +#endif + NOT_CORRUPTED; } static_assert(FIL_PAGE_LSN % 4 == 0, "alignment"); @@ -640,15 +654,16 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, /* Stored log sequence numbers at the start and the end of page do not match */ - return(true); + return CORRUPTED_OTHER; } - buf_page_check_lsn(check_lsn, read_buf); - /* Check whether the checksum fields have correct values */ if (zip_size) { - return !page_zip_verify_checksum(read_buf, zip_size); + if (!page_zip_verify_checksum(read_buf, zip_size)) { + return CORRUPTED_OTHER; + } + goto check_lsn; } const uint32_t checksum_field1 = mach_read_from_4( @@ -685,7 +700,7 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, } if (all_zeroes) { - return false; + return NOT_CORRUPTED; } } @@ -694,13 +709,17 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, case SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: #endif /* !UNIV_INNOCHECKSUM */ - return !buf_page_is_checksum_valid_crc32( - read_buf, checksum_field1, checksum_field2); + if (!buf_page_is_checksum_valid_crc32(read_buf, + checksum_field1, + checksum_field2)) { + return CORRUPTED_OTHER; + } + goto check_lsn; #ifndef UNIV_INNOCHECKSUM default: if (checksum_field1 == BUF_NO_CHECKSUM_MAGIC && checksum_field2 == BUF_NO_CHECKSUM_MAGIC) { - return false; + goto check_lsn; } const uint32_t crc32 = buf_calc_page_crc32(read_buf); @@ -718,28 +737,35 @@ bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, DBUG_EXECUTE_IF( "page_intermittent_checksum_mismatch", { static int page_counter; - if (page_counter++ == 6) - return true; + if (mach_read_from_4(FIL_PAGE_OFFSET + + read_buf) + && page_counter++ == 6) + return CORRUPTED_OTHER; }); if ((checksum_field1 != crc32 || checksum_field2 != crc32) && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { - return true; + return CORRUPTED_OTHER; } } switch (checksum_field1) { case 0: case BUF_NO_CHECKSUM_MAGIC: - return false; + break; + default: + if ((checksum_field1 != crc32 + || checksum_field2 != crc32) + && checksum_field1 + != buf_calc_page_new_checksum(read_buf)) { + return CORRUPTED_OTHER; + } } - return (checksum_field1 != crc32 || checksum_field2 != crc32) - && checksum_field1 - != buf_calc_page_new_checksum(read_buf); } #endif /* !UNIV_INNOCHECKSUM */ + goto check_lsn; } #ifndef UNIV_INNOCHECKSUM @@ -3494,6 +3520,7 @@ or decrypt/decompress just failed. @return whether the operation succeeded @retval DB_SUCCESS if page has been read and is not corrupted @retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted +@retval DB_CORRUPTION if the page LSN is in the future @retval DB_DECRYPTION_FAILED if page post encryption checksum matches but after decryption normal page checksum does not match. */ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, @@ -3530,8 +3557,18 @@ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, node.space->is_compressed())) { err = DB_PAGE_CORRUPTED; } - } else if (buf_page_is_corrupted(true, dst_frame, node.space->flags)) { - err = DB_PAGE_CORRUPTED; + } else { + switch (buf_page_is_corrupted(true, dst_frame, + node.space->flags)) { + case NOT_CORRUPTED: + break; + case CORRUPTED_OTHER: + err = DB_PAGE_CORRUPTED; + break; + case CORRUPTED_FUTURE_LSN: + err = DB_CORRUPTION; + break; + } } if (seems_encrypted && err == DB_PAGE_CORRUPTED @@ -3551,9 +3588,8 @@ static dberr_t buf_page_check_corrupt(buf_page_t *bpage, /** Complete a read of a page. @param node data file @return whether the operation succeeded -@retval DB_PAGE_CORRUPTED if the checksum fails -@retval DB_DECRYPTION_FAILED if the page cannot be decrypted -@retval DB_FAIL if the page contains the wrong ID */ +@retval DB_PAGE_CORRUPTED if the checksum or the page ID is incorrect +@retval DB_DECRYPTION_FAILED if the page cannot be decrypted */ dberr_t buf_page_t::read_complete(const fil_node_t &node) { const page_id_t expected_id{id()}; @@ -3581,9 +3617,8 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node) if (!ok) { - ib::info() << "Page " << expected_id << " zip_decompress failure."; err= DB_PAGE_CORRUPTED; - goto database_corrupted; + goto database_corrupted_compressed; } } @@ -3608,15 +3643,21 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node) node.space->crypt_data && node.space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { - ib::error() << "Cannot decrypt " << expected_id; err= DB_DECRYPTION_FAILED; goto release_page; } else { - ib::error() << "Space id and page no stored in the page, read in are " - << read_id << ", should be " << expected_id; - err= DB_PAGE_CORRUPTED; + sql_print_error("InnoDB: Space id and page no stored in the page," + " read in from %s are " + "[page id: space=" UINT32PF ", page number=" UINT32PF + "], should be " + "[page id: space=" UINT32PF ", page number=" UINT32PF + "]", + node.name, + read_id.space(), read_id.page_no(), + expected_id.space(), expected_id.page_no()); + err= DB_FAIL; goto release_page; } } @@ -3626,28 +3667,33 @@ dberr_t buf_page_t::read_complete(const fil_node_t &node) { database_corrupted: if (belongs_to_unzip_LRU()) +database_corrupted_compressed: memset_aligned(frame, 0, srv_page_size); - if (err == DB_PAGE_CORRUPTED) - { - ib::error() << "Database page corruption on disk" - " or a failed read of file '" - << node.name << "' page " << expected_id - << ". You may have to recover from a backup."; + if (!srv_force_recovery) + goto release_page; - buf_page_print(read_frame, zip_size()); - - node.space->set_corrupted(); - - ib::info() << " You can use CHECK TABLE to scan" - " your table for corruption. " - << FORCE_RECOVERY_MSG; - } - - if (err == DB_PAGE_CORRUPTED || err == DB_DECRYPTION_FAILED || - !srv_force_recovery) + if (err == DB_PAGE_CORRUPTED || err == DB_DECRYPTION_FAILED) { release_page: + if (recv_sys.free_corrupted_page(expected_id, node)); + else if (err == DB_FAIL) + err= DB_PAGE_CORRUPTED; + else + { + sql_print_error("InnoDB: Failed to read page " UINT32PF + " from file '%s': %s", expected_id.page_no(), + node.name, ut_strerr(err)); + + buf_page_print(read_frame, zip_size()); + + if (node.space->set_corrupted() && + !is_predefined_tablespace(node.space->id)) + sql_print_information("InnoDB: You can use CHECK TABLE to scan" + " your table for corruption. %s", + FORCE_RECOVERY_MSG); + } + buf_pool.corrupted_evict(this, buf_page_t::READ_FIX); return err; } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 2bf4960819f..5583c3fe604 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -33,6 +33,7 @@ Created 2011/12/19 #include "trx0sys.h" #include "fil0crypt.h" #include "fil0pagecompress.h" +#include "log.h" using st_::span; @@ -356,6 +357,9 @@ void buf_dblwr_t::recover() ut_ad(log_sys.last_checkpoint_lsn); if (!is_created()) return; + const lsn_t max_lsn{log_sys.get_lsn()}; + ut_ad(recv_sys.scanned_lsn == max_lsn); + ut_ad(recv_sys.scanned_lsn >= recv_sys.lsn); uint32_t page_no_dblwr= 0; byte *read_buf= static_cast(aligned_malloc(3 * srv_page_size, @@ -365,25 +369,15 @@ void buf_dblwr_t::recover() for (recv_dblwr_t::list::iterator i= recv_sys.dblwr.pages.begin(); i != recv_sys.dblwr.pages.end(); ++i, ++page_no_dblwr) { - byte *page= *i; + const page_t *const page= *i; const uint32_t page_no= page_get_page_no(page); - if (!page_no) /* recovered via recv_dblwr_t::restore_first_page() */ - continue; - const lsn_t lsn= mach_read_from_8(page + FIL_PAGE_LSN); - if (log_sys.last_checkpoint_lsn > lsn) - /* Pages written before the checkpoint are not useful for recovery. */ + if (log_sys.last_checkpoint_lsn > lsn || lsn > recv_sys.lsn) + /* Pages written before or after the recovery range are not usable. */ continue; const uint32_t space_id= page_get_space_id(page); const page_id_t page_id(space_id, page_no); - if (recv_sys.scanned_lsn < lsn) - { - ib::info() << "Ignoring a doublewrite copy of page " << page_id - << " with future log sequence number " << lsn; - continue; - } - fil_space_t *space= fil_space_t::get(space_id); if (!space) @@ -395,10 +389,14 @@ void buf_dblwr_t::recover() /* Do not report the warning for undo tablespaces, because they can be truncated in place. */ if (!srv_is_undo_tablespace(space_id)) - ib::warn() << "A copy of page " << page_no - << " in the doublewrite buffer slot " << page_no_dblwr - << " is beyond the end of " << space->chain.start->name - << " (" << space->size << " pages)"; + sql_print_warning("InnoDB: A copy of page " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " in the doublewrite buffer slot " UINT32PF + " is beyond the end of %s (" UINT32PF " pages)", + page_id.space(), page_id.page_no(), + page_no_dblwr, space->chain.start->name, + space->size); next_page: space->release(); continue; @@ -417,41 +415,48 @@ next_page: physical_size, read_buf); if (UNIV_UNLIKELY(fio.err != DB_SUCCESS)) - { - ib::warn() << "Double write buffer recovery: " << page_id - << " ('" << space->chain.start->name - << "') read failed with error: " << fio.err; - continue; - } - - if (buf_is_zeroes(span(read_buf, physical_size))) + sql_print_warning("InnoDB: Double write buffer recovery: " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " ('%s') read failed with error: %s", + page_id.space(), page_id.page_no(), fio.node->name, + ut_strerr(fio.err)); + else if (buf_is_zeroes(span(read_buf, physical_size))) { /* We will check if the copy in the doublewrite buffer is valid. If not, we will ignore this page (there should be redo log records to initialize it). */ } - else if (recv_sys.dblwr.validate_page(page_id, read_buf, space, buf)) + else if (recv_sys.dblwr.validate_page(page_id, max_lsn, space, + read_buf, buf)) goto next_page; else /* We intentionally skip this message for all-zero pages. */ - ib::info() << "Trying to recover page " << page_id - << " from the doublewrite buffer."; + sql_print_information("InnoDB: Trying to recover page " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " from the doublewrite buffer.", + page_id.space(), page_id.page_no()); - page= recv_sys.dblwr.find_page(page_id, space, buf); + if (const byte *page= + recv_sys.dblwr.find_page(page_id, max_lsn, space, buf)) + { + /* Write the good page from the doublewrite buffer to the intended + position. */ + space->reacquire(); + fio= space->io(IORequestWrite, + os_offset_t{page_id.page_no()} * physical_size, + physical_size, const_cast(page)); - if (!page) - goto next_page; + if (fio.err == DB_SUCCESS) + sql_print_information("InnoDB: Recovered page " + "[page id: space=" UINT32PF + ", page number=" UINT32PF "]" + " to '%s' from the doublewrite buffer.", + page_id.space(), page_id.page_no(), + fio.node->name); + } - /* Write the good page from the doublewrite buffer to the intended - position. */ - space->reacquire(); - fio= space->io(IORequestWrite, - os_offset_t{page_id.page_no()} * physical_size, - physical_size, page); - - if (fio.err == DB_SUCCESS) - ib::info() << "Recovered page " << page_id << " to '" << fio.node->name - << "' from the doublewrite buffer."; goto next_page; } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 78de045290b..390231b2dbd 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1594,7 +1594,7 @@ static ulint buf_flush_list(ulint max_n= ULINT_UNDEFINED, bool buf_flush_list_space(fil_space_t *space, ulint *n_flushed) { const auto space_id= space->id; - ut_ad(space_id <= SRV_SPACE_ID_UPPER_BOUND); + ut_ad(space_id < SRV_SPACE_ID_UPPER_BOUND); bool may_have_skipped= false; ulint max_n_flush= srv_io_capacity; diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 4a380a28637..f5b6ad2bb13 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1023,7 +1023,7 @@ buf_LRU_block_free_non_file_page( } /** Release a memory block to the buffer pool. */ -ATTRIBUTE_COLD void buf_pool_t::free_block(buf_block_t *block) +ATTRIBUTE_COLD void buf_pool_t::free_block(buf_block_t *block) noexcept { ut_ad(this == &buf_pool); mysql_mutex_lock(&mutex); @@ -1174,13 +1174,12 @@ static bool buf_LRU_block_remove_hashed(buf_page_t *bpage, const page_id_t id, @param bpage x-latched page that was found corrupted @param state expected current state of the page */ ATTRIBUTE_COLD -void buf_pool_t::corrupted_evict(buf_page_t *bpage, uint32_t state) +void buf_pool_t::corrupted_evict(buf_page_t *bpage, uint32_t state) noexcept { const page_id_t id{bpage->id()}; buf_pool_t::hash_chain &chain= buf_pool.page_hash.cell_get(id.fold()); page_hash_latch &hash_lock= buf_pool.page_hash.lock_get(chain); - recv_sys.free_corrupted_page(id); mysql_mutex_lock(&mutex); hash_lock.lock(); diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 1398861db4a..9549eee6f4a 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -261,15 +261,13 @@ buf_read_page_low( dst, bpage); if (UNIV_UNLIKELY(fio.err != DB_SUCCESS)) { + recv_sys.free_corrupted_page(page_id, *space->chain.start); buf_pool.corrupted_evict(bpage, buf_page_t::READ_FIX); } else if (sync) { thd_wait_end(nullptr); /* The i/o was already completed in space->io() */ fio.err = bpage->read_complete(*fio.node); space->release(); - if (fio.err == DB_FAIL) { - fio.err = DB_PAGE_CORRUPTED; - } if (mariadb_timer) { mariadb_increment_pages_read_time(mariadb_timer); } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index d9df7c85790..6ecd1eadaec 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -59,10 +59,14 @@ Created 10/25/1995 Heikki Tuuri #include "bzlib.h" #include "snappy-c.h" -ATTRIBUTE_COLD void fil_space_t::set_corrupted() const +ATTRIBUTE_COLD bool fil_space_t::set_corrupted() const noexcept { if (!is_stopping() && !is_corrupted.test_and_set()) + { sql_print_error("InnoDB: File '%s' is corrupted", chain.start->name); + return true; + } + return false; } /** Try to close a file to adhere to the innodb_open_files limit. @@ -82,7 +86,7 @@ bool fil_space_t::try_to_close(fil_space_t *ignore_space, bool print_info) case FIL_TYPE_IMPORT: break; case FIL_TYPE_TABLESPACE: - if (is_predefined_tablespace(space.id)) + if (space.id == TRX_SYS_SPACE || srv_is_undo_tablespace(space.id)) continue; } @@ -335,11 +339,14 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, return node; } -__attribute__((warn_unused_result, nonnull)) +__attribute__((warn_unused_result, nonnull(1))) /** Open a tablespace file. @param node data file +@param page first page of the tablespace, or nullptr +@param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the file was successfully opened */ -static bool fil_node_open_file_low(fil_node_t *node) +static bool fil_node_open_file_low(fil_node_t *node, const byte *page, + bool no_lsn) { ut_ad(!node->is_open()); ut_ad(node->space->is_closing()); @@ -395,7 +402,7 @@ static bool fil_node_open_file_low(fil_node_t *node) bool comp_algo_invalid = false; if (node->size); - else if (!node->read_page0() || + else if (!node->read_page0(page, no_lsn) || // validate compression algorithm for full crc32 format (node->space->full_crc32() && (comp_algo_invalid = !fil_comp_algo_loaded(comp_algo)))) @@ -426,8 +433,10 @@ static bool fil_node_open_file_low(fil_node_t *node) /** Open a tablespace file. @param node data file +@param page first page of the tablespace, or nullptr +@param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the file was successfully opened */ -static bool fil_node_open_file(fil_node_t *node) +static bool fil_node_open_file(fil_node_t *node, const byte *page, bool no_lsn) { mysql_mutex_assert_owner(&fil_system.mutex); ut_ad(!node->is_open()); @@ -466,7 +475,7 @@ static bool fil_node_open_file(fil_node_t *node) /* The node can be opened beween releasing and acquiring fil_system.mutex in the above code */ - return node->is_open() || fil_node_open_file_low(node); + return node->is_open() || fil_node_open_file_low(node, page, no_lsn); } /** Close the file handle. */ @@ -694,7 +703,8 @@ ATTRIBUTE_COLD bool fil_space_t::prepare_acquired() ut_ad(!id || purpose == FIL_TYPE_TEMPORARY || node == UT_LIST_GET_FIRST(chain)); - const bool is_open= node && (node->is_open() || fil_node_open_file(node)); + const bool is_open= node && + (node->is_open() || fil_node_open_file(node, nullptr, false)); if (!is_open) release(); @@ -1121,8 +1131,10 @@ bool fil_assign_new_space_id(uint32_t *space_id) } /** Read the first page of a data file. +@param dpage copy of a first page, from the doublewrite buffer, or nullptr +@param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the page was found valid */ -bool fil_space_t::read_page0() +bool fil_space_t::read_page0(const byte *dpage, bool no_lsn) noexcept { ut_ad(fil_system.is_initialised()); mysql_mutex_assert_owner(&fil_system.mutex); @@ -1139,32 +1151,26 @@ bool fil_space_t::read_page0() ut_ad("this should not happen" == 0); return false; } - const bool ok= node->is_open() || fil_node_open_file(node); + const bool ok= node->is_open() || fil_node_open_file(node, dpage, no_lsn); release(); return ok; } -/** Look up a tablespace and ensure that its first page has been validated. */ -static fil_space_t *fil_space_get_space(uint32_t id) -{ - if (fil_space_t *space= fil_space_get_by_id(id)) - if (space->read_page0()) - return space; - return nullptr; -} - void fil_space_set_recv_size_and_flags(uint32_t id, uint32_t size, uint32_t flags) { ut_ad(id < SRV_SPACE_ID_UPPER_BOUND); + mysql_mutex_assert_owner(&recv_sys.mutex); mysql_mutex_lock(&fil_system.mutex); - if (fil_space_t *space= fil_space_get_space(id)) - { - if (size) - space->recv_size= size; - if (flags != FSP_FLAGS_FCRC32_MASK_MARKER) - space->flags= flags; - } + if (fil_space_t *space= fil_space_get_by_id(id)) + if (space->read_page0(recv_sys.dblwr.find_page(page_id_t(id, 0), LSN_MAX), + true)) + { + if (size) + space->recv_size= size; + if (flags != FSP_FLAGS_FCRC32_MASK_MARKER) + space->flags= flags; + } mysql_mutex_unlock(&fil_system.mutex); } @@ -1179,12 +1185,16 @@ bool fil_space_t::open(bool create_new_db) bool success= true; bool skip_read= create_new_db; + const page_t *page= skip_read + ? nullptr + : recv_sys.dblwr.find_page(page_id_t{id, 0}, LSN_MAX); + mysql_mutex_lock(&fil_system.mutex); for (fil_node_t *node= UT_LIST_GET_FIRST(chain); node; node= UT_LIST_GET_NEXT(chain, node)) { - if (!node->is_open() && !fil_node_open_file_low(node)) + if (!node->is_open() && !fil_node_open_file_low(node, page, page)) { err_exit: success= false; @@ -1202,7 +1212,7 @@ err_exit: continue; } - if (!node->read_page0()) + if (!node->read_page0(page, true)) { fil_system.n_open--; os_file_close(node->handle); @@ -1211,6 +1221,7 @@ err_exit: } skip_read= true; + page= nullptr; } if (!create_new_db) @@ -3019,6 +3030,7 @@ void IORequest::read_complete(int io_error) const { sql_print_error("InnoDB: Read error %d of page " UINT32PF " in file %s", io_error, id.page_no(), node->name); + recv_sys.free_corrupted_page(id, *node); buf_pool.corrupted_evict(bpage, buf_page_t::READ_FIX); corrupted: if (recv_recovery_is_on() && !srv_force_recovery) @@ -3028,13 +3040,8 @@ void IORequest::read_complete(int io_error) const mysql_mutex_unlock(&recv_sys.mutex); } } - else if (dberr_t err= bpage->read_complete(*node)) - { - if (err != DB_FAIL) - ib::error() << "Failed to read page " << id.page_no() - << " from file '" << node->name << "': " << err; + else if (bpage->read_complete(*node)) goto corrupted; - } node->space->release(); } diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index 62f90f53a54..637d8eef00c 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -234,6 +234,46 @@ Datafile::same_as( #endif /* WIN32 */ } +dberr_t Datafile::read_first_page_flags(const page_t *page) noexcept +{ + ut_ad(m_order == 0); + + if (memcmp_aligned<2>(FIL_PAGE_SPACE_ID + page, + FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4)) + { + sql_print_error("InnoDB: Inconsistent tablespace ID in %s", m_filepath); + return DB_CORRUPTION; + } + + m_space_id= mach_read_from_4(FIL_PAGE_SPACE_ID + page); + m_flags= fsp_header_get_flags(page); + if (!fil_space_t::is_valid_flags(m_flags, m_space_id)) + { + uint32_t cflags= fsp_flags_convert_from_101(m_flags); + if (cflags == UINT32_MAX) + switch (fsp_flags_is_incompatible_mysql(m_flags)) { + case 0: + sql_print_error("InnoDB: Invalid flags 0x%zx in %s", + m_flags, m_filepath); + return DB_CORRUPTION; + case 3: + case 2: + sql_print_error("InnoDB: MySQL-8.0 tablespace in %s", m_filepath); + goto unsupported; + case 1: + sql_print_error("InnoDB: MySQL Encrypted tablespace in %s", + m_filepath); + unsupported: + sql_print_error("InnoDB: Restart in MySQL for migration/recovery."); + return DB_UNSUPPORTED; + } + else + m_flags= cflags; + } + + return DB_SUCCESS; +} + /** Reads a few significant fields from the first page of the first datafile. The Datafile must already be open. @param[in] read_only_mode If true, then readonly mode checks are enforced. @@ -286,56 +326,16 @@ Datafile::read_first_page(bool read_only_mode) } } - if (err != DB_SUCCESS) { - return(err); + if (err == DB_SUCCESS && m_order == 0) { + err = read_first_page_flags(m_first_page); } - if (m_order == 0) { - if (memcmp_aligned<2>(FIL_PAGE_SPACE_ID + m_first_page, - FSP_HEADER_OFFSET + FSP_SPACE_ID - + m_first_page, 4)) { - ib::error() - << "Inconsistent tablespace ID in " - << m_filepath; - return DB_CORRUPTION; - } - - m_space_id = mach_read_from_4(FIL_PAGE_SPACE_ID - + m_first_page); - m_flags = fsp_header_get_flags(m_first_page); - if (!fil_space_t::is_valid_flags(m_flags, m_space_id)) { - uint32_t cflags = fsp_flags_convert_from_101(m_flags); - if (cflags == UINT32_MAX) { - switch (fsp_flags_is_incompatible_mysql(m_flags)) { - case 0: - sql_print_error("InnoDB: Invalid flags 0x%" PRIx32 " in %s", - m_flags, m_filepath); - return DB_CORRUPTION; - case 3: - case 2: - sql_print_error("InnoDB: MySQL-8.0 tablespace in %s", - m_filepath); - break; - case 1: - sql_print_error("InnoDB: MySQL Encrypted tablespace in %s", - m_filepath); - break; - } - sql_print_error("InnoDB: Restart in MySQL for migration/recovery."); - return DB_UNSUPPORTED; - } else { - m_flags = cflags; - } - } - } - - const size_t physical_size = fil_space_t::physical_size(m_flags); - - if (physical_size > page_size) { + if (err == DB_SUCCESS + && fil_space_t::physical_size(m_flags) > page_size) { ib::error() << "File " << m_filepath << " should be longer than " << page_size << " bytes"; - return(DB_CORRUPTION); + err = DB_CORRUPTION; } return(err); @@ -366,7 +366,7 @@ dberr_t Datafile::validate_to_dd(uint32_t space_id, uint32_t flags) /* Validate this single-table-tablespace with the data dictionary, but do not compare the DATA_DIR flag, in case the tablespace was remotely located. */ - err = validate_first_page(); + err = validate_first_page(m_first_page); if (err != DB_SUCCESS) { return(err); } @@ -394,6 +394,72 @@ dberr_t Datafile::validate_to_dd(uint32_t space_id, uint32_t flags) return(DB_ERROR); } +inline +uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file) + const noexcept +{ + os_offset_t file_size= os_file_get_size(file); + if (file_size != (os_offset_t) -1) + { + for (const page_t *page : pages) + { + uint32_t space_id= page_get_space_id(page); + byte *read_page= nullptr; + if (page_get_page_no(page) > 0 || space_id == 0) + { +next_page: + aligned_free(read_page); + continue; + } + uint32_t flags= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + page); + size_t page_size= fil_space_t::physical_size(flags); + if (file_size < 4 * page_size) + goto next_page; + read_page= + static_cast(aligned_malloc(3 * page_size, page_size)); + /* Read 3 pages from the file and match the space id + with the space id which is stored in + doublewrite buffer page. */ + if (os_file_read(IORequestRead, file, read_page, page_size, + 3 * page_size, nullptr) != DB_SUCCESS) + goto next_page; + for (ulint j= 0; j <= 2; j++) + { + byte *cur_page= read_page + j * page_size; + if (buf_is_zeroes(span(cur_page, page_size))) + { + aligned_free(read_page); + return 0; + } + if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 || + memcmp(cur_page + FIL_PAGE_SPACE_ID, + page + FIL_PAGE_SPACE_ID, 4) || + buf_page_is_corrupted(false, cur_page, flags)) + goto next_page; + } + + aligned_free(read_page); + page= find_page(page_id_t{space_id, 0}, LSN_MAX); + + if (!page) + { + /* If the first page of the given user tablespace is not there + in the doublewrite buffer, then the recovery is going to fail + now. Report error only when doublewrite buffer is not empty */ + sql_print_error("InnoDB: Corrupted page " + "[page id: space=" UINT32PF ", page number=0]" + " of datafile '%s' could not be found" + " in the doublewrite buffer", space_id, name); + break; + } + + return space_id; + } + } + return 0; +} + /** Validates this datafile for the purpose of recovery. The file should exist and be successfully opened. We initially open it in read-only mode because we just want to read the SpaceID. However, if the first page is @@ -409,7 +475,7 @@ Datafile::validate_for_recovery() ut_ad(is_open()); ut_ad(!srv_read_only_mode); - err = validate_first_page(); + err = validate_first_page(m_first_page); switch (err) { case DB_TABLESPACE_EXISTS: @@ -426,14 +492,7 @@ Datafile::validate_for_recovery() m_space_id is set in read_first_page(). */ /* fall through */ default: - /* Re-open the file in read-write mode Attempt to restore - page 0 from doublewrite and read the space ID from a survey - of the first few pages. */ - close(); - err = open_read_write(); - if (err != DB_SUCCESS) { - return(err); - } + const page_t *first_page = nullptr; if (!m_space_id) { m_space_id = recv_sys.dblwr.find_first_page( @@ -459,15 +518,17 @@ Datafile::validate_for_recovery() return DB_SUCCESS; /* empty file */ } - if (recv_sys.dblwr.restore_first_page( - m_space_id, m_filepath, m_handle)) { + first_page = recv_sys.dblwr.find_page( + page_id_t(m_space_id, 0), LSN_MAX); + + if (!first_page) { return m_defer ? err : DB_CORRUPTION; } free_first_page: /* Free the previously read first page and then re-validate. */ free_first_page(); m_defer = false; - err = validate_first_page(); + err = validate_first_page(first_page); } return(err); @@ -477,22 +538,24 @@ free_first_page: tablespace is opened. This occurs before the fil_space_t is created so the Space ID found here must not already be open. m_is_valid is set true on success, else false. +@param[in] first_page the contents of the first page @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ -dberr_t Datafile::validate_first_page() +dberr_t Datafile::validate_first_page(const page_t *first_page) noexcept { const char* error_txt = NULL; m_is_valid = true; + ut_ad(first_page || !m_first_page); - if (m_first_page == NULL - && read_first_page(srv_read_only_mode) != DB_SUCCESS) { - + if (first_page) { + if (dberr_t err = read_first_page_flags(first_page)) { + m_is_valid = false; + return err; + } + } else if (read_first_page(srv_read_only_mode) != DB_SUCCESS) { error_txt = "Cannot read first page"; - } - - if (error_txt != NULL) { err_exit: free_first_page(); @@ -508,11 +571,14 @@ err_exit: error_txt, m_filepath, m_space_id, m_flags); m_is_valid = false; return DB_CORRUPTION; + } else { + first_page = m_first_page; + ut_ad(first_page); } /* Check if the whole page is blank. */ if (!m_space_id && !m_flags) { - const byte* b = m_first_page; + const byte* b = first_page; ulint nonzero_bytes = srv_page_size; while (*b == '\0' && --nonzero_bytes != 0) { @@ -550,7 +616,7 @@ err_exit: return(DB_ERROR); } - if (page_get_page_no(m_first_page) != 0) { + if (page_get_page_no(first_page) != 0) { /* First page must be number 0 */ error_txt = "Header page contains inconsistent data"; goto err_exit; @@ -561,8 +627,13 @@ err_exit: goto err_exit; } - if (buf_page_is_corrupted(false, m_first_page, m_flags)) { - /* Look for checksum and other corruptions. */ + switch (buf_page_is_corrupted(false, first_page, m_flags)) { + case NOT_CORRUPTED: + break; + case CORRUPTED_FUTURE_LSN: + error_txt = "LSN is in the future"; + goto err_exit; + case CORRUPTED_OTHER: error_txt = "Checksum mismatch"; goto err_exit; } diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index be52e7e4743..6bd2627ea85 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -596,17 +596,22 @@ inline dberr_t SysTablespace::read_lsn_and_check_flags() /* Check the contents of the first page of the first datafile. */ - err = it->validate_first_page(); + err = it->validate_first_page(it->m_first_page); + const page_t *first_page = it->m_first_page; if (err != DB_SUCCESS) { - if (recv_sys.dblwr.restore_first_page( - it->m_space_id, it->m_filepath, - it->handle())) { - it->close(); - return(err); + mysql_mutex_lock(&recv_sys.mutex); + first_page = recv_sys.dblwr.find_page( + page_id_t(space_id(), 0), LSN_MAX); + mysql_mutex_unlock(&recv_sys.mutex); + if (!first_page) { + err = DB_CORRUPTION; + } else { + err = it->read_first_page_flags(first_page); + if (err == DB_SUCCESS) { + err = it->validate_first_page(first_page); + } } - err = it->read_first_page( - m_ignore_read_only && srv_read_only_mode); } /* Make sure the tablespace space ID matches the @@ -629,7 +634,7 @@ inline dberr_t SysTablespace::read_lsn_and_check_flags() log_sys.latch.wr_lock(SRW_LOCK_CALL); /* Prepare for possible upgrade from 0-sized ib_logfile0. */ log_sys.next_checkpoint_lsn = mach_read_from_8( - it->m_first_page + 26/*FIL_PAGE_FILE_FLUSH_LSN*/); + first_page + 26/*FIL_PAGE_FILE_FLUSH_LSN*/); if (log_sys.next_checkpoint_lsn < 8204) { /* Before MDEV-14425, InnoDB had a minimum LSN of 8192+12=8204. Likewise, mariadb-backup diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 8316c02de95..3a5f8ee539a 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -277,13 +277,21 @@ buf_block_modify_clock_inc( @return whether the buffer is all zeroes */ bool buf_is_zeroes(st_::span buf); +/** Reason why buf_page_is_corrupted() fails */ +enum buf_page_is_corrupted_reason +{ + CORRUPTED_FUTURE_LSN= -1, + NOT_CORRUPTED= 0, + CORRUPTED_OTHER +}; + /** Check if a page is corrupt. @param check_lsn whether FIL_PAGE_LSN should be checked @param read_buf database page @param fsp_flags contents of FIL_SPACE_FLAGS @return whether the page is corrupted */ -bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, - uint32_t fsp_flags) +buf_page_is_corrupted_reason +buf_page_is_corrupted(bool check_lsn, const byte *read_buf, uint32_t fsp_flags) MY_ATTRIBUTE((warn_unused_result)); /** Read the key version from the page. In full crc32 format, @@ -718,9 +726,8 @@ public: /** Complete a read of a page. @param node data file @return whether the operation succeeded - @retval DB_PAGE_CORRUPTED if the checksum fails - @retval DB_DECRYPTION_FAILED if the page cannot be decrypted - @retval DB_FAIL if the page contains the wrong ID */ + @retval DB_PAGE_CORRUPTED if the checksum or the page ID is incorrect + @retval DB_DECRYPTION_FAILED if the page cannot be decrypted */ dberr_t read_complete(const fil_node_t &node); /** Release a write fix after a page write was completed. @@ -1302,10 +1309,11 @@ public: /** Release and evict a corrupted page. @param bpage x-latched page that was found corrupted @param state expected current state of the page */ - ATTRIBUTE_COLD void corrupted_evict(buf_page_t *bpage, uint32_t state); + ATTRIBUTE_COLD void corrupted_evict(buf_page_t *bpage, uint32_t state) + noexcept; /** Release a memory block to the buffer pool. */ - ATTRIBUTE_COLD void free_block(buf_block_t *block); + ATTRIBUTE_COLD void free_block(buf_block_t *block) noexcept; #ifdef UNIV_DEBUG /** Find a block that points to a ROW_FORMAT=COMPRESSED page diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 358297f3f23..d5f3a525f2c 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -481,8 +481,9 @@ public: written while the space ID is being updated in each page. */ inline void set_imported(); - /** Report the tablespace as corrupted */ - ATTRIBUTE_COLD void set_corrupted() const; + /** Report the tablespace as corrupted + @return whether this was the first call */ + ATTRIBUTE_COLD bool set_corrupted() const noexcept; /** @return whether the storage device is rotational (HDD, not SSD) */ inline bool is_rotational() const; @@ -939,8 +940,10 @@ public: void flush_low(); /** Read the first page of a data file. + @param dpage copy of a first page, from the doublewrite buffer, or nullptr + @param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the page was found valid */ - bool read_page0(); + bool read_page0(const byte *dpage, bool no_lsn) noexcept; /** Determine the next tablespace for encryption key rotation. @param space current tablespace (nullptr to start from the beginning) @@ -1044,8 +1047,10 @@ struct fil_node_t final bool is_open() const { return handle != OS_FILE_CLOSED; } /** Read the first page of a data file. + @param dpage copy of a first page, from the doublewrite buffer, or nullptr + @param no_lsn whether to skip the FIL_PAGE_LSN check @return whether the page was found valid */ - bool read_page0(); + bool read_page0(const byte *dpage, bool no_lsn) noexcept; /** Determine some file metadata when creating or reading the file. @param file the file that is being created, or OS_FILE_CLOSED */ @@ -1536,7 +1541,7 @@ inline uint32_t fil_space_t::get_size() if (!size) { mysql_mutex_lock(&fil_system.mutex); - read_page0(); + read_page0(nullptr, false); mysql_mutex_unlock(&fil_system.mutex); } return size; diff --git a/storage/innobase/include/fsp0file.h b/storage/innobase/include/fsp0file.h index 62dec39b91d..92d077d633c 100644 --- a/storage/innobase/include/fsp0file.h +++ b/storage/innobase/include/fsp0file.h @@ -216,10 +216,11 @@ public: tablespace is opened. This occurs before the fil_space_t is created so the Space ID found here must not already be open. m_is_valid is set true on success, else false. + @param first_page the contents of the first page @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ - dberr_t validate_first_page() + dberr_t validate_first_page(const byte *first_page) noexcept MY_ATTRIBUTE((warn_unused_result)); /** Get Datafile::m_filepath. @@ -360,6 +361,13 @@ private: dberr_t read_first_page(bool read_only_mode) MY_ATTRIBUTE((warn_unused_result)); + /** Read m_space_id, m_flags from a page frame. + @param page a copy of the first page of the tablespace + @retval DB_SUCCESS if the page seems to be valid + @retval DB_CORRUPTION if the page looks corrupted + @retval DB_UNSUPPORTED if the page is in an unsupported format */ + dberr_t read_first_page_flags(const byte *page) noexcept; + /** Free the first page from memory when it is no longer needed. */ void free_first_page(); diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 3c9fb5997b4..6f6c9f07d37 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -95,32 +95,28 @@ struct recv_dblwr_t /** Validate the page. @param page_id page identifier - @param page page contents + @param max_lsn the maximum allowed LSN @param space the tablespace of the page (not available for page 0) + @param page page contents @param tmp_buf 2*srv_page_size for decrypting and decompressing any page_compressed or encrypted pages @return whether the page is valid */ - bool validate_page(const page_id_t page_id, const byte *page, - const fil_space_t *space, byte *tmp_buf); + bool validate_page(const page_id_t page_id, lsn_t max_lsn, + const fil_space_t *space, + const byte *page, byte *tmp_buf) const noexcept; - /** Find a doublewrite copy of a page. + /** Find a doublewrite copy of a page with the smallest FIL_PAGE_LSN + that is large enough for recovery. @param page_id page identifier - @param space tablespace (not available for page_id.page_no()==0) + @param max_lsn the maximum allowed LSN + @param space tablespace (nullptr for page_id.page_no()==0) @param tmp_buf 2*srv_page_size for decrypting and decompressing any page_compressed or encrypted pages @return page frame - @retval NULL if no valid page for page_id was found */ - byte* find_page(const page_id_t page_id, const fil_space_t *space= NULL, - byte *tmp_buf= NULL); - - /** Restore the first page of the given tablespace from - doublewrite buffer. - @param space_id tablespace identifier - @param name tablespace filepath - @param file tablespace file handle - @return whether the operation failed */ - bool restore_first_page(uint32_t space_id, const char *name, - pfs_os_file_t file); + @retval nullptr if no valid page for page_id was found */ + const byte *find_page(const page_id_t page_id, lsn_t max_lsn, + const fil_space_t *space= nullptr, + byte *tmp_buf= nullptr) const noexcept; /** Restore the first page of the given tablespace from doublewrite buffer. @@ -131,7 +127,8 @@ struct recv_dblwr_t @param name tablespace filepath @param file tablespace file handle @return space_id or 0 in case of error */ - uint32_t find_first_page(const char *name, pfs_os_file_t file); + inline uint32_t find_first_page(const char *name, pfs_os_file_t file) + const noexcept; typedef std::deque > list; @@ -380,12 +377,15 @@ public: GOT_OOM }; + /** Whether to store parsed log records */ + enum store{NO,BACKUP,YES}; + private: /** Parse and register one log_t::FORMAT_10_8 mini-transaction. - @tparam store whether to store the records + @tparam storing whether to store the records @param l log data source @param if_exists if store: whether to check if the tablespace exists */ - template + template inline parse_mtr_result parse(source &l, bool if_exists) noexcept; /** Rewind a mini-transaction when parse() runs out of memory. @@ -399,20 +399,20 @@ private: public: /** Parse and register one log_t::FORMAT_10_8 mini-transaction, without handling any log_sys.is_mmap() buffer wrap-around. - @tparam store whether to store the records - @param if_exists if store: whether to check if the tablespace exists */ - template + @tparam storing whether to store the records + @param if_exists storing=YES: whether to check if the tablespace exists */ + template static parse_mtr_result parse_mtr(bool if_exists) noexcept; /** Parse and register one log_t::FORMAT_10_8 mini-transaction, handling log_sys.is_mmap() buffer wrap-around. - @tparam store whether to store the records - @param if_exists if store: whether to check if the tablespace exists */ - template + @tparam storing whether to store the records + @param if_exists storing=YES: whether to check if the tablespace exists */ + template static parse_mtr_result parse_mmap(bool if_exists) noexcept #ifdef HAVE_INNODB_MMAP ; #else - { return parse_mtr(if_exists); } + { return parse_mtr(if_exists); } #endif /** Erase log records for a page. */ @@ -435,20 +435,26 @@ public: inline void free(const void *data); /** Remove records for a corrupted page. - This function should only be called when innodb_force_recovery is set. - @param page_id corrupted page identifier */ - ATTRIBUTE_COLD void free_corrupted_page(page_id_t page_id); + @param page_id corrupted page identifier + @param node file for which an error is to be reported + @return whether an error message was reported */ + ATTRIBUTE_COLD bool free_corrupted_page(page_id_t page_id, + const fil_node_t &node) noexcept; /** Flag data file corruption during recovery. */ - ATTRIBUTE_COLD void set_corrupt_fs(); + ATTRIBUTE_COLD void set_corrupt_fs() noexcept; /** Flag log file corruption during recovery. */ - ATTRIBUTE_COLD void set_corrupt_log(); + ATTRIBUTE_COLD void set_corrupt_log() noexcept; /** @return whether data file corruption was found */ bool is_corrupt_fs() const { return UNIV_UNLIKELY(found_corrupt_fs); } /** @return whether log file corruption was found */ bool is_corrupt_log() const { return UNIV_UNLIKELY(found_corrupt_log); } + /** Check if recovery reached a consistent log sequence number. + @return whether the recovery failed to process enough log */ + inline bool validate_checkpoint() const noexcept; + /** Read a page or recover it based on redo log records. @param page_id page identifier @param mtr mini-transaction @@ -479,8 +485,3 @@ extern bool recv_needed_recovery; protected by exclusive log_sys.latch. */ extern bool recv_no_log_write; #endif /* UNIV_DEBUG */ - -/** TRUE if buf_page_is_corrupted() should check if the log sequence -number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by -recv_recovery_from_checkpoint_start(). */ -extern bool recv_lsn_checks_on; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index f559d9e85b4..e222c4c1531 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -65,16 +65,6 @@ Protected by log_sys.latch. */ bool recv_no_log_write = false; #endif /* UNIV_DEBUG */ -/** TRUE if buf_page_is_corrupted() should check if the log sequence -number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by -recv_recovery_from_checkpoint_start(). */ -bool recv_lsn_checks_on; - -/** The maximum lsn we see for a page during the recovery process. If this -is bigger than the lsn we are able to scan up to, that is an indication that -the recovery failed and the database may be corrupt. */ -static lsn_t recv_max_page_lsn; - /** Stored physical log record */ struct log_phys_t : public log_rec_t { @@ -845,7 +835,7 @@ processed: This is invoked if we found neither a valid first page in the data file nor redo log records that would initialize the first page. */ - void deferred_dblwr() + void deferred_dblwr(lsn_t max_lsn) { for (auto d= defers.begin(); d != defers.end(); ) { @@ -856,7 +846,7 @@ processed: continue; } const page_id_t page_id{d->first, 0}; - const byte *page= recv_sys.dblwr.find_page(page_id); + const byte *page= recv_sys.dblwr.find_page(page_id, max_lsn); if (!page) goto next_item; const uint32_t space_id= mach_read_from_4(page + FIL_PAGE_SPACE_ID); @@ -1350,7 +1340,6 @@ void recv_sys_t::create() progress_time = time(NULL); ut_ad(pages.empty()); pages_it = pages.end(); - recv_max_page_lsn = 0; memset(truncated_undo_spaces, 0, sizeof truncated_undo_spaces); truncated_sys_space= {0, 0}; @@ -1700,6 +1689,8 @@ dberr_t recv_sys_t::find_checkpoint() log_sys.last_checkpoint_lsn= log_sys.next_checkpoint_lsn; log_sys.set_recovered_lsn(log_sys.next_checkpoint_lsn); lsn= file_checkpoint= log_sys.next_checkpoint_lsn; + if (UNIV_LIKELY(lsn != 0)) + scanned_lsn= lsn; log_sys.next_checkpoint_no= 0; return DB_SUCCESS; } @@ -2093,17 +2084,10 @@ static void store_freed_or_init_rec(page_id_t page_id, bool freed) { uint32_t space_id= page_id.space(); uint32_t page_no= page_id.page_no(); - if (is_predefined_tablespace(space_id)) + if (space_id == TRX_SYS_SPACE || srv_is_undo_tablespace(space_id)) { - if (!srv_immediate_scrub_data_uncompressed) - return; - fil_space_t *space; - if (space_id == TRX_SYS_SPACE) - space= fil_system.sys_space; - else - space= fil_space_get(space_id); - - space->free_page(page_no, freed); + if (srv_immediate_scrub_data_uncompressed) + fil_space_get(space_id)->free_page(page_no, freed); return; } @@ -2414,28 +2398,29 @@ void recv_sys_t::rewind(source &l, source &begin) noexcept } /** Parse and register one log_t::FORMAT_10_8 mini-transaction. -@tparam store whether to store the records +@tparam storing whether to store the records @param l log data source @param if_exists if store: whether to check if the tablespace exists */ -template +template inline recv_sys_t::parse_mtr_result recv_sys_t::parse(source &l, bool if_exists) noexcept { restart: - ut_ad(log_sys.latch_have_wr() || - srv_operation == SRV_OPERATION_BACKUP || - srv_operation == SRV_OPERATION_BACKUP_NO_DEFER); + ut_ad(storing == BACKUP || log_sys.latch_have_wr()); + ut_ad(storing == BACKUP || !undo_space_trunc); + ut_ad(storing == BACKUP || !log_file_op); + ut_ad(storing == YES || !if_exists); + ut_ad((storing == BACKUP) == + (srv_operation == SRV_OPERATION_BACKUP || + srv_operation == SRV_OPERATION_BACKUP_NO_DEFER)); mysql_mutex_assert_owner(&mutex); ut_ad(log_sys.next_checkpoint_lsn); ut_ad(log_sys.is_latest()); - ut_ad(store || !if_exists); - ut_ad(store || - srv_operation != SRV_OPERATION_BACKUP || - srv_operation != SRV_OPERATION_BACKUP_NO_DEFER); alignas(8) byte iv[MY_AES_BLOCK_SIZE]; - byte *decrypt_buf= static_cast(alloca(srv_page_size)); + byte *decrypt_buf= storing == YES + ? static_cast(alloca(srv_page_size)) : nullptr; const lsn_t start_lsn{lsn}; @@ -2491,15 +2476,16 @@ restart: crc= my_crc32c(crc, iv, 8); } - DBUG_EXECUTE_IF("log_intermittent_checksum_mismatch", - { - static int c; - if (!c++) + if (storing == BACKUP) + DBUG_EXECUTE_IF("log_intermittent_checksum_mismatch", { - sql_print_information("Invalid log block checksum"); - return GOT_EOF; - } - }); + static int c; + if (!c++) + { + sql_print_information("Invalid log block checksum"); + return GOT_EOF; + } + }); if (crc != (l + 1).read4()) return GOT_EOF; @@ -2577,7 +2563,8 @@ restart: } sql_print_warning("InnoDB: Ignoring malformed log record at LSN " LSN_PF, lsn); - last_offset= 1; /* the next record must not be same_page */ + /* the next record must not be same_page */ + if (storing == YES) last_offset= 1; continue; } if (srv_operation == SRV_OPERATION_BACKUP) @@ -2587,7 +2574,7 @@ restart: lsn, b, l - recs + rlen, space_id, page_no)); goto same_page; } - last_offset= 0; + if (storing == YES) last_offset= 0; idlen= mlog_decode_varint_length(*l); if (UNIV_UNLIKELY(idlen > 5 || idlen >= rlen)) { @@ -2618,17 +2605,21 @@ restart: goto page_id_corrupted; l+= idlen; rlen-= idlen; - mach_write_to_4(iv + 8, space_id); - mach_write_to_4(iv + 12, page_no); + if (storing == YES) + { + mach_write_to_4(iv + 8, space_id); + mach_write_to_4(iv + 12, page_no); + } got_page_op= !(b & 0x80); if (!got_page_op); - else if (!store && srv_operation == SRV_OPERATION_BACKUP) + else if (storing == BACKUP && srv_operation == SRV_OPERATION_BACKUP) { - if (page_no == 0 && first_page_init && (b & 0x10)) + if (page_no == 0 && (b & 0xf0) == INIT_PAGE && first_page_init) first_page_init(space_id); continue; } - else if (store && file_checkpoint && !is_predefined_tablespace(space_id)) + else if (storing == YES && file_checkpoint && + space_id != TRX_SYS_SPACE && !srv_is_undo_tablespace(space_id)) { recv_spaces_t::iterator i= recv_spaces.lower_bound(space_id); if (i != recv_spaces.end() && i->first == space_id); @@ -2657,7 +2648,6 @@ restart: if (got_page_op) { same_page: - const byte *cl= l.ptr; if (!rlen); else if (UNIV_UNLIKELY(l - recs + rlen > srv_page_size)) goto record_corrupted; @@ -2665,27 +2655,65 @@ restart: ut_d(if ((b & 0x70) == INIT_PAGE || (b & 0x70) == OPTION) freed.erase(id)); ut_ad(freed.find(id) == freed.end()); + const byte *cl= storing == NO ? nullptr : l.ptr; switch (b & 0x70) { case FREE_PAGE: ut_ad(freed.emplace(id).second); - last_offset= 1; /* the next record must not be same_page */ + /* the next record must not be same_page */ + if (storing == YES) last_offset= 1; goto free_or_init_page; case INIT_PAGE: - last_offset= FIL_PAGE_TYPE; + if (storing == YES) last_offset= FIL_PAGE_TYPE; free_or_init_page: - store_freed_or_init_rec(id, (b & 0x70) == FREE_PAGE); + if (storing == BACKUP) + continue; if (UNIV_UNLIKELY(rlen != 0)) goto record_corrupted; + store_freed_or_init_rec(id, (b & 0x70) == FREE_PAGE); + + if (storing == NO) + { + /* We must update mlog_init for the correct operation of + multi-batch recovery, for example to avoid occasional + failures of the test innodb.recovery_memory. + + For storing == YES, this will be invoked in recv_sys_t::add(). */ + mlog_init.add(id, start_lsn); + + /* recv_scan_log() may have stored some log for this page + before entering the skip_the_rest: loop. Such records must + be discarded, because reading an INIT_PAGE or FREE_PAGE + record implies that the page can be recovered based on log + records, without reading it from a data file. */ + + if (pages_it == pages.end() || pages_it->first != id) + { + pages_it= pages.find(id); + if (pages_it == pages.end()) + continue; + } + map::iterator r= pages_it++; + ut_ad(!r->second.being_processed); + erase(r); + continue; + } copy_if_needed: cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen); break; case EXTENDED: + if (storing != YES) + continue; if (UNIV_UNLIKELY(!rlen)) goto record_corrupted; cl= l.copy_if_needed(iv, decrypt_buf, recs, rlen); if (rlen == 1 && *cl == TRIM_PAGES) { - if (srv_is_undo_tablespace(space_id)) + if (storing == BACKUP) + { + if (space_id && undo_space_trunc) + undo_space_trunc(space_id); + } + else if (srv_is_undo_tablespace(space_id)) { if (page_no != SRV_UNDO_TABLESPACE_SIZE_IN_PAGES) goto record_corrupted; @@ -2705,15 +2733,14 @@ restart: } static_assert(UT_ARR_SIZE(truncated_undo_spaces) == TRX_SYS_MAX_UNDO_SPACES, "compatibility"); - if (!store && undo_space_trunc && space_id) - undo_space_trunc(space_id); - last_offset= 1; /* the next record must not be same_page */ + /* the next record must not be same_page */ + if (storing == YES) last_offset= 1; continue; } - last_offset= FIL_PAGE_TYPE; + if (storing == YES) last_offset= FIL_PAGE_TYPE; break; case OPTION: - if (rlen == 5 && *l == OPT_PAGE_CHECKSUM) + if (storing == YES && rlen == 5 && *l == OPT_PAGE_CHECKSUM) goto copy_if_needed; /* fall through */ case RESERVED: @@ -2721,6 +2748,8 @@ restart: case WRITE: case MEMMOVE: case MEMSET: + if (storing != YES) + continue; if (UNIV_UNLIKELY(rlen == 0 || last_offset == 1)) goto record_corrupted; ut_d(const source payload{l}); @@ -2761,7 +2790,7 @@ restart: last_offset) : file_name_t::initial_flags; if (it == recv_spaces.end()) - ut_ad(!store || space_id == TRX_SYS_SPACE || + ut_ad(space_id == TRX_SYS_SPACE || srv_is_undo_tablespace(space_id)); else if (!it->second.space) { @@ -2821,7 +2850,7 @@ restart: ut_ad(modified.emplace(id).second || (b & 0x70) != INIT_PAGE); } #endif - if (store) + if (storing == YES) { if (if_exists) { @@ -2844,7 +2873,8 @@ restart: l - recs + rlen))) { lsn= start_lsn; - log_sys.set_recovered_lsn(start_lsn); + if (lsn > log_sys.get_lsn()) + log_sys.set_recovered_lsn(start_lsn); l+= rlen; offset= begin.ptr - log_sys.buf; rewind(l, begin); @@ -2856,23 +2886,11 @@ restart: goto restart; } sql_print_information("InnoDB: Multi-batch recovery needed at LSN " - LSN_PF, lsn); + LSN_PF, start_lsn); return GOT_OOM; } } } - else if ((b & 0x70) <= INIT_PAGE) - { - mlog_init.add(id, start_lsn); - if (pages_it == pages.end() || pages_it->first != id) - { - pages_it= pages.find(id); - if (pages_it == pages.end()) - continue; - } - map::iterator r= pages_it++; - erase(r); - } } else if (rlen) { @@ -2884,7 +2902,7 @@ restart: if (rlen < UNIV_PAGE_SIZE_MAX && !l.is_zero(rlen)) continue; } - else if (store) + else if (storing == YES) { ut_ad(file_checkpoint); continue; @@ -2962,7 +2980,7 @@ restart: goto file_rec_error; } - if (is_predefined_tablespace(space_id)) + if (space_id == TRX_SYS_SPACE || srv_is_undo_tablespace(space_id)) goto file_rec_error; if (fnend - fn < 4 || memcmp(fnend - 4, DOT_IBD, 4)) goto file_rec_error; @@ -2970,9 +2988,7 @@ restart: if (UNIV_UNLIKELY(!recv_needed_recovery && srv_read_only_mode)) continue; - if (!store && - (srv_operation == SRV_OPERATION_BACKUP || - srv_operation == SRV_OPERATION_BACKUP_NO_DEFER)) + if (storing == BACKUP) { if ((b & 0xf0) < FILE_CHECKPOINT && log_file_op) log_file_op(space_id, b & 0xf0, @@ -3014,22 +3030,23 @@ restart: return OK; } -template +template recv_sys_t::parse_mtr_result recv_sys_t::parse_mtr(bool if_exists) noexcept { recv_buf s{&log_sys.buf[recv_sys.offset]}; - return recv_sys.parse(s, if_exists); + return recv_sys.parse(s, if_exists); } /** for mariadb-backup; @see xtrabackup_copy_logfile() */ template -recv_sys_t::parse_mtr_result recv_sys_t::parse_mtr(bool) noexcept; +recv_sys_t::parse_mtr_result +recv_sys_t::parse_mtr(bool) noexcept; #ifdef HAVE_INNODB_MMAP -template +template recv_sys_t::parse_mtr_result recv_sys_t::parse_mmap(bool if_exists) noexcept { - recv_sys_t::parse_mtr_result r{parse_mtr(if_exists)}; + recv_sys_t::parse_mtr_result r{parse_mtr(if_exists)}; if (UNIV_LIKELY(r != PREMATURE_EOF) || !log_sys.is_mmap()) return r; ut_ad(recv_sys.len == log_sys.file_size); @@ -3039,12 +3056,13 @@ recv_sys_t::parse_mtr_result recv_sys_t::parse_mmap(bool if_exists) noexcept {recv_sys.offset == recv_sys.len ? &log_sys.buf[log_sys.START_OFFSET] : &log_sys.buf[recv_sys.offset]}; - return recv_sys.parse(s, if_exists); + return recv_sys.parse(s, if_exists); } /** for mariadb-backup; @see xtrabackup_copy_mmap_logfile() */ template -recv_sys_t::parse_mtr_result recv_sys_t::parse_mmap(bool) noexcept; +recv_sys_t::parse_mtr_result +recv_sys_t::parse_mmap(bool) noexcept; #endif /** Apply the hashed log records to the page, if the page lsn is less than the @@ -3213,11 +3231,18 @@ set_start_lsn: mtr.discard_modifications(); mtr.commit(); + fil_space_t* s = space + ? space + : fil_space_t::get(block->page.id().space()); + buf_pool.corrupted_evict(&block->page, block->page.state() & buf_page_t::LRU_MASK); - block = nullptr; - goto done; + if (!space) { + s->release(); + } + + return nullptr; } if (!start_lsn) { @@ -3257,29 +3282,26 @@ set_start_lsn: mtr.discard_modifications(); mtr.commit(); -done: - /* FIXME: do this in page read, protected with recv_sys.mutex! */ - if (recv_max_page_lsn < page_lsn) { - recv_max_page_lsn = page_lsn; - } - return block; } /** Remove records for a corrupted page. -This function should only be called when innodb_force_recovery is set. -@param page_id corrupted page identifier */ -ATTRIBUTE_COLD void recv_sys_t::free_corrupted_page(page_id_t page_id) +@param page_id corrupted page identifier +@param node file for which an error is to be reported +@return whether an error message was reported */ +ATTRIBUTE_COLD +bool recv_sys_t::free_corrupted_page(page_id_t page_id, + const fil_node_t &node) noexcept { if (!recovery_on) - return; + return false; mysql_mutex_lock(&mutex); map::iterator p= pages.find(page_id); if (p == pages.end()) { mysql_mutex_unlock(&mutex); - return; + return false; } p->second.being_processed= -1; @@ -3287,18 +3309,20 @@ ATTRIBUTE_COLD void recv_sys_t::free_corrupted_page(page_id_t page_id) set_corrupt_fs(); mysql_mutex_unlock(&mutex); - ib::error_or_warn(!srv_force_recovery) - << "Unable to apply log to corrupted page " << page_id; + (srv_force_recovery ? sql_print_warning : sql_print_error) + ("InnoDB: Unable to apply log to corrupted page " UINT32PF + " in file %s", page_id.page_no(), node.name); + return true; } -ATTRIBUTE_COLD void recv_sys_t::set_corrupt_log() +ATTRIBUTE_COLD void recv_sys_t::set_corrupt_log() noexcept { mysql_mutex_lock(&mutex); found_corrupt_log= true; mysql_mutex_unlock(&mutex); } -ATTRIBUTE_COLD void recv_sys_t::set_corrupt_fs() +ATTRIBUTE_COLD void recv_sys_t::set_corrupt_fs() noexcept { mysql_mutex_assert_owner(&mutex); if (!srv_force_recovery) @@ -4044,7 +4068,7 @@ static bool recv_scan_log(bool last_phase) for (;;) { const byte& b{log_sys.buf[recv_sys.offset]}; - r= recv_sys.parse_mmap(false); + r= recv_sys.parse_mmap(false); switch (r) { case recv_sys_t::PREMATURE_EOF: goto read_more; @@ -4074,7 +4098,7 @@ static bool recv_scan_log(bool last_phase) else { ut_ad(recv_sys.file_checkpoint != 0); - switch ((r= recv_sys.parse_mmap(false))) { + switch ((r= recv_sys.parse_mmap(false))) { case recv_sys_t::PREMATURE_EOF: goto read_more; case recv_sys_t::GOT_EOF: @@ -4096,11 +4120,13 @@ static bool recv_scan_log(bool last_phase) if (!store) skip_the_rest: - while ((r= recv_sys.parse_mmap(false)) == recv_sys_t::OK); + while ((r= recv_sys.parse_mmap(false)) == + recv_sys_t::OK); else { uint16_t count= 0; - while ((r= recv_sys.parse_mmap(last_phase)) == recv_sys_t::OK) + while ((r= recv_sys.parse_mmap(last_phase)) == + recv_sys_t::OK) if (!++count && recv_sys.report(time(nullptr))) { const size_t n= recv_sys.pages.size(); @@ -4130,7 +4156,8 @@ static bool recv_scan_log(bool last_phase) ut_ad(recv_sys.is_initialised()); if (recv_sys.scanned_lsn > 1) { - ut_ad(recv_sys.scanned_lsn == recv_sys.lsn); + ut_ad(recv_sys.is_corrupt_fs() || + recv_sys.scanned_lsn == recv_sys.lsn); break; } recv_sys.scanned_lsn= recv_sys.lsn; @@ -4247,7 +4274,7 @@ recv_validate_tablespace(bool rescan, bool& missing_tablespace) p != recv_sys.pages.end();) { ut_ad(!p->second.log.empty()); const uint32_t space = p->first.space(); - if (is_predefined_tablespace(space)) { + if (space == TRX_SYS_SPACE || srv_is_undo_tablespace(space)) { next: p++; continue; @@ -4499,6 +4526,18 @@ inline void log_t::set_recovered() noexcept set_buf_free(offset); } +inline bool recv_sys_t::validate_checkpoint() const noexcept +{ + if (lsn >= file_checkpoint && lsn >= log_sys.next_checkpoint_lsn) + return false; + sql_print_error("InnoDB: The log was only scanned up to " + LSN_PF ", while the current LSN at the " + "time of the latest checkpoint " LSN_PF + " was " LSN_PF "!", + lsn, log_sys.next_checkpoint_lsn, file_checkpoint); + return true; +} + /** Start recovering from a redo log checkpoint. of first system tablespace page @return error code or DB_SUCCESS */ @@ -4526,9 +4565,7 @@ dberr_t recv_recovery_from_checkpoint_start() log_sys.latch.wr_lock(SRW_LOCK_CALL); log_sys.set_capacity(); - /* Start reading the log from the checkpoint lsn. The variable - contiguous_lsn contains an lsn up to which the log is known to - be contiguously written. */ + /* Start reading the log from the checkpoint lsn. */ ut_ad(recv_sys.pages.empty()); @@ -4560,12 +4597,12 @@ read_only_recovery: goto err_exit; } ut_ad(recv_sys.file_checkpoint); + ut_ad(log_sys.get_lsn() >= recv_sys.scanned_lsn); if (rewind) { recv_sys.lsn = log_sys.next_checkpoint_lsn; recv_sys.offset = 0; recv_sys.len = 0; } - ut_ad(!recv_max_page_lsn); rescan = recv_scan_log(false); if (srv_read_only_mode && recv_needed_recovery) { @@ -4578,7 +4615,7 @@ read_only_recovery: } } - log_sys.set_recovered_lsn(recv_sys.lsn); + log_sys.set_recovered_lsn(recv_sys.scanned_lsn); if (recv_needed_recovery) { bool missing_tablespace = false; @@ -4622,14 +4659,17 @@ read_only_recovery: tablespaces (not individual pages), while retaining the initial recv_sys.pages. */ mysql_mutex_lock(&recv_sys.mutex); + ut_ad(log_sys.get_lsn() >= recv_sys.lsn); recv_sys.clear(); recv_sys.lsn = log_sys.next_checkpoint_lsn; mysql_mutex_unlock(&recv_sys.mutex); } if (srv_operation <= SRV_OPERATION_EXPORT_RESTORED) { - deferred_spaces.deferred_dblwr(); + mysql_mutex_lock(&recv_sys.mutex); + deferred_spaces.deferred_dblwr(log_sys.get_lsn()); buf_dblwr.recover(); + mysql_mutex_unlock(&recv_sys.mutex); } ut_ad(srv_force_recovery <= SRV_FORCE_NO_UNDO_LOG_SCAN); @@ -4651,21 +4691,8 @@ read_only_recovery: ut_ad(recv_sys.pages.empty()); } - if (log_sys.is_latest() - && (recv_sys.lsn < log_sys.next_checkpoint_lsn - || recv_sys.lsn < recv_max_page_lsn)) { - - sql_print_error("InnoDB: We scanned the log up to " LSN_PF "." - " A checkpoint was at " LSN_PF - " and the maximum LSN on a database page was " - LSN_PF ". It is possible that the" - " database is now corrupt!", - recv_sys.lsn, - log_sys.next_checkpoint_lsn, - recv_max_page_lsn); - } - - if (recv_sys.lsn < log_sys.next_checkpoint_lsn) { + if (!log_sys.is_latest()) { + } else if (recv_sys.validate_checkpoint()) { err_exit: err = DB_ERROR; goto func_exit; @@ -4701,7 +4728,6 @@ err_exit: err = recv_rename_files(); } - recv_lsn_checks_on = true; mysql_mutex_unlock(&recv_sys.mutex); /* The database is now ready to start almost normal processing of user @@ -4715,14 +4741,17 @@ err_exit: goto func_exit; } -bool recv_dblwr_t::validate_page(const page_id_t page_id, - const byte *page, +bool recv_dblwr_t::validate_page(const page_id_t page_id, lsn_t max_lsn, const fil_space_t *space, - byte *tmp_buf) + const byte *page, byte *tmp_buf) + const noexcept { + mysql_mutex_assert_owner(&recv_sys.mutex); + uint32_t flags; + if (page_id.page_no() == 0) { - uint32_t flags= fsp_header_get_flags(page); + flags= fsp_header_get_flags(page); if (!fil_space_t::is_valid_flags(flags, page_id.space())) { uint32_t cflags= fsp_flags_convert_from_101(flags); @@ -4737,9 +4766,15 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, } /* Page 0 is never page_compressed or encrypted. */ - return !buf_page_is_corrupted(true, page, flags); + goto check_if_corrupted; } + flags= space->flags; + + if (space->full_crc32()) + check_if_corrupted: + return !buf_page_is_corrupted(max_lsn < LSN_MAX, page, flags); + ut_ad(tmp_buf); byte *tmp_frame= tmp_buf; byte *tmp_page= tmp_buf + srv_page_size; @@ -4747,9 +4782,6 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, const bool expect_encrypted= space->crypt_data && space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED; - if (space->full_crc32()) - return !buf_page_is_corrupted(true, page, space->flags); - if (expect_encrypted && mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)) { @@ -4776,126 +4808,45 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, return false; /* decompression failed */ if (decomp == srv_page_size) return false; /* the page was not compressed (invalid page type) */ - return !buf_page_is_corrupted(true, tmp_page, space->flags); + page= tmp_page; } - return !buf_page_is_corrupted(true, page, space->flags); + goto check_if_corrupted; } -byte *recv_dblwr_t::find_page(const page_id_t page_id, - const fil_space_t *space, byte *tmp_buf) +const byte *recv_dblwr_t::find_page(const page_id_t page_id, lsn_t max_lsn, + const fil_space_t *space, byte *tmp_buf) + const noexcept { - byte *result= NULL; - lsn_t max_lsn= 0; + mysql_mutex_assert_owner(&recv_sys.mutex); + ut_ad(recv_sys.scanned_lsn <= max_lsn); for (byte *page : pages) { if (page_get_page_no(page) != page_id.page_no() || page_get_space_id(page) != page_id.space()) continue; + const lsn_t lsn= mach_read_from_8(page + FIL_PAGE_LSN); if (page_id.page_no() == 0) { + if (!lsn) + continue; uint32_t flags= mach_read_from_4( FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); if (!fil_space_t::is_valid_flags(flags, page_id.space())) continue; } - const lsn_t lsn= mach_read_from_8(page + FIL_PAGE_LSN); - if (lsn <= max_lsn || - !validate_page(page_id, page, space, tmp_buf)) + if (lsn > max_lsn || lsn < log_sys.next_checkpoint_lsn || + !validate_page(page_id, max_lsn, space, page, tmp_buf)) { /* Mark processed for subsequent iterations in buf_dblwr_t::recover() */ - memset(page + FIL_PAGE_LSN, 0, 8); + memset_aligned<8>(page + FIL_PAGE_LSN, 0, 8); continue; } - ut_a(page_get_page_no(page) == page_id.page_no()); - max_lsn= lsn; - result= page; + return page; } - return result; -} - -bool recv_dblwr_t::restore_first_page(uint32_t space_id, const char *name, - pfs_os_file_t file) -{ - const page_id_t page_id(space_id, 0); - const byte* page= find_page(page_id); - if (!page) - { - /* If the first page of the given user tablespace is not there - in the doublewrite buffer, then the recovery is going to fail - now. Report error only when doublewrite buffer is not empty */ - if (pages.size()) - ib::error() << "Corrupted page " << page_id << " of datafile '" - << name << "' could not be found in the doublewrite buffer."; - return true; - } - - ulint physical_size= fil_space_t::physical_size( - mach_read_from_4(page + FSP_HEADER_OFFSET + FSP_SPACE_FLAGS)); - ib::info() << "Restoring page " << page_id << " of datafile '" - << name << "' from the doublewrite buffer. Writing " - << physical_size << " bytes into file '" << name << "'"; - - return os_file_write( - IORequestWrite, name, file, page, 0, physical_size) != - DB_SUCCESS; -} - -uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file) -{ - os_offset_t file_size= os_file_get_size(file); - if (file_size != (os_offset_t) -1) - { - for (const page_t *page : pages) - { - uint32_t space_id= page_get_space_id(page); - byte *read_page= nullptr; - if (page_get_page_no(page) > 0 || space_id == 0) - { -next_page: - aligned_free(read_page); - continue; - } - uint32_t flags= mach_read_from_4( - FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - page_id_t page_id(space_id, 0); - size_t page_size= fil_space_t::physical_size(flags); - if (file_size < 4 * page_size) - goto next_page; - read_page= - static_cast(aligned_malloc(3 * page_size, page_size)); - /* Read 3 pages from the file and match the space id - with the space id which is stored in - doublewrite buffer page. */ - if (os_file_read(IORequestRead, file, read_page, page_size, - 3 * page_size, nullptr) != DB_SUCCESS) - goto next_page; - for (ulint j= 0; j <= 2; j++) - { - byte *cur_page= read_page + j * page_size; - if (buf_is_zeroes(span(cur_page, page_size))) - { - space_id= 0; - goto early_exit; - } - if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 || - memcmp(cur_page + FIL_PAGE_SPACE_ID, - page + FIL_PAGE_SPACE_ID, 4) || - buf_page_is_corrupted(false, cur_page, flags)) - goto next_page; - } - if (!restore_first_page(space_id, name, file)) - { -early_exit: - aligned_free(read_page); - return space_id; - } - break; - } - } - return 0; + return nullptr; } diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index dd2415d895c..04aafa0e580 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -1151,6 +1151,8 @@ std::pair mtr_t::do_write() ut_ad(is_logged()); ut_ad(m_log.size()); ut_ad(!m_latch_ex || log_sys.latch_have_wr()); + ut_ad(!m_user_space || + (m_user_space->id > 0 && m_user_space->id < SRV_SPACE_ID_UPPER_BOUND)); #ifndef DBUG_OFF do @@ -1189,7 +1191,7 @@ std::pair mtr_t::do_write() log_sys.latch.rd_lock(SRW_LOCK_CALL); if (UNIV_UNLIKELY(m_user_space && !m_user_space->max_lsn && - !is_predefined_tablespace(m_user_space->id))) + !srv_is_undo_tablespace((m_user_space->id)))) { if (!m_latch_ex) { diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 6ea53d80952..08df7e1ef32 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -3935,9 +3935,10 @@ void fil_node_t::find_metadata(os_file_t file /** Read the first page of a data file. @return whether the page was found valid */ -bool fil_node_t::read_page0() +bool fil_node_t::read_page0(const byte *dpage, bool no_lsn) noexcept { mysql_mutex_assert_owner(&fil_system.mutex); + ut_ad(!dpage || no_lsn); const unsigned psize= space->physical_size(); #ifndef _WIN32 struct stat statbuf; @@ -3961,15 +3962,18 @@ bool fil_node_t::read_page0() if (!deferred) { - page_t *page= static_cast(aligned_malloc(psize, psize)); - if (os_file_read(IORequestRead, handle, page, 0, psize, nullptr) - != DB_SUCCESS) + page_t *apage= static_cast(aligned_malloc(psize, psize)); + if (os_file_read(IORequestRead, handle, apage, 0, psize, nullptr) != + DB_SUCCESS) { sql_print_error("InnoDB: Unable to read first page of file %s", name); - aligned_free(page); + err_exit: + aligned_free(apage); return false; } + const page_t *page= apage; + retry: const ulint space_id= memcmp_aligned<2> (FIL_PAGE_SPACE_ID + page, FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4) @@ -3977,8 +3981,16 @@ bool fil_node_t::read_page0() : mach_read_from_4(FIL_PAGE_SPACE_ID + page); uint32_t flags= fsp_header_get_flags(page); const uint32_t size= fsp_header_get_field(page, FSP_SIZE); + if (!space_id && !flags && !size && dpage) + { + retry_dpage: + page= dpage; + dpage= nullptr; + goto retry; + } const uint32_t free_limit= fsp_header_get_field(page, FSP_FREE_LIMIT); const uint32_t free_len= flst_get_len(FSP_HEADER_OFFSET + FSP_FREE + page); + if (!fil_space_t::is_valid_flags(flags, space->id)) { uint32_t cflags= fsp_flags_convert_from_101(flags); @@ -3995,7 +4007,6 @@ bool fil_node_t::read_page0() } } - aligned_free(page); goto invalid; } @@ -4004,28 +4015,40 @@ bool fil_node_t::read_page0() !fil_space_t::is_flags_equal((space->flags & ~FSP_FLAGS_MEM_MASK), (flags & ~FSP_FLAGS_MEM_MASK))) { -invalid: + invalid: + if (dpage) + goto retry_dpage; sql_print_error("InnoDB: Expected tablespace flags 0x%zx but found 0x%zx" " in the file %s", space->flags, flags, name); - return false; + goto err_exit; } flags_ok: ut_ad(!(flags & FSP_FLAGS_MEM_MASK)); + if (buf_page_is_corrupted(!no_lsn, page, flags) != NOT_CORRUPTED) + { + if (dpage) + goto retry_dpage; + sql_print_error("InnoDB: The first page of file %s is corrupted", name); + goto err_exit; + } + + if (UNIV_UNLIKELY(space_id != space->id)) + { + if (dpage) + goto retry_dpage; + sql_print_error("InnoDB: Expected tablespace id %zu but found %zu" + " in the file %s", ulint{space->id}, ulint{space_id}, + name); + goto err_exit; + } + /* Try to read crypt_data from page 0 if it is not yet read. */ if (!space->crypt_data) space->crypt_data= fil_space_read_crypt_data( fil_space_t::zip_size(flags), page); - aligned_free(page); - - if (UNIV_UNLIKELY(space_id != space->id)) - { - ib::error() << "Expected tablespace id " << space->id - << " but found " << space_id - << " in the file " << name; - return false; - } + aligned_free(apage); space->flags= (space->flags & FSP_FLAGS_MEM_MASK) | flags; ut_ad(space->free_limit == 0 || space->free_limit == free_limit); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index fe89cdb7b55..44268d5b9fb 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -221,7 +221,12 @@ close_and_exit: if (!log_sys.attach(file, srv_log_file_size)) { goto close_and_exit; } - if (!fil_system.sys_space->open(create_new_db)) { + + mysql_mutex_lock(&recv_sys.mutex); + const bool all_opened = fil_system.sys_space->open(create_new_db); + mysql_mutex_unlock(&recv_sys.mutex); + + if (!all_opened) { goto err_exit; } @@ -664,13 +669,14 @@ static uint32_t srv_undo_tablespace_open(bool create, const char* name, ulint n_retries = 5; os_offset_t size= os_file_get_size(fh); ut_a(size != os_offset_t(-1)); + page_t *apage= nullptr; + const page_t *page= nullptr; if (!create) { - page_t *page= static_cast(aligned_malloc(srv_page_size, - srv_page_size)); + apage= static_cast(aligned_malloc(srv_page_size, srv_page_size)); undo_retry: - if (os_file_read(IORequestRead, fh, page, 0, srv_page_size, nullptr) != + if (os_file_read(IORequestRead, fh, apage, 0, srv_page_size, nullptr) != DB_SUCCESS) { err_exit: @@ -681,11 +687,12 @@ err_exit: n_retries--; goto undo_retry; } - ib::error() << "Unable to read first page of file " << name; - aligned_free(page); + sql_print_error("InnoDB: Unable to read first page of file %s", name); + aligned_free(apage); return ~0U; } + page= apage; DBUG_EXECUTE_IF("undo_space_read_fail", goto err_exit;); uint32_t id= mach_read_from_4(FIL_PAGE_SPACE_ID + page); @@ -693,18 +700,30 @@ err_exit: memcmp_aligned<2>(FIL_PAGE_SPACE_ID + page, FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4)) { - ib::error() << "Inconsistent tablespace ID in file " << name; + sql_print_error("InnoDB: Inconsistent tablespace ID in file %s", name); goto err_exit; } space_id= id; fsp_flags= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - if (buf_page_is_corrupted(false, page, fsp_flags) && - recv_sys.dblwr.restore_first_page(space_id, name, fh)) - goto err_exit; + if (buf_page_is_corrupted(false, page, fsp_flags)) + { + page= recv_sys.dblwr.find_page(page_id_t{space_id, 0}, LSN_MAX); + if (!page) + { + /* If the first page of the given user tablespace is not there + in the doublewrite buffer, then the recovery is going to fail + now. Report error only when doublewrite buffer is not empty */ + sql_print_error("InnoDB: Corrupted page " + "[page id: space=" UINT32PF ", page number=0]" + " of datafile '%s' could not be found" + " in the doublewrite buffer", space_id, name); + goto err_exit; + } - aligned_free(page); + fsp_flags= mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + } } /* Load the tablespace into InnoDB's internal data structures. */ @@ -727,7 +746,7 @@ err_exit: space->set_sizes(SRV_UNDO_TABLESPACE_SIZE_IN_PAGES); space->size= file->size= uint32_t(size >> srv_page_size_shift); } - else if (!file->read_page0()) + else if (!file->read_page0(page, true)) { os_file_close(file->handle); file->handle= OS_FILE_CLOSED; @@ -736,6 +755,7 @@ err_exit: } mysql_mutex_unlock(&fil_system.mutex); + aligned_free(apage); return space_id; } @@ -894,11 +914,12 @@ dberr_t srv_undo_tablespaces_init(bool create_new_undo, mtr_t *mtr) srv_undo_tablespaces_active= srv_undo_tablespaces; uint32_t n_undo= (create_new_undo || srv_operation == SRV_OPERATION_BACKUP || - srv_operation == SRV_OPERATION_RESTORE_DELTA) + srv_operation == SRV_OPERATION_RESTORE_DELTA) ? srv_undo_tablespaces : TRX_SYS_N_RSEGS; - if (dberr_t err= srv_all_undo_tablespaces_open(create_new_undo, n_undo)) - return err; + mysql_mutex_lock(&recv_sys.mutex); + dberr_t err= srv_all_undo_tablespaces_open(create_new_undo, n_undo); + mysql_mutex_unlock(&recv_sys.mutex); /* Initialize srv_undo_space_id_start=0 when there are no dedicated undo tablespaces. */ @@ -906,16 +927,11 @@ dberr_t srv_undo_tablespaces_init(bool create_new_undo, mtr_t *mtr) srv_undo_space_id_start= 0; if (create_new_undo) - { - for (uint32_t i= 0; i < srv_undo_tablespaces; ++i) - { - dberr_t err= fsp_header_init(fil_space_get(srv_undo_space_id_start + i), - SRV_UNDO_TABLESPACE_SIZE_IN_PAGES, mtr); - if (err) return err; - } - } + for (uint32_t i= 0; err == DB_SUCCESS && i < srv_undo_tablespaces; ++i) + err= fsp_header_init(fil_space_get(srv_undo_space_id_start + i), + SRV_UNDO_TABLESPACE_SIZE_IN_PAGES, mtr); - return DB_SUCCESS; + return err; } /** Create the temporary file tablespace. @@ -1451,9 +1467,14 @@ dberr_t srv_start(bool create_new_db) /* Open data files in the system tablespace: we keep them open until database shutdown */ + mysql_mutex_lock(&recv_sys.mutex); ut_d(fil_system.sys_space->recv_size = srv_sys_space_size_debug); + err = fil_system.sys_space->open(create_new_db) + ? DB_SUCCESS : DB_ERROR; + mysql_mutex_unlock(&recv_sys.mutex); - if (fil_system.sys_space->open(create_new_db)) { + + if (err == DB_SUCCESS) { mtr_t mtr; mtr.start(); err= srv_undo_tablespaces_init(create_new_db, &mtr);