database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-08 11:22:35 +03:00
Code Activity

MDEV-32027 Opening all .ibd files on InnoDB startup can be slow

Browse Source 
dict_find_max_space_id(): Return SELECT MAX(SPACE) FROM SYS_TABLES.

dict_check_tablespaces_and_store_max_id(): In the normal case
(no encryption plugin has been loaded and the change buffer is empty),
invoke dict_find_max_space_id() and do not open any .ibd files.
If a std::set<uint32_t> has been specified, open the files whose
tablespace ID is mentioned. Else, open all data files that are identified
by SYS_TABLES records.

fil_ibd_open(): Remove a call to os_file_get_last_error() that can
report a misleading error, such as EINVAL inside my_realpath() that is
not an actual error. This could be invoked when a data file is found
but the FSP_SPACE_FLAGS are incorrect, such as is the case for
table test.td in
./mtr --mysqld=--innodb-buffer-pool-dump-at-shutdown=0 innodb.table_flags

buf_load(): If any tablespaces could not be found, invoke
dict_check_tablespaces_and_store_max_id() on the missing tablespaces.

dict_load_tablespace(): Try to load the tablespace unless it was found
to be futile. This fixes failures related to FTS_*.ibd files for
FULLTEXT INDEX.

btr_cur_t::search_leaf(): Prevent a crash when the tablespace
does not exist. This was caught by the test innodb_fts.fts_concurrent_insert
when the change to dict_load_tablespaces() was not present.

We modify a few tests to ensure that tables will not be loaded at startup.
For some fault injection tests this means that the corrupted tables
will not be loaded, because dict_load_tablespace() would perform stricter
checks than dict_check_tablespaces_and_store_max_id().

Tested by: Matthias Leich
Reviewed by: Thirunarayanan Balathandayuthapani
This commit is contained in:
Marko Mäkelä
2023-11-17 15:07:51 +02:00
parent 44b9e41694
commit eb1f8b2919
13 changed files with 104 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
mysql-test/suite/innodb/r/row_format_redundant.result
View File

@@ -68,9 +68,8 @@ DROP TABLE t1;
Warnings:
Warning 1932 Table 'test.t1' doesn't exist in engine
DROP TABLE t2,t3;
FOUND 6 /\[ERROR\] InnoDB: Table test/t1 in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=1 SYS_TABLES\.MIX_LEN=511\b/ in mysqld.1.err
FOUND 5 /\[ERROR\] InnoDB: Table test/t1 in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=1 SYS_TABLES\.MIX_LEN=511\b/ in mysqld.1.err
# restart
ib_buffer_pool
ib_logfile0
ibdata1
db.opt

12
mysql-test/suite/innodb/r/table_flags.result
View File

@@ -101,13 +101,9 @@ ERROR 42S02: Table 'test.tc' doesn't exist in engine
SELECT * FROM tc;
ERROR 42S02: Table 'test.tc' doesn't exist in engine
SHOW CREATE TABLE td;
Table Create Table
td CREATE TABLE `td` (
`a` int(11) NOT NULL,
PRIMARY KEY (`a`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci ROW_FORMAT=DYNAMIC
ERROR HY000: Got error 194 "Tablespace is missing for a table" from storage engine InnoDB
SELECT * FROM td;
a
ERROR HY000: Got error 194 "Tablespace is missing for a table" from storage engine InnoDB
SHOW CREATE TABLE tz;
Table Create Table
tz CREATE TABLE `tz` (
@@ -122,8 +118,8 @@ a
42
SHOW CREATE TABLE tp;
ERROR 42S02: Table 'test.tp' doesn't exist in engine
FOUND 5 /InnoDB: Table test/t[cp] in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=(129|289|3873|1232[13]) SYS_TABLES\.N_COLS=2147483649/ in mysqld.1.err
FOUND 2 /InnoDB: Table test/tr in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=65 SYS_TABLES\.MIX_LEN=4294967295\b/ in mysqld.1.err
FOUND 3 /InnoDB: Table test/t[cp] in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=(129|289|3873|1232[13]) SYS_TABLES\.N_COLS=2147483649/ in mysqld.1.err
FOUND 1 /InnoDB: Table test/tr in InnoDB data dictionary contains invalid flags\. SYS_TABLES\.TYPE=65 SYS_TABLES\.MIX_LEN=4294967295\b/ in mysqld.1.err
Restoring SYS_TABLES clustered index root page (8)
# restart: with restart_parameters
SHOW CREATE TABLE tr;

2
mysql-test/suite/innodb/t/row_format_redundant.opt
View File

@@ -1 +1,3 @@
--innodb-checksum-algorithm=crc32
--skip-innodb-fast-shutdown
--skip-innodb-buffer-pool-dump-at-shutdown

1
mysql-test/suite/innodb/t/table_flags.opt
View File

@@ -1,2 +1,3 @@
--innodb-checksum-algorithm=crc32
--skip-innodb-read-only-compressed
--skip-innodb-buffer-pool-dump-at-shutdown

2
mysql-test/suite/innodb/t/table_flags.test
View File

@@ -157,7 +157,9 @@ SHOW CREATE TABLE tr;
SHOW CREATE TABLE tc;
--error ER_NO_SUCH_TABLE_IN_ENGINE
SELECT * FROM tc;
--error ER_GET_ERRNO
SHOW CREATE TABLE td;
--error ER_GET_ERRNO
SELECT * FROM td;
# This table was converted to NO_ROLLBACK due to the SYS_TABLES.TYPE change.
SHOW CREATE TABLE tz;

37
mysql-test/suite/innodb_zip/r/restart.result
View File

@@ -527,15 +527,6 @@ Variable_name Value
innodb_file_per_table ON
=== information_schema.innodb_sys_tablespaces and innodb_sys_datafiles ===
Space_Name Page_Size Zip_Size Path
test/t4_restart DEFAULT DEFAULT MYSQLD_DATADIR/test/t4_restart.ibd
test/t6_restart#p#p0 DEFAULT 2048 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p0.ibd
test/t6_restart#p#p1 DEFAULT 2048 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p1.ibd
test/t7_restart#p#p0#sp#s0 DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s0.ibd
test/t7_restart#p#p0#sp#s1 DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s1.ibd
test/t5_restart DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd
test/t6_restart#p#p2 DEFAULT 2048 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p2.ibd
test/t7_restart#p#p1#sp#s2 DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s2.ibd
test/t7_restart#p#p1#sp#s3 DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s3.ibd
innodb_temporary DEFAULT DEFAULT MYSQLD_DATADIR/ibtmp1
SELECT count(*) FROM t5_restart;
count(*)
@@ -629,7 +620,6 @@ RENAME TABLE t6_restart TO t66_restart;
RENAME TABLE t7_restart TO t77_restart;
=== information_schema.innodb_sys_tablespaces and innodb_sys_datafiles ===
Space_Name Page_Size Zip_Size Path
test/t4_restart DEFAULT DEFAULT MYSQLD_DATADIR/test/t4_restart.ibd
test/t66_restart#p#p0 DEFAULT 2048 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd
test/t66_restart#p#p1 DEFAULT 2048 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd
test/t77_restart#p#p0#sp#s0 DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd
@@ -728,15 +718,6 @@ Variable_name Value
innodb_file_per_table ON
=== information_schema.innodb_sys_tablespaces and innodb_sys_datafiles ===
Space_Name Page_Size Zip_Size Path
test/t4_restart DEFAULT DEFAULT MYSQLD_DATADIR/test/t4_restart.ibd
test/t66_restart#p#p0 DEFAULT 2048 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd
test/t66_restart#p#p1 DEFAULT 2048 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd
test/t77_restart#p#p0#sp#s0 DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd
test/t77_restart#p#p0#sp#s1 DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s1.ibd
test/t55_restart DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t55_restart.ibd
test/t66_restart#p#p2 DEFAULT 2048 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p2.ibd
test/t77_restart#p#p1#sp#s2 DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s2.ibd
test/t77_restart#p#p1#sp#s3 DEFAULT DEFAULT MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s3.ibd
innodb_temporary DEFAULT DEFAULT MYSQLD_DATADIR/ibtmp1
INSERT INTO t55_restart (SELECT 0, c2, c3, c4, c5 FROM t55_restart);
SELECT count(*) FROM t55_restart;
@@ -863,15 +844,6 @@ t77_restart#p#p1#sp#s3.ibd
# restart
=== information_schema.innodb_sys_tablespaces and innodb_sys_datafiles ===
Space_Name Page_Size Zip_Size Path
test/t4_restart DEFAULT DEFAULT MYSQL_TMP_DIR/new_dir/test/t4_restart.ibd
test/t66_restart#p#p0 DEFAULT 2048 MYSQL_TMP_DIR/new_dir/test/t66_restart#p#p0.ibd
test/t66_restart#p#p1 DEFAULT 2048 MYSQL_TMP_DIR/new_dir/test/t66_restart#p#p1.ibd
test/t77_restart#p#p0#sp#s0 DEFAULT DEFAULT MYSQL_TMP_DIR/new_dir/test/t77_restart#p#p0#sp#s0.ibd
test/t77_restart#p#p0#sp#s1 DEFAULT DEFAULT MYSQL_TMP_DIR/new_dir/test/t77_restart#p#p0#sp#s1.ibd
test/t55_restart DEFAULT DEFAULT MYSQL_TMP_DIR/new_dir/test/t55_restart.ibd
test/t66_restart#p#p2 DEFAULT 2048 MYSQL_TMP_DIR/new_dir/test/t66_restart#p#p2.ibd
test/t77_restart#p#p1#sp#s2 DEFAULT DEFAULT MYSQL_TMP_DIR/new_dir/test/t77_restart#p#p1#sp#s2.ibd
test/t77_restart#p#p1#sp#s3 DEFAULT DEFAULT MYSQL_TMP_DIR/new_dir/test/t77_restart#p#p1#sp#s3.ibd
innodb_temporary DEFAULT DEFAULT MYSQLD_DATADIR/ibtmp1
INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart);
SELECT count(*) FROM t4_restart;
@@ -1002,15 +974,6 @@ t77_restart.par
# restart
=== information_schema.innodb_sys_tablespaces and innodb_sys_datafiles ===
Space_Name Page_Size Zip_Size Path
test/t4_restart DEFAULT DEFAULT MYSQLD_DATADIR/test/t4_restart.ibd
test/t66_restart#p#p0 DEFAULT 2048 MYSQLD_DATADIR/test/t66_restart#p#p0.ibd
test/t66_restart#p#p1 DEFAULT 2048 MYSQLD_DATADIR/test/t66_restart#p#p1.ibd
test/t77_restart#p#p0#sp#s0 DEFAULT DEFAULT MYSQLD_DATADIR/test/t77_restart#p#p0#sp#s0.ibd
test/t77_restart#p#p0#sp#s1 DEFAULT DEFAULT MYSQLD_DATADIR/test/t77_restart#p#p0#sp#s1.ibd
test/t55_restart DEFAULT DEFAULT MYSQLD_DATADIR/test/t55_restart.ibd
test/t66_restart#p#p2 DEFAULT 2048 MYSQLD_DATADIR/test/t66_restart#p#p2.ibd
test/t77_restart#p#p1#sp#s2 DEFAULT DEFAULT MYSQLD_DATADIR/test/t77_restart#p#p1#sp#s2.ibd
test/t77_restart#p#p1#sp#s3 DEFAULT DEFAULT MYSQLD_DATADIR/test/t77_restart#p#p1#sp#s3.ibd
innodb_temporary DEFAULT DEFAULT MYSQLD_DATADIR/ibtmp1
INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart);
SELECT count(*) FROM t4_restart;

7
mysql-test/suite/innodb_zip/t/restart.opt
View File

@@ -1,2 +1,5 @@
--loose-innodb-sys-tables
--loose-innodb-sys-tablespaces
--innodb-sys-tables
--innodb-sys-tablespaces
--skip-innodb-stats-persistent
--skip-innodb-buffer-pool-dump-at-shutdown
--skip-innodb-fast-shutdown

26
storage/innobase/btr/btr0cur.cc
View File

@@ -1155,6 +1155,19 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode,
mtr_s_lock_index(index(), mtr);
}
dberr_t err;
if (!index()->table->space)
{
corrupted:
ut_ad("corrupted" == 0); // FIXME: remove this
err= DB_CORRUPTION;
func_exit:
if (UNIV_LIKELY_NULL(heap))
mem_heap_free(heap);
return err;
}
const ulint zip_size= index()->table->space->zip_size();
/* Start with the root page. */
@@ -1168,7 +1181,6 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode,
low_bytes= 0;
ulint buf_mode= BUF_GET;
search_loop:
dberr_t err;
auto block_savepoint= mtr->get_savepoint();
buf_block_t *block=
buf_page_get_gen(page_id, zip_size, rw_latch, guess, buf_mode, mtr,
@@ -1180,10 +1192,7 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode,
btr_decryption_failed(*index());
/* fall through */
default:
func_exit:
if (UNIV_LIKELY_NULL(heap))
mem_heap_free(heap);
return err;
goto func_exit;
case DB_SUCCESS:
/* This must be a search to perform an insert, delete mark, or delete;
try using the change buffer */
@@ -1250,12 +1259,7 @@ dberr_t btr_cur_t::search_leaf(const dtuple_t *tuple, page_cur_mode_t mode,
btr_page_get_index_id(block->page.frame) != index()->id ||
fil_page_get_type(block->page.frame) == FIL_PAGE_RTREE ||
!fil_page_index_page_check(block->page.frame))
{
corrupted:
ut_ad("corrupted" == 0); // FIXME: remove this
err= DB_CORRUPTION;
goto func_exit;
}
goto corrupted;
page_cur.block= block;
ut_ad(block == mtr->at_savepoint(block_savepoint));

18
storage/innobase/buf/buf0dump.cc
View File

@@ -33,7 +33,7 @@ Created April 08, 2011 Vasil Dimov
#include "buf0rea.h"
#include "buf0dump.h"
#include "dict0dict.h"
#include "dict0load.h"
#include "os0file.h"
#include "srv0srv.h"
#include "srv0start.h"
@@ -557,6 +557,22 @@ buf_load()
if (!SHUTTING_DOWN()) {
std::sort(dump, dump + dump_n);
std::set<uint32_t> missing;
for (const page_id_t id : st_::span<const page_id_t>
(dump, dump_n)) {
missing.emplace(id.space());
}
for (std::set<uint32_t>::iterator i = missing.begin();
i != missing.end(); ) {
auto j = i++;
if (fil_space_t* space = fil_space_t::get(*j)) {
space->release();
missing.erase(j);
}
}
if (!missing.empty()) {
dict_check_tablespaces_and_store_max_id(&missing);
}
}
/* Avoid calling the expensive fil_space_t::get() for each

82
storage/innobase/dict/dict0load.cc
View File

@@ -33,8 +33,8 @@ Created 4/24/1996 Heikki Tuuri
#include "dict0boot.h"
#include "dict0crea.h"
#include "dict0dict.h"
#include "dict0mem.h"
#include "dict0stats.h"
#include "ibuf0ibuf.h"
#include "fsp0file.h"
#include "fts0priv.h"
#include "mach0data.h"
@@ -867,18 +867,30 @@ err_exit:
return READ_OK;
}
/** Check each tablespace found in the data dictionary.
Then look at each table defined in SYS_TABLES that has a space_id > 0
to find all the file-per-table tablespaces.
/** @return SELECT MAX(space) FROM sys_tables */
static uint32_t dict_find_max_space_id(btr_pcur_t *pcur, mtr_t *mtr)
{
uint32_t max_space_id= 0;
In a crash recovery we already have some tablespace objects created from
processing the REDO log. We will compare the
space_id information in the data dictionary to what we find in the
tablespace file. In addition, more validation will be done if recovery
was needed and force_recovery is not set.
for (const rec_t *rec= dict_startscan_system(pcur, mtr, dict_sys.sys_tables);
rec; rec= dict_getnext_system_low(pcur, mtr))
if (!dict_sys_tables_rec_check(rec))
{
ulint len;
const byte *field=
rec_get_nth_field_old(rec, DICT_FLD__SYS_TABLES__SPACE, &len);
ut_ad(len == 4);
max_space_id= std::max(max_space_id, mach_read_from_4(field));
}
We also scan the biggest space id, and store it to fil_system. */
void dict_check_tablespaces_and_store_max_id()
return max_space_id;
}
/** Check MAX(SPACE) FROM SYS_TABLES and store it in fil_system.
Open each data file if an encryption plugin has been loaded.
@param spaces set of tablespace files to open */
void dict_check_tablespaces_and_store_max_id(const std::set<uint32_t> *spaces)
{
ulint max_space_id = 0;
btr_pcur_t pcur;
@@ -890,6 +902,12 @@ void dict_check_tablespaces_and_store_max_id()
dict_sys.lock(SRW_LOCK_CALL);
if (!spaces && ibuf.empty
&& !encryption_key_id_exists(FIL_DEFAULT_ENCRYPTION_KEY)) {
max_space_id = dict_find_max_space_id(&pcur, &mtr);
goto done;
}
for (const rec_t *rec = dict_startscan_system(&pcur, &mtr,
dict_sys.sys_tables);
rec; rec = dict_getnext_system_low(&pcur, &mtr)) {
@@ -921,14 +939,6 @@ void dict_check_tablespaces_and_store_max_id()
continue;
}
if (flags2 & DICT_TF2_DISCARDED) {
sql_print_information("InnoDB: Ignoring tablespace"
" for %.*s because "
"the DISCARD flag is set",
static_cast<int>(len), field);
continue;
}
/* For tables or partitions using .ibd files, the flag
DICT_TF2_USE_FILE_PER_TABLE was not set in MIX_LEN
before MySQL 5.6.5. The flag should not have been
@@ -941,6 +951,19 @@ void dict_check_tablespaces_and_store_max_id()
continue;
}
if (spaces && spaces->find(uint32_t(space_id))
== spaces->end()) {
continue;
}
if (flags2 & DICT_TF2_DISCARDED) {
sql_print_information("InnoDB: Ignoring tablespace"
" for %.*s because "
"the DISCARD flag is set",
static_cast<int>(len), field);
continue;
}
const span<const char> name{field, len};
char* filepath = fil_make_filepath(nullptr, name,
@@ -973,6 +996,7 @@ void dict_check_tablespaces_and_store_max_id()
ut_free(filepath);
}
done:
mtr.commit();
fil_set_max_space_id_if_bigger(max_space_id);
@@ -2248,22 +2272,10 @@ dict_load_tablespace(
/* The tablespace may already be open. */
table->space = fil_space_for_table_exists_in_mem(table->space_id,
table->flags);
if (table->space) {
if (table->space || table->file_unreadable) {
return;
}
if (ignore_err >= DICT_ERR_IGNORE_TABLESPACE) {
table->file_unreadable = true;
return;
}
if (!(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK)) {
ib::error() << "Failed to find tablespace for table "
<< table->name << " in the cache. Attempting"
" to load the tablespace with space id "
<< table->space_id;
}
/* Use the remote filepath if needed. This parameter is optional
in the call to fil_ibd_open(). If not supplied, it will be built
from the table->name. */
@@ -2286,6 +2298,12 @@ dict_load_tablespace(
if (!table->space) {
/* We failed to find a sensible tablespace file */
table->file_unreadable = true;
if (!(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK)) {
sql_print_error("InnoDB: Failed to load tablespace "
ULINTPF " for table %s",
table->space_id, table->name);
}
}
ut_free(filepath);

2
storage/innobase/fil/fil0fil.cc
View File

@@ -2221,8 +2221,6 @@ func_exit:
goto corrupted;
}
os_file_get_last_error(operation_not_for_export,
!operation_not_for_export);
if (!operation_not_for_export) {
goto corrupted;
}

16
storage/innobase/include/dict0load.h
View File

@@ -35,22 +35,16 @@ Created 4/24/1996 Heikki Tuuri
#include "btr0types.h"
#include <deque>
#include <set>
/** A stack of table names related through foreign key constraints */
typedef std::deque<const char*, ut_allocator<const char*> > dict_names_t;
/** Check each tablespace found in the data dictionary.
Then look at each table defined in SYS_TABLES that has a space_id > 0
to find all the file-per-table tablespaces.
/** Check MAX(SPACE) FROM SYS_TABLES and store it in fil_system.
Open each data file if an encryption plugin has been loaded.
In a crash recovery we already have some tablespace objects created from
processing the REDO log. We will compare the
space_id information in the data dictionary to what we find in the
tablespace file. In addition, more validation will be done if recovery
was needed and force_recovery is not set.
We also scan the biggest space id, and store it to fil_system. */
void dict_check_tablespaces_and_store_max_id();
@param spaces set of tablespace files to open */
void dict_check_tablespaces_and_store_max_id(const std::set<uint32_t> *spaces);
/** Make sure the data_file_name is saved in dict_table_t if needed.
@param[in,out] table Table object */

12
storage/innobase/srv/srv0start.cc
View File

@@ -1819,21 +1819,13 @@ file_checked:
}
if (srv_force_recovery < SRV_FORCE_NO_UNDO_LOG_SCAN) {
/* The following call is necessary for the insert
/* The following call is necessary for the change
buffer to work with multiple tablespaces. We must
know the mapping between space id's and .ibd file
names.
In a crash recovery, we check that the info in data
dictionary is consistent with what we already know
about space id's from the calls to fil_ibd_load().
In a normal startup, we create the space objects for
every table in the InnoDB data dictionary that has
an .ibd file.
We also determine the maximum tablespace id used. */
dict_check_tablespaces_and_store_max_id();
dict_check_tablespaces_and_store_max_id(nullptr);
}
if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO