database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-07-30 16:24:05 +03:00
Code Activity

Merge 10.1 into 10.2

Browse Source 
Disable the test encryption.innodb_encryption-page-compression
because the wait_condition would seem to time out deterministically.
MDEV-14814 has to be addressed in 10.2 separately.

Datafile::validate_first_page(): Do not invoke
page_size_t::page_size_t(flags) before validating the tablespace flags.
This avoids a crash in MDEV-15333 innodb.restart test case.
FIXME: Reduce the number of error messages. The first one is enough.
This commit is contained in:
Marko Mäkelä
2018-02-17 14:54:12 +02:00
parent 8bf2c08d54 9a46d97149
commit 970ce270c9
11 changed files with 188 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
mysql-test/suite/encryption/disabled.def
View File

@ -13,3 +13,4 @@
innodb_scrub : MDEV-8139 scrubbing does not work reliably
innodb_scrub_background : MDEV-8139 scrubbing does not work reliably
innodb-redo-badkey : MDEV-13893 / MDEV-12699 Improve crash recovery of corrupted data pages
innodb_encryption-page-compression : MDEV-14814 wait condition timeout

14
mysql-test/suite/encryption/r/innodb_encryption-page-compression.result
View File

@ -137,15 +137,9 @@ count(*)
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
variable_value > 0
1
SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted';
variable_value >= 0
1
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed';
variable_value > 0
1
SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
variable_value >= 0
1
SET GLOBAL innodb_encryption_threads = 4;
SET GLOBAL innodb_encrypt_tables = off;
update innodb_page_compressed1 set c1 = c1 + 1;
@ -157,8 +151,8 @@ update innodb_page_compressed6 set c1 = c1 + 1;
update innodb_page_compressed7 set c1 = c1 + 1;
update innodb_page_compressed8 set c1 = c1 + 1;
update innodb_page_compressed9 set c1 = c1 + 1;
SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
variable_value >= 0
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
variable_value > 0
1
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted';
variable_value > 0
@ -166,8 +160,8 @@ variable_value > 0
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed';
variable_value > 0
1
SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
variable_value >= 0
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
variable_value > 0
1
drop procedure innodb_insert_proc;
drop table innodb_normal;

23
mysql-test/suite/encryption/r/innodb_encryption_filekeys.result
View File

@ -1,4 +1,3 @@
call mtr.add_suppression("trying to do an operation on a dropped tablespace .*");
SET GLOBAL innodb_encrypt_tables = OFF;
SET GLOBAL innodb_encryption_threads = 4;
CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB;
@ -12,24 +11,21 @@ t1 CREATE TABLE `t1` (
CREATE TABLE t2 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=YES;
CREATE TABLE t3 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=NO;
CREATE TABLE t4 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
INSERT INTO t2 select * from t1;
INSERT INTO t3 select * from t1;
INSERT INTO t4 select * from t1;
SET GLOBAL innodb_encrypt_tables = on;
# Wait max 10 min for key encryption threads to encrypt required all spaces
# Success!
SELECT COUNT(1) FROM t1;
COUNT(1)
400
10
SELECT COUNT(1) FROM t2;
COUNT(1)
400
10
SELECT COUNT(1) FROM t3;
COUNT(1)
400
10
SELECT COUNT(1) FROM t4;
COUNT(1)
400
10
SET GLOBAL innodb_encrypt_tables = off;
# Wait max 10 min for key encryption threads to decrypt all required spaces
# Success!
@ -48,18 +44,17 @@ INSERT INTO t5 select * from t1;
# Success!
SELECT COUNT(1) FROM t1;
COUNT(1)
400
10
SELECT COUNT(1) FROM t2;
COUNT(1)
400
10
SELECT COUNT(1) FROM t3;
COUNT(1)
400
10
SELECT COUNT(1) FROM t4;
COUNT(1)
400
10
SELECT COUNT(1) FROM t5;
COUNT(1)
400
10
drop table t1,t2,t3,t4, t5;
set GLOBAL innodb_default_encryption_key_id=1;

26
mysql-test/suite/encryption/t/innodb_encryption-page-compression.test
View File

@ -1,9 +1,6 @@
-- source include/have_innodb.inc
-- source include/have_example_key_management_plugin.inc
let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`;
let $innodb_encrypt_tables_orig = `SELECT @@innodb_encrypt_tables`;
let $innodb_encryption_threads_orig = `SELECT @@innodb_encryption_threads`;
-- source include/not_embedded.inc
SET GLOBAL innodb_encryption_threads = 4;
SET GLOBAL innodb_encrypt_tables = on;
@ -78,11 +75,13 @@ select count(*) from innodb_page_compressed9 where c1 < 500000;
let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED';
--source include/wait_condition.inc
let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED';
--source include/wait_condition.inc
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted';
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed';
SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
--source include/restart_mysqld.inc
SET GLOBAL innodb_encryption_threads = 4;
SET GLOBAL innodb_encrypt_tables = off;
@ -97,13 +96,15 @@ update innodb_page_compressed7 set c1 = c1 + 1;
update innodb_page_compressed8 set c1 = c1 + 1;
update innodb_page_compressed9 set c1 = c1 + 1;
let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_DECRYPTED';
let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED';
--source include/wait_condition.inc
let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED';
--source include/wait_condition.inc
SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted';
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed';
SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
drop procedure innodb_insert_proc;
drop table innodb_normal;
@ -116,10 +117,3 @@ drop table innodb_page_compressed6;
drop table innodb_page_compressed7;
drop table innodb_page_compressed8;
drop table innodb_page_compressed9;
# reset system
--disable_query_log
EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig;
EVAL SET GLOBAL innodb_encrypt_tables = $innodb_encrypt_tables_orig;
EVAL SET GLOBAL innodb_encryption_threads = $innodb_encryption_threads_orig;
--enable_query_log

31
mysql-test/suite/encryption/t/innodb_encryption_filekeys.test
View File

@ -1,12 +1,11 @@
-- source include/have_innodb.inc
-- source include/have_file_key_management_plugin.inc
# embedded does not support restart
-- source include/not_embedded.inc
call mtr.add_suppression("trying to do an operation on a dropped tablespace .*");
--disable_query_log
let $encrypt_tables = `SELECT @@innodb_encrypt_tables`;
let $threads = `SELECT @@innodb_encryption_threads`;
let $key_id = `SELECT @@innodb_default_encryption_key_id`;
--enable_query_log
SET GLOBAL innodb_encrypt_tables = OFF;
SET GLOBAL innodb_encryption_threads = 4;
@ -19,21 +18,20 @@ CREATE TABLE t4 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNOD
--disable_warnings
--disable_query_log
set autocommit=0;
let $i = 400;
begin;
let $i = 10;
while ($i)
{
INSERT INTO t1 values(NULL, substring(MD5(RAND()), -128));
dec $i;
}
commit;
set autocommit=1;
--enable_warnings
--enable_query_log
INSERT INTO t2 select * from t1;
INSERT INTO t3 select * from t1;
INSERT INTO t4 select * from t1;
commit;
--enable_warnings
--enable_query_log
SET GLOBAL innodb_encrypt_tables = on;
@ -54,7 +52,7 @@ while ($cnt)
}
if (!$success)
{
SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
SHOW STATUS LIKE 'innodb_encryption%';
-- die Timeout waiting for encryption threads
}
@ -84,7 +82,7 @@ while ($cnt)
}
if (!$success)
{
SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
SHOW STATUS LIKE 'innodb_encryption%';
-- die Timeout waiting for encryption threads
}
@ -113,7 +111,7 @@ while ($cnt)
}
if (!$success)
{
SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
SHOW STATUS LIKE 'innodb_encryption%';
-- die Timeout waiting for encryption threads
}
@ -125,12 +123,11 @@ SELECT COUNT(1) FROM t3;
SELECT COUNT(1) FROM t4;
SELECT COUNT(1) FROM t5;
drop table t1,t2,t3,t4, t5;
# reset system
--disable_query_log
EVAL SET GLOBAL innodb_encrypt_tables = $encrypt_tables;
EVAL SET GLOBAL innodb_encryption_threads = $threads;
EVAL set GLOBAL innodb_default_encryption_key_id = $key_id;
--enable_query_log
drop table t1,t2,t3,t4, t5;
set GLOBAL innodb_default_encryption_key_id=1;

29
mysql-test/suite/innodb/r/restart.result Normal file
View File

@ -0,0 +1,29 @@
#
# MDEV-15333 MariaDB (still) slow start
#
# FIXME: Unlike MySQL, maybe MariaDB should not read the .ibd files
# of tables with .isl file or DATA DIRECTORY attribute.
call mtr.add_suppression("\\[ERROR\\] InnoDB: Invalid flags 0x7a207879 in .*td\\.ibd");
# FIXME: This is much more noisy than MariaDB 10.1!
call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot read first page in datafile: .*td\\.ibd, Space ID:2048948345, Flags: 2048948345");
call mtr.add_suppression("\\[ERROR\\] InnoDB: Operating system error number .* in a file operation\\.");
call mtr.add_suppression("\\[ERROR\\] InnoDB: The error means the system cannot find the path specified\\.");
call mtr.add_suppression("\\[ERROR\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them\\.");
call mtr.add_suppression("\\[Warning\\] InnoDB: Ignoring tablespace for `test`\\.`td` because it could not be opened\\.");
CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT
PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9;
CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC
STATS_PERSISTENT=0 DATA DIRECTORY='MYSQL_TMP_DIR';
SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
COUNT(*)
1
SELECT * FROM tr;
a
SELECT * FROM tc;
a
SELECT * FROM td;
a
DROP TABLE tr,tc,td;

80
mysql-test/suite/innodb/t/restart.test Normal file
View File

@ -0,0 +1,80 @@
--source include/innodb_page_size.inc
--source include/not_embedded.inc
let datadir= `select @@datadir`;
let page_size= `select @@innodb_page_size`;
--echo #
--echo # MDEV-15333 MariaDB (still) slow start
--echo #
# Ensure that on normal startup, no data files are read.
# Note: just like in MySQL, all .ibd files will still be
# opened at least once.
--echo # FIXME: Unlike MySQL, maybe MariaDB should not read the .ibd files
--echo # of tables with .isl file or DATA DIRECTORY attribute.
call mtr.add_suppression("\\[ERROR\\] InnoDB: Invalid flags 0x7a207879 in .*td\\.ibd");
--echo # FIXME: This is much more noisy than MariaDB 10.1!
call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot read first page in datafile: .*td\\.ibd, Space ID:2048948345, Flags: 2048948345");
call mtr.add_suppression("\\[ERROR\\] InnoDB: Operating system error number .* in a file operation\\.");
call mtr.add_suppression("\\[ERROR\\] InnoDB: The error means the system cannot find the path specified\\.");
call mtr.add_suppression("\\[ERROR\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them\\.");
call mtr.add_suppression("\\[Warning\\] InnoDB: Ignoring tablespace for `test`\\.`td` because it could not be opened\\.");
CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT
PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9;
--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR
eval CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC
STATS_PERSISTENT=0 DATA DIRECTORY='$MYSQL_TMP_DIR';
--source include/shutdown_mysqld.inc
--move_file $datadir/test/tr.ibd $datadir/test/tr0.ibd
--move_file $datadir/test/tc.ibd $datadir/test/tc0.ibd
--move_file $MYSQL_TMP_DIR/test/td.ibd $datadir/test/td0.ibd
# TODO: test that MariaDB does not even attempt to open the files
#--mkdir $datadir/test/tr.ibd
#--mkdir $datadir/test/tc.ibd
#--mkdir $MYSQL_TMP_DIR/test/td.ibd
perl;
die unless open OUT, ">", "$ENV{datadir}/test/tr.ibd";
print OUT "foo " x $ENV{page_size};
close OUT or die;
die unless open OUT, ">", "$ENV{datadir}/test/tc.ibd";
print OUT "bar " x $ENV{page_size};
close OUT or die;
die unless open OUT, ">", "$ENV{MYSQL_TMP_DIR}/test/td.ibd";
print OUT "xyz " x $ENV{page_size};
close OUT or die;
EOF
--let $restart_parameters= --skip-innodb-buffer-pool-load-at-startup
--source include/start_mysqld.inc
--let $restart_parameters=
SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
--source include/shutdown_mysqld.inc
# TODO: test that MariaDB does not even attempt to open the files
#--rmdir $datadir/test/tr.ibd
#--rmdir $datadir/test/tc.ibd
#--rmdir $MYSQL_TMP_DIR/test/td.ibd
--remove_file $datadir/test/tr.ibd
--remove_file $datadir/test/tc.ibd
--remove_file $MYSQL_TMP_DIR/test/td.ibd
--move_file $datadir/test/tr0.ibd $datadir/test/tr.ibd
--move_file $datadir/test/tc0.ibd $datadir/test/tc.ibd
--move_file $datadir/test/td0.ibd $MYSQL_TMP_DIR/test/td.ibd
--source include/start_mysqld.inc
SELECT * FROM tr;
SELECT * FROM tc;
SELECT * FROM td;
DROP TABLE tr,tc,td;

3
storage/innobase/fil/fil0fil.cc
View File

@ -4215,7 +4215,8 @@ skip_validate:
err = DB_ERROR;
}
if (purpose != FIL_TYPE_IMPORT && !srv_read_only_mode) {
if (err == DB_SUCCESS && validate
&& purpose != FIL_TYPE_IMPORT && !srv_read_only_mode) {
df_remote.close();
df_dict.close();
df_default.close();

67
storage/innobase/fsp/fsp0file.cc
View File

@ -516,8 +516,18 @@ Datafile::validate_first_page(lsn_t* flush_lsn)
}
}
if (error_txt != NULL) {
err_exit:
ib::error() << error_txt << " in datafile: " << m_filepath
<< ", Space ID:" << m_space_id << ", Flags: "
<< m_flags << ". " << TROUBLESHOOT_DATADICT_MSG;
m_is_valid = false;
free_first_page();
return(DB_CORRUPTION);
}
/* Check if the whole page is blank. */
if (error_txt == NULL && !m_space_id && !m_flags) {
if (!m_space_id && !m_flags) {
const byte* b = m_first_page;
ulint nonzero_bytes = UNIV_PAGE_SIZE;
@ -528,56 +538,45 @@ Datafile::validate_first_page(lsn_t* flush_lsn)
if (nonzero_bytes == 0) {
error_txt = "Header page consists of zero bytes";
goto err_exit;
}
}
if (!fsp_flags_is_valid(m_flags, m_space_id)) {
/* Tablespace flags must be valid. */
error_txt = "Tablespace flags are invalid";
goto err_exit;
}
const page_size_t page_size(m_flags);
if (error_txt != NULL) {
/* skip the next few tests */
} else if (univ_page_size.logical() != page_size.logical()) {
if (univ_page_size.logical() != page_size.logical()) {
/* Page size must be univ_page_size. */
ib::error()
<< "Data file '" << m_filepath << "' uses page size "
<< page_size.logical() << ", but the innodb_page_size"
" start-up parameter is "
<< univ_page_size.logical();
free_first_page();
return(DB_ERROR);
} else if (!fsp_flags_is_valid(m_flags, m_space_id)) {
/* Tablespace flags must be valid. */
error_txt = "Tablespace flags are invalid";
} else if (page_get_page_no(m_first_page) != 0) {
/* First page must be number 0 */
error_txt = "Header page contains inconsistent data";
} else if (m_space_id == ULINT_UNDEFINED) {
/* The space_id can be most anything, except -1. */
error_txt = "A bad Space ID was found";
} else if (buf_page_is_corrupted(false, m_first_page, page_size)) {
/* Look for checksum and other corruptions. */
error_txt = "Checksum mismatch";
}
if (error_txt != NULL) {
ib::error() << error_txt << " in datafile: " << m_filepath
<< ", Space ID:" << m_space_id << ", Flags: "
<< m_flags << ". " << TROUBLESHOOT_DATADICT_MSG;
m_is_valid = false;
if (page_get_page_no(m_first_page) != 0) {
/* First page must be number 0 */
error_txt = "Header page contains inconsistent data";
goto err_exit;
}
free_first_page();
return(DB_CORRUPTION);
if (m_space_id == ULINT_UNDEFINED) {
/* The space_id can be most anything, except -1. */
error_txt = "A bad Space ID was found";
goto err_exit;
}
if (buf_page_is_corrupted(false, m_first_page, page_size)) {
/* Look for checksum and other corruptions. */
error_txt = "Checksum mismatch";
goto err_exit;
}
if (fil_space_read_name_and_filepath(

5
storage/mroonga/vendor/groonga/lib/CMakeLists.txt vendored
View File

@ -177,3 +177,8 @@ if(GRN_WITH_MRUBY)
FILES ${EXPRESSION_TREE_RUBY_SCRIPTS}
DESTINATION "${GRN_RELATIVE_RUBY_SCRIPTS_DIR}/expression_tree")
endif()
# Workaround GCC ICE on ARM64
IF(CMAKE_COMPILER_IS_GNUCXX AND CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
ADD_COMPILE_FLAGS(ts/ts_expr_node.c COMPILE_FLAGS "-fno-tree-loop-vectorize")
ENDIF()

12
storage/xtradb/fil/fil0fil.cc
View File

@ -4468,17 +4468,7 @@ cleanup_and_exit:
mem_free(def.filepath);
/* We need to check fsp flags when no errors has happened and
server was not started on read only mode and tablespace validation
was requested or flags contain other table options except
low order bits to FSP_FLAGS_POS_PAGE_SSIZE position.
Note that flag comparison is pessimistic. Adjust is required
only when flags contain buggy MariaDB 10.1.0 -
MariaDB 10.1.20 flags. */
if (err == DB_SUCCESS
&& !srv_read_only_mode
&& (validate
|| flags >= (1U << FSP_FLAGS_POS_PAGE_SSIZE))) {
if (err == DB_SUCCESS && validate && !srv_read_only_mode) {
fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK);
}