database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-01 03:47:19 +03:00
Code Activity

Merge 10.1 into 10.2

Browse Source
This commit is contained in:
Marko Mäkelä
2018-12-17 20:04:03 +02:00
parent 10e01b56f7 8c43f96388
commit 7d245083a4
23 changed files with 239 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
extra/innochecksum.cc
View File

@ -523,7 +523,16 @@ is_page_corrupted(
normal method. */
if (is_encrypted && key_version != 0) {
is_corrupted = !fil_space_verify_crypt_checksum(buf,
page_size, space_id, (ulint)cur_page_num);
page_size);
if (is_corrupted && log_file) {
fprintf(log_file,
"Page " ULINTPF ":%llu may be corrupted;"
" key_version=%u\n",
space_id, cur_page_num,
mach_read_from_4(
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+ buf));
}
} else {
is_corrupted = true;
}

11
extra/mariabackup/fil_cur.cc
View File

@ -362,9 +362,14 @@ read_retry:
page_no >= FSP_EXTENT_SIZE &&
page_no < FSP_EXTENT_SIZE * 3) {
/* We ignore the doublewrite buffer pages */
} else if (fil_space_verify_crypt_checksum(
page, cursor->page_size,
space->id, page_no)) {
} else if (mach_read_from_4(
page
+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
&& space->crypt_data
&& space->crypt_data->type
!= CRYPT_SCHEME_UNENCRYPTED
&& fil_space_verify_crypt_checksum(
page, cursor->page_size)) {
ut_ad(mach_read_from_4(page + FIL_PAGE_SPACE_ID)
== space->id);

4
mysql-test/include/innodb_encrypt_tables.combinations
View File

@ -3,12 +3,12 @@ innodb_encrypt_tables=ON
plugin-load-add=$FILE_KEY_MANAGEMENT_SO
loose-file-key-management
loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt
file-key-management-encryption-algorithm=aes_ctr
file-key-management-encryption-algorithm=aes_cbc
[clear]
innodb_encrypt_tables=OFF
plugin-load-add=$FILE_KEY_MANAGEMENT_SO
loose-file-key-management
loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt
file-key-management-encryption-algorithm=aes_ctr
file-key-management-encryption-algorithm=aes_cbc

3
mysql-test/suite/encryption/r/innodb-force-corrupt.result
View File

@ -1,5 +1,4 @@
call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\.");
call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version=3221342974");
SET GLOBAL innodb_file_per_table = ON;
set global innodb_compression_algorithm = 1;
# Create and populate tables to be corrupted

9
mysql-test/suite/encryption/t/innodb-force-corrupt.test
View File

@ -7,8 +7,7 @@
# Don't test under embedded
-- source include/not_embedded.inc
call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\.");
call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=\\d+, page number=[36]\\] in file .*test.t[123]\\.ibd looks corrupted; key_version=3221342974");
SET GLOBAL innodb_file_per_table = ON;
set global innodb_compression_algorithm = 1;
@ -50,17 +49,17 @@ perl;
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
print FILE pack("H*", "c001cafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
print FILE pack("H*", "c001cafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
print FILE pack("H*", "c001cafedeadb017");
close FILE or die "close";
EOF

3
mysql-test/suite/galera/include/have_filekeymanagement.inc Normal file
View File

@ -0,0 +1,3 @@
#
# Used in galera/suite.pm to check file key management plugin
#

1
mysql-test/suite/galera/r/galera_defaults.result
View File

@ -52,4 +52,3 @@ WSREP_SST_DONOR
WSREP_SST_DONOR_REJECTS_QUERIES OFF
WSREP_SST_METHOD rsync
WSREP_SYNC_WAIT 15
<BASE_DIR>; <BASE_HOST>; <BASE_PORT>; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT30S; evs.info_log_mask = 0; evs.install_timeout = PT15S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT10S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; <GCACHE_DIR>; gcache.keep_pages_size = 0; gcache.mem_size = 0; <GCACHE_NAME>; gcache.page_size = 128M; gcache.recover = no; gcache.size = 10M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; <GCS_RECV_Q_HARD_LIMIT>; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; <GMCAST_LISTEN_ADDR>; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; <IST_RECV_ADDR>; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = PT30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT90S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; <REPL_PROTO_MAX>;socket.checksum = 2; socket.recv_buf_size = 212992;

2
mysql-test/suite/galera/suite.pm
View File

@ -96,6 +96,8 @@ if (which(socat)) {
sub skip_combinations {
my %skip = ();
$skip{'include/have_filekeymanagement.inc'} = 'needs file_key_management plugin'
unless $ENV{FILE_KEY_MANAGEMENT_SO};
$skip{'include/have_xtrabackup.inc'} = 'Need innobackupex'
unless which(innobackupex);
$skip{'include/have_xtrabackup.inc'} = 'Need socat or nc'

23
mysql-test/suite/galera/t/galera_defaults.test
View File

@ -36,26 +36,3 @@ AND VARIABLE_NAME NOT IN (
'WSREP_PATCH_VERSION'
)
ORDER BY VARIABLE_NAME;
# wsrep_provider_options
#
# We replace the ones that vary from run to run with placeholders
--let _WSREP_PROVIDER_OPTIONS = `SELECT @@wsrep_provider_options`
--perl
use strict;
my $wsrep_provider_options = $ENV{'_WSREP_PROVIDER_OPTIONS'};
$wsrep_provider_options =~ s/base_dir = .*?;/<BASE_DIR>;/sgio;
$wsrep_provider_options =~ s/base_host = .*?;/<BASE_HOST>;/sgio;
$wsrep_provider_options =~ s/base_port = .*?;/<BASE_PORT>;/sgio;
$wsrep_provider_options =~ s/gcache\.dir = .*?;/<GCACHE_DIR>;/sgio;
$wsrep_provider_options =~ s/gcache\.name = .*?;/<GCACHE_NAME>;/sgio;
$wsrep_provider_options =~ s/gmcast\.listen_addr = .*?;/<GMCAST_LISTEN_ADDR>;/sgio;
$wsrep_provider_options =~ s/gcs\.recv_q_hard_limit = .*?;/<GCS_RECV_Q_HARD_LIMIT>;/sgio;
$wsrep_provider_options =~ s/ist\.recv_addr = .*?;/<IST_RECV_ADDR>;/sgio;
$wsrep_provider_options =~ s/evs\.evict = .*?;/<EVS_EVICT>;/sgio;
$wsrep_provider_options =~ s/signal = .*?;\s*//sgio;
$wsrep_provider_options =~ s/dbug = .*?;\s*//sgio;
$wsrep_provider_options =~ s/repl.proto_max = .*?;\s*/<REPL_PROTO_MAX>;/sgio;
print $wsrep_provider_options."\n";
EOF

1
mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.test
View File

@ -1,5 +1,6 @@
--source include/big_test.inc
--source include/galera_cluster.inc
--source include/have_filekeymanagement.inc
--source include/innodb_encrypt_tables.inc
--source include/innodb_page_size_small.inc
--source include/have_mariabackup.inc

43
storage/innobase/buf/buf0buf.cc
View File

@ -525,9 +525,13 @@ decompress_with_slot:
+ dst_frame)) {
/* Verify encryption checksum before we even try to
decrypt. */
if (!fil_space_verify_crypt_checksum(
dst_frame, bpage->size, bpage->id.space(),
bpage->id.page_no())) {
if (!fil_space_verify_crypt_checksum(dst_frame, bpage->size)) {
ib::error() << "Encrypted page " << bpage->id
<< " in file " << space->chain.start->name
<< " looks corrupted; key_version="
<< mach_read_from_4(
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+ dst_frame);
decrypt_failed:
/* Mark page encrypted in case it should be. */
if (space->crypt_data->type
@ -735,23 +739,6 @@ buf_block_alloc(
}
#endif /* !UNIV_INNOCHECKSUM */
/** Checks if a page contains only zeroes.
@param[in] read_buf database page
@param[in] page_size page size
@return true if page is filled with zeroes */
bool
buf_page_is_zeroes(
const byte* read_buf,
const page_size_t& page_size)
{
for (ulint i = 0; i < page_size.logical(); i++) {
if (read_buf[i] != 0) {
return(false);
}
}
return(true);
}
/** Checks if the page is in crc32 checksum format.
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@ -5843,18 +5830,14 @@ or decrypt/decompress just failed.
@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but
after decryption normal page checksum does not match.
@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */
static
dberr_t
buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
static dberr_t buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
{
ut_ad(space->n_pending_ios > 0);
byte* dst_frame = (bpage->zip.data) ? bpage->zip.data :
((buf_block_t*) bpage)->frame;
bool still_encrypted = false;
dberr_t err = DB_SUCCESS;
bool corrupted = false;
fil_space_crypt_t* crypt_data = space->crypt_data;
/* In buf_decrypt_after_read we have either decrypted the page if
page post encryption checksum matches and used key_id is found
@ -5862,12 +5845,12 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
not decrypted and it could be either encrypted and corrupted
or corrupted or good page. If we decrypted, there page could
still be corrupted if used key does not match. */
still_encrypted = crypt_data
&& crypt_data->type != CRYPT_SCHEME_UNENCRYPTED
const bool still_encrypted = mach_read_from_4(
dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
&& space->crypt_data
&& space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED
&& !bpage->encrypted
&& fil_space_verify_crypt_checksum(
dst_frame, bpage->size,
bpage->id.space(), bpage->id.page_no());
&& fil_space_verify_crypt_checksum(dst_frame, bpage->size);
if (!still_encrypted) {
/* If traditional checksums match, we assume that page is

41
storage/innobase/buf/buf0dblwr.cc
View File

@ -1,7 +1,7 @@
/*****************************************************************************
Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2013, 2017, MariaDB Corporation.
Copyright (c) 2013, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -338,6 +338,20 @@ too_small:
goto start_again;
}
/** Check if a page is all zeroes.
@param[in] read_buf database page
@param[in] page_size page frame size
@return whether the page is all zeroes */
static bool buf_page_is_zeroes(const byte* read_buf, size_t page_size)
{
for (ulint i = 0; i < page_size; i++) {
if (read_buf[i] != 0) {
return false;
}
}
return true;
}
/**
At database startup initializes the doublewrite buffer memory structure if
we already have a doublewrite buffer created in the data files. If we are
@ -573,7 +587,7 @@ buf_dblwr_process()
}
const page_size_t page_size(space->flags);
ut_ad(!buf_page_is_zeroes(page, page_size));
ut_ad(!buf_page_is_zeroes(page, page_size.physical()));
/* We want to ensure that for partial reads the
unread portion of the page is NUL. */
@ -597,7 +611,9 @@ buf_dblwr_process()
}
const bool is_all_zero = buf_page_is_zeroes(
read_buf, page_size);
read_buf, page_size.physical());
const bool expect_encrypted = space->crypt_data
&& space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED;
if (is_all_zero) {
/* We will check if the copy in the
@ -613,10 +629,13 @@ buf_dblwr_process()
goto bad;
}
if (fil_space_verify_crypt_checksum(
read_buf, page_size, space_id, page_no)
|| !buf_page_is_corrupted(
true, read_buf, page_size, space)) {
if (expect_encrypted && mach_read_from_4(
read_buf
+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
? fil_space_verify_crypt_checksum(read_buf,
page_size)
: !buf_page_is_corrupted(true, read_buf,
page_size, space)) {
/* The page is good; there is no need
to consult the doublewrite buffer. */
continue;
@ -635,9 +654,11 @@ bad:
&& page_size.is_compressed())) {
goto bad_doublewrite;
}
if (!fil_space_verify_crypt_checksum(page, page_size,
space_id, page_no)
&& buf_page_is_corrupted(true, page, page_size, space)) {
if (expect_encrypted && mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
? !fil_space_verify_crypt_checksum(page, page_size)
: buf_page_is_corrupted(true, page, page_size, space)) {
if (!is_all_zero) {
bad_doublewrite:
ib::warn() << "A doublewrite copy of page "

183
storage/innobase/fil/fil0crypt.cc
View File

@ -616,8 +616,7 @@ fil_encrypt_buf(
// store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum);
ut_ad(fil_space_verify_crypt_checksum(dst_frame, page_size,
space, offset));
ut_ad(fil_space_verify_crypt_checksum(dst_frame, page_size));
srv_stats.pages_encrypted.inc();
@ -2533,164 +2532,68 @@ encrypted, or corrupted.
@param[in,out] page page frame (checksum is temporarily modified)
@param[in] page_size page size
@param[in] space tablespace identifier
@param[in] offset page number
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
@return whether the encrypted page is OK */
bool
fil_space_verify_crypt_checksum(
byte* page,
const page_size_t& page_size,
ulint space,
ulint offset)
fil_space_verify_crypt_checksum(const byte* page, const page_size_t& page_size)
{
uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* If page is not encrypted, return false */
if (key_version == 0) {
return false;
}
/* Read stored post encryption checksum. */
uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/* Declare empty pages non-corrupted */
if (checksum == 0
&& *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
&& buf_page_is_zeroes(page, page_size)) {
return(true);
}
ut_ad(mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION));
/* Compressed and encrypted pages do not have checksum. Assume not
corrupted. Page verification happens after decompression in
buf_page_io_complete() using buf_page_is_corrupted(). */
if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return (true);
if (mach_read_from_2(page + FIL_PAGE_TYPE)
== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return true;
}
uint32_t cchecksum1, cchecksum2;
/* Calculate checksums */
if (page_size.is_compressed()) {
cchecksum1 = page_zip_calc_checksum(
page, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_CRC32);
cchecksum2 = (cchecksum1 == checksum)
? 0
: page_zip_calc_checksum(
page, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_INNODB);
} else {
cchecksum1 = buf_calc_page_crc32(page);
cchecksum2 = (cchecksum1 == checksum)
? 0
: buf_calc_page_new_checksum(page);
}
/* Read stored post encryption checksum. */
const ib_uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/* If stored checksum matches one of the calculated checksums
page is not corrupted. */
bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
|| checksum == BUF_NO_CHECKSUM_MAGIC);
/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
first page of each system tablespace file at
FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
than page 0 of the system tablespace.
Starting from MariaDB 10.1 the field has been repurposed for
encryption key_version.
Starting with MySQL 5.7 (and MariaDB Server 10.2), the
field has been repurposed for SPATIAL INDEX pages for
FIL_RTREE_SPLIT_SEQ_NUM.
Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
checksum.
Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
field would usually be 0 for pages that are not encrypted, we cannot
assume that a nonzero value means that the page is encrypted.
Therefore we must validate the page both as encrypted and unencrypted
when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
*/
uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM);
uint32_t checksum2;
bool valid = false;
if (page_size.is_compressed()) {
valid = checksum1 == cchecksum1;
checksum2 = checksum1;
} else {
checksum2 = mach_read_from_4(
page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
srv_checksum_algorithm_t algorithm =
static_cast<srv_checksum_algorithm_t>(
srv_checksum_algorithm_t algorithm = srv_checksum_algorithm_t(
srv_checksum_algorithm);
switch (algorithm) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
valid = buf_page_is_checksum_valid_crc32(
page, checksum1, checksum2);
break;
if (page_size.is_compressed()) {
return checksum == page_zip_calc_checksum(
page, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_CRC32);
}
return checksum == buf_calc_page_crc32(page);
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
valid = buf_page_is_checksum_valid_innodb(
page, checksum1, checksum2);
break;
if (page_size.is_compressed()) {
return checksum == page_zip_calc_checksum(
page, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_INNODB);
}
return checksum == buf_calc_page_new_checksum(page);
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_INNODB:
return checksum == BUF_NO_CHECKSUM_MAGIC;
case SRV_CHECKSUM_ALGORITHM_NONE:
/* never supported
innodb_checksum_algorithm=none or strict_none
for encrypted pages. */
valid = buf_page_is_checksum_valid_crc32(
page, checksum1, checksum2)
|| buf_page_is_checksum_valid_innodb(
page, checksum1, checksum2);
break;
return true;
case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_CRC32:
if (checksum == BUF_NO_CHECKSUM_MAGIC) {
return true;
}
if (page_size.is_compressed()) {
if (checksum == page_zip_calc_checksum(
page, page_size.physical(), algorithm)) {
return true;
}
if (encrypted && valid) {
/* If page is encrypted and traditional checksums match,
page could be still encrypted, or not encrypted and valid or
corrupted. */
#ifdef UNIV_INNOCHECKSUM
fprintf(log_file ? log_file : stderr,
"Page " ULINTPF ":" ULINTPF " may be corrupted."
" Post encryption checksum %u"
" stored [%u:%u] key_version %u\n",
space, offset, checksum, checksum1, checksum2,
key_version);
#else /* UNIV_INNOCHECKSUM */
ib::error()
<< " Page " << space << ":" << offset
<< " may be corrupted.";
ib::info()
<< "If encrypted: stored checksum" << checksum
<< " calculated checksum [" << cchecksum1 << ":" << cchecksum2
<< "] key_version " << key_version;
ib::info()
<< "If unencrypted: stored checksum [" << checksum1
<< ":" << checksum2 << "] calculated crc32 ["
<< buf_calc_page_crc32(page)
# ifdef INNODB_BUG_ENDIAN_CRC32
<< ":" << buf_calc_page_crc32(page, true)
# endif /* INNODB_BUG_ENDIAN_CRC32 */
<< "] innodb ["
<< buf_calc_page_old_checksum(page) << ":"
<< buf_calc_page_new_checksum(page) << "] LSN "
<< mach_read_from_4(page + FIL_PAGE_LSN);
#endif
encrypted = false;
algorithm = algorithm == SRV_CHECKSUM_ALGORITHM_INNODB
? SRV_CHECKSUM_ALGORITHM_CRC32
: SRV_CHECKSUM_ALGORITHM_INNODB;
return checksum == page_zip_calc_checksum(
page, page_size.physical(), algorithm);
}
return(encrypted);
return checksum == buf_calc_page_crc32(page)
|| checksum == buf_calc_page_new_checksum(page);
}
}

14
storage/innobase/ibuf/ibuf0ibuf.cc
View File

@ -4944,6 +4944,20 @@ ibuf_print(
mutex_exit(&ibuf_mutex);
}
/** Check if a page is all zeroes.
@param[in] read_buf database page
@param[in] size page size
@return whether the page is all zeroes */
static bool buf_page_is_zeroes(const byte* read_buf, const page_size_t& size)
{
for (ulint i = 0; i < size.physical(); i++) {
if (read_buf[i] != 0) {
return false;
}
}
return true;
}
/******************************************************************//**
Checks the insert buffer bitmaps on IMPORT TABLESPACE.
@return DB_SUCCESS or error code */

11
storage/innobase/include/buf0buf.h
View File

@ -748,15 +748,6 @@ buf_page_is_checksum_valid_none(
ulint checksum_field2)
MY_ATTRIBUTE((nonnull(1), warn_unused_result));
/** Checks if a page contains only zeroes.
@param[in] read_buf database page
@param[in] page_size page size
@return true if page is filled with zeroes */
bool
buf_page_is_zeroes(
const byte* read_buf,
const page_size_t& page_size);
/** Check if a page is corrupt.
@param[in] check_lsn whether the LSN should be checked
@param[in] read_buf database page
@ -775,9 +766,7 @@ buf_page_is_corrupted(
#endif
MY_ATTRIBUTE((warn_unused_result));
#ifndef UNIV_INNOCHECKSUM
/**********************************************************************//**
Gets the space id, page offset, and byte offset within page of a
pointer pointing to a buffer frame containing a file page. */

11
storage/innobase/include/fil0crypt.h
View File

@ -1,6 +1,6 @@
/*****************************************************************************
Copyright (C) 2013, 2015, Google Inc. All Rights Reserved.
Copyright (c) 2015, 2017, MariaDB Corporation.
Copyright (c) 2015, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -490,16 +490,9 @@ encrypted, or corrupted.
@param[in,out] page page frame (checksum is temporarily modified)
@param[in] page_size page size
@param[in] space tablespace identifier
@param[in] offset page number
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
const page_size_t& page_size,
ulint space,
ulint offset)
fil_space_verify_crypt_checksum(const byte* page, const page_size_t& page_size)
MY_ATTRIBUTE((warn_unused_result));
#endif /* fil0crypt_h */

4
storage/innobase/row/row0import.cc
View File

@ -3447,9 +3447,7 @@ not_encrypted:
}
} else {
if (!fil_space_verify_crypt_checksum(
src, callback.get_page_size(),
block->page.id.space(),
block->page.id.page_no())) {
src, callback.get_page_size())) {
goto page_corrupted;
}

46
storage/xtradb/buf/buf0buf.cc
View File

@ -481,8 +481,15 @@ decompress_with_slot:
/* Verify encryption checksum before we even try to
decrypt. */
if (!fil_space_verify_crypt_checksum(
dst_frame, buf_page_get_zip_size(bpage), NULL,
bpage->offset)) {
dst_frame, buf_page_get_zip_size(bpage))) {
ib_logf(IB_LOG_LEVEL_ERROR,
"Encrypted page %u:%u in file %s"
" looks corrupted; key_version=" ULINTPF,
bpage->space, bpage->offset,
space->chain.start->name,
mach_read_from_4(
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+ dst_frame));
decrypt_failed:
/* Mark page encrypted in case it should be. */
if (space->crypt_data->type
@ -728,24 +735,6 @@ buf_block_alloc(
}
#endif /* !UNIV_HOTBACKUP */
/** Check if a page is all zeroes.
@param[in] read_buf database page
@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0
@return whether the page is all zeroes */
UNIV_INTERN
bool
buf_page_is_zeroes(const byte* read_buf, ulint zip_size)
{
const ulint page_size = zip_size ? zip_size : UNIV_PAGE_SIZE;
for (ulint i = 0; i < page_size; i++) {
if (read_buf[i] != 0) {
return(false);
}
}
return(true);
}
/** Checks if the page is in crc32 checksum format.
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@ -4780,19 +4769,15 @@ or decrypt/decompress just failed.
@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but
after decryption normal page checksum does not match.
@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */
static
dberr_t
buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
static dberr_t buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
{
ut_ad(space->n_pending_ios > 0);
ulint zip_size = buf_page_get_zip_size(bpage);
byte* dst_frame = (zip_size) ? bpage->zip.data :
((buf_block_t*) bpage)->frame;
bool still_encrypted = false;
dberr_t err = DB_SUCCESS;
bool corrupted = false;
fil_space_crypt_t* crypt_data = space->crypt_data;
/* In buf_decrypt_after_read we have either decrypted the page if
page post encryption checksum matches and used key_id is found
@ -4800,11 +4785,12 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
not decrypted and it could be either encrypted and corrupted
or corrupted or good page. If we decrypted, there page could
still be corrupted if used key does not match. */
still_encrypted = (crypt_data &&
crypt_data->type != CRYPT_SCHEME_UNENCRYPTED &&
!bpage->encrypted &&
fil_space_verify_crypt_checksum(dst_frame, zip_size,
space, bpage->offset));
const bool still_encrypted = mach_read_from_4(
dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
&& space->crypt_data
&& space->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED
&& !bpage->encrypted
&& fil_space_verify_crypt_checksum(dst_frame, zip_size);
if (!still_encrypted) {
/* If traditional checksums match, we assume that page is

43
storage/xtradb/buf/buf0dblwr.cc
View File

@ -1,7 +1,7 @@
/*****************************************************************************
Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2013, 2017, MariaDB Corporation.
Copyright (c) 2013, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
@ -362,6 +362,22 @@ too_small:
goto start_again;
}
/** Check if a page is all zeroes.
@param[in] read_buf database page
@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0
@return whether the page is all zeroes */
static bool buf_page_is_zeroes(const byte* read_buf, ulint zip_size)
{
const ulint page_size = zip_size ? zip_size : UNIV_PAGE_SIZE;
for (ulint i = 0; i < page_size; i++) {
if (read_buf[i] != 0) {
return false;
}
}
return true;
}
/****************************************************************//**
At a database startup initializes the doublewrite buffer memory structure if
we already have a doublewrite buffer created in the data files. If we are
@ -556,6 +572,9 @@ buf_dblwr_process()
const bool is_all_zero = buf_page_is_zeroes(
read_buf, zip_size);
const bool expect_encrypted = space()->crypt_data
&& space()->crypt_data->type
!= CRYPT_SCHEME_UNENCRYPTED;
if (is_all_zero) {
/* We will check if the copy in the
@ -566,17 +585,17 @@ buf_dblwr_process()
/* Decompress the page before
validating the checksum. */
ulint decomp = fil_page_decompress(buf, read_buf);
if (!decomp) {
goto bad;
}
if (!decomp || (decomp != srv_page_size && zip_size)) {
goto bad;
}
if (fil_space_verify_crypt_checksum(
read_buf, zip_size, NULL, page_no)
|| !buf_page_is_corrupted(
true, read_buf, zip_size, space())) {
if (expect_encrypted && mach_read_from_4(
read_buf
+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
? fil_space_verify_crypt_checksum(read_buf,
zip_size)
: !buf_page_is_corrupted(true, read_buf,
zip_size, space())) {
/* The page is good; there is no need
to consult the doublewrite buffer. */
continue;
@ -595,9 +614,11 @@ bad:
if (!decomp || (decomp != srv_page_size && zip_size)) {
goto bad_doublewrite;
}
if (!fil_space_verify_crypt_checksum(page, zip_size, NULL,
page_no)
&& buf_page_is_corrupted(true, page, zip_size, space)) {
if (expect_encrypted && mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)
? !fil_space_verify_crypt_checksum(page, zip_size)
: buf_page_is_corrupted(true, page, zip_size, space())) {
if (!is_all_zero) {
bad_doublewrite:
ib_logf(IB_LOG_LEVEL_WARN,

183
storage/xtradb/fil/fil0crypt.cc
View File

@ -662,7 +662,7 @@ fil_encrypt_buf(
// store the post-encryption checksum after the key-version
mach_write_to_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4, checksum);
ut_ad(fil_space_verify_crypt_checksum(dst_frame, zip_size, NULL, offset));
ut_ad(fil_space_verify_crypt_checksum(dst_frame, zip_size));
srv_stats.pages_encrypted.inc();
@ -2568,167 +2568,66 @@ encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
@return whether the encrypted page is OK */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
#ifndef UNIV_INNOCHECKSUM
const fil_space_t* space,
#else
const void* space,
#endif
ulint pageno)
bool fil_space_verify_crypt_checksum(const byte* page, ulint zip_size)
{
uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* If page is not encrypted, return false */
if (key_version == 0) {
return(false);
}
srv_checksum_algorithm_t algorithm =
static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
/* If no checksum is used, can't continue checking. */
if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
return(true);
}
/* Read stored post encryption checksum. */
ib_uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/* Declare empty pages non-corrupted */
if (checksum == 0
&& *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
&& buf_page_is_zeroes(page, zip_size)) {
return(true);
}
ut_ad(mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION));
/* Compressed and encrypted pages do not have checksum. Assume not
corrupted. Page verification happens after decompression in
buf_page_io_complete() using buf_page_is_corrupted(). */
if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return (true);
if (mach_read_from_2(page + FIL_PAGE_TYPE)
== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return true;
}
ib_uint32_t cchecksum1 = 0;
ib_uint32_t cchecksum2 = 0;
/* Calculate checksums */
if (zip_size) {
cchecksum1 = page_zip_calc_checksum(
page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
cchecksum2 = (cchecksum1 == checksum)
? 0
: page_zip_calc_checksum(
page, zip_size,
SRV_CHECKSUM_ALGORITHM_INNODB);
} else {
cchecksum1 = buf_calc_page_crc32(page);
cchecksum2 = (cchecksum1 == checksum)
? 0
: buf_calc_page_new_checksum(page);
}
/* Read stored post encryption checksum. */
const ib_uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/* If stored checksum matches one of the calculated checksums
page is not corrupted. */
srv_checksum_algorithm_t algorithm = srv_checksum_algorithm_t(
srv_checksum_algorithm);
bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
|| checksum == BUF_NO_CHECKSUM_MAGIC);
/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
first page of each system tablespace file at
FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
than page 0 of the system tablespace.
Starting from MariaDB 10.1 the field has been repurposed for
encryption key_version.
Starting with MySQL 5.7 (and MariaDB Server 10.2), the
field has been repurposed for SPATIAL INDEX pages for
FIL_RTREE_SPLIT_SEQ_NUM.
Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
checksum.
Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
field would usually be 0 for pages that are not encrypted, we cannot
assume that a nonzero value means that the page is encrypted.
Therefore we must validate the page both as encrypted and unencrypted
when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
*/
uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM);
uint32_t checksum2;
bool valid = false;
if (zip_size) {
valid = (checksum1 == cchecksum1);
checksum2 = checksum1;
} else {
checksum2 = mach_read_from_4(
page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
switch (algorithm) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
valid = buf_page_is_checksum_valid_crc32(page, checksum1,
checksum2);
break;
if (zip_size) {
return checksum == page_zip_calc_checksum(
page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
}
return checksum == buf_calc_page_crc32(page);
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
valid = buf_page_is_checksum_valid_innodb(page, checksum1,
checksum2);
break;
if (zip_size) {
return checksum == page_zip_calc_checksum(
page, zip_size, SRV_CHECKSUM_ALGORITHM_INNODB);
}
return checksum == buf_calc_page_new_checksum(page);
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
valid = buf_page_is_checksum_valid_none(page, checksum1,
checksum2);
break;
case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_INNODB:
valid = buf_page_is_checksum_valid_crc32(
page, checksum1, checksum2)
|| buf_page_is_checksum_valid_innodb(
page, checksum1, checksum2);
break;
return checksum == BUF_NO_CHECKSUM_MAGIC;
case SRV_CHECKSUM_ALGORITHM_NONE:
ut_error;
return true;
case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_CRC32:
if (checksum == BUF_NO_CHECKSUM_MAGIC) {
return true;
}
if (zip_size) {
if (checksum == page_zip_calc_checksum(
page, zip_size, algorithm)) {
return true;
}
if (encrypted && valid) {
/* If page is encrypted and traditional checksums match,
page could be still encrypted, or not encrypted and valid or
corrupted. */
#ifndef UNIV_INNOCHECKSUM
ib_logf(IB_LOG_LEVEL_ERROR,
" Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted."
" Post encryption checksum %u stored [%u:%u] key_version %u",
pageno,
space ? space->name : "N/A",
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2, key_version);
#else
if (log_file) {
fprintf(log_file,
"Page " ULINTPF ":" ULINTPF " may be corrupted."
" Post encryption checksum %u"
" stored [%u:%u] key_version %u\n",
pageno,
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2,
key_version);
}
#endif /* UNIV_INNOCHECKSUM */
encrypted = false;
algorithm = algorithm == SRV_CHECKSUM_ALGORITHM_INNODB
? SRV_CHECKSUM_ALGORITHM_CRC32
: SRV_CHECKSUM_ALGORITHM_INNODB;
return checksum == page_zip_calc_checksum(
page, zip_size, algorithm);
}
return(encrypted);
return checksum == buf_calc_page_crc32(page)
|| checksum == buf_calc_page_new_checksum(page);
}
}

7
storage/xtradb/include/buf0buf.h
View File

@ -692,13 +692,6 @@ buf_page_is_corrupted(
ulint zip_size,
const fil_space_t* space)
MY_ATTRIBUTE((warn_unused_result));
/** Check if a page is all zeroes.
@param[in] read_buf database page
@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0
@return whether the page is all zeroes */
UNIV_INTERN
bool
buf_page_is_zeroes(const byte* read_buf, ulint zip_size);
#ifndef UNIV_HOTBACKUP
/**********************************************************************//**
Gets the space id, page offset, and byte offset within page of a

11
storage/xtradb/include/fil0crypt.h
View File

@ -409,16 +409,9 @@ encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
@return whether the encrypted page is OK */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
const fil_space_t* space,
ulint pageno)
bool fil_space_verify_crypt_checksum(const byte* page, ulint zip_size)
MY_ATTRIBUTE((warn_unused_result));
/*********************************************************************

3
storage/xtradb/row/row0import.cc
View File

@ -3537,8 +3537,7 @@ not_encrypted:
}
} else {
if (!fil_space_verify_crypt_checksum(
src, callback.get_zip_size(),
NULL, block->page.offset)) {
src, callback.get_zip_size())) {
goto page_corrupted;
}