database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-11-28 17:36:30 +03:00
Code Activity
Files
7b33a6a1ea80dc628e6614368449e6b2bcb267f2
mariadb/mysql-test/suite/innodb/t/doublewrite.test
Thirunarayanan Balathandayuthapani c0f47a4a58 MDEV-12026: Implement innodb_checksum_algorithm=full_crc32 
MariaDB data-at-rest encryption (innodb_encrypt_tables)
had repurposed the same unused data field that was repurposed
in MySQL 5.7 (and MariaDB 10.2) for the Split Sequence Number (SSN)
field of SPATIAL INDEX. Because of this, MariaDB was unable to
support encryption on SPATIAL INDEX pages.

Furthermore, InnoDB page checksums skipped some bytes, and there
are multiple variations and checksum algorithms. By default,
InnoDB accepts all variations of all algorithms that ever existed.
This unnecessarily weakens the page checksums.

We hereby introduce two more innodb_checksum_algorithm variants
(full_crc32, strict_full_crc32) that are special in a way:
When either setting is active, newly created data files will
carry a flag (fil_space_t::full_crc32()) that indicates that
all pages of the file will use a full CRC-32C checksum over the
entire page contents (excluding the bytes where the checksum
is stored, at the very end of the page). Such files will always
use that checksum, no matter what the parameter
innodb_checksum_algorithm is assigned to.

For old files, the old checksum algorithms will continue to be
used. The value strict_full_crc32 will be equivalent to strict_crc32
and the value full_crc32 will be equivalent to crc32.

ROW_FORMAT=COMPRESSED tables will only use the old format.
These tables do not support new features, such as larger
innodb_page_size or instant ADD/DROP COLUMN. They may be
deprecated in the future. We do not want an unnecessary
file format change for them.

The new full_crc32() format also cleans up the MariaDB tablespace
flags. We will reserve flags to store the page_compressed
compression algorithm, and to store the compressed payload length,
so that checksum can be computed over the compressed (and
possibly encrypted) stream and can be validated without
decrypting or decompressing the page.

In the full_crc32 format, there no longer are separate before-encryption
and after-encryption checksums for pages. The single checksum is
computed on the page contents that is written to the file.

We do not make the new algorithm the default for two reasons.
First, MariaDB 10.4.2 was a beta release, and the default values
of parameters should not change after beta. Second, we did not
yet implement the full_crc32 format for page_compressed pages.
This will be fixed in MDEV-18644.

This is joint work with Marko Mäkelä.
2019-02-19 18:50:19 +02:00

463 lines
14 KiB
Plaintext
Raw Blame History

--echo #
--echo # Bug #17335427 INNODB CAN NOT USE THE DOUBLEWRITE BUFFER PROPERLY
--echo # Bug #18144349 INNODB CANNOT USE THE DOUBLEWRITE BUFFER FOR THE FIRST
--echo # PAGE OF SYSTEM TABLESPACE
--echo #
--source include/innodb_page_size.inc
--source include/have_debug.inc
--source include/not_embedded.inc
# Slow shutdown and restart to make sure ibuf merge is finished
SET GLOBAL innodb_fast_shutdown = 0;
--disable_query_log
call mtr.add_suppression("InnoDB: Header page consists of zero bytes");
call mtr.add_suppression("InnoDB: Checksum mismatch in datafile: .*, Space ID:0, Flags: 0");
call mtr.add_suppression("InnoDB: Data file .* uses page size .* but the innodb_page_size start-up parameter is");
call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS");
call mtr.add_suppression("InnoDB: New log files created");
call mtr.add_suppression("InnoDB: Cannot create doublewrite buffer: the first file in innodb_data_file_path must be at least (3|6|12)M\\.");
call mtr.add_suppression("InnoDB: Database creation was aborted");
call mtr.add_suppression("Plugin 'InnoDB' (init function returned error|registration as a STORAGE ENGINE failed)");
call mtr.add_suppression("InnoDB: A bad Space ID was found in datafile.*");
call mtr.add_suppression("InnoDB: Checksum mismatch in datafile: .*");
--enable_query_log
--source include/restart_mysqld.inc
let INNODB_PAGE_SIZE=`select @@innodb_page_size`;
let MYSQLD_DATADIR=`select @@datadir`;
let ALGO=`select @@innodb_checksum_algorithm`;
let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err;
show variables like 'innodb_doublewrite';
show variables like 'innodb_fil_make_page_dirty_debug';
show variables like 'innodb_saved_page_number_debug';
create table t1 (f1 int primary key, f2 blob) engine=innodb;
start transaction;
insert into t1 values(1, repeat('#',12));
insert into t1 values(2, repeat('+',12));
insert into t1 values(3, repeat('/',12));
insert into t1 values(4, repeat('-',12));
insert into t1 values(5, repeat('.',12));
commit work;
--echo # ---------------------------------------------------------------
--echo # Test Begin: Test if recovery works if first page of user
--echo # tablespace is full of zeroes.
select space from information_schema.innodb_sys_tables
where name = 'test/t1' into @space_id;
--echo # Ensure that dirty pages of table t1 is flushed.
flush tables t1 for export;
unlock tables;
begin;
insert into t1 values (6, repeat('%', 12));
--source ../include/no_checkpoint_start.inc
--echo # Make the first page dirty for table t1
set global innodb_saved_page_number_debug = 0;
set global innodb_fil_make_page_dirty_debug = @space_id;
--echo # Ensure that dirty pages of table t1 are flushed.
set global innodb_buf_flush_list_now = 1;
--let CLEANUP_IF_CHECKPOINT=drop table t1;
--source ../include/no_checkpoint_end.inc
--echo # Make the first page (page_no=0) of the user tablespace
--echo # full of zeroes.
--echo #
--echo # MDEV-11623: Use old FSP_SPACE_FLAGS in the doublewrite buffer.
perl;
use IO::Handle;
do "$ENV{MTR_SUITE_DIR}/include/crc32.pl";
my $polynomial = 0x82f63b78; # CRC-32C
my $algo = $ENV{ALGO};
die "Unsupported innodb_checksum_algorithm=$algo\n" unless $algo =~ /crc32/;
my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd";
my $page_size = $ENV{INNODB_PAGE_SIZE};
my $page;
do "$ENV{MTR_SUITE_DIR}/../innodb/include/crc32.pl";
open(FILE, "+<", $fname) or die;
sysread(FILE, $page, $page_size)==$page_size||die "Unable to read $name\n";
my $page1 = $page;
substr($page1, 34, 4) = pack("N", 0);
my $polynomial0 = 0x82f63b78; # CRC-32C
my $ck0 = mycrc32(substr($page1, 0, ($page_size-4)), 0, $polynomial0);
substr($page1, ($page_size - 4), 4) = pack("N", $ck0);
sysseek(FILE, 0, 0)||die "Unable to seek $fname\n";
die unless syswrite(FILE, $page1, $page_size) == $page_size;
close FILE;
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}ibdata1")||die "cannot open ibdata1\n";
sysseek(FILE, 6 * $page_size - 190, 0)||die "Unable to seek ibdata1\n";
sysread(FILE, $_, 12) == 12||die "Unable to read TRX_SYS\n";
my($magic,$d1,$d2)=unpack "NNN", $_;
die "magic=$magic, $d1, $d2\n" unless $magic == 536853855 && $d2 >= $d1 + 64;
sysseek(FILE, $d1 * $page_size, 0)||die "Unable to seek ibdata1\n";
# Find the page in the doublewrite buffer
for (my $d = $d1; $d < $d2 + 64; $d++)
{
sysread(FILE, $_, $page_size)==$page_size||die "Cannot read doublewrite\n";
next unless $_ eq $page;
sysseek(FILE, $d * $page_size, 0)||die "Unable to seek ibdata1\n";
# Write buggy MariaDB 10.1.x FSP_SPACE_FLAGS to the doublewrite buffer
my($flags) = unpack "x[54]N", $_;
my $badflags = ($flags & 0x3f);
my $compression_level=6;
$badflags |= 1<<6|$compression_level<<7 if ($flags & 1 << 16);
$badflags |= ($flags & 15 << 6) << 7; # PAGE_SSIZE
substr ($_, 54, 4) = pack("N", $badflags);
if ($algo =~ /full_crc32/)
{
my $ck = mycrc32(substr($_, 0, $page_size - 4), 0, $polynomial);
substr($_, $page_size - 4, 4) = pack("N", $ck);
}
else
{
# Replace the innodb_checksum_algorithm=crc32 checksum
my $ck= pack("N",
mycrc32(substr($_, 4, 22), 0, $polynomial) ^
mycrc32(substr($_, 38, $page_size - 38 - 8), 0,
$polynomial));
substr ($_, 0, 4) = $ck;
substr ($_, $page_size - 8, 4) = $ck;
}
syswrite(FILE, $_, $page_size)==$page_size||die;
close(FILE);
exit 0;
}
die "Did not find the page in the doublewrite buffer ($d1,$d2)\n";
EOF
--source include/start_mysqld.inc
check table t1;
select f1, f2 from t1;
--echo # Test End
--echo # ---------------------------------------------------------------
--echo # Test Begin: Test if recovery works if first page of user
--echo # tablespace is corrupted.
select space from information_schema.innodb_sys_tables
where name = 'test/t1' into @space_id;
--echo # Ensure that dirty pages of table t1 is flushed.
flush tables t1 for export;
unlock tables;
begin;
insert into t1 values (6, repeat('%', 12));
--source ../include/no_checkpoint_start.inc
--echo # Make the first page dirty for table t1
set global innodb_saved_page_number_debug = 0;
set global innodb_fil_make_page_dirty_debug = @space_id;
--echo # Ensure that dirty pages of table t1 are flushed.
set global innodb_buf_flush_list_now = 1;
--source include/no_checkpoint_end.inc
--echo # Corrupt the first page (page_no=0) of the user tablespace.
perl;
use IO::Handle;
my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd";
my $page_size = $ENV{INNODB_PAGE_SIZE};
open(FILE, "+<", $fname) or die;
sysread(FILE, $page, $page_size)==$page_size||die "Unable to read $name\n";
substr($page, 28, 4) = pack("N", 1000);
sysseek(FILE, 0, 0)||die "Unable to seek $fname\n";
die unless syswrite(FILE, $page, $page_size) == $page_size;
close FILE;
EOF
--source include/start_mysqld.inc
check table t1;
select f1, f2 from t1;
--echo # Test End
--echo # ---------------------------------------------------------------
--echo # Test Begin: Test if recovery works if 2nd page of user
--echo # tablespace is full of zeroes.
select space from information_schema.innodb_sys_tables
where name = 'test/t1' into @space_id;
--echo # Ensure that dirty pages of table t1 is flushed.
flush tables t1 for export;
unlock tables;
begin;
insert into t1 values (6, repeat('%', 400));
--source ../include/no_checkpoint_start.inc
--echo # Make the 2nd page dirty for table t1
set global innodb_saved_page_number_debug = 1;
set global innodb_fil_make_page_dirty_debug = @space_id;
--echo # Ensure that dirty pages of table t1 are flushed.
set global innodb_buf_flush_list_now = 1;
--source include/no_checkpoint_end.inc
--echo # Make the 2nd page (page_no=1) of the tablespace all zeroes.
perl;
use IO::Handle;
my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd";
open(FILE, "+<", $fname) or die;
FILE->autoflush(1);
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'}, SEEK_SET);
print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'});
close FILE;
EOF
--source include/start_mysqld.inc
check table t1;
select f1, f2 from t1;
--echo # Test End
--echo # ---------------------------------------------------------------
--echo # Test Begin: Test if recovery works if 2nd page of user
--echo # tablespace is corrupted.
select space from information_schema.innodb_sys_tables
where name = 'test/t1' into @space_id;
--echo # Ensure that dirty pages of table t1 is flushed.
flush tables t1 for export;
unlock tables;
begin;
insert into t1 values (6, repeat('%', 400));
--source ../include/no_checkpoint_start.inc
--echo # Make the 2nd page dirty for table t1
set global innodb_saved_page_number_debug = 1;
set global innodb_fil_make_page_dirty_debug = @space_id;
--echo # Ensure that the dirty pages of table t1 are flushed.
set global innodb_buf_flush_list_now = 1;
--source include/no_checkpoint_end.inc
--echo # Corrupt the 2nd page (page_no=1) of the user tablespace.
perl;
use IO::Handle;
my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd";
open(FILE, "+<", $fname) or die;
FILE->autoflush(1);
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'}, SEEK_SET);
print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}/2);
close FILE;
EOF
--source include/start_mysqld.inc
check table t1;
select f1, f2 from t1;
--echo # Test End
--echo # ---------------------------------------------------------------
--echo # Test Begin: Test if recovery works if first page of
--echo # system tablespace is full of zeroes.
begin;
insert into t1 values (6, repeat('%', 400));
--echo # Ensure that all dirty pages in the system are flushed.
set global innodb_buf_flush_list_now = 1;
--echo # Make the first page dirty for system tablespace
set global innodb_saved_page_number_debug = 0;
set global innodb_fil_make_page_dirty_debug = 0;
--echo # Ensure that the dirty page of system tablespace is also flushed.
# We do this after the transaction starts and all dirty pages have been flushed
# already. So flushing of this specified dirty page will surely keep the
# copy in doublewrite buffer, and no more writes to doublewrite buffer would
# overwrite the copy. Thus, we can safely modify the original page when server
# is down. So do the following testings.
set global innodb_buf_flush_list_now = 1;
--source include/kill_mysqld.inc
--echo # Make the first page (page_no=0) of the system tablespace
--echo # all zeroes.
perl;
use IO::Handle;
my $fname= "$ENV{'MYSQLD_DATADIR'}ibdata1";
open(FILE, "+<", $fname) or die;
FILE->autoflush(1);
binmode FILE;
print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'});
close FILE;
EOF
--source include/start_mysqld.inc
check table t1;
select f1, f2 from t1;
--echo # Test End
--echo # ---------------------------------------------------------------
--echo # Test Begin: Test if recovery works if first page of
--echo # system tablespace is corrupted.
begin;
insert into t1 values (6, repeat('%', 400));
--echo # Ensure that all dirty pages in the system are flushed.
set global innodb_buf_flush_list_now = 1;
--echo # Make the first page dirty for system tablespace
set global innodb_saved_page_number_debug = 0;
set global innodb_fil_make_page_dirty_debug = 0;
--echo # Ensure that the dirty page of system tablespace is also flushed.
set global innodb_buf_flush_list_now = 1;
--source include/kill_mysqld.inc
--echo # Corrupt the first page (page_no=0) of the system tablespace.
perl;
use IO::Handle;
my $fname= "$ENV{'MYSQLD_DATADIR'}ibdata1";
open(FILE, "+<", $fname) or die;
FILE->autoflush(1);
binmode FILE;
print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}/2);
close FILE;
EOF
--source include/start_mysqld.inc
check table t1;
select f1, f2 from t1;
--echo # Test End
--echo # ---------------------------------------------------------------
--echo # Test Begin: Test if recovery works if 2nd page of
--echo # system tablespace is full of zeroes.
begin;
insert into t1 values (6, repeat('%', 400));
--echo # Ensure that all dirty pages in the system are flushed.
set global innodb_buf_flush_list_now = 1;
--echo # Make the second page dirty for system tablespace
set global innodb_saved_page_number_debug = 1;
set global innodb_fil_make_page_dirty_debug = 0;
--echo # Ensure that the dirty page of system tablespace is also flushed.
set global innodb_buf_flush_list_now = 1;
--source include/kill_mysqld.inc
--echo # Make the 2nd page (page_no=1) of the system tablespace
--echo # all zeroes.
perl;
use IO::Handle;
my $fname= "$ENV{'MYSQLD_DATADIR'}ibdata1";
open(FILE, "+<", $fname) or die;
FILE->autoflush(1);
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'}, SEEK_SET);
print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'});
close FILE;
EOF
--source include/start_mysqld.inc
check table t1;
select f1, f2 from t1;
--echo # Test End
--echo # ---------------------------------------------------------------
--echo # Test Begin: Test if recovery works if 2nd page of
--echo # system tablespace is corrupted.
begin;
insert into t1 values (6, repeat('%', 400));
--echo # Ensure that all dirty pages in the system are flushed.
set global innodb_buf_flush_list_now = 1;
--echo # Make the second page dirty for system tablespace
set global innodb_saved_page_number_debug = 1;
set global innodb_fil_make_page_dirty_debug = 0;
--echo # Ensure that the dirty page of system tablespace is also flushed.
set global innodb_buf_flush_list_now = 1;
--source include/kill_mysqld.inc
--echo # Make the 2nd page (page_no=1) of the system tablespace
--echo # all zeroes.
perl;
use IO::Handle;
my $fname= "$ENV{'MYSQLD_DATADIR'}ibdata1";
open(FILE, "+<", $fname) or die;
FILE->autoflush(1);
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'}, SEEK_SET);
print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}/2);
close FILE;
EOF
--source include/start_mysqld.inc
check table t1;
--let SEARCH_PATTERN= \[ERROR\] InnoDB: .*test.t1\\.ibd.*
--source include/search_pattern_in_file.inc
select f1, f2 from t1;
drop table t1;
--echo #
--echo # MDEV-12600 crash during install_db with innodb_page_size=32K
--echo # and ibdata1=3M
--echo #
let bugdir= $MYSQLTEST_VARDIR/tmp/doublewrite;
--mkdir $bugdir
let $check_no_innodb=SELECT * FROM INFORMATION_SCHEMA.ENGINES
WHERE engine = 'innodb'
AND support IN ('YES', 'DEFAULT', 'ENABLED');
--let $ibp=--innodb-log-group-home-dir=$bugdir --innodb-data-home-dir=$bugdir
--let $ibd=$ibp --innodb-undo-tablespaces=0 --innodb-log-files-in-group=2
--let $ibp=$ibp --innodb-data-file-path=ibdata1:1M;ibdata2:1M:autoextend
--let $restart_parameters= $ibp
--source include/restart_mysqld.inc
eval $check_no_innodb;
--let SEARCH_PATTERN= \[ERROR\] InnoDB: Cannot create doublewrite buffer
--source include/search_pattern_in_file.inc
--let $restart_parameters=
--source include/restart_mysqld.inc
--remove_file $bugdir/ibdata1
--remove_file $bugdir/ibdata2
--remove_file $bugdir/ib_logfile0
--remove_file $bugdir/ib_logfile1
--rmdir $bugdir
View Git Blame Copy Permalink