database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-08-08 11:22:35 +03:00
Code Activity

MDEV-8770: Incorrect error message when importing page compressed tablespace

Browse Source 
Added decompression and after page update recompression support for
import.
This commit is contained in:
Jan Lindström
2015-09-21 11:24:08 +03:00
parent e96f3c79b9
commit 89af0f11a8
12 changed files with 519 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
mysql-test/suite/encryption/r/innodb-discard-import.result Normal file
View File

@@ -0,0 +1,144 @@
call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded");
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
SET GLOBAL innodb_compression_algorithm = 1;
create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=4;
show warnings;
Level Code Message
create table t2(c1 bigint not null, b char(200)) engine=innodb page_compressed=1 encrypted=yes encryption_key_id=4;
show warnings;
Level Code Message
create table t3(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4;
show warnings;
Level Code Message
create table t4(c1 bigint not null, b char(200)) engine=innodb page_compressed=1;
show warnings;
Level Code Message
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values(current_num, repeat('foobar',30));
insert into t2 values(current_num, repeat('barfoo',30));
insert into t3 values(current_num, repeat('tmpres',30));
insert into t4 values(current_num, repeat('mysql',30));
set current_num = current_num + 1;
end while;
end//
commit;
set autocommit=0;
call innodb_insert_proc(2000);
commit;
set autocommit=1;
select count(*) from t1;
count(*)
2000
select count(*) from t2;
count(*)
2000
select count(*) from t3;
count(*)
2000
select count(*) from t4;
count(*)
2000
FLUSH TABLE t1,t2,t3,t4 FOR EXPORT;
# List before copying files
t1.cfg
t1.frm
t1.ibd
t2.cfg
t2.frm
t2.ibd
t3.cfg
t3.frm
t3.ibd
t4.cfg
t4.frm
t4.ibd
UNLOCK TABLES;
# tables should be either encrypted and/or compressed
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 yes on expecting NOT FOUND
NOT FOUND /barfoo/ in t2.ibd
# t3 yes on expecting NOT FOUND
NOT FOUND /tmpres/ in t3.ibd
# t4 yes on expecting NOT FOUND
NOT FOUND /mysql/ in t4.ibd
ALTER TABLE t1 DISCARD TABLESPACE;
ALTER TABLE t2 DISCARD TABLESPACE;
ALTER TABLE t3 DISCARD TABLESPACE;
ALTER TABLE t4 DISCARD TABLESPACE;
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
SET GLOBAL innodb_compression_algorithm = 1;
# List after t1 DISCARD
t1.frm
t2.frm
t3.frm
t4.frm
ALTER TABLE t1 IMPORT TABLESPACE;
Warnings:
Warning 1814 Tablespace has been discarded for table 't1'
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`c1` bigint(20) NOT NULL,
`b` char(200) DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=4
SELECT COUNT(*) FROM t1;
COUNT(*)
2000
ALTER TABLE t2 IMPORT TABLESPACE;
Warnings:
Warning 1814 Tablespace has been discarded for table 't2'
SHOW CREATE TABLE t2;
Table Create Table
t2 CREATE TABLE `t2` (
`c1` bigint(20) NOT NULL,
`b` char(200) DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `encrypted`=yes `encryption_key_id`=4
SELECT COUNT(*) FROM t2;
COUNT(*)
2000
ALTER TABLE t3 IMPORT TABLESPACE;
Warnings:
Warning 1814 Tablespace has been discarded for table 't3'
SHOW CREATE TABLE t3;
Table Create Table
t3 CREATE TABLE `t3` (
`c1` bigint(20) NOT NULL,
`b` char(200) DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `encrypted`=yes `encryption_key_id`=4
SELECT COUNT(*) FROM t3;
COUNT(*)
2000
ALTER TABLE t4 IMPORT TABLESPACE;
Warnings:
Warning 1814 Tablespace has been discarded for table 't4'
SHOW CREATE TABLE t4;
Table Create Table
t4 CREATE TABLE `t4` (
`c1` bigint(20) NOT NULL,
`b` char(200) DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1
SELECT COUNT(*) FROM t4;
COUNT(*)
2000
flush data to disk
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
SET GLOBAL innodb_compression_algorithm = 1;
# tables should be still either encrypted and/or compressed
# t1 yes on expecting NOT FOUND
NOT FOUND /foobar/ in t1.ibd
# t2 yes on expecting NOT FOUND
NOT FOUND /barfoo/ in t2.ibd
# t3 yes on expecting NOT FOUND
NOT FOUND /tmpres/ in t3.ibd
# t4 yes on expecting NOT FOUND
NOT FOUND /mysql/ in t4.ibd
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1,t2,t3,t4;

167
mysql-test/suite/encryption/t/innodb-discard-import.test Normal file
View File

@@ -0,0 +1,167 @@
-- source include/have_innodb.inc
-- source include/have_file_key_management_plugin.inc
# embedded does not support restart
-- source include/not_embedded.inc
-- source include/not_valgrind.inc
# Avoid CrashReporter popup on Mac
-- source include/not_crashrep.inc
#
# MDEV-8770: Incorrect error message when importing page compressed tablespace
#
call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded");
--disable_query_log
let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`;
--enable_query_log
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
SET GLOBAL innodb_compression_algorithm = 1;
--let $MYSQLD_TMPDIR = `SELECT @@tmpdir`
--let $MYSQLD_DATADIR = `SELECT @@datadir`
--let SEARCH_RANGE = 10000000
--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd
--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd
--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd
--let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd
create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=4;
show warnings;
create table t2(c1 bigint not null, b char(200)) engine=innodb page_compressed=1 encrypted=yes encryption_key_id=4;
show warnings;
create table t3(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4;
show warnings;
create table t4(c1 bigint not null, b char(200)) engine=innodb page_compressed=1;
show warnings;
delimiter //;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into t1 values(current_num, repeat('foobar',30));
insert into t2 values(current_num, repeat('barfoo',30));
insert into t3 values(current_num, repeat('tmpres',30));
insert into t4 values(current_num, repeat('mysql',30));
set current_num = current_num + 1;
end while;
end//
delimiter ;//
commit;
set autocommit=0;
call innodb_insert_proc(2000);
commit;
set autocommit=1;
select count(*) from t1;
select count(*) from t2;
select count(*) from t3;
select count(*) from t4;
FLUSH TABLE t1,t2,t3,t4 FOR EXPORT;
--echo # List before copying files
--list_files $MYSQLD_DATADIR/test
--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_TMPDIR/t1.cfg
--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_TMPDIR/t1.ibd
--copy_file $MYSQLD_DATADIR/test/t2.cfg $MYSQLD_TMPDIR/t2.cfg
--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_TMPDIR/t2.ibd
--copy_file $MYSQLD_DATADIR/test/t3.cfg $MYSQLD_TMPDIR/t3.cfg
--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_TMPDIR/t3.ibd
--copy_file $MYSQLD_DATADIR/test/t4.cfg $MYSQLD_TMPDIR/t4.cfg
--copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_TMPDIR/t4.ibd
UNLOCK TABLES;
--echo # tables should be either encrypted and/or compressed
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t2 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=tmpres
--echo # t3 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t4 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- source include/search_pattern_in_file.inc
ALTER TABLE t1 DISCARD TABLESPACE;
ALTER TABLE t2 DISCARD TABLESPACE;
ALTER TABLE t3 DISCARD TABLESPACE;
ALTER TABLE t4 DISCARD TABLESPACE;
--source include/restart_mysqld.inc
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
SET GLOBAL innodb_compression_algorithm = 1;
--echo # List after t1 DISCARD
--list_files $MYSQLD_DATADIR/test
--copy_file $MYSQLD_TMPDIR/t1.cfg $MYSQLD_DATADIR/test/t1.cfg
--copy_file $MYSQLD_TMPDIR/t1.ibd $MYSQLD_DATADIR/test/t1.ibd
--copy_file $MYSQLD_TMPDIR/t2.cfg $MYSQLD_DATADIR/test/t2.cfg
--copy_file $MYSQLD_TMPDIR/t2.ibd $MYSQLD_DATADIR/test/t2.ibd
--copy_file $MYSQLD_TMPDIR/t3.cfg $MYSQLD_DATADIR/test/t3.cfg
--copy_file $MYSQLD_TMPDIR/t3.ibd $MYSQLD_DATADIR/test/t3.ibd
--copy_file $MYSQLD_TMPDIR/t4.cfg $MYSQLD_DATADIR/test/t4.cfg
--copy_file $MYSQLD_TMPDIR/t4.ibd $MYSQLD_DATADIR/test/t4.ibd
ALTER TABLE t1 IMPORT TABLESPACE;
SHOW CREATE TABLE t1;
SELECT COUNT(*) FROM t1;
ALTER TABLE t2 IMPORT TABLESPACE;
SHOW CREATE TABLE t2;
SELECT COUNT(*) FROM t2;
ALTER TABLE t3 IMPORT TABLESPACE;
SHOW CREATE TABLE t3;
SELECT COUNT(*) FROM t3;
ALTER TABLE t4 IMPORT TABLESPACE;
SHOW CREATE TABLE t4;
SELECT COUNT(*) FROM t4;
--echo flush data to disk
--source include/restart_mysqld.inc
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
SET GLOBAL innodb_compression_algorithm = 1;
--echo # tables should be still either encrypted and/or compressed
--let SEARCH_PATTERN=foobar
--echo # t1 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t1_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=barfoo
--echo # t2 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t2_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=tmpres
--echo # t3 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t3_IBD
-- source include/search_pattern_in_file.inc
--let SEARCH_PATTERN=mysql
--echo # t4 yes on expecting NOT FOUND
-- let SEARCH_FILE=$t4_IBD
-- source include/search_pattern_in_file.inc
DROP PROCEDURE innodb_insert_proc;
DROP TABLE t1,t2,t3,t4;
# reset system
--disable_query_log
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo;
--enable_query_log

1
storage/innobase/buf/buf0buf.cc
View File

@@ -5892,6 +5892,7 @@ buf_page_init_for_backup_restore(
Reserve unused slot from temporary memory array and allocate necessary Reserve unused slot from temporary memory array and allocate necessary
temporary memory if not yet allocated. temporary memory if not yet allocated.
@return reserved slot */ @return reserved slot */
UNIV_INTERN
buf_tmp_buffer_t* buf_tmp_buffer_t*
buf_pool_reserve_tmp_slot( buf_pool_reserve_tmp_slot(
/*======================*/ /*======================*/

3
storage/innobase/fil/fil0crypt.cc
View File

@@ -613,6 +613,9 @@ fil_encrypt_buf(
memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, memcpy(dst_frame + page_size - FIL_PAGE_DATA_END,
src_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END,
FIL_PAGE_DATA_END); FIL_PAGE_DATA_END);
} else {
/* Clean up rest of buffer */
memset(dst_frame+header_len+srclen, 0, page_size - (header_len+srclen));
} }
/* handle post encryption checksum */ /* handle post encryption checksum */

47
storage/innobase/fil/fil0fil.cc
View File

@@ -6446,15 +6446,22 @@ fil_iterate(
for (ulint i = 0; i < n_pages_read; ++i) { for (ulint i = 0; i < n_pages_read; ++i) {
ulint size = iter.page_size; ulint size = iter.page_size;
dberr_t err = DB_SUCCESS; dberr_t err = DB_SUCCESS;
byte* src = (readptr + (i * size));
byte* dst = (io_buffer + (i * size));
ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE);
bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ||
page_type == FIL_PAGE_PAGE_COMPRESSED);
/* If tablespace is encrypted, we need to decrypt /* If tablespace is encrypted, we need to decrypt
the page. */ the page. */
if (encrypted) { if (encrypted) {
decrypted = fil_space_decrypt( decrypted = fil_space_decrypt(
iter.crypt_data, iter.crypt_data,
io_buffer + i * size, //dst dst, //dst
iter.page_size, iter.page_size,
readptr + i * size, // src src, // src
&err); // src &err); // src
if (err != DB_SUCCESS) { if (err != DB_SUCCESS) {
@@ -6465,13 +6472,19 @@ fil_iterate(
updated = true; updated = true;
} else { } else {
/* TODO: remove unnecessary memcpy's */ /* TODO: remove unnecessary memcpy's */
memcpy(io_buffer + i * size, readptr + i * size, size); memcpy(dst, src, size);
} }
} }
/* If the original page is page_compressed, we need
to decompress page before we can update it. */
if (page_compressed) {
fil_decompress_page(NULL, dst, size, NULL);
updated = true;
}
buf_block_set_file_page(block, space_id, page_no++); buf_block_set_file_page(block, space_id, page_no++);
if ((err = callback(page_off, block)) != DB_SUCCESS) { if ((err = callback(page_off, block)) != DB_SUCCESS) {
return(err); return(err);
@@ -6484,12 +6497,28 @@ fil_iterate(
buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_NOT_USED);
buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
src = (io_buffer + (i * size));
if (page_compressed) {
ulint len = 0;
byte* res = fil_compress_page(space_id,
src,
NULL,
size,
fil_space_get_page_compression_level(space_id),
fil_space_get_block_size(space_id, offset, size),
encrypted,
&len,
NULL);
updated = true;
}
/* If tablespace is encrypted, encrypt page before we /* If tablespace is encrypted, encrypt page before we
write it back. Note that we should not encrypt the write it back. Note that we should not encrypt the
buffer that is in buffer pool. */ buffer that is in buffer pool. */
if (decrypted && encrypted) { if (decrypted && encrypted) {
unsigned char *src = io_buffer + (i * size); unsigned char *dest = (writeptr + (i * size));
unsigned char *dst = writeptr + (i * size);
ulint space = mach_read_from_4( ulint space = mach_read_from_4(
src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET);
@@ -6502,12 +6531,14 @@ fil_iterate(
lsn, lsn,
src, src,
iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size, iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size,
dst); dest);
if (tmp == src) { if (tmp == src) {
/* TODO: remove unnecessary memcpy's */ /* TODO: remove unnecessary memcpy's */
memcpy(writeptr, src, size); memcpy(dest, src, size);
} }
updated = true;
} }
page_off += iter.page_size; page_off += iter.page_size;

68
storage/innobase/fil/fil0pagecompress.cc
View File

@@ -111,11 +111,23 @@ fil_compress_page(
/* Cache to avoid change during function execution */ /* Cache to avoid change during function execution */
ulint comp_method = innodb_compression_algorithm; ulint comp_method = innodb_compression_algorithm;
ulint orig_page_type; ulint orig_page_type;
bool allocated=false;
if (encrypted) { if (encrypted) {
header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE;
} }
if (!out_buf) {
allocated = true;
out_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
#ifdef HAVE_LZO
if (comp_method == PAGE_LZO_ALGORITHM) {
lzo_mem = static_cast<byte *>(ut_malloc(LZO1X_1_15_MEM_COMPRESS));
memset(lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS);
}
#endif
}
ut_ad(buf); ut_ad(buf);
ut_ad(out_buf); ut_ad(out_buf);
ut_ad(len); ut_ad(len);
@@ -124,6 +136,10 @@ fil_compress_page(
/* read original page type */ /* read original page type */
orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE);
fil_system_enter();
fil_space_t* space = fil_space_get_by_id(space_id);
fil_system_exit();
/* Let's not compress file space header or /* Let's not compress file space header or
extent descriptor */ extent descriptor */
if (orig_page_type == 0 || if (orig_page_type == 0 ||
@@ -131,12 +147,9 @@ fil_compress_page(
orig_page_type == FIL_PAGE_TYPE_XDES || orig_page_type == FIL_PAGE_TYPE_XDES ||
orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { orig_page_type == FIL_PAGE_PAGE_COMPRESSED) {
*out_len = len; *out_len = len;
return (buf);
}
fil_system_enter(); goto err_exit;
fil_space_t* space = fil_space_get_by_id(space_id); }
fil_system_exit();
/* If no compression level was provided to this table, use system /* If no compression level was provided to this table, use system
default level */ default level */
@@ -174,7 +187,7 @@ fil_compress_page(
#endif #endif
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
#endif /* HAVE_LZ4 */ #endif /* HAVE_LZ4 */
@@ -190,9 +203,10 @@ fil_compress_page(
space_id, fil_space_name(space), len, err, write_size); space_id, fil_space_name(space), len, err, write_size);
space->printed_compression_failure = true; space->printed_compression_failure = true;
} }
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
@@ -218,9 +232,10 @@ fil_compress_page(
space_id, fil_space_name(space), len, err, out_pos); space_id, fil_space_name(space), len, err, out_pos);
space->printed_compression_failure = true; space->printed_compression_failure = true;
} }
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
write_size = out_pos; write_size = out_pos;
@@ -248,9 +263,10 @@ fil_compress_page(
space_id, fil_space_name(space), len, err, write_size); space_id, fil_space_name(space), len, err, write_size);
space->printed_compression_failure = true; space->printed_compression_failure = true;
} }
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
} }
@@ -270,9 +286,10 @@ fil_compress_page(
space_id, fil_space_name(space), len, (int)cstatus, write_size); space_id, fil_space_name(space), len, (int)cstatus, write_size);
space->printed_compression_failure = true; space->printed_compression_failure = true;
} }
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
} }
@@ -293,7 +310,7 @@ fil_compress_page(
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
@@ -339,15 +356,17 @@ fil_compress_page(
byte *comp_page; byte *comp_page;
byte *uncomp_page; byte *uncomp_page;
comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); memcpy(comp_page, out_buf, UNIV_PAGE_SIZE);
fil_decompress_page(uncomp_page, comp_page, len, NULL); fil_decompress_page(uncomp_page, comp_page, len, NULL);
if(buf_page_is_corrupted(false, uncomp_page, 0)) { if(buf_page_is_corrupted(false, uncomp_page, 0)) {
buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH);
ut_error; ut_error;
} }
ut_free(comp_page); ut_free(comp_page);
ut_free(uncomp_page); ut_free(uncomp_page);
} }
@@ -366,6 +385,8 @@ fil_compress_page(
size_t tmp = write_size; size_t tmp = write_size;
write_size = (size_t)ut_uint64_align_up((ib_uint64_t)write_size, block_size); write_size = (size_t)ut_uint64_align_up((ib_uint64_t)write_size, block_size);
/* Clean up the end of buffer */
memset(out_buf+tmp, 0, write_size - tmp);
#ifdef UNIV_DEBUG #ifdef UNIV_DEBUG
ut_a(write_size > 0 && ((write_size % block_size) == 0)); ut_a(write_size > 0 && ((write_size % block_size) == 0));
ut_a(write_size >= tmp); ut_a(write_size >= tmp);
@@ -384,12 +405,30 @@ fil_compress_page(
/* If we do not persistently trim rest of page, we need to write it /* If we do not persistently trim rest of page, we need to write it
all */ all */
if (!srv_use_trim) { if (!srv_use_trim) {
memset(out_buf+write_size,0,len-write_size);
write_size = len; write_size = len;
} }
*out_len = write_size; *out_len = write_size;
if (allocated) {
/* TODO: reduce number of memcpy's */
memcpy(buf, out_buf, len);
} else {
return(out_buf); return(out_buf);
}
err_exit:
if (allocated) {
ut_free(out_buf);
#ifdef HAVE_LZO
if (comp_method == PAGE_LZO_ALGORITHM) {
ut_free(lzo_mem);
}
#endif
}
return (buf);
} }
@@ -432,7 +471,8 @@ fil_decompress_page(
// If no buffer was given, we need to allocate temporal buffer // If no buffer was given, we need to allocate temporal buffer
if (page_buf == NULL) { if (page_buf == NULL) {
in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
memset(in_buf, 0, UNIV_PAGE_SIZE);
} else { } else {
in_buf = page_buf; in_buf = page_buf;
} }

9
storage/innobase/include/fsp0pagecompress.ic
View File

@@ -21,7 +21,8 @@ this program; if not, write to the Free Software Foundation, Inc.,
Implementation for helper functions for extracting/storing page Implementation for helper functions for extracting/storing page
compression and atomic writes information to file space. compression and atomic writes information to file space.
Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com
***********************************************************************/ ***********************************************************************/
/********************************************************************//** /********************************************************************//**
@@ -100,7 +101,7 @@ fil_page_is_compressed_encrypted(
/*******************************************************************//** /*******************************************************************//**
Returns the page compression level of the space, or 0 if the space Returns the page compression level of the space, or 0 if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
@return page compression level, ULINT_UNDEFINED if space not found */ @return page compression level, 0 if space not found */
UNIV_INLINE UNIV_INLINE
ulint ulint
fil_space_get_page_compression_level( fil_space_get_page_compression_level(
@@ -116,7 +117,7 @@ fil_space_get_page_compression_level(
return(fsp_flags_get_page_compression_level(flags)); return(fsp_flags_get_page_compression_level(flags));
} }
return(flags); return(0);
} }
/*******************************************************************//** /*******************************************************************//**
@@ -138,7 +139,7 @@ fil_space_is_page_compressed(
return(fsp_flags_is_page_compressed(flags)); return(fsp_flags_is_page_compressed(flags));
} }
return(flags); return(0);
} }
#endif /* UNIV_INNOCHECKSUM */ #endif /* UNIV_INNOCHECKSUM */

1
storage/xtradb/buf/buf0buf.cc
View File

@@ -6070,6 +6070,7 @@ buf_pool_mutex_exit(
Reserve unused slot from temporary memory array and allocate necessary Reserve unused slot from temporary memory array and allocate necessary
temporary memory if not yet allocated. temporary memory if not yet allocated.
@return reserved slot */ @return reserved slot */
UNIV_INTERN
buf_tmp_buffer_t* buf_tmp_buffer_t*
buf_pool_reserve_tmp_slot( buf_pool_reserve_tmp_slot(
/*======================*/ /*======================*/

3
storage/xtradb/fil/fil0crypt.cc
View File

@@ -613,6 +613,9 @@ fil_encrypt_buf(
memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, memcpy(dst_frame + page_size - FIL_PAGE_DATA_END,
src_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END,
FIL_PAGE_DATA_END); FIL_PAGE_DATA_END);
} else {
/* Clean up rest of buffer */
memset(dst_frame+header_len+srclen, 0, page_size - (header_len+srclen));
} }
/* handle post encryption checksum */ /* handle post encryption checksum */

47
storage/xtradb/fil/fil0fil.cc
View File

@@ -6503,15 +6503,22 @@ fil_iterate(
for (ulint i = 0; i < n_pages_read; ++i) { for (ulint i = 0; i < n_pages_read; ++i) {
ulint size = iter.page_size; ulint size = iter.page_size;
dberr_t err = DB_SUCCESS; dberr_t err = DB_SUCCESS;
byte* src = (readptr + (i * size));
byte* dst = (io_buffer + (i * size));
ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE);
bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ||
page_type == FIL_PAGE_PAGE_COMPRESSED);
/* If tablespace is encrypted, we need to decrypt /* If tablespace is encrypted, we need to decrypt
the page. */ the page. */
if (encrypted) { if (encrypted) {
decrypted = fil_space_decrypt( decrypted = fil_space_decrypt(
iter.crypt_data, iter.crypt_data,
io_buffer + i * size, //dst dst, //dst
iter.page_size, iter.page_size,
readptr + i * size, // src src, // src
&err); // src &err); // src
if (err != DB_SUCCESS) { if (err != DB_SUCCESS) {
@@ -6522,13 +6529,19 @@ fil_iterate(
updated = true; updated = true;
} else { } else {
/* TODO: remove unnecessary memcpy's */ /* TODO: remove unnecessary memcpy's */
memcpy(io_buffer + i * size, readptr + i * size, size); memcpy(dst, src, size);
} }
} }
/* If the original page is page_compressed, we need
to decompress page before we can update it. */
if (page_compressed) {
fil_decompress_page(NULL, dst, size, NULL);
updated = true;
}
buf_block_set_file_page(block, space_id, page_no++); buf_block_set_file_page(block, space_id, page_no++);
if ((err = callback(page_off, block)) != DB_SUCCESS) { if ((err = callback(page_off, block)) != DB_SUCCESS) {
return(err); return(err);
@@ -6541,12 +6554,28 @@ fil_iterate(
buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_NOT_USED);
buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE);
src = (io_buffer + (i * size));
if (page_compressed) {
ulint len = 0;
byte* res = fil_compress_page(space_id,
src,
NULL,
size,
fil_space_get_page_compression_level(space_id),
fil_space_get_block_size(space_id, offset, size),
encrypted,
&len,
NULL);
updated = true;
}
/* If tablespace is encrypted, encrypt page before we /* If tablespace is encrypted, encrypt page before we
write it back. Note that we should not encrypt the write it back. Note that we should not encrypt the
buffer that is in buffer pool. */ buffer that is in buffer pool. */
if (decrypted && encrypted) { if (decrypted && encrypted) {
unsigned char *src = io_buffer + (i * size); unsigned char *dest = (writeptr + (i * size));
unsigned char *dst = writeptr + (i * size);
ulint space = mach_read_from_4( ulint space = mach_read_from_4(
src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET);
@@ -6559,12 +6588,14 @@ fil_iterate(
lsn, lsn,
src, src,
iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size, iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size,
dst); dest);
if (tmp == src) { if (tmp == src) {
/* TODO: remove unnecessary memcpy's */ /* TODO: remove unnecessary memcpy's */
memcpy(writeptr, src, size); memcpy(dest, src, size);
} }
updated = true;
} }
page_off += iter.page_size; page_off += iter.page_size;

70
storage/xtradb/fil/fil0pagecompress.cc
View File

@@ -111,11 +111,23 @@ fil_compress_page(
/* Cache to avoid change during function execution */ /* Cache to avoid change during function execution */
ulint comp_method = innodb_compression_algorithm; ulint comp_method = innodb_compression_algorithm;
ulint orig_page_type; ulint orig_page_type;
bool allocated=false;
if (encrypted) { if (encrypted) {
header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE;
} }
if (!out_buf) {
allocated = true;
out_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
#ifdef HAVE_LZO
if (comp_method == PAGE_LZO_ALGORITHM) {
lzo_mem = static_cast<byte *>(ut_malloc(LZO1X_1_15_MEM_COMPRESS));
memset(lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS);
}
#endif
}
ut_ad(buf); ut_ad(buf);
ut_ad(out_buf); ut_ad(out_buf);
ut_ad(len); ut_ad(len);
@@ -124,6 +136,10 @@ fil_compress_page(
/* read original page type */ /* read original page type */
orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE);
fil_system_enter();
fil_space_t* space = fil_space_get_by_id(space_id);
fil_system_exit();
/* Let's not compress file space header or /* Let's not compress file space header or
extent descriptor */ extent descriptor */
if (orig_page_type == 0 || if (orig_page_type == 0 ||
@@ -131,12 +147,9 @@ fil_compress_page(
orig_page_type == FIL_PAGE_TYPE_XDES || orig_page_type == FIL_PAGE_TYPE_XDES ||
orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { orig_page_type == FIL_PAGE_PAGE_COMPRESSED) {
*out_len = len; *out_len = len;
return (buf);
}
fil_system_enter(); goto err_exit;
fil_space_t* space = fil_space_get_by_id(space_id); }
fil_system_exit();
/* If no compression level was provided to this table, use system /* If no compression level was provided to this table, use system
default level */ default level */
@@ -174,7 +187,7 @@ fil_compress_page(
#endif #endif
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
#endif /* HAVE_LZ4 */ #endif /* HAVE_LZ4 */
@@ -190,9 +203,10 @@ fil_compress_page(
space_id, fil_space_name(space), len, err, write_size); space_id, fil_space_name(space), len, err, write_size);
space->printed_compression_failure = true; space->printed_compression_failure = true;
} }
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
@@ -218,9 +232,10 @@ fil_compress_page(
space_id, fil_space_name(space), len, err, out_pos); space_id, fil_space_name(space), len, err, out_pos);
space->printed_compression_failure = true; space->printed_compression_failure = true;
} }
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
write_size = out_pos; write_size = out_pos;
@@ -248,9 +263,10 @@ fil_compress_page(
space_id, fil_space_name(space), len, err, write_size); space_id, fil_space_name(space), len, err, write_size);
space->printed_compression_failure = true; space->printed_compression_failure = true;
} }
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
} }
@@ -270,9 +286,10 @@ fil_compress_page(
space_id, fil_space_name(space), len, (int)cstatus, write_size); space_id, fil_space_name(space), len, (int)cstatus, write_size);
space->printed_compression_failure = true; space->printed_compression_failure = true;
} }
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
} }
@@ -293,7 +310,7 @@ fil_compress_page(
srv_stats.pages_page_compression_error.inc(); srv_stats.pages_page_compression_error.inc();
*out_len = len; *out_len = len;
return (buf); goto err_exit;
} }
break; break;
@@ -339,15 +356,17 @@ fil_compress_page(
byte *comp_page; byte *comp_page;
byte *uncomp_page; byte *uncomp_page;
comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); memcpy(comp_page, out_buf, UNIV_PAGE_SIZE);
fil_decompress_page(uncomp_page, comp_page, len, NULL); fil_decompress_page(uncomp_page, comp_page, len, NULL);
if(buf_page_is_corrupted(false, uncomp_page, 0)) { if(buf_page_is_corrupted(false, uncomp_page, 0)) {
buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH);
ut_error; ut_error;
} }
ut_free(comp_page); ut_free(comp_page);
ut_free(uncomp_page); ut_free(uncomp_page);
} }
@@ -363,10 +382,10 @@ fil_compress_page(
/* Actual write needs to be alligned on block size */ /* Actual write needs to be alligned on block size */
if (write_size % block_size) { if (write_size % block_size) {
#ifdef UNIV_DEBUG
size_t tmp = write_size; size_t tmp = write_size;
#endif
write_size = (size_t)ut_uint64_align_up((ib_uint64_t)write_size, block_size); write_size = (size_t)ut_uint64_align_up((ib_uint64_t)write_size, block_size);
/* Clean up the end of buffer */
memset(out_buf+tmp, 0, write_size - tmp);
#ifdef UNIV_DEBUG #ifdef UNIV_DEBUG
ut_a(write_size > 0 && ((write_size % block_size) == 0)); ut_a(write_size > 0 && ((write_size % block_size) == 0));
ut_a(write_size >= tmp); ut_a(write_size >= tmp);
@@ -385,12 +404,30 @@ fil_compress_page(
/* If we do not persistently trim rest of page, we need to write it /* If we do not persistently trim rest of page, we need to write it
all */ all */
if (!srv_use_trim) { if (!srv_use_trim) {
memset(out_buf+write_size,0,len-write_size);
write_size = len; write_size = len;
} }
*out_len = write_size; *out_len = write_size;
if (allocated) {
/* TODO: reduce number of memcpy's */
memcpy(buf, out_buf, len);
} else {
return(out_buf); return(out_buf);
}
err_exit:
if (allocated) {
ut_free(out_buf);
#ifdef HAVE_LZO
if (comp_method == PAGE_LZO_ALGORITHM) {
ut_free(lzo_mem);
}
#endif
}
return (buf);
} }
@@ -433,7 +470,8 @@ fil_decompress_page(
// If no buffer was given, we need to allocate temporal buffer // If no buffer was given, we need to allocate temporal buffer
if (page_buf == NULL) { if (page_buf == NULL) {
in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE));
memset(in_buf, 0, UNIV_PAGE_SIZE);
} else { } else {
in_buf = page_buf; in_buf = page_buf;
} }

9
storage/xtradb/include/fsp0pagecompress.ic
View File

@@ -21,7 +21,8 @@ this program; if not, write to the Free Software Foundation, Inc.,
Implementation for helper functions for extracting/storing page Implementation for helper functions for extracting/storing page
compression and atomic writes information to file space. compression and atomic writes information to file space.
Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com
***********************************************************************/ ***********************************************************************/
/********************************************************************//** /********************************************************************//**
@@ -100,7 +101,7 @@ fil_page_is_compressed_encrypted(
/*******************************************************************//** /*******************************************************************//**
Returns the page compression level of the space, or 0 if the space Returns the page compression level of the space, or 0 if the space
is not compressed. The tablespace must be cached in the memory cache. is not compressed. The tablespace must be cached in the memory cache.
@return page compression level, ULINT_UNDEFINED if space not found */ @return page compression level, 0 if space not found */
UNIV_INLINE UNIV_INLINE
ulint ulint
fil_space_get_page_compression_level( fil_space_get_page_compression_level(
@@ -116,7 +117,7 @@ fil_space_get_page_compression_level(
return(fsp_flags_get_page_compression_level(flags)); return(fsp_flags_get_page_compression_level(flags));
} }
return(flags); return(0);
} }
/*******************************************************************//** /*******************************************************************//**
@@ -138,7 +139,7 @@ fil_space_is_page_compressed(
return(fsp_flags_is_page_compressed(flags)); return(fsp_flags_is_page_compressed(flags));
} }
return(flags); return(0);
} }
#endif /* UNIV_INNOCHECKSUM */ #endif /* UNIV_INNOCHECKSUM */