From 37f294fec20e5b27532c156cd6956d65d804a5df Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Tue, 27 Dec 2016 03:21:13 +0200 Subject: [PATCH 01/13] Disable the test for valgrind builds Test is very slow with valgrind, and pointless because it is initially about a race condition which is hardly achievable with valgrind --- mysql-test/t/mdev-504.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/t/mdev-504.test b/mysql-test/t/mdev-504.test index fb5c7666d33..b96c8779c68 100644 --- a/mysql-test/t/mdev-504.test +++ b/mysql-test/t/mdev-504.test @@ -1,3 +1,4 @@ +--source include/not_valgrind.inc --disable_ps_protocol SET GLOBAL net_write_timeout = 900; From 2f5670dc269ce19f14132d54e54060c91cc028e3 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 27 Dec 2016 14:13:32 +0530 Subject: [PATCH 02/13] MDEV-11636 Extra persistent columns on slave always gets NULL in RBR Problem:- In replication if slave has extra persistent column then these column are not computed while applying write-set from master. Solution:- While applying row events from server, we will generate values for extra persistent columns. --- .../rpl/r/rpl_alter_extra_persistent.result | 220 ++++++++++++++++++ .../rpl/t/rpl_alter_extra_persistent.test | 106 +++++++++ sql/rpl_record.cc | 32 +++ sql/rpl_record.h | 1 + 4 files changed, 359 insertions(+) create mode 100644 mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result create mode 100644 mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test diff --git a/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result new file mode 100644 index 00000000000..84915073d87 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result @@ -0,0 +1,220 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +connection slave; +select * from t1 order by a; +a +1 +2 +3 +4 +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +connection master; +insert into t1 values(5); +insert into t1 values(6); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#UPDATE query +connection master; +update t1 set a = a+10; +select * from t1 order by a; +a +11 +12 +13 +14 +15 +16 +connection slave; +select * from t1 order by a; +a z1 z2 +11 12 13 +12 13 14 +13 14 15 +14 15 16 +15 16 17 +16 17 18 +connection master; +update t1 set a = a-10; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#DELETE quert +connection master; +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +#REPLACE query +connection master; +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#SELECT query +connection master; +select * from t1 where a > 2 and a < 4; +a +3 +connection slave; +select * from t1 where a > 2 and a < 4; +a z1 z2 +3 4 5 +#UPDATE with SELECT query +connection master; +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +13 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +13 14 15 +connection master; +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; +connection master; +#entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +connection slave; +include/wait_for_slave_sql_error.inc [errno=1062] +connection slave; +connection slave; +select * from t1 order by a; +a z1 z2 z4 +1 2 3 1 +2 3 4 2 +3 4 5 3 +4 5 6 4 +5 6 7 5 +6 7 8 0 +alter table t1 drop column z4; +start slave; +include/wait_for_slave_sql_to_start.inc +connection slave; +connection slave; +connection master; +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +7 8 9 +connection master; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +drop table t1; +include/rpl_end.inc +connection server_2; +connection server_2; +connection server_2; +connection server_2; +connection server_1; +connection server_1; +connection server_1; +connection server_2; +connection server_1; +connection server_2; +connection server_2; +connection server_1; +connection server_1; diff --git a/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test new file mode 100644 index 00000000000..3b2fff1cb13 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test @@ -0,0 +1,106 @@ +--source include/master-slave.inc +--source include/have_binlog_format_row.inc + +--enable_connect_log +--connection master +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); + +--sync_slave_with_master +select * from t1 order by a; +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; + +--connection master +insert into t1 values(5); +insert into t1 values(6); + +--sync_slave_with_master +select * from t1 order by a; + + +--echo #UPDATE query + +--connection master +update t1 set a = a+10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a-10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #DELETE quert +--connection master +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #REPLACE query +--connection master +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); + +--sync_slave_with_master +select * from t1 order by a; + +--echo #SELECT query +--connection master +select * from t1 where a > 2 and a < 4; + +--connection slave +select * from t1 where a > 2 and a < 4; + +--echo #UPDATE with SELECT query +--connection master +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; + +--connection master + +--echo #entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; + +--connection slave +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc +select * from t1 order by a; +alter table t1 drop column z4; +start slave; + +--source include/wait_for_slave_sql_to_start.inc + +--connection master +--sync_slave_with_master +select * from t1 order by a; + +--connection master +select * from t1 order by a; +drop table t1; + +--source include/rpl_end.inc diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc index f82c5a3982a..e62f722675d 100644 --- a/sql/rpl_record.cc +++ b/sql/rpl_record.cc @@ -416,6 +416,13 @@ unpack_row(rpl_group_info *rgi, } } + /* + Add Extra slave persistent columns + */ + int error= 0; + if ((error= fill_extra_persistent_columns(table, cols->n_bits))) + DBUG_RETURN(error); + /* We should now have read all the null bytes, otherwise something is really wrong. @@ -489,5 +496,30 @@ int prepare_record(TABLE *const table, const uint skip, const bool check) DBUG_RETURN(0); } +/** + Fills @c table->record[0] with computed values of extra persistent column which are present on slave but not on master. + @param table Table whose record[0] buffer is prepared. + @param master_cols No of columns on master + @returns 0 on success + */ +int fill_extra_persistent_columns(TABLE *table, int master_cols) +{ + int error= 0; + Field **vfield_ptr, *vfield; + if (!table->vfield) + return 0; + for (vfield_ptr= table->vfield; *vfield_ptr; ++vfield_ptr) + { + vfield= *vfield_ptr; + if (vfield->field_index >= master_cols && vfield->stored_in_db) + { + /*Set bitmap for writing*/ + bitmap_set_bit(table->vcol_set, vfield->field_index); + error= vfield->vcol_info->expr_item->save_in_field(vfield,0); + bitmap_clear_bit(table->vcol_set, vfield->field_index); + } + } + return error; +} #endif // HAVE_REPLICATION diff --git a/sql/rpl_record.h b/sql/rpl_record.h index c10eb8225b0..be69716d9d5 100644 --- a/sql/rpl_record.h +++ b/sql/rpl_record.h @@ -38,6 +38,7 @@ int unpack_row(rpl_group_info *rgi, // Fill table's record[0] with default values. int prepare_record(TABLE *const table, const uint skip, const bool check); +int fill_extra_persistent_columns(TABLE *table, int master_cols); #endif #endif From d02a77bc5fbaf2e9504035e98f132fdf7145fd94 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 27 Dec 2016 14:13:32 +0530 Subject: [PATCH 03/13] MDEV-11636 Extra persistent columns on slave always gets NULL in RBR Problem:- In replication if slave has extra persistent column then these column are not computed while applying write-set from master. Solution:- While applying row events from server, we will generate values for extra persistent columns. --- .../rpl/r/rpl_alter_extra_persistent.result | 220 ++++++++++++++++++ .../rpl/t/rpl_alter_extra_persistent.test | 106 +++++++++ sql/rpl_record.cc | 31 +++ sql/rpl_record.h | 1 + 4 files changed, 358 insertions(+) create mode 100644 mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result create mode 100644 mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test diff --git a/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result new file mode 100644 index 00000000000..84915073d87 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result @@ -0,0 +1,220 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +connection slave; +select * from t1 order by a; +a +1 +2 +3 +4 +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +connection master; +insert into t1 values(5); +insert into t1 values(6); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#UPDATE query +connection master; +update t1 set a = a+10; +select * from t1 order by a; +a +11 +12 +13 +14 +15 +16 +connection slave; +select * from t1 order by a; +a z1 z2 +11 12 13 +12 13 14 +13 14 15 +14 15 16 +15 16 17 +16 17 18 +connection master; +update t1 set a = a-10; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#DELETE quert +connection master; +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +#REPLACE query +connection master; +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#SELECT query +connection master; +select * from t1 where a > 2 and a < 4; +a +3 +connection slave; +select * from t1 where a > 2 and a < 4; +a z1 z2 +3 4 5 +#UPDATE with SELECT query +connection master; +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +13 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +13 14 15 +connection master; +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; +connection master; +#entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +connection slave; +include/wait_for_slave_sql_error.inc [errno=1062] +connection slave; +connection slave; +select * from t1 order by a; +a z1 z2 z4 +1 2 3 1 +2 3 4 2 +3 4 5 3 +4 5 6 4 +5 6 7 5 +6 7 8 0 +alter table t1 drop column z4; +start slave; +include/wait_for_slave_sql_to_start.inc +connection slave; +connection slave; +connection master; +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +7 8 9 +connection master; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +drop table t1; +include/rpl_end.inc +connection server_2; +connection server_2; +connection server_2; +connection server_2; +connection server_1; +connection server_1; +connection server_1; +connection server_2; +connection server_1; +connection server_2; +connection server_2; +connection server_1; +connection server_1; diff --git a/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test new file mode 100644 index 00000000000..3b2fff1cb13 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test @@ -0,0 +1,106 @@ +--source include/master-slave.inc +--source include/have_binlog_format_row.inc + +--enable_connect_log +--connection master +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); + +--sync_slave_with_master +select * from t1 order by a; +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; + +--connection master +insert into t1 values(5); +insert into t1 values(6); + +--sync_slave_with_master +select * from t1 order by a; + + +--echo #UPDATE query + +--connection master +update t1 set a = a+10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a-10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #DELETE quert +--connection master +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #REPLACE query +--connection master +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); + +--sync_slave_with_master +select * from t1 order by a; + +--echo #SELECT query +--connection master +select * from t1 where a > 2 and a < 4; + +--connection slave +select * from t1 where a > 2 and a < 4; + +--echo #UPDATE with SELECT query +--connection master +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; + +--connection master + +--echo #entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; + +--connection slave +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc +select * from t1 order by a; +alter table t1 drop column z4; +start slave; + +--source include/wait_for_slave_sql_to_start.inc + +--connection master +--sync_slave_with_master +select * from t1 order by a; + +--connection master +select * from t1 order by a; +drop table t1; + +--source include/rpl_end.inc diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc index 183248ad1b8..5e48cfb02e5 100644 --- a/sql/rpl_record.cc +++ b/sql/rpl_record.cc @@ -389,6 +389,12 @@ unpack_row(rpl_group_info *rgi, } } + /* + Add Extra slave persistent columns + */ + if ((error= fill_extra_persistent_columns(table, cols->n_bits))) + DBUG_RETURN(error); + /* We should now have read all the null bytes, otherwise something is really wrong. @@ -461,5 +467,30 @@ int prepare_record(TABLE *const table, const uint skip, const bool check) DBUG_RETURN(0); } +/** + Fills @c table->record[0] with computed values of extra persistent column which are present on slave but not on master. + @param table Table whose record[0] buffer is prepared. + @param master_cols No of columns on master + @returns 0 on success + */ +int fill_extra_persistent_columns(TABLE *table, int master_cols) +{ + int error= 0; + Field **vfield_ptr, *vfield; + if (!table->vfield) + return 0; + for (vfield_ptr= table->vfield; *vfield_ptr; ++vfield_ptr) + { + vfield= *vfield_ptr; + if (vfield->field_index >= master_cols && vfield->stored_in_db) + { + /*Set bitmap for writing*/ + bitmap_set_bit(table->vcol_set, vfield->field_index); + error= vfield->vcol_info->expr_item->save_in_field(vfield,0); + bitmap_clear_bit(table->vcol_set, vfield->field_index); + } + } + return error; +} #endif // HAVE_REPLICATION diff --git a/sql/rpl_record.h b/sql/rpl_record.h index c10eb8225b0..be69716d9d5 100644 --- a/sql/rpl_record.h +++ b/sql/rpl_record.h @@ -38,6 +38,7 @@ int unpack_row(rpl_group_info *rgi, // Fill table's record[0] with default values. int prepare_record(TABLE *const table, const uint skip, const bool check); +int fill_extra_persistent_columns(TABLE *table, int master_cols); #endif #endif From 3871477c40efc826805f4c4e35b006c2c233dd26 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 1 Jan 2017 20:06:03 +0200 Subject: [PATCH 04/13] MDEV-10100 main.pool_of_threads fails sporadically in buildbot The patch fixes two test failures: - on slow builders, sometimes a connection attempt which should fail due to the exceeded number of thread_pool_max_threads actually succeeds; - on even slow builders, MTR sometimes cannot establish the initial connection, and check-testcase fails prior to the test start The problem with check-testcase was caused by connect-timeout=2 which was set for all clients in the test config file. On slow builders it might be not enough. There is no way to override it for the pre-test check, so it needed to be substantially increased or removed. The other problem was caused by a race condition between sleeps that the test performs in existing connections and the connect timeout for the connection attempt which was expected to fail. If sleeps finished before the connect-timeout was exceeded, it would allow the connection to succeed. To solve each problem without making the other one worse, connect-timeout should be configured dynamically during the test. Due to the nature of the test (all connections must be busy at the moment when we need to change the timeout, and cannot execute SET GLOBAL ...), it needs to be done independently from the server. The solution: - recognize 'connect_timeout' as a connection option in mysqltest's "connect" command; - remove connect-timeout from the test configuration file; - use the new connect_timeout option for those connections which are expected to fail; - re-arrange the test flow to allow running a huge SLEEP without affecting the test execution time (because it would be interrupted after the main test flow is finished). The test is still subject to false negatives, e.g. if the connection fails due to timeout rather than due to the exceeded number of allowed threads, or if the connection on extra port succeeds due to a race condition and not because the special logic for the extra port. But those false negatives have always been possible there on slow builders, they should not be critical because faster builders should catch such failures if they appear. --- client/mysqltest.cc | 12 +++++++ mysql-test/r/pool_of_threads.result | 21 ++++++------- mysql-test/t/pool_of_threads.cnf | 3 -- mysql-test/t/pool_of_threads.test | 49 +++++++++++++++-------------- 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index ed401b6e4a2..15db5759ac9 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -5945,6 +5945,7 @@ void do_connect(struct st_command *command) my_bool con_shm __attribute__ ((unused))= 0; int read_timeout= 0; int write_timeout= 0; + int connect_timeout= 0; struct st_connection* con_slot; static DYNAMIC_STRING ds_connection_name; @@ -6051,6 +6052,11 @@ void do_connect(struct st_command *command) { write_timeout= atoi(con_options + sizeof("write_timeout=")-1); } + else if (strncasecmp(con_options, "connect_timeout=", + sizeof("connect_timeout=")-1) == 0) + { + connect_timeout= atoi(con_options + sizeof("connect_timeout=")-1); + } else die("Illegal option to connect: %.*s", (int) (end - con_options), con_options); @@ -6135,6 +6141,12 @@ void do_connect(struct st_command *command) (char*)&write_timeout); } + if (connect_timeout) + { + mysql_options(con_slot->mysql, MYSQL_OPT_CONNECT_TIMEOUT, + (char*)&connect_timeout); + } + #ifdef HAVE_SMEM if (con_shm) { diff --git a/mysql-test/r/pool_of_threads.result b/mysql-test/r/pool_of_threads.result index 9611d7ff43b..ddc451516e2 100644 --- a/mysql-test/r/pool_of_threads.result +++ b/mysql-test/r/pool_of_threads.result @@ -2157,23 +2157,22 @@ Warnings: Warning 1052 Column 'kundentyp' in group statement is ambiguous drop table t1; SET optimizer_switch=@save_optimizer_switch; -SELECT sleep(5.5); -SELECT sleep(5); +SELECT sleep(50); +SELECT sleep(50); # -- Success: more than --thread_pool_max_threads normal connections not possible -sleep(5.5) -0 -sleep(5) -0 -SELECT sleep(5); -SELECT sleep(5); SELECT 'Connection on extra port ok'; Connection on extra port ok Connection on extra port ok +SELECT sleep(5.5); SELECT 'Connection on extra port 2 ok'; Connection on extra port 2 ok Connection on extra port 2 ok # -- Success: more than --extra-max-connections + 1 normal connections not possible -sleep(5) -0 -sleep(5) +KILL QUERY ; +KILL QUERY ; +sleep(50) +1 +sleep(50) +1 +sleep(5.5) 0 diff --git a/mysql-test/t/pool_of_threads.cnf b/mysql-test/t/pool_of_threads.cnf index c03e1da6450..f6651c878de 100644 --- a/mysql-test/t/pool_of_threads.cnf +++ b/mysql-test/t/pool_of_threads.cnf @@ -7,8 +7,5 @@ loose-thread_pool_max_threads= 2 extra-port= @ENV.MASTER_EXTRA_PORT extra-max-connections=1 -[client] -connect-timeout= 2 - [ENV] MASTER_EXTRA_PORT= @OPT.port diff --git a/mysql-test/t/pool_of_threads.test b/mysql-test/t/pool_of_threads.test index 24e0218db62..f13a096985c 100644 --- a/mysql-test/t/pool_of_threads.test +++ b/mysql-test/t/pool_of_threads.test @@ -15,23 +15,26 @@ SET optimizer_switch=@save_optimizer_switch; # connections on the extra port. # First set two connections running, and check that extra connection -# on normal port fails due to--thread-pool-max_threads=2 -connection default; +# on normal port fails due to --thread-pool-max-threads=2. +# We can afford using a really long sleep, because we won't wait +# till it ends, we'll interrupt it as soon as we don't need it anymore -# Sleep for slightly longer than 5 sec to trigger MDEV-4566 -# (abort in interruptible wait connection check) -send SELECT sleep(5.5); +connection default; +--let $con1_id= `SELECT CONNECTION_ID()` + +send SELECT sleep(50); --sleep 1 connect(con2,localhost,root,,); -connection con2; -send SELECT sleep(5); +--let $con2_id= `SELECT CONNECTION_ID()` + +send SELECT sleep(50); --sleep 0.5 --disable_abort_on_error --disable_result_log --disable_query_log -connect(con3,localhost,root,,); +connect(con3,localhost,root,,,,,connect_timeout=2); --enable_query_log --enable_result_log --enable_abort_on_error @@ -45,24 +48,15 @@ if ($error) --echo # -- Success: more than --thread_pool_max_threads normal connections not possible } -connection default; ---reap -connection con2; ---reap - -# Now try again, but this time use the extra port to successfully connect. - -connection default; -send SELECT sleep(5); - -connection con2; -send SELECT sleep(5); ---sleep 1 - connect(extracon,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); connection extracon; SELECT 'Connection on extra port ok'; +# Here, sleep just for slightly longer than 5 sec to trigger MDEV-4566 +# (abort in interruptible wait connection check). +send SELECT sleep(5.5); + + connect(extracon2,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); connection extracon2; SELECT 'Connection on extra port 2 ok'; @@ -70,7 +64,7 @@ SELECT 'Connection on extra port 2 ok'; --disable_abort_on_error --disable_result_log --disable_query_log -connect(extracon3,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); +connect(extracon3,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,,connect_timeout=2); --enable_query_log --enable_result_log --enable_abort_on_error @@ -84,7 +78,16 @@ if ($error) --echo # -- Success: more than --extra-max-connections + 1 normal connections not possible } +connection extracon2; +--replace_result $con1_id +eval KILL QUERY $con1_id; +--replace_result $con2_id +eval KILL QUERY $con2_id; + connection default; --reap connection con2; --reap + +connection extracon; +--reap From d9a1a201aae87a655cdf3e5a344b2265912a94a7 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 3 Jan 2017 10:10:58 +0530 Subject: [PATCH 05/13] MDEV-11016 wsrep_node_is_ready() check is too strict Problem:- The condition that checks for node readiness is too strict as it does not allow SELECTs even if these selects do not access any tables. For example,if we run SELECT 1; OR SELECT @@max_allowed_packet; Solution:- We need not to report this error when all_tables(lex->query_tables) is NULL: --- mysql-test/suite/galera/r/MW-284.result | 2 ++ mysql-test/suite/galera/r/galera_var_dirty_reads.result | 9 +++++++++ mysql-test/suite/galera/t/MW-284.test | 5 +++++ mysql-test/suite/galera/t/galera_var_dirty_reads.test | 5 +++++ sql/sql_parse.cc | 5 ++++- 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/r/MW-284.result b/mysql-test/suite/galera/r/MW-284.result index 3ff131674ea..9465370c25b 100644 --- a/mysql-test/suite/galera/r/MW-284.result +++ b/mysql-test/suite/galera/r/MW-284.result @@ -2,12 +2,14 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; SET SESSION wsrep_on = OFF; SET SESSION wsrep_on = ON; +SET global wsrep_sync_wait=0; START SLAVE; include/wait_for_slave_param.inc [Slave_IO_Running] SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; include/wait_for_slave_to_start.inc INSERT INTO t1 VALUES (1); DROP TABLE t1; +SET global wsrep_sync_wait=7; STOP SLAVE; RESET SLAVE ALL; CALL mtr.add_suppression('failed registering on master'); diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 8a3175912c7..52729395b19 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,6 +18,15 @@ Variable_name Value wsrep_cluster_status non-Primary SELECT * FROM t1; ERROR 08S01: WSREP has not yet prepared node for application use +SELECT @@wsrep_dirty_reads; +@@wsrep_dirty_reads +0 +SELECT 2; +2 +2 +SELECT 2+2 FROM DUAL; +2+2 +4 SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i diff --git a/mysql-test/suite/galera/t/MW-284.test b/mysql-test/suite/galera/t/MW-284.test index f3ce1b0dc91..7add82f1227 100644 --- a/mysql-test/suite/galera/t/MW-284.test +++ b/mysql-test/suite/galera/t/MW-284.test @@ -11,12 +11,16 @@ --enable_query_log --connection node_1 +--let $wsrep_sync_wait_state= `SELECT @@global.wsrep_sync_wait;` CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; SET SESSION wsrep_on = OFF; --let $wait_condition = SELECT VARIABLE_VALUE = 'non-Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status' --source include/wait_condition.inc SET SESSION wsrep_on = ON; +#wsrep_sync_wait is set to zero because when slave tries to connect it it ask for queries like SELECT UNIX_TIMESTAMP() on node 1 which will fail, causing +#a warning in slave error log. +SET global wsrep_sync_wait=0; --connection node_3 START SLAVE; @@ -47,6 +51,7 @@ INSERT INTO t1 VALUES (1); --connection node_1 DROP TABLE t1; +--eval SET global wsrep_sync_wait=$wsrep_sync_wait_state --connection node_3 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' --source include/wait_condition.inc diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index bcdb1574a3d..1fd30c69a7b 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -37,6 +37,11 @@ SHOW STATUS LIKE 'wsrep_cluster_status'; --error ER_UNKNOWN_COM_ERROR SELECT * FROM t1; +#Select query which does not access table should be allowed MDEV-11016 +SELECT @@wsrep_dirty_reads; +SELECT 2; +SELECT 2+2 FROM DUAL; + SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index b214a1759be..5717c3dd6c4 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -288,6 +288,7 @@ void init_update_queries(void) server_command_flags[COM_STMT_RESET]= CF_SKIP_QUESTIONS | CF_SKIP_WSREP_CHECK; server_command_flags[COM_STMT_EXECUTE]= CF_SKIP_WSREP_CHECK; server_command_flags[COM_STMT_SEND_LONG_DATA]= CF_SKIP_WSREP_CHECK; + server_command_flags[COM_REGISTER_SLAVE]= CF_SKIP_WSREP_CHECK; /* Initialize the sql command flags array. */ memset(sql_command_flags, 0, sizeof(sql_command_flags)); @@ -2651,7 +2652,7 @@ mysql_execute_command(THD *thd) /* Bail out if DB snapshot has not been installed. SET and SHOW commands, however, are always allowed. - + Select query is also allowed if it does not access any table. We additionally allow all other commands that do not change data in case wsrep_dirty_reads is enabled. */ @@ -2659,6 +2660,8 @@ mysql_execute_command(THD *thd) !wsrep_is_show_query(lex->sql_command) && !(thd->variables.wsrep_dirty_reads && !is_update_query(lex->sql_command)) && + !(lex->sql_command == SQLCOM_SELECT && + !all_tables) && !wsrep_node_is_ready(thd)) goto error; } From b4616c40be00c5c8a2a73d537d676d8ddb7c84cf Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 3 Jan 2017 10:45:55 +0530 Subject: [PATCH 06/13] MDEV-7955 WSREP() appears on radar in OLTP RO This commit is for optimizing WSREP(thd) macro. #define WSREP(thd) \ (WSREP_ON && wsrep && (thd && thd->variables.wsrep_on)) In this we can safely remove wsrep and thd. We are not removing WSREP_ON because this will change WSREP(thd) behaviour. Patch Credit:- Nirbhay Choubay, Sergey Vojtovich --- sql/sql_parse.cc | 2 +- sql/wsrep_mysqld.h | 2 +- storage/innobase/handler/ha_innodb.cc | 2 +- storage/innobase/trx/trx0trx.cc | 2 +- storage/xtradb/handler/ha_innodb.cc | 2 +- storage/xtradb/trx/trx0trx.cc | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 5717c3dd6c4..4f630b68264 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2624,7 +2624,7 @@ mysql_execute_command(THD *thd) } /* endif unlikely slave */ #endif #ifdef WITH_WSREP - if (WSREP(thd)) + if (wsrep && WSREP(thd)) { /* change LOCK TABLE WRITE to transaction diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index cf549ffe544..fc4b2a452fe 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -179,7 +179,7 @@ extern wsrep_seqno_t wsrep_locked_seqno; strcmp(wsrep_provider, WSREP_NONE)) #define WSREP(thd) \ - (WSREP_ON && wsrep && (thd && thd->variables.wsrep_on)) + (WSREP_ON && thd->variables.wsrep_on) #define WSREP_CLIENT(thd) \ (WSREP(thd) && thd->wsrep_client_thread) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3713dd959d4..b283bbda7f4 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4103,7 +4103,7 @@ innobase_commit_low( #ifdef WITH_WSREP THD* thd = (THD*)trx->mysql_thd; const char* tmp = 0; - if (wsrep_on(thd)) { + if (thd && wsrep_on(thd)) { #ifdef WSREP_PROC_INFO char info[64]; info[sizeof(info) - 1] = '\0'; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index cde32448769..11025b4c21c 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1349,7 +1349,7 @@ trx_commit_in_memory( ut_ad(!trx->in_rw_trx_list); #ifdef WITH_WSREP - if (wsrep_on(trx->mysql_thd)) { + if (trx->mysql_thd && wsrep_on(trx->mysql_thd)) { trx->lock.was_chosen_as_deadlock_victim = FALSE; } #endif diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 8d564df2bb3..8f51b931038 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -4742,7 +4742,7 @@ innobase_commit_low( #ifdef WITH_WSREP THD* thd = (THD*)trx->mysql_thd; const char* tmp = 0; - if (wsrep_on(thd)) { + if (thd && wsrep_on(thd)) { #ifdef WSREP_PROC_INFO char info[64]; info[sizeof(info) - 1] = '\0'; diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index ec57a8e5c54..bdf407ff7fb 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -1572,7 +1572,7 @@ trx_commit_in_memory( ut_ad(!trx->in_rw_trx_list); #ifdef WITH_WSREP - if (wsrep_on(trx->mysql_thd)) { + if (trx->mysql_thd && wsrep_on(trx->mysql_thd)) { trx->lock.was_chosen_as_deadlock_victim = FALSE; } #endif From fc779252aec56e01c5924d80378cafc8b6ec80b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 13:18:47 +0200 Subject: [PATCH 07/13] MDEV-11688 fil_crypt_threads_end() tries to create threads after aborted InnoDB startup This bug was repeatable by starting MariaDB 10.2 with an invalid option, such as --innodb-flush-method=foo. It is not repeatable in MariaDB 10.1 in the same way, but the problem exists already there. --- storage/innobase/fil/fil0crypt.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 6483cdc5a53..6482956eb0a 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2465,8 +2465,9 @@ void fil_crypt_threads_end() /*===================*/ { - /* stop threads */ - fil_crypt_set_thread_cnt(0); + if (fil_crypt_threads_inited) { + fil_crypt_set_thread_cnt(0); + } } /********************************************************************* From ba8198a34cf651cce8a1f59f61ba7f31dbe41579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 15:44:44 +0200 Subject: [PATCH 08/13] Post-fix for MDEV-11688 fil_crypt_threads_end() tries to create threads fil_crypt_threads_cleanup(): Do nothing if nothing was initialized. --- storage/innobase/fil/fil0crypt.cc | 3 +++ storage/xtradb/fil/fil0crypt.cc | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 6482956eb0a..1eaabf68678 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2477,6 +2477,9 @@ void fil_crypt_threads_cleanup() /*=======================*/ { + if (!fil_crypt_threads_inited) { + return; + } os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 6483cdc5a53..1eaabf68678 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -2465,8 +2465,9 @@ void fil_crypt_threads_end() /*===================*/ { - /* stop threads */ - fil_crypt_set_thread_cnt(0); + if (fil_crypt_threads_inited) { + fil_crypt_set_thread_cnt(0); + } } /********************************************************************* @@ -2476,6 +2477,9 @@ void fil_crypt_threads_cleanup() /*=======================*/ { + if (!fil_crypt_threads_inited) { + return; + } os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); From 80d5d1452a4a6b7bd1627116f2a5a950003fc3cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 19:32:47 +0200 Subject: [PATCH 09/13] MDEV-11694 InnoDB tries to create unused table SYS_ZIP_DICT MariaDB Server 10.0.28 and 10.1.19 merged code from Percona XtraDB that introduced support for compressed columns. Much but not all of this code was disabled by placing #ifdef HAVE_PERCONA_COMPRESSED_COLUMNS around it. Among the unused but not disabled code is code to access some new system tables related to compressed columns. The creation of these system tables SYS_ZIP_DICT and SYS_ZIP_DICT_COLS would cause a crash in --innodb-read-only mode when upgrading from an earlier version to 10.0.28 or 10.1.19. Let us remove all the dead code related to compressed columns. Users who already upgraded to 10.0.28 and 10.1.19 will have the two above mentioned empty tables in their InnoDB system tablespace. Subsequent versions of MariaDB Server will completely ignore those tables. --- storage/xtradb/dict/dict0boot.cc | 5 +- storage/xtradb/dict/dict0crea.cc | 583 +----------------------- storage/xtradb/dict/dict0dict.cc | 160 +------ storage/xtradb/dict/dict0load.cc | 160 +------ storage/xtradb/handler/ha_innodb.cc | 415 +---------------- storage/xtradb/handler/ha_innodb.h | 38 +- storage/xtradb/handler/handler0alter.cc | 62 +-- storage/xtradb/handler/xtradb_i_s.cc | 330 +------------- storage/xtradb/handler/xtradb_i_s.h | 3 +- storage/xtradb/include/data0type.h | 15 +- storage/xtradb/include/data0type.ic | 16 +- storage/xtradb/include/dict0boot.h | 33 +- storage/xtradb/include/dict0boot.ic | 3 +- storage/xtradb/include/dict0crea.h | 90 +--- storage/xtradb/include/dict0dict.h | 48 +- storage/xtradb/include/dict0load.h | 29 +- storage/xtradb/include/row0mysql.h | 85 +--- storage/xtradb/rem/rem0rec.cc | 18 +- storage/xtradb/row/row0merge.cc | 7 +- storage/xtradb/row/row0mysql.cc | 583 +----------------------- storage/xtradb/row/row0sel.cc | 39 +- storage/xtradb/srv/srv0start.cc | 7 +- 22 files changed, 60 insertions(+), 2669 deletions(-) diff --git a/storage/xtradb/dict/dict0boot.cc b/storage/xtradb/dict/dict0boot.cc index c0bb0298bea..20ab5d75205 100644 --- a/storage/xtradb/dict/dict0boot.cc +++ b/storage/xtradb/dict/dict0boot.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -272,10 +273,6 @@ dict_boot(void) ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_FOR_NAME == 2); ut_ad(DICT_NUM_COLS__SYS_FOREIGN_COLS == 4); ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_COLS == 6); - ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT == 3); - ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT == 5); - ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT_COLS == 3); - ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS == 5); mtr_start(&mtr); diff --git a/storage/xtradb/dict/dict0crea.cc b/storage/xtradb/dict/dict0crea.cc index db0ca638de4..9a4040421d7 100644 --- a/storage/xtradb/dict/dict0crea.cc +++ b/storage/xtradb/dict/dict0crea.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1932,135 +1933,6 @@ dict_create_or_check_sys_tablespace(void) return(err); } -/** Creates the zip_dict system table inside InnoDB -at server bootstrap or server start if it is not found or is -not of the right form. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -dict_create_or_check_sys_zip_dict(void) -{ - trx_t* trx; - my_bool srv_file_per_table_backup; - dberr_t err; - dberr_t sys_zip_dict_err; - dberr_t sys_zip_dict_cols_err; - - ut_a(srv_get_active_thread_type() == SRV_NONE); - - /* Note: The master thread has not been started at this point. */ - - sys_zip_dict_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); - sys_zip_dict_cols_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT_COLS", DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, - 1); - - if (sys_zip_dict_err == DB_SUCCESS && - sys_zip_dict_cols_err == DB_SUCCESS) - return (DB_SUCCESS); - - trx = trx_allocate_for_mysql(); - - trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); - - trx->op_info = "creating zip_dict and zip_dict_cols sys tables"; - - row_mysql_lock_data_dictionary(trx); - - /* Check which incomplete table definition to drop. */ - - if (sys_zip_dict_err == DB_CORRUPTION) { - ib_logf(IB_LOG_LEVEL_WARN, - "Dropping incompletely created " - "SYS_ZIP_DICT table."); - row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE, TRUE); - } - if (sys_zip_dict_cols_err == DB_CORRUPTION) { - ib_logf(IB_LOG_LEVEL_WARN, - "Dropping incompletely created " - "SYS_ZIP_DICT_COLS table."); - row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE, TRUE); - } - - ib_logf(IB_LOG_LEVEL_INFO, - "Creating zip_dict and zip_dict_cols system tables."); - - /* We always want SYSTEM tables to be created inside the system - tablespace. */ - srv_file_per_table_backup = srv_file_per_table; - srv_file_per_table = 0; - - err = que_eval_sql( - NULL, - "PROCEDURE CREATE_SYS_ZIP_DICT_PROC () IS\n" - "BEGIN\n" - "CREATE TABLE SYS_ZIP_DICT(\n" - " ID INT UNSIGNED NOT NULL,\n" - " NAME CHAR(" - STRINGIFY_ARG(ZIP_DICT_MAX_NAME_LENGTH) - ") NOT NULL,\n" - " DATA BLOB NOT NULL\n" - ");\n" - "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_ID" - " ON SYS_ZIP_DICT (ID);\n" - "CREATE UNIQUE INDEX SYS_ZIP_DICT_NAME" - " ON SYS_ZIP_DICT (NAME);\n" - "CREATE TABLE SYS_ZIP_DICT_COLS(\n" - " TABLE_ID INT UNSIGNED NOT NULL,\n" - " COLUMN_POS INT UNSIGNED NOT NULL,\n" - " DICT_ID INT UNSIGNED NOT NULL\n" - ");\n" - "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_COLS_COMPOSITE" - " ON SYS_ZIP_DICT_COLS (TABLE_ID, COLUMN_POS);\n" - "END;\n", - FALSE, trx); - - if (err != DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Creation of SYS_ZIP_DICT and SYS_ZIP_DICT_COLS" - "has failed with error %lu. Tablespace is full. " - "Dropping incompletely created tables.", - (ulong) err); - - ut_a(err == DB_OUT_OF_FILE_SPACE - || err == DB_TOO_MANY_CONCURRENT_TRXS); - - row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE, TRUE); - row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE, TRUE); - - if (err == DB_OUT_OF_FILE_SPACE) { - err = DB_MUST_GET_MORE_FILE_SPACE; - } - } - - trx_commit_for_mysql(trx); - - row_mysql_unlock_data_dictionary(trx); - - trx_free_for_mysql(trx); - - srv_file_per_table = srv_file_per_table_backup; - - if (err == DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_INFO, - "zip_dict and zip_dict_cols system tables created."); - } - - /* Note: The master thread has not been started at this point. */ - /* Confirm and move to the non-LRU part of the table LRU list. */ - - sys_zip_dict_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); - ut_a(sys_zip_dict_err == DB_SUCCESS); - sys_zip_dict_cols_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT_COLS", - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, 1); - ut_a(sys_zip_dict_cols_err == DB_SUCCESS); - - return(err); -} - /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -2114,456 +1986,3 @@ dict_create_add_tablespace_to_dictionary( return(error); } - -/** Add a single compression dictionary definition to the SYS_ZIP_DICT -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - ut_ad(data); - - pars_info_t* info = pars_info_create(); - - pars_info_add_literal(info, "name", name, name_len, - DATA_VARCHAR, DATA_ENGLISH); - pars_info_add_literal(info, "data", data, data_len, - DATA_BLOB, DATA_BINARY_TYPE | DATA_NOT_NULL); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - " max_id INT;\n" - "DECLARE CURSOR cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " ORDER BY ID DESC;\n" - "BEGIN\n" - " max_id := 0;\n" - " OPEN cur;\n" - " FETCH cur INTO max_id;\n" - " IF (cur % NOTFOUND) THEN\n" - " max_id := 0;\n" - " END IF;\n" - " CLOSE cur;\n" - " INSERT INTO SYS_ZIP_DICT VALUES" - " (max_id + 1, :name, :data);\n" - "END;\n", - FALSE, trx); - - return error; -} - -/** Fetch callback, just stores extracted zip_dict id in the external -variable. -@return TRUE if all OK */ -static -ibool -dict_create_extract_int_aux( - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: int32 id */ -{ - sel_node_t* node = static_cast(row); - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - ut_a(len == sizeof(ib_uint32_t)); - - memcpy(user_arg, dfield_get_data(dfield), sizeof(ib_uint32_t)); - - return(TRUE); -} - -/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "table_id", table_id); - pars_info_add_int4_literal(info, "column_pos", column_pos); - pars_info_add_int4_literal(info, "dict_id", dict_id); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "BEGIN\n" - " INSERT INTO SYS_ZIP_DICT_COLS VALUES" - " (:table_id, :column_pos, :dict_id);\n" - "END;\n", - FALSE, trx); - return error; -} - -/** Get a single compression dictionary id for the given -(table id, column pos) pair. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(dict_id); - - pars_info_t* info = pars_info_create(); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast(&dict_id_buf ), - ULINT32_UNDEFINED); - - pars_info_add_int4_literal(info, "table_id", table_id); - pars_info_add_int4_literal(info, "column_pos", column_pos); - pars_info_bind_function( - info, "my_func", dict_create_extract_int_aux, &dict_id_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT DICT_ID FROM SYS_ZIP_DICT_COLS\n" - " WHERE TABLE_ID = :table_id AND\n" - " COLUMN_POS = :column_pos;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) - error = DB_RECORD_NOT_FOUND; - else - *dict_id = local_dict_id; - } - return error; -} - -/** Get compression dictionary id for the given name. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_name( - const char* dict_name, /*!< in: dict name */ - ulint dict_name_len, /*!< in: dict name length */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(dict_name); - ut_ad(dict_name_len); - ut_ad(dict_id); - - pars_info_t* info = pars_info_create(); - - pars_info_add_literal(info, "dict_name", dict_name, dict_name_len, - DATA_VARCHAR, DATA_ENGLISH); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast(&dict_id_buf), - ULINT32_UNDEFINED); - pars_info_bind_function( - info, "my_func", dict_create_extract_int_aux, &dict_id_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " WHERE NAME = :dict_name;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) - error = DB_RECORD_NOT_FOUND; - else - *dict_id = local_dict_id; - } - return error; -} - -/** Auxiliary enum used to indicate zip dict data extraction result code */ -enum zip_dict_info_aux_code { - zip_dict_info_success, /*!< success */ - zip_dict_info_not_found, /*!< zip dict record not found */ - zip_dict_info_oom, /*!< out of memory */ - zip_dict_info_corrupted_name, /*!< corrupted zip dict name */ - zip_dict_info_corrupted_data /*!< corrupted zip dict data */ -}; - -/** Auxiliary struct used to return zip dict info aling with result code */ -struct zip_dict_info_aux { - LEX_STRING name; /*!< zip dict name */ - LEX_STRING data; /*!< zip dict data */ - int code; /*!< result code (0 - success) */ -}; - -/** Fetch callback, just stores extracted zip_dict data in the external -variable. -@return always returns TRUE */ -static -ibool -dict_create_get_zip_dict_info_by_id_aux( - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to zip_dict_info_aux* */ -{ - sel_node_t* node = static_cast(row); - zip_dict_info_aux* result = - static_cast(user_arg); - - result->code = zip_dict_info_success; - result->name.str = 0; - result->name.length = 0; - result->data.str = 0; - result->data.length = 0; - - /* NAME field */ - que_node_t* exp = node->select_list; - ut_a(exp != 0); - - dfield_t* dfield = que_node_get_val(exp); - dtype_t* type = dfield_get_type(dfield); - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - - ulint len = dfield_get_len(dfield); - void* data = dfield_get_data(dfield); - - - if (len == UNIV_SQL_NULL) { - result->code = zip_dict_info_corrupted_name; - } - else { - result->name.str = - static_cast(my_malloc(len + 1, MYF(0))); - if (result->name.str == 0) { - result->code = zip_dict_info_oom; - } - else { - memcpy(result->name.str, data, len); - result->name.str[len] = '\0'; - result->name.length = len; - } - } - - /* DATA field */ - exp = que_node_get_next(exp); - ut_a(exp != 0); - - dfield = que_node_get_val(exp); - type = dfield_get_type(dfield); - ut_a(dtype_get_mtype(type) == DATA_BLOB); - - len = dfield_get_len(dfield); - data = dfield_get_data(dfield); - - if (len == UNIV_SQL_NULL) { - result->code = zip_dict_info_corrupted_data; - } - else { - result->data.str = - static_cast(my_malloc( - len == 0 ? 1 : len, MYF(0))); - if (result->data.str == 0) { - result->code = zip_dict_info_oom; - } - else { - memcpy(result->data.str, data, len); - result->data.length = len; - } - } - - ut_ad(que_node_get_next(exp) == 0); - - if (result->code != zip_dict_info_success) { - if (result->name.str == 0) { - mem_free(result->name.str); - result->name.str = 0; - result->name.length = 0; - } - if (result->data.str == 0) { - mem_free(result->data.str); - result->data.str = 0; - result->data.length = 0; - } - } - - return TRUE; -} - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory for name and data on success. -Must be freed with mem_free(). -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_info_by_id( - ulint dict_id, /*!< in: dict id */ - char** name, /*!< out: dict name */ - ulint* name_len, /*!< out: dict name length*/ - char** data, /*!< out: dict data */ - ulint* data_len, /*!< out: dict data length*/ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - ut_ad(data); - - zip_dict_info_aux rec; - rec.code = zip_dict_info_not_found; - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "id", dict_id); - pars_info_bind_function( - info, "my_func", dict_create_get_zip_dict_info_by_id_aux, - &rec); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT NAME, DATA FROM SYS_ZIP_DICT\n" - " WHERE ID = :id;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - switch (rec.code) { - case zip_dict_info_success: - *name = rec.name.str; - *name_len = rec.name.length; - *data = rec.data.str; - *data_len = rec.data.length; - break; - case zip_dict_info_not_found: - error = DB_RECORD_NOT_FOUND; - break; - case zip_dict_info_oom: - error = DB_OUT_OF_MEMORY; - break; - case zip_dict_info_corrupted_name: - case zip_dict_info_corrupted_data: - error = DB_INVALID_NULL; - break; - default: - ut_error; - } - } - return error; -} - -/** Remove a single compression dictionary from the data dictionary -tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - - pars_info_t* info = pars_info_create(); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast(&dict_id_buf), - ULINT32_UNDEFINED); - ib_uint32_t counter_buf; - mach_write_to_4(reinterpret_cast(&counter_buf), - ULINT32_UNDEFINED); - - pars_info_add_literal(info, "name", name, name_len, - DATA_VARCHAR, DATA_ENGLISH); - pars_info_bind_int4_literal(info, "dict_id", &dict_id_buf); - pars_info_bind_function(info, "find_dict_func", - dict_create_extract_int_aux, &dict_id_buf); - pars_info_bind_function(info, "count_func", - dict_create_extract_int_aux, &counter_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION find_dict_func;\n" - "DECLARE FUNCTION count_func;\n" - "DECLARE CURSOR dict_cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " WHERE NAME = :name\n" - " FOR UPDATE;\n" - "DECLARE CURSOR ref_cur IS\n" - " SELECT 1 FROM SYS_ZIP_DICT_COLS\n" - " WHERE DICT_ID = :dict_id;\n" - "BEGIN\n" - " OPEN dict_cur;\n" - " FETCH dict_cur INTO find_dict_func();\n" - " IF NOT (SQL % NOTFOUND) THEN\n" - " OPEN ref_cur;\n" - " FETCH ref_cur INTO count_func();\n" - " IF SQL % NOTFOUND THEN\n" - " DELETE FROM SYS_ZIP_DICT WHERE CURRENT OF dict_cur;\n" - " END IF;\n" - " CLOSE ref_cur;\n" - " END IF;\n" - " CLOSE dict_cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) { - error = DB_RECORD_NOT_FOUND; - } - else { - ib_uint32_t local_counter = mach_read_from_4( - reinterpret_cast(&counter_buf)); - if (local_counter != ULINT32_UNDEFINED) - error = DB_ROW_IS_REFERENCED; - } - } - return error; -} - -/** Remove all compression dictionary references for the given table ID from -the data dictionary tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict_references_for_table( - ulint table_id, /*!< in: table id */ - trx_t* trx) /*!< in/out: transaction */ -{ - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "table_id", table_id); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "BEGIN\n" - " DELETE FROM SYS_ZIP_DICT_COLS\n" - " WHERE TABLE_ID = :table_id;\n" - "END;\n", - FALSE, trx); - return error; -} diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 3845138c8bf..339e5615bea 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -7288,161 +7288,3 @@ dict_tf_to_row_format_string( return(0); } #endif /* !UNIV_HOTBACKUP */ - -/** Insert a records into SYS_ZIP_DICT. -@retval DB_SUCCESS if OK -@retval dberr_t if the insert failed */ -UNIV_INTERN -dberr_t -dict_create_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len, /*!< in: zip_dict name length*/ - const char* data, /*!< in: zip_dict data */ - ulint data_len) /*!< in: zip_dict data length */ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - ut_ad(name); - ut_ad(data); - - rw_lock_x_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "insert zip_dict"; - trx->dict_operation_lock_mode = RW_X_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_add_zip_dict(name, name_len, data, data_len, trx); - - if (err == DB_SUCCESS) { - trx_commit_for_mysql(trx); - } - else { - trx->op_info = "rollback of internal trx on zip_dict table"; - trx_rollback_to_savepoint(trx, NULL); - ut_a(trx->error_state == DB_SUCCESS); - } - trx->op_info = ""; - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_x_unlock(&dict_operation_lock); - - return err; -} -/** Get single compression dictionary id for the given -(table id, column pos) pair. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_id_by_key( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id) /*!< out: zip_dict id */ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - rw_lock_s_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "get zip dict id by composite key"; - trx->dict_operation_lock_mode = RW_S_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_get_zip_dict_id_by_reference(table_id, column_pos, - dict_id, trx); - - trx_commit_for_mysql(trx); - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_s_unlock(&dict_operation_lock); - - return err; -} -/** Get compression dictionary info (name and data) for the given id. -Allocates memory in name->str and data->str on success. -Must be freed with mem_free(). -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_info_by_id( - ulint dict_id, /*!< in: table name */ - char** name, /*!< out: dictionary name */ - ulint* name_len, /*!< out: dictionary name length*/ - char** data, /*!< out: dictionary data */ - ulint* data_len) /*!< out: dictionary data length*/ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - rw_lock_s_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "get zip dict name and data by id"; - trx->dict_operation_lock_mode = RW_S_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_get_zip_dict_info_by_id(dict_id, name, name_len, - data, data_len, trx); - - trx_commit_for_mysql(trx); - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_s_unlock(&dict_operation_lock); - - return err; -} -/** Delete a record in SYS_ZIP_DICT with the given name. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found -@retval DB_ROW_IS_REFERENCED if in use */ -UNIV_INTERN -dberr_t -dict_drop_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len) /*!< in: zip_dict name length*/ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - ut_ad(name); - - rw_lock_x_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "delete zip_dict"; - trx->dict_operation_lock_mode = RW_X_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_remove_zip_dict(name, name_len, trx); - - if (err == DB_SUCCESS) { - trx_commit_for_mysql(trx); - } - else { - trx->op_info = "rollback of internal trx on zip_dict table"; - trx_rollback_to_savepoint(trx, NULL); - ut_a(trx->error_state == DB_SUCCESS); - } - trx->op_info = ""; - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_x_unlock(&dict_operation_lock); - - return err; -} diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 9ea4f4d873a..6cbd0a3d488 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,9 +57,7 @@ static const char* SYSTEM_TABLE_NAME[] = { "SYS_FOREIGN", "SYS_FOREIGN_COLS", "SYS_TABLESPACES", - "SYS_DATAFILES", - "SYS_ZIP_DICT", - "SYS_ZIP_DICT_COLS" + "SYS_DATAFILES" }; /* If this flag is TRUE, then we will load the cluster index's (and tables') @@ -730,161 +729,6 @@ err_len: return(NULL); } -/** This function parses a SYS_ZIP_DICT record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict( - mem_heap_t* heap, /*!< in/out: heap memory */ - ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* id, /*!< out: dict id */ - const char** name, /*!< out: dict name */ - const char** data, /*!< out: dict data */ - ulint* data_len) /*!< out: dict data length */ -{ - ulint len; - const byte* field; - - /* Initialize the output values */ - *id = ULINT_UNDEFINED; - *name = NULL; - *data = NULL; - *data_len = 0; - - if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { - return("delete-marked record in SYS_ZIP_DICT"); - } - - if (UNIV_UNLIKELY( - rec_get_n_fields_old(rec)!= DICT_NUM_FIELDS__SYS_ZIP_DICT)) { - return("wrong number of columns in SYS_ZIP_DICT record"); - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *id = mach_read_from_4(field); - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID, &len); - if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR, &len); - if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__NAME, &len); - if (UNIV_UNLIKELY(len == 0 || len == UNIV_SQL_NULL)) { - goto err_len; - } - *name = mem_heap_strdupl(heap, (char*) field, len); - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__DATA, &len); - if (UNIV_UNLIKELY(len == UNIV_SQL_NULL)) { - goto err_len; - } - - if (rec_get_1byte_offs_flag(rec) == 0 && - rec_2_is_field_extern(rec, DICT_FLD__SYS_ZIP_DICT__DATA)) { - ut_a(len >= BTR_EXTERN_FIELD_REF_SIZE); - - if (UNIV_UNLIKELY - (!memcmp(field + len - BTR_EXTERN_FIELD_REF_SIZE, - field_ref_zero, - BTR_EXTERN_FIELD_REF_SIZE))) { - goto err_len; - } - *data = reinterpret_cast( - btr_copy_externally_stored_field(data_len, field, - zip_size, len, heap, 0)); - } - else { - *data_len = len; - *data = static_cast(mem_heap_dup(heap, field, len)); - } - - return(NULL); - -err_len: - return("incorrect column length in SYS_ZIP_DICT"); -} - -/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict_cols( - mem_heap_t* heap, /*!< in/out: heap memory */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* table_id, /*!< out: table id */ - ulint* column_pos, /*!< out: column position */ - ulint* dict_id) /*!< out: dict id */ -{ - ulint len; - const byte* field; - - /* Initialize the output values */ - *table_id = ULINT_UNDEFINED; - *column_pos = ULINT_UNDEFINED; - *dict_id = ULINT_UNDEFINED; - - if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { - return("delete-marked record in SYS_ZIP_DICT_COLS"); - } - - if (UNIV_UNLIKELY(rec_get_n_fields_old(rec) != - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS)) { - return("wrong number of columns in SYS_ZIP_DICT_COLS" - " record"); - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { -err_len: - return("incorrect column length in SYS_ZIP_DICT_COLS"); - } - *table_id = mach_read_from_4(field); - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *column_pos = mach_read_from_4(field); - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID, &len); - if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR, &len); - if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *dict_id = mach_read_from_4(field); - - return(NULL); -} /********************************************************************//** Determine the flags of a table as stored in SYS_TABLES.TYPE and N_COLS. @return ULINT_UNDEFINED if error, else a valid dict_table_t::flags. */ diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index d072e61a237..ff4f218b557 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -4,7 +4,7 @@ Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -107,14 +107,6 @@ this program; if not, write to the Free Software Foundation, Inc., #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) -#ifndef HAVE_PERCONA_COMPRESSED_COLUMNS -#define COLUMN_FORMAT_TYPE_COMPRESSED 0xBADF00D -#define SQLCOM_CREATE_COMPRESSION_DICTIONARY 0xDECAF -#define SQLCOM_DROP_COMPRESSION_DICTIONARY 0xC0FFEE -#define ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST 0xDEADFACE -const static LEX_CSTRING null_lex_cstr={0,0}; -#endif - #ifdef MYSQL_DYNAMIC_PLUGIN #define tc_size 400 #define tdc_size 400 @@ -1476,30 +1468,6 @@ normalize_table_name_low( ibool set_lower_case); /* in: TRUE if we want to set name to lower case */ -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/** Creates a new compression dictionary. */ -static -handler_create_zip_dict_result -innobase_create_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len, - /*!< in/out: zip dictionary name length */ - const char* data, /*!< in: zip dictionary data */ - ulint* data_len); - /*!< in/out: zip dictionary data length */ - -/** Drops a existing compression dictionary. */ -static -handler_drop_zip_dict_result -innobase_drop_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len); - /*!< in/out: zip dictionary name length */ -#endif /*************************************************************//** Checks if buffer pool is big enough to enable backoff algorithm. InnoDB empty free list algorithm backoff requires free pages @@ -3591,10 +3559,6 @@ innobase_init( if (srv_file_per_table) innobase_hton->tablefile_extensions = ha_innobase_exts; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - innobase_hton->create_zip_dict = innobase_create_zip_dict; - innobase_hton->drop_zip_dict = innobase_drop_zip_dict; -#endif ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); #ifndef DBUG_OFF @@ -4288,90 +4252,6 @@ innobase_purge_changed_page_bitmaps( return (my_bool)log_online_purge_changed_page_bitmaps(lsn); } -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/** Creates a new compression dictionary. */ -static -handler_create_zip_dict_result -innobase_create_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len, - /*!< in/out: zip dictionary name length */ - const char* data, /*!< in: zip dictionary data */ - ulint* data_len) - /*!< in/out: zip dictionary data length */ -{ - handler_create_zip_dict_result result = - HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; - - DBUG_ENTER("innobase_create_zip_dict"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (UNIV_UNLIKELY(high_level_read_only)) { - DBUG_RETURN(HA_CREATE_ZIP_DICT_READ_ONLY); - } - - if (UNIV_UNLIKELY(*name_len > ZIP_DICT_MAX_NAME_LENGTH)) { - *name_len = ZIP_DICT_MAX_NAME_LENGTH; - DBUG_RETURN(HA_CREATE_ZIP_DICT_NAME_TOO_LONG); - } - - if (UNIV_UNLIKELY(*data_len > ZIP_DICT_MAX_DATA_LENGTH)) { - *data_len = ZIP_DICT_MAX_DATA_LENGTH; - DBUG_RETURN(HA_CREATE_ZIP_DICT_DATA_TOO_LONG); - } - - switch (dict_create_zip_dict(name, *name_len, data, *data_len)) { - case DB_SUCCESS: - result = HA_CREATE_ZIP_DICT_OK; - break; - case DB_DUPLICATE_KEY: - result = HA_CREATE_ZIP_DICT_ALREADY_EXISTS; - break; - default: - ut_ad(0); - result = HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; - } - DBUG_RETURN(result); -} - -/** Drops a existing compression dictionary. */ -static -handler_drop_zip_dict_result -innobase_drop_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len) - /*!< in/out: zip dictionary name length */ -{ - handler_drop_zip_dict_result result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; - - DBUG_ENTER("innobase_drop_zip_dict"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (UNIV_UNLIKELY(high_level_read_only)) { - DBUG_RETURN(HA_DROP_ZIP_DICT_READ_ONLY); - } - - switch (dict_drop_zip_dict(name, *name_len)) { - case DB_SUCCESS: - result = HA_DROP_ZIP_DICT_OK; - break; - case DB_RECORD_NOT_FOUND: - result = HA_DROP_ZIP_DICT_DOES_NOT_EXIST; - break; - case DB_ROW_IS_REFERENCED: - result = HA_DROP_ZIP_DICT_IS_REFERENCED; - break; - default: - ut_ad(0); - result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; - } - DBUG_RETURN(result); -} -#endif /*****************************************************************//** Check whether this is a fake change transaction. @return TRUE if a fake change transaction */ @@ -5941,88 +5821,6 @@ func_exit: DBUG_RETURN(ret); } -/** This function checks if all the compression dictionaries referenced -in table->fields exist in SYS_ZIP_DICT InnoDB system table. -@return true if all referenced dictionaries exist */ -UNIV_INTERN -bool -innobase_check_zip_dicts( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - ulint* dict_ids, /*!< out: identified zip dict ids - (at least n_fields long) */ - trx_t* trx, /*!< in: transaction */ - const char** err_dict_name) /*!< out: the name of the - zip_dict which does not exist. */ -{ - DBUG_ENTER("innobase_check_zip_dicts"); - - bool res = true; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - dberr_t err = DB_SUCCESS; - const size_t n_fields = table->s->fields; - - Field* field_ptr; - for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; - ++field_idx) - { - field_ptr = table->field[field_idx]; - if (field_ptr->has_associated_compression_dictionary()) { - err = dict_create_get_zip_dict_id_by_name( - field_ptr->zip_dict_name.str, - field_ptr->zip_dict_name.length, - &dict_ids[field_idx], - trx); - ut_a(err == DB_SUCCESS || err == DB_RECORD_NOT_FOUND); - } - else { - dict_ids[field_idx] = ULINT_UNDEFINED; - } - - } - - if (err != DB_SUCCESS) { - res = false; - *err_dict_name = field_ptr->zip_dict_name.str; - } - -#endif - DBUG_RETURN(res); -} - -/** This function creates compression dictionary references in -SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info -in table->fields and provided zip dict ids. */ -UNIV_INTERN -void -innobase_create_zip_dict_references( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - table_id_t ib_table_id, /*!< in: table ID in Innodb data - dictionary */ - ulint* zip_dict_ids, /*!< in: zip dict ids - (at least n_fields long) */ - trx_t* trx) /*!< in: transaction */ -{ - DBUG_ENTER("innobase_create_zip_dict_references"); - - dberr_t err = DB_SUCCESS; - const size_t n_fields = table->s->fields; - - for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; - ++field_idx) - { - if (zip_dict_ids[field_idx] != ULINT_UNDEFINED) { - err = dict_create_add_zip_dict_reference(ib_table_id, - table->field[field_idx]->field_index, - zip_dict_ids[field_idx], trx); - ut_a(err == DB_SUCCESS); - } - } - - DBUG_VOID_RETURN; -} - /*******************************************************************//** This function uses index translation table to quickly locate the requested index structure. @@ -7289,16 +7087,7 @@ ha_innobase::store_key_val_for_row( blob_data = row_mysql_read_blob_ref(&blob_len, (byte*) (record + (ulint) get_field_offset(table, field)), - (ulint) field->pack_length(), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast( - field->zip_dict_data.str), - field->zip_dict_data.length, prebuilt); -#else - 0, 0, 0, prebuilt); -#endif + (ulint) field->pack_length()); true_len = blob_len; @@ -7613,14 +7402,6 @@ build_template_field( templ->mbminlen = dict_col_get_mbminlen(col); templ->mbmaxlen = dict_col_get_mbmaxlen(col); templ->is_unsigned = col->prtype & DATA_UNSIGNED; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - templ->compressed = (field->column_format() - == COLUMN_FORMAT_TYPE_COMPRESSED); - templ->zip_dict_data = field->zip_dict_data; -#else - templ->compressed = 0; - templ->zip_dict_data = null_lex_cstr; -#endif if (!dict_index_is_clust(index) && templ->rec_field_no == ULINT_UNDEFINED) { @@ -8409,10 +8190,8 @@ calc_row_difference( case DATA_BLOB: /* Do not compress blob column while comparing*/ - o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len, - false, 0, 0, prebuilt); - n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len, - false, 0, 0, prebuilt); + o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len); + n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len); break; @@ -8482,17 +8261,7 @@ calc_row_difference( TRUE, new_mysql_row_col, col_pack_len, - dict_table_is_comp(prebuilt->table), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast( - field->zip_dict_data.str), - field->zip_dict_data.length, -#else - 0, 0, 0, -#endif - prebuilt); + dict_table_is_comp(prebuilt->table)); dfield_copy(&ufield->new_val, &dfield); } else { dfield_set_null(&ufield->new_val); @@ -10179,7 +9948,6 @@ create_table_def( ulint unsigned_type; ulint binary_type; ulint long_true_varchar; - ulint compressed; ulint charset_no; ulint i; ulint doc_id_col = 0; @@ -10329,13 +10097,6 @@ create_table_def( } } - /* Check if the the field has COMPRESSED attribute */ - compressed = 0; - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - compressed = DATA_COMPRESSED; - } - /* First check whether the column to be added has a system reserved name. */ if (dict_col_name_is_reserved(field->field_name)){ @@ -10356,8 +10117,7 @@ err_col: dtype_form_prtype( (ulint) field->type() | nulls_allowed | unsigned_type - | binary_type | long_true_varchar - | compressed, + | binary_type | long_true_varchar, charset_no), col_len); } @@ -11187,9 +10947,6 @@ ha_innobase::create( const char* stmt; size_t stmt_len; - mem_heap_t* heap = 0; - ulint* zip_dict_ids = 0; - DBUG_ENTER("ha_innobase::create"); DBUG_ASSERT(thd != NULL); @@ -11280,25 +11037,6 @@ ha_innobase::create( row_mysql_lock_data_dictionary(trx); - heap = mem_heap_create(form->s->fields * sizeof(ulint)); - zip_dict_ids = static_cast( - mem_heap_alloc(heap, form->s->fields * sizeof(ulint))); - - /* This is currently required for valgrind because MariaDB does - not currently support compressed columns. */ - for (size_t field_idx = 0; field_idx < form->s->fields; ++field_idx) { - zip_dict_ids[field_idx] = ULINT_UNDEFINED; - } - - const char* err_zip_dict_name = 0; - if (!innobase_check_zip_dicts(form, zip_dict_ids, - trx, &err_zip_dict_name)) { - error = -1; - my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, - MYF(0), err_zip_dict_name); - goto cleanup; - } - error = create_table_def(trx, form, norm_name, temp_path, remote_path, flags, flags2); if (error) { @@ -11406,22 +11144,6 @@ ha_innobase::create( dict_table_get_all_fts_indexes(innobase_table, fts->indexes); } - /* - Adding compression dictionary <-> compressed table column links - to the SYS_ZIP_DICT_COLS table. - */ - ut_a(zip_dict_ids != 0); - { - dict_table_t* local_table = dict_table_open_on_name( - norm_name, TRUE, FALSE, DICT_ERR_IGNORE_NONE); - - ut_a(local_table); - table_id_t table_id = local_table->id; - dict_table_close(local_table, TRUE, FALSE); - innobase_create_zip_dict_references(form, - table_id, zip_dict_ids, trx); - } - stmt = innobase_get_stmt(thd, &stmt_len); if (stmt) { @@ -11538,9 +11260,6 @@ ha_innobase::create( trx_free_for_mysql(trx); - if (heap != 0) - mem_heap_free(heap); - DBUG_RETURN(0); cleanup: @@ -11550,9 +11269,6 @@ cleanup: trx_free_for_mysql(trx); - if (heap != 0) - mem_heap_free(heap); - DBUG_RETURN(error); } @@ -13834,10 +13550,6 @@ ha_innobase::extra( row_mysql_prebuilt_free_blob_heap(prebuilt); } - if (prebuilt->compress_heap) { - row_mysql_prebuilt_free_compress_heap(prebuilt); - } - break; case HA_EXTRA_RESET_STATE: reset_template(); @@ -13889,10 +13601,6 @@ ha_innobase::reset() row_mysql_prebuilt_free_blob_heap(prebuilt); } - if (prebuilt->compress_heap) { - row_mysql_prebuilt_free_compress_heap(prebuilt); - } - reset_template(); ds_mrr.dsmrr_close(); @@ -14092,11 +13800,7 @@ ha_innobase::external_lock( && lock_type == F_WRLCK) || thd_sql_command(thd) == SQLCOM_CREATE_INDEX || thd_sql_command(thd) == SQLCOM_DROP_INDEX - || thd_sql_command(thd) == SQLCOM_DELETE - || thd_sql_command(thd) == - SQLCOM_CREATE_COMPRESSION_DICTIONARY - || thd_sql_command(thd) == - SQLCOM_DROP_COMPRESSION_DICTIONARY)) { + || thd_sql_command(thd) == SQLCOM_DELETE)) { if (thd_sql_command(thd) == SQLCOM_CREATE_TABLE) { @@ -14864,9 +14568,7 @@ ha_innobase::store_lock( && lock_type <= TL_WRITE)) || sql_command == SQLCOM_CREATE_INDEX || sql_command == SQLCOM_DROP_INDEX - || sql_command == SQLCOM_DELETE - || sql_command == SQLCOM_CREATE_COMPRESSION_DICTIONARY - || sql_command == SQLCOM_DROP_COMPRESSION_DICTIONARY)) { + || sql_command == SQLCOM_DELETE)) { ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); @@ -15807,84 +15509,6 @@ ha_innobase::check_if_incompatible_data( return(COMPATIBLE_DATA_YES); } -/** This function reads zip dict-related info from SYS_ZIP_DICT -and SYS_ZIP_DICT_COLS for all columns marked with -COLUMN_FORMAT_TYPE_COMPRESSED flag and updates -zip_dict_name / zip_dict_data for those which have associated -compression dictionaries. -*/ -UNIV_INTERN -void -ha_innobase::update_field_defs_with_zip_dict_info() -{ - DBUG_ENTER("update_field_defs_with_zip_dict_info"); - ut_ad(!mutex_own(&dict_sys->mutex)); - - char norm_name[FN_REFLEN]; - normalize_table_name(norm_name, table_share->normalized_path.str); - - dict_table_t* ib_table = dict_table_open_on_name( - norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); - - /* if dict_table_open_on_name() returns NULL, then it means that - TABLE_SHARE is populated for a table being created and we can - skip filling zip dict info here */ - if (ib_table == 0) - DBUG_VOID_RETURN; - -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - table_id_t ib_table_id = ib_table->id; - dict_table_close(ib_table, FALSE, FALSE); - Field* field; - for (uint i = 0; i < table_share->fields; ++i) { - field = table_share->field[i]; - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - bool reference_found = false; - ulint dict_id = 0; - switch (dict_get_dictionary_id_by_key(ib_table_id, i, - &dict_id)) { - case DB_SUCCESS: - reference_found = true; - break; - case DB_RECORD_NOT_FOUND: - reference_found = false; - break; - default: - ut_error; - } - if (reference_found) { - char* local_name = 0; - ulint local_name_len = 0; - char* local_data = 0; - ulint local_data_len = 0; - if (dict_get_dictionary_info_by_id(dict_id, - &local_name, &local_name_len, - &local_data, &local_data_len) != - DB_SUCCESS) { - ut_error; - } - else { - field->zip_dict_name.str = - local_name; - field->zip_dict_name.length = - local_name_len; - field->zip_dict_data.str = - local_data; - field->zip_dict_data.length = - local_data_len; - } - } - else { - field->zip_dict_name = null_lex_cstr; - field->zip_dict_data = null_lex_cstr; - } - } - } -#endif - DBUG_VOID_RETURN; -} - /****************************************************************//** Update the system variable innodb_io_capacity_max using the "saved" value. This function is registered as a callback with MySQL. */ @@ -19276,21 +18900,6 @@ static MYSQL_SYSVAR_BOOL(use_stacktrace, srv_use_stacktrace, "Print stacktrace on long semaphore wait (off by default supported only on linux)", NULL, NULL, FALSE); -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -static MYSQL_SYSVAR_UINT(compressed_columns_zip_level, - srv_compressed_columns_zip_level, - PLUGIN_VAR_RQCMDARG, - "Compression level used for compressed columns. 0 is no compression" - ", 1 is fastest and 9 is best compression. Default is 6.", - NULL, NULL, DEFAULT_COMPRESSION_LEVEL, 0, 9, 0); - -static MYSQL_SYSVAR_ULONG(compressed_columns_threshold, - srv_compressed_columns_threshold, - PLUGIN_VAR_RQCMDARG, - "Compress column data if its length exceeds this value. Default is 96", - NULL, NULL, 96, 1, ~0UL, 0); -#endif - static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(log_block_size), MYSQL_SYSVAR(additional_mem_pool_size), @@ -19494,10 +19103,6 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(tmpdir), MYSQL_SYSVAR(use_stacktrace), MYSQL_SYSVAR(simulate_comp_failures), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - MYSQL_SYSVAR(compressed_columns_zip_level), - MYSQL_SYSVAR(compressed_columns_threshold), -#endif NULL }; @@ -19520,10 +19125,6 @@ maria_declare_plugin(xtradb) i_s_xtradb_read_view, i_s_xtradb_internal_hash_tables, i_s_xtradb_rseg, -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -i_s_xtradb_zip_dict, -i_s_xtradb_zip_dict_cols, -#endif i_s_innodb_trx, i_s_innodb_locks, i_s_innodb_lock_waits, diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index f01faaf41d8..f4c73fda89d 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -290,14 +290,6 @@ class ha_innobase: public handler bool check_if_supported_virtual_columns(void) { return TRUE; } - /** This function reads zip dict-related info from SYS_ZIP_DICT - and SYS_ZIP_DICT_COLS for all columns marked with - COLUMN_FORMAT_TYPE_COMPRESSED flag and updates - zip_dict_name / zip_dict_data for those which have associated - compression dictionaries. - */ - virtual void update_field_defs_with_zip_dict_info(); - private: /** Builds a 'template' to the prebuilt struct. @@ -691,31 +683,3 @@ ib_push_frm_error( TABLE* table, /*!< in: MySQL table */ ulint n_keys, /*!< in: InnoDB #keys */ bool push_warning); /*!< in: print warning ? */ - -/** This function checks if all the compression dictionaries referenced -in table->fields exist in SYS_ZIP_DICT InnoDB system table. -@return true if all referenced dictionaries exist */ -UNIV_INTERN -bool -innobase_check_zip_dicts( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - ulint* dict_ids, /*!< out: identified zip dict ids - (at least n_fields long) */ - trx_t* trx, /*!< in: transaction */ - const char** err_dict_name); /*!< out: the name of the - zip_dict which does not exist. */ - -/** This function creates compression dictionary references in -SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info -in table->fields and provided zip dict ids. */ -UNIV_INTERN -void -innobase_create_zip_dict_references( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - table_id_t ib_table_id, /*!< in: table ID in Innodb data - dictionary */ - ulint* zip_dict_ids, /*!< in: zip dict ids - (at least n_fields long) */ - trx_t* trx); /*!< in: transaction */ diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 362ac9c1b89..c5ac48dc4e3 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -21,11 +22,6 @@ this program; if not, write to the Free Software Foundation, Inc., Smart ALTER TABLE *******************************************************/ -#ifndef HAVE_PERCONA_COMPRESSED_COLUMNS -#define COLUMN_FORMAT_TYPE_COMPRESSED 0xBADF00D -#define ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST 0xDEADFACE -#endif - #include #include #include @@ -1185,15 +1181,6 @@ innobase_col_to_mysql( field->reset(); if (field->type() == MYSQL_TYPE_VARCHAR) { - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - /* Skip compressed varchar column when - reporting an erroneous row - during index creation or table rebuild. */ - field->set_null(); - break; - } - /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -2492,14 +2479,7 @@ innobase_build_col_map_add( byte* buf = static_cast(mem_heap_alloc(heap, size)); row_mysql_store_col_in_innobase_format( - dfield, buf, TRUE, field->ptr, size, comp, -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast(field->zip_dict_data.str), - field->zip_dict_data.length, prebuilt); -#else - 0,0,0, prebuilt); -#endif + dfield, buf, TRUE, field->ptr, size, comp); } /** Construct the translation table for reordering, dropping or @@ -2753,7 +2733,6 @@ prepare_inplace_alter_table_dict( ulint num_fts_index; ha_innobase_inplace_ctx*ctx; uint sql_idx; - ulint* zip_dict_ids = 0; DBUG_ENTER("prepare_inplace_alter_table_dict"); @@ -2890,26 +2869,6 @@ prepare_inplace_alter_table_dict( ulint n_cols; dtuple_t* add_cols; - zip_dict_ids = static_cast( - mem_heap_alloc(ctx->heap, - altered_table->s->fields * sizeof(ulint))); - - /* This is currently required for valgrind because MariaDB does - not currently support compressed columns. */ - for (size_t field_idx = 0; - field_idx < altered_table->s->fields; - ++field_idx) { - zip_dict_ids[field_idx] = ULINT_UNDEFINED; - } - - const char* err_zip_dict_name = 0; - if (!innobase_check_zip_dicts(altered_table, zip_dict_ids, - ctx->trx, &err_zip_dict_name)) { - my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, - MYF(0), err_zip_dict_name); - goto new_clustered_failed; - } - if (innobase_check_foreigns( ha_alter_info, altered_table, old_table, user_table, ctx->drop_fk, ctx->num_to_drop_fk)) { @@ -3016,12 +2975,6 @@ prepare_inplace_alter_table_dict( } } - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - field_type |= DATA_COMPRESSED; - } - - if (dict_col_name_is_reserved(field->field_name)) { dict_mem_table_free(ctx->new_table); my_error(ER_WRONG_COLUMN_NAME, MYF(0), @@ -3279,17 +3232,6 @@ op_ok: DBUG_ASSERT(error == DB_SUCCESS); -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - /* - Adding compression dictionary <-> compressed table column links - to the SYS_ZIP_DICT_COLS table. - */ - if (zip_dict_ids != 0) { - innobase_create_zip_dict_references(altered_table, - ctx->trx->table_id, zip_dict_ids, ctx->trx); - } -#endif - /* Commit the data dictionary transaction in order to release the table locks on the system tables. This means that if MySQL crashes while creating a new primary key inside diff --git a/storage/xtradb/handler/xtradb_i_s.cc b/storage/xtradb/handler/xtradb_i_s.cc index eb6637dad03..84c1e5852bc 100644 --- a/storage/xtradb/handler/xtradb_i_s.cc +++ b/storage/xtradb/handler/xtradb_i_s.cc @@ -2,6 +2,7 @@ Copyright (c) 2007, 2012, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2010-2012, Percona Inc. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -36,7 +37,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include /* btr_search_sys */ #include /* recv_sys */ #include -#include /* for ZIP_DICT_MAX_* constants */ /* for XTRADB_RSEG table */ #include "trx0trx.h" /* for TRX_QUE_STATE_STR_MAX_LEN */ @@ -629,331 +629,3 @@ UNIV_INTERN struct st_mysql_plugin i_s_xtradb_rseg = STRUCT_FLD(version_info, INNODB_VERSION_STR), STRUCT_FLD(maturity, MariaDB_PLUGIN_MATURITY_STABLE), }; - - -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/************************************************************************/ -enum zip_dict_field_type -{ - zip_dict_field_id, - zip_dict_field_name, - zip_dict_field_zip_dict -}; - -static ST_FIELD_INFO xtradb_sys_zip_dict_fields_info[] = -{ - { STRUCT_FLD(field_name, "id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "name"), - STRUCT_FLD(field_length, ZIP_DICT_MAX_NAME_LENGTH), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, 0), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "zip_dict"), - STRUCT_FLD(field_length, ZIP_DICT_MAX_DATA_LENGTH), - STRUCT_FLD(field_type, MYSQL_TYPE_BLOB), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, 0), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - END_OF_ST_FIELD_INFO -}; - -/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT with information -collected by scanning SYS_ZIP_DICT table. -@return 0 on success */ -static -int -xtradb_i_s_dict_fill_sys_zip_dict( - THD* thd, /*!< in: thread */ - ulint id, /*!< in: dict ID */ - const char* name, /*!< in: dict name */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - TABLE* table_to_fill) /*!< in/out: fill this table */ -{ - DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict"); - - Field** fields = table_to_fill->field; - - OK(field_store_ulint(fields[zip_dict_field_id], id)); - OK(field_store_string(fields[zip_dict_field_name], name)); - OK(field_store_blob(fields[zip_dict_field_zip_dict], data, - data_len)); - - OK(schema_table_store_record(thd, table_to_fill)); - - DBUG_RETURN(0); -} - -/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. -Loop through each record in SYS_ZIP_DICT, and extract the column -information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. -@return 0 on success */ -static -int -xtradb_i_s_sys_zip_dict_fill_table( - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (not used) */ -{ - btr_pcur_t pcur; - const rec_t* rec; - mem_heap_t* heap; - mtr_t mtr; - - DBUG_ENTER("xtradb_i_s_sys_zip_dict_fill_table"); - RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); - - /* deny access to user without SUPER_ACL privilege */ - if (check_global_access(thd, SUPER_ACL)) { - DBUG_RETURN(0); - } - - heap = mem_heap_create(1000); - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - - rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT); - ulint zip_size = dict_table_zip_size(pcur.btr_cur.index->table); - - while (rec) { - const char* err_msg; - ulint id; - const char* name; - const char* data; - ulint data_len; - - /* Extract necessary information from a SYS_ZIP_DICT row */ - err_msg = dict_process_sys_zip_dict( - heap, zip_size, rec, &id, &name, &data, &data_len); - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - - if (!err_msg) { - xtradb_i_s_dict_fill_sys_zip_dict( - thd, id, name, data, data_len, - tables->table); - } else { - push_warning_printf(thd, - Sql_condition::WARN_LEVEL_WARN, - ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); - } - - mem_heap_empty(heap); - - /* Get the next record */ - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - rec = dict_getnext_system(&pcur, &mtr); - } - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - mem_heap_free(heap); - - DBUG_RETURN(0); -} - -static int i_s_xtradb_zip_dict_init(void* p) -{ - DBUG_ENTER("i_s_xtradb_zip_dict_init"); - - ST_SCHEMA_TABLE* schema = static_cast(p); - - schema->fields_info = xtradb_sys_zip_dict_fields_info; - schema->fill_table = xtradb_i_s_sys_zip_dict_fill_table; - - DBUG_RETURN(0); -} - -UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict = -{ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - STRUCT_FLD(info, &i_s_info), - STRUCT_FLD(name, "XTRADB_ZIP_DICT"), - STRUCT_FLD(author, PLUGIN_AUTHOR), - STRUCT_FLD(descr, "InnoDB compression dictionaries information"), - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - STRUCT_FLD(init, i_s_xtradb_zip_dict_init), - STRUCT_FLD(deinit, i_s_common_deinit), - STRUCT_FLD(version, INNODB_VERSION_SHORT), - STRUCT_FLD(status_vars, NULL), - STRUCT_FLD(system_vars, NULL), - STRUCT_FLD(__reserved1, NULL), - STRUCT_FLD(flags, 0UL), -}; - -enum zip_dict_cols_field_type -{ - zip_dict_cols_field_table_id, - zip_dict_cols_field_column_pos, - zip_dict_cols_field_dict_id -}; - -static ST_FIELD_INFO xtradb_sys_zip_dict_cols_fields_info[] = -{ - { STRUCT_FLD(field_name, "table_id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "column_pos"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "dict_id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - END_OF_ST_FIELD_INFO -}; - -/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS with information -collected by scanning SYS_ZIP_DICT_COLS table. -@return 0 on success */ -static -int -xtradb_i_s_dict_fill_sys_zip_dict_cols( - THD* thd, /*!< in: thread */ - ulint table_id, /*!< in: table ID */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict ID */ - TABLE* table_to_fill) /*!< in/out: fill this table */ -{ - DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict_cols"); - - Field** fields = table_to_fill->field; - - OK(field_store_ulint(fields[zip_dict_cols_field_table_id], - table_id)); - OK(field_store_ulint(fields[zip_dict_cols_field_column_pos], - column_pos)); - OK(field_store_ulint(fields[zip_dict_cols_field_dict_id], - dict_id)); - - OK(schema_table_store_record(thd, table_to_fill)); - - DBUG_RETURN(0); -} - -/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. -Loop through each record in SYS_ZIP_DICT_COLS, and extract the column -information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. -@return 0 on success */ -static -int -xtradb_i_s_sys_zip_dict_cols_fill_table( - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (not used) */ -{ - btr_pcur_t pcur; - const rec_t* rec; - mem_heap_t* heap; - mtr_t mtr; - - DBUG_ENTER("xtradb_i_s_sys_zip_dict_cols_fill_table"); - RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); - - /* deny access to user without SUPER_ACL privilege */ - if (check_global_access(thd, SUPER_ACL)) { - DBUG_RETURN(0); - } - - heap = mem_heap_create(1000); - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - - rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT_COLS); - - while (rec) { - const char* err_msg; - ulint table_id; - ulint column_pos; - ulint dict_id; - - /* Extract necessary information from a SYS_ZIP_DICT_COLS - row */ - err_msg = dict_process_sys_zip_dict_cols( - heap, rec, &table_id, &column_pos, &dict_id); - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - - if (!err_msg) { - xtradb_i_s_dict_fill_sys_zip_dict_cols( - thd, table_id, column_pos, dict_id, - tables->table); - } else { - push_warning_printf(thd, - Sql_condition::WARN_LEVEL_WARN, - ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); - } - - mem_heap_empty(heap); - - /* Get the next record */ - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - rec = dict_getnext_system(&pcur, &mtr); - } - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - mem_heap_free(heap); - - DBUG_RETURN(0); -} - -static int i_s_xtradb_zip_dict_cols_init(void* p) -{ - DBUG_ENTER("i_s_xtradb_zip_dict_cols_init"); - - ST_SCHEMA_TABLE* schema = static_cast(p); - - schema->fields_info = xtradb_sys_zip_dict_cols_fields_info; - schema->fill_table = xtradb_i_s_sys_zip_dict_cols_fill_table; - - DBUG_RETURN(0); -} - -UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict_cols = -{ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - STRUCT_FLD(info, &i_s_info), - STRUCT_FLD(name, "XTRADB_ZIP_DICT_COLS"), - STRUCT_FLD(author, PLUGIN_AUTHOR), - STRUCT_FLD(descr, "InnoDB compressed columns information"), - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - STRUCT_FLD(init, i_s_xtradb_zip_dict_cols_init), - STRUCT_FLD(deinit, i_s_common_deinit), - STRUCT_FLD(version, INNODB_VERSION_SHORT), - STRUCT_FLD(status_vars, NULL), - STRUCT_FLD(system_vars, NULL), - STRUCT_FLD(__reserved1, NULL), - STRUCT_FLD(flags, 0UL), -}; -#endif diff --git a/storage/xtradb/handler/xtradb_i_s.h b/storage/xtradb/handler/xtradb_i_s.h index 905d84587af..994bc11c1b8 100644 --- a/storage/xtradb/handler/xtradb_i_s.h +++ b/storage/xtradb/handler/xtradb_i_s.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2010-2012, Percona Inc. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -22,7 +23,5 @@ this program; if not, write to the Free Software Foundation, Inc., extern struct st_mysql_plugin i_s_xtradb_read_view; extern struct st_mysql_plugin i_s_xtradb_internal_hash_tables; extern struct st_mysql_plugin i_s_xtradb_rseg; -extern struct st_mysql_plugin i_s_xtradb_zip_dict; -extern struct st_mysql_plugin i_s_xtradb_zip_dict_cols; #endif /* XTRADB_I_S_H */ diff --git a/storage/xtradb/include/data0type.h b/storage/xtradb/include/data0type.h index f269c266efb..df6b6a41c11 100644 --- a/storage/xtradb/include/data0type.h +++ b/storage/xtradb/include/data0type.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -170,9 +171,6 @@ be less than 256 */ type when the column is true VARCHAR where MySQL uses 2 bytes to store the data len; for shorter VARCHARs MySQL uses only 1 byte */ -#define DATA_COMPRESSED 16384 /* this is ORed to the precise data - type when the column has COLUMN_FORMAT = - COMPRESSED attribute*/ /*-------------------------------------------*/ /* This many bytes we need to store the type information affecting the @@ -503,17 +501,6 @@ dtype_print( /*========*/ const dtype_t* type); /*!< in: type */ -/** -Calculates the number of extra bytes needed for compression header -depending on precise column type. -@reval 0 if prtype does not include DATA_COMPRESSED flag -@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag -*/ -UNIV_INLINE -ulint -prtype_get_compression_extra( - ulint prtype); /*!< in: precise type */ - /* Structure for an SQL data type. If you add fields to this structure, be sure to initialize them everywhere. This structure is initialized in the following functions: diff --git a/storage/xtradb/include/data0type.ic b/storage/xtradb/include/data0type.ic index 29dc480a19c..555852474aa 100644 --- a/storage/xtradb/include/data0type.ic +++ b/storage/xtradb/include/data0type.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -710,18 +711,3 @@ dtype_get_sql_null_size( 0, 0)); #endif /* !UNIV_HOTBACKUP */ } - -/** -Calculates the number of extra bytes needed for compression header -depending on precise column type. -@reval 0 if prtype does not include DATA_COMPRESSED flag -@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag -*/ -UNIV_INLINE -ulint -prtype_get_compression_extra( - ulint prtype) /*!< in: precise type */ -{ - return (prtype & DATA_COMPRESSED) != 0 ? - ZIP_COLUMN_HEADER_LENGTH : 0; -} diff --git a/storage/xtradb/include/dict0boot.h b/storage/xtradb/include/dict0boot.h index d5bee886cbf..4fd9b0b7f98 100644 --- a/storage/xtradb/include/dict0boot.h +++ b/storage/xtradb/include/dict0boot.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -324,38 +325,6 @@ enum dict_fld_sys_datafiles_enum { DICT_FLD__SYS_DATAFILES__PATH = 3, DICT_NUM_FIELDS__SYS_DATAFILES = 4 }; -/* The columns in SYS_DICT */ -enum dict_col_sys_zip_dict_enum { - DICT_COL__SYS_ZIP_DICT__ID = 0, - DICT_COL__SYS_ZIP_DICT__NAME = 1, - DICT_COL__SYS_ZIP_DICT__DATA = 2, - DICT_NUM_COLS__SYS_ZIP_DICT = 3 -}; -/* The field numbers in the SYS_DICT clustered index */ -enum dict_fld_sys_zip_dict_enum { - DICT_FLD__SYS_ZIP_DICT__ID = 0, - DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID = 1, - DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR = 2, - DICT_FLD__SYS_ZIP_DICT__NAME = 3, - DICT_FLD__SYS_ZIP_DICT__DATA = 4, - DICT_NUM_FIELDS__SYS_ZIP_DICT = 5 -}; -/* The columns in SYS_DICT_COLS */ -enum dict_col_sys_zip_dict_cols_enum { - DICT_COL__SYS_ZIP_DICT_COLS__TABLE_ID = 0, - DICT_COL__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, - DICT_COL__SYS_ZIP_DICT_COLS__DICT_ID = 2, - DICT_NUM_COLS__SYS_ZIP_DICT_COLS = 3 -}; -/* The field numbers in the SYS_DICT_COLS clustered index */ -enum dict_fld_sys_zip_dict_cols_enum { - DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID = 0, - DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, - DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID = 2, - DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR = 3, - DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID = 4, - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS = 5 -}; /* A number of the columns above occur in multiple tables. These are the length of thos fields. */ diff --git a/storage/xtradb/include/dict0boot.ic b/storage/xtradb/include/dict0boot.ic index 2b156a4f672..42e91ee930e 100644 --- a/storage/xtradb/include/dict0boot.ic +++ b/storage/xtradb/include/dict0boot.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -92,5 +93,3 @@ dict_is_sys_table( { return(id < DICT_HDR_FIRST_ID); } - - diff --git a/storage/xtradb/include/dict0crea.h b/storage/xtradb/include/dict0crea.h index 686f56ad58c..e5977cbabf6 100644 --- a/storage/xtradb/include/dict0crea.h +++ b/storage/xtradb/include/dict0crea.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -164,18 +165,6 @@ dberr_t dict_create_or_check_sys_tablespace(void); /*=====================================*/ -#define ZIP_DICT_MAX_NAME_LENGTH 64 -/* Max window size (2^15) minus 262 */ -#define ZIP_DICT_MAX_DATA_LENGTH 32506 - -/** Creates the zip_dict system table inside InnoDB -at server bootstrap or server start if it is not found or is -not of the right form. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -dict_create_or_check_sys_zip_dict(void); - /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -192,83 +181,6 @@ dict_create_add_tablespace_to_dictionary( bool commit); /*!< in: if true then commit the transaction */ -/** Add a single compression dictionary definition to the SYS_ZIP_DICT -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get a single compression dictionary id for the given -(table id, column pos) pair. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get compression dictionary id for the given name. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_name( - const char* dict_name, /*!< in: dict name */ - ulint dict_name_len, /*!< in: dict name length */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory for name and data on success. -Must be freed with mem_free(). -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_info_by_id( - ulint dict_id, /*!< in: dict id */ - char** name, /*!< out: dict name */ - ulint* name_len, /*!< out: dict name length */ - char** data, /*!< out: dict data */ - ulint* data_len, /*!< out: dict data length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Remove a single compression dictionary from the data dictionary -tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Remove all compression dictionary references for the given table ID from -the data dictionary tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict_references_for_table( - ulint table_id, /*!< in: table id */ - trx_t* trx); /*!< in/out: transaction */ - /********************************************************************//** Add a foreign key definition to the data dictionary tables. @return error code or DB_SUCCESS */ diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index 1b6110dd010..a85f6f1eef9 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1871,52 +1871,6 @@ dict_table_set_corrupt_by_space( ulint space_id, ibool need_mutex); -/** Insert a records into SYS_ZIP_DICT. -@retval DB_SUCCESS if OK -@retval dberr_t if the insert failed */ -UNIV_INTERN -dberr_t -dict_create_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len, /*!< in: zip_dict name length*/ - const char* data, /*!< in: zip_dict data */ - ulint data_len); /*!< in: zip_dict data length */ - -/** Get single compression dictionary id for the given -(table id, column pos) pair. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_id_by_key( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id); /*!< out: zip_dict id */ - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory in name->str and data->str on success. -Must be freed with mem_free(). -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_info_by_id( - ulint dict_id, /*!< in: table name */ - char** name, /*!< out: dictionary name */ - ulint* name_len, /*!< out: dictionary name length*/ - char** data, /*!< out: dictionary data */ - ulint* data_len); /*!< out: dictionary data length*/ - -/** Delete a record in SYS_ZIP_DICT with the given name. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found -@retval DB_ROW_IS_REFERENCED if in use */ -UNIV_INTERN -dberr_t -dict_drop_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len); /*!< in: zip_dict name length*/ - #ifndef UNIV_NONINL #include "dict0dict.ic" #endif diff --git a/storage/xtradb/include/dict0load.h b/storage/xtradb/include/dict0load.h index 85e3e565637..1a720de5bb6 100644 --- a/storage/xtradb/include/dict0load.h +++ b/storage/xtradb/include/dict0load.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,8 +45,6 @@ enum dict_system_id_t { SYS_FOREIGN_COLS, SYS_TABLESPACES, SYS_DATAFILES, - SYS_ZIP_DICT, - SYS_ZIP_DICT_COLS, /* This must be last item. Defines the number of system tables. */ SYS_NUM_SYSTEM_TABLES @@ -389,32 +388,6 @@ dict_process_sys_datafiles( ulint* space, /*!< out: pace id */ const char** path); /*!< out: datafile path */ -/** This function parses a SYS_ZIP_DICT record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict( - mem_heap_t* heap, /*!< in/out: heap memory */ - ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* id, /*!< out: dict id */ - const char** name, /*!< out: dict name */ - const char** data, /*!< out: dict data */ - ulint* data_len); /*!< out: dict data length */ - -/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict_cols( - mem_heap_t* heap, /*!< in/out: heap memory */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* table_id, /*!< out: table id */ - ulint* column_pos, /*!< out: column position */ - ulint* dict_id); /*!< out: dict id */ - /********************************************************************//** Get the filepath for a spaceid from SYS_DATAFILES. This function provides a temporary heap which is used for the table lookup, but not for the path. diff --git a/storage/xtradb/include/row0mysql.h b/storage/xtradb/include/row0mysql.h index 70da84640e5..e1148517d99 100644 --- a/storage/xtradb/include/row0mysql.h +++ b/storage/xtradb/include/row0mysql.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +42,6 @@ struct SysIndexCallback; extern ibool row_rollback_on_timeout; -extern uint srv_compressed_columns_zip_level; -extern ulong srv_compressed_columns_threshold; - struct row_prebuilt_t; /*******************************************************************//** @@ -55,48 +53,6 @@ row_mysql_prebuilt_free_blob_heap( row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a ha_innobase:: table handle */ -/** Frees the compress heap in prebuilt when no longer needed. */ -UNIV_INTERN -void -row_mysql_prebuilt_free_compress_heap( - row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a - ha_innobase:: table handle */ - -/** Uncompress blob/text/varchar column using zlib -@return pointer to the uncompressed data */ -const byte* -row_decompress_column( - const byte* data, /*!< in: data in innodb(compressed) format */ - ulint *len, /*!< in: data length; out: length of - decompressed data*/ - const byte* dict_data, - /*!< in: optional dictionary data used for - decompression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt); - /*!< in: use prebuilt->compress_heap only - here*/ - -/** Compress blob/text/varchar column using zlib -@return pointer to the compressed data */ -byte* -row_compress_column( - const byte* data, /*!< in: data in mysql(uncompressed) - format */ - ulint *len, /*!< in: data length; out: length of - compressed data*/ - ulint lenlen, /*!< in: bytes used to store the length of - data */ - const byte* dict_data, - /*!< in: optional dictionary data used for - compression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt); - /*!< in: use prebuilt->compress_heap only - here*/ - /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -135,21 +91,10 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len, /*!< in: BLOB length; if the value to store + ulint len); /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ - bool need_decompression, - /*!< in: if the data need to be compressed*/ - const byte* dict_data, - /*!< in: optional compression dictionary - data */ - ulint dict_data_len, - /*!< in: optional compression dictionary data - length */ - row_prebuilt_t* prebuilt); - /*compress_heap only - here */ /*******************************************************************//** Reads a reference to a BLOB in the MySQL format. @return pointer to BLOB data */ @@ -160,17 +105,8 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len, /*!< in: BLOB reference length + ulint col_len); /*!< in: BLOB reference length (not BLOB length) */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap - only here */ /**************************************************************//** Pad a column with spaces. */ UNIV_INTERN @@ -218,16 +154,7 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp, /*!< in: nonzero=compact format */ - bool need_compression, - /*!< in: if the data need to be - compressed */ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap - only here */ + ulint comp); /*!< in: nonzero=compact format */ /****************************************************************//** Handles user errors and lock waits detected by the database engine. @return true if it was a lock wait and we should continue running the @@ -721,8 +648,6 @@ struct mysql_row_templ_t { ulint is_unsigned; /*!< if a column type is an integer type and this field is != 0, then it is an unsigned integer type */ - bool compressed; /*!< if column format is compressed */ - LEX_CSTRING zip_dict_data; /*!< associated compression dictionary */ }; #define MYSQL_FETCH_CACHE_SIZE 8 @@ -920,8 +845,6 @@ struct row_prebuilt_t { in fetch_cache */ mem_heap_t* blob_heap; /*!< in SELECTS BLOB fields are copied to this heap */ - mem_heap_t* compress_heap; /*!< memory heap used to compress - /decompress blob column*/ mem_heap_t* old_vers_heap; /*!< memory heap where a previous version is built in consistent read */ bool in_fts_query; /*!< Whether we are in a FTS query */ diff --git a/storage/xtradb/rem/rem0rec.cc b/storage/xtradb/rem/rem0rec.cc index 09cd810cd7b..6db11be18e9 100644 --- a/storage/xtradb/rem/rem0rec.cc +++ b/storage/xtradb/rem/rem0rec.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -320,8 +321,7 @@ rec_init_offsets_comp_ordinary( stored in one byte for 0..127. The length will be encoded in two bytes when it is 128 or more, or when the field is stored externally. */ - if (UNIV_UNLIKELY(col->len > 255 - - prtype_get_compression_extra(col->prtype)) + if (UNIV_UNLIKELY(col->len > 255) || UNIV_UNLIKELY(col->mtype == DATA_BLOB)) { if (len & 0x80) { @@ -846,8 +846,7 @@ rec_get_converted_size_comp_prefix_low( ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY || col->mtype == DATA_VARMYSQL) && (col->len == 0 - || len <= col->len + - prtype_get_compression_extra(col->prtype)))); + || len <= col->len))); fixed_len = field->fixed_len; if (temp && fixed_len @@ -879,8 +878,7 @@ rec_get_converted_size_comp_prefix_low( ut_ad(col->len >= 256 || col->mtype == DATA_BLOB); extra_size += 2; } else if (len < 128 - || (col->len < 256 - - prtype_get_compression_extra(col->prtype) + || (col->len < 256 && col->mtype != DATA_BLOB)) { extra_size++; } else { @@ -1276,16 +1274,12 @@ rec_convert_dtuple_to_rec_comp( *lens-- = (byte) (len >> 8) | 0xc0; *lens-- = (byte) len; } else { - ut_ad(len <= dtype_get_len(type) + - prtype_get_compression_extra( - dtype_get_prtype(type)) + ut_ad(len <= dtype_get_len(type) || dtype_get_mtype(type) == DATA_BLOB || !strcmp(index->name, FTS_INDEX_TABLE_IND_NAME)); if (len < 128 - || (dtype_get_len(type) < 256 - - prtype_get_compression_extra( - dtype_get_prtype(type)) + || (dtype_get_len(type) < 256 && dtype_get_mtype(type) != DATA_BLOB)) { *lens-- = (byte) len; diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index e397053949e..c1436f9e1b7 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -529,8 +530,7 @@ row_merge_buf_add( ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY || col->mtype == DATA_VARMYSQL) && (col->len == 0 - || len <= col->len + - prtype_get_compression_extra(col->prtype)))); + || len <= col->len))); fixed_len = ifield->fixed_len; if (fixed_len && !dict_table_is_comp(index->table) @@ -559,8 +559,7 @@ row_merge_buf_add( } else if (dfield_is_ext(field)) { extra_size += 2; } else if (len < 128 - || (col->len < 256 - - prtype_get_compression_extra(col->prtype) + || (col->len < 256 && col->mtype != DATA_BLOB)) { extra_size++; } else { diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 46bf523750c..b366a55b2d9 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -69,48 +70,6 @@ Created 9/17/2000 Heikki Tuuri /** Provide optional 4.x backwards compatibility for 5.0 and above */ UNIV_INTERN ibool row_rollback_on_timeout = FALSE; -/** -Z_NO_COMPRESSION = 0 -Z_BEST_SPEED = 1 -Z_BEST_COMPRESSION = 9 -Z_DEFAULT_COMPRESSION = -1 -Compression level to be used by zlib for compressed-blob columns. -Settable by user. -*/ -UNIV_INTERN uint srv_compressed_columns_zip_level = DEFAULT_COMPRESSION_LEVEL; -/** -(Z_FILTERED | Z_HUFFMAN_ONLY | Z_RLE | Z_FIXED | Z_DEFAULT_STRATEGY) - -The strategy parameter is used to tune the compression algorithm. Use the -value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a -filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only -(no string match), or Z_RLE to limit match distances to one -(run-length encoding). Filtered data consists mostly of small values with a -somewhat random distribution. In this case, the compression algorithm is -tuned to compress them better. -The effect of Z_FILTERED is to force more Huffman coding and less string -matching; it is somewhat intermediate between Z_DEFAULT_STRATEGY and -Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as Z_HUFFMAN_ONLY, -but give better compression for PNG image data. The strategy parameter only -affects the compression ratio but not the correctness of the compressed -output even if it is not set appropriately. Z_FIXED prevents the use of -dynamic Huffman codes, allowing for a simpler decoder for special -applications. -*/ -const uint srv_compressed_columns_zlib_strategy = Z_DEFAULT_STRATEGY; -/** Compress the column if the data length exceeds this value. */ -UNIV_INTERN ulong srv_compressed_columns_threshold = 96; -/** -Determine if zlib needs to compute adler32 value for the compressed data. -This variables is similar to page_zip_zlib_wrap, but only used by -compressed blob columns. -*/ -const bool srv_compressed_columns_zlib_wrap = true; -/** -Determine if zlib will use custom memory allocation functions based on -InnoDB memory heap routines (mem_heap_t*). -*/ -const bool srv_compressed_columns_zlib_use_heap = false; /** Chain node of the list of tables to drop in the background. */ struct row_mysql_drop_t{ char* table_name; /*!< table name */ @@ -214,17 +173,6 @@ row_mysql_prebuilt_free_blob_heap( prebuilt->blob_heap = NULL; } -/** Frees the compress heap in prebuilt when no longer needed. */ -UNIV_INTERN -void -row_mysql_prebuilt_free_compress_heap( - row_prebuilt_t* prebuilt) /*!< in: prebuilt struct of a - ha_innobase:: table handle */ -{ - mem_heap_free(prebuilt->compress_heap); - prebuilt->compress_heap = NULL; -} - /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -281,425 +229,6 @@ row_mysql_read_true_varchar( return(field + 1); } -/** - Compressed BLOB header format: - --------------------------------------------------------------- - | reserved | wrap | algorithm | len-len | compressed | unused | - | [1] | [1] | [5] | [3] | [1] | [5] | - --------------------------------------------------------------- - | 0 0 | 1 1 | 2 6 | 7 9 | 10 10 | 11 15 | - --------------------------------------------------------------- - * 'reserved' bit is planned to be used in future versions of the BLOB - header. In this version it must always be - 'default_zip_column_reserved_value' (0). - * 'wrap' identifies if compression algorithm calculated a checksum - (adler32 in case of zlib) and appended it to the compressed data. - * 'algorithm' identifies which algoritm was used to compress this BLOB. - Currently, the only value 'default_zip_column_algorithm_value' (0) is - supported. - * 'len-len' field identifies the length of the column length data portion - followed by this header (see below). - * If 'compressed' bit is set to 1, then this header is immediately followed - by 1..8 bytes (depending on the value of 'len-len' bitfield) which - determine original (uncompressed) block size. These 'len-len' bytes are - followed by compressed representation of the original data. - * If 'compressed' bit is set to 0, every other bitfield ('wrap', - 'algorithm' and 'le-len') must be ignored. In this case the header is - immediately followed by uncompressed (original) data. -*/ - -/** - Currently the only supported value for the 'reserved' field is - false (0). -*/ -static const bool default_zip_column_reserved_value = false; - -/** - Currently the only supported value for the 'algorithm' field is 0, which - means 'zlib'. -*/ -static const uint default_zip_column_algorithm_value = 0; - -static const size_t zip_column_prefix_max_length = - ZIP_COLUMN_HEADER_LENGTH + 8; -static const size_t zip_column_header_length = ZIP_COLUMN_HEADER_LENGTH; - -/* 'reserved', bit 0 */ -static const uint zip_column_reserved = 0; -/* 0000 0000 0000 0001 */ -static const uint zip_column_reserved_mask = 0x0001; - -/* 'wrap', bit 1 */ -static const uint zip_column_wrap = 1; -/* 0000 0000 0000 0010 */ -static const uint zip_column_wrap_mask = 0x0002; - -/* 'algorithm', bit 2,3,4,5,6 */ -static const uint zip_column_algorithm = 2; -/* 0000 0000 0111 1100 */ -static const uint zip_column_algorithm_mask = 0x007C; - -/* 'len-len', bit 7,8,9 */ -static const uint zip_column_data_length = 7; -/* 0000 0011 1000 0000 */ -static const uint zip_column_data_length_mask = 0x0380; - -/* 'compressed', bit 10 */ -static const uint zip_column_compressed = 10; -/* 0000 0100 0000 0000 */ -static const uint zip_column_compressed_mask = 0x0400; - -/** Updates compressed block header with the given components */ -static void -column_set_compress_header( - byte* data, - bool compressed, - ulint lenlen, - uint alg, - bool wrap, - bool reserved) -{ - ulint header = 0; - header |= (compressed << zip_column_compressed); - header |= (lenlen << zip_column_data_length); - header |= (alg << zip_column_algorithm); - header |= (wrap << zip_column_wrap); - header |= (reserved << zip_column_reserved); - mach_write_to_2(data, header); -} - -/** Parse compressed block header into components */ -static void -column_get_compress_header( - const byte* data, - bool* compressed, - ulint* lenlen, - uint* alg, - bool* wrap, - bool* reserved -) -{ - ulint header = mach_read_from_2(data); - *compressed = ((header & zip_column_compressed_mask) >> - zip_column_compressed); - *lenlen = ((header & zip_column_data_length_mask) >> - zip_column_data_length); - *alg = ((header & zip_column_algorithm_mask) >> - zip_column_algorithm); - *wrap = ((header & zip_column_wrap_mask) >> - zip_column_wrap); - *reserved = ((header & zip_column_reserved_mask) >> - zip_column_reserved); -} - -/** Allocate memory for zlib. */ -static -void* -column_zip_zalloc( - void* opaque, /*!< in/out: memory heap */ - uInt items, /*!< in: number of items to allocate */ - uInt size) /*!< in: size of an item in bytes */ -{ - return(mem_heap_zalloc(static_cast(opaque), - items * size)); -} - -/** Deallocate memory for zlib. */ -static -void -column_zip_free( - void* opaque MY_ATTRIBUTE((unused)), /*!< in: memory heap */ - void* address MY_ATTRIBUTE((unused))) /*!< in: object to free */ -{ -} - -/** Configure the zlib allocator to use the given memory heap. */ -UNIV_INTERN -void -column_zip_set_alloc( - void* stream, /*!< in/out: zlib stream */ - mem_heap_t* heap) /*!< in: memory heap to use */ -{ - z_stream* strm = static_cast(stream); - - if (srv_compressed_columns_zlib_use_heap) { - strm->zalloc = column_zip_zalloc; - strm->zfree = column_zip_free; - strm->opaque = heap; - } else { - strm->zalloc = (alloc_func)0; - strm->zfree = (free_func)0; - strm->opaque = (voidpf)0; - } -} - -/** Compress blob/text/varchar column using zlib -@return pointer to the compressed data */ -byte* -row_compress_column( - const byte* data, /*!< in: data in mysql(uncompressed) - format */ - ulint *len, /*!< in: data length; out: length of - compressed data*/ - ulint lenlen, /*!< in: bytes used to store the length of - data */ - const byte* dict_data, - /*!< in: optional dictionary data used for - compression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap only - here*/ -{ - int err = 0; - ulint comp_len = *len; - ulint buf_len = *len + zip_column_prefix_max_length; - byte* buf; - byte* ptr; - z_stream c_stream; - bool wrap = srv_compressed_columns_zlib_wrap; - - int window_bits = wrap ? MAX_WBITS : -MAX_WBITS; - - if (!prebuilt->compress_heap) { - prebuilt->compress_heap = - mem_heap_create(max(UNIV_PAGE_SIZE, buf_len)); - } - - buf = static_cast(mem_heap_zalloc( - prebuilt->compress_heap,buf_len)); - - if (*len < srv_compressed_columns_threshold || - srv_compressed_columns_zip_level == Z_NO_COMPRESSION) - goto do_not_compress; - - ptr = buf + zip_column_header_length + lenlen; - - /*init deflate object*/ - c_stream.next_in = const_cast(data); - c_stream.avail_in = *len; - c_stream.next_out = ptr; - c_stream.avail_out = comp_len; - - column_zip_set_alloc(&c_stream, prebuilt->compress_heap); - - err = deflateInit2(&c_stream, srv_compressed_columns_zip_level, - Z_DEFLATED, window_bits, MAX_MEM_LEVEL, - srv_compressed_columns_zlib_strategy); - ut_a(err == Z_OK); - - if (dict_data != 0 && dict_data_len != 0) { - err = deflateSetDictionary(&c_stream, dict_data, - dict_data_len); - ut_a(err == Z_OK); - } - - err = deflate(&c_stream, Z_FINISH); - if (err != Z_STREAM_END) { - deflateEnd(&c_stream); - if (err == Z_OK) - err = Z_BUF_ERROR; - } else { - comp_len = c_stream.total_out; - err = deflateEnd(&c_stream); - } - - switch (err) { - case Z_OK: - break; - case Z_BUF_ERROR: - /* data after compress is larger than uncompressed data*/ - break; - default: - ib_logf(IB_LOG_LEVEL_ERROR, - "failed to compress the column, error: %d\n", err); - } - - /* make sure the compressed data size is smaller than - uncompressed data */ - if (err == Z_OK && - *len > (comp_len + zip_column_header_length + lenlen)) { - column_set_compress_header(buf, true, lenlen - 1, - default_zip_column_algorithm_value, wrap, - default_zip_column_reserved_value); - ptr = buf + zip_column_header_length; - /*store the uncompressed data length*/ - switch (lenlen) { - case 1: - mach_write_to_1(ptr, *len); - break; - case 2: - mach_write_to_2(ptr, *len); - break; - case 3: - mach_write_to_3(ptr, *len); - break; - case 4: - mach_write_to_4(ptr, *len); - break; - default: - ut_error; - } - - *len = comp_len + zip_column_header_length + lenlen; - return buf; - } - -do_not_compress: - ptr = buf; - column_set_compress_header(ptr, false, 0, - default_zip_column_algorithm_value, false, - default_zip_column_reserved_value); - ptr += zip_column_header_length; - memcpy(ptr, data, *len); - *len += zip_column_header_length; - return buf; -} - -/** Uncompress blob/text/varchar column using zlib -@return pointer to the uncompressed data */ -const byte* -row_decompress_column( - const byte* data, /*!< in: data in innodb(compressed) format */ - ulint *len, /*!< in: data length; out: length of - decompressed data*/ - const byte* dict_data, - /*!< in: optional dictionary data used for - decompression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap only - here*/ -{ - ulint buf_len = 0; - byte* buf; - int err = 0; - int window_bits = 0; - z_stream d_stream; - bool is_compressed = false; - bool wrap = false; - bool reserved = false; - ulint lenlen = 0; - uint alg = 0; - - ut_ad(*len != ULINT_UNDEFINED); - ut_ad(*len >= zip_column_header_length); - - column_get_compress_header(data, &is_compressed, &lenlen, &alg, - &wrap, &reserved); - - if (reserved != default_zip_column_reserved_value) { - ib_logf(IB_LOG_LEVEL_FATAL, - "unsupported compressed BLOB header format\n"); - } - - if (alg != default_zip_column_algorithm_value) { - ib_logf(IB_LOG_LEVEL_FATAL, - "unsupported 'algorithm' value in the" - " compressed BLOB header\n"); - } - - ut_a(lenlen < 4); - - data += zip_column_header_length; - if (!is_compressed) { /* column not compressed */ - *len -= zip_column_header_length; - return data; - } - - lenlen++; - - ulint comp_len = *len - zip_column_header_length - lenlen; - - ulint uncomp_len = 0; - switch (lenlen) { - case 1: - uncomp_len = mach_read_from_1(data); - break; - case 2: - uncomp_len = mach_read_from_2(data); - break; - case 3: - uncomp_len = mach_read_from_3(data); - break; - case 4: - uncomp_len = mach_read_from_4(data); - break; - default: - ut_error; - } - - data += lenlen; - - /* data is compressed, decompress it*/ - if (!prebuilt->compress_heap) { - prebuilt->compress_heap = - mem_heap_create(max(UNIV_PAGE_SIZE, uncomp_len)); - } - - buf_len = uncomp_len; - buf = static_cast(mem_heap_zalloc( - prebuilt->compress_heap, buf_len)); - - /* init d_stream */ - d_stream.next_in = const_cast(data); - d_stream.avail_in = comp_len; - d_stream.next_out = buf; - d_stream.avail_out = buf_len; - - column_zip_set_alloc(&d_stream, prebuilt->compress_heap); - - window_bits = wrap ? MAX_WBITS : -MAX_WBITS; - err = inflateInit2(&d_stream, window_bits); - ut_a(err == Z_OK); - - err = inflate(&d_stream, Z_FINISH); - if (err == Z_NEED_DICT) { - ut_a(dict_data != 0 && dict_data_len != 0); - err = inflateSetDictionary(&d_stream, dict_data, - dict_data_len); - ut_a(err == Z_OK); - err = inflate(&d_stream, Z_FINISH); - } - - if (err != Z_STREAM_END) { - inflateEnd(&d_stream); - if (err == Z_BUF_ERROR && d_stream.avail_in == 0) - err = Z_DATA_ERROR; - } else { - buf_len = d_stream.total_out; - err = inflateEnd(&d_stream); - } - - switch (err) { - case Z_OK: - break; - case Z_BUF_ERROR: - ib_logf(IB_LOG_LEVEL_FATAL, - "zlib buf error, this shouldn't happen\n"); - break; - default: - ib_logf(IB_LOG_LEVEL_FATAL, - "failed to decompress column, error: %d\n", err); - } - - if (err == Z_OK) { - if (buf_len != uncomp_len) { - ib_logf(IB_LOG_LEVEL_FATAL, - "failed to decompress blob column, may" - " be corrupted\n"); - } - *len = buf_len; - return buf; - } - - *len -= (zip_column_header_length + lenlen); - return data; -} - - /*******************************************************************//** Stores a reference to a BLOB in the MySQL format. */ UNIV_INTERN @@ -713,21 +242,10 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len, /*!< in: BLOB length; if the value to store + ulint len) /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ - bool need_decompression, - /*!< in: if the data need to be compressed*/ - const byte* dict_data, - /*!< in: optional compression dictionary - data */ - ulint dict_data_len, - /*!< in: optional compression dictionary data - length */ - row_prebuilt_t* prebuilt) - /*compress_heap only - here */ { /* MySQL might assume the field is set to zero except the length and the pointer fields */ @@ -739,27 +257,11 @@ row_mysql_store_blob_ref( In 32-bit architectures we only use the first 4 bytes of the pointer slot. */ - ut_a(col_len - 8 > 1 || - len < 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - ut_a(col_len - 8 > 2 || - len < 256 * 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - ut_a(col_len - 8 > 3 || - len < 256 * 256 * 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - - const byte *ptr = NULL; - - if (need_decompression) - ptr = row_decompress_column((const byte*)data, &len, - dict_data, dict_data_len, prebuilt); - - if (ptr) - memcpy(dest + col_len - 8, &ptr, sizeof ptr); - else - memcpy(dest + col_len - 8, &data, sizeof data); + ut_a(col_len - 8 > 1 || len < 256); + ut_a(col_len - 8 > 2 || len < 256 * 256); + ut_a(col_len - 8 > 3 || len < 256 * 256 * 256); + memcpy(dest + col_len - 8, &data, sizeof data); mach_write_to_n_little_endian(dest, col_len - 8, len); } @@ -773,32 +275,15 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len, /*!< in: BLOB reference length + ulint col_len) /*!< in: BLOB reference length (not BLOB length) */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap - only here */ { byte* data = NULL; - byte* ptr = NULL; *len = mach_read_from_n_little_endian(ref, col_len - 8); memcpy(&data, ref + col_len - 8, sizeof data); - if (need_compression) { - ptr = row_compress_column(data, len, col_len - 8, dict_data, - dict_data_len, prebuilt); - if (ptr) - data = ptr; - } - return(data); } @@ -881,16 +366,7 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp, /*!< in: nonzero=compact format */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap - only here */ + ulint comp) /*!< in: nonzero=compact format */ { const byte* ptr = mysql_data; const dtype_t* dtype; @@ -943,14 +419,8 @@ row_mysql_store_col_in_innobase_format( lenlen = 2; } - const byte* tmp_ptr = row_mysql_read_true_varchar( + ptr = row_mysql_read_true_varchar( &col_len, mysql_data, lenlen); - if (need_compression) - ptr = row_compress_column(tmp_ptr, &col_len, - lenlen, dict_data, dict_data_len, - prebuilt); - else - ptr = tmp_ptr; } else { /* Remove trailing spaces from old style VARCHAR columns. */ @@ -1032,9 +502,7 @@ row_mysql_store_col_in_innobase_format( } } else if (type == DATA_BLOB && row_format_col) { - ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len, - need_compression, dict_data, dict_data_len, - prebuilt); + ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len); } dfield_set_data(dfield, ptr, col_len); @@ -1092,11 +560,7 @@ row_mysql_convert_row_to_innobase( TRUE, /* MySQL row format data */ mysql_rec + templ->mysql_col_offset, templ->mysql_col_len, - dict_table_is_comp(prebuilt->table), - templ->compressed, - reinterpret_cast( - templ->zip_dict_data.str), - templ->zip_dict_data.length, prebuilt); + dict_table_is_comp(prebuilt->table)); next_column: ; } @@ -1442,10 +906,6 @@ row_prebuilt_free( mem_heap_free(prebuilt->blob_heap); } - if (prebuilt->compress_heap) { - mem_heap_free(prebuilt->compress_heap); - } - if (prebuilt->old_vers_heap) { mem_heap_free(prebuilt->old_vers_heap); } @@ -1875,9 +1335,6 @@ row_insert_for_mysql( return(DB_READ_ONLY); } - if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) - mem_heap_empty(prebuilt->compress_heap); - trx->op_info = "inserting"; row_mysql_delay_if_needed(); @@ -3443,8 +2900,6 @@ row_mysql_table_id_reassign( " WHERE TABLE_ID = :old_id;\n" "UPDATE SYS_INDEXES SET TABLE_ID = :new_id\n" " WHERE TABLE_ID = :old_id;\n" - "UPDATE SYS_ZIP_DICT_COLS SET TABLE_ID = :new_id_narrow\n" - " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n", FALSE, trx); return(err); @@ -4228,9 +3683,6 @@ next_rec: "UPDATE SYS_INDEXES" " SET TABLE_ID = :new_id, SPACE = :new_space\n" " WHERE TABLE_ID = :old_id;\n" - "UPDATE SYS_ZIP_DICT_COLS\n" - " SET TABLE_ID = :new_id_narrow\n" - " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n" , FALSE, trx); @@ -4858,19 +4310,6 @@ row_drop_table_for_mysql( filepath = fil_make_ibd_name(tablename, false); } - /* Remove all compression dictionary references for the - table */ - err = dict_create_remove_zip_dict_references_for_table( - table->id, trx); - if (err != DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_ERROR, "Error: (%s) not " - "able to remove compression dictionary " - "references for table %s", ut_strerr(err), - tablename); - - goto funct_exit; - } - if (dict_table_has_fts_index(table) || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) { ut_ad(table->n_ref_count == 0); diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index ad1e9e2bf9d..fd23d83e0e5 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -2,6 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2459,8 +2460,7 @@ row_sel_convert_mysql_key_to_innobase( /* MySQL key value format col */ FALSE, key_ptr + data_offset, data_len, - dict_table_is_comp(index->table), - false, 0, 0 ,0); + dict_table_is_comp(index->table)); ut_a(buf <= original_buf + buf_len); } @@ -2554,15 +2554,15 @@ row_sel_store_row_id_to_prebuilt( #ifdef UNIV_DEBUG /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ # define row_sel_field_store_in_mysql_format( \ - dest,templ,idx,field,src,len,prebuilt) \ + dest,templ,idx,field,src,len) \ row_sel_field_store_in_mysql_format_func \ - (dest,templ,idx,field,src,len, prebuilt) + (dest,templ,idx,field,src,len) #else /* UNIV_DEBUG */ /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ # define row_sel_field_store_in_mysql_format( \ - dest,templ,idx,field,src,len,prebuilt) \ + dest,templ,idx,field,src,len) \ row_sel_field_store_in_mysql_format_func \ - (dest,templ,src,len, prebuilt) + (dest,templ,src,len) #endif /* UNIV_DEBUG */ /**************************************************************//** @@ -2592,10 +2592,7 @@ row_sel_field_store_in_mysql_format_func( templ->icp_rec_field_no */ #endif /* UNIV_DEBUG */ const byte* data, /*!< in: data to store */ - ulint len, /*!< in: length of the data */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap - only here */ + ulint len) /*!< in: length of the data */ { byte* ptr; #ifdef UNIV_DEBUG @@ -2639,15 +2636,6 @@ row_sel_field_store_in_mysql_format_func( field_end = dest + templ->mysql_col_len; if (templ->mysql_type == DATA_MYSQL_TRUE_VARCHAR) { - /* If this is a compressed column, - decompress it first */ - if (templ->compressed) - data = row_decompress_column(data, &len, - reinterpret_cast( - templ->zip_dict_data.str), - templ->zip_dict_data.length, - prebuilt); - /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -2698,11 +2686,7 @@ row_sel_field_store_in_mysql_format_func( already copied to the buffer in row_sel_store_mysql_rec */ row_mysql_store_blob_ref(dest, templ->mysql_col_len, data, - len, templ->compressed, - reinterpret_cast( - templ->zip_dict_data.str), - templ->zip_dict_data.length, - prebuilt); + len); break; case DATA_MYSQL: @@ -2855,7 +2839,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len, prebuilt); + templ, index, field_no, data, len); if (heap != prebuilt->blob_heap) { mem_heap_free(heap); @@ -2905,7 +2889,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len, prebuilt); + templ, index, field_no, data, len); } ut_ad(len != UNIV_SQL_NULL); @@ -2953,9 +2937,6 @@ row_sel_store_mysql_rec( prebuilt->blob_heap = NULL; } - if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) - mem_heap_empty(prebuilt->compress_heap); - for (i = 0; i < prebuilt->n_template; i++) { const mysql_row_templ_t*templ = &prebuilt->mysql_template[i]; const ulint field_no diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index cae3b71c2bb..9f426a8b513 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -3,6 +3,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2776,12 +2777,6 @@ files_checked: } } - /* Create the SYS_ZIP_DICT system table */ - err = dict_create_or_check_sys_zip_dict(); - if (err != DB_SUCCESS) { - return(err); - } - srv_is_being_started = FALSE; ut_a(trx_purge_state() == PURGE_STATE_INIT); From 0f8e17af92cd0126aadf283e25edcd64872b2ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 4 Jan 2017 18:16:37 +0200 Subject: [PATCH 10/13] Part 1 of MDEV-8139 Fix scrubbing tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port a bug fix from MySQL 5.7, so that all undo log pages will be freed during a slow shutdown. We cannot scrub pages that are left allocated. commit 173e171c6fb55f064eea278c76fbb28e2b1c757b Author: Thirunarayanan Balathandayuthapani Date: Fri Sep 9 18:01:27 2016 +0530 Bug #24450908 UNDO LOG EXISTS AFTER SLOW SHUTDOWN Problem: ======== 1) cached undo segment is not removed from rollback segment history (RSEG_HISTORY) during slow shutdown. In other words, If the segment is not completely free, we are failing to remove an entry from the history list. While starting the server, we traverse all rollback segment slots history list and make it as list of undo logs to be purged in purge queue. In that case, purge queue will never be empty after slow shutdown. 2) Freeing of undo log segment is linked with removing undo log header from history. Fix: ==== 1) Have separate logic of removing the undo log header from history list from rollback segment slots and remove it from rollback segment history even though it is not completely free. Reviewed-by: Debarun Banerjee Reviewed-by: Marko Mäkelä RB:13672 --- storage/innobase/fut/fut0lst.cc | 98 ------------------------------ storage/innobase/include/fut0lst.h | 25 -------- storage/innobase/trx/trx0purge.cc | 67 ++++++++++---------- storage/xtradb/fut/fut0lst.cc | 98 ------------------------------ storage/xtradb/include/fut0lst.h | 25 -------- storage/xtradb/trx/trx0purge.cc | 67 ++++++++++---------- 6 files changed, 62 insertions(+), 318 deletions(-) diff --git a/storage/innobase/fut/fut0lst.cc b/storage/innobase/fut/fut0lst.cc index 8f96a6426d2..dd3fa1238d9 100644 --- a/storage/innobase/fut/fut0lst.cc +++ b/storage/innobase/fut/fut0lst.cc @@ -338,104 +338,6 @@ flst_remove( mlog_write_ulint(base + FLST_LEN, len - 1, MLOG_4BYTES, mtr); } -/********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - ulint space; - flst_node_t* node1; - fil_addr_t node1_addr; - fil_addr_t node2_addr; - ulint len; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX)); - ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX)); - ut_ad(n_nodes > 0); - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - node1_addr = flst_get_prev_addr(node2, mtr); - - if (!fil_addr_is_null(node1_addr)) { - - /* Update next field of node1 */ - - if (node1_addr.page == node2_addr.page) { - - node1 = page_align(node2) + node1_addr.boffset; - } else { - node1 = fut_get_ptr(space, - fil_space_get_zip_size(space), - node1_addr, RW_X_LATCH, mtr); - } - - flst_write_addr(node1 + FLST_NEXT, fil_addr_null, mtr); - } else { - /* node2 was first in list: update the field in base */ - flst_write_addr(base + FLST_FIRST, fil_addr_null, mtr); - } - - flst_write_addr(base + FLST_LAST, node1_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base, mtr); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - fil_addr_t node2_addr; - ulint len; - ulint space; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX)); - ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX)); - if (n_nodes == 0) { - - ut_ad(fil_addr_is_null(flst_get_next_addr(node2, mtr))); - - return; - } - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - /* Update next field of node2 */ - flst_write_addr(node2 + FLST_NEXT, fil_addr_null, mtr); - - flst_write_addr(base + FLST_LAST, node2_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base, mtr); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - /********************************************************************//** Validates a file-based list. @return TRUE if ok */ diff --git a/storage/innobase/include/fut0lst.h b/storage/innobase/include/fut0lst.h index 90f9a65d4fa..8554cc60cdd 100644 --- a/storage/innobase/include/fut0lst.h +++ b/storage/innobase/include/fut0lst.h @@ -102,31 +102,6 @@ flst_remove( flst_node_t* node2, /*!< in: node to remove */ mtr_t* mtr); /*!< in: mini-transaction handle */ /********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** Gets list length. @return length */ UNIV_INLINE diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index efc600d16b1..42b94d9b6a3 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -281,18 +281,33 @@ trx_purge_add_update_undo_to_history( } } -/**********************************************************************//** -Frees an undo log segment which is in the history list. Cuts the end of the -history list at the youngest undo log in this segment. */ +/** Remove undo log header from the history list. +@param[in,out] rseg_hdr rollback segment header +@param[in] log_hdr undo log segment header +@param[in,out] mtr mini transaction. */ +static +void +trx_purge_remove_log_hdr( + trx_rsegf_t* rseg_hdr, + trx_ulogf_t* log_hdr, + mtr_t* mtr) +{ + flst_remove(rseg_hdr + TRX_RSEG_HISTORY, + log_hdr + TRX_UNDO_HISTORY_NODE, mtr); + + os_atomic_decrement_ulint(&trx_sys->rseg_history_len, 1); +} + +/** Frees an undo log segment which is in the history list. Removes the +undo log hdr from the history list. +@param[in,out] rseg rollback segment +@param[in] hdr_addr file address of log_hdr +@param[in] noredo skip redo logging. */ static void trx_purge_free_segment( -/*===================*/ - trx_rseg_t* rseg, /*!< in: rollback segment */ - fil_addr_t hdr_addr, /*!< in: the file address of log_hdr */ - ulint n_removed_logs) /*!< in: count of how many undo logs we - will cut off from the end of the - history list */ + trx_rseg_t* rseg, + fil_addr_t hdr_addr) { mtr_t mtr; trx_rsegf_t* rseg_hdr; @@ -356,16 +371,7 @@ trx_purge_free_segment( history list: otherwise, in case of a database crash, the segment could become inaccessible garbage in the file space. */ - flst_cut_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, n_removed_logs, &mtr); - -#ifdef HAVE_ATOMIC_BUILTINS - os_atomic_decrement_ulint(&trx_sys->rseg_history_len, n_removed_logs); -#else - mutex_enter(&trx_sys->mutex); - trx_sys->rseg_history_len -= n_removed_logs; - mutex_exit(&trx_sys->mutex); -#endif /* HAVE_ATOMIC_BUILTINS */ + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); do { @@ -407,7 +413,6 @@ trx_purge_truncate_rseg_history( page_t* undo_page; trx_ulogf_t* log_hdr; trx_usegf_t* seg_hdr; - ulint n_removed_logs = 0; mtr_t mtr; trx_id_t undo_trx_no; @@ -445,19 +450,6 @@ loop: hdr_addr.boffset, limit->undo_no); } -#ifdef HAVE_ATOMIC_BUILTINS - os_atomic_decrement_ulint( - &trx_sys->rseg_history_len, n_removed_logs); -#else - mutex_enter(&trx_sys->mutex); - trx_sys->rseg_history_len -= n_removed_logs; - mutex_exit(&trx_sys->mutex); -#endif /* HAVE_ATOMIC_BUILTINS */ - - flst_truncate_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, - n_removed_logs, &mtr); - mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); @@ -466,7 +458,6 @@ loop: prev_hdr_addr = trx_purge_get_log_from_hist( flst_get_prev_addr(log_hdr + TRX_UNDO_HISTORY_NODE, &mtr)); - n_removed_logs++; seg_hdr = undo_page + TRX_UNDO_SEG_HDR; @@ -478,10 +469,14 @@ loop: mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - trx_purge_free_segment(rseg, hdr_addr, n_removed_logs); + /* calls the trx_purge_remove_log_hdr() + inside trx_purge_free_segment(). */ + trx_purge_free_segment(rseg, hdr_addr); - n_removed_logs = 0; } else { + /* Remove the log hdr from the rseg history. */ + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); + mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); } diff --git a/storage/xtradb/fut/fut0lst.cc b/storage/xtradb/fut/fut0lst.cc index 8f96a6426d2..dd3fa1238d9 100644 --- a/storage/xtradb/fut/fut0lst.cc +++ b/storage/xtradb/fut/fut0lst.cc @@ -338,104 +338,6 @@ flst_remove( mlog_write_ulint(base + FLST_LEN, len - 1, MLOG_4BYTES, mtr); } -/********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - ulint space; - flst_node_t* node1; - fil_addr_t node1_addr; - fil_addr_t node2_addr; - ulint len; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX)); - ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX)); - ut_ad(n_nodes > 0); - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - node1_addr = flst_get_prev_addr(node2, mtr); - - if (!fil_addr_is_null(node1_addr)) { - - /* Update next field of node1 */ - - if (node1_addr.page == node2_addr.page) { - - node1 = page_align(node2) + node1_addr.boffset; - } else { - node1 = fut_get_ptr(space, - fil_space_get_zip_size(space), - node1_addr, RW_X_LATCH, mtr); - } - - flst_write_addr(node1 + FLST_NEXT, fil_addr_null, mtr); - } else { - /* node2 was first in list: update the field in base */ - flst_write_addr(base + FLST_FIRST, fil_addr_null, mtr); - } - - flst_write_addr(base + FLST_LAST, node1_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base, mtr); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - fil_addr_t node2_addr; - ulint len; - ulint space; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX)); - ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX)); - if (n_nodes == 0) { - - ut_ad(fil_addr_is_null(flst_get_next_addr(node2, mtr))); - - return; - } - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - /* Update next field of node2 */ - flst_write_addr(node2 + FLST_NEXT, fil_addr_null, mtr); - - flst_write_addr(base + FLST_LAST, node2_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base, mtr); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - /********************************************************************//** Validates a file-based list. @return TRUE if ok */ diff --git a/storage/xtradb/include/fut0lst.h b/storage/xtradb/include/fut0lst.h index 90f9a65d4fa..8554cc60cdd 100644 --- a/storage/xtradb/include/fut0lst.h +++ b/storage/xtradb/include/fut0lst.h @@ -102,31 +102,6 @@ flst_remove( flst_node_t* node2, /*!< in: node to remove */ mtr_t* mtr); /*!< in: mini-transaction handle */ /********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** Gets list length. @return length */ UNIV_INLINE diff --git a/storage/xtradb/trx/trx0purge.cc b/storage/xtradb/trx/trx0purge.cc index d9e40c5d6f5..57338a73450 100644 --- a/storage/xtradb/trx/trx0purge.cc +++ b/storage/xtradb/trx/trx0purge.cc @@ -285,18 +285,33 @@ trx_purge_add_update_undo_to_history( } } -/**********************************************************************//** -Frees an undo log segment which is in the history list. Cuts the end of the -history list at the youngest undo log in this segment. */ +/** Remove undo log header from the history list. +@param[in,out] rseg_hdr rollback segment header +@param[in] log_hdr undo log segment header +@param[in,out] mtr mini transaction. */ +static +void +trx_purge_remove_log_hdr( + trx_rsegf_t* rseg_hdr, + trx_ulogf_t* log_hdr, + mtr_t* mtr) +{ + flst_remove(rseg_hdr + TRX_RSEG_HISTORY, + log_hdr + TRX_UNDO_HISTORY_NODE, mtr); + + os_atomic_decrement_ulint(&trx_sys->rseg_history_len, 1); +} + +/** Frees an undo log segment which is in the history list. Removes the +undo log hdr from the history list. +@param[in,out] rseg rollback segment +@param[in] hdr_addr file address of log_hdr +@param[in] noredo skip redo logging. */ static void trx_purge_free_segment( -/*===================*/ - trx_rseg_t* rseg, /*!< in: rollback segment */ - fil_addr_t hdr_addr, /*!< in: the file address of log_hdr */ - ulint n_removed_logs) /*!< in: count of how many undo logs we - will cut off from the end of the - history list */ + trx_rseg_t* rseg, + fil_addr_t hdr_addr) { mtr_t mtr; trx_rsegf_t* rseg_hdr; @@ -360,16 +375,7 @@ trx_purge_free_segment( history list: otherwise, in case of a database crash, the segment could become inaccessible garbage in the file space. */ - flst_cut_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, n_removed_logs, &mtr); - -#ifdef HAVE_ATOMIC_BUILTINS - os_atomic_decrement_ulint(&trx_sys->rseg_history_len, n_removed_logs); -#else - mutex_enter(&trx_sys->mutex); - trx_sys->rseg_history_len -= n_removed_logs; - mutex_exit(&trx_sys->mutex); -#endif /* HAVE_ATOMIC_BUILTINS */ + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); do { @@ -411,7 +417,6 @@ trx_purge_truncate_rseg_history( page_t* undo_page; trx_ulogf_t* log_hdr; trx_usegf_t* seg_hdr; - ulint n_removed_logs = 0; mtr_t mtr; trx_id_t undo_trx_no; @@ -449,19 +454,6 @@ loop: hdr_addr.boffset, limit->undo_no); } -#ifdef HAVE_ATOMIC_BUILTINS - os_atomic_decrement_ulint( - &trx_sys->rseg_history_len, n_removed_logs); -#else - mutex_enter(&trx_sys->mutex); - trx_sys->rseg_history_len -= n_removed_logs; - mutex_exit(&trx_sys->mutex); -#endif /* HAVE_ATOMIC_BUILTINS */ - - flst_truncate_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, - n_removed_logs, &mtr); - mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); @@ -470,7 +462,6 @@ loop: prev_hdr_addr = trx_purge_get_log_from_hist( flst_get_prev_addr(log_hdr + TRX_UNDO_HISTORY_NODE, &mtr)); - n_removed_logs++; seg_hdr = undo_page + TRX_UNDO_SEG_HDR; @@ -482,10 +473,14 @@ loop: mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - trx_purge_free_segment(rseg, hdr_addr, n_removed_logs); + /* calls the trx_purge_remove_log_hdr() + inside trx_purge_free_segment(). */ + trx_purge_free_segment(rseg, hdr_addr); - n_removed_logs = 0; } else { + /* Remove the log hdr from the rseg history. */ + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); + mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); } From 719321e78e69249e796a907c17400dac14ef0921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 4 Jan 2017 18:43:32 +0200 Subject: [PATCH 11/13] MDEV-11638 Encryption causes race conditions in InnoDB shutdown InnoDB shutdown failed to properly take fil_crypt_thread() into account. The encryption threads were signalled to shut down together with other non-critical tasks. This could be much too early in case of slow shutdown, which could need minutes to complete the purge. Furthermore, InnoDB failed to wait for the fil_crypt_thread() to actually exit before proceeding to the final steps of shutdown, causing the race conditions. Furthermore, the log_scrub_thread() was shut down way too early. Also it should remain until the SRV_SHUTDOWN_FLUSH_PHASE. fil_crypt_threads_end(): Remove. This would cause the threads to be terminated way too early. srv_buf_dump_thread_active, srv_dict_stats_thread_active, lock_sys->timeout_thread_active, log_scrub_thread_active, srv_monitor_active, srv_error_monitor_active: Remove a race condition between startup and shutdown, by setting these in the startup thread that creates threads, not in each created thread. In this way, once the flag is cleared, it will remain cleared during shutdown. srv_n_fil_crypt_threads_started, fil_crypt_threads_event: Declare in global rather than static scope. log_scrub_event, srv_log_scrub_thread_active, log_scrub_thread(): Declare in static rather than global scope. Let these be created by log_init() and freed by log_shutdown(). rotate_thread_t::should_shutdown(): Do not shut down before the SRV_SHUTDOWN_FLUSH_PHASE. srv_any_background_threads_are_active(): Remove. These checks now exist in logs_empty_and_mark_files_at_shutdown(). logs_empty_and_mark_files_at_shutdown(): Shut down the threads in the proper order. Keep fil_crypt_thread() and log_scrub_thread() alive until SRV_SHUTDOWN_FLUSH_PHASE, and check that they actually terminate. --- storage/innobase/buf/buf0dump.cc | 10 +- storage/innobase/dict/dict0stats_bg.cc | 10 +- storage/innobase/fil/fil0crypt.cc | 35 ++-- storage/innobase/handler/ha_innodb.cc | 4 +- storage/innobase/include/fil0crypt.h | 12 +- storage/innobase/include/log0log.h | 14 +- storage/innobase/include/srv0srv.h | 24 +-- storage/innobase/lock/lock0wait.cc | 9 +- storage/innobase/log/log0log.cc | 259 ++++++++++++------------- storage/innobase/srv/srv0srv.cc | 71 +------ storage/innobase/srv/srv0start.cc | 29 +-- storage/xtradb/buf/buf0dump.cc | 10 +- storage/xtradb/dict/dict0stats_bg.cc | 10 +- storage/xtradb/fil/fil0crypt.cc | 35 ++-- storage/xtradb/handler/ha_innodb.cc | 2 - storage/xtradb/include/fil0crypt.h | 12 +- storage/xtradb/include/log0log.h | 14 +- storage/xtradb/include/srv0srv.h | 25 +-- storage/xtradb/lock/lock0wait.cc | 9 +- storage/xtradb/log/log0log.cc | 247 +++++++++++------------ storage/xtradb/srv/srv0srv.cc | 71 +------ storage/xtradb/srv/srv0start.cc | 29 +-- 22 files changed, 351 insertions(+), 590 deletions(-) diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 0abf7118b4f..934e47cea86 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -682,15 +683,10 @@ again. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(buf_dump_thread)( -/*============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(buf_dump_thread)(void*) { ut_ad(!srv_read_only_mode); - srv_buf_dump_thread_active = TRUE; - buf_dump_status(STATUS_INFO, "Dumping buffer pool(s) not yet started"); buf_load_status(STATUS_INFO, "Loading buffer pool(s) not yet started"); @@ -720,7 +716,7 @@ DECLARE_THREAD(buf_dump_thread)( keep going even if we are in a shutdown state */); } - srv_buf_dump_thread_active = FALSE; + srv_buf_dump_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 7aefa6a1d4d..45fa0fc03f0 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -525,15 +526,10 @@ statistics. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(dict_stats_thread)( -/*==============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(dict_stats_thread)(void*) { ut_a(!srv_read_only_mode); - srv_dict_stats_thread_active = TRUE; - while (!SHUTTING_DOWN()) { /* Wake up periodically even if not signaled. This is @@ -556,7 +552,7 @@ DECLARE_THREAD(dict_stats_thread)( os_event_reset(dict_stats_event); } - srv_dict_stats_thread_active = FALSE; + srv_dict_stats_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit instead of return(). */ diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 1eaabf68678..2999bea2765 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,7 +56,7 @@ UNIV_INTERN ulong srv_encrypt_tables = 0; UNIV_INTERN uint srv_n_fil_crypt_threads = 0; /** No of key rotation threads started */ -static uint srv_n_fil_crypt_threads_started = 0; +UNIV_INTERN uint srv_n_fil_crypt_threads_started = 0; /** At this age or older a space/page will be rotated */ UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; @@ -65,7 +65,7 @@ UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; static os_event_t fil_crypt_event; /** Event to signal TO the key rotation threads. */ -static os_event_t fil_crypt_threads_event; +UNIV_INTERN os_event_t fil_crypt_threads_event; /** Event for waking up threads throttle */ static os_event_t fil_crypt_throttle_sleep_event; @@ -1303,10 +1303,20 @@ struct rotate_thread_t { btr_scrub_t scrub_data; /* thread local data used by btr_scrub-functions * when iterating pages of tablespace */ - /* check if this thread should shutdown */ + /** @return whether this thread should terminate */ bool should_shutdown() const { - return ! (srv_shutdown_state == SRV_SHUTDOWN_NONE && - thread_no < srv_n_fil_crypt_threads); + switch (srv_shutdown_state) { + case SRV_SHUTDOWN_NONE: + case SRV_SHUTDOWN_CLEANUP: + return thread_no >= srv_n_fil_crypt_threads; + case SRV_SHUTDOWN_FLUSH_PHASE: + return true; + case SRV_SHUTDOWN_LAST_PHASE: + case SRV_SHUTDOWN_EXIT_THREADS: + break; + } + ut_ad(0); + return true; } }; @@ -2458,18 +2468,6 @@ fil_crypt_threads_init() } } -/********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end() -/*===================*/ -{ - if (fil_crypt_threads_inited) { - fil_crypt_set_thread_cnt(0); - } -} - /********************************************************************* Clean up key rotation threads resources */ UNIV_INTERN @@ -2480,6 +2478,7 @@ fil_crypt_threads_cleanup() if (!fil_crypt_threads_inited) { return; } + ut_a(!srv_n_fil_crypt_threads_started); os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index b283bbda7f4..625ae5f5d27 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1,10 +1,10 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -73,7 +73,6 @@ MYSQL_PLUGIN_IMPORT extern char mysql_unpacked_real_data_home[]; #include "srv0srv.h" #include "trx0roll.h" #include "trx0trx.h" - #include "trx0sys.h" #include "rem0types.h" #include "row0ins.h" @@ -248,7 +247,6 @@ static char* internal_innobase_data_file_path = NULL; static char* innodb_version_str = (char*) INNODB_VERSION_STR; -extern uint srv_n_fil_crypt_threads; extern uint srv_fil_crypt_rotate_key_age; extern uint srv_n_fil_crypt_iops; diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 9a35f6591e7..42cdafde4d0 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,6 +26,8 @@ Created 04/01/2015 Jan Lindström #ifndef fil0crypt_h #define fil0crypt_h +#include "os0sync.h" + /** * Magic pattern in start of crypt data on page 0 */ @@ -45,6 +47,8 @@ typedef enum { FIL_SPACE_ENCRYPTION_OFF = 2 /* Tablespace is not encrypted */ } fil_encryption_t; +extern os_event_t fil_crypt_threads_event; + /** * CRYPT_SCHEME_UNENCRYPTED * @@ -391,12 +395,6 @@ fil_crypt_set_thread_cnt( /*=====================*/ uint new_cnt); /*!< in: requested #threads */ -/********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end(); - /********************************************************************* Cleanup resources for threads for key rotation */ UNIV_INTERN diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 0015ea15c35..0f4cb611586 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. +Copyright (c) 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1013,22 +1014,9 @@ struct log_t{ /* @} */ #endif /* UNIV_LOG_ARCHIVE */ -extern os_event_t log_scrub_event; /* log scrubbing speed, in bytes/sec */ extern ulonglong innodb_scrub_log_speed; -/*****************************************************************//** -This is the main thread for log scrub. It waits for an event and -when waked up fills current log block with dummy records and -sleeps again. -@return this function does not return, it calls os_thread_exit() */ -extern "C" UNIV_INTERN -os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg); /*!< in: a dummy parameter - required by os_thread_create */ - #ifndef UNIV_NONINL #include "log0log.ic" #endif diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 905cc80f0df..1ba1e3ae9c6 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, 2009, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -328,6 +328,9 @@ extern char** srv_data_file_names; extern ulint* srv_data_file_sizes; extern ulint* srv_data_file_is_raw_partition; +extern uint srv_n_fil_crypt_threads; +extern uint srv_n_fil_crypt_threads_started; + extern ibool srv_auto_extend_last_data_file; extern ulint srv_last_file_size_max; extern char* srv_log_group_home_dir; @@ -478,19 +481,17 @@ extern ibool srv_print_verbose_log; "tables instead, see " REFMAN "innodb-i_s-tables.html" extern ibool srv_print_innodb_table_monitor; -extern ibool srv_monitor_active; -extern ibool srv_error_monitor_active; +extern bool srv_monitor_active; +extern bool srv_error_monitor_active; /* TRUE during the lifetime of the buffer pool dump/load thread */ -extern ibool srv_buf_dump_thread_active; +extern bool srv_buf_dump_thread_active; /* TRUE during the lifetime of the stats thread */ -extern ibool srv_dict_stats_thread_active; +extern bool srv_dict_stats_thread_active; /* TRUE if enable log scrubbing */ extern my_bool srv_scrub_log; -/* TRUE during the lifetime of the log scrub thread */ -extern ibool srv_log_scrub_thread_active; extern ulong srv_n_spin_wait_rounds; extern ulong srv_n_free_tickets_to_enter; @@ -899,15 +900,6 @@ srv_release_threads( enum srv_thread_type type, /*!< in: thread type */ ulint n); /*!< in: number of threads to release */ -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -UNIV_INTERN -const char* -srv_any_background_threads_are_active(void); -/*=======================================*/ - /**********************************************************************//** Wakeup the purge threads. */ UNIV_INTERN diff --git a/storage/innobase/lock/lock0wait.cc b/storage/innobase/lock/lock0wait.cc index c7bd223c491..8f9ea7e10aa 100644 --- a/storage/innobase/lock/lock0wait.cc +++ b/storage/innobase/lock/lock0wait.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -510,11 +511,7 @@ A thread which wakes up threads whose lock wait may have lasted too long. @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(lock_wait_timeout_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /* in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(lock_wait_timeout_thread)(void*) { ib_int64_t sig_count = 0; os_event_t event = lock_sys->timeout_event; @@ -525,8 +522,6 @@ DECLARE_THREAD(lock_wait_timeout_thread)( pfs_register_thread(srv_lock_timeout_thread_key); #endif /* UNIV_PFS_THREAD */ - lock_sys->timeout_thread_active = true; - do { srv_slot_t* slot; diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index a8326571366..3c44baaf1de 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -42,10 +42,11 @@ Created 12/9/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0flu.h" #include "srv0srv.h" +#include "lock0lock.h" #include "log0recv.h" #include "fil0fil.h" #include "dict0boot.h" -#include "srv0srv.h" +#include "dict0stats_bg.h" /* dict_stats_event */ #include "srv0start.h" #include "trx0sys.h" #include "trx0trx.h" @@ -154,6 +155,15 @@ the previous */ #define LOG_ARCHIVE_READ 1 #define LOG_ARCHIVE_WRITE 2 +/** Event to wake up the log scrub thread */ +static os_event_t log_scrub_event; + +static bool log_scrub_thread_active; + +extern "C" UNIV_INTERN +os_thread_ret_t +DECLARE_THREAD(log_scrub_thread)(void*); + /******************************************************//** Completes a checkpoint write i/o to a log file. */ static @@ -943,6 +953,12 @@ log_init(void) mutex_exit(&(log_sys->mutex)); + log_scrub_thread_active = !srv_read_only_mode && srv_scrub_log; + if (log_scrub_thread_active) { + log_scrub_event = os_event_create(); + os_thread_create(log_scrub_thread, NULL, NULL); + } + #ifdef UNIV_LOG_DEBUG recv_sys_create(); recv_sys_init(buf_pool_get_curr_size()); @@ -3232,8 +3248,6 @@ logs_empty_and_mark_files_at_shutdown(void) ulint count = 0; ulint total_trx; ulint pending_io; - enum srv_thread_type active_thd; - const char* thread_name; ibool server_busy; ib_logf(IB_LOG_LEVEL_INFO, "Starting shutdown..."); @@ -3249,29 +3263,17 @@ logs_empty_and_mark_files_at_shutdown(void) srv_shutdown_state = SRV_SHUTDOWN_CLEANUP; loop: + if (!srv_read_only_mode) { + os_event_set(srv_error_event); + os_event_set(srv_monitor_event); + os_event_set(srv_buf_dump_event); + os_event_set(lock_sys->timeout_event); + os_event_set(dict_stats_event); + } os_thread_sleep(100000); count++; - /* We need the monitor threads to stop before we proceed with - a shutdown. */ - - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - /* Print a message every 60 seconds if we are waiting - for the monitor thread to exit. Master and worker - threads check will be done later. */ - - if (srv_print_verbose_log && count > 600) { - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for %s to exit", thread_name); - count = 0; - } - - goto loop; - } - /* Check that there are no longer transactions, except for PREPARED ones. We need this wait even for the 'very fast' shutdown, because the InnoDB layer may have committed or @@ -3292,55 +3294,61 @@ loop: goto loop; } - /* Check that the background threads are suspended */ + /* We need these threads to stop early in shutdown. */ + const char* thread_name; - active_thd = srv_get_active_thread_type(); + if (srv_error_monitor_active) { + thread_name = "srv_error_monitor_thread"; + } else if (srv_monitor_active) { + thread_name = "srv_monitor_thread"; + } else if (srv_dict_stats_thread_active) { + thread_name = "dict_stats_thread"; + } else if (lock_sys->timeout_thread_active) { + thread_name = "lock_wait_timeout_thread"; + } else if (srv_buf_dump_thread_active) { + thread_name = "buf_dump_thread"; + } else { + thread_name = NULL; + } - if (active_thd != SRV_NONE) { - - if (active_thd == SRV_PURGE) { - srv_purge_wakeup(); - } - - /* The srv_lock_timeout_thread, srv_error_monitor_thread - and srv_monitor_thread should already exit by now. The - only threads to be suspended are the master threads - and worker threads (purge threads). Print the thread - type if any of such threads not in suspended mode */ + if (thread_name) { + ut_ad(!srv_read_only_mode); +wait_suspend_loop: if (srv_print_verbose_log && count > 600) { - const char* thread_type = ""; - - switch (active_thd) { - case SRV_NONE: - /* This shouldn't happen because we've - already checked for this case before - entering the if(). We handle it here - to avoid a compiler warning. */ - ut_error; - case SRV_WORKER: - thread_type = "worker threads"; - break; - case SRV_MASTER: - thread_type = "master thread"; - break; - case SRV_PURGE: - thread_type = "purge thread"; - break; - } - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for %s to be suspended", - thread_type); + "Waiting for %s to exit", thread_name); count = 0; } - goto loop; } + /* Check that the background threads are suspended */ + + switch (srv_get_active_thread_type()) { + case SRV_NONE: + srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + if (!srv_n_fil_crypt_threads_started) { + break; + } + os_event_set(fil_crypt_threads_event); + thread_name = "fil_crypt_thread"; + goto wait_suspend_loop; + case SRV_PURGE: + srv_purge_wakeup(); + thread_name = "purge thread"; + goto wait_suspend_loop; + case SRV_MASTER: + thread_name = "master thread"; + goto wait_suspend_loop; + case SRV_WORKER: + thread_name = "worker threads"; + goto wait_suspend_loop; + } + /* At this point only page_cleaner should be active. We wait here to let it complete the flushing of the buffer pools before proceeding further. */ - srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + count = 0; while (buf_page_cleaner_is_active) { ++count; @@ -3353,8 +3361,14 @@ loop: } } + if (log_scrub_thread_active) { + ut_ad(!srv_read_only_mode); + os_event_set(log_scrub_event); + } + mutex_enter(&log_sys->mutex); - server_busy = log_sys->n_pending_checkpoint_writes + server_busy = log_scrub_thread_active + || log_sys->n_pending_checkpoint_writes #ifdef UNIV_LOG_ARCHIVE || log_sys->n_pending_archive_ios #endif /* UNIV_LOG_ARCHIVE */ @@ -3373,6 +3387,8 @@ loop: goto loop; } + ut_ad(!log_scrub_thread_active); + pending_io = buf_pool_check_no_pending_io(); if (pending_io) { @@ -3408,93 +3424,67 @@ loop: from the stamps if the previous shutdown was clean. */ log_buffer_flush_to_disk(); - - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - ib_logf(IB_LOG_LEVEL_WARN, - "Background thread %s woke up " - "during shutdown", thread_name); - goto loop; - } } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; fil_close_all_files(); - - thread_name = srv_any_background_threads_are_active(); - - ut_a(!thread_name); - return; } if (!srv_read_only_mode) { log_make_checkpoint_at(LSN_MAX, TRUE); - } - mutex_enter(&log_sys->mutex); + mutex_enter(&log_sys->mutex); - lsn = log_sys->lsn; + lsn = log_sys->lsn; - if (lsn != log_sys->last_checkpoint_lsn + if (lsn != log_sys->last_checkpoint_lsn #ifdef UNIV_LOG_ARCHIVE - || (srv_log_archive_on - && lsn != log_sys->archived_lsn + LOG_BLOCK_HDR_SIZE) + || (srv_log_archive_on + && lsn != log_sys->archived_lsn + LOG_BLOCK_HDR_SIZE) +#endif /* UNIV_LOG_ARCHIVE */ + ) { + + mutex_exit(&log_sys->mutex); + + goto loop; + } + + arch_log_no = 0; + +#ifdef UNIV_LOG_ARCHIVE + UT_LIST_GET_FIRST(log_sys->log_groups)->archived_file_no; + + if (!UT_LIST_GET_FIRST(log_sys->log_groups)->archived_offset) { + + arch_log_no--; + } + + log_archive_close_groups(TRUE); #endif /* UNIV_LOG_ARCHIVE */ - ) { mutex_exit(&log_sys->mutex); - goto loop; - } - - arch_log_no = 0; - -#ifdef UNIV_LOG_ARCHIVE - UT_LIST_GET_FIRST(log_sys->log_groups)->archived_file_no; - - if (0 == UT_LIST_GET_FIRST(log_sys->log_groups)->archived_offset) { - - arch_log_no--; - } - - log_archive_close_groups(TRUE); -#endif /* UNIV_LOG_ARCHIVE */ - - mutex_exit(&log_sys->mutex); - - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - if (thread_name != NULL) { - ib_logf(IB_LOG_LEVEL_WARN, - "Background thread %s woke up during shutdown", - thread_name); - - goto loop; - } - - if (!srv_read_only_mode) { fil_flush_file_spaces(FIL_TABLESPACE); fil_flush_file_spaces(FIL_LOG); - } - /* The call fil_write_flushed_lsn_to_data_files() will pass the buffer - pool: therefore it is essential that the buffer pool has been - completely flushed to disk! (We do not call fil_write... if the - 'very fast' shutdown is enabled.) */ + /* The call fil_write_flushed_lsn_to_data_files() will + bypass the buffer pool: therefore it is essential that + the buffer pool has been completely flushed to disk! */ - if (!buf_all_freed()) { + if (!buf_all_freed()) { + if (srv_print_verbose_log && count > 600) { + ib_logf(IB_LOG_LEVEL_INFO, + "Waiting for dirty buffer pages" + " to be flushed"); + count = 0; + } - if (srv_print_verbose_log && count > 600) { - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for dirty buffer pages to be flushed"); - count = 0; + goto loop; } - - goto loop; + } else { + lsn = srv_start_lsn; } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; @@ -3739,6 +3729,10 @@ log_shutdown(void) mutex_free(&log_sys->mutex); + if (!srv_read_only_mode && srv_scrub_log) { + os_event_free(log_scrub_event); + } + #ifdef UNIV_LOG_ARCHIVE rw_lock_free(&log_sys->archive_lock); os_event_create(); @@ -3766,11 +3760,6 @@ log_mem_free(void) } } -/** Event to wake up the log scrub thread */ -UNIV_INTERN os_event_t log_scrub_event = NULL; - -UNIV_INTERN ibool srv_log_scrub_thread_active = FALSE; - /*****************************************************************//* If no log record has been written for a while, fill current log block with dummy records. */ @@ -3797,17 +3786,11 @@ sleeps again. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg __attribute__((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(log_scrub_thread)(void*) { ut_ad(!srv_read_only_mode); - srv_log_scrub_thread_active = TRUE; - - while(srv_shutdown_state == SRV_SHUTDOWN_NONE) - { + while (srv_shutdown_state < SRV_SHUTDOWN_FLUSH_PHASE) { /* log scrubbing interval in µs. */ ulonglong interval = 1000*1000*512/innodb_scrub_log_speed; @@ -3818,7 +3801,7 @@ DECLARE_THREAD(log_scrub_thread)( os_event_reset(log_scrub_event); } - srv_log_scrub_thread_active = FALSE; + log_scrub_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 1e323625b13..bf2a0a7fa36 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -60,7 +60,6 @@ Created 10/8/1995 Heikki Tuuri #include "btr0sea.h" #include "dict0load.h" #include "dict0boot.h" -#include "dict0stats_bg.h" /* dict_stats_event */ #include "srv0start.h" #include "row0mysql.h" #include "row0log.h" @@ -89,15 +88,14 @@ UNIV_INTERN ulong srv_fatal_semaphore_wait_threshold = DEFAULT_SRV_FATAL_SEMAPH in microseconds, in order to reduce the lagging of the purge thread. */ UNIV_INTERN ulint srv_dml_needed_delay = 0; -UNIV_INTERN ibool srv_monitor_active = FALSE; -UNIV_INTERN ibool srv_error_monitor_active = FALSE; +UNIV_INTERN bool srv_monitor_active; +UNIV_INTERN bool srv_error_monitor_active; -UNIV_INTERN ibool srv_buf_dump_thread_active = FALSE; +UNIV_INTERN bool srv_buf_dump_thread_active; -UNIV_INTERN ibool srv_dict_stats_thread_active = FALSE; +UNIV_INTERN bool srv_dict_stats_thread_active; -UNIV_INTERN ibool srv_log_scrub_active = FALSE; -UNIV_INTERN my_bool srv_scrub_log = FALSE; +UNIV_INTERN my_bool srv_scrub_log; UNIV_INTERN const char* srv_main_thread_op_info = ""; @@ -1687,11 +1685,7 @@ A thread which prints the info output by various InnoDB monitors. @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(srv_monitor_thread)( -/*===============================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_monitor_thread)(void*) { ib_int64_t sig_count; double time_elapsed; @@ -1712,9 +1706,7 @@ DECLARE_THREAD(srv_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_monitor_active = TRUE; - UT_NOT_USED(arg); srv_last_monitor_time = ut_time(); last_table_monitor_time = ut_time(); last_tablespace_monitor_time = ut_time(); @@ -1846,7 +1838,7 @@ loop: goto loop; exit_func: - srv_monitor_active = FALSE; + srv_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -1864,11 +1856,7 @@ we should avoid waiting any mutexes in this function! @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(srv_error_monitor_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_error_monitor_thread)(void*) { /* number of successive fatal timeouts observed */ ulint fatal_cnt = 0; @@ -1894,7 +1882,6 @@ DECLARE_THREAD(srv_error_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_error_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_error_monitor_active = TRUE; loop: /* Try to track a strange bug reported by Harald Fuchs and others, @@ -1980,7 +1967,7 @@ loop: goto loop; } - srv_error_monitor_active = FALSE; + srv_error_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -2039,44 +2026,6 @@ srv_get_active_thread_type(void) return(ret); } -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -UNIV_INTERN -const char* -srv_any_background_threads_are_active(void) -/*=======================================*/ -{ - const char* thread_active = NULL; - - if (srv_read_only_mode) { - return(NULL); - } else if (srv_error_monitor_active) { - thread_active = "srv_error_monitor_thread"; - } else if (lock_sys->timeout_thread_active) { - thread_active = "srv_lock_timeout thread"; - } else if (srv_monitor_active) { - thread_active = "srv_monitor_thread"; - } else if (srv_buf_dump_thread_active) { - thread_active = "buf_dump_thread"; - } else if (srv_dict_stats_thread_active) { - thread_active = "dict_stats_thread"; - } else if (srv_scrub_log && srv_log_scrub_thread_active) { - thread_active = "log_scrub_thread"; - } - - os_event_set(srv_error_event); - os_event_set(srv_monitor_event); - os_event_set(srv_buf_dump_event); - os_event_set(lock_sys->timeout_event); - os_event_set(dict_stats_event); - if (srv_scrub_log) - os_event_set(log_scrub_event); - - return(thread_active); -} - /*******************************************************************//** Tells the InnoDB server that there has been activity in the database and wakes up the master thread if it is suspended (not sleeping). Used diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index acfce274992..ffd97b61091 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -2356,11 +2356,6 @@ files_checked: dict_stats_thread_init(); } - if (!srv_read_only_mode && srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_init() instead? */ - log_scrub_event = os_event_create(); - } - trx_sys_file_format_init(); trx_sys_create(); @@ -2725,14 +2720,17 @@ files_checked: lock_wait_timeout_thread, NULL, thread_ids + 2 + SRV_MAX_N_IO_THREADS); thread_started[2 + SRV_MAX_N_IO_THREADS] = true; + lock_sys->timeout_thread_active = true; /* Create the thread which warns of long semaphore waits */ + srv_error_monitor_active = true; thread_handles[3 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_error_monitor_thread, NULL, thread_ids + 3 + SRV_MAX_N_IO_THREADS); thread_started[3 + SRV_MAX_N_IO_THREADS] = true; /* Create the thread which prints InnoDB monitor info */ + srv_monitor_active = true; thread_handles[4 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_monitor_thread, NULL, thread_ids + 4 + SRV_MAX_N_IO_THREADS); @@ -2965,6 +2963,8 @@ files_checked: /* Create the buffer pool dump/load thread */ buf_dump_thread_handle= os_thread_create(buf_dump_thread, NULL, NULL); + + srv_buf_dump_thread_active = true; buf_dump_thread_started = true; #ifdef WITH_WSREP } else { @@ -2975,7 +2975,9 @@ files_checked: #endif /* WITH_WSREP */ /* Create the dict stats gathering thread */ - dict_stats_thread_handle = os_thread_create(dict_stats_thread, NULL, NULL); + dict_stats_thread_handle = os_thread_create( + dict_stats_thread, NULL, NULL); + srv_dict_stats_thread_active = true; dict_stats_thread_started = true; /* Create the thread that will optimize the FTS sub-system. */ @@ -2985,10 +2987,6 @@ files_checked: fil_system_enter(); fil_crypt_threads_init(); fil_system_exit(); - - /* Create the log scrub thread */ - if (srv_scrub_log) - os_thread_create(log_scrub_thread, NULL, NULL); } /* Init data for datafile scrub threads */ @@ -3057,9 +3055,6 @@ innobase_shutdown_for_mysql(void) fts_optimize_start_shutdown(); fts_optimize_end(); - - /* Shutdown key rotation threads */ - fil_crypt_threads_end(); } /* 1. Flush the buffer pool to disk, write the current lsn to @@ -3168,14 +3163,6 @@ innobase_shutdown_for_mysql(void) if (!srv_read_only_mode) { dict_stats_thread_deinit(); - if (srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_deinit() instead? */ - os_event_free(log_scrub_event); - log_scrub_event = NULL; - } - } - - if (!srv_read_only_mode) { fil_crypt_threads_cleanup(); } diff --git a/storage/xtradb/buf/buf0dump.cc b/storage/xtradb/buf/buf0dump.cc index 114c96cec98..6abf7375775 100644 --- a/storage/xtradb/buf/buf0dump.cc +++ b/storage/xtradb/buf/buf0dump.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -682,15 +683,10 @@ again. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(buf_dump_thread)( -/*============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(buf_dump_thread)(void*) { ut_ad(!srv_read_only_mode); - srv_buf_dump_thread_active = TRUE; - buf_dump_status(STATUS_INFO, "Dumping buffer pool(s) not yet started"); buf_load_status(STATUS_INFO, "Loading buffer pool(s) not yet started"); @@ -720,7 +716,7 @@ DECLARE_THREAD(buf_dump_thread)( keep going even if we are in a shutdown state */); } - srv_buf_dump_thread_active = FALSE; + srv_buf_dump_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/xtradb/dict/dict0stats_bg.cc b/storage/xtradb/dict/dict0stats_bg.cc index 5f24bb2b742..c2265d6abd6 100644 --- a/storage/xtradb/dict/dict0stats_bg.cc +++ b/storage/xtradb/dict/dict0stats_bg.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -526,15 +527,10 @@ statistics. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(dict_stats_thread)( -/*==============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(dict_stats_thread)(void*) { ut_a(!srv_read_only_mode); - srv_dict_stats_thread_active = TRUE; - while (!SHUTTING_DOWN()) { /* Wake up periodically even if not signaled. This is @@ -557,7 +553,7 @@ DECLARE_THREAD(dict_stats_thread)( os_event_reset(dict_stats_event); } - srv_dict_stats_thread_active = FALSE; + srv_dict_stats_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit instead of return(). */ diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 1eaabf68678..2999bea2765 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,7 +56,7 @@ UNIV_INTERN ulong srv_encrypt_tables = 0; UNIV_INTERN uint srv_n_fil_crypt_threads = 0; /** No of key rotation threads started */ -static uint srv_n_fil_crypt_threads_started = 0; +UNIV_INTERN uint srv_n_fil_crypt_threads_started = 0; /** At this age or older a space/page will be rotated */ UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; @@ -65,7 +65,7 @@ UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; static os_event_t fil_crypt_event; /** Event to signal TO the key rotation threads. */ -static os_event_t fil_crypt_threads_event; +UNIV_INTERN os_event_t fil_crypt_threads_event; /** Event for waking up threads throttle */ static os_event_t fil_crypt_throttle_sleep_event; @@ -1303,10 +1303,20 @@ struct rotate_thread_t { btr_scrub_t scrub_data; /* thread local data used by btr_scrub-functions * when iterating pages of tablespace */ - /* check if this thread should shutdown */ + /** @return whether this thread should terminate */ bool should_shutdown() const { - return ! (srv_shutdown_state == SRV_SHUTDOWN_NONE && - thread_no < srv_n_fil_crypt_threads); + switch (srv_shutdown_state) { + case SRV_SHUTDOWN_NONE: + case SRV_SHUTDOWN_CLEANUP: + return thread_no >= srv_n_fil_crypt_threads; + case SRV_SHUTDOWN_FLUSH_PHASE: + return true; + case SRV_SHUTDOWN_LAST_PHASE: + case SRV_SHUTDOWN_EXIT_THREADS: + break; + } + ut_ad(0); + return true; } }; @@ -2458,18 +2468,6 @@ fil_crypt_threads_init() } } -/********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end() -/*===================*/ -{ - if (fil_crypt_threads_inited) { - fil_crypt_set_thread_cnt(0); - } -} - /********************************************************************* Clean up key rotation threads resources */ UNIV_INTERN @@ -2480,6 +2478,7 @@ fil_crypt_threads_cleanup() if (!fil_crypt_threads_inited) { return; } + ut_a(!srv_n_fil_crypt_threads_started); os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 6db23a3af2a..a361ab17a7c 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -68,7 +68,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include "srv0srv.h" #include "trx0roll.h" #include "trx0trx.h" - #include "trx0sys.h" #include "rem0types.h" #include "row0ins.h" @@ -252,7 +251,6 @@ static char* internal_innobase_data_file_path = NULL; static char* innodb_version_str = (char*) INNODB_VERSION_STR; -extern uint srv_n_fil_crypt_threads; extern uint srv_fil_crypt_rotate_key_age; extern uint srv_n_fil_crypt_iops; diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index 9a35f6591e7..42cdafde4d0 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,6 +26,8 @@ Created 04/01/2015 Jan Lindström #ifndef fil0crypt_h #define fil0crypt_h +#include "os0sync.h" + /** * Magic pattern in start of crypt data on page 0 */ @@ -45,6 +47,8 @@ typedef enum { FIL_SPACE_ENCRYPTION_OFF = 2 /* Tablespace is not encrypted */ } fil_encryption_t; +extern os_event_t fil_crypt_threads_event; + /** * CRYPT_SCHEME_UNENCRYPTED * @@ -391,12 +395,6 @@ fil_crypt_set_thread_cnt( /*=====================*/ uint new_cnt); /*!< in: requested #threads */ -/********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end(); - /********************************************************************* Cleanup resources for threads for key rotation */ UNIV_INTERN diff --git a/storage/xtradb/include/log0log.h b/storage/xtradb/include/log0log.h index 470274761fc..ec3ecbd1d66 100644 --- a/storage/xtradb/include/log0log.h +++ b/storage/xtradb/include/log0log.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. +Copyright (c) 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1070,22 +1071,9 @@ struct log_t{ /* @} */ #endif /* UNIV_LOG_ARCHIVE */ -extern os_event_t log_scrub_event; /* log scrubbing speed, in bytes/sec */ extern ulonglong innodb_scrub_log_speed; -/*****************************************************************//** -This is the main thread for log scrub. It waits for an event and -when waked up fills current log block with dummy records and -sleeps again. -@return this function does not return, it calls os_thread_exit() */ -extern "C" UNIV_INTERN -os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg); /*!< in: a dummy parameter - required by os_thread_create */ - #ifndef UNIV_NONINL #include "log0log.ic" #endif diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index e2d141b4140..f3b6098a875 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, 2009, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -349,7 +349,6 @@ extern char** srv_data_file_names; extern ulint* srv_data_file_sizes; extern ulint* srv_data_file_is_raw_partition; - /** Whether the redo log tracking is currently enabled. Note that it is possible for the log tracker thread to be running and the tracking to be disabled */ @@ -359,6 +358,9 @@ extern ulonglong srv_max_bitmap_file_size; extern ulonglong srv_max_changed_pages; +extern uint srv_n_fil_crypt_threads; +extern uint srv_n_fil_crypt_threads_started; + extern ibool srv_auto_extend_last_data_file; extern ulint srv_last_file_size_max; extern char* srv_log_group_home_dir; @@ -577,19 +579,17 @@ extern ibool srv_print_verbose_log; "tables instead, see " REFMAN "innodb-i_s-tables.html" extern ibool srv_print_innodb_table_monitor; -extern ibool srv_monitor_active; -extern ibool srv_error_monitor_active; +extern bool srv_monitor_active; +extern bool srv_error_monitor_active; /* TRUE during the lifetime of the buffer pool dump/load thread */ -extern ibool srv_buf_dump_thread_active; +extern bool srv_buf_dump_thread_active; /* TRUE during the lifetime of the stats thread */ -extern ibool srv_dict_stats_thread_active; +extern bool srv_dict_stats_thread_active; /* TRUE if enable log scrubbing */ extern my_bool srv_scrub_log; -/* TRUE during the lifetime of the log scrub thread */ -extern ibool srv_log_scrub_thread_active; extern ulong srv_n_spin_wait_rounds; extern ulong srv_n_free_tickets_to_enter; @@ -1085,15 +1085,6 @@ srv_release_threads( enum srv_thread_type type, /*!< in: thread type */ ulint n); /*!< in: number of threads to release */ -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -UNIV_INTERN -const char* -srv_any_background_threads_are_active(void); -/*=======================================*/ - /**********************************************************************//** Wakeup the purge threads. */ UNIV_INTERN diff --git a/storage/xtradb/lock/lock0wait.cc b/storage/xtradb/lock/lock0wait.cc index c7bd223c491..8f9ea7e10aa 100644 --- a/storage/xtradb/lock/lock0wait.cc +++ b/storage/xtradb/lock/lock0wait.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -510,11 +511,7 @@ A thread which wakes up threads whose lock wait may have lasted too long. @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(lock_wait_timeout_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /* in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(lock_wait_timeout_thread)(void*) { ib_int64_t sig_count = 0; os_event_t event = lock_sys->timeout_event; @@ -525,8 +522,6 @@ DECLARE_THREAD(lock_wait_timeout_thread)( pfs_register_thread(srv_lock_timeout_thread_key); #endif /* UNIV_PFS_THREAD */ - lock_sys->timeout_thread_active = true; - do { srv_slot_t* slot; diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index f3a3486017c..9245ae160e6 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -52,9 +52,11 @@ Created 12/9/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0flu.h" #include "srv0srv.h" +#include "lock0lock.h" #include "log0recv.h" #include "fil0fil.h" #include "dict0boot.h" +#include "dict0stats_bg.h" /* dict_stats_event */ #include "srv0srv.h" #include "srv0start.h" #include "trx0sys.h" @@ -95,6 +97,10 @@ UNIV_INTERN log_t* log_sys = NULL; UNIV_INTERN log_checksum_func_t log_checksum_algorithm_ptr = log_block_calc_checksum_innodb; +extern "C" UNIV_INTERN +os_thread_ret_t +DECLARE_THREAD(log_scrub_thread)(void*); + /* Next log block number to do dummy record filling if no log records written for a while */ static ulint next_lbn_to_pad = 0; @@ -167,6 +173,11 @@ the previous */ #define LOG_ARCHIVE_READ 1 #define LOG_ARCHIVE_WRITE 2 +/** Event to wake up the log scrub thread */ +static os_event_t log_scrub_event; + +static bool log_scrub_thread_active; + /******************************************************//** Completes a checkpoint write i/o to a log file. */ static @@ -1049,6 +1060,12 @@ log_init(void) mutex_exit(&(log_sys->mutex)); + log_scrub_thread_active = !srv_read_only_mode && srv_scrub_log; + if (log_scrub_thread_active) { + log_scrub_event = os_event_create(); + os_thread_create(log_scrub_thread, NULL, NULL); + } + #ifdef UNIV_LOG_DEBUG recv_sys_create(); recv_sys_init(buf_pool_get_curr_size()); @@ -3545,8 +3562,6 @@ logs_empty_and_mark_files_at_shutdown(void) ulint count = 0; ulint total_trx; ulint pending_io; - enum srv_thread_type active_thd; - const char* thread_name; ibool server_busy; ib_logf(IB_LOG_LEVEL_INFO, "Starting shutdown..."); @@ -3566,29 +3581,17 @@ logs_empty_and_mark_files_at_shutdown(void) srv_shutdown_state = SRV_SHUTDOWN_CLEANUP; loop: + if (!srv_read_only_mode) { + os_event_set(srv_error_event); + os_event_set(srv_monitor_event); + os_event_set(srv_buf_dump_event); + os_event_set(lock_sys->timeout_event); + os_event_set(dict_stats_event); + } os_thread_sleep(100000); count++; - /* We need the monitor threads to stop before we proceed with - a shutdown. */ - - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - /* Print a message every 60 seconds if we are waiting - for the monitor thread to exit. Master and worker - threads check will be done later. */ - - if (srv_print_verbose_log && count > 600) { - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for %s to exit", thread_name); - count = 0; - } - - goto loop; - } - /* Check that there are no longer transactions, except for PREPARED ones. We need this wait even for the 'very fast' shutdown, because the InnoDB layer may have committed or @@ -3609,55 +3612,61 @@ loop: goto loop; } - /* Check that the background threads are suspended */ + /* We need these threads to stop early in shutdown. */ + const char* thread_name; - active_thd = srv_get_active_thread_type(); + if (srv_error_monitor_active) { + thread_name = "srv_error_monitor_thread"; + } else if (srv_monitor_active) { + thread_name = "srv_monitor_thread"; + } else if (srv_dict_stats_thread_active) { + thread_name = "dict_stats_thread"; + } else if (lock_sys->timeout_thread_active) { + thread_name = "lock_wait_timeout_thread"; + } else if (srv_buf_dump_thread_active) { + thread_name = "buf_dump_thread"; + } else { + thread_name = NULL; + } - if (active_thd != SRV_NONE) { - - if (active_thd == SRV_PURGE) { - srv_purge_wakeup(); - } - - /* The srv_lock_timeout_thread, srv_error_monitor_thread - and srv_monitor_thread should already exit by now. The - only threads to be suspended are the master threads - and worker threads (purge threads). Print the thread - type if any of such threads not in suspended mode */ + if (thread_name) { + ut_ad(!srv_read_only_mode); +wait_suspend_loop: if (srv_print_verbose_log && count > 600) { - const char* thread_type = ""; - - switch (active_thd) { - case SRV_NONE: - /* This shouldn't happen because we've - already checked for this case before - entering the if(). We handle it here - to avoid a compiler warning. */ - ut_error; - case SRV_WORKER: - thread_type = "worker threads"; - break; - case SRV_MASTER: - thread_type = "master thread"; - break; - case SRV_PURGE: - thread_type = "purge thread"; - break; - } - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for %s to be suspended", - thread_type); + "Waiting for %s to exit", thread_name); count = 0; } - goto loop; } + /* Check that the background threads are suspended */ + + switch (srv_get_active_thread_type()) { + case SRV_NONE: + srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + if (!srv_n_fil_crypt_threads_started) { + break; + } + os_event_set(fil_crypt_threads_event); + thread_name = "fil_crypt_thread"; + goto wait_suspend_loop; + case SRV_PURGE: + srv_purge_wakeup(); + thread_name = "purge thread"; + goto wait_suspend_loop; + case SRV_MASTER: + thread_name = "master thread"; + goto wait_suspend_loop; + case SRV_WORKER: + thread_name = "worker threads"; + goto wait_suspend_loop; + } + /* At this point only page_cleaner should be active. We wait here to let it complete the flushing of the buffer pools before proceeding further. */ - srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + count = 0; while (buf_page_cleaner_is_active || buf_lru_manager_is_active) { if (srv_print_verbose_log && count == 0) { @@ -3672,8 +3681,14 @@ loop: } } + if (log_scrub_thread_active) { + ut_ad(!srv_read_only_mode); + os_event_set(log_scrub_event); + } + mutex_enter(&log_sys->mutex); - server_busy = log_sys->n_pending_checkpoint_writes + server_busy = log_scrub_thread_active + || log_sys->n_pending_checkpoint_writes #ifdef UNIV_LOG_ARCHIVE || log_sys->n_pending_archive_ios #endif /* UNIV_LOG_ARCHIVE */ @@ -3692,6 +3707,8 @@ loop: goto loop; } + ut_ad(!log_scrub_thread_active); + pending_io = buf_pool_check_no_pending_io(); if (pending_io) { @@ -3727,16 +3744,6 @@ loop: from the stamps if the previous shutdown was clean. */ log_buffer_flush_to_disk(); - - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - ib_logf(IB_LOG_LEVEL_WARN, - "Background thread %s woke up " - "during shutdown", thread_name); - goto loop; - } } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; @@ -3749,74 +3756,57 @@ loop: } fil_close_all_files(); - - thread_name = srv_any_background_threads_are_active(); - - ut_a(!thread_name); - return; } if (!srv_read_only_mode) { log_make_checkpoint_at(LSN_MAX, TRUE); - } - mutex_enter(&log_sys->mutex); + mutex_enter(&log_sys->mutex); - tracked_lsn = log_get_tracked_lsn(); + tracked_lsn = log_get_tracked_lsn(); - lsn = log_sys->lsn; + lsn = log_sys->lsn; - if (lsn != log_sys->last_checkpoint_lsn - || (srv_track_changed_pages - && (tracked_lsn != log_sys->last_checkpoint_lsn)) + if (lsn != log_sys->last_checkpoint_lsn + || (srv_track_changed_pages + && (tracked_lsn != log_sys->last_checkpoint_lsn)) #ifdef UNIV_LOG_ARCHIVE - || (srv_log_archive_on - && lsn != log_sys->archived_lsn + LOG_BLOCK_HDR_SIZE) + || (srv_log_archive_on + && lsn != log_sys->archived_lsn + LOG_BLOCK_HDR_SIZE) +#endif /* UNIV_LOG_ARCHIVE */ + ) { + + mutex_exit(&log_sys->mutex); + + goto loop; + } + +#ifdef UNIV_LOG_ARCHIVE + log_archive_close_groups(TRUE); #endif /* UNIV_LOG_ARCHIVE */ - ) { mutex_exit(&log_sys->mutex); - goto loop; - } - -#ifdef UNIV_LOG_ARCHIVE - - log_archive_close_groups(TRUE); -#endif /* UNIV_LOG_ARCHIVE */ - - mutex_exit(&log_sys->mutex); - - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - if (thread_name != NULL) { - ib_logf(IB_LOG_LEVEL_WARN, - "Background thread %s woke up during shutdown", - thread_name); - - goto loop; - } - - if (!srv_read_only_mode) { fil_flush_file_spaces(FIL_TABLESPACE); fil_flush_file_spaces(FIL_LOG); - } - /* The call fil_write_flushed_lsn_to_data_files() will pass the buffer - pool: therefore it is essential that the buffer pool has been - completely flushed to disk! (We do not call fil_write... if the - 'very fast' shutdown is enabled.) */ + /* The call fil_write_flushed_lsn_to_data_files() will + bypass the buffer pool: therefore it is essential that + the buffer pool has been completely flushed to disk! */ - if (!buf_all_freed()) { + if (!buf_all_freed()) { + if (srv_print_verbose_log && count > 600) { + ib_logf(IB_LOG_LEVEL_INFO, + "Waiting for dirty buffer pages" + " to be flushed"); + count = 0; + } - if (srv_print_verbose_log && count > 600) { - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for dirty buffer pages to be flushed"); - count = 0; + goto loop; } - - goto loop; + } else { + lsn = srv_start_lsn; } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; @@ -4094,6 +4084,10 @@ log_shutdown(void) mutex_free(&log_sys->mutex); mutex_free(&log_sys->log_flush_order_mutex); + if (!srv_read_only_mode && srv_scrub_log) { + os_event_free(log_scrub_event); + } + #ifdef UNIV_LOG_ARCHIVE rw_lock_free(&log_sys->archive_lock); os_event_free(log_sys->archiving_on); @@ -4121,11 +4115,6 @@ log_mem_free(void) } } -/** Event to wake up the log scrub thread */ -UNIV_INTERN os_event_t log_scrub_event = NULL; - -UNIV_INTERN ibool srv_log_scrub_thread_active = FALSE; - /*****************************************************************//* If no log record has been written for a while, fill current log block with dummy records. */ @@ -4152,17 +4141,11 @@ sleeps again. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg __attribute__((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(log_scrub_thread)(void*) { ut_ad(!srv_read_only_mode); - srv_log_scrub_thread_active = TRUE; - - while(srv_shutdown_state == SRV_SHUTDOWN_NONE) - { + while (srv_shutdown_state < SRV_SHUTDOWN_FLUSH_PHASE) { /* log scrubbing interval in µs. */ ulonglong interval = 1000*1000*512/innodb_scrub_log_speed; @@ -4173,7 +4156,7 @@ DECLARE_THREAD(log_scrub_thread)( os_event_reset(log_scrub_event); } - srv_log_scrub_thread_active = FALSE; + log_scrub_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index eff680a4ef0..bc222b864ef 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -61,7 +61,6 @@ Created 10/8/1995 Heikki Tuuri #include "btr0sea.h" #include "dict0load.h" #include "dict0boot.h" -#include "dict0stats_bg.h" /* dict_stats_event */ #include "srv0start.h" #include "row0mysql.h" #include "row0log.h" @@ -107,15 +106,14 @@ UNIV_INTERN long long srv_kill_idle_transaction = 0; in microseconds, in order to reduce the lagging of the purge thread. */ UNIV_INTERN ulint srv_dml_needed_delay = 0; -UNIV_INTERN ibool srv_monitor_active = FALSE; -UNIV_INTERN ibool srv_error_monitor_active = FALSE; +UNIV_INTERN bool srv_monitor_active; +UNIV_INTERN bool srv_error_monitor_active; -UNIV_INTERN ibool srv_buf_dump_thread_active = FALSE; +UNIV_INTERN bool srv_buf_dump_thread_active; -UNIV_INTERN ibool srv_dict_stats_thread_active = FALSE; +UNIV_INTERN bool srv_dict_stats_thread_active; -UNIV_INTERN ibool srv_log_scrub_active = FALSE; -UNIV_INTERN my_bool srv_scrub_log = FALSE; +UNIV_INTERN my_bool srv_scrub_log; UNIV_INTERN const char* srv_main_thread_op_info = ""; @@ -2100,11 +2098,7 @@ A thread which prints the info output by various InnoDB monitors. @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(srv_monitor_thread)( -/*===============================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_monitor_thread)(void*) { ib_int64_t sig_count; double time_elapsed; @@ -2125,9 +2119,7 @@ DECLARE_THREAD(srv_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_monitor_active = TRUE; - UT_NOT_USED(arg); srv_last_monitor_time = ut_time(); last_table_monitor_time = ut_time(); last_tablespace_monitor_time = ut_time(); @@ -2259,7 +2251,7 @@ loop: goto loop; exit_func: - srv_monitor_active = FALSE; + srv_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -2277,11 +2269,7 @@ we should avoid waiting any mutexes in this function! @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(srv_error_monitor_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_error_monitor_thread)(void*) { /* number of successive fatal timeouts observed */ ulint fatal_cnt = 0; @@ -2307,7 +2295,6 @@ DECLARE_THREAD(srv_error_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_error_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_error_monitor_active = TRUE; loop: /* Try to track a strange bug reported by Harald Fuchs and others, @@ -2423,7 +2410,7 @@ rescan_idle: goto loop; } - srv_error_monitor_active = FALSE; + srv_error_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -2487,44 +2474,6 @@ srv_get_active_thread_type(void) return(ret); } -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -UNIV_INTERN -const char* -srv_any_background_threads_are_active(void) -/*=======================================*/ -{ - const char* thread_active = NULL; - - if (srv_read_only_mode) { - return(NULL); - } else if (srv_error_monitor_active) { - thread_active = "srv_error_monitor_thread"; - } else if (lock_sys->timeout_thread_active) { - thread_active = "srv_lock_timeout thread"; - } else if (srv_monitor_active) { - thread_active = "srv_monitor_thread"; - } else if (srv_buf_dump_thread_active) { - thread_active = "buf_dump_thread"; - } else if (srv_dict_stats_thread_active) { - thread_active = "dict_stats_thread"; - } else if (srv_scrub_log && srv_log_scrub_thread_active) { - thread_active = "log_scrub_thread"; - } - - os_event_set(srv_error_event); - os_event_set(srv_monitor_event); - os_event_set(srv_buf_dump_event); - os_event_set(lock_sys->timeout_event); - os_event_set(dict_stats_event); - if (srv_scrub_log) - os_event_set(log_scrub_event); - - return(thread_active); -} - /******************************************************************//** A thread which follows the redo log and outputs the changed page bitmap. @return a dummy value */ diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 8f8cf265041..31421c7a605 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -2435,11 +2435,6 @@ files_checked: dict_stats_thread_init(); } - if (!srv_read_only_mode && srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_init() instead? */ - log_scrub_event = os_event_create(); - } - trx_sys_file_format_init(); trx_sys_create(); @@ -2782,14 +2777,17 @@ files_checked: lock_wait_timeout_thread, NULL, thread_ids + 2 + SRV_MAX_N_IO_THREADS); thread_started[2 + SRV_MAX_N_IO_THREADS] = true; + lock_sys->timeout_thread_active = true; /* Create the thread which warns of long semaphore waits */ + srv_error_monitor_active = true; thread_handles[3 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_error_monitor_thread, NULL, thread_ids + 3 + SRV_MAX_N_IO_THREADS); thread_started[3 + SRV_MAX_N_IO_THREADS] = true; /* Create the thread which prints InnoDB monitor info */ + srv_monitor_active = true; thread_handles[4 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_monitor_thread, NULL, thread_ids + 4 + SRV_MAX_N_IO_THREADS); @@ -3033,6 +3031,8 @@ files_checked: /* Create the buffer pool dump/load thread */ buf_dump_thread_handle= os_thread_create(buf_dump_thread, NULL, NULL); + + srv_buf_dump_thread_active = true; buf_dump_thread_started = true; #ifdef WITH_WSREP } else { @@ -3043,7 +3043,9 @@ files_checked: #endif /* WITH_WSREP */ /* Create the dict stats gathering thread */ - dict_stats_thread_handle = os_thread_create(dict_stats_thread, NULL, NULL); + dict_stats_thread_handle = os_thread_create( + dict_stats_thread, NULL, NULL); + srv_dict_stats_thread_active = true; dict_stats_thread_started = true; /* Create the thread that will optimize the FTS sub-system. */ @@ -3053,10 +3055,6 @@ files_checked: fil_system_enter(); fil_crypt_threads_init(); fil_system_exit(); - - /* Create the log scrub thread */ - if (srv_scrub_log) - os_thread_create(log_scrub_thread, NULL, NULL); } /* Init data for datafile scrub threads */ @@ -3125,9 +3123,6 @@ innobase_shutdown_for_mysql(void) fts_optimize_start_shutdown(); fts_optimize_end(); - - /* Shutdown key rotation threads */ - fil_crypt_threads_end(); } /* 1. Flush the buffer pool to disk, write the current lsn to @@ -3230,14 +3225,6 @@ innobase_shutdown_for_mysql(void) if (!srv_read_only_mode) { dict_stats_thread_deinit(); - if (srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_deinit() instead? */ - os_event_free(log_scrub_event); - log_scrub_event = NULL; - } - } - - if (!srv_read_only_mode) { fil_crypt_threads_cleanup(); } From ffb38c9771bc7248482dc6a3d2f26ca10fe09a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 4 Jan 2017 20:49:13 +0200 Subject: [PATCH 12/13] MDEV-8139 Fix scrubbing tests encryption.innodb_scrub: Clean up. Make it also cover ROW_FORMAT=COMPRESSED, removing the need for encryption.innodb_scrub_compressed. Add a FIXME comment saying that we should create a secondary index, to demonstrate that also undo log pages get scrubbed. Currently that is not working! Also clean up encryption.innodb_scrub_background, but keep it disabled, because the background scrubbing does not work reliably. Fix both tests so that if something is not scrubbed, the test will be aborted, so that the data files will be preserved. Allow the tests to run on Windows as well. --- mysql-test/include/search_pattern_in_file.inc | 6 + mysql-test/suite/encryption/disabled.def | 6 +- .../suite/encryption/r/innodb_scrub.result | 238 ++++++------------ .../r/innodb_scrub_background.result | 169 +++++++++---- .../r/innodb_scrub_compressed.result | 71 ------ .../suite/encryption/t/innodb_scrub.test | 233 +++++++++-------- .../encryption/t/innodb_scrub_background.test | 152 +++++------ .../encryption/t/innodb_scrub_compressed.opt | 9 - .../encryption/t/innodb_scrub_compressed.test | 161 ------------ 9 files changed, 398 insertions(+), 647 deletions(-) delete mode 100644 mysql-test/suite/encryption/r/innodb_scrub_compressed.result delete mode 100644 mysql-test/suite/encryption/t/innodb_scrub_compressed.opt delete mode 100644 mysql-test/suite/encryption/t/innodb_scrub_compressed.test diff --git a/mysql-test/include/search_pattern_in_file.inc b/mysql-test/include/search_pattern_in_file.inc index 3280dbfd574..f77a7c60916 100644 --- a/mysql-test/include/search_pattern_in_file.inc +++ b/mysql-test/include/search_pattern_in_file.inc @@ -82,8 +82,14 @@ perl; } $ENV{'SEARCH_FILE'} =~ s{^.*?([^/\\]+)$}{$1}; if ($content =~ m{$search_pattern}) { + die "FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + if $ENV{SEARCH_ABORT} eq 'FOUND'; print "FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + unless defined $ENV{SEARCH_ABORT}; } else { + die "NOT FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + if $ENV{SEARCH_ABORT} eq 'NOT FOUND'; print "NOT FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + unless defined $ENV{SEARCH_ABORT}; } EOF diff --git a/mysql-test/suite/encryption/disabled.def b/mysql-test/suite/encryption/disabled.def index 8c0d47983fd..348e4c979ea 100644 --- a/mysql-test/suite/encryption/disabled.def +++ b/mysql-test/suite/encryption/disabled.def @@ -10,8 +10,4 @@ # ############################################################################## -innodb_scrub : MDEV-8139 -innodb_scrub_compressed : MDEV-8139 -innodb_scrub_background : MDEV-8139 -innodb_encryption-page-compression : Fails with lost connection at line 156 - +innodb_scrub_background : MDEV-8139 background scrubbing does not work reliably diff --git a/mysql-test/suite/encryption/r/innodb_scrub.result b/mysql-test/suite/encryption/r/innodb_scrub.result index 95f0aed3226..6e8febc762d 100644 --- a/mysql-test/suite/encryption/r/innodb_scrub.result +++ b/mysql-test/suite/encryption/r/innodb_scrub.result @@ -1,207 +1,121 @@ create table snapshot_status engine = myisam select * from information_schema.global_status where variable_name like 'innodb_scrub%'; -# -# Test delete of records -# -create table t1 ( +# MDEV-8139 Fix scrubbing tests +# FIXME: Add index(b) to each table; ensure that undo logs are scrubbed. +create table delete_3 ( +a int auto_increment primary key, +b varchar(256), +c text) engine = innodb row_format=compressed; +delete from delete_3; +create table delete_rollback_delete_3 ( +a int auto_increment primary key, +b varchar(256), +c text) engine = innodb row_format=compressed; +begin; +delete from delete_rollback_delete_3; +rollback; +delete from delete_rollback_delete_3; +create table insert_rollback_3 ( +a int auto_increment primary key, +b varchar(256), +c text) engine = innodb row_format=compressed; +begin; +rollback; +create table delete_2 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=compact; -# Populate table with rows -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# compact: delete from: grep -c bicycle t1.ibd -0 -# compact: delete from: grep -c bicycle ibdata1 -0 -# compact: delete from: grep -c repairman t1.ibd -0 -# compact: delete from: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete+rollback+delete -# -create table t1 ( +delete from delete_2; +create table delete_rollback_delete_2 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=compact; -# Populate table with rows begin; -delete from t1; +delete from delete_rollback_delete_2; rollback; -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# compact: delete rollback: grep -c bicycle t1.ibd -0 -# compact: delete rollback: grep -c bicycle ibdata1 -0 -# compact: delete rollback: grep -c repairman t1.ibd -0 -# compact: delete rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test insert+rollback -# -create table t1 ( +delete from delete_rollback_delete_2; +create table insert_rollback_2 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=compact; -# Populate table with rows begin; rollback; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# compact: insert rollback: grep -c bicycle t1.ibd -0 -# compact: insert rollback: grep -c bicycle ibdata1 -0 -# compact: insert rollback: grep -c repairman t1.ibd -0 -# compact: insert rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete of records -# -create table t1 ( +create table delete_1 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=redundant; -# Populate table with rows -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# redundant: delete from: grep -c bicycle t1.ibd -0 -# redundant: delete from: grep -c bicycle ibdata1 -0 -# redundant: delete from: grep -c repairman t1.ibd -0 -# redundant: delete from: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete+rollback+delete -# -create table t1 ( +delete from delete_1; +create table delete_rollback_delete_1 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=redundant; -# Populate table with rows begin; -delete from t1; +delete from delete_rollback_delete_1; rollback; -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# redundant: delete rollback: grep -c bicycle t1.ibd -0 -# redundant: delete rollback: grep -c bicycle ibdata1 -0 -# redundant: delete rollback: grep -c repairman t1.ibd -0 -# redundant: delete rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test insert+rollback -# -create table t1 ( +delete from delete_rollback_delete_1; +create table insert_rollback_1 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=redundant; -# Populate table with rows begin; rollback; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# redundant: insert rollback: grep -c bicycle t1.ibd -0 -# redundant: insert rollback: grep -c bicycle ibdata1 -0 -# redundant: insert rollback: grep -c repairman t1.ibd -0 -# redundant: insert rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete of records -# -create table t1 ( +create table delete_0 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=dynamic; -# Populate table with rows -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# dynamic: delete from: grep -c bicycle t1.ibd -0 -# dynamic: delete from: grep -c bicycle ibdata1 -0 -# dynamic: delete from: grep -c repairman t1.ibd -0 -# dynamic: delete from: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete+rollback+delete -# -create table t1 ( +delete from delete_0; +create table delete_rollback_delete_0 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=dynamic; -# Populate table with rows begin; -delete from t1; +delete from delete_rollback_delete_0; rollback; -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# dynamic: delete rollback: grep -c bicycle t1.ibd -0 -# dynamic: delete rollback: grep -c bicycle ibdata1 -0 -# dynamic: delete rollback: grep -c repairman t1.ibd -0 -# dynamic: delete rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test insert+rollback -# -create table t1 ( +delete from delete_rollback_delete_0; +create table insert_rollback_0 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=dynamic; -# Populate table with rows begin; rollback; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# dynamic: insert rollback: grep -c bicycle t1.ibd -0 -# dynamic: insert rollback: grep -c bicycle ibdata1 -0 -# dynamic: insert rollback: grep -c repairman t1.ibd -0 -# dynamic: insert rollback: grep -c repairman ibdata1 -0 -drop table t1; +SET GLOBAL innodb_fast_shutdown=0; +# delete_3.ibd +# delete_rollback_delete_3.ibd +# insert_rollback_3.ibd +# delete_2.ibd +# delete_rollback_delete_2.ibd +# insert_rollback_2.ibd +# delete_1.ibd +# delete_rollback_delete_1.ibd +# insert_rollback_1.ibd +# delete_0.ibd +# delete_rollback_delete_0.ibd +# insert_rollback_0.ibd +check table delete_3, delete_rollback_delete_3, insert_rollback_3; +Table Op Msg_type Msg_text +test.delete_3 check status OK +test.delete_rollback_delete_3 check status OK +test.insert_rollback_3 check status OK +drop table delete_3, delete_rollback_delete_3, insert_rollback_3; +check table delete_2, delete_rollback_delete_2, insert_rollback_2; +Table Op Msg_type Msg_text +test.delete_2 check status OK +test.delete_rollback_delete_2 check status OK +test.insert_rollback_2 check status OK +drop table delete_2, delete_rollback_delete_2, insert_rollback_2; +check table delete_1, delete_rollback_delete_1, insert_rollback_1; +Table Op Msg_type Msg_text +test.delete_1 check status OK +test.delete_rollback_delete_1 check status OK +test.insert_rollback_1 check status OK +drop table delete_1, delete_rollback_delete_1, insert_rollback_1; +check table delete_0, delete_rollback_delete_0, insert_rollback_0; +Table Op Msg_type Msg_text +test.delete_0 check status OK +test.delete_rollback_delete_0 check status OK +test.insert_rollback_0 check status OK +drop table delete_0, delete_rollback_delete_0, insert_rollback_0; show variables like 'innodb_%scrub_data%'; Variable_name Value innodb_background_scrub_data_check_interval 3600 diff --git a/mysql-test/suite/encryption/r/innodb_scrub_background.result b/mysql-test/suite/encryption/r/innodb_scrub_background.result index 1ff6c0bda98..6a2f263d0a1 100644 --- a/mysql-test/suite/encryption/r/innodb_scrub_background.result +++ b/mysql-test/suite/encryption/r/innodb_scrub_background.result @@ -11,66 +11,146 @@ innodb_background_scrub_data_uncompressed ON innodb_immediate_scrub_data_uncompressed OFF # make sure spaces are checked quickly SET GLOBAL innodb_background_scrub_data_check_interval=1; -create table snapshot_status engine = myisam -select * from information_schema.global_status -where variable_name like 'innodb_scrub%'; -truncate table snapshot_status; -insert into snapshot_status -select * from information_schema.global_status -where variable_name like 'innodb_scrub%'; -# -# Test delete of records -# -create table t1 ( +create table delete_3 ( a int auto_increment primary key, b varchar(256), -c text, index(b)) engine = innodb row_format=dynamic; +c text, +index(b)) engine = innodb row_format=compressed; # Populate table with rows -delete from t1; -# -# Test delete+rollback+delete -# -create table t2 ( +delete from delete_3; +create table delete_rollback_delete_3 ( a int auto_increment primary key, b varchar(256), -c text, index(b)) engine = innodb row_format=dynamic; +c text, +index(b)) engine = innodb row_format=compressed; # Populate table with rows begin; -delete from t2; +delete from delete_rollback_delete_3; rollback; -delete from t2; -# -# Test insert+rollback -# -create table t3 ( +delete from delete_rollback_delete_3; +create table insert_rollback_3 ( a int auto_increment primary key, b varchar(256), -c text, index(b)) engine = innodb row_format=dynamic; +c text, +index(b)) engine = innodb row_format=compressed; +# Populate table with rows +begin; +rollback; +create table delete_2 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=compact; +# Populate table with rows +delete from delete_2; +create table delete_rollback_delete_2 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=compact; +# Populate table with rows +begin; +delete from delete_rollback_delete_2; +rollback; +delete from delete_rollback_delete_2; +create table insert_rollback_2 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=compact; +# Populate table with rows +begin; +rollback; +create table delete_1 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=redundant; +# Populate table with rows +delete from delete_1; +create table delete_rollback_delete_1 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=redundant; +# Populate table with rows +begin; +delete from delete_rollback_delete_1; +rollback; +delete from delete_rollback_delete_1; +create table insert_rollback_1 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=redundant; +# Populate table with rows +begin; +rollback; +create table delete_0 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=dynamic; +# Populate table with rows +delete from delete_0; +create table delete_rollback_delete_0 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=dynamic; +# Populate table with rows +begin; +delete from delete_rollback_delete_0; +rollback; +delete from delete_rollback_delete_0; +create table insert_rollback_0 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=dynamic; # Populate table with rows begin; rollback; # start scrubbing threads SET GLOBAL innodb_encryption_threads=5; # Wait max 10 min for scrubbing -# Success! -# stop scrubbing threads -SET GLOBAL innodb_encryption_threads=0; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# dynamic: delete: grep -c bicycle t1.ibd -0 -# dynamic: delete: grep -c repairman t1.ibd -0 -# dynamic: delete rollback: grep -c bicycle t2.ibd -0 -# dynamic: delete rollback: grep -c repairman t2.ibd -0 -# dynamic: insert rollback: grep -c bicycle t3.ibd -0 -# dynamic: insert rollback: grep -c repairman t3.ibd -0 -drop table t1, t2, t3; +SET GLOBAL innodb_fast_shutdown=0; +# delete_3.ibd +# delete_rollback_delete_3.ibd +# insert_rollback_3.ibd +# delete_2.ibd +# delete_rollback_delete_2.ibd +# insert_rollback_2.ibd +# delete_1.ibd +# delete_rollback_delete_1.ibd +# insert_rollback_1.ibd +# delete_0.ibd +# delete_rollback_delete_0.ibd +# insert_rollback_0.ibd +check table delete_3, delete_rollback_delete_3, insert_rollback_3; +Table Op Msg_type Msg_text +test.delete_3 check status OK +test.delete_rollback_delete_3 check status OK +test.insert_rollback_3 check status OK +drop table delete_3, delete_rollback_delete_3, insert_rollback_3; +check table delete_2, delete_rollback_delete_2, insert_rollback_2; +Table Op Msg_type Msg_text +test.delete_2 check status OK +test.delete_rollback_delete_2 check status OK +test.insert_rollback_2 check status OK +drop table delete_2, delete_rollback_delete_2, insert_rollback_2; +check table delete_1, delete_rollback_delete_1, insert_rollback_1; +Table Op Msg_type Msg_text +test.delete_1 check status OK +test.delete_rollback_delete_1 check status OK +test.insert_rollback_1 check status OK +drop table delete_1, delete_rollback_delete_1, insert_rollback_1; +check table delete_0, delete_rollback_delete_0, insert_rollback_0; +Table Op Msg_type Msg_text +test.delete_0 check status OK +test.delete_rollback_delete_0 check status OK +test.insert_rollback_0 check status OK +drop table delete_0, delete_rollback_delete_0, insert_rollback_0; show variables like 'innodb_%scrub_data%'; Variable_name Value innodb_background_scrub_data_check_interval 3600 @@ -78,4 +158,3 @@ innodb_background_scrub_data_compressed ON innodb_background_scrub_data_interval 604800 innodb_background_scrub_data_uncompressed ON innodb_immediate_scrub_data_uncompressed OFF -drop table snapshot_status; diff --git a/mysql-test/suite/encryption/r/innodb_scrub_compressed.result b/mysql-test/suite/encryption/r/innodb_scrub_compressed.result deleted file mode 100644 index 0b5e9f11a05..00000000000 --- a/mysql-test/suite/encryption/r/innodb_scrub_compressed.result +++ /dev/null @@ -1,71 +0,0 @@ -# make sure spaces are checked quickly -SET GLOBAL innodb_background_scrub_data_check_interval=1; -# -# Test delete of records -# -create table t1 ( -a int auto_increment primary key, -b varchar(256), -c text) engine = innodb row_format=compressed; -# Populate table with rows -delete from t1; -# -# Test delete+rollback+delete -# -create table t2 ( -a int auto_increment primary key, -b varchar(256), -c text) engine = innodb row_format=compressed; -# Populate table with rows -begin; -delete from t2; -rollback; -delete from t2; -# -# Test insert+rollback -# -create table t3 ( -a int auto_increment primary key, -b varchar(256), -c text) engine = innodb row_format=compressed; -# Populate table with rows -begin; -rollback; -# start scrubbing threads -SET GLOBAL innodb_encryption_threads=5; -# Wait max 10 min for scrubbing of this table -# Success! -# stop scrubbing threads -SET GLOBAL innodb_encryption_threads=0; -# Now there should be background scrubs -# restart mysqld so that all pages are flushed (encryption off) -# so that grep will find stuff -# read all rows from table -select * from t1; -select * from t2; -select * from t3; -# grep -c bicycle t1.ibd -0 -# grep -c bicycle ibdata1 -0 -# grep -c repairman t1.ibd -0 -# grep -c repairman ibdata1 -0 -# grep -c boondoggle t2.ibd -0 -# grep -c boondoggle ibdata1 -0 -# grep -c waste t2.ibd -0 -# grep -c waste ibdata1 -0 -# grep -c keso t3.ibd -0 -# grep -c keso ibdata1 -0 -# grep -c kent t3.ibd -0 -# grep -c kent ibdata1 -0 -drop table t1, t2, t3; diff --git a/mysql-test/suite/encryption/t/innodb_scrub.test b/mysql-test/suite/encryption/t/innodb_scrub.test index 32170567b4c..6cb48530f86 100644 --- a/mysql-test/suite/encryption/t/innodb_scrub.test +++ b/mysql-test/suite/encryption/t/innodb_scrub.test @@ -1,147 +1,138 @@ -- source include/have_innodb.inc -- source include/not_embedded.inc -- source include/have_example_key_management_plugin.inc --- source include/not_windows.inc let $MYSQLD_DATADIR=`select @@datadir`; -let ib1_IBD = $MYSQLD_DATADIR/ibdata1; -let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd; +let INNODB_PAGE_SIZE= `select @@innodb_page_size`; create table snapshot_status engine = myisam select * from information_schema.global_status where variable_name like 'innodb_scrub%'; let $rowcount=500; -let $formatno = 3; +let $maxformatno= 4; +let $formatno= $maxformatno; + +--echo # MDEV-8139 Fix scrubbing tests +--echo # FIXME: Add index(b) to each table; ensure that undo logs are scrubbed. +let $tableformat= ( + a int auto_increment primary key, + b varchar(256), + c text) engine = innodb row_format; + while ($formatno) { +dec $formatno; let $format = `select case $formatno - when 1 then 'dynamic' - when 2 then 'redundant' - when 3 then 'compact' + when 0 then 'dynamic' + when 1 then 'redundant' + when 2 then 'compact' + when 3 then 'compressed' end`; + +let $t= delete_$formatno; +eval create table $t $tableformat=$format; + +let $numinserts = $rowcount; +--disable_query_log +begin; +while ($numinserts) +{ + dec $numinserts; + eval insert into $t(b,c) values ('repairman', repeat('unicycle', 1000)); +} +commit; +--enable_query_log + +eval delete from $t; + +let $t= delete_rollback_delete_$formatno; +eval create table $t $tableformat=$format; + +let $numinserts = $rowcount; +--disable_query_log +begin; +while ($numinserts) +{ + dec $numinserts; + eval insert into $t(b,c) values ('breakhuman', repeat('bicycle', 1000)); +} +commit; +--enable_query_log + +begin; +eval delete from $t; +rollback; +eval delete from $t; + +let $t= insert_rollback_$formatno; + +eval create table $t $tableformat=$format; + +let $numinserts = $rowcount; +begin; +--disable_query_log +while ($numinserts) +{ + dec $numinserts; + eval insert into $t(b,c) values ('wonderwoman', repeat('tricycle', 1000)); +} +--enable_query_log + +rollback; +} + +SET GLOBAL innodb_fast_shutdown=0; +-- source include/shutdown_mysqld.inc + +let SEARCH_ABORT= FOUND; +let SEARCH_PATTERN= (un|b|tr)icycle|(repair|breakhu|wonderwo)man; +let SEARCH_RANGE= 12582912; +let SEARCH_FILE= $MYSQLD_DATADIR/ibdata1; + +# We may randomly find copies of unscrubbed pages in the doublewrite buffer. +# Let us scrub the doublewrite buffer ourselves. +perl; +use Fcntl 'SEEK_SET'; +my $page_size = $ENV{INNODB_PAGE_SIZE}; +open(FILE, "+<", "$ENV{SEARCH_FILE}") or die "cannot open: $!\n"; +seek(FILE, $page_size * 64, SEEK_SET) or die "cannot seek: $!\n"; +print(FILE chr(0) x ($page_size * 128)) or die "cannot write: $!\n"; +close FILE or die "cannot close: $!\n";; +EOF + +-- source include/search_pattern_in_file.inc + +let $formatno= $maxformatno; +while ($formatno) +{ dec $formatno; --- echo # --- echo # Test delete of records --- echo # - -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=$format; - -let $numinserts = $rowcount; --- echo # Populate table with rows ---disable_query_log -while ($numinserts) -{ - dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); +let $t= delete_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +let $t= delete_rollback_delete_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +let $t= insert_rollback_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc } ---enable_query_log -delete from t1; +-- source include/start_mysqld.inc --- echo # restart mysqld so that all pages are flushed --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; --- enable_result_log - --- echo # $format: delete from: grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # $format: delete from: grep -c bicycle ibdata1 --- exec grep -c bicycle $ib1_IBD || true --- echo # $format: delete from: grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true --- echo # $format: delete from: grep -c repairman ibdata1 --- exec grep -c repairman $ib1_IBD || true - -drop table t1; - --- echo # --- echo # Test delete+rollback+delete --- echo # - -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=$format; - -let $numinserts = $rowcount; --- echo # Populate table with rows ---disable_query_log -while ($numinserts) +let $formatno= $maxformatno; +while ($formatno) { - dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); -} ---enable_query_log +dec $formatno; -begin; -delete from t1; -rollback; -delete from t1; +let $t= delete_$formatno, delete_rollback_delete_$formatno, insert_rollback_$formatno; --- echo # restart mysqld so that all pages are flushed --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; --- enable_result_log - --- echo # $format: delete rollback: grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # $format: delete rollback: grep -c bicycle ibdata1 --- exec grep -c bicycle $ib1_IBD || true --- echo # $format: delete rollback: grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true --- echo # $format: delete rollback: grep -c repairman ibdata1 --- exec grep -c repairman $ib1_IBD || true - -drop table t1; - --- echo # --- echo # Test insert+rollback --- echo # - -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=$format; - -let $numinserts = $rowcount; --- echo # Populate table with rows -begin; ---disable_query_log -while ($numinserts) -{ - dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); -} ---enable_query_log - -rollback; - --- echo # restart mysqld so that all pages are flushed --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; --- enable_result_log - --- echo # $format: insert rollback: grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # $format: insert rollback: grep -c bicycle ibdata1 --- exec grep -c bicycle $ib1_IBD || true --- echo # $format: insert rollback: grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true --- echo # $format: insert rollback: grep -c repairman ibdata1 --- exec grep -c repairman $ib1_IBD || true - -drop table t1; +eval check table $t; +eval drop table $t; } show variables like 'innodb_%scrub_data%'; diff --git a/mysql-test/suite/encryption/t/innodb_scrub_background.test b/mysql-test/suite/encryption/t/innodb_scrub_background.test index 19e56683117..c705ee51006 100644 --- a/mysql-test/suite/encryption/t/innodb_scrub_background.test +++ b/mysql-test/suite/encryption/t/innodb_scrub_background.test @@ -1,13 +1,9 @@ -- source include/have_innodb.inc -- source include/not_embedded.inc -- source include/have_example_key_management_plugin.inc --- source include/not_windows.inc let $MYSQLD_DATADIR=`select @@datadir`; -let ib1_IBD = $MYSQLD_DATADIR/ibdata1; -let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd; -let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; -let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; +let INNODB_PAGE_SIZE= `select @@innodb_page_size`; --echo # --echo # immediate scrubbing is off @@ -18,80 +14,67 @@ show variables like 'innodb_%scrub_data%'; -- echo # make sure spaces are checked quickly SET GLOBAL innodb_background_scrub_data_check_interval=1; -create table snapshot_status engine = myisam -select * from information_schema.global_status -where variable_name like 'innodb_scrub%'; - let $rowcount=500; -let $formatno = 1; +let $maxformatno= 4; +let $formatno= $maxformatno; + +let $tableformat= ( + a int auto_increment primary key, + b varchar(256), + c text, + index(b)) engine = innodb row_format; + while ($formatno) { -let $format = `select case $formatno - when 1 then 'dynamic' - when 2 then 'redundant' - when 3 then 'compact' - when 4 then 'compressed' - end`; dec $formatno; +let $format = `select case $formatno + when 0 then 'dynamic' + when 1 then 'redundant' + when 2 then 'compact' + when 3 then 'compressed' + end`; -truncate table snapshot_status; -insert into snapshot_status -select * from information_schema.global_status -where variable_name like 'innodb_scrub%'; - --- echo # --- echo # Test delete of records --- echo # - -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text, index(b)) engine = innodb row_format=$format; +let $t= delete_$formatno; +eval create table $t $tableformat=$format; let $numinserts = $rowcount; -- echo # Populate table with rows --disable_query_log +begin; while ($numinserts) { dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); + eval insert into $t(b,c) values ('unicycle', repeat('wonderwoman', 1000)); } +commit; --enable_query_log -delete from t1; +eval delete from $t; --- echo # --- echo # Test delete+rollback+delete --- echo # +let $t= delete_rollback_delete_$formatno; -eval create table t2 ( - a int auto_increment primary key, - b varchar(256), - c text, index(b)) engine = innodb row_format=$format; +eval create table $t $tableformat=$format; let $numinserts = $rowcount; -- echo # Populate table with rows --disable_query_log +begin; while ($numinserts) { dec $numinserts; - insert into t2(b,c) values ('bicycle', repeat('repairman', 1000)); + eval insert into $t(b,c) values ('bicycle', repeat('repairman', 1000)); } +commit; --enable_query_log begin; -delete from t2; +eval delete from $t; rollback; -delete from t2; +eval delete from $t; --- echo # --- echo # Test insert+rollback --- echo # +let $t= insert_rollback_$formatno; -eval create table t3 ( - a int auto_increment primary key, - b varchar(256), - c text, index(b)) engine = innodb row_format=$format; +eval create table $t $tableformat=$format; let $numinserts = $rowcount; -- echo # Populate table with rows @@ -100,11 +83,12 @@ begin; while ($numinserts) { dec $numinserts; - insert into t3(b,c) values ('bicycle', repeat('repairman', 1000)); + eval insert into $t(b,c) values ('tricycle', repeat('superhuman', 1000)); } --enable_query_log rollback; +} -- echo # start scrubbing threads SET GLOBAL innodb_encryption_threads=5; @@ -130,35 +114,57 @@ if (!$success) -- die Timeout waiting for background threads } --- echo # Success! --- echo # stop scrubbing threads -SET GLOBAL innodb_encryption_threads=0; +SET GLOBAL innodb_fast_shutdown=0; +-- source include/shutdown_mysqld.inc --- echo # restart mysqld so that all pages are flushed --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; --- enable_result_log +let SEARCH_ABORT= FOUND; +let SEARCH_PATTERN= (un|b|tr)icycle|(repair|breakhu|wonderwo)man; +let SEARCH_RANGE= 12582912; +let SEARCH_FILE= $MYSQLD_DATADIR/ibdata1; --- echo # $format: delete: grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # $format: delete: grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true +# We may randomly find copies of unscrubbed pages in the doublewrite buffer. +# Let us scrub the doublewrite buffer ourselves. +perl; +use Fcntl 'SEEK_SET'; +my $page_size = $ENV{INNODB_PAGE_SIZE}; +open(FILE, "+<", "$ENV{SEARCH_FILE}") or die "cannot open: $!\n"; +seek(FILE, $page_size * 64, SEEK_SET) or die "cannot seek: $!\n"; +print(FILE chr(0) x ($page_size * 128)) or die "cannot write: $!\n"; +close FILE or die "cannot close: $!\n";; +EOF --- echo # $format: delete rollback: grep -c bicycle t2.ibd --- exec grep -c bicycle $t2_IBD || true --- echo # $format: delete rollback: grep -c repairman t2.ibd --- exec grep -c repairman $t2_IBD || true +-- source include/search_pattern_in_file.inc --- echo # $format: insert rollback: grep -c bicycle t3.ibd --- exec grep -c bicycle $t3_IBD || true --- echo # $format: insert rollback: grep -c repairman t3.ibd --- exec grep -c repairman $t3_IBD || true +let $formatno= $maxformatno; +while ($formatno) +{ +dec $formatno; -drop table t1, t2, t3; +let $t= delete_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +let $t= delete_rollback_delete_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +let $t= insert_rollback_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +} + +-- source include/start_mysqld.inc + +let $formatno= $maxformatno; +while ($formatno) +{ +dec $formatno; + +let $t= delete_$formatno, delete_rollback_delete_$formatno, insert_rollback_$formatno; + +eval check table $t; +eval drop table $t; } show variables like 'innodb_%scrub_data%'; - -drop table snapshot_status; diff --git a/mysql-test/suite/encryption/t/innodb_scrub_compressed.opt b/mysql-test/suite/encryption/t/innodb_scrub_compressed.opt deleted file mode 100644 index 48b04fa7049..00000000000 --- a/mysql-test/suite/encryption/t/innodb_scrub_compressed.opt +++ /dev/null @@ -1,9 +0,0 @@ ---innodb-file-per-table=1 ---innodb-file-format=Barracuda ---innodb-immediate-scrub-data-uncompressed=ON ---innodb-background-scrub-data-uncompressed=ON ---innodb-background-scrub-data-compressed=ON ---loose-innodb-debug-force-scrubbing=ON ---innodb-encrypt-tables=OFF ---innodb-encrypt-log=OFF ---innodb-tablespaces-scrubbing diff --git a/mysql-test/suite/encryption/t/innodb_scrub_compressed.test b/mysql-test/suite/encryption/t/innodb_scrub_compressed.test deleted file mode 100644 index d41edac7494..00000000000 --- a/mysql-test/suite/encryption/t/innodb_scrub_compressed.test +++ /dev/null @@ -1,161 +0,0 @@ --- source include/have_innodb.inc --- source include/not_embedded.inc --- source include/have_example_key_management_plugin.inc --- source include/not_windows.inc - -let $MYSQLD_DATADIR=`select @@datadir`; -let ib1_IBD = $MYSQLD_DATADIR/ibdata1; -let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd; -let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; -let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; - -let $rowcount=500; - --- echo # make sure spaces are checked quickly -SET GLOBAL innodb_background_scrub_data_check_interval=1; - --- echo # --- echo # Test delete of records --- echo # - -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=compressed; - -let $numinserts = $rowcount; --- echo # Populate table with rows ---disable_query_log -while ($numinserts) -{ - dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); -} ---enable_query_log - -delete from t1; - --- echo # --- echo # Test delete+rollback+delete --- echo # - -eval create table t2 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=compressed; - -let $numinserts = $rowcount; --- echo # Populate table with rows ---disable_query_log -while ($numinserts) -{ - dec $numinserts; - insert into t2(b,c) values ('boondoggle', repeat('waste of time', 1000)); -} ---enable_query_log - -begin; -delete from t2; -rollback; -delete from t2; - --- echo # --- echo # Test insert+rollback --- echo # - -eval create table t3 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=compressed; - -let $numinserts = $rowcount; --- echo # Populate table with rows -begin; ---disable_query_log -while ($numinserts) -{ - dec $numinserts; - insert into t3(b,c) values ('keso', repeat('kent', 1000)); -} ---enable_query_log - -rollback; - --- echo # start scrubbing threads -SET GLOBAL innodb_encryption_threads=5; --- echo # Wait max 10 min for scrubbing of this table -let $cnt=600; -while ($cnt) -{ - let $success=`SELECT COUNT(*) = 0 -FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_SCRUBBING -WHERE LAST_SCRUB_COMPLETED IS NULL AND ( NAME like 'test/%' OR SPACE = 0 )`; - - if ($success) - { - let $cnt=0; - } - if (!$success) - { - real_sleep 1; - dec $cnt; - } -} -if (!$success) -{ - SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_SCRUBBING; - SHOW STATUS LIKE 'innodb_%scrub%'; - -- die Timeout waiting for background threads -} --- echo # Success! --- echo # stop scrubbing threads -SET GLOBAL innodb_encryption_threads=0; - ---echo # Now there should be background scrubs -let $success=`select sum(variable_value) > 0 -from information_schema.global_status -where variable_name in ('innodb_scrub_background_page_reorganizations', -'innodb_scrub_background_page_splits')`; - -if (!$success) { - show status like 'innodb_scrub%'; -} - --- echo # restart mysqld so that all pages are flushed (encryption off) --- echo # so that grep will find stuff --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; -select * from t2; -select * from t3; --- enable_result_log - --- echo # grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # grep -c bicycle ibdata1 --- exec grep -c bicycle $ib1_IBD || true --- echo # grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true --- echo # grep -c repairman ibdata1 --- exec grep -c repairman $ib1_IBD || true - --- echo # grep -c boondoggle t2.ibd --- exec grep -c boondoggle $t2_IBD || true --- echo # grep -c boondoggle ibdata1 --- exec grep -c boondoggle $ib1_IBD || true --- echo # grep -c waste t2.ibd --- exec grep -c waste $t2_IBD || true --- echo # grep -c waste ibdata1 --- exec grep -c waste $ib1_IBD || true - --- echo # grep -c keso t3.ibd --- exec grep -c keso $t3_IBD || true --- echo # grep -c keso ibdata1 --- exec grep -c keso $ib1_IBD || true --- echo # grep -c kent t3.ibd --- exec grep -c kent $t3_IBD || true --- echo # grep -c kent ibdata1 --- exec grep -c kent $ib1_IBD || true - -drop table t1, t2, t3; From 758af98ff7c47cc1fb5debdc138312fa389d528f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 5 Jan 2017 10:42:19 +0200 Subject: [PATCH 13/13] Post-push fix for Part 1 of MDEV-8139 Fix scrubbing tests In the backport of Bug#24450908 UNDO LOG EXISTS AFTER SLOW SHUTDOWN from MySQL 5.7 to the MySQL 5.6 based MariaDB Server 10.1, we must use a mutex when HAVE_ATOMIC_BUILTINS is not defined. Also, correct a function comment. In MySQL 5.6 and MariaDB Server 10.1, also temporary InnoDB tables are redo-logged. --- storage/innobase/trx/trx0purge.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 42b94d9b6a3..b3d16546a63 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -294,15 +294,19 @@ trx_purge_remove_log_hdr( { flst_remove(rseg_hdr + TRX_RSEG_HISTORY, log_hdr + TRX_UNDO_HISTORY_NODE, mtr); - +#ifdef HAVE_ATOMIC_BUILTINS os_atomic_decrement_ulint(&trx_sys->rseg_history_len, 1); +#else + mutex_enter(&trx_sys->mutex); + --trx_sys->rseg_history_len; + mutex_exit(&trx_sys->mutex); +#endif } /** Frees an undo log segment which is in the history list. Removes the undo log hdr from the history list. @param[in,out] rseg rollback segment -@param[in] hdr_addr file address of log_hdr -@param[in] noredo skip redo logging. */ +@param[in] hdr_addr file address of log_hdr */ static void trx_purge_free_segment(