From 0ca3250e5910e5e636cbf774027747c3835fe616 Mon Sep 17 00:00:00 2001 From: "aelkin/elkin@koti.dsl.inet.fi" <> Date: Fri, 21 Dec 2007 22:34:43 +0200 Subject: [PATCH] BUG#30435 loading large LOAD DATA INFILE breaks slave with read_buffer_size set on master BUG#33413 show binlog events fails if binlog has event size of close to max_allowed_packet The size of Append_block replication event was determined solely by read_buffer_size whereas the rest of replication code deals with max_allowed_packet. When the former parameter was set to larger than the latter there were two artifacts: the master could not read events from binlog; show master events did not show. Fixed with - fragmenting the used io-cached buffer into pieces each size of less than max_allowed_packet (bug#30435) - incrementing show-binlog-events handling thread's max_allowed_packet with the max estimated for the replication header size --- include/my_sys.h | 6 +++ mysql-test/r/rpl_loaddata_map.result | 31 +++++++++++ mysql-test/t/rpl_loaddata_map-master.opt | 1 + mysql-test/t/rpl_loaddata_map-slave.opt | 1 + mysql-test/t/rpl_loaddata_map.test | 51 ++++++++++++++++++ sql/sql_repl.cc | 69 +++++++++++++++--------- 6 files changed, 134 insertions(+), 25 deletions(-) create mode 100644 mysql-test/r/rpl_loaddata_map.result create mode 100644 mysql-test/t/rpl_loaddata_map-master.opt create mode 100644 mysql-test/t/rpl_loaddata_map-slave.opt create mode 100644 mysql-test/t/rpl_loaddata_map.test diff --git a/include/my_sys.h b/include/my_sys.h index 4f7e75a836e..b18df822b40 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -530,6 +530,12 @@ typedef int (*qsort2_cmp)(const void *, const void *, const void *); #define my_b_tell(info) ((info)->pos_in_file + \ (uint) (*(info)->current_pos - (info)->request_pos)) +#define my_b_get_buffer_start(info) (info)->request_pos +#define my_b_get_bytes_in_buffer(info) (char*) (info)->read_end - \ + (char*) my_b_get_buffer_start(info) +#define my_b_get_pos_in_file(info) (info)->pos_in_file + + /* tell write offset in the SEQ_APPEND cache */ my_off_t my_b_append_tell(IO_CACHE* info); my_off_t my_b_safe_tell(IO_CACHE* info); /* picks the correct tell() */ diff --git a/mysql-test/r/rpl_loaddata_map.result b/mysql-test/r/rpl_loaddata_map.result new file mode 100644 index 00000000000..11e07e06234 --- /dev/null +++ b/mysql-test/r/rpl_loaddata_map.result @@ -0,0 +1,31 @@ +stop slave; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; +reset master; +reset slave; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; +start slave; +create table t2 (id int not null primary key auto_increment); +show variables like 'max_allowed_packet' /* 8K */; +Variable_name Value +max_allowed_packet 7168 +show variables like 'read_buffer_size' /* 9K */; +Variable_name Value +read_buffer_size 8228 +load data infile '/home/elkin/MySQL/TEAM/FIXES/5.0/bug33435-load_data_read_buffer_size/mysql-test/var/tmp/bug30435_5k.txt' into table t2; +select count(*) from t2 /* 5 000 */; +count(*) +5000 +show binlog events in 'master-bin.000002' from 98; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 98 Query 1 # use `test`; create table t2 (id int not null primary key auto_increment) +master-bin.000002 221 Begin_load_query 1 # ;file_id=1;block_len=7168 +master-bin.000002 7412 Append_block 1 # ;file_id=1;block_len=7168 +master-bin.000002 14603 Append_block 1 # ;file_id=1;block_len=2048 +master-bin.000002 16674 Append_block 1 # ;file_id=1;block_len=7168 +master-bin.000002 23865 Append_block 1 # ;file_id=1;block_len=341 +master-bin.000002 24229 Execute_load_query 1 # use `test`; load data infile '/home/elkin/MySQL/TEAM/FIXES/5.0/bug33435-load_data_read_buffer_size/mysql-test/var/tmp/bug30435_5k.txt' into table t2 ;file_id=1 +select count(*) from t2 /* 5 000 */; +count(*) +5000 +drop table t1, t2; +end of the tests diff --git a/mysql-test/t/rpl_loaddata_map-master.opt b/mysql-test/t/rpl_loaddata_map-master.opt new file mode 100644 index 00000000000..f3c797b19c8 --- /dev/null +++ b/mysql-test/t/rpl_loaddata_map-master.opt @@ -0,0 +1 @@ +--read_buffer_size=9K --max_allowed_packet=8K diff --git a/mysql-test/t/rpl_loaddata_map-slave.opt b/mysql-test/t/rpl_loaddata_map-slave.opt new file mode 100644 index 00000000000..95f55bcf7d8 --- /dev/null +++ b/mysql-test/t/rpl_loaddata_map-slave.opt @@ -0,0 +1 @@ +--max_allowed_packet=8K diff --git a/mysql-test/t/rpl_loaddata_map.test b/mysql-test/t/rpl_loaddata_map.test new file mode 100644 index 00000000000..17a127cc5b6 --- /dev/null +++ b/mysql-test/t/rpl_loaddata_map.test @@ -0,0 +1,51 @@ +# +# check replication of load data with the server parameters subjected to +# read_buffer_size > max_allowed_packet +# +# BUG#30435 loading large LOAD DATA INFILE breaks slave with +# read_buffer_size set on master +# BUG#33413 show binlog events fails if binlog has event size of close +# to max_allowed_packet + +source include/master-slave.inc; +source include/have_innodb.inc; + +--disable_query_log +let $rows= 5000; +create table t1 (id int not null primary key auto_increment); + +while($rows) +{ + eval insert into t1 values (null); + dec $rows; +} +eval select * into outfile '$MYSQLTEST_VARDIR/tmp/bug30435_5k.txt' from t1; +flush logs; +--enable_query_log + +connection master; +create table t2 (id int not null primary key auto_increment); + +show variables like 'max_allowed_packet' /* 8K */; +show variables like 'read_buffer_size' /* 9K */; + +eval load data infile '$MYSQLTEST_VARDIR/tmp/bug30435_5k.txt' into table t2; +select count(*) from t2 /* 5 000 */; + +# the binglog will show fragmented Append_block events +--let $binlog_start=98 +--replace_column 5 # +--replace_regex /\/\* xid=.* \*\//\/* XID *\// +--eval show binlog events in 'master-bin.000002' from $binlog_start + + +sync_slave_with_master; +#connection slave; +select count(*) from t2 /* 5 000 */; + +connection master; +drop table t1, t2; +sync_slave_with_master; +remove_file $MYSQLTEST_VARDIR/tmp/bug30435_5k.txt; + +--echo end of the tests diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 903d254db8f..c1ba33ce27d 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -1355,6 +1355,11 @@ bool mysql_show_binlog_events(THD* thd) if ((file=open_binlog(&log, linfo.log_file_name, &errmsg)) < 0) goto err; + /* + to account binlog event header size + */ + thd->variables.max_allowed_packet += MAX_LOG_EVENT_HEADER; + pthread_mutex_lock(log_lock); /* @@ -1365,7 +1370,6 @@ bool mysql_show_binlog_events(THD* thd) This code will fail on a mixed relay log (one which has Format_desc then Rotate then Format_desc). */ - ev = Log_event::read_log_event(&log,(pthread_mutex_t*)0,description_event); if (ev) { @@ -1556,37 +1560,52 @@ err: DBUG_RETURN(TRUE); } - +/** + Load data's io cache specific hook to be executed + before a chunk of data is being read into the cache's buffer + The fuction instantianates and writes into the binlog + replication events along LOAD DATA processing. + + @param file pointer to io-cache + @return 0 +*/ int log_loaded_block(IO_CACHE* file) { + DBUG_ENTER("log_loaded_block"); LOAD_FILE_INFO *lf_info; - uint block_len ; - - /* file->request_pos contains position where we started last read */ - char* buffer = (char*) file->request_pos; - if (!(block_len = (char*) file->read_end - (char*) buffer)) - return 0; - lf_info = (LOAD_FILE_INFO*) file->arg; + uint block_len; + /* buffer contains position where we started last read */ + char* buffer= my_b_get_buffer_start(file); + uint max_event_size= current_thd->variables.max_allowed_packet; + lf_info= (LOAD_FILE_INFO*) file->arg; if (lf_info->last_pos_in_file != HA_POS_ERROR && - lf_info->last_pos_in_file >= file->pos_in_file) + lf_info->last_pos_in_file >= my_b_get_pos_in_file(file)) return 0; - lf_info->last_pos_in_file = file->pos_in_file; - if (lf_info->wrote_create_file) + + for (block_len= my_b_get_bytes_in_buffer(file); block_len > 0; + buffer += min(block_len, max_event_size), + block_len -= min(block_len, max_event_size)) { - Append_block_log_event a(lf_info->thd, lf_info->thd->db, buffer, - block_len, lf_info->log_delayed); - mysql_bin_log.write(&a); + lf_info->last_pos_in_file= my_b_get_pos_in_file(file); + if (lf_info->wrote_create_file) + { + Append_block_log_event a(lf_info->thd, lf_info->thd->db, buffer, + min(block_len, max_event_size), + lf_info->log_delayed); + mysql_bin_log.write(&a); + } + else + { + Begin_load_query_log_event b(lf_info->thd, lf_info->thd->db, + buffer, + min(block_len, max_event_size), + lf_info->log_delayed); + mysql_bin_log.write(&b); + lf_info->wrote_create_file= 1; + DBUG_SYNC_POINT("debug_lock.created_file_event",10); + } } - else - { - Begin_load_query_log_event b(lf_info->thd, lf_info->thd->db, - buffer, block_len, - lf_info->log_delayed); - mysql_bin_log.write(&b); - lf_info->wrote_create_file = 1; - DBUG_SYNC_POINT("debug_lock.created_file_event",10); - } - return 0; + DBUG_RETURN(0); } #endif /* HAVE_REPLICATION */