From f35b4218eaa2bf831ab682b3674851a1cc6ee65a Mon Sep 17 00:00:00 2001 From: Georgi Kodinov Date: Fri, 13 Mar 2009 15:51:25 +0200 Subject: [PATCH 1/2] Bug #22047 : Time in SHOW PROCESSLIST for SQL thread in replication seems to become negative THD::start_time has a dual meaning : it's either the time since the process entered a given state or is the transaction time returned by e.g. NOW(). This causes problems, as sometimes THD::start_time may be set to a value that is correct and needed when used as a base for NOW(), but these times may be arbitrary (SET @@timestamp) or non-local (coming from the master through the replication feed). If one such non-local time is set there's no way to return a correct value for e.g. SHOW PROCESSLIST or SELECT ... FROM INFORMATION_SCHEMA.PROCESSLIST. Fixed by making the Time column in SHOW PROCESSLIST SIGNED LONG instead of UNSIGNED LONG and doing the correct conversions. Note that no reliable test suite can be constructed, since it would require knowing the local time and can't be achieved by the means of the current test suite. sql/sql_show.cc: Bug #22047: make the Time in SHOW PROCESSLIST LONG from LONG UNSIGNED --- sql/sql_show.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/sql_show.cc b/sql/sql_show.cc index a3ccf770a3c..fee3d076436 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -1338,7 +1338,8 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose) field_list.push_back(field=new Item_empty_string("db",NAME_LEN)); field->maybe_null=1; field_list.push_back(new Item_empty_string("Command",16)); - field_list.push_back(new Item_return_int("Time",7, FIELD_TYPE_LONG)); + field_list.push_back(field= new Item_return_int("Time",7, FIELD_TYPE_LONG)); + field->unsigned_flag= 0; field_list.push_back(field=new Item_empty_string("State",30)); field->maybe_null=1; field_list.push_back(field=new Item_empty_string("Info",max_query_length)); @@ -1439,7 +1440,7 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose) else protocol->store(command_name[thd_info->command], system_charset_info); if (thd_info->start_time) - protocol->store((uint32) (now - thd_info->start_time)); + protocol->store_long ((longlong) (now - thd_info->start_time)); else protocol->store_null(); protocol->store(thd_info->state_info, system_charset_info); From 73a7d99331d8fd0819fb9f7a10081e393590a94d Mon Sep 17 00:00:00 2001 From: Alexey Kopytov Date: Wed, 18 Mar 2009 11:18:24 +0300 Subject: [PATCH 2/2] Fix for bug#41486: extra character appears in BLOB for every ~40Mb after mysqldump/import When the input string exceeds the maximum allowed size for the internal buffer, batch_readline() returns a truncated string. Since there was no way for a caller to determine whether the string was truncated or not, the command line client assumed batch_readline() to always return the whole input string and appended a newline character. This resulted in garbled data when importing dumps containing strings longer than the maximum input buffer size. Fixed by adding a flag to the batch_readline() interface to signal a truncated string to the caller. Other minor problems fixed during patch implementation: - The maximum allowed buffer size for batch_readline() was set up depending on the client's max_allowed_packet value. It does not actully make any sense, as those variables are not related. The input buffer size limit is now always set to 1 MB. - fill_buffer() did not always set the EOF flag. - The input buffer could actually grow twice as the specified limit due to insufficient checks in intern_read_line(). client/my_readline.h: Changed the interface of batch_readline(). client/mysql.cc: Honor the truncated flag returned by batch_readline() and do not append the newline character if it was set. Since we can't change the interfaces for readline()/fgets() used in the interactive mode, always assume the returned string was not truncated. In addition, always set the batch_readline() internal buffer to 1 MB, independently from the client's max_allowed_packet. client/readline.cc: Added the 'truncated' argument do batch_readline() to signal truncated string to a caller. Fixed fill_buffer() to set the EOF flag correctly. Fixed checks in intern_read_line() to not allow the internal buffer grow past the specified limit. mysql-test/r/mysql.result: Added a test case for bug #41486. mysql-test/t/mysql.test: Added a test case for bug #41486. --- client/my_readline.h | 2 +- client/mysql.cc | 22 +++++++++++-------- client/readline.cc | 46 +++++++++++++++++++++++++++++---------- mysql-test/r/mysql.result | 11 ++++++++++ mysql-test/t/mysql.test | 27 +++++++++++++++++++++++ 5 files changed, 86 insertions(+), 22 deletions(-) diff --git a/client/my_readline.h b/client/my_readline.h index 47be7fa9294..32d6da4c626 100644 --- a/client/my_readline.h +++ b/client/my_readline.h @@ -29,5 +29,5 @@ typedef struct st_line_buffer extern LINE_BUFFER *batch_readline_init(ulong max_size,FILE *file); extern LINE_BUFFER *batch_readline_command(LINE_BUFFER *buffer, my_string str); -extern char *batch_readline(LINE_BUFFER *buffer); +extern char *batch_readline(LINE_BUFFER *buffer, bool *truncated); extern void batch_readline_end(LINE_BUFFER *buffer); diff --git a/client/mysql.cc b/client/mysql.cc index 1a025345190..0a4587d1c92 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -112,6 +112,8 @@ extern "C" { #define PROMPT_CHAR '\\' #define DEFAULT_DELIMITER ";" +#define MAX_BATCH_BUFFER_SIZE (1024L * 1024L) + typedef struct st_status { int exit_status; @@ -1035,7 +1037,7 @@ static void fix_history(String *final_command); static COMMANDS *find_command(char *name,char cmd_name); static bool add_line(String &buffer,char *line,char *in_string, - bool *ml_comment); + bool *ml_comment, bool truncated); static void remove_cntrl(String &buffer); static void print_table_data(MYSQL_RES *result); static void print_table_data_html(MYSQL_RES *result); @@ -1117,7 +1119,7 @@ int main(int argc,char *argv[]) exit(1); } if (status.batch && !status.line_buff && - !(status.line_buff=batch_readline_init(opt_max_allowed_packet+512,stdin))) + !(status.line_buff= batch_readline_init(MAX_BATCH_BUFFER_SIZE, stdin))) { free_defaults(defaults_argv); my_end(0); @@ -1766,13 +1768,14 @@ static int read_and_execute(bool interactive) ulong line_number=0; bool ml_comment= 0; COMMANDS *com; + bool truncated= 0; status.exit_status=1; for (;;) { if (!interactive) { - line=batch_readline(status.line_buff); + line=batch_readline(status.line_buff, &truncated); /* Skip UTF8 Byte Order Marker (BOM) 0xEFBBBF. Editors like "notepad" put this marker in @@ -1891,7 +1894,7 @@ static int read_and_execute(bool interactive) #endif continue; } - if (add_line(glob_buffer,line,&in_string,&ml_comment)) + if (add_line(glob_buffer,line,&in_string,&ml_comment, truncated)) break; } /* if in batch mode, send last query even if it doesn't end with \g or go */ @@ -1977,7 +1980,7 @@ static COMMANDS *find_command(char *name,char cmd_char) static bool add_line(String &buffer,char *line,char *in_string, - bool *ml_comment) + bool *ml_comment, bool truncated) { uchar inchar; char buff[80], *pos, *out; @@ -2224,9 +2227,10 @@ static bool add_line(String &buffer,char *line,char *in_string, { uint length=(uint) (out-line); - if (length < 9 || - my_strnncoll (charset_info, - (uchar *)line, 9, (const uchar *) "delimiter", 9)) + if (!truncated && + (length < 9 || + my_strnncoll (charset_info, + (uchar *)line, 9, (const uchar *) "delimiter", 9))) { /* Don't add a new line in case there's a DELIMITER command to be @@ -3886,7 +3890,7 @@ static int com_source(String *buffer, char *line) return put_info(buff, INFO_ERROR, 0); } - if (!(line_buff=batch_readline_init(opt_max_allowed_packet+512,sql_file))) + if (!(line_buff= batch_readline_init(MAX_BATCH_BUFFER_SIZE, sql_file))) { my_fclose(sql_file,MYF(0)); return put_info("Can't initialize batch_readline", INFO_ERROR, 0); diff --git a/client/readline.cc b/client/readline.cc index ad42ed2ee10..726d9cd9415 100644 --- a/client/readline.cc +++ b/client/readline.cc @@ -24,7 +24,7 @@ static bool init_line_buffer(LINE_BUFFER *buffer,File file,ulong size, ulong max_size); static bool init_line_buffer_from_string(LINE_BUFFER *buffer,my_string str); static uint fill_buffer(LINE_BUFFER *buffer); -static char *intern_read_line(LINE_BUFFER *buffer,ulong *out_length); +static char *intern_read_line(LINE_BUFFER *buffer, ulong *out_length, bool *truncated); LINE_BUFFER *batch_readline_init(ulong max_size,FILE *file) @@ -42,12 +42,13 @@ LINE_BUFFER *batch_readline_init(ulong max_size,FILE *file) } -char *batch_readline(LINE_BUFFER *line_buff) +char *batch_readline(LINE_BUFFER *line_buff, bool *truncated) { char *pos; ulong out_length; + DBUG_ASSERT(truncated != NULL); - if (!(pos=intern_read_line(line_buff,&out_length))) + if (!(pos=intern_read_line(line_buff,&out_length, truncated))) return 0; if (out_length && pos[out_length-1] == '\n') if (--out_length && pos[out_length-1] == '\r') /* Remove '\n' */ @@ -149,6 +150,14 @@ static uint fill_buffer(LINE_BUFFER *buffer) read_count=(buffer->bufread - bufbytes)/IO_SIZE; if ((read_count*=IO_SIZE)) break; + if (buffer->bufread * 2 > buffer->max_size) + { + /* + So we must grow the buffer but we cannot due to the max_size limit. + Return 0 w/o setting buffer->eof to signal this condition. + */ + return 0; + } buffer->bufread *= 2; if (!(buffer->buffer = (char*) my_realloc(buffer->buffer, buffer->bufread+1, @@ -172,11 +181,15 @@ static uint fill_buffer(LINE_BUFFER *buffer) DBUG_PRINT("fill_buff", ("Got %d bytes", read_count)); - /* Kludge to pretend every nonempty file ends with a newline. */ - if (!read_count && bufbytes && buffer->end[-1] != '\n') + if (!read_count) { - buffer->eof = read_count = 1; - *buffer->end = '\n'; + buffer->eof = 1; + /* Kludge to pretend every nonempty file ends with a newline. */ + if (bufbytes && buffer->end[-1] != '\n') + { + read_count = 1; + *buffer->end = '\n'; + } } buffer->end_of_line=(buffer->start_of_line=buffer->buffer)+bufbytes; buffer->end+=read_count; @@ -186,7 +199,7 @@ static uint fill_buffer(LINE_BUFFER *buffer) -char *intern_read_line(LINE_BUFFER *buffer,ulong *out_length) +char *intern_read_line(LINE_BUFFER *buffer, ulong *out_length, bool *truncated) { char *pos; uint length; @@ -200,14 +213,23 @@ char *intern_read_line(LINE_BUFFER *buffer,ulong *out_length) pos++; if (pos == buffer->end) { - if ((uint) (pos - buffer->start_of_line) < buffer->max_size) + /* + fill_buffer() can return 0 either on EOF in which case we abort + or when the internal buffer has hit the size limit. In the latter case + return what we have read so far and signal string truncation. + */ + if (!(length=fill_buffer(buffer)) || length == (uint) -1) { - if (!(length=fill_buffer(buffer)) || length == (uint) -1) - DBUG_RETURN(0); - continue; + if (buffer->eof) + DBUG_RETURN(0); } + else + continue; pos--; /* break line here */ + *truncated= 1; } + else + *truncated= 0; buffer->end_of_line=pos+1; *out_length=(ulong) (pos + 1 - buffer->eof - buffer->start_of_line); DBUG_RETURN(buffer->start_of_line); diff --git a/mysql-test/r/mysql.result b/mysql-test/r/mysql.result index 10537f6da16..a6087e0134b 100644 --- a/mysql-test/r/mysql.result +++ b/mysql-test/r/mysql.result @@ -192,4 +192,15 @@ delimiter 1 1 1 +set @old_max_allowed_packet = @@global.max_allowed_packet; +set @@global.max_allowed_packet = 2 * 1024 * 1024 + 1024; +set @@max_allowed_packet = @@global.max_allowed_packet; +CREATE TABLE t1(data LONGBLOB); +INSERT INTO t1 SELECT REPEAT('1', 2*1024*1024); +SELECT LENGTH(data) FROM t1; +LENGTH(data) +2097152 +DROP TABLE t1; +set @@global.max_allowed_packet = @old_max_allowed_packet; +set @@max_allowed_packet = @@global.max_allowed_packet; End of 5.0 tests diff --git a/mysql-test/t/mysql.test b/mysql-test/t/mysql.test index 594d10e46a5..0030f624be6 100644 --- a/mysql-test/t/mysql.test +++ b/mysql-test/t/mysql.test @@ -331,4 +331,31 @@ EOF remove_file $MYSQLTEST_VARDIR/tmp/bug31060.sql; +# +# Bug #41486: extra character appears in BLOB for every ~40Mb after +# mysqldump/import +# + +# Have to change the global variable as the mysql client will use +# a separate session +set @old_max_allowed_packet = @@global.max_allowed_packet; +# 2 MB blob length + some space for the rest of INSERT query +set @@global.max_allowed_packet = 2 * 1024 * 1024 + 1024; +set @@max_allowed_packet = @@global.max_allowed_packet; + +CREATE TABLE t1(data LONGBLOB); +INSERT INTO t1 SELECT REPEAT('1', 2*1024*1024); + +--exec $MYSQL_DUMP test t1 >$MYSQLTEST_VARDIR/tmp/bug41486.sql +# Check that the mysql client does not insert extra newlines when loading +# strings longer than client's max_allowed_packet +--exec $MYSQL --max_allowed_packet=1M test < $MYSQLTEST_VARDIR/tmp/bug41486.sql 2>&1 +SELECT LENGTH(data) FROM t1; + +remove_file $MYSQLTEST_VARDIR/tmp/bug41486.sql; +DROP TABLE t1; + +set @@global.max_allowed_packet = @old_max_allowed_packet; +set @@max_allowed_packet = @@global.max_allowed_packet; + --echo End of 5.0 tests