From 06fb8c2d1009fe6d285fd29ab9080b2b040a3d60 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 11 Oct 2007 18:18:05 +0200 Subject: [PATCH] BUG#29549 (Endians: test failures on Solaris): Refactoring code to add parameter to pack() and unpack() functions with purpose of indicating if data should be packed in little-endian or native order. Using new functions to always pack data for binary log in little-endian order. The purpose of this refactoring is to allow proper implementation of endian-agnostic pack() and unpack() functions. Eliminating several versions of virtual pack() and unpack() functions in favor for one single virtual function which is overridden in subclasses. Implementing pack() and unpack() functions for some field types that packed data in native format regardless of the value of the st_table_share::db_low_byte_first flag. The field types that were packed in native format regardless are: Field_real, Field_decimal, Field_tiny, Field_short, Field_medium, Field_long, Field_longlong, and Field_blob. Before the patch, row-based logging wrote the rows incorrectly on big-endian machines where the storage engine defined its own low_byte_first() to be FALSE on big-endian machines (the default is TRUE), while little-endian machines wrote the fields in correct order. The only known storage engine that does this is NDB. In effect, this means that row-based replication from or to a big-endian machine where the table was using NDB as storage engine failed if the other engine was either non-NDB or on a little-endian machine. With this patch, row-based logging is now always done in little-endian order, while ORDER BY uses the native order if the storage engine defines low_byte_first() to return FALSE for big-endian machines. In addition, the max_data_length() function available in Field_blob was generalized to the entire Field hierarchy to give the maximum number of bytes that Field::pack() will write. mysql-test/extra/rpl_tests/rpl_extraMaster_Col.test: Sorting by columns that produces deterministic order. mysql-test/suite/rpl/r/rpl_extraColmaster_innodb.result: Result change. mysql-test/suite/rpl/r/rpl_extraColmaster_myisam.result: Result change. mysql-test/suite/rpl/r/rpl_row_extraColmaster_ndb.result: Result change. mysql-test/suite/rpl/t/disabled.def: Enabling tests. mysql-test/suite/rpl/t/rpl_row_mysqlbinlog.test: Adding missing sync_slave_with_master causing slave to keep tables after shutdown. mysql-test/suite/rpl_ndb/t/disabled.def: Enabling tests. mysql-test/suite/rpl_ndb/t/rpl_ndb_innodb2ndb-slave.opt: Adding --new option mysql-test/suite/rpl_ndb/t/rpl_ndb_innodb2ndb.test: Adding have_log_bin. mysql-test/suite/rpl_ndb/t/rpl_ndb_myisam2ndb-slave.opt: Adding --new option mysql-test/suite/rpl_ndb/t/rpl_ndb_myisam2ndb.test: Adding have_log_bin mysql-test/t/partition.test: Adding have_archive, since that is used in the test. sql/field.cc: Eliminating all two-argument pack() and unpack() functions and moving functionality into the four-argument version. The four argument version is introduced so that it is possible to avoid using the storage engine default when writing and reading the packed format (the unpacked format still uses the storage engine's default). This is used by row-based replication to write the fields in a storage engine- and endian-agnostic format. Packing integral and floating-point numbers in little-endian format (if requested). Using pad_char for the field instead of spaces (0x20) when unpacking. Adding some Doxygen documentation. --- Adding max_data_length() to denote the maximum number of bytes that pack() will write. Adding casts to remove warnings for debug printouts. sql/field.h: Eliminating all virtual pack() and unpack() functions except the four- argument version, which now is the function that shall be overridden. The two-argument versions are convenience functions, to prevent changes to code that uses these. Adding code to pack integer numbers and floating-point numbers in little-endian format, if requested. --- Adding max_data_length() to denote the maximum number of bytes that pack() will write. sql/log.cc: Removing debug printout causing crash when starting NDB on Solaris. sql/log_event.cc: Adding missing #ifndef causing compile failure. Adding debug printouts. sql/rpl_record.cc: Debriding code. Using new pack() and unpack() functions to always pack fields little-endian. Adding debug printouts. --- Using max_data_length() when packing field into row. Adding casts to debug printouts. sql/sql_show.cc: Adding code that causes crash on Solaris machines since printf() cannot handle NULL values for strings properly. mysql-test/suite/rpl_ndb/r/rpl_ndb_innodb2ndb.result: New BitKeeper file ``mysql-test/suite/rpl_ndb/r/rpl_ndb_innodb2ndb.result'' mysql-test/suite/rpl_ndb/r/rpl_ndb_myisam2ndb.result: New BitKeeper file ``mysql-test/suite/rpl_ndb/r/rpl_ndb_myisam2ndb.result'' --- .../extra/rpl_tests/rpl_extraMaster_Col.test | 2 +- .../rpl/r/rpl_extraColmaster_innodb.result | Bin 38226 -> 38226 bytes .../rpl/r/rpl_extraColmaster_myisam.result | Bin 38226 -> 38226 bytes .../rpl/r/rpl_row_extraColmaster_ndb.result | Bin 19129 -> 19129 bytes mysql-test/suite/rpl/t/disabled.def | 1 - .../suite/rpl/t/rpl_row_mysqlbinlog.test | 1 + .../suite/rpl_ndb/r/rpl_ndb_innodb2ndb.result | 855 ++++++++++++++++++ .../suite/rpl_ndb/r/rpl_ndb_myisam2ndb.result | 855 ++++++++++++++++++ mysql-test/suite/rpl_ndb/t/disabled.def | 2 - .../rpl_ndb/t/rpl_ndb_innodb2ndb-slave.opt | 2 +- .../suite/rpl_ndb/t/rpl_ndb_innodb2ndb.test | 10 +- .../rpl_ndb/t/rpl_ndb_myisam2ndb-slave.opt | 2 +- .../suite/rpl_ndb/t/rpl_ndb_myisam2ndb.test | 7 +- mysql-test/t/partition.test | 3 +- sql/field.cc | 407 ++++++--- sql/field.h | 271 +++++- sql/log.cc | 3 - sql/log_event.cc | 4 + sql/rpl_record.cc | 54 +- sql/sql_show.cc | 6 +- 20 files changed, 2251 insertions(+), 234 deletions(-) create mode 100644 mysql-test/suite/rpl_ndb/r/rpl_ndb_innodb2ndb.result create mode 100644 mysql-test/suite/rpl_ndb/r/rpl_ndb_myisam2ndb.result diff --git a/mysql-test/extra/rpl_tests/rpl_extraMaster_Col.test b/mysql-test/extra/rpl_tests/rpl_extraMaster_Col.test index 26b1e6d5ba0..a03aa337c2b 100644 --- a/mysql-test/extra/rpl_tests/rpl_extraMaster_Col.test +++ b/mysql-test/extra/rpl_tests/rpl_extraMaster_Col.test @@ -466,7 +466,7 @@ binary data'; select * from t2 order by f1; select * from t3 order by f1; select * from t4 order by f1; - select * from t31 order by f1; + select * from t31 order by f3; connection master; diff --git a/mysql-test/suite/rpl/r/rpl_extraColmaster_innodb.result b/mysql-test/suite/rpl/r/rpl_extraColmaster_innodb.result index 189066179255c33b7d56e0fdfd8a3893dda3535c..ef6e2a6de25a11ce8c9b48e9bd4a0ab8e0380089 100644 GIT binary patch delta 27 dcmcb#is{lSrVXZUlVyXYHwU=6c|jQqSpb}X3M&8r delta 27 ecmcb#is{lSrVXZUlW+A)Zw_#C^MW!KvH$?ISqvQj diff --git a/mysql-test/suite/rpl/r/rpl_extraColmaster_myisam.result b/mysql-test/suite/rpl/r/rpl_extraColmaster_myisam.result index 5bc1a13b1078a054d99dfef348ad74e85295e302..475ea4cdcbaab56b53f74844b68bcae4aaa0d753 100644 GIT binary patch delta 27 dcmcb#is{lSrVXZUlVyXYHwU=6c|jQqSpb}X3M&8r delta 27 ecmcb#is{lSrVXZUlW+A)Zw_#C^MW!KvH$?ISqvQj diff --git a/mysql-test/suite/rpl/r/rpl_row_extraColmaster_ndb.result b/mysql-test/suite/rpl/r/rpl_row_extraColmaster_ndb.result index 0b1f2e6c8bf9514ef53ef326dde109d3a78bdea8..fb83c5088e502338e8db678565120f65b47dcba1 100644 GIT binary patch delta 82 zcmdlvm2u}(#tna6C;ySTNkkg2>I5jyxFNF)lG3G4E$Si`eO*qr?ON&Y} Xz!IjMX_;wIwi#z}X2s@Uw=@<2Nf{Ui delta 86 zcmdlvm2u}(#tna6Cm(Yab~fcq%Slow_byte_first is dependent on how the + packed data is going to be used: for local use, e.g., temporary + store on disk or in memory, use the native format since that is + faster. For data that is going to be transfered to other machines + (e.g., when writing data to the binary log), data should always be + stored in little-endian format. + + @note The default method for packing fields just copy the raw bytes + of the record into the destination, but never more than + max_length characters. + + @param to + Pointer to memory area where representation of field should be put. + + @param from + Pointer to memory area where record representation of field is + stored. + + @param max_length + Maximum length of the field, as given in the column definition. For + example, for CHAR(1000), the max_length + is 1000. This information is sometimes needed to decide how to pack + the data. + + @param low_byte_first + @c TRUE if integers should be stored little-endian, @c FALSE if + native format should be used. Note that for little-endian machines, + the value of this flag is a moot point since the native format is + little-endian. +*/ +uchar * +Field::pack(uchar *to, const uchar *from, uint max_length, + bool low_byte_first __attribute__((unused))) +{ + uint32 length= pack_length(); + set_if_smaller(length, max_length); + memcpy(to, from, length); + return to+length; +} + /** Unpack a field from row data. - This method is used to unpack a field from a master whose size - of the field is less than that of the slave. - + This method is used to unpack a field from a master whose size of + the field is less than that of the slave. + + The param_data parameter is a two-byte integer (stored + in the least significant 16 bits of the unsigned integer) usually + consisting of two parts: the real type in the most significant byte + and a original pack length in the least significant byte. + + The exact layout of the param_data field is given by + the Table_map_log_event::save_field_metadata(). + + This is the default method for unpacking a field. It just copies + the memory block in byte order (of original pack length bytes or + length of field, whichever is smaller). + @param to Destination of the data @param from Source of the data - @param param_data Pack length of the field data + @param param_data Real type and original pack length of the field + data + + @param low_byte_first + If this flag is @c true, all composite entities (e.g., lengths) + should be unpacked in little-endian format; otherwise, the entities + are unpacked in native order. @return New pointer into memory based on from + length of the data */ -const uchar *Field::unpack(uchar* to, - const uchar *from, - uint param_data) +const uchar * +Field::unpack(uchar* to, const uchar *from, uint param_data, + bool low_byte_first __attribute__((unused))) { uint length=pack_length(); int from_type= 0; @@ -1399,19 +1464,18 @@ const uchar *Field::unpack(uchar* to, from_type= (param_data & 0xff00) >> 8U; // real_type. param_data= param_data & 0x00ff; // length. } + + if ((param_data == 0) || + (length == param_data) || + (from_type != real_type())) + { + memcpy(to, from, length); + return from+length; + } + uint len= (param_data && (param_data < length)) ? param_data : length; - /* - If the length is the same, use old unpack method. - If the param_data is 0, use the old unpack method. - This is possible if the table map was generated from a down-level - master or if the data was not available on the master. - If the real_types are not the same, use the old unpack method. - */ - if ((length == param_data) || - (param_data == 0) || - (from_type != real_type())) - return(unpack(to, from)); + memcpy(to, from, param_data > length ? length : len); return from+len; } @@ -2714,10 +2778,15 @@ uint Field_new_decimal::is_equal(Create_field *new_field) @return New pointer into memory based on from + length of the data */ -const uchar *Field_new_decimal::unpack(uchar* to, - const uchar *from, - uint param_data) +const uchar * +Field_new_decimal::unpack(uchar* to, + const uchar *from, + uint param_data, + bool low_byte_first) { + if (param_data == 0) + return Field::unpack(to, from, param_data, low_byte_first); + uint from_precision= (param_data & 0xff00) >> 8U; uint from_decimal= param_data & 0x00ff; uint length=pack_length(); @@ -3857,6 +3926,49 @@ void Field_longlong::sql_type(String &res) const } +/* + Floating-point numbers + */ + +uchar * +Field_real::pack(uchar *to, const uchar *from, + uint max_length, bool low_byte_first) +{ + DBUG_ENTER("Field_real::pack"); + DBUG_ASSERT(max_length >= pack_length()); + DBUG_PRINT("debug", ("pack_length(): %u", pack_length())); +#ifdef WORDS_BIGENDIAN + if (low_byte_first != table->s->db_low_byte_first) + { + const uchar *dptr= from + pack_length(); + while (dptr-- > from) + *to++ = *dptr; + DBUG_RETURN(to); + } + else +#endif + DBUG_RETURN(Field::pack(to, from, max_length, low_byte_first)); +} + +const uchar * +Field_real::unpack(uchar *to, const uchar *from, + uint param_data, bool low_byte_first) +{ + DBUG_ENTER("Field_real::unpack"); + DBUG_PRINT("debug", ("pack_length(): %u", pack_length())); +#ifdef WORDS_BIGENDIAN + if (low_byte_first != table->s->db_low_byte_first) + { + const uchar *dptr= from + pack_length(); + while (dptr-- > from) + *to++ = *dptr; + DBUG_RETURN(from + pack_length()); + } + else +#endif + DBUG_RETURN(Field::unpack(to, from, param_data, low_byte_first)); +} + /**************************************************************************** single precision float ****************************************************************************/ @@ -6211,6 +6323,11 @@ int Field_longstr::store_decimal(const my_decimal *d) return store(str.ptr(), str.length(), str.charset()); } +uint32 Field_longstr::max_data_length() const +{ + return field_length + (field_length > 255 ? 2 : 1); +} + double Field_string::val_real(void) { @@ -6355,7 +6472,9 @@ void Field_string::sql_type(String &res) const } -uchar *Field_string::pack(uchar *to, const uchar *from, uint max_length) +uchar *Field_string::pack(uchar *to, const uchar *from, + uint max_length, + bool low_byte_first __attribute__((unused))) { uint length= min(field_length,max_length); uint local_char_length= max_length/field_charset->mbmaxlen; @@ -6363,11 +6482,15 @@ uchar *Field_string::pack(uchar *to, const uchar *from, uint max_length) local_char_length= my_charpos(field_charset, from, from+length, local_char_length); set_if_smaller(length, local_char_length); - while (length && from[length-1] == ' ') + while (length && from[length-1] == field_charset->pad_char) length--; + + // Length always stored little-endian *to++= (uchar) length; if (field_length > 255) *to++= (uchar) (length >> 8); + + // Store the actual bytes of the string memcpy(to, from, length); return to+length; } @@ -6389,34 +6512,27 @@ uchar *Field_string::pack(uchar *to, const uchar *from, uint max_length) @return New pointer into memory based on from + length of the data */ -const uchar *Field_string::unpack(uchar *to, - const uchar *from, - uint param_data) -{ - uint from_len= param_data & 0x00ff; // length. - uint length= 0; - uint f_length; - f_length= (from_len < field_length) ? from_len : field_length; - DBUG_ASSERT(f_length <= 255); - length= (uint) *from++; - bitmap_set_bit(table->write_set,field_index); - store((const char *)from, length, system_charset_info); - return from+length; -} - - -const uchar *Field_string::unpack(uchar *to, const uchar *from) +const uchar * +Field_string::unpack(uchar *to, + const uchar *from, + uint param_data, + bool low_byte_first __attribute__((unused))) { + uint from_length= + param_data ? min(param_data & 0x00ff, field_length) : field_length; uint length; - if (field_length > 255) + + if (from_length > 255) { length= uint2korr(from); from+= 2; } else length= (uint) *from++; - memcpy(to, from, (int) length); - bfill(to+length, field_length - length, ' '); + + memcpy(to, from, length); + // Pad the string with the pad character of the fields charset + bfill(to + length, field_length - length, field_charset->pad_char); return from+length; } @@ -6796,22 +6912,30 @@ uint32 Field_varstring::data_length() Here the number of length bytes are depending on the given max_length */ -uchar *Field_varstring::pack(uchar *to, const uchar *from, uint max_length) +uchar *Field_varstring::pack(uchar *to, const uchar *from, + uint max_length, + bool low_byte_first __attribute__((unused))) { uint length= length_bytes == 1 ? (uint) *from : uint2korr(from); set_if_smaller(max_length, field_length); if (length > max_length) length=max_length; - *to++= (char) (length & 255); + + /* Length always stored little-endian */ + *to++= length & 0xFF; if (max_length > 255) - *to++= (char) (length >> 8); - if (length) + *to++= (length >> 8) & 0xFF; + + /* Store bytes of string */ + if (length > 0) memcpy(to, from+length_bytes, length); return to+length; } -uchar *Field_varstring::pack_key(uchar *to, const uchar *key, uint max_length) +uchar * +Field_varstring::pack_key(uchar *to, const uchar *key, uint max_length, + bool low_byte_first __attribute__((unused))) { uint length= length_bytes == 1 ? (uint) *key : uint2korr(key); uint local_char_length= ((field_charset->mbmaxlen > 1) ? @@ -6850,8 +6974,9 @@ uchar *Field_varstring::pack_key(uchar *to, const uchar *key, uint max_length) Pointer to end of 'key' (To the next key part if multi-segment key) */ -const uchar *Field_varstring::unpack_key(uchar *to, const uchar *key, - uint max_length) +const uchar * +Field_varstring::unpack_key(uchar *to, const uchar *key, uint max_length, + bool low_byte_first __attribute__((unused))) { /* get length of the blob key */ uint32 length= *key++; @@ -6880,8 +7005,9 @@ const uchar *Field_varstring::unpack_key(uchar *to, const uchar *key, end of key storage */ -uchar *Field_varstring::pack_key_from_key_image(uchar *to, const uchar *from, - uint max_length) +uchar * +Field_varstring::pack_key_from_key_image(uchar *to, const uchar *from, uint max_length, + bool low_byte_first __attribute__((unused))) { /* Key length is always stored as 2 bytes */ uint length= uint2korr(from); @@ -6901,6 +7027,9 @@ uchar *Field_varstring::pack_key_from_key_image(uchar *to, const uchar *from, This method is used to unpack a varstring field from a master whose size of the field is less than that of the slave. + + @note + The string length is always packed little-endian. @param to Destination of the data @param from Source of the data @@ -6908,9 +7037,10 @@ uchar *Field_varstring::pack_key_from_key_image(uchar *to, const uchar *from, @return New pointer into memory based on from + length of the data */ -const uchar *Field_varstring::unpack(uchar *to, - const uchar *from, - uint param_data) +const uchar * +Field_varstring::unpack(uchar *to, const uchar *from, + uint param_data, + bool low_byte_first __attribute__((unused))) { uint length; uint l_bytes= (param_data && (param_data < field_length)) ? @@ -6922,28 +7052,7 @@ const uchar *Field_varstring::unpack(uchar *to, if (length_bytes == 2) to[1]= 0; } - else - { - length= uint2korr(from); - to[0]= *from++; - to[1]= *from++; - } - if (length) - memcpy(to+ length_bytes, from, length); - return from+length; -} - - -/* - unpack field packed with Field_varstring::pack() -*/ - -const uchar *Field_varstring::unpack(uchar *to, const uchar *from) -{ - uint length; - if (length_bytes == 1) - length= (uint) (*to= *from++); - else + else /* l_bytes == 2 */ { length= uint2korr(from); to[0]= *from++; @@ -7192,9 +7301,9 @@ void Field_blob::store_length(uchar *i_ptr, } -uint32 Field_blob::get_length(const uchar *pos, bool low_byte_first) +uint32 Field_blob::get_length(const uchar *pos, uint packlength_arg, bool low_byte_first) { - switch (packlength) { + switch (packlength_arg) { case 1: return (uint32) pos[0]; case 2: @@ -7608,28 +7717,37 @@ void Field_blob::sql_type(String &res) const } } - -uchar *Field_blob::pack(uchar *to, const uchar *from, uint max_length) +uchar *Field_blob::pack(uchar *to, const uchar *from, + uint max_length, bool low_byte_first) { + DBUG_ENTER("Field_blob::pack"); + DBUG_PRINT("enter", ("to: 0x%lx; from: 0x%lx;" + " max_length: %u; low_byte_first: %d", + (ulong) to, (ulong) from, + max_length, low_byte_first)); + DBUG_DUMP("record", from, table->s->reclength); uchar *save= ptr; ptr= (uchar*) from; uint32 length=get_length(); // Length of from string - if (length > max_length) - { - ptr=to; - length=max_length; - store_length(length); // Store max length - ptr= (uchar*) from; - } - else - memcpy(to,from,packlength); // Copy length - if (length) + + /* + Store max length, which will occupy packlength bytes. If the max + length given is smaller than the actual length of the blob, we + just store the initial bytes of the blob. + */ + store_length(to, packlength, min(length, max_length), low_byte_first); + + /* + Store the actual blob data, which will occupy 'length' bytes. + */ + if (length > 0) { get_ptr((uchar**) &from); memcpy(to+packlength, from,length); } ptr=save; // Restore org row pointer - return to+packlength+length; + DBUG_DUMP("packed", to, packlength + length); + DBUG_RETURN(to+packlength+length); } @@ -7644,28 +7762,26 @@ uchar *Field_blob::pack(uchar *to, const uchar *from, uint max_length) @param to Destination of the data @param from Source of the data - @param param_data @return New pointer into memory based on from + length of the data */ const uchar *Field_blob::unpack(uchar *to, const uchar *from, - uint param_data) + uint param_data, + bool low_byte_first) { - return unpack(to, from); -} - - -const uchar *Field_blob::unpack(uchar *to, const uchar *from) -{ - memcpy(to,from,packlength); - uint32 length=get_length(from); - from+=packlength; - if (length) - memcpy_fixed(to+packlength, &from, sizeof(from)); - else - bzero(to+packlength,sizeof(from)); - return from+length; + DBUG_ENTER("Field_blob::unpack"); + DBUG_PRINT("enter", ("to: 0x%lx; from: 0x%lx;" + " param_data: %u; low_byte_first: %d", + (ulong) to, (ulong) from, param_data, low_byte_first)); + uint const master_packlength= + param_data > 0 ? param_data & 0xFF : packlength; + uint32 const length= get_length(from, master_packlength, low_byte_first); + DBUG_DUMP("packed", from, length + master_packlength); + store(reinterpret_cast(from) + master_packlength, + length, field_charset); + DBUG_DUMP("record", to, table->s->reclength); + DBUG_RETURN(from + master_packlength + length); } /* Keys for blobs are like keys on varchars */ @@ -7715,7 +7831,9 @@ int Field_blob::pack_cmp(const uchar *b, uint key_length_arg, /* Create a packed key that will be used for storage from a MySQL row */ -uchar *Field_blob::pack_key(uchar *to, const uchar *from, uint max_length) +uchar * +Field_blob::pack_key(uchar *to, const uchar *from, uint max_length, + bool low_byte_first __attribute__((unused))) { uchar *save= ptr; ptr= (uchar*) from; @@ -7760,8 +7878,9 @@ uchar *Field_blob::pack_key(uchar *to, const uchar *from, uint max_length) Pointer into 'from' past the last byte copied from packed key. */ -const uchar *Field_blob::unpack_key(uchar *to, const uchar *from, - uint max_length) +const uchar * +Field_blob::unpack_key(uchar *to, const uchar *from, uint max_length, + bool low_byte_first __attribute__((unused))) { /* get length of the blob key */ uint32 length= *from++; @@ -7784,8 +7903,9 @@ const uchar *Field_blob::unpack_key(uchar *to, const uchar *from, /* Create a packed key that will be used for storage from a MySQL key */ -uchar *Field_blob::pack_key_from_key_image(uchar *to, const uchar *from, - uint max_length) +uchar * +Field_blob::pack_key_from_key_image(uchar *to, const uchar *from, uint max_length, + bool low_byte_first __attribute__((unused))) { uint length=uint2korr(from); if (length > max_length) @@ -8672,9 +8792,11 @@ void Field_bit::sql_type(String &res) const } -uchar *Field_bit::pack(uchar *to, const uchar *from, uint max_length) +uchar * +Field_bit::pack(uchar *to, const uchar *from, uint max_length, + bool low_byte_first __attribute__((unused))) { - DBUG_ASSERT(max_length); + DBUG_ASSERT(max_length > 0); uint length; if (bit_len > 0) { @@ -8709,28 +8831,44 @@ uchar *Field_bit::pack(uchar *to, const uchar *from, uint max_length) /** Unpack a bit field from row data. - This method is used to unpack a bit field from a master whose size + This method is used to unpack a bit field from a master whose size of the field is less than that of the slave. - + @param to Destination of the data @param from Source of the data @param param_data Bit length (upper) and length (lower) values @return New pointer into memory based on from + length of the data */ -const uchar *Field_bit::unpack(uchar *to, - const uchar *from, - uint param_data) +const uchar * +Field_bit::unpack(uchar *to, const uchar *from, uint param_data, + bool low_byte_first __attribute__((unused))) { uint const from_len= (param_data >> 8U) & 0x00ff; uint const from_bit_len= param_data & 0x00ff; /* - If the master and slave have the same sizes, then use the old - unpack() method. + If the parameter data is zero (i.e., undefined), or if the master + and slave have the same sizes, then use the old unpack() method. */ - if ((from_bit_len == bit_len) && - (from_len == bytes_in_rec)) - return(unpack(to, from)); + if (param_data == 0 || + (from_bit_len == bit_len) && (from_len == bytes_in_rec)) + { + if (bit_len > 0) + { + /* + set_rec_bits is a macro, don't put the post-increment in the + argument since that might cause strange side-effects. + + For the choice of the second argument, see the explanation for + Field_bit::pack(). + */ + set_rec_bits(*from, bit_ptr + (to - ptr), bit_ofs, bit_len); + from++; + } + memcpy(to, from, bytes_in_rec); + return from + bytes_in_rec; + } + /* We are converting a smaller bit field to a larger one here. To do that, we first need to construct a raw value for the original @@ -8758,25 +8896,6 @@ const uchar *Field_bit::unpack(uchar *to, } -const uchar *Field_bit::unpack(uchar *to, const uchar *from) -{ - if (bit_len > 0) - { - /* - set_rec_bits is a macro, don't put the post-increment in the - argument since that might cause strange side-effects. - - For the choice of the second argument, see the explanation for - Field_bit::pack(). - */ - set_rec_bits(*from, bit_ptr + (to - ptr), bit_ofs, bit_len); - from++; - } - memcpy(to, from, bytes_in_rec); - return from + bytes_in_rec; -} - - void Field_bit::set_default() { if (bit_len > 0) diff --git a/sql/field.h b/sql/field.h index 60f6fc19d76..37a0cdae070 100644 --- a/sql/field.h +++ b/sql/field.h @@ -157,6 +157,17 @@ public: */ virtual uint32 data_length() { return pack_length(); } virtual uint32 sort_length() const { return pack_length(); } + + /** + Get the maximum size of the data in packed format. + + @return Maximum data length of the field when packed using the + Field::pack() function. + */ + virtual uint32 max_data_length() const { + return pack_length(); + }; + virtual int reset(void) { bzero(ptr,pack_length()); return 0; } virtual void reset_fields() {} virtual void set_default() @@ -339,32 +350,45 @@ public: return str; } virtual bool send_binary(Protocol *protocol); - virtual uchar *pack(uchar *to, const uchar *from, uint max_length=~(uint) 0) + + virtual uchar *pack(uchar *to, const uchar *from, + uint max_length, bool low_byte_first); + /** + @overload Field::pack(uchar*, const uchar*, uint, bool) + */ + uchar *pack(uchar *to, const uchar *from) { - uint32 length=pack_length(); - memcpy(to,from,length); - return to+length; + DBUG_ENTER("Field::pack"); + uchar *result= this->pack(to, from, UINT_MAX, table->s->db_low_byte_first); + DBUG_RETURN(result); } - virtual const uchar *unpack(uchar* to, const uchar *from, uint param_data); - virtual const uchar *unpack(uchar* to, const uchar *from) + + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first); + /** + @overload Field::unpack(uchar*, const uchar*, uint, bool) + */ + const uchar *unpack(uchar* to, const uchar *from) { - uint length=pack_length(); - memcpy(to,from,length); - return from+length; + DBUG_ENTER("Field::unpack"); + const uchar *result= unpack(to, from, 0U, table->s->db_low_byte_first); + DBUG_RETURN(result); } - virtual uchar *pack_key(uchar* to, const uchar *from, uint max_length) + + virtual uchar *pack_key(uchar* to, const uchar *from, + uint max_length, bool low_byte_first) { - return pack(to,from,max_length); + return pack(to, from, max_length, low_byte_first); } virtual uchar *pack_key_from_key_image(uchar* to, const uchar *from, - uint max_length) + uint max_length, bool low_byte_first) { - return pack(to,from,max_length); + return pack(to, from, max_length, low_byte_first); } virtual const uchar *unpack_key(uchar* to, const uchar *from, - uint max_length) + uint max_length, bool low_byte_first) { - return unpack(to,from); + return unpack(to, from, max_length, low_byte_first); } virtual uint packed_col_length(const uchar *to, uint length) { return length;} @@ -536,6 +560,7 @@ public: {} int store_decimal(const my_decimal *d); + uint32 max_data_length() const; }; /* base class for float and double and decimal (old one) */ @@ -556,6 +581,10 @@ public: int truncate(double *nr, double max_length); uint32 max_display_length() { return field_length; } uint size_of() const { return sizeof(*this); } + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first); + virtual uchar *pack(uchar* to, const uchar *from, + uint max_length, bool low_byte_first); }; @@ -584,6 +613,16 @@ public: void overflow(bool negative); bool zero_pack() const { return 0; } void sql_type(String &str) const; + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first) + { + return Field::unpack(to, from, param_data, low_byte_first); + } + virtual uchar *pack(uchar* to, const uchar *from, + uint max_length, bool low_byte_first) + { + return Field::pack(to, from, max_length, low_byte_first); + } }; @@ -629,7 +668,8 @@ public: uint size_of() const { return sizeof(*this); } uint32 pack_length() const { return (uint32) bin_size; } uint is_equal(Create_field *new_field); - virtual const uchar *unpack(uchar* to, const uchar *from, uint param_data); + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first); }; @@ -660,6 +700,20 @@ public: uint32 pack_length() const { return 1; } void sql_type(String &str) const; uint32 max_display_length() { return 4; } + + virtual uchar *pack(uchar* to, const uchar *from, + uint max_length, bool low_byte_first) + { + *to= *from; + return to + 1; + } + + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first) + { + *to= *from; + return from + 1; + } }; @@ -695,8 +749,47 @@ public: uint32 pack_length() const { return 2; } void sql_type(String &str) const; uint32 max_display_length() { return 6; } -}; + virtual uchar *pack(uchar* to, const uchar *from, + uint max_length, bool low_byte_first) + { + int16 val; +#ifdef WORDS_BIGENDIAN + if (table->s->db_low_byte_first) + val = sint2korr(from); + else +#endif + shortget(val, from); + +#ifdef WORDS_BIGENDIAN + if (low_byte_first) + int2store(to, val); + else +#endif + shortstore(to, val); + return to + sizeof(val); + } + + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first) + { + int16 val; +#ifdef WORDS_BIGENDIAN + if (low_byte_first) + val = sint2korr(from); + else +#endif + shortget(val, from); + +#ifdef WORDS_BIGENDIAN + if (table->s->db_low_byte_first) + int2store(to, val); + else +#endif + shortstore(to, val); + return from + sizeof(val); + } +}; class Field_medium :public Field_num { public: @@ -725,6 +818,18 @@ public: uint32 pack_length() const { return 3; } void sql_type(String &str) const; uint32 max_display_length() { return 8; } + + virtual uchar *pack(uchar* to, const uchar *from, + uint max_length, bool low_byte_first) + { + return Field::pack(to, from, max_length, low_byte_first); + } + + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first) + { + return Field::unpack(to, from, param_data, low_byte_first); + } }; @@ -760,6 +865,45 @@ public: uint32 pack_length() const { return 4; } void sql_type(String &str) const; uint32 max_display_length() { return MY_INT32_NUM_DECIMAL_DIGITS; } + virtual uchar *pack(uchar* to, const uchar *from, + uint max_length, bool low_byte_first) + { + int32 val; +#ifdef WORDS_BIGENDIAN + if (table->s->db_low_byte_first) + val = sint4korr(from); + else +#endif + longget(val, from); + +#ifdef WORDS_BIGENDIAN + if (low_byte_first) + int4store(to, val); + else +#endif + longstore(to, val); + return to + sizeof(val); + } + + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first) + { + int32 val; +#ifdef WORDS_BIGENDIAN + if (low_byte_first) + val = sint4korr(from); + else +#endif + longget(val, from); + +#ifdef WORDS_BIGENDIAN + if (table->s->db_low_byte_first) + int4store(to, val); + else +#endif + longstore(to, val); + return from + sizeof(val); + } }; @@ -802,6 +946,45 @@ public: void sql_type(String &str) const; bool can_be_compared_as_longlong() const { return TRUE; } uint32 max_display_length() { return 20; } + virtual uchar *pack(uchar* to, const uchar *from, + uint max_length, bool low_byte_first) + { + int64 val; +#ifdef WORDS_BIGENDIAN + if (table->s->db_low_byte_first) + val = sint8korr(from); + else +#endif + longlongget(val, from); + +#ifdef WORDS_BIGENDIAN + if (low_byte_first) + int8store(to, val); + else +#endif + longlongstore(to, val); + return to + sizeof(val); + } + + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first) + { + int64 val; +#ifdef WORDS_BIGENDIAN + if (low_byte_first) + val = sint8korr(from); + else +#endif + longlongget(val, from); + +#ifdef WORDS_BIGENDIAN + if (table->s->db_low_byte_first) + int8store(to, val); + else +#endif + longlongstore(to, val); + return from + sizeof(val); + } }; #endif @@ -1176,9 +1359,10 @@ public: int cmp(const uchar *,const uchar *); void sort_string(uchar *buff,uint length); void sql_type(String &str) const; - uchar *pack(uchar *to, const uchar *from, uint max_length=~(uint) 0); - virtual const uchar *unpack(uchar* to, const uchar *from, uint param_data); - const uchar *unpack(uchar* to, const uchar *from); + virtual uchar *pack(uchar *to, const uchar *from, + uint max_length, bool low_byte_first); + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first); int pack_cmp(const uchar *a,const uchar *b,uint key_length, my_bool insert_or_update); int pack_cmp(const uchar *b,uint key_length,my_bool insert_or_update); @@ -1250,13 +1434,15 @@ public: uint get_key_image(uchar *buff,uint length, imagetype type); void set_key_image(const uchar *buff,uint length); void sql_type(String &str) const; - uchar *pack(uchar *to, const uchar *from, uint max_length=~(uint) 0); - uchar *pack_key(uchar *to, const uchar *from, uint max_length); + virtual uchar *pack(uchar *to, const uchar *from, + uint max_length, bool low_byte_first); + uchar *pack_key(uchar *to, const uchar *from, uint max_length, bool low_byte_first); uchar *pack_key_from_key_image(uchar* to, const uchar *from, - uint max_length); - virtual const uchar *unpack(uchar* to, const uchar *from, uint param_data); - const uchar *unpack(uchar* to, const uchar *from); - const uchar *unpack_key(uchar* to, const uchar *from, uint max_length); + uint max_length, bool low_byte_first); + virtual const uchar *unpack(uchar* to, const uchar *from, + uint param_data, bool low_byte_first); + const uchar *unpack_key(uchar* to, const uchar *from, + uint max_length, bool low_byte_first); int pack_cmp(const uchar *a, const uchar *b, uint key_length, my_bool insert_or_update); int pack_cmp(const uchar *b, uint key_length,my_bool insert_or_update); @@ -1346,7 +1532,7 @@ public: uint32 pack_length_no_ptr() const { return (uint32) (packlength); } uint32 sort_length() const; - inline uint32 max_data_length() const + virtual uint32 max_data_length() const { return (uint32) (((ulonglong) 1 << (packlength*8)) -1); } @@ -1374,13 +1560,13 @@ public: @retval The length in the row plus the size of the data. */ uint32 get_packed_size(const uchar *ptr_arg, bool low_byte_first) - {return packlength + get_length(ptr_arg, low_byte_first);} + {return packlength + get_length(ptr_arg, packlength, low_byte_first);} inline uint32 get_length(uint row_offset= 0) - { return get_length(ptr+row_offset, table->s->db_low_byte_first); } - uint32 get_length(const uchar *ptr, bool low_byte_first); + { return get_length(ptr+row_offset, this->packlength, table->s->db_low_byte_first); } + uint32 get_length(const uchar *ptr, uint packlength, bool low_byte_first); uint32 get_length(const uchar *ptr_arg) - { return get_length(ptr_arg, table->s->db_low_byte_first); } + { return get_length(ptr_arg, this->packlength, table->s->db_low_byte_first); } void put_length(uchar *pos, uint32 length); inline void get_ptr(uchar **str) { @@ -1421,13 +1607,16 @@ public: memcpy_fixed(ptr+packlength,&tmp,sizeof(char*)); return 0; } - uchar *pack(uchar *to, const uchar *from, uint max_length= ~(uint) 0); - uchar *pack_key(uchar *to, const uchar *from, uint max_length); + virtual uchar *pack(uchar *to, const uchar *from, + uint max_length, bool low_byte_first); + uchar *pack_key(uchar *to, const uchar *from, + uint max_length, bool low_byte_first); uchar *pack_key_from_key_image(uchar* to, const uchar *from, - uint max_length); - virtual const uchar *unpack(uchar *to, const uchar *from, uint param_data); - const uchar *unpack(uchar *to, const uchar *from); - const uchar *unpack_key(uchar* to, const uchar *from, uint max_length); + uint max_length, bool low_byte_first); + virtual const uchar *unpack(uchar *to, const uchar *from, + uint param_data, bool low_byte_first); + const uchar *unpack_key(uchar* to, const uchar *from, + uint max_length, bool low_byte_first); int pack_cmp(const uchar *a, const uchar *b, uint key_length, my_bool insert_or_update); int pack_cmp(const uchar *b, uint key_length,my_bool insert_or_update); @@ -1572,6 +1761,7 @@ public: enum_field_types type() const { return MYSQL_TYPE_BIT; } enum ha_base_keytype key_type() const { return HA_KEYTYPE_BIT; } uint32 key_length() const { return (uint32) (field_length + 7) / 8; } + uint32 max_data_length() const { return (field_length + 7) / 8; } uint32 max_display_length() { return field_length; } uint size_of() const { return sizeof(*this); } Item_result result_type () const { return INT_RESULT; } @@ -1602,9 +1792,10 @@ public: uint32 pack_length() const { return (uint32) (field_length + 7) / 8; } uint32 pack_length_in_rec() const { return bytes_in_rec; } void sql_type(String &str) const; - uchar *pack(uchar *to, const uchar *from, uint max_length=~(uint) 0); - virtual const uchar *unpack(uchar *to, const uchar *from, uint param_data); - const uchar *unpack(uchar* to, const uchar *from); + virtual uchar *pack(uchar *to, const uchar *from, + uint max_length, bool low_byte_first); + virtual const uchar *unpack(uchar *to, const uchar *from, + uint param_data, bool low_byte_first); virtual void set_default(); Field *new_key_field(MEM_ROOT *root, struct st_table *new_table, diff --git a/sql/log.cc b/sql/log.cc index dcf27161f1a..4d1ef9eed9f 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -3571,9 +3571,6 @@ bool MYSQL_BIN_LOG::write(Log_event *event_info) (!binlog_filter->db_ok(local_db))) { VOID(pthread_mutex_unlock(&LOCK_log)); - DBUG_PRINT("info",("OPTION_BIN_LOG is %s, db_ok('%s') == %d", - (thd->options & OPTION_BIN_LOG) ? "set" : "clear", - local_db, binlog_filter->db_ok(local_db))); DBUG_RETURN(0); } #endif /* HAVE_REPLICATION */ diff --git a/sql/log_event.cc b/sql/log_event.cc index cfac7df21a8..cf4bb229b2f 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -5697,7 +5697,9 @@ Rows_log_event::Rows_log_event(const char *buf, uint event_len, *description_event) : Log_event(buf, description_event), m_row_count(0), +#ifndef MYSQL_CLIENT m_table(NULL), +#endif m_rows_buf(0), m_rows_cur(0), m_rows_end(0), m_curr_row(NULL), m_curr_row_end(NULL), m_key(NULL) @@ -6168,6 +6170,8 @@ int Rows_log_event::do_apply_event(RELAY_LOG_INFO const *rli) unpack_current_row(rli); // at this moment m_curr_row_end should be set + DBUG_PRINT("debug", ("m_curr_row_end: 0x%lx; m_curr_row: 0x%lx; m_rows_end: 0x%lx", + m_curr_row_end, m_curr_row, m_rows_end)); DBUG_ASSERT(error || m_curr_row_end != NULL); DBUG_ASSERT(error || m_curr_row < m_curr_row_end); DBUG_ASSERT(error || m_curr_row_end <= m_rows_end); diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc index de4f6e0c337..dbfe7902ee9 100644 --- a/sql/rpl_record.cc +++ b/sql/rpl_record.cc @@ -65,6 +65,8 @@ pack_row(TABLE *table, MY_BITMAP const* cols, my_ptrdiff_t const rec_offset= record - table->record[0]; my_ptrdiff_t const def_offset= table->s->default_values - table->record[0]; + DBUG_ENTER("pack_row"); + /* We write the null bits and the packed records using one pass through all the fields. The null bytes are written little-endian, @@ -96,26 +98,14 @@ pack_row(TABLE *table, MY_BITMAP const* cols, For big-endian machines, we have to make sure that the length is stored in little-endian format, since this is the format used for the binlog. - - We do this by setting the db_low_byte_first, which is used - inside some store_length() to decide what order to write the - bytes in. - - In reality, db_log_byte_first is only set for legacy table - type Isam, but in the event of a bug, we need to guarantee - the endianess when writing to the binlog. - - This is currently broken for NDB due to BUG#29549, so we - will fix it when NDB has fixed their way of handling BLOBs. */ -#if 0 - bool save= table->s->db_low_byte_first; - table->s->db_low_byte_first= TRUE; -#endif - pack_ptr= field->pack(pack_ptr, field->ptr + offset); -#if 0 - table->s->db_low_byte_first= save; -#endif + const uchar *old_pack_ptr= pack_ptr; + pack_ptr= field->pack(pack_ptr, field->ptr + offset, + field->max_data_length(), TRUE); + DBUG_PRINT("debug", ("field: %s; pack_ptr: 0x%lx;" + " pack_ptr':0x%lx; bytes: %d", + field->field_name, (ulong) old_pack_ptr, + (ulong) pack_ptr, pack_ptr - old_pack_ptr)); } null_mask <<= 1; @@ -143,8 +133,8 @@ pack_row(TABLE *table, MY_BITMAP const* cols, packed data. If it doesn't, something is very wrong. */ DBUG_ASSERT(null_ptr == row_data + null_byte_count); - - return static_cast(pack_ptr - row_data); + DBUG_DUMP("row_data", row_data, pack_ptr - row_data); + DBUG_RETURN(static_cast(pack_ptr - row_data)); } #endif @@ -242,18 +232,14 @@ unpack_row(RELAY_LOG_INFO const *rli, Use the master's size information if available else call normal unpack operation. */ -#if 0 - bool save= table->s->db_low_byte_first; - table->s->db_low_byte_first= TRUE; -#endif uint16 const metadata= tabledef->field_metadata(i); - if (tabledef && metadata) - pack_ptr= f->unpack(f->ptr, pack_ptr, metadata); - else - pack_ptr= f->unpack(f->ptr, pack_ptr); -#if 0 - table->s->db_low_byte_first= save; -#endif + uchar const *const old_pack_ptr= pack_ptr; + pack_ptr= f->unpack(f->ptr, pack_ptr, metadata, TRUE); + DBUG_PRINT("debug", ("field: %s; metadata: 0x%x;" + " pack_ptr: 0x%lx; pack_ptr': 0x%lx; bytes: %d", + f->field_name, metadata, + (ulong) old_pack_ptr, (ulong) pack_ptr, + pack_ptr - old_pack_ptr)); } null_mask <<= 1; @@ -265,6 +251,7 @@ unpack_row(RELAY_LOG_INFO const *rli, throw away master's extra fields */ uint max_cols= min(tabledef->size(), cols->n_bits); + DBUG_PRINT("debug", ("Master has %u fields, slave %u", tabledef->size(), cols->n_bits)); for (; i < max_cols; i++) { if (bitmap_is_set(cols, i)) @@ -279,6 +266,7 @@ unpack_row(RELAY_LOG_INFO const *rli, if (!((null_bits & null_mask) && tabledef->maybe_null(i))) pack_ptr+= tabledef->calc_field_size(i, (uchar *) pack_ptr); + DBUG_PRINT("debug", ("pack_ptr: 0x%lx", (ulong) pack_ptr)); null_mask <<= 1; } } @@ -289,6 +277,8 @@ unpack_row(RELAY_LOG_INFO const *rli, */ DBUG_ASSERT(null_ptr == row_data + master_null_byte_count); + DBUG_DUMP("row_data", row_data, pack_ptr - row_data); + *row_end = pack_ptr; if (master_reclength) { diff --git a/sql/sql_show.cc b/sql/sql_show.cc index bb4f5107df8..49910e75286 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -29,6 +29,8 @@ #include "event_data_objects.h" #include +#define STR_OR_NIL(S) ((S) ? (S) : "") + #ifdef WITH_PARTITION_STORAGE_ENGINE #include "ha_partition.h" #endif @@ -3096,8 +3098,8 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) schema_table_idx= get_schema_table_idx(schema_table); get_lookup_field_values(thd, cond, tables, &lookup_field_vals); DBUG_PRINT("INDEX VALUES",("db_name='%s', table_name='%s'", - lookup_field_vals.db_value.str, - lookup_field_vals.table_value.str)); + STR_OR_NIL(lookup_field_vals.db_value.str), + STR_OR_NIL(lookup_field_vals.table_value.str))); if (!lookup_field_vals.wild_db_value && !lookup_field_vals.wild_table_value) {