diff --git a/mysql-test/suite/binlog/r/binlog_base64_flag.result b/mysql-test/suite/binlog/r/binlog_base64_flag.result index c4c4fddb5d6..ef92e63eb93 100644 --- a/mysql-test/suite/binlog/r/binlog_base64_flag.result +++ b/mysql-test/suite/binlog/r/binlog_base64_flag.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("BINLOG_BASE64_EVENT: According to the master's version"); +call mtr.add_suppression("BINLOG_BASE64_EVENT: Master's version is earlier than"); call mtr.add_suppression("BINLOG_BASE64_EVENT: Column 1 of table 'test.char128_utf8' cannot be converted"); DROP TABLE IF EXISTS t1; ==== Test BUG#32407 ==== diff --git a/mysql-test/suite/binlog/t/binlog_base64_flag.test b/mysql-test/suite/binlog/t/binlog_base64_flag.test index 5311da54f5f..c2e3d583822 100644 --- a/mysql-test/suite/binlog/t/binlog_base64_flag.test +++ b/mysql-test/suite/binlog/t/binlog_base64_flag.test @@ -9,7 +9,7 @@ # BINLOG statement does not work in embedded mode. source include/not_embedded.inc; -call mtr.add_suppression("BINLOG_BASE64_EVENT: According to the master's version"); +call mtr.add_suppression("BINLOG_BASE64_EVENT: Master's version is earlier than"); call mtr.add_suppression("BINLOG_BASE64_EVENT: Column 1 of table 'test.char128_utf8' cannot be converted"); disable_warnings; diff --git a/sql/field.cc b/sql/field.cc index 57a08128e69..fb3224afce5 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7724,7 +7724,7 @@ Field_string::compatible_field_size(uint field_metadata, { #ifdef HAVE_REPLICATION const Check_field_param check_param = { this }; - if (rpl_master_has_bug(rli_arg, 37426, TRUE, + if (rpl_master_has_bug(rli_arg, RPL_BUG_MYSQL_37426, TRUE, check_field_for_37426, &check_param)) return FALSE; // Not compatible field sizes #endif diff --git a/sql/log_event.h b/sql/log_event.h index 412e348505c..a4112c5a1a6 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -159,6 +159,26 @@ class String; #define NUM_LOAD_DELIM_STRS 5 + +/* + Master-side bugs that we work-around with rpl_master_has_bug(). + Each define is a bit number (0, 1, 2, ...) set in + Relay_log_info::rpl_master_bug_bitmask. + These bit numbers are internal-only, can be changed (eg. to drop a bug + work-around for ancient version) without concern for backwards compatibility. +*/ +enum enum_rpl_master_bug_ids { + RPL_BUG_MYSQL_24432, + RPL_BUG_MYSQL_33029, + RPL_BUG_MYSQL_37426, + RPL_BUG_MDEV_29621, + RPL_BUG_END /* To check size of Relay_log_info::rpl_master_bug_bitmask */ +}; + +/* Must be large enough to hold all enum_rpl_master_bug_ids bits. */ +typedef uint32_t rpl_bug_id_t; + + /* The following is the max table_map_id. This is limited by that we are using 6 bytes for it in replication diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index bf0984a1030..dd510b6868c 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -2799,6 +2799,14 @@ int Format_description_log_event::do_apply_event(rpl_group_info *rgi) copy_crypto_data(rli->relay_log.description_event_for_exec); delete rli->relay_log.description_event_for_exec; rli->relay_log.description_event_for_exec= this; + /* + The start of the relay log has our own generated format description + event followed by the real format description from the master. + Don't re-compute the bug bitmask from the generated event so we don't + leave a window where a wrong value is temporarily set. + */ + if (!is_relay_log_event()) + rli->calc_master_bug_bitmask(this); } DBUG_RETURN(ret); @@ -7992,8 +8000,8 @@ int Rows_log_event::update_sequence() (table_rgi && !(table_rgi->gtid_ev_flags2 & Gtid_log_event::FL_DDL) && !(old_master= - rpl_master_has_bug(thd_rgi->rli, - 29621, FALSE, FALSE, FALSE, TRUE)))) + rpl_master_has_bug(thd_rgi->rli, RPL_BUG_MDEV_29621, FALSE, + NULL, NULL)))) { /* This event come from a setval function executed on the master. Update the sequence next_number and round, like we do with setval() diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index 05a8b5948e3..a93decb2f30 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -54,7 +54,8 @@ Relay_log_info::Relay_log_info(bool is_slave_recovery, const char* thread_name) cur_log_old_open_count(0), error_on_rli_init_info(false), group_relay_log_pos(0), event_relay_log_pos(0), group_master_log_pos(0), log_space_total(0), ignore_log_space_limit(0), - last_master_timestamp(0), sql_thread_caught_up(true), slave_skip_counter(0), + last_master_timestamp(0), sql_thread_caught_up(true), + rpl_master_bug_bitmask(0), slave_skip_counter(0), abort_pos_wait(0), slave_run_id(0), sql_driver_thd(), gtid_skip_flag(GTID_SKIP_NOT), inited(0), abort_slave(0), stop_for_until(0), slave_running(MYSQL_SLAVE_NOT_RUN), until_condition(UNTIL_NONE), @@ -741,6 +742,12 @@ int init_relay_log_pos(Relay_log_info* rli,const char* log, goto err; delete rli->relay_log.description_event_for_exec; rli->relay_log.description_event_for_exec= fdev; + /* + The read_relay_log_description_event() returns the real FD from the + master found near the start of the relay log. Initialize the bitmask + of any known bugs in that master version. + */ + rli->calc_master_bug_bitmask(fdev); } my_b_seek(rli->cur_log,(off_t)pos); DBUG_PRINT("info", ("my_b_tell(rli->cur_log)=%llu rli->event_relay_log_pos=%llu", @@ -2657,4 +2664,159 @@ bool Relay_log_info::flush() DBUG_RETURN(error); } + +struct st_version_range_for_one_bug { + rpl_bug_id_t bug_id; + uint bug_num; + bool mysql_bug; // Else Mariadb MDEV + Version introduced_in; // first version with bug + Version fixed_in; // first version with fix +}; +static struct st_version_range_for_one_bug versions_for_all_bugs[]= +{ + /* + Sorted with the most recent server version first, since this is the version + that will be printed in any error messages. + */ + /* MySQL bugs. */ + {RPL_BUG_MYSQL_24432, 24432, true, { 5, 1, 12 }, { 5, 1, 17 } }, + {RPL_BUG_MYSQL_24432, 24432, true, { 5, 0, 24 }, { 5, 0, 38 } }, + {RPL_BUG_MYSQL_33029, 33029, true, { 5, 1, 0 }, { 5, 1, 12 } }, + {RPL_BUG_MYSQL_33029, 33029, true, { 5, 0, 0 }, { 5, 0, 58 } }, + {RPL_BUG_MYSQL_37426, 37426, true, { 5, 1, 0 }, { 5, 1, 26 } }, + /* MariaDB bugs (MDEV-nnnnn). */ + {RPL_BUG_MDEV_29621, 29621, false, { 10, 11,1 }, { 10, 11,3 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 10,1 }, { 10, 10,4 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 9, 2 }, { 10, 9, 6 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 8, 4 }, { 10, 8, 8 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 7, 5 }, { 10, 7, 9 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 6, 9 }, { 10, 6, 13 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 5, 17 }, { 10, 5, 20 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 4, 26 }, { 10, 4, 29 } }, + {RPL_BUG_MDEV_29621, 29621, false, { 10, 3, 36 }, { 10, 3, 39 } }, +}; + + +/** + Detect, based on the version in format description event, any bugs on the + master that we have a work-around for on the slave. Calculate a bitmask + for this with a bit set for any detected bug. +*/ +void +Relay_log_info::calc_master_bug_bitmask(const Format_description_log_event * + fd_event) +{ + const Version &master_ver= fd_event->server_version_split; + uint all_size= + sizeof(versions_for_all_bugs)/sizeof(versions_for_all_bugs[0]); + rpl_bug_id_t bitmask= 0; + for (uint i= 0; i < all_size; i++) + { + const Version &introduced_in= versions_for_all_bugs[i].introduced_in; + const Version &fixed_in= versions_for_all_bugs[i].fixed_in; + if (introduced_in <= master_ver && fixed_in > master_ver) + { + bitmask|= (1 << versions_for_all_bugs[i].bug_id); + } + } + rpl_master_bug_bitmask.store(bitmask, std::memory_order_relaxed); +} + + +/** + Detects, based on version of master's format description event, if master + has a certain bug. The bitset of bugs was pre-calculated by + Relay_log_info::calc_master_bug_bitmask(). + + The bitset is read out of the Relay_log_info without locking. However this + is ok, as the set of master bugs can only change with a master server + restart, and we do not replicate in parallel events crossing a master + restart (as identified by its restart format description event). + + @param rli The Relay_log_info contains the pre-computed bug bitset. + @param bug_id Identifier of the bug as defined in enum_rpl_master_bug_ids. + @param report bool report error message. + + @param pred Predicate function that will be called with @c param to + check for the bug. If the function return @c true, the bug is present, + otherwise, it is not. + + @param param State passed to @c pred function. + + @return TRUE if master has the bug, FALSE if it does not. +*/ +bool +rpl_master_has_bug_ext(const Relay_log_info *rli, rpl_bug_id_t bug_id, + bool report, + bool (*pred)(const void *), const void *param) +{ + rpl_bug_id_t mask= + rli->rpl_master_bug_bitmask.load(std::memory_order_relaxed); + if (!(mask & (1 << bug_id))) + return false; + if (pred != NULL && !(*pred)(param)) + return false; + if (!report) + return true; + uint all_size= + sizeof(versions_for_all_bugs)/sizeof(versions_for_all_bugs[0]); + for (uint i= 0; i < all_size; i++) + { + const Version &fixed_in= versions_for_all_bugs[i].fixed_in; + if (versions_for_all_bugs[i].bug_id == bug_id) + { + const char *bug_source= versions_for_all_bugs[i].mysql_bug ? + "http://bugs.mysql.com/bug.php?id=" : + "https://jira.mariadb.org/browse/MDEV-"; + // a short message for SHOW SLAVE STATUS (message length constraints). + my_printf_error(ER_UNKNOWN_ERROR, "master may suffer from" + " %s%u" + " so slave stops; check error log on slave" + " for more info", MYF(0), bug_source, + versions_for_all_bugs[i].bug_num); + // a verbose message for the error log. + rli->report(ERROR_LEVEL, ER_UNKNOWN_ERROR, NULL, + "Master's version is earlier than %d.%d.%d," + " it is probable that master suffers from this bug:" + " %s%u" + " and thus replicating the current binary log event" + " may make the slave's data become different from the" + " master's data." + " To take no risk, slave refuses to replicate" + " this event and stops." + " We recommend that all updates be stopped on the" + " master and slave, that the data of both be" + " manually synchronized," + " that master's binary logs be deleted," + " that master be upgraded to a version at least" + " equal to '%d.%d.%d'. Then replication can be" + " restarted.", + fixed_in[0], fixed_in[1], fixed_in[2], + bug_source, + versions_for_all_bugs[i].bug_num, + fixed_in[0], fixed_in[1], fixed_in[2]); + return true; + } + } + DBUG_ASSERT(0 /* Impossible, as bitmap calculated from the same table. */); + my_printf_error(ER_UNKNOWN_ERROR, "master may suffer from a serious bug" + " so slave stops", MYF(0)); + rli->report(ERROR_LEVEL, ER_UNKNOWN_ERROR, NULL, + "Master's version is old," + " it is probable that master suffers from a serious bug" + " and thus replicating the current binary log event" + " may make the slave's data become different from the" + " master's data." + " To take no risk, slave refuses to replicate" + " this event and stops." + " We recommend that all updates be stopped on the" + " master and slave, that the data of both be" + " manually synchronized," + " that master's binary logs be deleted," + " that master be upgraded to a recent version" + " Then replication can be" + " restarted."); + return true; +} + #endif diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h index 20c58e4aaf7..e75844de417 100644 --- a/sql/rpl_rli.h +++ b/sql/rpl_rli.h @@ -282,6 +282,19 @@ public: */ bool are_sql_threads_caught_up(); + /** + Bitmask of bugs present in the master version of the current + description_event_for_exec. Of atomic type, since it is written when + applying master's format description events, and read concurrently without + locking from parallel replication worker threads. Though note that the + value will only change during a master restart, and we do not replicate + events concurrently across such restart event, so the value will not change + in practice while worker threads are working. + */ + std::atomic rpl_master_bug_bitmask; + static_assert(sizeof(rpl_bug_id_t) > ((RPL_BUG_END - 1) >> 3), + "rpl_master_bug_bitmask too small to hold all bits"); + void clear_until_condition(); /** Reset the delay. @@ -557,6 +570,7 @@ public: int32 get_sql_delay() { return sql_delay; } void set_sql_delay(int32 _sql_delay) { sql_delay= _sql_delay; } time_t get_sql_delay_end() { return sql_delay_end; } + void calc_master_bug_bitmask(const Format_description_log_event * fd_event); rpl_gtid last_seen_gtid; ulong last_trans_retry_count; private: @@ -1089,4 +1103,19 @@ int event_group_new_gtid(rpl_group_info *rgi, Gtid_log_event *gev); void delete_or_keep_event_post_apply(rpl_group_info *rgi, Log_event_type typ, Log_event *ev); +bool +rpl_master_has_bug_ext(const Relay_log_info *rli, rpl_bug_id_t bug_id, + bool report, + bool (*pred)(const void *), const void *param); +static inline bool +rpl_master_has_bug(const Relay_log_info *rli, rpl_bug_id_t bug_id, + bool report, bool (*pred)(const void *), const void *param) +{ + rpl_bug_id_t mask= + rli->rpl_master_bug_bitmask.load(std::memory_order_relaxed); + if (likely(!(mask & (1 << bug_id)))) + return false; + return rpl_master_has_bug_ext(rli, bug_id, report, pred, param); +} + #endif /* RPL_RLI_H */ diff --git a/sql/rpl_utility_server.cc b/sql/rpl_utility_server.cc index ccad7bd0709..de6ba19357f 100644 --- a/sql/rpl_utility_server.cc +++ b/sql/rpl_utility_server.cc @@ -775,9 +775,8 @@ static void show_sql_type(const Conv_source &src, const Field &dst, acceptable according to the current settings. @param order The computed order of the conversion needed. - @param rli The relay log info data structure: for error reporting. */ -static bool is_conversion_ok(enum_conv_type type, const Relay_log_info *rli, +static bool is_conversion_ok(enum_conv_type type, ulonglong type_conversion_options) { DBUG_ENTER("is_conversion_ok"); @@ -948,12 +947,10 @@ table_def::compatible_with(THD *thd, rpl_group_info *rgi, return false; } - if (!h) - return false; // An unknown data type found in the binary log Conv_source source(h, field_metadata(col), field->charset()); enum_conv_type convtype= can_convert_field_to(field, source, rli, Conv_param(m_flags)); - if (is_conversion_ok(convtype, rli, slave_type_conversions_options)) + if (is_conversion_ok(convtype, slave_type_conversions_options)) { DBUG_PRINT("debug", ("Checking column %d -" " field '%s' can be converted - order: %d", diff --git a/sql/slave.cc b/sql/slave.cc index 3af008a77eb..0d25165628b 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -8732,104 +8732,6 @@ end: } -/** - Detects, based on master's version (as found in the relay log), if master - has a certain bug. - @param rli Relay_log_info which tells the master's version - @param bug_id Number of the bug as found in bugs.mysql.com - @param report bool report error message, default TRUE - - @param pred Predicate function that will be called with @c param to - check for the bug. If the function return @c true, the bug is present, - otherwise, it is not. - - @param param State passed to @c pred function. - - @return TRUE if master has the bug, FALSE if it does not. -*/ -bool rpl_master_has_bug(const Relay_log_info *rli, uint bug_id, bool report, - bool (*pred)(const void *), const void *param, - bool maria_master) -{ - struct st_version_range_for_one_bug { - uint bug_id; - Version introduced_in; // first version with bug - Version fixed_in; // first version with fix - }; - static struct st_version_range_for_one_bug versions_for_their_bugs[]= - { - {24432, { 5, 0, 24 }, { 5, 0, 38 } }, - {24432, { 5, 1, 12 }, { 5, 1, 17 } }, - {33029, { 5, 0, 0 }, { 5, 0, 58 } }, - {33029, { 5, 1, 0 }, { 5, 1, 12 } }, - {37426, { 5, 1, 0 }, { 5, 1, 26 } }, - }; - static struct st_version_range_for_one_bug versions_for_our_bugs[]= - { - {29621, { 10, 3, 36 }, { 10, 3, 39 } }, - {29621, { 10, 4, 26 }, { 10, 4, 29 } }, - {29621, { 10, 5, 17 }, { 10, 5, 20 } }, - {29621, { 10, 6, 9 }, { 10, 6, 13 } }, - {29621, { 10, 7, 5 }, { 10, 7, 9 } }, - {29621, { 10, 8, 4 }, { 10, 8, 8 } }, - {29621, { 10, 9, 2 }, { 10, 9, 6 } }, - {29621, { 10, 10,1 }, { 10, 10,4 } }, - {29621, { 10, 11,1 }, { 10, 11,3 } }, - }; - const Version &master_ver= - rli->relay_log.description_event_for_exec->server_version_split; - struct st_version_range_for_one_bug* versions_for_all_bugs= maria_master ? - versions_for_our_bugs : versions_for_their_bugs; - uint all_size= maria_master ? - sizeof(versions_for_our_bugs)/sizeof(*versions_for_our_bugs) : - sizeof(versions_for_their_bugs)/sizeof(*versions_for_their_bugs); - - for (uint i= 0; i < all_size; i++) - { - const Version &introduced_in= versions_for_all_bugs[i].introduced_in; - const Version &fixed_in= versions_for_all_bugs[i].fixed_in; - if ((versions_for_all_bugs[i].bug_id == bug_id) && - introduced_in <= master_ver && - fixed_in > master_ver && - (pred == NULL || (*pred)(param))) - { - const char *bug_source= maria_master ? - "https://jira.mariadb.org/browse/MDEV-" : - "http://bugs.mysql.com/bug.php?id="; - if (!report) - return TRUE; - // a short message for SHOW SLAVE STATUS (message length constraints) - my_printf_error(ER_UNKNOWN_ERROR, "master may suffer from" - " %s%u" - " so slave stops; check error log on slave" - " for more info", MYF(0), bug_source, bug_id); - // a verbose message for the error log - rli->report(ERROR_LEVEL, ER_UNKNOWN_ERROR, NULL, - "According to the master's version ('%s')," - " it is probable that master suffers from this bug:" - " %s%u" - " and thus replicating the current binary log event" - " may make the slave's data become different from the" - " master's data." - " To take no risk, slave refuses to replicate" - " this event and stops." - " We recommend that all updates be stopped on the" - " master and slave, that the data of both be" - " manually synchronized," - " that master's binary logs be deleted," - " that master be upgraded to a version at least" - " equal to '%d.%d.%d'. Then replication can be" - " restarted.", - rli->relay_log.description_event_for_exec->server_version, - bug_source, - bug_id, - fixed_in[0], fixed_in[1], fixed_in[2]); - return TRUE; - } - } - return FALSE; -} - /** BUG#33029, For all 5.0 up to 5.0.58 exclusive, and 5.1 up to 5.1.12 exclusive, if one statement in a SP generated AUTO_INCREMENT value @@ -8845,7 +8747,8 @@ bool rpl_master_erroneous_autoinc(THD *thd) if (thd->rgi_slave) { DBUG_EXECUTE_IF("simulate_bug33029", return TRUE;); - return rpl_master_has_bug(thd->rgi_slave->rli, 33029, FALSE, NULL, NULL); + return rpl_master_has_bug(thd->rgi_slave->rli, RPL_BUG_MYSQL_33029, FALSE, + NULL, NULL); } return FALSE; } diff --git a/sql/slave.h b/sql/slave.h index ce25bebf730..c5670447d09 100644 --- a/sql/slave.h +++ b/sql/slave.h @@ -230,9 +230,6 @@ bool show_master_info(THD* thd, Master_info* mi, bool full); bool show_all_master_info(THD* thd); void show_binlog_info_get_fields(THD *thd, List *field_list); bool show_binlog_info(THD* thd); -bool rpl_master_has_bug(const Relay_log_info *rli, uint bug_id, bool report, - bool (*pred)(const void *), const void *param, - bool maria_master= false); bool rpl_master_erroneous_autoinc(THD* thd); const char *print_slave_db_safe(const char *db); diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 14104b08b7c..c8ad6d4a267 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -944,7 +944,8 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, if (thd->rgi_slave && (info.handle_duplicates == DUP_UPDATE) && (table->next_number_field != NULL) && - rpl_master_has_bug(thd->rgi_slave->rli, 24432, TRUE, NULL, NULL)) + rpl_master_has_bug(thd->rgi_slave->rli, RPL_BUG_MYSQL_24432, + TRUE, NULL, NULL)) goto abort; #endif @@ -4341,7 +4342,8 @@ select_insert::prepare(List &values, SELECT_LEX_UNIT *u) if (thd->rgi_slave && (info.handle_duplicates == DUP_UPDATE) && (table->next_number_field != NULL) && - rpl_master_has_bug(thd->rgi_slave->rli, 24432, TRUE, NULL, NULL)) + rpl_master_has_bug(thd->rgi_slave->rli, RPL_BUG_MYSQL_24432, + TRUE, NULL, NULL)) DBUG_RETURN(1); #endif